diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml index 9c734166..191efb53 100644 --- a/.github/workflows/build-and-test.yml +++ b/.github/workflows/build-and-test.yml @@ -43,6 +43,7 @@ jobs: build/*/*.ini build/*/*.so build/*/*.dll + build/*/*.dylib build/*/vanity/ build/*/navmeshes/ build/*/migrations/ diff --git a/CMakeLists.txt b/CMakeLists.txt index eeba9eee..ac08daf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -69,6 +69,7 @@ set(RECASTNAVIGATION_EXAMPLES OFF CACHE BOOL "" FORCE) # Disabled no-register # Disabled unknown pragmas because Linux doesn't understand Windows pragmas. if(UNIX) + add_link_options("-Wl,-rpath,$ORIGIN/") add_compile_options("-fPIC") add_compile_definitions(_GLIBCXX_USE_CXX11_ABI=0 _GLIBCXX_USE_CXX17_ABI=0) @@ -318,7 +319,7 @@ add_subdirectory(dPhysics) add_subdirectory(dServer) # Create a list of common libraries shared between all binaries -set(COMMON_LIBRARIES "dCommon" "dDatabase" "dNet" "raknet" "MariaDB::ConnCpp" "magic_enum") +set(COMMON_LIBRARIES "dCommon" "dDatabase" "dNet" "raknet" "magic_enum") # Add platform specific common libraries if(UNIX) diff --git a/CMakePresets.json b/CMakePresets.json index bafc2c7e..40db4152 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -92,4 +92,5 @@ "jobs": 2 } ] + } diff --git a/dAuthServer/AuthServer.cpp b/dAuthServer/AuthServer.cpp index d306eb70..741a6e59 100644 --- a/dAuthServer/AuthServer.cpp +++ b/dAuthServer/AuthServer.cpp @@ -60,7 +60,7 @@ int main(int argc, char** argv) { try { Database::Connect(); - } catch (sql::SQLException& ex) { + } catch (std::exception& ex) { LOG("Got an error while connecting to the database: %s", ex.what()); Database::Destroy("AuthServer"); delete Game::server; diff --git a/dChatServer/ChatServer.cpp b/dChatServer/ChatServer.cpp index 022175e5..b4959992 100644 --- a/dChatServer/ChatServer.cpp +++ b/dChatServer/ChatServer.cpp @@ -81,7 +81,7 @@ int main(int argc, char** argv) { //Connect to the MySQL Database try { Database::Connect(); - } catch (sql::SQLException& ex) { + } catch (std::exception& ex) { LOG("Got an error while connecting to the database: %s", ex.what()); Database::Destroy("ChatServer"); delete Game::server; diff --git a/dCommon/BrickByBrickFix.cpp b/dCommon/BrickByBrickFix.cpp index b771c986..5aef091a 100644 --- a/dCommon/BrickByBrickFix.cpp +++ b/dCommon/BrickByBrickFix.cpp @@ -123,7 +123,7 @@ uint32_t BrickByBrickFix::UpdateBrickByBrickModelsToSd0() { Database::Get()->UpdateUgcModelData(model.id, outputStringStream); LOG("Updated model %i to sd0", model.id); updatedModels++; - } catch (sql::SQLException exception) { + } catch (std::exception& exception) { LOG("Failed to update model %i. This model should be inspected manually to see why." "The database error is %s", model.id, exception.what()); } diff --git a/dDatabase/CMakeLists.txt b/dDatabase/CMakeLists.txt index 004bdc14..42bdb983 100644 --- a/dDatabase/CMakeLists.txt +++ b/dDatabase/CMakeLists.txt @@ -2,6 +2,12 @@ add_subdirectory(CDClientDatabase) add_subdirectory(GameDatabase) add_library(dDatabase STATIC "MigrationRunner.cpp") + +add_custom_target(conncpp_dylib + ${CMAKE_COMMAND} -E copy $ ${PROJECT_BINARY_DIR}) + +add_dependencies(dDatabase conncpp_dylib) + target_include_directories(dDatabase PUBLIC ".") target_link_libraries(dDatabase PUBLIC dDatabaseCDClient dDatabaseGame) diff --git a/dDatabase/GameDatabase/Database.h b/dDatabase/GameDatabase/Database.h index 65b04722..cd0e93e3 100644 --- a/dDatabase/GameDatabase/Database.h +++ b/dDatabase/GameDatabase/Database.h @@ -1,7 +1,6 @@ #pragma once #include -#include #include "GameDatabase.h" diff --git a/dDatabase/GameDatabase/GameDatabase.h b/dDatabase/GameDatabase/GameDatabase.h index f52c8c4e..dcfac4a2 100644 --- a/dDatabase/GameDatabase/GameDatabase.h +++ b/dDatabase/GameDatabase/GameDatabase.h @@ -24,6 +24,7 @@ #include "IIgnoreList.h" #include "IAccountsRewardCodes.h" #include "IBehaviors.h" +#include "IUgcModularBuild.h" namespace sql { class Statement; @@ -42,7 +43,7 @@ class GameDatabase : public IPropertyContents, public IProperty, public IPetNames, public ICharXml, public IMigrationHistory, public IUgc, public IFriends, public ICharInfo, public IAccounts, public IActivityLog, public IAccountsRewardCodes, public IIgnoreList, - public IBehaviors { + public IBehaviors, public IUgcModularBuild { public: virtual ~GameDatabase() = default; // TODO: These should be made private. diff --git a/dDatabase/GameDatabase/ITables/ILeaderboard.h b/dDatabase/GameDatabase/ITables/ILeaderboard.h index 84d44eb2..fc4164bc 100644 --- a/dDatabase/GameDatabase/ITables/ILeaderboard.h +++ b/dDatabase/GameDatabase/ITables/ILeaderboard.h @@ -3,12 +3,44 @@ #include #include +#include +#include class ILeaderboard { public: + struct Entry { + uint32_t charId{}; + uint32_t lastPlayedTimestamp{}; + float primaryScore{}; + float secondaryScore{}; + uint32_t tertiaryScore{}; + uint32_t numWins{}; + uint32_t numTimesPlayed{}; + uint32_t ranking{}; + std::string name{}; + }; + + struct Score { + auto operator<=>(const Score& rhs) const = default; + + float primaryScore{ 0.0f }; + float secondaryScore{ 0.0f }; + float tertiaryScore{ 0.0f }; + }; + // Get the donation total for the given activity id. virtual std::optional GetDonationTotal(const uint32_t activityId) = 0; + + virtual std::vector GetDescendingLeaderboard(const uint32_t activityId) = 0; + virtual std::vector GetAscendingLeaderboard(const uint32_t activityId) = 0; + virtual std::vector GetNsLeaderboard(const uint32_t activityId) = 0; + virtual std::vector GetAgsLeaderboard(const uint32_t activityId) = 0; + virtual std::optional GetPlayerScore(const uint32_t playerId, const uint32_t gameId) = 0; + + virtual void SaveScore(const uint32_t playerId, const uint32_t gameId, const Score& score) = 0; + virtual void UpdateScore(const uint32_t playerId, const uint32_t gameId, const Score& score) = 0; + virtual void IncrementNumWins(const uint32_t playerId, const uint32_t gameId) = 0; }; #endif //!__ILEADERBOARD__H__ diff --git a/dDatabase/GameDatabase/ITables/IUgcModularBuild.h b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h new file mode 100644 index 00000000..4aa2e312 --- /dev/null +++ b/dDatabase/GameDatabase/ITables/IUgcModularBuild.h @@ -0,0 +1,14 @@ +#ifndef IUGCMODULARBUILD_H +#define IUGCMODULARBUILD_H + +#include +#include +#include + +class IUgcModularBuild { +public: + virtual void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) = 0; + virtual void DeleteUgcBuild(const LWOOBJID bigId) = 0; +}; + +#endif //!IUGCMODULARBUILD_H diff --git a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h index f30e33ce..4e7d19a4 100644 --- a/dDatabase/GameDatabase/MySQL/MySQLDatabase.h +++ b/dDatabase/GameDatabase/MySQL/MySQLDatabase.h @@ -7,6 +7,7 @@ #include "GameDatabase.h" typedef std::unique_ptr& UniquePreppedStmtRef; +typedef std::unique_ptr UniqueResultSet; // Purposefully no definition for this to provide linker errors in the case someone tries to // bind a parameter to a type that isn't defined. @@ -113,6 +114,16 @@ public: void RemoveBehavior(const int32_t characterId) override; void UpdateAccountGmLevel(const uint32_t accountId, const eGameMasterLevel gmLevel) override; std::optional GetProperties(const IProperty::PropertyLookup& params) override; + std::vector GetDescendingLeaderboard(const uint32_t activityId) override; + std::vector GetAscendingLeaderboard(const uint32_t activityId) override; + std::vector GetNsLeaderboard(const uint32_t activityId) override; + std::vector GetAgsLeaderboard(const uint32_t activityId) override; + void SaveScore(const uint32_t playerId, const uint32_t gameId, const Score& score) override; + void UpdateScore(const uint32_t playerId, const uint32_t gameId, const Score& score) override; + std::optional GetPlayerScore(const uint32_t playerId, const uint32_t gameId) override; + void IncrementNumWins(const uint32_t playerId, const uint32_t gameId) override; + void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) override; + void DeleteUgcBuild(const LWOOBJID bigId) override; private: // Generic query functions that can be used for any query. diff --git a/dDatabase/GameDatabase/MySQL/Tables/CMakeLists.txt b/dDatabase/GameDatabase/MySQL/Tables/CMakeLists.txt index 47cd220e..2f1fa6de 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/CMakeLists.txt +++ b/dDatabase/GameDatabase/MySQL/Tables/CMakeLists.txt @@ -20,6 +20,7 @@ set(DDATABASES_DATABASES_MYSQL_TABLES_SOURCES "PropertyContents.cpp" "Servers.cpp" "Ugc.cpp" + "UgcModularBuild.cpp" PARENT_SCOPE ) diff --git a/dDatabase/GameDatabase/MySQL/Tables/Leaderboard.cpp b/dDatabase/GameDatabase/MySQL/Tables/Leaderboard.cpp index 22403abb..a6734030 100644 --- a/dDatabase/GameDatabase/MySQL/Tables/Leaderboard.cpp +++ b/dDatabase/GameDatabase/MySQL/Tables/Leaderboard.cpp @@ -1,5 +1,9 @@ #include "MySQLDatabase.h" +#include "Game.h" +#include "Logger.h" +#include "dConfig.h" + std::optional MySQLDatabase::GetDonationTotal(const uint32_t activityId) { auto donation_total = ExecuteSelect("SELECT SUM(primaryScore) as donation_total FROM leaderboard WHERE game_id = ?;", activityId); @@ -9,3 +13,75 @@ std::optional MySQLDatabase::GetDonationTotal(const uint32_t activityI return donation_total->getUInt("donation_total"); } + +std::vector ProcessQuery(UniqueResultSet& rows) { + std::vector entries; + entries.reserve(rows->rowsCount()); + + while (rows->next()) { + auto& entry = entries.emplace_back(); + + entry.charId = rows->getUInt("character_id"); + entry.lastPlayedTimestamp = rows->getUInt("lp_unix"); + entry.primaryScore = rows->getFloat("primaryScore"); + entry.secondaryScore = rows->getFloat("secondaryScore"); + entry.tertiaryScore = rows->getFloat("tertiaryScore"); + entry.numWins = rows->getUInt("numWins"); + entry.numTimesPlayed = rows->getUInt("timesPlayed"); + entry.name = rows->getString("char_name"); + // entry.ranking is never set because its calculated in leaderboard in code. + } + + return entries; +} + +std::vector MySQLDatabase::GetDescendingLeaderboard(const uint32_t activityId) { + auto leaderboard = ExecuteSelect("SELECT *, UNIX_TIMESTAMP(last_played) as lp_unix, ci.name as char_name FROM leaderboard lb JOIN charinfo ci on ci.id = lb.character_id where game_id = ? ORDER BY primaryscore DESC, secondaryscore DESC, tertiaryScore DESC, last_played ASC;", activityId); + return ProcessQuery(leaderboard); +} + +std::vector MySQLDatabase::GetAscendingLeaderboard(const uint32_t activityId) { + auto leaderboard = ExecuteSelect("SELECT *, UNIX_TIMESTAMP(last_played) as lp_unix, ci.name as char_name FROM leaderboard lb JOIN charinfo ci on ci.id = lb.character_id where game_id = ? ORDER BY primaryscore ASC, secondaryscore ASC, tertiaryScore ASC, last_played ASC;", activityId); + return ProcessQuery(leaderboard); +} + +std::vector MySQLDatabase::GetAgsLeaderboard(const uint32_t activityId) { + auto query = Game::config->GetValue("classic_survival_scoring") != "1" ? + "SELECT *, UNIX_TIMESTAMP(last_played) as lp_unix, ci.name as char_name FROM leaderboard lb JOIN charinfo ci on ci.id = lb.character_id where game_id = ? ORDER BY primaryscore DESC, secondaryscore DESC, tertiaryScore DESC, last_played ASC;" : + "SELECT *, UNIX_TIMESTAMP(last_played) as lp_unix, ci.name as char_name FROM leaderboard lb JOIN charinfo ci on ci.id = lb.character_id where game_id = ? ORDER BY secondaryscore DESC, primaryscore DESC, tertiaryScore DESC, last_played ASC;"; + auto leaderboard = ExecuteSelect(query, activityId); + return ProcessQuery(leaderboard); +} + +std::vector MySQLDatabase::GetNsLeaderboard(const uint32_t activityId) { + auto leaderboard = ExecuteSelect("SELECT *, UNIX_TIMESTAMP(last_played) as lp_unix, ci.name as char_name FROM leaderboard lb JOIN charinfo ci on ci.id = lb.character_id where game_id = ? ORDER BY primaryscore DESC, secondaryscore ASC, tertiaryScore DESC, last_played ASC;", activityId); + return ProcessQuery(leaderboard); +} + +void MySQLDatabase::SaveScore(const uint32_t playerId, const uint32_t gameId, const Score& score) { + ExecuteInsert("INSERT leaderboard SET primaryScore = ?, secondaryScore = ?, tertiaryScore = ?, character_id = ?, game_id = ?;", + score.primaryScore, score.secondaryScore, score.tertiaryScore, playerId, gameId); +} + +void MySQLDatabase::UpdateScore(const uint32_t playerId, const uint32_t gameId, const Score& score) { + ExecuteInsert("UPDATE leaderboard SET primaryScore = ?, secondaryScore = ?, tertiaryScore = ?, timesPlayed = timesPlayed + 1 WHERE character_id = ? AND game_id = ?;", + score.primaryScore, score.secondaryScore, score.tertiaryScore, playerId, gameId); +} + +std::optional MySQLDatabase::GetPlayerScore(const uint32_t playerId, const uint32_t gameId) { + std::optional toReturn = std::nullopt; + auto res = ExecuteSelect("SELECT * FROM leaderboard WHERE character_id = ? AND game_id = ?;", playerId, gameId); + if (res->next()) { + toReturn = ILeaderboard::Score{ + .primaryScore = res->getFloat("primaryScore"), + .secondaryScore = res->getFloat("secondaryScore"), + .tertiaryScore = res->getFloat("tertiaryScore") + }; + } + + return toReturn; +} + +void MySQLDatabase::IncrementNumWins(const uint32_t playerId, const uint32_t gameId) { + ExecuteUpdate("UPDATE leaderboard SET numWins = numWins + 1 WHERE character_id = ? AND game_id = ?;", playerId, gameId); +} diff --git a/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp new file mode 100644 index 00000000..a9573515 --- /dev/null +++ b/dDatabase/GameDatabase/MySQL/Tables/UgcModularBuild.cpp @@ -0,0 +1,9 @@ +#include "MySQLDatabase.h" + +void MySQLDatabase::InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) { + ExecuteInsert("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)", bigId, modules, characterId); +} + +void MySQLDatabase::DeleteUgcBuild(const LWOOBJID bigId) { + ExecuteDelete("DELETE FROM ugc_modular_build WHERE ugc_id = ?;", bigId); +} diff --git a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h index 1fbb1845..c2a3950a 100644 --- a/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h +++ b/dDatabase/GameDatabase/TestSQL/TestSQLDatabase.h @@ -91,6 +91,16 @@ class TestSQLDatabase : public GameDatabase { void RemoveBehavior(const int32_t behaviorId) override; void UpdateAccountGmLevel(const uint32_t accountId, const eGameMasterLevel gmLevel) override; std::optional GetProperties(const IProperty::PropertyLookup& params) override { return {}; }; + std::vector GetDescendingLeaderboard(const uint32_t activityId) override { return {}; }; + std::vector GetAscendingLeaderboard(const uint32_t activityId) override { return {}; }; + std::vector GetNsLeaderboard(const uint32_t activityId) override { return {}; }; + std::vector GetAgsLeaderboard(const uint32_t activityId) override { return {}; }; + void SaveScore(const uint32_t playerId, const uint32_t gameId, const Score& score) override {}; + void UpdateScore(const uint32_t playerId, const uint32_t gameId, const Score& score) override {}; + std::optional GetPlayerScore(const uint32_t playerId, const uint32_t gameId) override { return {}; }; + void IncrementNumWins(const uint32_t playerId, const uint32_t gameId) override {}; + void InsertUgcBuild(const std::string& modules, const LWOOBJID bigId, const std::optional characterId) override {}; + void DeleteUgcBuild(const LWOOBJID bigId) override {}; }; #endif //!TESTSQLDATABASE_H diff --git a/dDatabase/MigrationRunner.cpp b/dDatabase/MigrationRunner.cpp index 8034a3e2..44ccaa99 100644 --- a/dDatabase/MigrationRunner.cpp +++ b/dDatabase/MigrationRunner.cpp @@ -34,7 +34,7 @@ Migration LoadMigration(std::string path) { void MigrationRunner::RunMigrations() { Database::Get()->CreateMigrationHistoryTable(); - sql::SQLString finalSQL = ""; + std::string finalSQL = ""; bool runSd0Migrations = false; for (const auto& entry : GeneralUtils::GetSqlFileNamesFromFolder((BinaryPathFinder::GetBinaryDir() / "./migrations/dlu/").string())) { auto migration = LoadMigration("dlu/" + entry); @@ -61,12 +61,12 @@ void MigrationRunner::RunMigrations() { } if (!finalSQL.empty()) { - auto migration = GeneralUtils::SplitString(static_cast(finalSQL), ';'); + auto migration = GeneralUtils::SplitString(finalSQL, ';'); for (auto& query : migration) { try { if (query.empty()) continue; - Database::Get()->ExecuteCustomQuery(query.c_str()); - } catch (sql::SQLException& e) { + Database::Get()->ExecuteCustomQuery(query); + } catch (std::exception& e) { LOG("Encountered error running migration: %s", e.what()); } } diff --git a/dGame/Entity.cpp b/dGame/Entity.cpp index 54629888..55b3b3b5 100644 --- a/dGame/Entity.cpp +++ b/dGame/Entity.cpp @@ -1351,6 +1351,11 @@ void Entity::OnCollisionPhantom(const LWOOBJID otherEntity) { callback(other); } + SwitchComponent* switchComp = GetComponent(); + if (switchComp) { + switchComp->OnUse(other); + } + TriggerEvent(eTriggerEventType::ENTER, other); // POI system diff --git a/dGame/LeaderboardManager.cpp b/dGame/LeaderboardManager.cpp index 347bd68e..f3a27919 100644 --- a/dGame/LeaderboardManager.cpp +++ b/dGame/LeaderboardManager.cpp @@ -1,5 +1,6 @@ #include "LeaderboardManager.h" +#include #include #include @@ -72,197 +73,191 @@ void Leaderboard::Serialize(RakNet::BitStream& bitStream) const { bitStream.Write0(); } -void Leaderboard::QueryToLdf(std::unique_ptr& rows) { - Clear(); - if (rows->rowsCount() == 0) return; +// Takes the resulting query from a leaderboard lookup and converts it to the LDF we need +// to send it to a client. +void QueryToLdf(Leaderboard& leaderboard, const std::vector& leaderboardEntries) { + using enum Leaderboard::Type; + leaderboard.Clear(); + if (leaderboardEntries.empty()) return; - this->entries.reserve(rows->rowsCount()); - while (rows->next()) { + for (const auto& leaderboardEntry : leaderboardEntries) { constexpr int32_t MAX_NUM_DATA_PER_ROW = 9; - this->entries.push_back(std::vector()); - auto& entry = this->entries.back(); + auto& entry = leaderboard.PushBackEntry(); entry.reserve(MAX_NUM_DATA_PER_ROW); - entry.push_back(new LDFData(u"CharacterID", rows->getInt("character_id"))); - entry.push_back(new LDFData(u"LastPlayed", rows->getUInt64("lastPlayed"))); - entry.push_back(new LDFData(u"NumPlayed", rows->getInt("timesPlayed"))); - entry.push_back(new LDFData(u"name", GeneralUtils::ASCIIToUTF16(rows->getString("name").c_str()))); - entry.push_back(new LDFData(u"RowNumber", rows->getInt("ranking"))); - switch (leaderboardType) { - case Type::ShootingGallery: - entry.push_back(new LDFData(u"Score", rows->getInt("primaryScore"))); + entry.push_back(new LDFData(u"CharacterID", leaderboardEntry.charId)); + entry.push_back(new LDFData(u"LastPlayed", leaderboardEntry.lastPlayedTimestamp)); + entry.push_back(new LDFData(u"NumPlayed", leaderboardEntry.numTimesPlayed)); + entry.push_back(new LDFData(u"name", GeneralUtils::ASCIIToUTF16(leaderboardEntry.name))); + entry.push_back(new LDFData(u"RowNumber", leaderboardEntry.ranking)); + switch (leaderboard.GetLeaderboardType()) { + case ShootingGallery: + entry.push_back(new LDFData(u"Score", leaderboardEntry.primaryScore)); // Score:1 - entry.push_back(new LDFData(u"Streak", rows->getInt("secondaryScore"))); + entry.push_back(new LDFData(u"Streak", leaderboardEntry.secondaryScore)); // Streak:1 - entry.push_back(new LDFData(u"HitPercentage", (rows->getInt("tertiaryScore") / 100.0f))); + entry.push_back(new LDFData(u"HitPercentage", (leaderboardEntry.tertiaryScore / 100.0f))); // HitPercentage:3 between 0 and 1 break; - case Type::Racing: - entry.push_back(new LDFData(u"BestTime", rows->getDouble("primaryScore"))); + case Racing: + entry.push_back(new LDFData(u"BestTime", leaderboardEntry.primaryScore)); // BestLapTime:3 - entry.push_back(new LDFData(u"BestLapTime", rows->getDouble("secondaryScore"))); + entry.push_back(new LDFData(u"BestLapTime", leaderboardEntry.secondaryScore)); // BestTime:3 entry.push_back(new LDFData(u"License", 1)); // License:1 - 1 if player has completed mission 637 and 0 otherwise - entry.push_back(new LDFData(u"NumWins", rows->getInt("numWins"))); + entry.push_back(new LDFData(u"NumWins", leaderboardEntry.numWins)); // NumWins:1 break; - case Type::UnusedLeaderboard4: - entry.push_back(new LDFData(u"Points", rows->getInt("primaryScore"))); + case UnusedLeaderboard4: + entry.push_back(new LDFData(u"Points", leaderboardEntry.primaryScore)); // Points:1 break; - case Type::MonumentRace: - entry.push_back(new LDFData(u"Time", rows->getInt("primaryScore"))); + case MonumentRace: + entry.push_back(new LDFData(u"Time", leaderboardEntry.primaryScore)); // Time:1(?) break; - case Type::FootRace: - entry.push_back(new LDFData(u"Time", rows->getInt("primaryScore"))); + case FootRace: + entry.push_back(new LDFData(u"Time", leaderboardEntry.primaryScore)); // Time:1 break; - case Type::Survival: - entry.push_back(new LDFData(u"Points", rows->getInt("primaryScore"))); + case Survival: + entry.push_back(new LDFData(u"Points", leaderboardEntry.primaryScore)); // Points:1 - entry.push_back(new LDFData(u"Time", rows->getInt("secondaryScore"))); + entry.push_back(new LDFData(u"Time", leaderboardEntry.secondaryScore)); // Time:1 break; - case Type::SurvivalNS: - entry.push_back(new LDFData(u"Wave", rows->getInt("primaryScore"))); + case SurvivalNS: + entry.push_back(new LDFData(u"Wave", leaderboardEntry.primaryScore)); // Wave:1 - entry.push_back(new LDFData(u"Time", rows->getInt("secondaryScore"))); + entry.push_back(new LDFData(u"Time", leaderboardEntry.secondaryScore)); // Time:1 break; - case Type::Donations: - entry.push_back(new LDFData(u"Score", rows->getInt("primaryScore"))); + case Donations: + entry.push_back(new LDFData(u"Score", leaderboardEntry.primaryScore)); // Score:1 break; - case Type::None: - // This type is included here simply to resolve a compiler warning on mac about unused enum types - break; + case None: + [[fallthrough]]; default: break; } } } -const std::string_view Leaderboard::GetOrdering(Leaderboard::Type leaderboardType) { - // Use a switch case and return desc for all 3 columns if higher is better and asc if lower is better - switch (leaderboardType) { - case Type::Racing: - case Type::MonumentRace: - return "primaryScore ASC, secondaryScore ASC, tertiaryScore ASC"; - case Type::Survival: - return Game::config->GetValue("classic_survival_scoring") == "1" ? - "secondaryScore DESC, primaryScore DESC, tertiaryScore DESC" : - "primaryScore DESC, secondaryScore DESC, tertiaryScore DESC"; - case Type::SurvivalNS: - return "primaryScore DESC, secondaryScore ASC, tertiaryScore DESC"; - case Type::ShootingGallery: - case Type::FootRace: - case Type::UnusedLeaderboard4: - case Type::Donations: - case Type::None: - default: - return "primaryScore DESC, secondaryScore DESC, tertiaryScore DESC"; +std::vector FilterTo10(const std::vector& leaderboard, const uint32_t relatedPlayer, const Leaderboard::InfoType infoType) { + std::vector toReturn; + + int32_t index = 0; + // for friends and top, we dont need to find this players index. + if (infoType == Leaderboard::InfoType::MyStanding || infoType == Leaderboard::InfoType::Friends) { + for (; index < leaderboard.size(); index++) { + if (leaderboard[index].charId == relatedPlayer) break; + } } + + if (leaderboard.size() < 10) { + toReturn.assign(leaderboard.begin(), leaderboard.end()); + index = 0; + } else if (index < 10) { + toReturn.assign(leaderboard.begin(), leaderboard.begin() + 10); // get the top 10 since we are in the top 10 + index = 0; + } else if (index > leaderboard.size() - 10) { + toReturn.assign(leaderboard.end() - 10, leaderboard.end()); // get the bottom 10 since we are in the bottom 10 + index = leaderboard.size() - 10; + } else { + toReturn.assign(leaderboard.begin() + index - 5, leaderboard.begin() + index + 5); // get the 5 above and below + index -= 5; + } + + int32_t i = index; + for (auto& entry : toReturn) { + entry.ranking = ++i; + } + + return toReturn; } -void Leaderboard::SetupLeaderboard(bool weekly, uint32_t resultStart, uint32_t resultEnd) { - resultStart++; - resultEnd++; - // We need everything except 1 column so i'm selecting * from leaderboard - const std::string queryBase = - R"QUERY( - WITH leaderboardsRanked AS ( - SELECT leaderboard.*, charinfo.name, - RANK() OVER - ( - ORDER BY %s, UNIX_TIMESTAMP(last_played) ASC, id DESC - ) AS ranking - FROM leaderboard JOIN charinfo on charinfo.id = leaderboard.character_id - WHERE game_id = ? %s - ), - myStanding AS ( - SELECT - ranking as myRank - FROM leaderboardsRanked - WHERE id = ? - ), - lowestRanking AS ( - SELECT MAX(ranking) AS lowestRank - FROM leaderboardsRanked - ) - SELECT leaderboardsRanked.*, character_id, UNIX_TIMESTAMP(last_played) as lastPlayed, leaderboardsRanked.name, leaderboardsRanked.ranking FROM leaderboardsRanked, myStanding, lowestRanking - WHERE leaderboardsRanked.ranking - BETWEEN - LEAST(GREATEST(CAST(myRank AS SIGNED) - 5, %i), CAST(lowestRanking.lowestRank AS SIGNED) - 9) - AND - LEAST(GREATEST(myRank + 5, %i), lowestRanking.lowestRank) - ORDER BY ranking ASC; - )QUERY"; +std::vector FilterWeeklies(const std::vector& leaderboard) { + // Filter the leaderboard to only include entries from the last week + const auto currentTime = std::chrono::system_clock::now(); + auto epochTime = currentTime.time_since_epoch().count(); + constexpr auto SECONDS_IN_A_WEEK = 60 * 60 * 24 * 7; // if you think im taking leap seconds into account thats cute. - std::string friendsFilter = - R"QUERY( - AND ( - character_id IN ( - SELECT fr.requested_player FROM ( - SELECT CASE - WHEN player_id = ? THEN friend_id - WHEN friend_id = ? THEN player_id - END AS requested_player - FROM friends - ) AS fr - JOIN charinfo AS ci - ON ci.id = fr.requested_player - WHERE fr.requested_player IS NOT NULL - ) - OR character_id = ? - ) - )QUERY"; - - std::string weeklyFilter = " AND UNIX_TIMESTAMP(last_played) BETWEEN UNIX_TIMESTAMP(date_sub(now(),INTERVAL 1 WEEK)) AND UNIX_TIMESTAMP(now()) "; - - std::string filter; - // Setup our filter based on the query type - if (this->infoType == InfoType::Friends) filter += friendsFilter; - if (this->weekly) filter += weeklyFilter; - const auto orderBase = GetOrdering(this->leaderboardType); - - // For top query, we want to just rank all scores, but for all others we need the scores around a specific player - std::string baseLookup; - if (this->infoType == InfoType::Top) { - baseLookup = "SELECT id, last_played FROM leaderboard WHERE game_id = ? " + (this->weekly ? weeklyFilter : std::string("")) + " ORDER BY "; - baseLookup += orderBase.data(); - } else { - baseLookup = "SELECT id, last_played FROM leaderboard WHERE game_id = ? " + (this->weekly ? weeklyFilter : std::string("")) + " AND character_id = "; - baseLookup += std::to_string(static_cast(this->relatedPlayer)); + std::vector weeklyLeaderboard; + for (const auto& entry : leaderboard) { + if (epochTime - entry.lastPlayedTimestamp < SECONDS_IN_A_WEEK) { + weeklyLeaderboard.push_back(entry); + } } - baseLookup += " LIMIT 1"; - LOG_DEBUG("query is %s", baseLookup.c_str()); - std::unique_ptr baseQuery(Database::Get()->CreatePreppedStmt(baseLookup)); - baseQuery->setInt(1, this->gameID); - std::unique_ptr baseResult(baseQuery->executeQuery()); - if (!baseResult->next()) return; // In this case, there are no entries in the leaderboard for this game. + return weeklyLeaderboard; +} - uint32_t relatedPlayerLeaderboardId = baseResult->getInt("id"); - - // Create and execute the actual save here. Using a heap allocated buffer to avoid stack overflow - constexpr uint16_t STRING_LENGTH = 4096; - std::unique_ptr lookupBuffer = std::make_unique(STRING_LENGTH); - int32_t res = snprintf(lookupBuffer.get(), STRING_LENGTH, queryBase.c_str(), orderBase.data(), filter.c_str(), resultStart, resultEnd); - DluAssert(res != -1); - std::unique_ptr query(Database::Get()->CreatePreppedStmt(lookupBuffer.get())); - LOG_DEBUG("Query is %s vars are %i %i %i", lookupBuffer.get(), this->gameID, this->relatedPlayer, relatedPlayerLeaderboardId); - query->setInt(1, this->gameID); - if (this->infoType == InfoType::Friends) { - query->setInt(2, this->relatedPlayer); - query->setInt(3, this->relatedPlayer); - query->setInt(4, this->relatedPlayer); - query->setInt(5, relatedPlayerLeaderboardId); - } else { - query->setInt(2, relatedPlayerLeaderboardId); +std::vector FilterFriends(const std::vector& leaderboard, const uint32_t relatedPlayer) { + // Filter the leaderboard to only include friends of the player + auto friendOfPlayer = Database::Get()->GetFriendsList(relatedPlayer); + std::vector friendsLeaderboard; + for (const auto& entry : leaderboard) { + const auto res = std::ranges::find_if(friendOfPlayer, [&entry, relatedPlayer](const FriendData& data) { + return entry.charId == data.friendID || entry.charId == relatedPlayer; + }); + if (res != friendOfPlayer.cend()) { + friendsLeaderboard.push_back(entry); + } } - std::unique_ptr result(query->executeQuery()); - QueryToLdf(result); + + return friendsLeaderboard; +} + +std::vector ProcessLeaderboard( + const std::vector& leaderboard, + const bool weekly, + const Leaderboard::InfoType infoType, + const uint32_t relatedPlayer) { + std::vector toReturn; + + if (infoType == Leaderboard::InfoType::Friends) { + const auto friendsLeaderboard = FilterFriends(leaderboard, relatedPlayer); + toReturn = FilterTo10(weekly ? FilterWeeklies(friendsLeaderboard) : friendsLeaderboard, relatedPlayer, infoType); + } else { + toReturn = FilterTo10(weekly ? FilterWeeklies(leaderboard) : leaderboard, relatedPlayer, infoType); + } + + return toReturn; +} + +void Leaderboard::SetupLeaderboard(bool weekly) { + const auto leaderboardType = LeaderboardManager::GetLeaderboardType(gameID); + std::vector leaderboardRes; + + switch (leaderboardType) { + case Type::SurvivalNS: + leaderboardRes = Database::Get()->GetNsLeaderboard(gameID); + break; + case Type::Survival: + leaderboardRes = Database::Get()->GetAgsLeaderboard(gameID); + break; + case Type::Racing: + [[fallthrough]]; + case Type::MonumentRace: + leaderboardRes = Database::Get()->GetAscendingLeaderboard(gameID); + break; + case Type::ShootingGallery: + [[fallthrough]]; + case Type::FootRace: + [[fallthrough]]; + case Type::Donations: + [[fallthrough]]; + case Type::None: + [[fallthrough]]; + default: + leaderboardRes = Database::Get()->GetDescendingLeaderboard(gameID); + break; + } + + const auto processedLeaderboard = ProcessLeaderboard(leaderboardRes, weekly, infoType, relatedPlayer); + + QueryToLdf(*this, processedLeaderboard); } void Leaderboard::Send(const LWOOBJID targetID) const { @@ -272,129 +267,41 @@ void Leaderboard::Send(const LWOOBJID targetID) const { } } -std::string FormatInsert(const Leaderboard::Type& type, const Score& score, const bool useUpdate) { - std::string insertStatement; - if (useUpdate) { - insertStatement = - R"QUERY( - UPDATE leaderboard - SET primaryScore = %f, secondaryScore = %f, tertiaryScore = %f, - timesPlayed = timesPlayed + 1 WHERE character_id = ? AND game_id = ?; - )QUERY"; - } else { - insertStatement = - R"QUERY( - INSERT leaderboard SET - primaryScore = %f, secondaryScore = %f, tertiaryScore = %f, - character_id = ?, game_id = ?; - )QUERY"; - } - - constexpr uint16_t STRING_LENGTH = 400; - // Then fill in our score - char finishedQuery[STRING_LENGTH]; - int32_t res = snprintf(finishedQuery, STRING_LENGTH, insertStatement.c_str(), score.GetPrimaryScore(), score.GetSecondaryScore(), score.GetTertiaryScore()); - DluAssert(res != -1); - return finishedQuery; -} - void LeaderboardManager::SaveScore(const LWOOBJID& playerID, const GameID activityId, const float primaryScore, const float secondaryScore, const float tertiaryScore) { const Leaderboard::Type leaderboardType = GetLeaderboardType(activityId); - std::unique_ptr query(Database::Get()->CreatePreppedStmt("SELECT * FROM leaderboard WHERE character_id = ? AND game_id = ?;")); - query->setInt(1, playerID); - query->setInt(2, activityId); - std::unique_ptr myScoreResult(query->executeQuery()); + const auto oldScore = Database::Get()->GetPlayerScore(playerID, activityId); - std::string saveQuery("UPDATE leaderboard SET timesPlayed = timesPlayed + 1 WHERE character_id = ? AND game_id = ?;"); - Score newScore(primaryScore, secondaryScore, tertiaryScore); - if (myScoreResult->next()) { - Score oldScore; - bool lowerScoreBetter = false; - switch (leaderboardType) { - // Higher score better - case Leaderboard::Type::ShootingGallery: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - oldScore.SetSecondaryScore(myScoreResult->getInt("secondaryScore")); - oldScore.SetTertiaryScore(myScoreResult->getInt("tertiaryScore")); - break; - } - case Leaderboard::Type::FootRace: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - break; - } - case Leaderboard::Type::Survival: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - oldScore.SetSecondaryScore(myScoreResult->getInt("secondaryScore")); - break; - } - case Leaderboard::Type::SurvivalNS: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - oldScore.SetSecondaryScore(myScoreResult->getInt("secondaryScore")); - break; - } - case Leaderboard::Type::UnusedLeaderboard4: - case Leaderboard::Type::Donations: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - newScore.SetPrimaryScore(oldScore.GetPrimaryScore() + newScore.GetPrimaryScore()); - break; - } - case Leaderboard::Type::Racing: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - oldScore.SetSecondaryScore(myScoreResult->getInt("secondaryScore")); - - // For wins we dont care about the score, just the time, so zero out the tertiary. - // Wins are updated later. - oldScore.SetTertiaryScore(0); - newScore.SetTertiaryScore(0); - lowerScoreBetter = true; - break; - } - case Leaderboard::Type::MonumentRace: { - oldScore.SetPrimaryScore(myScoreResult->getInt("primaryScore")); - lowerScoreBetter = true; - // Do score checking here - break; - } - case Leaderboard::Type::None: - default: - LOG("Unknown leaderboard type %i for game %i. Cannot save score!", leaderboardType, activityId); - return; - } + ILeaderboard::Score newScore{ .primaryScore = primaryScore, .secondaryScore = secondaryScore, .tertiaryScore = tertiaryScore }; + if (oldScore.has_value()) { + bool lowerScoreBetter = leaderboardType == Leaderboard::Type::Racing || leaderboardType == Leaderboard::Type::MonumentRace; bool newHighScore = lowerScoreBetter ? newScore < oldScore : newScore > oldScore; // Nimbus station has a weird leaderboard where we need a custom scoring system if (leaderboardType == Leaderboard::Type::SurvivalNS) { - newHighScore = newScore.GetPrimaryScore() > oldScore.GetPrimaryScore() || - (newScore.GetPrimaryScore() == oldScore.GetPrimaryScore() && newScore.GetSecondaryScore() < oldScore.GetSecondaryScore()); + newHighScore = newScore.primaryScore > oldScore->primaryScore || + (newScore.primaryScore == oldScore->primaryScore && newScore.secondaryScore < oldScore->secondaryScore); } else if (leaderboardType == Leaderboard::Type::Survival && Game::config->GetValue("classic_survival_scoring") == "1") { - Score oldScoreFlipped(oldScore.GetSecondaryScore(), oldScore.GetPrimaryScore()); - Score newScoreFlipped(newScore.GetSecondaryScore(), newScore.GetPrimaryScore()); + ILeaderboard::Score oldScoreFlipped{oldScore->secondaryScore, oldScore->primaryScore, oldScore->tertiaryScore}; + ILeaderboard::Score newScoreFlipped{newScore.secondaryScore, newScore.primaryScore, newScore.tertiaryScore}; newHighScore = newScoreFlipped > oldScoreFlipped; } + if (newHighScore) { - saveQuery = FormatInsert(leaderboardType, newScore, true); + Database::Get()->UpdateScore(playerID, activityId, newScore); } } else { - saveQuery = FormatInsert(leaderboardType, newScore, false); + Database::Get()->SaveScore(playerID, activityId, newScore); } - LOG("save query %s %i %i", saveQuery.c_str(), playerID, activityId); - std::unique_ptr saveStatement(Database::Get()->CreatePreppedStmt(saveQuery)); - saveStatement->setInt(1, playerID); - saveStatement->setInt(2, activityId); - saveStatement->execute(); // track wins separately if (leaderboardType == Leaderboard::Type::Racing && tertiaryScore != 0.0f) { - std::unique_ptr winUpdate(Database::Get()->CreatePreppedStmt("UPDATE leaderboard SET numWins = numWins + 1 WHERE character_id = ? AND game_id = ?;")); - winUpdate->setInt(1, playerID); - winUpdate->setInt(2, activityId); - winUpdate->execute(); + Database::Get()->IncrementNumWins(playerID, activityId); } } -void LeaderboardManager::SendLeaderboard(const GameID gameID, const Leaderboard::InfoType infoType, const bool weekly, const LWOOBJID playerID, const LWOOBJID targetID, const uint32_t resultStart, const uint32_t resultEnd) { +void LeaderboardManager::SendLeaderboard(const GameID gameID, const Leaderboard::InfoType infoType, const bool weekly, const LWOOBJID playerID, const LWOOBJID targetID) { Leaderboard leaderboard(gameID, infoType, weekly, playerID, GetLeaderboardType(gameID)); - leaderboard.SetupLeaderboard(weekly, resultStart, resultEnd); + leaderboard.SetupLeaderboard(weekly); leaderboard.Send(targetID); } diff --git a/dGame/LeaderboardManager.h b/dGame/LeaderboardManager.h index 527ae02d..af879573 100644 --- a/dGame/LeaderboardManager.h +++ b/dGame/LeaderboardManager.h @@ -9,46 +9,10 @@ #include "dCommonVars.h" #include "LDFFormat.h" -namespace sql { - class ResultSet; -}; - namespace RakNet { class BitStream; }; -class Score { -public: - Score() { - primaryScore = 0; - secondaryScore = 0; - tertiaryScore = 0; - } - Score(const float primaryScore, const float secondaryScore = 0, const float tertiaryScore = 0) { - this->primaryScore = primaryScore; - this->secondaryScore = secondaryScore; - this->tertiaryScore = tertiaryScore; - } - bool operator<(const Score& rhs) const { - return primaryScore < rhs.primaryScore || (primaryScore == rhs.primaryScore && secondaryScore < rhs.secondaryScore) || (primaryScore == rhs.primaryScore && secondaryScore == rhs.secondaryScore && tertiaryScore < rhs.tertiaryScore); - } - bool operator>(const Score& rhs) const { - return primaryScore > rhs.primaryScore || (primaryScore == rhs.primaryScore && secondaryScore > rhs.secondaryScore) || (primaryScore == rhs.primaryScore && secondaryScore == rhs.secondaryScore && tertiaryScore > rhs.tertiaryScore); - } - void SetPrimaryScore(const float score) { primaryScore = score; } - float GetPrimaryScore() const { return primaryScore; } - - void SetSecondaryScore(const float score) { secondaryScore = score; } - float GetSecondaryScore() const { return secondaryScore; } - - void SetTertiaryScore(const float score) { tertiaryScore = score; } - float GetTertiaryScore() const { return tertiaryScore; } -private: - float primaryScore; - float secondaryScore; - float tertiaryScore; -}; - using GameID = uint32_t; class Leaderboard { @@ -79,7 +43,7 @@ public: /** * @brief Resets the leaderboard state and frees its allocated memory - * + * */ void Clear(); @@ -96,20 +60,16 @@ public: * @param resultStart The index to start the leaderboard at. Zero indexed. * @param resultEnd The index to end the leaderboard at. Zero indexed. */ - void SetupLeaderboard(bool weekly, uint32_t resultStart = 0, uint32_t resultEnd = 10); + void SetupLeaderboard(bool weekly); /** * Sends the leaderboard to the client specified by targetID. */ void Send(const LWOOBJID targetID) const; - // Helper function to get the columns, ordering and insert format for a leaderboard - static const std::string_view GetOrdering(Type leaderboardType); -private: - // Takes the resulting query from a leaderboard lookup and converts it to the LDF we need - // to send it to a client. - void QueryToLdf(std::unique_ptr& rows); + +private: using LeaderboardEntry = std::vector; using LeaderboardEntries = std::vector; @@ -119,10 +79,18 @@ private: InfoType infoType; Leaderboard::Type leaderboardType; bool weekly; +public: + LeaderboardEntry& PushBackEntry() { + return entries.emplace_back(); + } + + Type GetLeaderboardType() const { + return leaderboardType; + } }; namespace LeaderboardManager { - void SendLeaderboard(const GameID gameID, const Leaderboard::InfoType infoType, const bool weekly, const LWOOBJID playerID, const LWOOBJID targetID, const uint32_t resultStart = 0, const uint32_t resultEnd = 10); + void SendLeaderboard(const GameID gameID, const Leaderboard::InfoType infoType, const bool weekly, const LWOOBJID playerID, const LWOOBJID targetID); void SaveScore(const LWOOBJID& playerID, const GameID activityId, const float primaryScore, const float secondaryScore = 0, const float tertiaryScore = 0); diff --git a/dGame/dComponents/SwitchComponent.cpp b/dGame/dComponents/SwitchComponent.cpp index cb13cc7f..4f48fb46 100644 --- a/dGame/dComponents/SwitchComponent.cpp +++ b/dGame/dComponents/SwitchComponent.cpp @@ -82,7 +82,6 @@ void SwitchComponent::EntityEnter(Entity* entity) { RenderComponent::PlayAnimation(m_Parent, u"engaged"); m_PetBouncer->SetPetBouncerEnabled(true); } else { - GameMessages::SendKnockback(entity->GetObjectID(), m_Parent->GetObjectID(), m_Parent->GetObjectID(), 0.0f, NiPoint3(0.0f, 17.0f, 0.0f)); Game::entityManager->SerializeEntity(m_Parent); } diff --git a/dGame/dGameMessages/GameMessages.cpp b/dGame/dGameMessages/GameMessages.cpp index fc77e690..8e94cee3 100644 --- a/dGame/dGameMessages/GameMessages.cpp +++ b/dGame/dGameMessages/GameMessages.cpp @@ -1691,7 +1691,7 @@ void GameMessages::HandleRequestActivitySummaryLeaderboardData(RakNet::BitStream bool weekly = inStream.ReadBit(); - LeaderboardManager::SendLeaderboard(gameID, queryType, weekly, entity->GetObjectID(), entity->GetObjectID(), resultsStart, resultsEnd); + LeaderboardManager::SendLeaderboard(gameID, queryType, weekly, entity->GetObjectID(), entity->GetObjectID()); } void GameMessages::HandleActivityStateChangeRequest(RakNet::BitStream& inStream, Entity* entity) { @@ -5066,9 +5066,7 @@ void GameMessages::HandleModularBuildConvertModel(RakNet::BitStream& inStream, E item->Disassemble(TEMP_MODELS); - std::unique_ptr stmt(Database::Get()->CreatePreppedStmt("DELETE FROM ugc_modular_build where ugc_id = ?")); - stmt->setUInt64(1, item->GetSubKey()); - stmt->execute(); + Database::Get()->DeleteUgcBuild(item->GetSubKey()); item->SetCount(item->GetCount() - 1, false, false, true, eLootSourceType::QUICKBUILD); } @@ -5394,6 +5392,8 @@ void GameMessages::HandleRemoveItemFromInventory(RakNet::BitStream& inStream, En const auto itemType = static_cast(item->GetInfo().itemType); if (itemType == eItemType::MODEL || itemType == eItemType::LOOT_MODEL) { item->DisassembleModel(iStackCount); + } else if (itemType == eItemType::VEHICLE) { + Database::Get()->DeleteUgcBuild(item->GetSubKey()); } auto lot = item->GetLot(); item->SetCount(item->GetCount() - iStackCount, true); @@ -5569,12 +5569,8 @@ void GameMessages::HandleModularBuildFinish(RakNet::BitStream& inStream, Entity* inv->AddItem(8092, 1, eLootSourceType::QUICKBUILD, eInventoryType::MODELS, config, LWOOBJID_EMPTY, true, false, newIdBig); } - std::unique_ptr stmt(Database::Get()->CreatePreppedStmt("INSERT INTO ugc_modular_build (ugc_id, ldf_config, character_id) VALUES (?,?,?)")); - stmt->setUInt64(1, newIdBig); - stmt->setString(2, GeneralUtils::UTF16ToWTF8(modules).c_str()); auto* pCharacter = character->GetCharacter(); - pCharacter ? stmt->setUInt(3, pCharacter->GetID()) : stmt->setNull(3, sql::DataType::BIGINT); - stmt->execute(); + Database::Get()->InsertUgcBuild(GeneralUtils::UTF16ToWTF8(modules), newIdBig, pCharacter ? std::optional(character->GetCharacter()->GetID()) : std::nullopt); auto* missionComponent = character->GetComponent(); diff --git a/dMasterServer/MasterServer.cpp b/dMasterServer/MasterServer.cpp index 7edcaced..9a54a64e 100644 --- a/dMasterServer/MasterServer.cpp +++ b/dMasterServer/MasterServer.cpp @@ -103,7 +103,7 @@ int main(int argc, char** argv) { //Connect to the MySQL Database try { Database::Connect(); - } catch (sql::SQLException& ex) { + } catch (std::exception& ex) { LOG("Got an error while connecting to the database: %s", ex.what()); LOG("Migrations not run"); return EXIT_FAILURE; @@ -264,7 +264,7 @@ int main(int argc, char** argv) { //Create account try { Database::Get()->InsertNewAccount(username, std::string(hash, BCRYPT_HASHSIZE)); - } catch (sql::SQLException& e) { + } catch (std::exception& e) { LOG("A SQL error occurred!:\n %s", e.what()); return EXIT_FAILURE; } diff --git a/dMasterServer/PersistentIDManager.cpp b/dMasterServer/PersistentIDManager.cpp index 3e56fadc..1b3a1c0b 100644 --- a/dMasterServer/PersistentIDManager.cpp +++ b/dMasterServer/PersistentIDManager.cpp @@ -24,9 +24,9 @@ void PersistentIDManager::Initialize() { LOG("Invalid persistent object ID in database. Aborting to prevent bad id generation."); throw std::runtime_error("Invalid persistent object ID in database. Aborting to prevent bad id generation."); } - } catch (sql::SQLException& e) { + } catch (std::exception& e) { LOG("Unable to fetch max persistent object ID in use. This will cause issues. Aborting to prevent collisions."); - LOG("SQL error: %s", e.what()); + LOG("Error: %s", e.what()); throw e; } } diff --git a/dScripts/ActivityManager.cpp b/dScripts/ActivityManager.cpp index 0f251dbf..8ba4834e 100644 --- a/dScripts/ActivityManager.cpp +++ b/dScripts/ActivityManager.cpp @@ -121,7 +121,7 @@ void ActivityManager::GetLeaderboardData(Entity* self, const LWOOBJID playerID, auto* sac = self->GetComponent(); uint32_t gameID = sac != nullptr ? sac->GetActivityID() : self->GetLOT(); // Save the new score to the leaderboard and show the leaderboard to the player - LeaderboardManager::SendLeaderboard(activityID, Leaderboard::InfoType::MyStanding, false, playerID, self->GetObjectID(), 0, numResults); + LeaderboardManager::SendLeaderboard(activityID, Leaderboard::InfoType::MyStanding, false, playerID, self->GetObjectID()); } void ActivityManager::ActivityTimerStart(Entity* self, const std::string& timerName, const float_t updateInterval, diff --git a/dWorldServer/WorldServer.cpp b/dWorldServer/WorldServer.cpp index c723a01e..c3dbeaca 100644 --- a/dWorldServer/WorldServer.cpp +++ b/dWorldServer/WorldServer.cpp @@ -194,7 +194,7 @@ int main(int argc, char** argv) { //Connect to the MySQL Database: try { Database::Connect(); - } catch (sql::SQLException& ex) { + } catch (std::exception& ex) { LOG("Got an error while connecting to the database: %s", ex.what()); return EXIT_FAILURE; } diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 24adec61..d95a24bc 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -4,15 +4,6 @@ enable_testing() find_package(GoogleTest REQUIRED) include(GoogleTest) -if(APPLE) - set(CMAKE_INSTALL_RPATH_USE_LINK_PATH True) - set(CMAKE_BUILD_WITH_INSTALL_RPATH True) - set(CMAKE_INSTALL_RPATH "@executable_path") -endif() - -add_custom_target(conncpp_tests - ${CMAKE_COMMAND} -E copy $ ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) - # Add the subdirectories add_subdirectory(dCommonTests) add_subdirectory(dGameTests) diff --git a/tests/dCommonTests/CMakeLists.txt b/tests/dCommonTests/CMakeLists.txt index ef7c4cba..17b31ced 100644 --- a/tests/dCommonTests/CMakeLists.txt +++ b/tests/dCommonTests/CMakeLists.txt @@ -17,18 +17,13 @@ list(APPEND DCOMMONTEST_SOURCES ${DENUMS_TESTS}) # Set our executable add_executable(dCommonTests ${DCOMMONTEST_SOURCES}) -add_dependencies(dCommonTests conncpp_tests) -# Apple needs some special linkage for the mariadb connector for tests. +# Needs to be in binary dir for ctest if(APPLE) -add_custom_command(TARGET dCommonTests POST_BUILD - COMMAND otool ARGS -l dCommonTests - COMMAND otool ARGS -L dCommonTests - COMMAND ls - COMMAND otool ARGS -D libmariadbcpp.dylib - COMMAND install_name_tool ARGS -change libmariadbcpp.dylib @rpath/libmariadbcpp.dylib dCommonTests - COMMAND otool ARGS -L dCommonTests - WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + add_custom_target(dCommonTestsLink + ${CMAKE_COMMAND} -E copy $ ${CMAKE_CURRENT_BINARY_DIR}) + + add_dependencies(dCommonTests dCommonTestsLink) endif() # Link needed libraries diff --git a/tests/dGameTests/CMakeLists.txt b/tests/dGameTests/CMakeLists.txt index e1c29433..f4749ce8 100644 --- a/tests/dGameTests/CMakeLists.txt +++ b/tests/dGameTests/CMakeLists.txt @@ -13,14 +13,12 @@ file(COPY ${COMPONENT_TEST_DATA} DESTINATION ${CMAKE_CURRENT_BINARY_DIR}) # Add the executable. Remember to add all tests above this! add_executable(dGameTests ${DGAMETEST_SOURCES}) -add_dependencies(dGameTests conncpp_tests) -# Apple needs some special linkage for the mariadb connector for tests. if(APPLE) -add_custom_command(TARGET dGameTests POST_BUILD - COMMAND install_name_tool ARGS -change libmariadbcpp.dylib @rpath/libmariadbcpp.dylib dGameTests - COMMAND otool ARGS -L dGameTests - WORKING_DIRECTORY ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) + add_custom_target(dGameTestsLink + ${CMAKE_COMMAND} -E copy $ ${CMAKE_CURRENT_BINARY_DIR}) + + add_dependencies(dGameTests dGameTestsLink) endif() target_link_libraries(dGameTests ${COMMON_LIBRARIES} GTest::gtest_main diff --git a/thirdparty/SQLite/sqlite3.c b/thirdparty/SQLite/sqlite3.c index 786da6bb..099c5482 100644 --- a/thirdparty/SQLite/sqlite3.c +++ b/thirdparty/SQLite/sqlite3.c @@ -1,6 +1,6 @@ /****************************************************************************** ** This file is an amalgamation of many separate C source files from SQLite -** version 3.13.0. By combining all the individual C code files into this +** version 3.47.1. By combining all the individual C code files into this ** single large file, the entire code can be compiled as a single translation ** unit. This allows many compilers to do optimizations that would not be ** possible if the files were compiled separately. Performance improvements @@ -9,13 +9,16 @@ ** ** This file is all you need to compile SQLite. To use SQLite in other ** programs, you need this file and the "sqlite3.h" header file that defines -** the programming interface to the SQLite library. (If you do not have +** the programming interface to the SQLite library. (If you do not have ** the "sqlite3.h" header file at hand, you will find a copy embedded within ** the text of this file. Search for "Begin file sqlite3.h" to find the start ** of the embedded sqlite3.h header file.) Additional code files may be needed ** if you want a wrapper to interface SQLite with your choice of programming ** language. The code for the "sqlite3" command-line shell is also in a ** separate file. This file contains only code for the core SQLite library. +** +** The content in this amalgamation comes from Fossil check-in +** b95d11e958643b969c47a8e5857f3793b9e6. */ #define SQLITE_CORE 1 #define SQLITE_AMALGAMATION 1 @@ -37,8 +40,8 @@ ** Internal interface definitions for SQLite. ** */ -#ifndef _SQLITEINT_H_ -#define _SQLITEINT_H_ +#ifndef SQLITEINT_H +#define SQLITEINT_H /* Special Comments: ** @@ -50,29 +53,29 @@ ** used on lines of code that actually ** implement parts of coverage testing. ** -** OPTIMIZATION-IF-TRUE - This branch is allowed to alway be false +** OPTIMIZATION-IF-TRUE - This branch is allowed to always be false ** and the correct answer is still obtained, ** though perhaps more slowly. ** -** OPTIMIZATION-IF-FALSE - This branch is allowed to alway be true +** OPTIMIZATION-IF-FALSE - This branch is allowed to always be true ** and the correct answer is still obtained, ** though perhaps more slowly. ** ** PREVENTS-HARMLESS-OVERREAD - This branch prevents a buffer overread ** that would be harmless and undetectable -** if it did occur. +** if it did occur. ** ** In all cases, the special comment must be enclosed in the usual -** slash-asterisk...asterisk-slash comment marks, with no spaces between the +** slash-asterisk...asterisk-slash comment marks, with no spaces between the ** asterisks and the comment text. */ /* -** Make sure that rand_s() is available on Windows systems with MSVC 2005 -** or higher. +** Make sure the Tcl calling convention macro is defined. This macro is +** only used by test code and Tcl integration code. */ -#if defined(_MSC_VER) && _MSC_VER>=1400 -# define _CRT_RAND_S +#ifndef SQLITE_TCLAPI +# define SQLITE_TCLAPI #endif /* @@ -97,8 +100,8 @@ ** ** This file contains code that is specific to MSVC. */ -#ifndef _MSVC_H_ -#define _MSVC_H_ +#ifndef SQLITE_MSVC_H +#define SQLITE_MSVC_H #if defined(_MSC_VER) #pragma warning(disable : 4054) @@ -118,7 +121,16 @@ #pragma warning(disable : 4706) #endif /* defined(_MSC_VER) */ -#endif /* _MSVC_H_ */ +#if defined(_MSC_VER) && !defined(_WIN64) +#undef SQLITE_4_BYTE_ALIGNED_MALLOC +#define SQLITE_4_BYTE_ALIGNED_MALLOC +#endif /* defined(_MSC_VER) && !defined(_WIN64) */ + +#if !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 +#define HAVE_LOG2 0 +#endif /* !defined(HAVE_LOG2) && defined(_MSC_VER) && _MSC_VER<1800 */ + +#endif /* SQLITE_MSVC_H */ /************** End of msvc.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -196,12 +208,41 @@ # define _LARGEFILE_SOURCE 1 #endif -/* What version of GCC is being used. 0 means GCC is not being used */ -#ifdef __GNUC__ +/* The GCC_VERSION and MSVC_VERSION macros are used to +** conditionally include optimizations for each of these compilers. A +** value of 0 means that compiler is not being used. The +** SQLITE_DISABLE_INTRINSIC macro means do not use any compiler-specific +** optimizations, and hence set all compiler macros to 0 +** +** There was once also a CLANG_VERSION macro. However, we learn that the +** version numbers in clang are for "marketing" only and are inconsistent +** and unreliable. Fortunately, all versions of clang also recognize the +** gcc version numbers and have reasonable settings for gcc version numbers, +** so the GCC_VERSION macro will be set to a correct non-zero value even +** when compiling with clang. +*/ +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) # define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) #else # define GCC_VERSION 0 #endif +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif + +/* +** Some C99 functions in "math.h" are only present for MSVC when its version +** is associated with Visual Studio 2013 or higher. +*/ +#ifndef SQLITE_HAVE_C99_MATH_FUNCS +# if MSVC_VERSION==0 || MSVC_VERSION>=1800 +# define SQLITE_HAVE_C99_MATH_FUNCS (1) +# else +# define SQLITE_HAVE_C99_MATH_FUNCS (0) +# endif +#endif /* Needed for various definitions... */ #if defined(__GNUC__) && !defined(_GNU_SOURCE) @@ -212,6 +253,18 @@ # define _BSD_SOURCE #endif +/* +** Macro to disable warnings about missing "break" at the end of a "case". +*/ +#if defined(__has_attribute) +# if __has_attribute(fallthrough) +# define deliberate_fall_through __attribute__((fallthrough)); +# endif +#endif +#if !defined(deliberate_fall_through) +# define deliberate_fall_through +#endif + /* ** For MinGW, check to see if we can include the header file containing its ** version information, among other things. Normally, this internal MinGW @@ -244,6 +297,17 @@ # define _USE_32BIT_TIME_T #endif +/* Optionally #include a user-defined header, whereby compilation options +** may be set prior to where they take effect, but after platform setup. +** If SQLITE_CUSTOM_INCLUDE=? is defined, its value names the #include +** file. +*/ +#ifdef SQLITE_CUSTOM_INCLUDE +# define INC_STRINGIFY_(f) #f +# define INC_STRINGIFY(f) INC_STRINGIFY_(f) +# include INC_STRINGIFY(SQLITE_CUSTOM_INCLUDE) +#endif + /* The public SQLite interface. The _FILE_OFFSET_BITS macro must appear ** first in QNX. Also, the _USE_32BIT_TIME_T macro must appear first for ** MinGW. @@ -251,7 +315,7 @@ /************** Include sqlite3.h in the middle of sqliteInt.h ***************/ /************** Begin file sqlite3.h *****************************************/ /* -** 2001 September 15 +** 2001-09-15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -282,8 +346,8 @@ ** the version number) and changes its name to "sqlite3.h" as ** part of the build process. */ -#ifndef _SQLITE3_H_ -#define _SQLITE3_H_ +#ifndef SQLITE3_H +#define SQLITE3_H #include /* Needed for the definition of va_list */ /* @@ -295,7 +359,30 @@ extern "C" { /* -** Provide the ability to override linkage features of the interface. +** Facilitate override of interface linkage and calling conventions. +** Be aware that these macros may not be used within this particular +** translation of the amalgamation and its associated header file. +** +** The SQLITE_EXTERN and SQLITE_API macros are used to instruct the +** compiler that the target identifier should have external linkage. +** +** The SQLITE_CDECL macro is used to set the calling convention for +** public functions that accept a variable number of arguments. +** +** The SQLITE_APICALL macro is used to set the calling convention for +** public functions that accept a fixed number of arguments. +** +** The SQLITE_STDCALL macro is no longer used and is now deprecated. +** +** The SQLITE_CALLBACK macro is used to set the calling convention for +** function pointers. +** +** The SQLITE_SYSAPI macro is used to set the calling convention for +** functions provided by the operating system. +** +** Currently, the SQLITE_CDECL, SQLITE_APICALL, SQLITE_CALLBACK, and +** SQLITE_SYSAPI macros are used only when building for environments +** that require non-default calling conventions. */ #ifndef SQLITE_EXTERN # define SQLITE_EXTERN extern @@ -306,8 +393,17 @@ extern "C" { #ifndef SQLITE_CDECL # define SQLITE_CDECL #endif +#ifndef SQLITE_APICALL +# define SQLITE_APICALL +#endif #ifndef SQLITE_STDCALL -# define SQLITE_STDCALL +# define SQLITE_STDCALL SQLITE_APICALL +#endif +#ifndef SQLITE_CALLBACK +# define SQLITE_CALLBACK +#endif +#ifndef SQLITE_SYSAPI +# define SQLITE_SYSAPI #endif /* @@ -351,25 +447,28 @@ extern "C" { ** be held constant and Z will be incremented or else Y will be incremented ** and Z will be reset to zero. ** -** Since version 3.6.18, SQLite source code has been stored in the +** Since [version 3.6.18] ([dateof:3.6.18]), +** SQLite source code has been stored in the ** Fossil configuration management ** system. ^The SQLITE_SOURCE_ID macro evaluates to ** a string which identifies a particular check-in of SQLite ** within its configuration management system. ^The SQLITE_SOURCE_ID -** string contains the date and time of the check-in (UTC) and an SHA1 -** hash of the entire source tree. +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. If the source code has +** been edited in any way since it was last checked in, then the last +** four hexadecimal digits of the hash may be modified. ** ** See also: [sqlite3_libversion()], ** [sqlite3_libversion_number()], [sqlite3_sourceid()], ** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.13.0" -#define SQLITE_VERSION_NUMBER 3013000 -#define SQLITE_SOURCE_ID "2016-05-18 10:57:30 fc49f556e48970561d7ab6a2f24fdd7d9eb81ff2" +#define SQLITE_VERSION "3.47.1" +#define SQLITE_VERSION_NUMBER 3047001 +#define SQLITE_SOURCE_ID "2024-11-25 12:07:48 b95d11e958643b969c47a8e5857f3793b9e69700b8f1469371386369a26e577e" /* ** CAPI3REF: Run-Time Library Version Numbers -** KEYWORDS: sqlite3_version, sqlite3_sourceid +** KEYWORDS: sqlite3_version sqlite3_sourceid ** ** These interfaces provide the same information as the [SQLITE_VERSION], ** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros @@ -381,7 +480,7 @@ extern "C" { ** ** 
 ** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
-** assert( strcmp(sqlite3_sourceid(),SQLITE_SOURCE_ID)==0 );
+** assert( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,80)==0 );
 ** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
 **
)^ ** @@ -391,42 +490,47 @@ extern "C" { ** function is provided for use in DLLs since DLL users usually do not have ** direct access to string constants within the DLL. ^The ** sqlite3_libversion_number() function returns an integer equal to -** [SQLITE_VERSION_NUMBER]. ^The sqlite3_sourceid() function returns -** a pointer to a string constant whose value is the same as the -** [SQLITE_SOURCE_ID] C preprocessor macro. +** [SQLITE_VERSION_NUMBER]. ^(The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. Except if SQLite is built +** using an edited copy of [the amalgamation], then the last four characters +** of the hash might be different from [SQLITE_SOURCE_ID].)^ ** ** See also: [sqlite_version()] and [sqlite_source_id()]. */ SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; -SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void); -SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void); -SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void); +SQLITE_API const char *sqlite3_libversion(void); +SQLITE_API const char *sqlite3_sourceid(void); +SQLITE_API int sqlite3_libversion_number(void); /* ** CAPI3REF: Run-Time Library Compilation Options Diagnostics ** -** ^The sqlite3_compileoption_used() function returns 0 or 1 -** indicating whether the specified option was defined at -** compile time. ^The SQLITE_ prefix may be omitted from the -** option name passed to sqlite3_compileoption_used(). +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). ** ** ^The sqlite3_compileoption_get() function allows iterating ** over the list of options that were defined at compile time by ** returning the N-th compile time option string. ^If N is out of range, -** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ -** prefix is omitted from any strings returned by +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by ** sqlite3_compileoption_get(). ** ** ^Support for the diagnostic functions sqlite3_compileoption_used() -** and sqlite3_compileoption_get() may be omitted by specifying the +** and sqlite3_compileoption_get() may be omitted by specifying the ** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. ** ** See also: SQL functions [sqlite_compileoption_used()] and ** [sqlite_compileoption_get()] and the [compile_options pragma]. */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS -SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName); -SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N); +SQLITE_API int sqlite3_compileoption_used(const char *zOptName); +SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) #endif /* @@ -439,7 +543,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N); ** SQLite can be compiled with or without mutexes. When ** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes ** are enabled and SQLite is threadsafe. When the -** [SQLITE_THREADSAFE] macro is 0, +** [SQLITE_THREADSAFE] macro is 0, ** the mutexes are omitted. Without the mutexes, it is not safe ** to use SQLite concurrently from more than one thread. ** @@ -465,7 +569,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_compileoption_get(int N); ** ** See the [threading mode] documentation for additional information. */ -SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void); +SQLITE_API int sqlite3_threadsafe(void); /* ** CAPI3REF: Database Connection Handle @@ -496,12 +600,16 @@ typedef struct sqlite3 sqlite3; ** ** ^The sqlite3_int64 and sqlite_int64 types can store integer values ** between -9223372036854775808 and +9223372036854775807 inclusive. ^The -** sqlite3_uint64 and sqlite_uint64 types can store integer values +** sqlite3_uint64 and sqlite_uint64 types can store integer values ** between 0 and +18446744073709551615 inclusive. */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; - typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# ifdef SQLITE_UINT64_TYPE + typedef SQLITE_UINT64_TYPE sqlite_uint64; +# else + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# endif #elif defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 sqlite_int64; typedef unsigned __int64 sqlite_uint64; @@ -530,26 +638,22 @@ typedef sqlite_uint64 sqlite3_uint64; ** the [sqlite3] object is successfully destroyed and all associated ** resources are deallocated. ** -** ^If the database connection is associated with unfinalized prepared -** statements or unfinished sqlite3_backup objects then sqlite3_close() -** will leave the database connection open and return [SQLITE_BUSY]. -** ^If sqlite3_close_v2() is called with unfinalized prepared statements -** and/or unfinished sqlite3_backups, then the database connection becomes -** an unusable "zombie" which will automatically be deallocated when the -** last prepared statement is finalized or the last sqlite3_backup is -** finished. The sqlite3_close_v2() interface is intended for use with -** host languages that are garbage collected, and where the order in which -** destructors are called is arbitrary. -** -** Applications should [sqlite3_finalize | finalize] all [prepared statements], -** [sqlite3_blob_close | close] all [BLOB handles], and +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and ** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated -** with the [sqlite3] object prior to attempting to close the object. ^If -** sqlite3_close_v2() is called on a [database connection] that still has -** outstanding [prepared statements], [BLOB handles], and/or -** [sqlite3_backup] objects then it returns [SQLITE_OK] and the deallocation -** of resources is deferred until all [prepared statements], [BLOB handles], -** and [sqlite3_backup] objects are also destroyed. +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. ** ** ^If an [sqlite3] object is destroyed while a transaction is open, ** the transaction is automatically rolled back. @@ -562,8 +666,8 @@ typedef sqlite_uint64 sqlite3_uint64; ** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer ** argument is a harmless no-op. */ -SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3*); -SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3*); +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); /* ** The type for a callback function. @@ -579,7 +683,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** The sqlite3_exec() interface is a convenience wrapper around ** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], ** that allows an application to run multiple statements of SQL -** without having to use a lot of C code. +** without having to use a lot of C code. ** ** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, ** semicolon-separate SQL statements passed into its 2nd argument, @@ -619,7 +723,7 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** from [sqlite3_column_name()]. ** ** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer -** to an empty string, or a pointer that contains only whitespace and/or +** to an empty string, or a pointer that contains only whitespace and/or ** SQL comments, then no SQL statements are evaluated and the database ** is not changed. ** @@ -632,9 +736,11 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); ** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. **
  • The application must not modify the SQL statement text passed into ** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
  • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. ** */ -SQLITE_API int SQLITE_STDCALL sqlite3_exec( +SQLITE_API int sqlite3_exec( sqlite3*, /* An open database */ const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ @@ -655,7 +761,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ -#define SQLITE_ERROR 1 /* SQL error or missing database */ +#define SQLITE_ERROR 1 /* Generic error */ #define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ #define SQLITE_PERM 3 /* Access permission denied */ #define SQLITE_ABORT 4 /* Callback routine requested an abort */ @@ -670,7 +776,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_FULL 13 /* Insertion failed because database is full */ #define SQLITE_CANTOPEN 14 /* Unable to open the database file */ #define SQLITE_PROTOCOL 15 /* Database lock protocol error */ -#define SQLITE_EMPTY 16 /* Database is empty */ +#define SQLITE_EMPTY 16 /* Internal use only */ #define SQLITE_SCHEMA 17 /* The database schema changed */ #define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ #define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ @@ -678,7 +784,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_MISUSE 21 /* Library used incorrectly */ #define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ #define SQLITE_AUTH 23 /* Authorization denied */ -#define SQLITE_FORMAT 24 /* Auxiliary database format error */ +#define SQLITE_FORMAT 24 /* Not used */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ #define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ @@ -695,7 +801,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( ** [result codes]. However, experience has shown that many of ** these result codes are too coarse-grained. They do not provide as ** much information about problems as programmers might like. In an effort to -** address this, newer versions of SQLite (version 3.3.8 and later) include +** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] +** and later) include ** support for additional result codes that provide more detailed information ** about errors. These [extended result codes] are enabled or disabled ** on a per database connection basis using the @@ -703,6 +810,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( ** the most recent error can be obtained using ** [sqlite3_extended_errcode()]. */ +#define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) +#define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) #define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) #define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) #define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) @@ -731,18 +841,32 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) #define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) #define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) +#define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) +#define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) +#define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) +#define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) #define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) +#define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) #define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) #define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_BUSY_TIMEOUT (SQLITE_BUSY | (3<<8)) #define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) #define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) #define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) #define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ +#define SQLITE_CANTOPEN_SYMLINK (SQLITE_CANTOPEN | (6<<8)) #define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) +#define SQLITE_CORRUPT_INDEX (SQLITE_CORRUPT | (3<<8)) #define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) #define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) #define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) #define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) +#define SQLITE_READONLY_DIRECTORY (SQLITE_READONLY | (6<<8)) #define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) #define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) #define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) @@ -754,10 +878,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) #define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) #define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) +#define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) #define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) #define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) #define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) #define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) +#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ /* ** CAPI3REF: Flags For File Open Operations @@ -765,6 +894,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( ** These bit values are intended for use in the ** 3rd parameter to the [sqlite3_open_v2()] interface and ** in the 4th parameter to the [sqlite3_vfs.xOpen] method. +** +** Only those flags marked as "Ok for sqlite3_open_v2()" may be +** used as the third argument to the [sqlite3_open_v2()] interface. +** The other flags have historically been ignored by sqlite3_open_v2(), +** though future versions of SQLite might change so that an error is +** raised if any of the disallowed bits are passed into sqlite3_open_v2(). +** Applications should not depend on the historical behavior. +** +** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into +** [sqlite3_open_v2()] does *not* cause the underlying database file +** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into +** [sqlite3_open_v2()] has historically be a no-op and might become an +** error in future versions of SQLite. */ #define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ @@ -780,14 +922,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 /* VFS only */ #define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 /* VFS only */ #define SQLITE_OPEN_SUBJOURNAL 0x00002000 /* VFS only */ -#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ +#define SQLITE_OPEN_SUPER_JOURNAL 0x00004000 /* VFS only */ #define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ #define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_EXRESCODE 0x02000000 /* Extended result codes */ /* Reserved: 0x00F00000 */ +/* Legacy compatibility: */ +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ + /* ** CAPI3REF: Device Characteristics @@ -812,10 +959,22 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( ** file that were written at the application level might have changed ** and that adjacent bytes, even bytes within the same sector are ** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN -** flag indicate that a file cannot be deleted when open. The +** flag indicates that a file cannot be deleted when open. The ** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on ** read-only media and cannot be changed even by processes with ** elevated privileges. +** +** The SQLITE_IOCAP_BATCH_ATOMIC property means that the underlying +** filesystem supports doing multiple write operations atomically when those +** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and +** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. +** +** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read +** from the database file in amounts that are not a multiple of the +** page size and that do not begin at a page boundary. Without this +** property, SQLite is careful to only do full-page reads and write +** on aligned pages, with the one exception that it will do a sub-page +** read of the first page to access the database header. */ #define SQLITE_IOCAP_ATOMIC 0x00000001 #define SQLITE_IOCAP_ATOMIC512 0x00000002 @@ -831,19 +990,25 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( #define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 #define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 #define SQLITE_IOCAP_IMMUTABLE 0x00002000 +#define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 +#define SQLITE_IOCAP_SUBPAGE_READ 0x00008000 /* ** CAPI3REF: File Locking Levels ** ** SQLite uses one of these integer values as the second ** argument to calls it makes to the xLock() and xUnlock() methods -** of an [sqlite3_io_methods] object. +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. */ -#define SQLITE_LOCK_NONE 0 -#define SQLITE_LOCK_SHARED 1 -#define SQLITE_LOCK_RESERVED 2 -#define SQLITE_LOCK_PENDING 3 -#define SQLITE_LOCK_EXCLUSIVE 4 +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ /* ** CAPI3REF: Synchronization Type Flags @@ -878,7 +1043,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( /* ** CAPI3REF: OS Interface Open File Handle ** -** An [sqlite3_file] object represents an open file in the +** An [sqlite3_file] object represents an open file in the ** [sqlite3_vfs | OS interface layer]. Individual OS interface ** implementations will ** want to subclass this object by appending additional fields @@ -900,7 +1065,7 @@ struct sqlite3_file { ** This object defines the methods used to perform various operations ** against the open file represented by the [sqlite3_file] object. ** -** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element ** to a non-NULL pointer, then the sqlite3_io_methods.xClose method ** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The ** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] @@ -921,11 +1086,18 @@ struct sqlite3_file { **
  • [SQLITE_LOCK_PENDING], or **
  • [SQLITE_LOCK_EXCLUSIVE]. ** -** xLock() increases the lock. xUnlock() decreases the lock. +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +** If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. ** The xCheckReservedLock() method checks whether any database connection, ** either in this process or in some other process, is holding a RESERVED, -** PENDING, or EXCLUSIVE lock on the file. It returns true -** if such a lock exists and false otherwise. +** PENDING, or EXCLUSIVE lock on the file. It returns, via its output +** pointer parameter, true if such a lock exists and false otherwise. ** ** The xFileControl() method is a generic interface that allows custom ** VFS implementations to directly control an open file using the @@ -962,6 +1134,11 @@ struct sqlite3_file { **
  • [SQLITE_IOCAP_ATOMIC64K] **
  • [SQLITE_IOCAP_SAFE_APPEND] **
  • [SQLITE_IOCAP_SEQUENTIAL] +**
  • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
  • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
  • [SQLITE_IOCAP_IMMUTABLE] +**
  • [SQLITE_IOCAP_BATCH_ATOMIC] +**
  • [SQLITE_IOCAP_SUBPAGE_READ] ** ** ** The SQLITE_IOCAP_ATOMIC property means that all writes of @@ -1022,9 +1199,8 @@ struct sqlite3_io_methods { ** opcode causes the xFileControl method to write the current state of ** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], ** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) -** into an integer that the pArg argument points to. This capability -** is used during testing and is only available when the SQLITE_TEST -** compile-time option is used. +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. ** **
  • [[SQLITE_FCNTL_SIZE_HINT]] ** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS @@ -1034,10 +1210,19 @@ struct sqlite3_io_methods { ** file space based on this hint in order to help writes to the database ** file run faster. ** +**
  • [[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** **
  • [[SQLITE_FCNTL_CHUNK_SIZE]] ** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS ** extends and truncates the database file in chunks of a size specified -** by the user. The fourth argument to [sqlite3_file_control()] should +** by the user. The fourth argument to [sqlite3_file_control()] should ** point to an integer (type int) containing the new chunk-size to use ** for the nominated database. Allocating database file space in large ** chunks (say 1MB at a time), may reduce file-system fragmentation and @@ -1060,24 +1245,24 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_SYNC]] ** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and ** sent to the VFS immediately before the xSync method is invoked on a -** database file descriptor. Or, if the xSync method is not invoked -** because the user has configured SQLite with -** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place ** of the xSync method. In most cases, the pointer argument passed with ** this file-control is NULL. However, if the database file is being synced ** as part of a multi-database commit, the argument points to a nul-terminated -** string containing the transactions master-journal file name. VFSes that -** do not need this signal should silently ignore this opcode. Applications -** should not call [sqlite3_file_control()] with this opcode as doing so may -** disrupt the operation of the specialized VFSes that do require it. +** string containing the transactions super-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. ** **
  • [[SQLITE_FCNTL_COMMIT_PHASETWO]] ** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite ** and sent to the VFS after a transaction has been committed immediately ** but before the database is unlocked. VFSes that do not need this signal ** should silently ignore this opcode. Applications should not call -** [sqlite3_file_control()] with this opcode as doing so may disrupt the -** operation of the specialized VFSes that do require it. +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. ** **
  • [[SQLITE_FCNTL_WIN32_AV_RETRY]] ** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic @@ -1090,7 +1275,7 @@ struct sqlite3_io_methods { ** opcode allows these two values (10 retries and 25 milliseconds of delay) ** to be adjusted. The values are changed for all database connections ** within the same process. The argument is a pointer to an array of two -** integers where the first integer i the new retry count and the second +** integers where the first integer is the new retry count and the second ** integer is the delay. If either integer is negative, then the setting ** is not changed but instead the prior value of that setting is written ** into the array entry, allowing the current retry settings to be @@ -1099,7 +1284,8 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_PERSIST_WAL]] ** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the ** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary -** write ahead log and shared memory files used for transaction control +** write ahead log ([WAL file]) and shared memory +** files used for transaction control ** are automatically deleted when the latest connection to the database ** closes. Setting persistent WAL mode causes those files to persist after ** close. Persisting the files is useful when other processes that do not @@ -1124,13 +1310,13 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_OVERWRITE]] ** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening ** a write transaction to indicate that, unless it is rolled back for some -** reason, the entire database file will be overwritten by the current +** reason, the entire database file will be overwritten by the current ** transaction. This is used by VACUUM operations. ** **
  • [[SQLITE_FCNTL_VFSNAME]] ** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of ** all [VFSes] in the VFS stack. The names are of all VFS shims and the -** final bottom-level VFS are written into memory obtained from +** final bottom-level VFS are written into memory obtained from ** [sqlite3_malloc()] and the result is stored in the char* variable ** that the fourth parameter of [sqlite3_file_control()] points to. ** The caller is responsible for freeing the memory when done. As with @@ -1149,7 +1335,7 @@ struct sqlite3_io_methods { ** upper-most shim only. ** **
  • [[SQLITE_FCNTL_PRAGMA]] -** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] ** file control is sent to the open [sqlite3_file] object corresponding ** to the database file to which the pragma statement refers. ^The argument ** to the [SQLITE_FCNTL_PRAGMA] file control is an array of @@ -1160,7 +1346,7 @@ struct sqlite3_io_methods { ** of the char** argument point to a string obtained from [sqlite3_mprintf()] ** or the equivalent and that string will become the result of the pragma or ** the error message if the pragma fails. ^If the -** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal ** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] ** file control returns [SQLITE_OK], then the parser assumes that the ** VFS has handled the PRAGMA itself and the parser generates a no-op @@ -1177,16 +1363,16 @@ struct sqlite3_io_methods { ** ^The [SQLITE_FCNTL_BUSYHANDLER] ** file-control may be invoked by SQLite on the database file handle ** shortly after it is opened in order to provide a custom VFS with access -** to the connections busy-handler callback. The argument is of type (void **) +** to the connection's busy-handler callback. The argument is of type (void**) ** - an array of two (void *) values. The first (void *) actually points -** to a function of type (int (*)(void *)). In order to invoke the connections +** to a function of type (int (*)(void *)). In order to invoke the connection's ** busy-handler, this function should be invoked with the second (void *) in ** the array as the only argument. If it returns non-zero, then the operation ** should be retried. If it returns zero, the custom VFS should abandon the ** current operation. ** **
  • [[SQLITE_FCNTL_TEMPFILENAME]] -** ^Application can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** ^Applications can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control ** to have SQLite generate a ** temporary filename using the same algorithm that is followed to generate ** temporary filenames for TEMP tables and other internal uses. The @@ -1200,7 +1386,7 @@ struct sqlite3_io_methods { ** The argument is a pointer to a value of type sqlite3_int64 that ** is an advisory maximum number of bytes in the file to memory map. The ** pointer is overwritten with the old value. The limit is not changed if -** the value originally pointed to is negative, and so the current limit +** the value originally pointed to is negative, and so the current limit ** can be queried by passing in a pointer to a negative number. This ** file-control is used internally to implement [PRAGMA mmap_size]. ** @@ -1218,6 +1404,12 @@ struct sqlite3_io_methods { ** on whether or not the file has been renamed, moved, or deleted since it ** was first opened. ** +**
  • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the +** underlying native file handle associated with a file handle. This file +** control interprets its argument as a pointer to a native file handle and +** writes the resulting value there. +** **
  • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] ** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This ** opcode causes the xFileControl method to swap the file handle with the one @@ -1238,7 +1430,102 @@ struct sqlite3_io_methods { **
  • [[SQLITE_FCNTL_RBU]] ** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by ** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for -** this opcode. +** this opcode. +** +**
  • [[SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]] +** If the [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] opcode returns SQLITE_OK, then +** the file descriptor is placed in "batch write mode", which +** means all subsequent write operations will be deferred and done +** atomically at the next [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. Systems +** that do not support batch atomic writes will return SQLITE_NOTFOUND. +** ^Following a successful SQLITE_FCNTL_BEGIN_ATOMIC_WRITE and prior to +** the closing [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] or +** [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE], SQLite will make +** no VFS interface calls on the same [sqlite3_file] file descriptor +** except for calls to the xWrite method and the xFileControl method +** with [SQLITE_FCNTL_SIZE_HINT]. +** +**
  • [[SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be performed atomically. +** This file control returns [SQLITE_OK] if and only if the writes were +** all performed successfully and have been committed to persistent storage. +** ^Regardless of whether or not it is successful, this file control takes +** the file descriptor out of batch write mode so that all subsequent +** write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_COMMIT_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be rolled back. +** ^This file control takes the file descriptor out of batch write mode +** so that all subsequent write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
  • [[SQLITE_FCNTL_LOCK_TIMEOUT]] +** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode is used to configure a VFS +** to block for up to M milliseconds before failing when attempting to +** obtain a file lock using the xLock or xShmLock methods of the VFS. +** The parameter is a pointer to a 32-bit signed integer that contains +** the value that M is to be set to. Before returning, the 32-bit signed +** integer is overwritten with the previous value of M. +** +**
  • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provides a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. +** +**
  • [[SQLITE_FCNTL_CKPT_START]] +** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint +** in wal mode before the client starts to copy pages from the wal +** file to the database file. +** +**
  • [[SQLITE_FCNTL_CKPT_DONE]] +** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint +** in wal mode after the client has finished copying pages from the wal +** file to the database file, but before the *-shm file is updated to +** record the fact that the pages have been checkpointed. +** +**
  • [[SQLITE_FCNTL_EXTERNAL_READER]] +** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect +** whether or not there is a database client in another process with a wal-mode +** transaction open on the database or not. It is only available on unix.The +** (void*) argument passed with this file-control should be a pointer to a +** value of type (int). The integer value is set to 1 if the database is a wal +** mode database and there exists at least one client in another process that +** currently has an SQL transaction open on the database. It is set to 0 if +** the database is not a wal-mode db, or if there is no such connection in any +** other process. This opcode cannot be used to detect transactions opened +** by clients within the current process, only within other processes. +** +**
  • [[SQLITE_FCNTL_CKSM_FILE]] +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the +** [checksum VFS shim] only. +** +**
  • [[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control +** purges the contents of the in-memory page cache. If there is an open +** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. ** */ #define SQLITE_FCNTL_LOCKSTATE 1 @@ -1268,6 +1555,20 @@ struct sqlite3_io_methods { #define SQLITE_FCNTL_RBU 26 #define SQLITE_FCNTL_VFS_POINTER 27 #define SQLITE_FCNTL_JOURNAL_POINTER 28 +#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 +#define SQLITE_FCNTL_PDB 30 +#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31 +#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 +#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 +#define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 +#define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 +#define SQLITE_FCNTL_CKPT_START 39 +#define SQLITE_FCNTL_EXTERNAL_READER 40 +#define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 /* deprecated names */ #define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE @@ -1287,6 +1588,36 @@ struct sqlite3_io_methods { */ typedef struct sqlite3_mutex sqlite3_mutex; +/* +** CAPI3REF: Loadable Extension Thunk +** +** A pointer to the opaque sqlite3_api_routines structure is passed as +** the third parameter to entry points of [loadable extensions]. This +** structure must be typedefed in order to work around compiler warnings +** on some platforms. +*/ +typedef struct sqlite3_api_routines sqlite3_api_routines; + +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
      +**
    • sqlite3_filename_database() +**
    • sqlite3_filename_journal() +**
    • sqlite3_filename_wal() +**
    • sqlite3_uri_parameter() +**
    • sqlite3_uri_boolean() +**
    • sqlite3_uri_int64() +**
    • sqlite3_uri_key() +**
    +*/ +typedef const char *sqlite3_filename; + /* ** CAPI3REF: OS Interface Object ** @@ -1295,12 +1626,18 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** in the name of the object stands for "virtual file system". See ** the [VFS | VFS documentation] for further information. ** -** The value of the iVersion field is initially 1 but may be larger in -** future versions of SQLite. Additional fields may be appended to this -** object when the iVersion value is increased. Note that the structure -** of the sqlite3_vfs object changes in the transaction between -** SQLite version 3.5.9 and 3.6.0 and yet the iVersion field was not -** modified. +** The VFS interface is sometimes extended by adding new methods onto +** the end. Each time such an extension occurs, the iVersion field +** is incremented. The iVersion value started out as 1 in +** SQLite [version 3.5.0] on [dateof:3.5.0], then increased to 2 +** with SQLite [version 3.7.0] on [dateof:3.7.0], and then increased +** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields +** may be appended to the sqlite3_vfs object and the iVersion value +** may increase again in future versions of SQLite. +** Note that due to an oversight, the structure +** of the sqlite3_vfs object changed in the transition from +** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] +** and yet the iVersion field was not increased. ** ** The szOsFile field is the size of the subclassed [sqlite3_file] ** structure used by this VFS. mxPathname is the maximum length of @@ -1335,14 +1672,14 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** the [sqlite3_file] can safely store a pointer to the ** filename if it needs to remember the filename for some reason. ** If the zFilename parameter to xOpen is a NULL pointer then xOpen -** must invent its own temporary name for the file. ^Whenever the +** must invent its own temporary name for the file. ^Whenever the ** xFilename parameter is NULL it will also be the case that the ** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. ** ** The flags argument to xOpen() includes all bits set in ** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] ** or [sqlite3_open16()] is used, then flags includes at least -** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. ** If xOpen() opens a file read-only then it sets *pOutFlags to ** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. ** @@ -1356,7 +1693,7 @@ typedef struct sqlite3_mutex sqlite3_mutex; **
  • [SQLITE_OPEN_TEMP_JOURNAL] **
  • [SQLITE_OPEN_TRANSIENT_DB] **
  • [SQLITE_OPEN_SUBJOURNAL] -**
  • [SQLITE_OPEN_MASTER_JOURNAL] +**
  • [SQLITE_OPEN_SUPER_JOURNAL] **
  • [SQLITE_OPEN_WAL] ** )^ ** @@ -1384,14 +1721,14 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction ** with the [SQLITE_OPEN_CREATE] flag, which are both directly ** analogous to the O_EXCL and O_CREAT flags of the POSIX open() -** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the +** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the ** SQLITE_OPEN_CREATE, is used to indicate that file should always ** be created, and that it is an error if it already exists. -** It is not used to indicate the file should be opened +** It is not used to indicate the file should be opened ** for exclusive access. ** ** ^At least szOsFile bytes of memory are allocated by SQLite -** to hold the [sqlite3_file] structure passed as the third +** to hold the [sqlite3_file] structure passed as the third ** argument to xOpen. The xOpen method does not have to ** allocate the structure; it should just fill it in. Note that ** the xOpen method must set the sqlite3_file.pMethods to either @@ -1404,8 +1741,14 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] ** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to ** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] -** to test whether a file is at least readable. The file can be a -** directory. +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. ** ** ^SQLite will always allocate at least mxPathname+1 bytes for the ** output buffer xFullPathname. The exact size of the output buffer @@ -1425,16 +1768,16 @@ typedef struct sqlite3_mutex sqlite3_mutex; ** method returns a Julian Day Number for the current date and time as ** a floating point value. ** ^The xCurrentTimeInt64() method returns, as an integer, the Julian -** Day Number multiplied by 86400000 (the number of milliseconds in -** a 24-hour day). +** Day Number multiplied by 86400000 (the number of milliseconds in +** a 24-hour day). ** ^SQLite will use the xCurrentTimeInt64() method to get the current -** date and time if that method is available (if iVersion is 2 or +** date and time if that method is available (if iVersion is 2 or ** greater and the function pointer is not NULL) and will fall back ** to xCurrentTime() if xCurrentTimeInt64() is unavailable. ** ** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces ** are not used by the SQLite core. These optional interfaces are provided -** by some VFSes to facilitate testing of the VFS code. By overriding +** by some VFSes to facilitate testing of the VFS code. By overriding ** system calls with functions under its control, a test program can ** simulate faults and error conditions that would otherwise be difficult ** or impossible to induce. The set of system calls that can be overridden @@ -1453,7 +1796,7 @@ struct sqlite3_vfs { sqlite3_vfs *pNext; /* Next registered VFS */ const char *zName; /* Name of this virtual file system */ void *pAppData; /* Pointer to application-specific data */ - int (*xOpen)(sqlite3_vfs*, const char *zName, sqlite3_file*, + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, int flags, int *pOutFlags); int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); @@ -1481,7 +1824,7 @@ struct sqlite3_vfs { /* ** The methods above are in versions 1 through 3 of the sqlite_vfs object. ** New fields may be appended in future versions. The iVersion - ** value will increment whenever this happens. + ** value will increment whenever this happens. */ }; @@ -1525,7 +1868,7 @@ struct sqlite3_vfs { ** ** ** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as -** was given on the corresponding lock. +** was given on the corresponding lock. ** ** The xShmLock method can transition between unlocked and SHARED or ** between unlocked and EXCLUSIVE. It cannot transition between SHARED @@ -1622,10 +1965,10 @@ struct sqlite3_vfs { ** must return [SQLITE_OK] on success and some other [error code] upon ** failure. */ -SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void); -SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void); -SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void); -SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void); +SQLITE_API int sqlite3_initialize(void); +SQLITE_API int sqlite3_shutdown(void); +SQLITE_API int sqlite3_os_init(void); +SQLITE_API int sqlite3_os_end(void); /* ** CAPI3REF: Configuring The SQLite Library @@ -1640,25 +1983,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void); ** must ensure that no other SQLite interfaces are invoked by other ** threads while sqlite3_config() is running. ** -** The sqlite3_config() interface -** may only be invoked prior to library initialization using -** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. -** ^If sqlite3_config() is called after [sqlite3_initialize()] and before -** [sqlite3_shutdown()] then it will return SQLITE_MISUSE. -** Note, however, that ^sqlite3_config() can be called as part of the -** implementation of an application-defined [sqlite3_os_init()]. -** ** The first argument to sqlite3_config() is an integer ** [configuration option] that determines ** what property of SQLite is to be configured. Subsequent arguments ** vary depending on the [configuration option] ** in the first argument. ** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** ** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. ** ^If the option is unknown or SQLite is unable to set the option ** then this routine returns a non-zero [error code]. */ -SQLITE_API int SQLITE_CDECL sqlite3_config(int, ...); +SQLITE_API int sqlite3_config(int, ...); /* ** CAPI3REF: Configure database connections @@ -1670,14 +2016,14 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int, ...); ** [database connection] (specified in the first argument). ** ** The second argument to sqlite3_db_config(D,V,...) is the -** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code ** that indicates what aspect of the [database connection] is being configured. ** Subsequent arguments vary depending on the configuration verb. ** ** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if ** the call is considered successful. */ -SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3*, int op, ...); +SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); /* ** CAPI3REF: Memory Allocation Routines @@ -1688,7 +2034,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3*, int op, ...); ** This object is used in only one place in the SQLite interface. ** A pointer to an instance of this object is the argument to ** [sqlite3_config()] when the configuration option is -** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. +** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. ** By creating an instance of this object ** and passing it to [sqlite3_config]([SQLITE_CONFIG_MALLOC]) ** during configuration, an application can specify an alternative @@ -1718,17 +2064,17 @@ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3*, int op, ...); ** allocators round up memory allocations at least to the next multiple ** of 8. Some allocators round up to a larger multiple or to a power of 2. ** Every memory allocation request coming in through [sqlite3_malloc()] -** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, +** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, ** that causes the corresponding memory allocation to fail. ** ** The xInit method initializes the memory allocator. For example, -** it might allocate any require mutexes or initialize internal data +** it might allocate any required mutexes or initialize internal data ** structures. The xShutdown method is invoked (indirectly) by ** [sqlite3_shutdown()] and should deallocate any resources acquired ** by xInit. The pAppData pointer is used as the only parameter to ** xInit and xShutdown. ** -** SQLite holds the [SQLITE_MUTEX_STATIC_MASTER] mutex when it invokes +** SQLite holds the [SQLITE_MUTEX_STATIC_MAIN] mutex when it invokes ** the xInit method, so the xInit method need not be threadsafe. The ** xShutdown method is only called from [sqlite3_shutdown()] so it does ** not need to be threadsafe either. For all other methods, SQLite @@ -1761,6 +2107,23 @@ struct sqlite3_mem_methods { ** These constants are the available integer configuration options that ** can be passed as the first argument to the [sqlite3_config()] interface. ** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
      +**
    • SQLITE_CONFIG_LOG +**
    • SQLITE_CONFIG_PCACHE_HDRSZ +**
    +** ** New configuration options may be added in future releases of SQLite. ** Existing configuration options might be discontinued. Applications ** should check the return code from [sqlite3_config()] to make sure that @@ -1776,7 +2139,7 @@ struct sqlite3_mem_methods { ** by a single thread. ^If SQLite is compiled with ** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then ** it is not possible to change the [threading mode] from its default -** value of Single-thread and so [sqlite3_config()] will return +** value of Single-thread and so [sqlite3_config()] will return ** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD ** configuration option. ** @@ -1811,7 +2174,7 @@ struct sqlite3_mem_methods { ** SQLITE_CONFIG_SERIALIZED configuration option. ** ** [[SQLITE_CONFIG_MALLOC]]
    SQLITE_CONFIG_MALLOC
    -**
    ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +**
    ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is ** a pointer to an instance of the [sqlite3_mem_methods] structure. ** The argument specifies ** alternative low-level memory allocation routines to be used in place of @@ -1828,12 +2191,23 @@ struct sqlite3_mem_methods { ** routines with a wrapper that simulations memory allocation failure or ** tracks memory usage, for example.
    ** +** [[SQLITE_CONFIG_SMALL_MALLOC]]
    SQLITE_CONFIG_SMALL_MALLOC
    +**
    ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** type int, interpreted as a boolean, which if true provides a hint to +** SQLite that it should avoid large memory allocations if possible. +** SQLite will run faster if it is free to make large memory allocations, +** but some application might prefer to run slower in exchange for +** guarantees about memory fragmentation that are possible if large +** allocations are avoided. This hint is normally off. +**
    +** ** [[SQLITE_CONFIG_MEMSTATUS]]
    SQLITE_CONFIG_MEMSTATUS
    **
    ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, ** interpreted as a boolean, which enables or disables the collection of ** memory allocation statistics. ^(When memory allocation statistics are ** disabled, the following SQLite interfaces become non-operational: **
      +**
    • [sqlite3_hard_heap_limit64()] **
    • [sqlite3_memory_used()] **
    • [sqlite3_memory_highwater()] **
    • [sqlite3_soft_heap_limit64()] @@ -1845,32 +2219,14 @@ struct sqlite3_mem_methods { **
    ** ** [[SQLITE_CONFIG_SCRATCH]]
    SQLITE_CONFIG_SCRATCH
    -**
    ^The SQLITE_CONFIG_SCRATCH option specifies a static memory buffer -** that SQLite can use for scratch memory. ^(There are three arguments -** to SQLITE_CONFIG_SCRATCH: A pointer an 8-byte -** aligned memory buffer from which the scratch allocations will be -** drawn, the size of each scratch allocation (sz), -** and the maximum number of scratch allocations (N).)^ -** The first argument must be a pointer to an 8-byte aligned buffer -** of at least sz*N bytes of memory. -** ^SQLite will not use more than one scratch buffers per thread. -** ^SQLite will never request a scratch buffer that is more than 6 -** times the database page size. -** ^If SQLite needs needs additional -** scratch memory beyond what is provided by this configuration option, then -** [sqlite3_malloc()] will be used to obtain the memory needed.

    -** ^When the application provides any amount of scratch memory using -** SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary large -** [sqlite3_malloc|heap allocations]. -** This can help [Robson proof|prevent memory allocation failures] due to heap -** fragmentation in low-memory embedded systems. +**

    The SQLITE_CONFIG_SCRATCH option is no longer used. **
    ** ** [[SQLITE_CONFIG_PAGECACHE]]
    SQLITE_CONFIG_PAGECACHE
    **
    ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool ** that SQLite can use for the database page cache with the default page -** cache implementation. -** This configuration option is a no-op if an application-define page +** cache implementation. +** This configuration option is a no-op if an application-defined page ** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. ** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to ** 8-byte aligned memory (pMem), the size of each page cache line (sz), @@ -1897,10 +2253,9 @@ struct sqlite3_mem_methods { ** additional cache line.
    ** ** [[SQLITE_CONFIG_HEAP]]
    SQLITE_CONFIG_HEAP
    -**
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +**
    ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer ** that SQLite will use for all of its dynamic memory allocation needs -** beyond those provided for by [SQLITE_CONFIG_SCRATCH] and -** [SQLITE_CONFIG_PAGECACHE]. +** beyond those provided for by [SQLITE_CONFIG_PAGECACHE]. ** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled ** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns ** [SQLITE_ERROR] if invoked otherwise. @@ -1953,7 +2308,7 @@ struct sqlite3_mem_methods { ** configuration on individual connections.)^
    ** ** [[SQLITE_CONFIG_PCACHE2]]
    SQLITE_CONFIG_PCACHE2
    -**
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +**
    ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is ** a pointer to an [sqlite3_pcache_methods2] object. This object specifies ** the interface to a custom page cache implementation.)^ ** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
    @@ -1967,7 +2322,7 @@ struct sqlite3_mem_methods { **
    The SQLITE_CONFIG_LOG option is used to configure the SQLite ** global [error log]. ** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a -** function with a call signature of void(*)(void*,int,const char*), +** function with a call signature of void(*)(void*,int,const char*), ** and a pointer to void. ^If the function pointer is not NULL, it is ** invoked by [sqlite3_log()] to process each logging event. ^If the ** function pointer is NULL, the [sqlite3_log()] interface becomes a no-op. @@ -2076,7 +2431,7 @@ struct sqlite3_mem_methods { ** [[SQLITE_CONFIG_STMTJRNL_SPILL]] **
    SQLITE_CONFIG_STMTJRNL_SPILL **
    ^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which -** becomes the [statement journal] spill-to-disk threshold. +** becomes the [statement journal] spill-to-disk threshold. ** [Statement journals] are held in memory until their size (in bytes) ** exceeds this threshold, at which point they are written to disk. ** Or if the threshold is -1, statement journals are always held @@ -2086,34 +2441,81 @@ struct sqlite3_mem_methods { ** I/O required to support statement rollback. ** The default value for this setting is controlled by the ** [SQLITE_STMTJRNL_SPILL] compile-time option. +** +** [[SQLITE_CONFIG_SORTERREF_SIZE]] +**
    SQLITE_CONFIG_SORTERREF_SIZE +**
    The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter +** of type (int) - the new value of the sorter-reference size threshold. +** Usually, when SQLite uses an external sort to order records according +** to an ORDER BY clause, all fields required by the caller are present in the +** sorted records. However, if SQLite determines based on the declared type +** of a table column that its values are likely to be very large - larger +** than the configured sorter-reference size threshold - then a reference +** is stored in each sorted record and the required column values loaded +** from the database as records are returned in sorted order. The default +** value for this option is to never use this optimization. Specifying a +** negative value for this option restores the default behavior. +** This option is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. +** +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]] +**
    SQLITE_CONFIG_MEMDB_MAXSIZE +**
    The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter +** [sqlite3_int64] parameter which is the default maximum size for an in-memory +** database created using [sqlite3_deserialize()]. This default maximum +** size can be adjusted up or down for individual databases using the +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this +** configuration setting is never used, then the default maximum is determined +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that +** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
    SQLITE_CONFIG_ROWID_IN_VIEW +**
    The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. ** */ -#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ -#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ -#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ -#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ -#define SQLITE_CONFIG_SCRATCH 6 /* void*, int sz, int N */ -#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ -#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ -#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ -#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ -#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ -/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ -#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ -#define SQLITE_CONFIG_PCACHE 14 /* no-op */ -#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ -#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ -#define SQLITE_CONFIG_URI 17 /* int */ -#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ -#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ #define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ -#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ -#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ #define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ #define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ #define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ #define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ +#define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ +#define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ +#define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ /* ** CAPI3REF: Database Connection Configuration Options @@ -2129,8 +2531,9 @@ struct sqlite3_mem_methods { ** is invoked. ** **
    +** [[SQLITE_DBCONFIG_LOOKASIDE]] **
    SQLITE_DBCONFIG_LOOKASIDE
    -**
    ^This option takes three additional arguments that determine the +**
    ^This option takes three additional arguments that determine the ** [lookaside memory allocator] configuration for the [database connection]. ** ^The first argument (the third parameter to [sqlite3_db_config()] is a ** pointer to a memory buffer to use for lookaside memory. @@ -2146,11 +2549,12 @@ struct sqlite3_mem_methods { ** configuration for a database connection can only be changed when that ** connection is not currently using lookaside memory, or in other words ** when the "current value" returned by -** [sqlite3_db_status](D,[SQLITE_CONFIG_LOOKASIDE],...) is zero. +** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. ** Any attempt to change the lookaside memory configuration when lookaside -** memory is in use leaves the configuration unchanged and returns +** memory is in use leaves the configuration unchanged and returns ** [SQLITE_BUSY].)^
    ** +** [[SQLITE_DBCONFIG_ENABLE_FKEY]] **
    SQLITE_DBCONFIG_ENABLE_FKEY
    **
    ^This option is used to enable or disable the enforcement of ** [foreign key constraints]. There should be two additional arguments. @@ -2161,6 +2565,7 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the FK enforcement setting is not reported back.
    ** +** [[SQLITE_DBCONFIG_ENABLE_TRIGGER]] **
    SQLITE_DBCONFIG_ENABLE_TRIGGER
    **
    ^This option is used to enable or disable [CREATE TRIGGER | triggers]. ** There should be two additional arguments. @@ -2169,11 +2574,35 @@ struct sqlite3_mem_methods { ** The second parameter is a pointer to an integer into which ** is written 0 or 1 to indicate whether triggers are disabled or enabled ** following this call. The second parameter may be a NULL pointer, in -** which case the trigger setting is not reported back.
    +** which case the trigger setting is not reported back. ** +**

    Originally this option disabled all triggers. ^(However, since +** SQLite version 3.35.0, TEMP triggers are still allowed even if +** this option is off. So, in other words, this option now only disables +** triggers in the main database schema or in the schemas of ATTACH-ed +** databases.)^

    +** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
    SQLITE_DBCONFIG_ENABLE_VIEW
    +**
    ^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back. +** +**

    Originally this option disabled all views. ^(However, since +** SQLite version 3.35.0, TEMP views are still allowed even if +** this option is off. So, in other words, this option now only disables +** views in the main database schema or in the schemas of ATTACH-ed +** databases.)^

    +** +** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] **
    SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
    -**
    ^This option is used to enable or disable the two-argument -** version of the [fts3_tokenizer()] function which is part of the +**
    ^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the ** [FTS3] full-text search engine extension. ** There should be two additional arguments. ** The first argument is an integer which is 0 to disable fts3_tokenizer() or @@ -2184,6 +2613,7 @@ struct sqlite3_mem_methods { ** following this call. The second parameter may be a NULL pointer, in ** which case the new setting is not reported back.
    ** +** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] **
    SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
    **
    ^This option is used to enable or disable the [sqlite3_load_extension()] ** interface independently of the [load_extension()] SQL function. @@ -2191,7 +2621,7 @@ struct sqlite3_mem_methods { ** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. ** There should be two additional arguments. ** When the first argument to this interface is 1, then only the C-API is -** enabled and the SQL function remains disabled. If the first argment to +** enabled and the SQL function remains disabled. If the first argument to ** this interface is 0, then both the C-API and the SQL function are disabled. ** If the first argument is -1, then no changes are made to state of either the ** C-API or the SQL function. @@ -2201,14 +2631,231 @@ struct sqlite3_mem_methods { ** be a NULL pointer, in which case the new setting is not reported back. **
    ** +** [[SQLITE_DBCONFIG_MAINDBNAME]]
    SQLITE_DBCONFIG_MAINDBNAME
    +**
    ^This option is used to change the name of the "main" database +** schema. ^The sole argument is a pointer to a constant UTF8 string +** which will become the new schema name in place of "main". ^SQLite +** does not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into this DBCONFIG option is unchanged +** until after the database connection closes. +**
    +** +** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] +**
    SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
    +**
    Usually, when a database in wal mode is closed or detached from a +** database handle, SQLite checks if this will mean that there are now no +** connections at all to the database. If so, it performs a checkpoint +** operation before closing the connection. This option may be used to +** override this behavior. The first parameter passed to this operation +** is an integer - positive to disable checkpoints-on-close, or zero (the +** default) to enable them, and negative to leave the setting unchanged. +** The second parameter is a pointer to an integer +** into which is written 0 or 1 to indicate whether checkpoints-on-close +** have been disabled - 0 if they are not disabled, 1 if they are. +**
    +** +** [[SQLITE_DBCONFIG_ENABLE_QPSG]]
    SQLITE_DBCONFIG_ENABLE_QPSG
    +**
    ^(The SQLITE_DBCONFIG_ENABLE_QPSG option activates or deactivates +** the [query planner stability guarantee] (QPSG). When the QPSG is active, +** a single SQL query statement will always use the same algorithm regardless +** of values of [bound parameters].)^ The QPSG disables some query optimizations +** that look at the values of bound parameters, which can make some queries +** slower. But the QPSG has the advantage of more predictable behavior. With +** the QPSG active, SQLite will always use the same query plan in the field as +** was used during testing in the lab. +** The first argument to this setting is an integer which is 0 to disable +** the QPSG, positive to enable QPSG, or negative to leave the setting +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether the QPSG is disabled or enabled +** following this call. +**
    +** +** [[SQLITE_DBCONFIG_TRIGGER_EQP]]
    SQLITE_DBCONFIG_TRIGGER_EQP
    +**
    By default, the output of EXPLAIN QUERY PLAN commands does not +** include output for any operations performed by trigger programs. This +** option is used to set or clear (the default) a flag that governs this +** behavior. The first parameter passed to this operation is an integer - +** positive to enable output for trigger programs, or zero to disable it, +** or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which is written +** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if +** it is not disabled, 1 if it is. +**
    +** +** [[SQLITE_DBCONFIG_RESET_DATABASE]]
    SQLITE_DBCONFIG_RESET_DATABASE
    +**
    Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run +** [VACUUM] in order to reset a database back to an empty database +** with no schema and no content. The following process works even for +** a badly corrupted database file: +**
      +**
    1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. +**
    2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); +**
    3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); +**
    4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); +**
    +** Because resetting a database is destructive and irreversible, the +** process requires the use of this obscure API and multiple steps to +** help ensure that it does not happen by accident. Because this +** feature must be capable of resetting corrupt databases, and +** shutting down virtual tables may require access to that corrupt +** storage, the library must abandon any installed virtual tables +** without calling their xDestroy() methods. +** +** [[SQLITE_DBCONFIG_DEFENSIVE]]
    SQLITE_DBCONFIG_DEFENSIVE
    +**
    The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the +** "defensive" flag for a database connection. When the defensive +** flag is enabled, language features that allow ordinary SQL to +** deliberately corrupt the database file are disabled. The disabled +** features include but are not limited to the following: +**
      +**
    • The [PRAGMA writable_schema=ON] statement. +**
    • The [PRAGMA journal_mode=OFF] statement. +**
    • The [PRAGMA schema_version=N] statement. +**
    • Writes to the [sqlite_dbpage] virtual table. +**
    • Direct writes to [shadow tables]. +**
    +**
    +** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
    SQLITE_DBCONFIG_WRITABLE_SCHEMA
    +**
    The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
    +** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
    SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
    +**
    The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
    +** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
    SQLITE_DBCONFIG_DQS_DML
    +**
    The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statements +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
    +** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
    SQLITE_DBCONFIG_DQS_DDL
    +**
    The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
    +** +** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] +**
    SQLITE_DBCONFIG_TRUSTED_SCHEMA
    +**
    The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to +** assume that database schemas are untainted by malicious content. +** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite +** takes additional defensive steps to protect the application from harm +** including: +**
      +**
    • Prohibit the use of SQL functions inside triggers, views, +** CHECK constraints, DEFAULT clauses, expression indexes, +** partial indexes, or generated columns +** unless those functions are tagged with [SQLITE_INNOCUOUS]. +**
    • Prohibit the use of virtual tables inside of triggers or views +** unless those virtual tables are tagged with [SQLITE_VTAB_INNOCUOUS]. +**
    +** This setting defaults to "on" for legacy compatibility, however +** all applications are advised to turn it off if possible. This setting +** can also be controlled using the [PRAGMA trusted_schema] statement. +**
    +** +** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] +**
    SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
    +**
    The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates +** the legacy file format flag. When activated, this flag causes all newly +** created database file to have a schema format version number (the 4-byte +** integer found at offset 44 into the database header) of 1. This in turn +** means that the resulting database file will be readable and writable by +** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, +** newly created databases are generally not understandable by SQLite versions +** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there +** is now scarcely any need to generate database files that are compatible +** all the way back to version 3.0.0, and so this setting is of little +** practical use, but is provided so that SQLite can continue to claim the +** ability to generate new database files that are compatible with version +** 3.0.0. +**

    Note that when the SQLITE_DBCONFIG_LEGACY_FILE_FORMAT setting is on, +** the [VACUUM] command will fail with an obscure error when attempting to +** process a table with generated columns and a descending index. This is +** not considered a bug since SQLite versions 3.3.0 and earlier do not support +** either generated columns or descending indexes. +**

    +** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
    SQLITE_DBCONFIG_STMT_SCANSTATUS
    +**
    The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
    +** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
    SQLITE_DBCONFIG_REVERSE_SCANORDER
    +**
    The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +**
    +** ** */ +#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ #define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ #define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ #define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ - +#define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_QPSG 1007 /* int int* */ +#define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ +#define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ +#define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ +#define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes @@ -2218,7 +2865,7 @@ struct sqlite3_mem_methods { ** [extended result codes] feature of SQLite. ^The extended result ** codes are disabled by default for historical compatibility. */ -SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3*, int onoff); +SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid @@ -2232,20 +2879,30 @@ SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3*, int onoff) ** the table has a column of type [INTEGER PRIMARY KEY] then that column ** is another alias for the rowid. ** -** ^The sqlite3_last_insert_rowid(D) interface returns the [rowid] of the -** most recent successful [INSERT] into a rowid table or [virtual table] -** on database connection D. -** ^Inserts into [WITHOUT ROWID] tables are not recorded. -** ^If no successful [INSERT]s into rowid tables -** have ever occurred on the database connection D, -** then sqlite3_last_insert_rowid(D) returns zero. +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. ** -** ^(If an [INSERT] occurs within a trigger or within a [virtual table] -** method, then this routine will return the [rowid] of the inserted -** row as long as the trigger or virtual table method is running. -** But once the trigger or virtual table method ends, the value returned -** by this routine reverts to what it was before the trigger or virtual -** table method began.)^ +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. +** +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ ** ** ^An [INSERT] that fails due to a constraint violation is not a ** successful [INSERT] and does not change the value returned by this @@ -2270,84 +2927,121 @@ SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3*, int onoff) ** unpredictable and might not equal either the old or the new ** last insert [rowid]. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); + +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); /* ** CAPI3REF: Count The Number Of Rows Modified ** METHOD: sqlite3 ** -** ^This function returns the number of rows modified, inserted or +** ^These functions return the number of rows modified, inserted or ** deleted by the most recently completed INSERT, UPDATE or DELETE ** statement on the database connection specified by the only parameter. -** ^Executing any other type of SQL statement does not modify the value -** returned by this function. +** The two functions are identical except for the type of the return value +** and that if the number of rows modified by the most recent INSERT, UPDATE +** or DELETE is greater than the maximum value supported by type "int", then +** the return value of sqlite3_changes() is undefined. ^Executing any other +** type of SQL statement does not modify the value returned by these functions. ** ** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are -** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], ** [foreign key actions] or [REPLACE] constraint resolution are not counted. -** -** Changes to a view that are intercepted by -** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value -** returned by sqlite3_changes() immediately after an INSERT, UPDATE or -** DELETE statement run on a view is always zero. Only changes made to real +** +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real ** tables are counted. ** ** Things are more complicated if the sqlite3_changes() function is ** executed while a trigger program is running. This may happen if the ** program uses the [changes() SQL function], or if some other callback ** function invokes sqlite3_changes() directly. Essentially: -** +** **
      **
    • ^(Before entering a trigger program the value returned by -** sqlite3_changes() function is saved. After the trigger program +** sqlite3_changes() function is saved. After the trigger program ** has finished, the original value is restored.)^ -** -**
    • ^(Within a trigger program each INSERT, UPDATE and DELETE -** statement sets the value returned by sqlite3_changes() -** upon completion as normal. Of course, this value will not include -** any changes performed by sub-triggers, as the sqlite3_changes() +** +**
    • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() ** value will be saved and restored after each sub-trigger has run.)^ **
    -** -** ^This means that if the changes() SQL function (or similar) is used -** by the first INSERT, UPDATE or DELETE statement within a trigger, it -** returns the value as set when the calling statement began executing. -** ^If it is used by the second or subsequent such statement within a trigger -** program, the value returned reflects the number of rows modified by the -** previous INSERT, UPDATE or DELETE statement within the same trigger. ** -** See also the [sqlite3_total_changes()] interface, the -** [count_changes pragma], and the [changes() SQL function]. +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. ** ** If a separate thread makes changes on the same database connection ** while [sqlite3_changes()] is running then the value returned ** is unpredictable and not meaningful. +** +** See also: +**
      +**
    • the [sqlite3_total_changes()] interface +**
    • the [count_changes pragma] +**
    • the [changes() SQL function] +**
    • the [data_version pragma] +**
    */ -SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3*); +SQLITE_API int sqlite3_changes(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_changes64(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified ** METHOD: sqlite3 ** -** ^This function returns the total number of rows inserted, modified or +** ^These functions return the total number of rows inserted, modified or ** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed ** since the database connection was opened, including those executed as -** part of trigger programs. ^Executing any other type of SQL statement -** does not affect the value returned by sqlite3_total_changes(). -** +** part of trigger programs. The two functions are identical except for the +** type of the return value and that if the number of rows modified by the +** connection exceeds the maximum value supported by type "int", then +** the return value of sqlite3_total_changes() is undefined. ^Executing +** any other type of SQL statement does not affect the value returned by +** sqlite3_total_changes(). +** ** ^Changes made as part of [foreign key actions] are included in the ** count, but those made as part of REPLACE constraint resolution are -** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers ** are not counted. -** -** See also the [sqlite3_changes()] interface, the -** [count_changes pragma], and the [total_changes() SQL function]. +** +** The [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. ** ** If a separate thread makes changes on the same database connection ** while [sqlite3_total_changes()] is running then the value ** returned is unpredictable and not meaningful. +** +** See also: +**
      +**
    • the [sqlite3_changes()] interface +**
    • the [count_changes pragma] +**
    • the [changes() SQL function] +**
    • the [data_version pragma] +**
    • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
    */ -SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3*); +SQLITE_API int sqlite3_total_changes(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); /* ** CAPI3REF: Interrupt A Long-Running Query @@ -2375,8 +3069,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3*); ** ** ^The sqlite3_interrupt(D) call is in effect until all currently running ** SQL statements on [database connection] D complete. ^Any new SQL statements -** that are started after the sqlite3_interrupt() call and before the -** running statements reaches zero are interrupted as if they had been +** that are started after the sqlite3_interrupt() call and before the +** running statement count reaches zero are interrupted as if they had been ** running prior to the sqlite3_interrupt() call. ^New SQL statements ** that are started after the running statement count reaches zero are ** not effected by the sqlite3_interrupt(). @@ -2384,10 +3078,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3*); ** SQL statements is a no-op and has no effect on SQL statements ** that are started after the sqlite3_interrupt() call returns. ** -** If the database connection closes while [sqlite3_interrupt()] -** is running then bad things will likely happen. +** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether +** or not an interrupt is currently in effect for [database connection] D. +** It returns 1 if an interrupt is currently in effect, or 0 otherwise. */ -SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3*); +SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API int sqlite3_is_interrupted(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete @@ -2410,7 +3106,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3*); ** ^These routines do not parse the SQL statements thus ** will not detect syntactically incorrect SQL. ** -** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior ** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked ** automatically by sqlite3_complete16(). If that initialization fails, ** then the return value from sqlite3_complete16() will be non-zero @@ -2422,8 +3118,8 @@ SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3*); ** The input to [sqlite3_complete16()] must be a zero-terminated ** UTF-16 string in native byte order. */ -SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *sql); -SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *sql); +SQLITE_API int sqlite3_complete(const char *sql); +SQLITE_API int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors @@ -2455,7 +3151,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *sql); ** The presence of a busy handler does not guarantee that it will be invoked ** when there is lock contention. ^If SQLite determines that invoking the busy ** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] -** to the application instead of invoking the +** to the application instead of invoking the ** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and @@ -2480,11 +3176,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *sql); ** database connection that invoked the busy handler. In other words, ** the busy handler is not reentrant. Any such actions ** result in undefined behavior. -** +** ** A busy handler must not close the database connection ** or [prepared statement] that invoked the busy handler. */ -SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); +SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); /* ** CAPI3REF: Set A Busy Timeout @@ -2507,7 +3203,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler(sqlite3*, int(*)(void*,int), ** ** See also: [PRAGMA busy_timeout] */ -SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3*, int ms); +SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries @@ -2547,9 +3243,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3*, int ms); ** Cindy | 21 ** ** -** There are two column (M==2) and three rows (N==3). Thus the +** There are two columns (M==2) and three rows (N==3). Thus the ** result table has 8 entries. Suppose the result table is stored -** in an array names azResult. Then azResult holds this content: +** in an array named azResult. Then azResult holds this content: ** ** 
     **        azResult[0] = "Name";
@@ -2582,7 +3278,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3*, int ms);
 ** reflected in subsequent calls to [sqlite3_errcode()] or
 ** [sqlite3_errmsg()].
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_get_table(
+SQLITE_API int sqlite3_get_table(
   sqlite3 *db,          /* An open database */
   const char *zSql,     /* SQL to be evaluated */
   char ***pazResult,    /* Results of the query */
@@ -2590,23 +3286,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_table(
   int *pnColumn,        /* Number of result columns written here */
   char **pzErrmsg       /* Error msg written here */
 );
-SQLITE_API void SQLITE_STDCALL sqlite3_free_table(char **result);
+SQLITE_API void sqlite3_free_table(char **result);
 
 /*
 ** CAPI3REF: Formatted String Printing Functions
 **
 ** These routines are work-alikes of the "printf()" family of functions
 ** from the standard C library.
-** These routines understand most of the common K&R formatting options,
-** plus some additional non-standard formats, detailed below.
-** Note that some of the more obscure formatting options from recent
-** C-library standards are omitted from this implementation.
+** These routines understand most of the common formatting options from
+** the standard library printf()
+** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]).
+** See the [built-in printf()] documentation for details.
 **
 ** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their
-** results into memory obtained from [sqlite3_malloc()].
+** results into memory obtained from [sqlite3_malloc64()].
 ** The strings returned by these two routines should be
 ** released by [sqlite3_free()].  ^Both routines return a
-** NULL pointer if [sqlite3_malloc()] is unable to allocate enough
+** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough
 ** memory to hold the resulting string.
 **
 ** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from
@@ -2630,83 +3326,19 @@ SQLITE_API void SQLITE_STDCALL sqlite3_free_table(char **result);
 **
 ** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf().
 **
-** These routines all implement some additional formatting
-** options that are useful for constructing SQL statements.
-** All of the usual printf() formatting options apply.  In addition, there
-** is are "%q", "%Q", "%w" and "%z" options.
-**
-** ^(The %q option works like %s in that it substitutes a nul-terminated
-** string from the argument list.  But %q also doubles every '\'' character.
-** %q is designed for use inside a string literal.)^  By doubling each '\''
-** character it escapes that character and allows it to be inserted into
-** the string.
-**
-** For example, assume the string variable zText contains text as follows:
-**
-** 
    -**  char *zText = "It's a happy day!";
-** 
    
-**
-** One can use this text in an SQL statement as follows:
-**
-** 
    -**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-** 
    
-**
-** Because the %q format string is used, the '\'' character in zText
-** is escaped and the SQL generated is as follows:
-**
-** 
    -**  INSERT INTO table1 VALUES('It''s a happy day!')
-** 
    
-**
-** This is correct.  Had we used %s instead of %q, the generated SQL
-** would have looked like this:
-**
-** 
    -**  INSERT INTO table1 VALUES('It's a happy day!');
-** 
    
-**
-** This second example is an SQL syntax error.  As a general rule you should
-** always use %q instead of %s when inserting text into a string literal.
-**
-** ^(The %Q option works like %q except it also adds single quotes around
-** the outside of the total string.  Additionally, if the parameter in the
-** argument list is a NULL pointer, %Q substitutes the text "NULL" (without
-** single quotes).)^  So, for example, one could say:
-**
-** 
    -**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-** 
    
-**
-** The code above will render a correct SQL statement in the zSQL
-** variable even if the zText variable is a NULL pointer.
-**
-** ^(The "%w" formatting option is like "%q" except that it expects to
-** be contained within double-quotes instead of single quotes, and it
-** escapes the double-quote character instead of the single-quote
-** character.)^  The "%w" formatting option is intended for safely inserting
-** table and column names into a constructed SQL statement.
-**
-** ^(The "%z" formatting option works like "%s" but with the
-** addition that after the string has been read and copied into
-** the result, [sqlite3_free()] is called on the input string.)^
+** See also:  [built-in printf()], [printf() SQL function]
 */
-SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char*,...);
-SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char*, va_list);
-SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int,char*,const char*, ...);
-SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int,char*,const char*, va_list);
+SQLITE_API char *sqlite3_mprintf(const char*,...);
+SQLITE_API char *sqlite3_vmprintf(const char*, va_list);
+SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...);
+SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list);
 
 /*
 ** CAPI3REF: Memory Allocation Subsystem
 **
 ** The SQLite core uses these three routines for all of its own
 ** internal memory allocation needs. "Core" in the previous sentence
-** does not include operating-system specific VFS implementation.  The
+** does not include operating-system specific [VFS] implementation.  The
 ** Windows VFS uses native malloc() and free() for some operations.
 **
 ** ^The sqlite3_malloc() routine returns a pointer to a block
@@ -2767,19 +3399,6 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int,char*,const char*, va_list
 ** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time
 ** option is used.
 **
-** In SQLite version 3.5.0 and 3.5.1, it was possible to define
-** the SQLITE_OMIT_MEMORY_ALLOCATION which would cause the built-in
-** implementation of these routines to be omitted.  That capability
-** is no longer provided.  Only built-in memory allocators can be used.
-**
-** Prior to SQLite version 3.7.10, the Windows OS interface layer called
-** the system malloc() and free() directly when converting
-** filenames between the UTF-8 encoding used by SQLite
-** and whatever filename encoding is used by the particular Windows
-** installation.  Memory allocation errors were detected, but
-** they were reported back as [SQLITE_CANTOPEN] or
-** [SQLITE_IOERR] rather than [SQLITE_NOMEM].
-**
 ** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()]
 ** must be either NULL or else pointers obtained from a prior
 ** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have
@@ -2789,12 +3408,12 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int,char*,const char*, va_list
 ** a block of memory after it has been released using
 ** [sqlite3_free()] or [sqlite3_realloc()].
 */
-SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int);
-SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64);
-SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void*, int);
-SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void*, sqlite3_uint64);
-SQLITE_API void SQLITE_STDCALL sqlite3_free(void*);
-SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void*);
+SQLITE_API void *sqlite3_malloc(int);
+SQLITE_API void *sqlite3_malloc64(sqlite3_uint64);
+SQLITE_API void *sqlite3_realloc(void*, int);
+SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64);
+SQLITE_API void sqlite3_free(void*);
+SQLITE_API sqlite3_uint64 sqlite3_msize(void*);
 
 /*
 ** CAPI3REF: Memory Allocator Statistics
@@ -2819,8 +3438,8 @@ SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void*);
 ** by [sqlite3_memory_highwater(1)] is the high-water mark
 ** prior to the reset.
 */
-SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void);
-SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag);
+SQLITE_API sqlite3_int64 sqlite3_memory_used(void);
+SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag);
 
 /*
 ** CAPI3REF: Pseudo-Random Number Generator
@@ -2828,7 +3447,7 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag);
 ** SQLite contains a high-quality pseudo-random number generator (PRNG) used to
 ** select random [ROWID | ROWIDs] when inserting new records into a table that
 ** already uses the largest possible [ROWID].  The PRNG is also used for
-** the build-in random() and randomblob() SQL functions.  This interface allows
+** the built-in random() and randomblob() SQL functions.  This interface allows
 ** applications to access the same PRNG for other purposes.
 **
 ** ^A call to this routine stores N bytes of randomness into buffer P.
@@ -2843,17 +3462,19 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag);
 ** internally and without recourse to the [sqlite3_vfs] xRandomness
 ** method.
 */
-SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P);
+SQLITE_API void sqlite3_randomness(int N, void *P);
 
 /*
 ** CAPI3REF: Compile-Time Authorization Callbacks
 ** METHOD: sqlite3
+** KEYWORDS: {authorizer callback}
 **
 ** ^This routine registers an authorizer callback with a particular
 ** [database connection], supplied in the first argument.
 ** ^The authorizer callback is invoked as SQL statements are being compiled
 ** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()],
-** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()].  ^At various
+** [sqlite3_prepare_v3()], [sqlite3_prepare16()], [sqlite3_prepare16_v2()],
+** and [sqlite3_prepare16_v3()].  ^At various
 ** points during the compilation process, as logic is being created
 ** to perform various actions, the authorizer callback is invoked to
 ** see if those actions are allowed.  ^The authorizer callback should
@@ -2869,14 +3490,16 @@ SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P);
 ** requested is ok.  ^When the callback returns [SQLITE_DENY], the
 ** [sqlite3_prepare_v2()] or equivalent call that triggered the
 ** authorizer will fail with an error message explaining that
-** access is denied. 
+** access is denied.
 **
 ** ^The first parameter to the authorizer callback is a copy of the third
 ** parameter to the sqlite3_set_authorizer() interface. ^The second parameter
 ** to the callback is an integer [SQLITE_COPY | action code] that specifies
 ** the particular action to be authorized. ^The third through sixth parameters
-** to the callback are zero-terminated strings that contain additional
-** details about the action to be authorized.
+** to the callback are either NULL pointers or zero-terminated strings
+** that contain additional details about the action to be authorized.
+** Applications must always be prepared to encounter a NULL pointer in any
+** of the third through the sixth parameters of the authorization callback.
 **
 ** ^If the action code is [SQLITE_READ]
 ** and the callback returns [SQLITE_IGNORE] then the
@@ -2885,6 +3508,10 @@ SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P);
 ** been read if [SQLITE_OK] had been returned.  The [SQLITE_IGNORE]
 ** return can be used to deny an untrusted user access to individual
 ** columns of a table.
+** ^When a table is referenced by a [SELECT] but no column values are
+** extracted from that table (for example in a query like
+** "SELECT count(*) FROM tab") then the [SQLITE_READ] authorizer callback
+** is invoked once for that table with a column name that is an empty string.
 ** ^If the action code is [SQLITE_DELETE] and the callback returns
 ** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the
 ** [truncate optimization] is disabled and all rows are deleted individually.
@@ -2916,7 +3543,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P);
 ** database connections for the meaning of "modify" in this paragraph.
 **
 ** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the
-** statement might be re-prepared during [sqlite3_step()] due to a 
+** statement might be re-prepared during [sqlite3_step()] due to a
 ** schema change.  Hence, the application should ensure that the
 ** correct authorizer callback remains in place during the [sqlite3_step()].
 **
@@ -2926,7 +3553,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *P);
 ** as stated in the previous paragraph, sqlite3_step() invokes
 ** sqlite3_prepare_v2() to reprepare a statement after a schema change.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer(
+SQLITE_API int sqlite3_set_authorizer(
   sqlite3*,
   int (*xAuth)(void*,int,const char*,const char*,const char*,const char*),
   void *pUserData
@@ -3003,8 +3630,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer(
 #define SQLITE_RECURSIVE            33   /* NULL            NULL            */
 
 /*
-** CAPI3REF: Tracing And Profiling Functions
-** METHOD: sqlite3
+** CAPI3REF: Deprecated Tracing And Profiling Functions
+** DEPRECATED
+**
+** These routines are deprecated. Use the [sqlite3_trace_v2()] interface
+** instead of the routines described here.
 **
 ** These routines register callback functions that can be used for
 ** tracing and profiling the execution of SQL statements.
@@ -3027,26 +3657,122 @@ SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer(
 ** time is in units of nanoseconds, however the current implementation
 ** is only capable of millisecond resolution so the six least significant
 ** digits in the time are meaningless.  Future versions of SQLite
-** might provide greater resolution on the profiler callback.  The
-** sqlite3_profile() function is considered experimental and is
-** subject to change in future versions of SQLite.
+** might provide greater resolution on the profiler callback.  Invoking
+** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the
+** profile callback.
 */
-SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*);
-SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_profile(sqlite3*,
+SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*,
+   void(*xTrace)(void*,const char*), void*);
+SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*,
    void(*xProfile)(void*,const char*,sqlite3_uint64), void*);
 
+/*
+** CAPI3REF: SQL Trace Event Codes
+** KEYWORDS: SQLITE_TRACE
+**
+** These constants identify classes of events that can be monitored
+** using the [sqlite3_trace_v2()] tracing logic.  The M argument
+** to [sqlite3_trace_v2(D,M,X,P)] is an OR-ed combination of one or more of
+** the following constants.  ^The first argument to the trace callback
+** is one of the following constants.
+**
+** New tracing constants may be added in future releases.
+**
+** ^A trace callback has four arguments: xCallback(T,C,P,X).
+** ^The T argument is one of the integer type codes above.
+** ^The C argument is a copy of the context pointer passed in as the
+** fourth argument to [sqlite3_trace_v2()].
+** The P and X arguments are pointers whose meanings depend on T.
+**
+** 
    
+** [[SQLITE_TRACE_STMT]] 
    SQLITE_TRACE_STMT
    
+** 
    ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement
+** first begins running and possibly at other times during the
+** execution of the prepared statement, such as at the start of each
+** trigger subprogram. ^The P argument is a pointer to the
+** [prepared statement]. ^The X argument is a pointer to a string which
+** is the unexpanded SQL text of the prepared statement or an SQL comment
+** that indicates the invocation of a trigger.  ^The callback can compute
+** the same text that would have been returned by the legacy [sqlite3_trace()]
+** interface by using the X argument when X begins with "--" and invoking
+** [sqlite3_expanded_sql(P)] otherwise.
+**
+** [[SQLITE_TRACE_PROFILE]] 
    SQLITE_TRACE_PROFILE
    
+** 
    ^An SQLITE_TRACE_PROFILE callback provides approximately the same
+** information as is provided by the [sqlite3_profile()] callback.
+** ^The P argument is a pointer to the [prepared statement] and the
+** X argument points to a 64-bit integer which is approximately
+** the number of nanoseconds that the prepared statement took to run.
+** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes.
+**
+** [[SQLITE_TRACE_ROW]] 
    SQLITE_TRACE_ROW
    
+** 
    ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared
+** statement generates a single row of result.
+** ^The P argument is a pointer to the [prepared statement] and the
+** X argument is unused.
+**
+** [[SQLITE_TRACE_CLOSE]] 
    SQLITE_TRACE_CLOSE
    
+** 
    ^An SQLITE_TRACE_CLOSE callback is invoked when a database
+** connection closes.
+** ^The P argument is a pointer to the [database connection] object
+** and the X argument is unused.
+** 
    
+*/
+#define SQLITE_TRACE_STMT       0x01
+#define SQLITE_TRACE_PROFILE    0x02
+#define SQLITE_TRACE_ROW        0x04
+#define SQLITE_TRACE_CLOSE      0x08
+
+/*
+** CAPI3REF: SQL Trace Hook
+** METHOD: sqlite3
+**
+** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback
+** function X against [database connection] D, using property mask M
+** and context pointer P.  ^If the X callback is
+** NULL or if the M mask is zero, then tracing is disabled.  The
+** M argument should be the bitwise OR-ed combination of
+** zero or more [SQLITE_TRACE] constants.
+**
+** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P)
+** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or
+** sqlite3_trace_v2(D,M,X,P) for the [database connection] D.  Each
+** database connection may have at most one trace callback.
+**
+** ^The X callback is invoked whenever any of the events identified by
+** mask M occur.  ^The integer return value from the callback is currently
+** ignored, though this may change in future releases.  Callback
+** implementations should return zero to ensure future compatibility.
+**
+** ^A trace callback is invoked with four arguments: callback(T,C,P,X).
+** ^The T argument is one of the [SQLITE_TRACE]
+** constants to indicate why the callback was invoked.
+** ^The C argument is a copy of the context pointer.
+** The P and X arguments are pointers whose meanings depend on T.
+**
+** The sqlite3_trace_v2() interface is intended to replace the legacy
+** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which
+** are deprecated.
+*/
+SQLITE_API int sqlite3_trace_v2(
+  sqlite3*,
+  unsigned uMask,
+  int(*xCallback)(unsigned,void*,void*,void*),
+  void *pCtx
+);
+
 /*
 ** CAPI3REF: Query Progress Callbacks
 ** METHOD: sqlite3
 **
 ** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback
 ** function X to be invoked periodically during long running calls to
-** [sqlite3_exec()], [sqlite3_step()] and [sqlite3_get_table()] for
+** [sqlite3_step()] and [sqlite3_prepare()] and similar for
 ** database connection D.  An example use for this
 ** interface is to keep a GUI updated during a large query.
 **
-** ^The parameter P is passed through as the only parameter to the 
-** callback function X.  ^The parameter N is the approximate number of 
+** ^The parameter P is passed through as the only parameter to the
+** callback function X.  ^The parameter N is the approximate number of
 ** [virtual machine instructions] that are evaluated between successive
 ** invocations of the callback X.  ^If N is less than one then the progress
 ** handler is disabled.
@@ -3066,14 +3792,21 @@ SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_profile(sqlite3*,
 ** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their
 ** database connections for the meaning of "modify" in this paragraph.
 **
+** The progress handler callback would originally only be invoked from the
+** bytecode engine.  It still might be invoked during [sqlite3_prepare()]
+** and similar because those routines might force a reparse of the schema
+** which involves running the bytecode engine.  However, beginning with
+** SQLite version 3.41.0, the progress handler callback might also be
+** invoked directly from [sqlite3_prepare()] while analyzing and generating
+** code for complex queries.
 */
-SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
+SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*);
 
 /*
 ** CAPI3REF: Opening A New Database Connection
 ** CONSTRUCTOR: sqlite3
 **
-** ^These routines open an SQLite database file as specified by the 
+** ^These routines open an SQLite database file as specified by the
 ** filename argument. ^The filename argument is interpreted as UTF-8 for
 ** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte
 ** order for sqlite3_open16(). ^(A [database connection] handle is usually
@@ -3097,20 +3830,23 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 ** The sqlite3_open_v2() interface works like sqlite3_open()
 ** except that it accepts two additional parameters for additional control
 ** over the new database connection.  ^(The flags parameter to
-** sqlite3_open_v2() can take one of
-** the following three values, optionally combined with the 
-** [SQLITE_OPEN_NOMUTEX], [SQLITE_OPEN_FULLMUTEX], [SQLITE_OPEN_SHAREDCACHE],
-** [SQLITE_OPEN_PRIVATECACHE], and/or [SQLITE_OPEN_URI] flags:)^
+** sqlite3_open_v2() must include, at a minimum, one of the following
+** three flag combinations:)^
 **
 ** 
    
 ** ^(
    [SQLITE_OPEN_READONLY]
    
-** 
    The database is opened in read-only mode.  If the database does not
-** already exist, an error is returned.
    )^
+** 
    The database is opened in read-only mode.  If the database does
+** not already exist, an error is returned.
    )^
 **
 ** ^(
    [SQLITE_OPEN_READWRITE]
    
-** 
    The database is opened for reading and writing if possible, or reading
-** only if the file is write protected by the operating system.  In either
-** case the database must already exist, otherwise an error is returned.
    )^
+** 
    The database is opened for reading and writing if possible, or
+** reading only if the file is write protected by the operating
+** system.  In either case the database must already exist, otherwise
+** an error is returned.  For historical reasons, if opening in
+** read-write mode fails due to OS-level permissions, an attempt is
+** made to open it in read-only mode. [sqlite3_db_readonly()] can be
+** used to determine whether the database is actually
+** read-write.
    )^
 **
 ** ^(
    [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
    
 ** 
    The database is opened for reading and writing, and is created if
@@ -3118,22 +3854,69 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 ** sqlite3_open() and sqlite3_open16().
    )^
 ** 
    
 **
-** If the 3rd parameter to sqlite3_open_v2() is not one of the
-** combinations shown above optionally combined with other
-** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits]
-** then the behavior is undefined.
+** In addition to the required flags, the following optional flags are
+** also supported:
 **
-** ^If the [SQLITE_OPEN_NOMUTEX] flag is set, then the database connection
-** opens in the multi-thread [threading mode] as long as the single-thread
-** mode has not been set at compile-time or start-time.  ^If the
-** [SQLITE_OPEN_FULLMUTEX] flag is set then the database connection opens
-** in the serialized [threading mode] unless single-thread was
-** previously selected at compile-time or start-time.
-** ^The [SQLITE_OPEN_SHAREDCACHE] flag causes the database connection to be
-** eligible to use [shared cache mode], regardless of whether or not shared
-** cache is enabled using [sqlite3_enable_shared_cache()].  ^The
-** [SQLITE_OPEN_PRIVATECACHE] flag causes the database connection to not
-** participate in [shared cache mode] even if it is enabled.
+** 
    
+** ^(
    [SQLITE_OPEN_URI]
    
+** 
    The filename can be interpreted as a URI if this flag is set.
    )^
+**
+** ^(
    [SQLITE_OPEN_MEMORY]
    
+** 
    The database will be opened as an in-memory database.  The database
+** is named by the "filename" argument for the purposes of cache-sharing,
+** if shared cache mode is enabled, but the "filename" is otherwise ignored.
+** 
    )^
+**
+** ^(
    [SQLITE_OPEN_NOMUTEX]
    
+** 
    The new database connection will use the "multi-thread"
+** [threading mode].)^  This means that separate threads are allowed
+** to use SQLite at the same time, as long as each thread is using
+** a different [database connection].
+**
+** ^(
    [SQLITE_OPEN_FULLMUTEX]
    
+** 
    The new database connection will use the "serialized"
+** [threading mode].)^  This means the multiple threads can safely
+** attempt to use the same database connection at the same time.
+** (Mutexes will block any actual concurrency, but in this mode
+** there is no harm in trying.)
+**
+** ^(
    [SQLITE_OPEN_SHAREDCACHE]
    
+** 
    The database is opened [shared cache] enabled, overriding
+** the default shared cache setting provided by
+** [sqlite3_enable_shared_cache()].)^
+** The [use of shared cache mode is discouraged] and hence shared cache
+** capabilities may be omitted from many builds of SQLite.  In such cases,
+** this option is a no-op.
+**
+** ^(
    [SQLITE_OPEN_PRIVATECACHE]
    
+** 
    The database is opened [shared cache] disabled, overriding
+** the default shared cache setting provided by
+** [sqlite3_enable_shared_cache()].)^
+**
+** [[OPEN_EXRESCODE]] ^(
    [SQLITE_OPEN_EXRESCODE]
    
+** 
    The database connection comes up in "extended result code mode".
+** In other words, the database behaves as if
+** [sqlite3_extended_result_codes(db,1)] were called on the database
+** connection as soon as the connection is created. In addition to setting
+** the extended result code mode, this flag also causes [sqlite3_open_v2()]
+** to return an extended result code.
    
+**
+** [[OPEN_NOFOLLOW]] ^(
    [SQLITE_OPEN_NOFOLLOW]
    
+** 
    The database filename is not allowed to contain a symbolic link
    
+** 
    )^
+**
+** If the 3rd parameter to sqlite3_open_v2() is not one of the
+** required combinations shown above optionally combined with other
+** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits]
+** then the behavior is undefined.  Historic versions of SQLite
+** have silently ignored surplus bits in the flags parameter to
+** sqlite3_open_v2(), however that behavior might not be carried through
+** into future versions of SQLite and so applications should not rely
+** upon it.  Note in particular that the SQLITE_OPEN_EXCLUSIVE flag is a no-op
+** for sqlite3_open_v2().  The SQLITE_OPEN_EXCLUSIVE does *not* cause
+** the open to fail if the database already exists.  The SQLITE_OPEN_EXCLUSIVE
+** flag is intended for use by the [sqlite3_vfs|VFS interface] only, and not
+** by sqlite3_open_v2().
 **
 ** ^The fourth parameter to sqlite3_open_v2() is the name of the
 ** [sqlite3_vfs] object that defines the operating system interface that
@@ -3157,26 +3940,26 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 ** ^If [URI filename] interpretation is enabled, and the filename argument
 ** begins with "file:", then the filename is interpreted as a URI. ^URI
 ** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is
-** set in the fourth argument to sqlite3_open_v2(), or if it has
+** set in the third argument to sqlite3_open_v2(), or if it has
 ** been enabled globally using the [SQLITE_CONFIG_URI] option with the
 ** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option.
-** As of SQLite version 3.7.7, URI filename interpretation is turned off
+** URI filename interpretation is turned off
 ** by default, but future releases of SQLite might enable URI filename
 ** interpretation by default.  See "[URI filenames]" for additional
 ** information.
 **
 ** URI filenames are parsed according to RFC 3986. ^If the URI contains an
-** authority, then it must be either an empty string or the string 
-** "localhost". ^If the authority is not an empty string or "localhost", an 
-** error is returned to the caller. ^The fragment component of a URI, if 
+** authority, then it must be either an empty string or the string
+** "localhost". ^If the authority is not an empty string or "localhost", an
+** error is returned to the caller. ^The fragment component of a URI, if
 ** present, is ignored.
 **
 ** ^SQLite uses the path component of the URI as the name of the disk file
-** which contains the database. ^If the path begins with a '/' character, 
-** then it is interpreted as an absolute path. ^If the path does not begin 
+** which contains the database. ^If the path begins with a '/' character,
+** then it is interpreted as an absolute path. ^If the path does not begin
 ** with a '/' (meaning that the authority section is omitted from the URI)
-** then the path is interpreted as a relative path. 
-** ^(On windows, the first component of an absolute path 
+** then the path is interpreted as a relative path.
+** ^(On windows, the first component of an absolute path
 ** is a drive specification (e.g. "C:").)^
 **
 ** [[core URI query parameters]]
@@ -3196,13 +3979,13 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 **
 **   
  •  mode: ^(The mode parameter may be set to either "ro", "rw",
 **     "rwc", or "memory". Attempting to set it to any other value is
-**     an error)^. 
-**     ^If "ro" is specified, then the database is opened for read-only 
-**     access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the 
-**     third argument to sqlite3_open_v2(). ^If the mode option is set to 
-**     "rw", then the database is opened for read-write (but not create) 
-**     access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had 
-**     been set. ^Value "rwc" is equivalent to setting both 
+**     an error)^.
+**     ^If "ro" is specified, then the database is opened for read-only
+**     access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the
+**     third argument to sqlite3_open_v2(). ^If the mode option is set to
+**     "rw", then the database is opened for read-write (but not create)
+**     access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had
+**     been set. ^Value "rwc" is equivalent to setting both
 **     SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE.  ^If the mode option is
 **     set to "memory" then a pure [in-memory database] that never reads
 **     or writes from disk is used. ^It is an error to specify a value for
@@ -3212,7 +3995,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 **   
  •  cache: ^The cache parameter may be set to either "shared" or
 **     "private". ^Setting it to "shared" is equivalent to setting the
 **     SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to
-**     sqlite3_open_v2(). ^Setting the cache parameter to "private" is 
+**     sqlite3_open_v2(). ^Setting the cache parameter to "private" is
 **     equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit.
 **     ^If sqlite3_open_v2() is used and the "cache" parameter is present in
 **     a URI filename, its value overrides any behavior requested by setting
@@ -3238,7 +4021,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 **     property on a database file that does in fact change can result
 **     in incorrect query results and/or [SQLITE_CORRUPT] errors.
 **     See also: [SQLITE_IOCAP_IMMUTABLE].
-**       
+**
 ** 
 **
 ** ^Specifying an unknown parameter in the query component of a URI is not an
@@ -3250,36 +4033,37 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 **
 ** 
 ** 
     URI filenames  Results
-** 
     file:data.db  
+** 
     file:data.db 
 **          Open the file "data.db" in the current directory.
 ** 
     file:/home/fred/data.db
    
-**          file:///home/fred/data.db 
     
-**          file://localhost/home/fred/data.db 
         		 
+**          file:///home/fred/data.db 
    
+**          file://localhost/home/fred/data.db 
         		
 **          Open the database file "/home/fred/data.db".
-** 
     file://darkstar/home/fred/data.db  
+** 
     file://darkstar/home/fred/data.db 
 **          An error. "darkstar" is not a recognized authority.
-** 
     
+** 
    
 **          file:///C:/Documents%20and%20Settings/fred/Desktop/data.db
 **      Windows only: Open the file "data.db" on fred's desktop on drive
-**          C:. Note that the %20 escaping in this example is not strictly 
+**          C:. Note that the %20 escaping in this example is not strictly
 **          necessary - space characters can be used literally
 **          in URI filenames.
-** 
     file:data.db?mode=ro&cache=private  
+** 
     file:data.db?mode=ro&cache=private 
 **          Open file "data.db" in the current directory for read-only access.
 **          Regardless of whether or not shared-cache mode is enabled by
 **          default, use a private cache.
 ** 
     file:/home/fred/data.db?vfs=unix-dotfile 
 **          Open file "/home/fred/data.db". Use the special VFS "unix-dotfile"
 **          that uses dot-files in place of posix advisory locking.
-** 
     file:data.db?mode=readonly  
+** 
     file:data.db?mode=readonly 
 **          An error. "readonly" is not a valid option for the "mode" parameter.
+**          Use "ro" instead:  "file:data.db?mode=ro".
 ** 
    
 **
 ** ^URI hexadecimal escape sequences (%HH) are supported within the path and
 ** query components of a URI. A hexadecimal escape sequence consists of a
-** percent sign - "%" - followed by exactly two hexadecimal digits 
+** percent sign - "%" - followed by exactly two hexadecimal digits
 ** specifying an octet value. ^Before the path or query components of a
-** URI filename are interpreted, they are encoded using UTF-8 and all 
+** URI filename are interpreted, they are encoded using UTF-8 and all
 ** hexadecimal escape sequences replaced by a single byte containing the
 ** corresponding octet. If this process generates an invalid UTF-8 encoding,
 ** the results are undefined.
@@ -3296,15 +4080,15 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler(sqlite3*, int, int(*)(vo
 **
 ** See also: [sqlite3_temp_directory]
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_open(
+SQLITE_API int sqlite3_open(
   const char *filename,   /* Database filename (UTF-8) */
   sqlite3 **ppDb          /* OUT: SQLite db handle */
 );
-SQLITE_API int SQLITE_STDCALL sqlite3_open16(
+SQLITE_API int sqlite3_open16(
   const void *filename,   /* Database filename (UTF-16) */
   sqlite3 **ppDb          /* OUT: SQLite db handle */
 );
-SQLITE_API int SQLITE_STDCALL sqlite3_open_v2(
+SQLITE_API int sqlite3_open_v2(
   const char *filename,   /* Database filename (UTF-8) */
   sqlite3 **ppDb,         /* OUT: SQLite db handle */
   int flags,              /* Flags */
@@ -3314,17 +4098,27 @@ SQLITE_API int SQLITE_STDCALL sqlite3_open_v2(
 /*
 ** CAPI3REF: Obtain Values For URI Parameters
 **
-** These are utility routines, useful to VFS implementations, that check
-** to see if a database file was a URI that contained a specific query 
+** These are utility routines, useful to [VFS|custom VFS implementations],
+** that check if a database file was a URI that contained a specific query
 ** parameter, and if so obtains the value of that query parameter.
 **
-** If F is the database filename pointer passed into the xOpen() method of 
-** a VFS implementation when the flags parameter to xOpen() has one or 
-** more of the [SQLITE_OPEN_URI] or [SQLITE_OPEN_MAIN_DB] bits set and
-** P is the name of the query parameter, then
+** The first parameter to these interfaces (hereafter referred to
+** as F) must be one of:
+** 
      
+** 
    •  A database filename pointer created by the SQLite core and
+** passed into the xOpen() method of a VFS implementation, or
+** 
    •  A filename obtained from [sqlite3_db_filename()], or
+** 
    •  A new filename constructed using [sqlite3_create_filename()].
+** 
    
+** If the F parameter is not one of the above, then the behavior is
+** undefined and probably undesirable.  Older versions of SQLite were
+** more tolerant of invalid F parameters than newer versions.
+**
+** If F is a suitable filename (as described in the previous paragraph)
+** and if P is the name of the query parameter, then
 ** sqlite3_uri_parameter(F,P) returns the value of the P
-** parameter if it exists or a NULL pointer if P does not appear as a 
-** query parameter on F.  If P is a query parameter of F
+** parameter if it exists or a NULL pointer if P does not appear as a
+** query parameter on F.  If P is a query parameter of F and it
 ** has no explicit value, then sqlite3_uri_parameter(F,P) returns
 ** a pointer to an empty string.
 **
@@ -3332,56 +4126,199 @@ SQLITE_API int SQLITE_STDCALL sqlite3_open_v2(
 ** parameter and returns true (1) or false (0) according to the value
 ** of P.  The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the
 ** value of query parameter P is one of "yes", "true", or "on" in any
-** case or if the value begins with a non-zero number.  The 
+** case or if the value begins with a non-zero number.  The
 ** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of
 ** query parameter P is one of "no", "false", or "off" in any case or
 ** if the value begins with a numeric zero.  If P is not a query
-** parameter on F or if the value of P is does not match any of the
+** parameter on F or if the value of P does not match any of the
 ** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0).
 **
 ** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a
 ** 64-bit signed integer and returns that integer, or D if P does not
 ** exist.  If the value of P is something other than an integer, then
 ** zero is returned.
-** 
+**
+** The sqlite3_uri_key(F,N) returns a pointer to the name (not
+** the value) of the N-th query parameter for filename F, or a NULL
+** pointer if N is less than zero or greater than the number of query
+** parameters minus 1.  The N value is zero-based so N should be 0 to obtain
+** the name of the first query parameter, 1 for the second parameter, and
+** so forth.
+**
 ** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and
 ** sqlite3_uri_boolean(F,P,B) returns B.  If F is not a NULL pointer and
-** is not a database file pathname pointer that SQLite passed into the xOpen
-** VFS method, then the behavior of this routine is undefined and probably
-** undesirable.
+** is not a database file pathname pointer that the SQLite core passed
+** into the xOpen VFS method, then the behavior of this routine is undefined
+** and probably undesirable.
+**
+** Beginning with SQLite [version 3.31.0] ([dateof:3.31.0]) the input F
+** parameter can also be the name of a rollback journal file or WAL file
+** in addition to the main database file.  Prior to version 3.31.0, these
+** routines would only work if F was the name of the main database file.
+** When the F parameter is the name of the rollback journal or WAL file,
+** it has access to all the same query parameters as were found on the
+** main database file.
+**
+** See the [URI filename] documentation for additional information.
 */
-SQLITE_API const char *SQLITE_STDCALL sqlite3_uri_parameter(const char *zFilename, const char *zParam);
-SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFile, const char *zParam, int bDefault);
-SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64(const char*, const char*, sqlite3_int64);
+SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam);
+SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault);
+SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64);
+SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N);
 
+/*
+** CAPI3REF:  Translate filenames
+**
+** These routines are available to [VFS|custom VFS implementations] for
+** translating filenames between the main database file, the journal file,
+** and the WAL file.
+**
+** If F is the name of an sqlite database file, journal file, or WAL file
+** passed by the SQLite core into the VFS, then sqlite3_filename_database(F)
+** returns the name of the corresponding database file.
+**
+** If F is the name of an sqlite database file, journal file, or WAL file
+** passed by the SQLite core into the VFS, or if F is a database filename
+** obtained from [sqlite3_db_filename()], then sqlite3_filename_journal(F)
+** returns the name of the corresponding rollback journal file.
+**
+** If F is the name of an sqlite database file, journal file, or WAL file
+** that was passed by the SQLite core into the VFS, or if F is a database
+** filename obtained from [sqlite3_db_filename()], then
+** sqlite3_filename_wal(F) returns the name of the corresponding
+** WAL file.
+**
+** In all of the above, if F is not the name of a database, journal or WAL
+** filename passed into the VFS from the SQLite core and F is not the
+** return value from [sqlite3_db_filename()], then the result is
+** undefined and is likely a memory access violation.
+*/
+SQLITE_API const char *sqlite3_filename_database(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename);
+SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename);
+
+/*
+** CAPI3REF:  Database File Corresponding To A Journal
+**
+** ^If X is the name of a rollback or WAL-mode journal file that is
+** passed into the xOpen method of [sqlite3_vfs], then
+** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file]
+** object that represents the main database file.
+**
+** This routine is intended for use in custom [VFS] implementations
+** only.  It is not a general-purpose interface.
+** The argument sqlite3_file_object(X) must be a filename pointer that
+** has been passed into [sqlite3_vfs].xOpen method where the
+** flags parameter to xOpen contains one of the bits
+** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL].  Any other use
+** of this routine results in undefined and probably undesirable
+** behavior.
+*/
+SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*);
+
+/*
+** CAPI3REF: Create and Destroy VFS Filenames
+**
+** These interfaces are provided for use by [VFS shim] implementations and
+** are not useful outside of that context.
+**
+** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of
+** database filename D with corresponding journal file J and WAL file W and
+** with N URI parameters key/values pairs in the array P.  The result from
+** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that
+** is safe to pass to routines like:
+** 
      
+** 
    •  [sqlite3_uri_parameter()],
+** 
    •  [sqlite3_uri_boolean()],
+** 
    •  [sqlite3_uri_int64()],
+** 
    •  [sqlite3_uri_key()],
+** 
    •  [sqlite3_filename_database()],
+** 
    •  [sqlite3_filename_journal()], or
+** 
    •  [sqlite3_filename_wal()].
+** 
    
+** If a memory allocation error occurs, sqlite3_create_filename() might
+** return a NULL pointer.  The memory obtained from sqlite3_create_filename(X)
+** must be released by a corresponding call to sqlite3_free_filename(Y).
+**
+** The P parameter in sqlite3_create_filename(D,J,W,N,P) should be an array
+** of 2*N pointers to strings.  Each pair of pointers in this array corresponds
+** to a key and value for a query parameter.  The P parameter may be a NULL
+** pointer if N is zero.  None of the 2*N pointers in the P array may be
+** NULL pointers and key pointers should not be empty strings.
+** None of the D, J, or W parameters to sqlite3_create_filename(D,J,W,N,P) may
+** be NULL pointers, though they can be empty strings.
+**
+** The sqlite3_free_filename(Y) routine releases a memory allocation
+** previously obtained from sqlite3_create_filename().  Invoking
+** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op.
+**
+** If the Y parameter to sqlite3_free_filename(Y) is anything other
+** than a NULL pointer or a pointer previously acquired from
+** sqlite3_create_filename(), then bad things such as heap
+** corruption or segfaults may occur. The value Y should not be
+** used again after sqlite3_free_filename(Y) has been called.  This means
+** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y,
+** then the corresponding [sqlite3_module.xClose() method should also be
+** invoked prior to calling sqlite3_free_filename(Y).
+*/
+SQLITE_API sqlite3_filename sqlite3_create_filename(
+  const char *zDatabase,
+  const char *zJournal,
+  const char *zWal,
+  int nParam,
+  const char **azParam
+);
+SQLITE_API void sqlite3_free_filename(sqlite3_filename);
 
 /*
 ** CAPI3REF: Error Codes And Messages
 ** METHOD: sqlite3
 **
-** ^If the most recent sqlite3_* API call associated with 
+** ^If the most recent sqlite3_* API call associated with
 ** [database connection] D failed, then the sqlite3_errcode(D) interface
 ** returns the numeric [result code] or [extended result code] for that
 ** API call.
-** If the most recent API call was successful,
-** then the return value from sqlite3_errcode() is undefined.
 ** ^The sqlite3_extended_errcode()
-** interface is the same except that it always returns the 
+** interface is the same except that it always returns the
 ** [extended result code] even when extended result codes are
 ** disabled.
 **
+** The values returned by sqlite3_errcode() and/or
+** sqlite3_extended_errcode() might change with each API call.
+** Except, there are some interfaces that are guaranteed to never
+** change the value of the error code.  The error-code preserving
+** interfaces include the following:
+**
+** 
      
+** 
    •  sqlite3_errcode()
+** 
    •  sqlite3_extended_errcode()
+** 
    •  sqlite3_errmsg()
+** 
    •  sqlite3_errmsg16()
+** 
    •  sqlite3_error_offset()
+** 
    
+**
 ** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language
-** text that describes the error, as either UTF-8 or UTF-16 respectively.
+** text that describes the error, as either UTF-8 or UTF-16 respectively,
+** or NULL if no error message is available.
+** (See how SQLite handles [invalid UTF] for exceptions to this rule.)
 ** ^(Memory to hold the error message string is managed internally.
 ** The application does not need to worry about freeing the result.
 ** However, the error string might be overwritten or deallocated by
 ** subsequent calls to other SQLite interface functions.)^
 **
-** ^The sqlite3_errstr() interface returns the English-language text
-** that describes the [result code], as UTF-8.
+** ^The sqlite3_errstr(E) interface returns the English-language text
+** that describes the [result code] E, as UTF-8, or NULL if E is not an
+** result code for which a text error message is available.
 ** ^(Memory to hold the error message string is managed internally
 ** and must not be freed by the application)^.
 **
+** ^If the most recent error references a specific token in the input
+** SQL, the sqlite3_error_offset() interface returns the byte offset
+** of the start of that token.  ^The byte offset returned by
+** sqlite3_error_offset() assumes that the input SQL is UTF8.
+** ^If the most recent error does not reference a specific token in the input
+** SQL, then the sqlite3_error_offset() function returns -1.
+**
 ** When the serialized [threading mode] is in use, it might be the
 ** case that a second error occurs on a separate thread in between
 ** the time of the first error and the call to these interfaces.
@@ -3396,11 +4333,12 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64(const char*, const cha
 ** was invoked incorrectly by the application.  In that case, the
 ** error code and message may or may not be set.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db);
-SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db);
-SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3*);
-SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3*);
-SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int);
+SQLITE_API int sqlite3_errcode(sqlite3 *db);
+SQLITE_API int sqlite3_extended_errcode(sqlite3 *db);
+SQLITE_API const char *sqlite3_errmsg(sqlite3*);
+SQLITE_API const void *sqlite3_errmsg16(sqlite3*);
+SQLITE_API const char *sqlite3_errstr(int);
+SQLITE_API int sqlite3_error_offset(sqlite3 *db);
 
 /*
 ** CAPI3REF: Prepared Statement Object
@@ -3410,7 +4348,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int);
 ** has been compiled into binary form and is ready to be evaluated.
 **
 ** Think of each SQL statement as a separate computer program.  The
-** original SQL text is source code.  A prepared statement object 
+** original SQL text is source code.  A prepared statement object
 ** is the compiled object code.  All SQL must be converted into a
 ** prepared statement before it can be run.
 **
@@ -3440,7 +4378,7 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 ** new limit for that construct.)^
 **
 ** ^If the new limit is a negative number, the limit is unchanged.
-** ^(For each limit category SQLITE_LIMIT_NAME there is a 
+** ^(For each limit category SQLITE_LIMIT_NAME there is a
 ** [limits | hard upper bound]
 ** set at compile-time by a C preprocessor macro called
 ** [limits | SQLITE_MAX_NAME].
@@ -3448,7 +4386,7 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 ** ^Attempts to increase a limit above its hard upper bound are
 ** silently truncated to the hard upper bound.
 **
-** ^Regardless of whether or not the limit was changed, the 
+** ^Regardless of whether or not the limit was changed, the
 ** [sqlite3_limit()] interface returns the prior value of the limit.
 ** ^Hence, to find the current value of a limit without changing it,
 ** simply invoke this interface with the third parameter set to -1.
@@ -3468,7 +4406,7 @@ typedef struct sqlite3_stmt sqlite3_stmt;
 **
 ** New run-time limit categories may be added in future releases.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal);
+SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal);
 
 /*
 ** CAPI3REF: Run-Time Limit Categories
@@ -3499,9 +4437,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal);
 **
 ** [[SQLITE_LIMIT_VDBE_OP]] ^(
    SQLITE_LIMIT_VDBE_OP
    
 ** 
    The maximum number of instructions in a virtual machine program
-** used to implement an SQL statement.  This limit is not currently
-** enforced, though that might be added in some future release of
-** SQLite.
    )^
+** used to implement an SQL statement.  If [sqlite3_prepare_v2()] or
+** the equivalent tries to allocate space for more than this many opcodes
+** in a single prepared statement, an SQLITE_NOMEM error is returned.)^
 **
 ** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
    SQLITE_LIMIT_FUNCTION_ARG
    
 ** 
    The maximum number of arguments on a function.
    )^
@@ -3539,32 +4477,87 @@ SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal);
 #define SQLITE_LIMIT_TRIGGER_DEPTH            10
 #define SQLITE_LIMIT_WORKER_THREADS           11
 
+/*
+** CAPI3REF: Prepare Flags
+**
+** These constants define various flags that can be passed into
+** "prepFlags" parameter of the [sqlite3_prepare_v3()] and
+** [sqlite3_prepare16_v3()] interfaces.
+**
+** New flags may be added in future releases of SQLite.
+**
+** 
    
+** [[SQLITE_PREPARE_PERSISTENT]] ^(
    SQLITE_PREPARE_PERSISTENT
    
+** 
    The SQLITE_PREPARE_PERSISTENT flag is a hint to the query planner
+** that the prepared statement will be retained for a long time and
+** probably reused many times.)^ ^Without this flag, [sqlite3_prepare_v3()]
+** and [sqlite3_prepare16_v3()] assume that the prepared statement will
+** be used just once or at most a few times and then destroyed using
+** [sqlite3_finalize()] relatively soon. The current implementation acts
+** on this hint by avoiding the use of [lookaside memory] so as not to
+** deplete the limited store of lookaside memory. Future versions of
+** SQLite may act on this hint differently.
+**
+** [[SQLITE_PREPARE_NORMALIZE]] 
    SQLITE_PREPARE_NORMALIZE
    
+** 
    The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used
+** to be required for any prepared statement that wanted to use the
+** [sqlite3_normalized_sql()] interface.  However, the
+** [sqlite3_normalized_sql()] interface is now available to all
+** prepared statements, regardless of whether or not they use this
+** flag.
+**
+** [[SQLITE_PREPARE_NO_VTAB]] 
    SQLITE_PREPARE_NO_VTAB
    
+** 
    The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler
+** to return an error (error code SQLITE_ERROR) if the statement uses
+** any virtual tables.
+** 
    
+*/
+#define SQLITE_PREPARE_PERSISTENT              0x01
+#define SQLITE_PREPARE_NORMALIZE               0x02
+#define SQLITE_PREPARE_NO_VTAB                 0x04
+
 /*
 ** CAPI3REF: Compiling An SQL Statement
 ** KEYWORDS: {SQL statement compiler}
 ** METHOD: sqlite3
 ** CONSTRUCTOR: sqlite3_stmt
 **
-** To execute an SQL query, it must first be compiled into a byte-code
-** program using one of these routines.
+** To execute an SQL statement, it must first be compiled into a byte-code
+** program using one of these routines.  Or, in other words, these routines
+** are constructors for the [prepared statement] object.
+**
+** The preferred routine to use is [sqlite3_prepare_v2()].  The
+** [sqlite3_prepare()] interface is legacy and should be avoided.
+** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used
+** for special purposes.
+**
+** The use of the UTF-8 interfaces is preferred, as SQLite currently
+** does all parsing using UTF-8.  The UTF-16 interfaces are provided
+** as a convenience.  The UTF-16 interfaces work by converting the
+** input text into UTF-8, then invoking the corresponding UTF-8 interface.
 **
 ** The first argument, "db", is a [database connection] obtained from a
 ** prior successful call to [sqlite3_open()], [sqlite3_open_v2()] or
 ** [sqlite3_open16()].  The database connection must not have been closed.
 **
 ** The second argument, "zSql", is the statement to be compiled, encoded
-** as either UTF-8 or UTF-16.  The sqlite3_prepare() and sqlite3_prepare_v2()
-** interfaces use UTF-8, and sqlite3_prepare16() and sqlite3_prepare16_v2()
-** use UTF-16.
+** as either UTF-8 or UTF-16.  The sqlite3_prepare(), sqlite3_prepare_v2(),
+** and sqlite3_prepare_v3()
+** interfaces use UTF-8, and sqlite3_prepare16(), sqlite3_prepare16_v2(),
+** and sqlite3_prepare16_v3() use UTF-16.
 **
 ** ^If the nByte argument is negative, then zSql is read up to the
-** first zero terminator. ^If nByte is positive, then it is the
-** number of bytes read from zSql.  ^If nByte is zero, then no prepared
+** first zero terminator. ^If nByte is positive, then it is the maximum
+** number of bytes read from zSql.  When nByte is positive, zSql is read
+** up to the first zero terminator or until the nByte bytes have been read,
+** whichever comes first.  ^If nByte is zero, then no prepared
 ** statement is generated.
 ** If the caller knows that the supplied string is nul-terminated, then
 ** there is a small performance advantage to passing an nByte parameter that
 ** is the number of bytes in the input string including
 ** the nul-terminator.
+** Note that nByte measure the length of the input in bytes, not
+** characters, even for the UTF-16 interfaces.
 **
 ** ^If pzTail is not NULL then *pzTail is made to point to the first byte
 ** past the end of the first SQL statement in zSql.  These routines only
@@ -3582,10 +4575,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal);
 ** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK];
 ** otherwise an [error code] is returned.
 **
-** The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are
-** recommended for all new programs. The two older interfaces are retained
-** for backwards compatibility, but their use is discouraged.
-** ^In the "v2" interfaces, the prepared statement
+** The sqlite3_prepare_v2(), sqlite3_prepare_v3(), sqlite3_prepare16_v2(),
+** and sqlite3_prepare16_v3() interfaces are recommended for all new programs.
+** The older interfaces (sqlite3_prepare() and sqlite3_prepare16())
+** are retained for backwards compatibility, but their use is discouraged.
+** ^In the "vX" interfaces, the prepared statement
 ** that is returned (the [sqlite3_stmt] object) contains a copy of the
 ** original SQL text. This causes the [sqlite3_step()] interface to
 ** behave differently in three ways:
@@ -3608,56 +4602,115 @@ SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3*, int id, int newVal);
 ** 
    • 
 **
 ** 
  • 
-** ^If the specific value bound to [parameter | host parameter] in the 
+** ^If the specific value bound to a [parameter | host parameter] in the
 ** WHERE clause might influence the choice of query plan for a statement,
-** then the statement will be automatically recompiled, as if there had been 
-** a schema change, on the first  [sqlite3_step()] call following any change
-** to the [sqlite3_bind_text | bindings] of that [parameter]. 
-** ^The specific value of WHERE-clause [parameter] might influence the 
+** then the statement will be automatically recompiled, as if there had been
+** a schema change, on the first [sqlite3_step()] call following any change
+** to the [sqlite3_bind_text | bindings] of that [parameter].
+** ^The specific value of a WHERE-clause [parameter] might influence the
 ** choice of query plan if the parameter is the left-hand side of a [LIKE]
 ** or [GLOB] operator or if the parameter is compared to an indexed column
-** and the [SQLITE_ENABLE_STAT3] compile-time option is enabled.
+** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled.
 ** 
    • 
 ** 
+**
+** 
    ^sqlite3_prepare_v3() differs from sqlite3_prepare_v2() only in having
+** the extra prepFlags parameter, which is a bit array consisting of zero or
+** more of the [SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_*] flags.  ^The
+** sqlite3_prepare_v2() interface works exactly the same as
+** sqlite3_prepare_v3() with a zero prepFlags parameter.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_prepare(
+SQLITE_API int sqlite3_prepare(
   sqlite3 *db,            /* Database handle */
   const char *zSql,       /* SQL statement, UTF-8 encoded */
   int nByte,              /* Maximum length of zSql in bytes. */
   sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
   const char **pzTail     /* OUT: Pointer to unused portion of zSql */
 );
-SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2(
+SQLITE_API int sqlite3_prepare_v2(
   sqlite3 *db,            /* Database handle */
   const char *zSql,       /* SQL statement, UTF-8 encoded */
   int nByte,              /* Maximum length of zSql in bytes. */
   sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
   const char **pzTail     /* OUT: Pointer to unused portion of zSql */
 );
-SQLITE_API int SQLITE_STDCALL sqlite3_prepare16(
+SQLITE_API int sqlite3_prepare_v3(
+  sqlite3 *db,            /* Database handle */
+  const char *zSql,       /* SQL statement, UTF-8 encoded */
+  int nByte,              /* Maximum length of zSql in bytes. */
+  unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */
+  sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
+  const char **pzTail     /* OUT: Pointer to unused portion of zSql */
+);
+SQLITE_API int sqlite3_prepare16(
   sqlite3 *db,            /* Database handle */
   const void *zSql,       /* SQL statement, UTF-16 encoded */
   int nByte,              /* Maximum length of zSql in bytes. */
   sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
   const void **pzTail     /* OUT: Pointer to unused portion of zSql */
 );
-SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2(
+SQLITE_API int sqlite3_prepare16_v2(
   sqlite3 *db,            /* Database handle */
   const void *zSql,       /* SQL statement, UTF-16 encoded */
   int nByte,              /* Maximum length of zSql in bytes. */
   sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
   const void **pzTail     /* OUT: Pointer to unused portion of zSql */
 );
+SQLITE_API int sqlite3_prepare16_v3(
+  sqlite3 *db,            /* Database handle */
+  const void *zSql,       /* SQL statement, UTF-16 encoded */
+  int nByte,              /* Maximum length of zSql in bytes. */
+  unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */
+  sqlite3_stmt **ppStmt,  /* OUT: Statement handle */
+  const void **pzTail     /* OUT: Pointer to unused portion of zSql */
+);
 
 /*
 ** CAPI3REF: Retrieving Statement SQL
 ** METHOD: sqlite3_stmt
 **
-** ^This interface can be used to retrieve a saved copy of the original
-** SQL text used to create a [prepared statement] if that statement was
-** compiled using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()].
+** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8
+** SQL text used to create [prepared statement] P if P was
+** created by [sqlite3_prepare_v2()], [sqlite3_prepare_v3()],
+** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()].
+** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8
+** string containing the SQL text of prepared statement P with
+** [bound parameters] expanded.
+** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8
+** string containing the normalized SQL text of prepared statement P.  The
+** semantics used to normalize a SQL statement are unspecified and subject
+** to change.  At a minimum, literal values will be replaced with suitable
+** placeholders.
+**
+** ^(For example, if a prepared statement is created using the SQL
+** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345
+** and parameter :xyz is unbound, then sqlite3_sql() will return
+** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql()
+** will return "SELECT 2345,NULL".)^
+**
+** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory
+** is available to hold the result, or if the result would exceed the
+** the maximum string length determined by the [SQLITE_LIMIT_LENGTH].
+**
+** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of
+** bound parameter expansions.  ^The [SQLITE_OMIT_TRACE] compile-time
+** option causes sqlite3_expanded_sql() to always return NULL.
+**
+** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P)
+** are managed by SQLite and are automatically freed when the prepared
+** statement is finalized.
+** ^The string returned by sqlite3_expanded_sql(P), on the other hand,
+** is obtained from [sqlite3_malloc()] and must be freed by the application
+** by passing it to [sqlite3_free()].
+**
+** ^The sqlite3_normalized_sql() interface is only available if
+** the [SQLITE_ENABLE_NORMALIZE] compile-time option is defined.
 */
-SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt);
+SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt);
+SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt);
+#ifdef SQLITE_ENABLE_NORMALIZE
+SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt);
+#endif
 
 /*
 ** CAPI3REF: Determine If An SQL Statement Writes The Database
@@ -3668,8 +4721,8 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt);
 ** the content of the database file.
 **
 ** Note that [application-defined SQL functions] or
-** [virtual tables] might change the database indirectly as a side effect.  
-** ^(For example, if an application defines a function "eval()" that 
+** [virtual tables] might change the database indirectly as a side effect.
+** ^(For example, if an application defines a function "eval()" that
 ** calls [sqlite3_exec()], then the following SQL statement would
 ** change the database file through side-effects:
 **
@@ -3683,34 +4736,98 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt);
 ** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK],
 ** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true,
 ** since the statements themselves do not actually modify the database but
-** rather they control the timing of when other statements modify the 
+** rather they control the timing of when other statements modify the
 ** database.  ^The [ATTACH] and [DETACH] statements also cause
 ** sqlite3_stmt_readonly() to return true since, while those statements
-** change the configuration of a database connection, they do not make 
+** change the configuration of a database connection, they do not make
 ** changes to the content of the database files on disk.
+** ^The sqlite3_stmt_readonly() interface returns true for [BEGIN] since
+** [BEGIN] merely sets internal flags, but the [BEGIN|BEGIN IMMEDIATE] and
+** [BEGIN|BEGIN EXCLUSIVE] commands do touch the database and so
+** sqlite3_stmt_readonly() returns false for those commands.
+**
+** ^This routine returns false if there is any possibility that the
+** statement might change the database file.  ^A false return does
+** not guarantee that the statement will change the database file.
+** ^For example, an UPDATE statement might have a WHERE clause that
+** makes it a no-op, but the sqlite3_stmt_readonly() result would still
+** be false.  ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a
+** read-only no-op if the table already exists, but
+** sqlite3_stmt_readonly() still returns false for such a statement.
+**
+** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN]
+** statement, then sqlite3_stmt_readonly(X) returns the same value as
+** if the EXPLAIN or EXPLAIN QUERY PLAN prefix were omitted.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
+SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt);
+
+/*
+** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement
+** METHOD: sqlite3_stmt
+**
+** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the
+** prepared statement S is an EXPLAIN statement, or 2 if the
+** statement S is an EXPLAIN QUERY PLAN.
+** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is
+** an ordinary statement or a NULL pointer.
+*/
+SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt);
+
+/*
+** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement
+** METHOD: sqlite3_stmt
+**
+** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN
+** setting for [prepared statement] S.  If E is zero, then S becomes
+** a normal prepared statement.  If E is 1, then S behaves as if
+** its SQL text began with "[EXPLAIN]".  If E is 2, then S behaves as if
+** its SQL text began with "[EXPLAIN QUERY PLAN]".
+**
+** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared.
+** SQLite tries to avoid a reprepare, but a reprepare might be necessary
+** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode.
+**
+** Because of the potential need to reprepare, a call to
+** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be
+** reprepared because it was created using [sqlite3_prepare()] instead of
+** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and
+** hence has no saved SQL text with which to reprepare.
+**
+** Changing the explain setting for a prepared statement does not change
+** the original SQL text for the statement.  Hence, if the SQL text originally
+** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0)
+** is called to convert the statement into an ordinary statement, the EXPLAIN
+** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S)
+** output, even though the statement now acts like a normal SQL statement.
+**
+** This routine returns SQLITE_OK if the explain mode is successfully
+** changed, or an error code if the explain mode could not be changed.
+** The explain mode cannot be changed while a statement is active.
+** Hence, it is good practice to call [sqlite3_reset(S)]
+** immediately prior to calling sqlite3_stmt_explain(S,E).
+*/
+SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode);
 
 /*
 ** CAPI3REF: Determine If A Prepared Statement Has Been Reset
 ** METHOD: sqlite3_stmt
 **
 ** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the
-** [prepared statement] S has been stepped at least once using 
+** [prepared statement] S has been stepped at least once using
 ** [sqlite3_step(S)] but has neither run to completion (returned
 ** [SQLITE_DONE] from [sqlite3_step(S)]) nor
 ** been reset using [sqlite3_reset(S)].  ^The sqlite3_stmt_busy(S)
-** interface returns false if S is a NULL pointer.  If S is not a 
+** interface returns false if S is a NULL pointer.  If S is not a
 ** NULL pointer and is not a pointer to a valid [prepared statement]
 ** object, then the behavior is undefined and probably undesirable.
 **
 ** This interface can be used in combination [sqlite3_next_stmt()]
-** to locate all prepared statements associated with a database 
+** to locate all prepared statements associated with a database
 ** connection that are in need of being reset.  This can be used,
-** for example, in diagnostic routines to search for prepared 
+** for example, in diagnostic routines to search for prepared
 ** statements that are holding a transaction open.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*);
+SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Dynamically Typed Value Object
@@ -3726,7 +4843,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*);
 ** will accept either a protected or an unprotected sqlite3_value.
 ** Every interface that accepts sqlite3_value arguments specifies
 ** whether or not it requires a protected sqlite3_value.  The
-** [sqlite3_value_dup()] interface can be used to construct a new 
+** [sqlite3_value_dup()] interface can be used to construct a new
 ** protected sqlite3_value from an unprotected sqlite3_value.
 **
 ** The terms "protected" and "unprotected" refer to whether or not
@@ -3734,7 +4851,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*);
 ** sqlite3_value object but no mutex is held for an unprotected
 ** sqlite3_value object.  If SQLite is compiled to be single-threaded
 ** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0)
-** or if SQLite is run in one of reduced mutex modes 
+** or if SQLite is run in one of reduced mutex modes
 ** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD]
 ** then there is no distinction between protected and unprotected
 ** sqlite3_value objects and they can be used interchangeably.  However,
@@ -3744,14 +4861,17 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt*);
 **
 ** ^The sqlite3_value objects that are passed as parameters into the
 ** implementation of [application-defined SQL functions] are protected.
+** ^The sqlite3_value objects returned by [sqlite3_vtab_rhs_value()]
+** are protected.
 ** ^The sqlite3_value object returned by
 ** [sqlite3_column_value()] is unprotected.
-** Unprotected sqlite3_value objects may only be used with
-** [sqlite3_result_value()] and [sqlite3_bind_value()].
+** Unprotected sqlite3_value objects may only be used as arguments
+** to [sqlite3_result_value()], [sqlite3_bind_value()], and
+** [sqlite3_value_dup()].
 ** The [sqlite3_value_blob | sqlite3_value_type()] family of
 ** interfaces require protected sqlite3_value objects.
 */
-typedef struct Mem sqlite3_value;
+typedef struct sqlite3_value sqlite3_value;
 
 /*
 ** CAPI3REF: SQL Function Context Object
@@ -3802,12 +4922,30 @@ typedef struct sqlite3_context sqlite3_context;
 ** [sqlite3_bind_parameter_index()] API if desired.  ^The index
 ** for "?NNN" parameters is the value of NNN.
 ** ^The NNN value must be between 1 and the [sqlite3_limit()]
-** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 999).
+** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 32766).
 **
 ** ^The third argument is the value to bind to the parameter.
 ** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16()
 ** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter
 ** is ignored and the end result is the same as sqlite3_bind_null().
+** ^If the third parameter to sqlite3_bind_text() is not NULL, then
+** it should be a pointer to well-formed UTF8 text.
+** ^If the third parameter to sqlite3_bind_text16() is not NULL, then
+** it should be a pointer to well-formed UTF16 text.
+** ^If the third parameter to sqlite3_bind_text64() is not NULL, then
+** it should be a pointer to a well-formed unicode string that is
+** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16
+** otherwise.
+**
+** [[byte-order determination rules]] ^The byte-order of
+** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF)
+** found in first character, which is removed, or in the absence of a BOM
+** the byte order is the native byte order of the host
+** machine for sqlite3_bind_text16() or the byte order specified in
+** the 6th parameter for sqlite3_bind_text64().)^
+** ^If UTF16 input text contains invalid unicode
+** characters, then SQLite might change those invalid characters
+** into the unicode replacement character: U+FFFD.
 **
 ** ^(In those routines that have a fourth argument, its value is the
 ** number of bytes in the parameter.  To be clear: the value is the
@@ -3821,21 +4959,27 @@ typedef struct sqlite3_context sqlite3_context;
 ** or sqlite3_bind_text16() or sqlite3_bind_text64() then
 ** that parameter must be the byte offset
 ** where the NUL terminator would occur assuming the string were NUL
-** terminated.  If any NUL characters occur at byte offsets less than 
+** terminated.  If any NUL characters occurs at byte offsets less than
 ** the value of the fourth parameter then the resulting string value will
 ** contain embedded NULs.  The result of expressions involving strings
 ** with embedded NULs is undefined.
 **
-** ^The fifth argument to the BLOB and string binding interfaces
-** is a destructor used to dispose of the BLOB or
-** string after SQLite has finished with it.  ^The destructor is called
-** to dispose of the BLOB or string even if the call to bind API fails.
-** ^If the fifth argument is
-** the special value [SQLITE_STATIC], then SQLite assumes that the
-** information is in static, unmanaged space and does not need to be freed.
-** ^If the fifth argument has the value [SQLITE_TRANSIENT], then
-** SQLite makes its own private copy of the data immediately, before
-** the sqlite3_bind_*() routine returns.
+** ^The fifth argument to the BLOB and string binding interfaces controls
+** or indicates the lifetime of the object referenced by the third parameter.
+** These three options exist:
+** ^ (1) A destructor to dispose of the BLOB or string after SQLite has finished
+** with it may be passed. ^It is called to dispose of the BLOB or string even
+** if the call to the bind API fails, except the destructor is not called if
+** the third parameter is a NULL pointer or the fourth parameter is negative.
+** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that
+** the application remains responsible for disposing of the object. ^In this
+** case, the object and the provided pointer to it must remain valid until
+** either the prepared statement is finalized or the same SQL parameter is
+** bound to something else, whichever occurs sooner.
+** ^ (3) The constant, [SQLITE_TRANSIENT], may be passed to indicate that the
+** object is to be copied prior to the return from sqlite3_bind_*(). ^The
+** object and pointer to it must remain valid until then. ^SQLite will then
+** manage the lifetime of its private copy.
 **
 ** ^The sixth argument to sqlite3_bind_text64() must be one of
 ** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]
@@ -3853,6 +4997,15 @@ typedef struct sqlite3_context sqlite3_context;
 ** [sqlite3_blob_open | incremental BLOB I/O] routines.
 ** ^A negative value for the zeroblob results in a zero-length BLOB.
 **
+** ^The sqlite3_bind_pointer(S,I,P,T,D) routine causes the I-th parameter in
+** [prepared statement] S to have an SQL value of NULL, but to also be
+** associated with the pointer P of type T.  ^D is either a NULL pointer or
+** a pointer to a destructor function for P. ^SQLite will invoke the
+** destructor D with a single argument of P when it is finished using
+** P.  The T parameter should be a static string, preferably a string
+** literal. The sqlite3_bind_pointer() routine is part of the
+** [pointer passing interface] added for SQLite 3.20.0.
+**
 ** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer
 ** for the [prepared statement] or with a prepared statement for which
 ** [sqlite3_step()] has been called more recently than [sqlite3_reset()],
@@ -3874,20 +5027,21 @@ typedef struct sqlite3_context sqlite3_context;
 ** See also: [sqlite3_bind_parameter_count()],
 ** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()].
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64,
+SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*));
+SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64,
                         void(*)(void*));
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt*, int, double);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt*, int, int);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt*, int);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*));
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*));
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64,
+SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double);
+SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int);
+SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64);
+SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int);
+SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*));
+SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*));
+SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64,
                          void(*)(void*), unsigned char encoding);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64);
+SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*);
+SQLITE_API int sqlite3_bind_pointer(sqlite3_stmt*, int, void*, const char*,void(*)(void*));
+SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n);
+SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64);
 
 /*
 ** CAPI3REF: Number Of SQL Parameters
@@ -3908,7 +5062,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite
 ** [sqlite3_bind_parameter_name()], and
 ** [sqlite3_bind_parameter_index()].
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt*);
+SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Name Of A Host Parameter
@@ -3929,14 +5083,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt*);
 ** ^If the value N is out of range or if the N-th parameter is
 ** nameless, then NULL is returned.  ^The returned string is
 ** always in UTF-8 encoding even if the named parameter was
-** originally specified as UTF-16 in [sqlite3_prepare16()] or
-** [sqlite3_prepare16_v2()].
+** originally specified as UTF-16 in [sqlite3_prepare16()],
+** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()].
 **
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_count()], and
 ** [sqlite3_bind_parameter_index()].
 */
-SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*, int);
+SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int);
 
 /*
 ** CAPI3REF: Index Of A Parameter With A Given Name
@@ -3947,13 +5101,14 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt*,
 ** parameter to [sqlite3_bind_blob|sqlite3_bind()].  ^A zero
 ** is returned if no matching parameter is found.  ^The parameter
 ** name must be given in UTF-8 even if the original statement
-** was prepared from UTF-16 text using [sqlite3_prepare16_v2()].
+** was prepared from UTF-16 text using [sqlite3_prepare16_v2()] or
+** [sqlite3_prepare16_v3()].
 **
 ** See also: [sqlite3_bind_blob|sqlite3_bind()],
 ** [sqlite3_bind_parameter_count()], and
 ** [sqlite3_bind_parameter_name()].
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName);
+SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName);
 
 /*
 ** CAPI3REF: Reset All Bindings On A Prepared Statement
@@ -3963,19 +5118,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt*, const
 ** the [sqlite3_bind_blob | bindings] on a [prepared statement].
 ** ^Use this routine to reset all host parameters to NULL.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt*);
+SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Number Of Columns In A Result Set
 ** METHOD: sqlite3_stmt
 **
 ** ^Return the number of columns in the result set returned by the
-** [prepared statement]. ^This routine returns 0 if pStmt is an SQL
-** statement that does not return data (for example an [UPDATE]).
+** [prepared statement]. ^If this routine returns 0, that means the
+** [prepared statement] returns no data (for example an [UPDATE]).
+** ^However, just because this routine returns a positive number does not
+** mean that one or more rows of data will be returned.  ^A SELECT statement
+** will always have a positive sqlite3_column_count() but depending on the
+** WHERE clause constraints and the table content, it might return no rows.
 **
 ** See also: [sqlite3_data_count()]
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt);
+SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt);
 
 /*
 ** CAPI3REF: Column Names In A Result Set
@@ -4004,8 +5163,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt);
 ** then the name of the column is unspecified and may change from
 ** one release of SQLite to the next.
 */
-SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt*, int N);
-SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt*, int N);
+SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N);
+SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N);
 
 /*
 ** CAPI3REF: Source Of Data In A Query Result
@@ -4034,7 +5193,7 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt*, int N
 **
 ** ^If the Nth column returned by the statement is an expression or
 ** subquery and is not a column value, then all of these functions return
-** NULL.  ^These routine might also return NULL if a memory allocation error
+** NULL.  ^These routines might also return NULL if a memory allocation error
 ** occurs.  ^Otherwise, they return the name of the attached database, table,
 ** or column that query result column was extracted from.
 **
@@ -4044,21 +5203,17 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt*, int N
 ** ^These APIs are only available if the library was compiled with the
 ** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol.
 **
-** If two or more threads call one or more of these routines against the same
-** prepared statement and column at the same time then the results are
-** undefined.
-**
 ** If two or more threads call one or more
 ** [sqlite3_column_database_name | column metadata interfaces]
 ** for the same [prepared statement] and result column
 ** at the same time then the results are undefined.
 */
-SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt*,int);
-SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt*,int);
-SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt*,int);
-SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt*,int);
-SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt*,int);
-SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt*,int);
+SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int);
+SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int);
+SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int);
+SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int);
+SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int);
+SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int);
 
 /*
 ** CAPI3REF: Declared Datatype Of A Query Result
@@ -4090,23 +5245,25 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt*
 ** is associated with individual values, not with the containers
 ** used to hold those values.
 */
-SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt*,int);
-SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt*,int);
+SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int);
+SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int);
 
 /*
 ** CAPI3REF: Evaluate An SQL Statement
 ** METHOD: sqlite3_stmt
 **
-** After a [prepared statement] has been prepared using either
-** [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or one of the legacy
+** After a [prepared statement] has been prepared using any of
+** [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], [sqlite3_prepare16_v2()],
+** or [sqlite3_prepare16_v3()] or one of the legacy
 ** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function
 ** must be called one or more times to evaluate the statement.
 **
 ** The details of the behavior of the sqlite3_step() interface depend
-** on whether the statement was prepared using the newer "v2" interface
-** [sqlite3_prepare_v2()] and [sqlite3_prepare16_v2()] or the older legacy
-** interface [sqlite3_prepare()] and [sqlite3_prepare16()].  The use of the
-** new "v2" interface is recommended for new applications but the legacy
+** on whether the statement was prepared using the newer "vX" interfaces
+** [sqlite3_prepare_v3()], [sqlite3_prepare_v2()], [sqlite3_prepare16_v3()],
+** [sqlite3_prepare16_v2()] or the older legacy
+** interfaces [sqlite3_prepare()] and [sqlite3_prepare16()].  The use of the
+** new "vX" interface is recommended for new applications but the legacy
 ** interface will continue to be supported.
 **
 ** ^In the legacy interface, the return value will be either [SQLITE_BUSY],
@@ -4150,9 +5307,10 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt*,in
 ** For all versions of SQLite up to and including 3.6.23.1, a call to
 ** [sqlite3_reset()] was required after sqlite3_step() returned anything
 ** other than [SQLITE_ROW] before any subsequent invocation of
-** sqlite3_step().  Failure to reset the prepared statement using 
+** sqlite3_step().  Failure to reset the prepared statement using
 ** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from
-** sqlite3_step().  But after version 3.6.23.1, sqlite3_step() began
+** sqlite3_step().  But after [version 3.6.23.1] ([dateof:3.6.23.1],
+** sqlite3_step() began
 ** calling [sqlite3_reset()] automatically in this circumstance rather
 ** than returning [SQLITE_MISUSE].  This is not considered a compatibility
 ** break because any application that ever receives an SQLITE_MISUSE error
@@ -4166,12 +5324,13 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt*,in
 ** specific [error codes] that better describes the error.
 ** We admit that this is a goofy design.  The problem has been fixed
 ** with the "v2" interface.  If you prepare all of your SQL statements
-** using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead
+** using [sqlite3_prepare_v3()] or [sqlite3_prepare_v2()]
+** or [sqlite3_prepare16_v2()] or [sqlite3_prepare16_v3()] instead
 ** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces,
 ** then the more specific [error codes] are returned directly
-** by sqlite3_step().  The use of the "v2" interface is recommended.
+** by sqlite3_step().  The use of the "vX" interfaces is recommended.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt*);
+SQLITE_API int sqlite3_step(sqlite3_stmt*);
 
 /*
 ** CAPI3REF: Number of columns in a result set
@@ -4180,7 +5339,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt*);
 ** ^The sqlite3_data_count(P) interface returns the number of columns in the
 ** current row of the result set of [prepared statement] P.
 ** ^If prepared statement P does not have results ready to return
-** (via calls to the [sqlite3_column_int | sqlite3_column_*()] of
+** (via calls to the [sqlite3_column_int | sqlite3_column()] family of
 ** interfaces) then sqlite3_data_count(P) returns 0.
 ** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer.
 ** ^The sqlite3_data_count(P) routine returns 0 if the previous call to
@@ -4192,7 +5351,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt*);
 **
 ** See also: [sqlite3_column_count()]
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
+SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt);
 
 /*
 ** CAPI3REF: Fundamental Datatypes
@@ -4231,6 +5390,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** KEYWORDS: {column access functions}
 ** METHOD: sqlite3_stmt
 **
+** Summary:
+** 
    
+** 
    sqlite3_column_blob→BLOB result
+** 
    sqlite3_column_double→REAL result
+** 
    sqlite3_column_int→32-bit INTEGER result
+** 
    sqlite3_column_int64→64-bit INTEGER result
+** 
    sqlite3_column_text→UTF-8 TEXT result
+** 
    sqlite3_column_text16→UTF-16 TEXT result
+** 
    sqlite3_column_value→The result as an
+** [sqlite3_value|unprotected sqlite3_value] object.
+** 
       
+** 
    sqlite3_column_bytes→Size of a BLOB
+** or a UTF-8 TEXT result in bytes
+** 
    sqlite3_column_bytes16  
+** →  Size of UTF-16
+** TEXT in bytes
+** 
    sqlite3_column_type→Default
+** datatype of the result
+** 
    
+**
+** Details:
+**
 ** ^These routines return information about a single column of the current
 ** result row of a query.  ^In every case the first argument is a pointer
 ** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*]
@@ -4252,16 +5433,29 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** are called from a different thread while any of these routines
 ** are pending, then the results are undefined.
 **
+** The first six interfaces (_blob, _double, _int, _int64, _text, and _text16)
+** each return the value of a result column in a specific data format.  If
+** the result column is not initially in the requested format (for example,
+** if the query returns an integer but the sqlite3_column_text() interface
+** is used to extract the value) then an automatic type conversion is performed.
+**
 ** ^The sqlite3_column_type() routine returns the
 ** [SQLITE_INTEGER | datatype code] for the initial data type
 ** of the result column.  ^The returned value is one of [SQLITE_INTEGER],
-** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].  The value
-** returned by sqlite3_column_type() is only meaningful if no type
-** conversions have occurred as described below.  After a type conversion,
-** the value returned by sqlite3_column_type() is undefined.  Future
+** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].
+** The return value of sqlite3_column_type() can be used to decide which
+** of the first six interface should be used to extract the column value.
+** The value returned by sqlite3_column_type() is only meaningful if no
+** automatic type conversions have occurred for the value in question.
+** After a type conversion, the result of calling sqlite3_column_type()
+** is undefined, though harmless.  Future
 ** versions of SQLite may change the behavior of sqlite3_column_type()
 ** following a type conversion.
 **
+** If the result is a BLOB or a TEXT string, then the sqlite3_column_bytes()
+** or sqlite3_column_bytes16() interfaces can be used to determine the size
+** of that BLOB or string.
+**
 ** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes()
 ** routine returns the number of bytes in that BLOB or string.
 ** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts
@@ -4280,7 +5474,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** the number of bytes in that string.
 ** ^If the result is NULL, then sqlite3_column_bytes16() returns zero.
 **
-** ^The values returned by [sqlite3_column_bytes()] and 
+** ^The values returned by [sqlite3_column_bytes()] and
 ** [sqlite3_column_bytes16()] do not include the zero terminators at the end
 ** of the string.  ^For clarity: the values returned by
 ** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of
@@ -4290,6 +5484,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** even empty strings, are always zero-terminated.  ^The return
 ** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer.
 **
+** ^Strings returned by sqlite3_column_text16() always have the endianness
+** which is native to the platform, regardless of the text encoding set
+** for the database.
+**
 ** Warning: ^The object returned by [sqlite3_column_value()] is an
 ** [unprotected sqlite3_value] object.  In a multithreaded environment,
 ** an unprotected sqlite3_value object may only be used safely with
@@ -4298,9 +5496,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 ** [sqlite3_column_value()] is used in any other way, including calls
 ** to routines like [sqlite3_value_int()], [sqlite3_value_text()],
 ** or [sqlite3_value_bytes()], the behavior is not threadsafe.
+** Hence, the sqlite3_column_value() interface
+** is normally only useful within the implementation of
+** [application-defined SQL functions] or [virtual tables], not within
+** top-level application code.
 **
-** These routines attempt to convert the value where appropriate.  ^For
-** example, if the internal representation is FLOAT and a text result
+** These routines may attempt to convert the datatype of the result.
+** ^For example, if the internal representation is FLOAT and a text result
 ** is requested, [sqlite3_snprintf()] is used internally to perform the
 ** conversion automatically.  ^(The following table details the conversions
 ** that are applied:
@@ -4324,7 +5526,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt);
 **   TEXT       BLOB     No change
 **   BLOB     INTEGER    [CAST] to INTEGER
 **   BLOB      FLOAT     [CAST] to REAL
-**   BLOB       TEXT     Add a zero terminator if needed
+**   BLOB       TEXT     [CAST] to TEXT, ensure zero terminator
 ** 
 **
    )^ ** @@ -4372,26 +5574,40 @@ SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt); ** ^The pointers returned are valid until a type conversion occurs as ** described above, or until [sqlite3_step()] or [sqlite3_reset()] or ** [sqlite3_finalize()] is called. ^The memory space used to hold strings -** and BLOBs is freed automatically. Do not pass the pointers returned +** and BLOBs is freed automatically. Do not pass the pointers returned ** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into ** [sqlite3_free()]. ** -** ^(If a memory allocation error occurs during the evaluation of any -** of these routines, a default value is returned. The default value -** is either the integer 0, the floating point number 0.0, or a NULL -** pointer. Subsequent calls to [sqlite3_errcode()] will return -** [SQLITE_NOMEM].)^ +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
      +**
    • sqlite3_column_blob() +**
    • sqlite3_column_text() +**
    • sqlite3_column_text16() +**
    • sqlite3_column_bytes() +**
    • sqlite3_column_bytes16() +**
    +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt*, int iCol); -SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt*, int iCol); -SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt*, int iCol); -SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt*, int iCol); -SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt*, int iCol); -SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt*, int iCol); -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt*, int iCol); -SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt*, int iCol); -SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); /* ** CAPI3REF: Destroy A Prepared Statement Object @@ -4419,7 +5635,7 @@ SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt*, int ** statement after it has been finalized can result in undefined and ** undesirable behavior such as segfaults and heap corruption. */ -SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt); +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object @@ -4434,34 +5650,47 @@ SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt); ** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S ** back to the beginning of its program. ** -** ^If the most recent call to [sqlite3_step(S)] for the -** [prepared statement] S returned [SQLITE_ROW] or [SQLITE_DONE], -** or if [sqlite3_step(S)] has never before been called on S, -** then [sqlite3_reset(S)] returns [SQLITE_OK]. +** ^The return code from [sqlite3_reset(S)] indicates whether or not +** the previous evaluation of prepared statement S completed successfully. +** ^If [sqlite3_step(S)] has never before been called on S or if +** [sqlite3_step(S)] has not been called since the previous call +** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return +** [SQLITE_OK]. ** ** ^If the most recent call to [sqlite3_step(S)] for the ** [prepared statement] S indicated an error, then ** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^The [sqlite3_reset(S)] interface might also return an [error code] +** if there were no prior errors but the process of resetting +** the prepared statement caused a new error. ^For example, if an +** [INSERT] statement with a [RETURNING] clause is only stepped one time, +** that one call to [sqlite3_step(S)] might return SQLITE_ROW but +** the overall statement might still fail and the [sqlite3_reset(S)] call +** might return SQLITE_BUSY if locking constraints prevent the +** database change from committing. Therefore, it is important that +** applications check the return code from [sqlite3_reset(S)] even if +** no prior call to [sqlite3_step(S)] indicated a problem. ** ** ^The [sqlite3_reset(S)] interface does not change the values ** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ -SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Create Or Redefine SQL Functions ** KEYWORDS: {function creation routines} -** KEYWORDS: {application-defined SQL function} -** KEYWORDS: {application-defined SQL functions} ** METHOD: sqlite3 ** ** ^These functions (collectively known as "function creation routines") ** are used to add SQL functions or aggregates or to redefine the behavior -** of existing SQL functions or aggregates. The only differences between -** these routines are the text encoding expected for -** the second parameter (the name of the function being created) -** and the presence or absence of a destructor callback for -** the application data pointer. +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. ** ** ^The first parameter is the [database connection] to which the SQL ** function is to be added. ^If an application uses more than one database @@ -4471,7 +5700,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); ** ^The second parameter is the name of the SQL function to be created or ** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 ** representation, exclusive of the zero-terminator. ^Note that the name -** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. ** ^Any attempt to create a function with a longer name ** will result in [SQLITE_MISUSE] being returned. ** @@ -4486,7 +5715,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); ** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for ** its parameters. The application should set this parameter to -** [SQLITE_UTF16LE] if the function implementation invokes +** [SQLITE_UTF16LE] if the function implementation invokes ** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the ** implementation invokes [sqlite3_value_text16be()] on an input, or ** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] @@ -4504,10 +5733,26 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); ** perform additional optimizations on deterministic functions, so use ** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs, TRIGGERs, CHECK constraints, generated column expressions, +** index expressions, or the WHERE clause of partial indexes. +** +** For best security, the [SQLITE_DIRECTONLY] flag is recommended for +** all application-defined SQL functions that do not need to be +** used inside of triggers, view, CHECK constraints, or other elements of +** the database schema. This flags is especially recommended for SQL +** functions that have side effects or reveal internal application state. +** Without this flag, an attacker might be able to modify the schema of +** a database file to include invocations of the function with parameters +** chosen by the attacker, which the application will then execute when +** the database file is opened and read. +** ** ^(The fifth parameter is an arbitrary pointer. The implementation of the ** function can gain access to this pointer using [sqlite3_user_data()].)^ ** -** ^The sixth, seventh and eighth parameters, xFunc, xStep and xFinal, are +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are ** pointers to C-language functions that implement the SQL function or ** aggregate. ^A scalar SQL function requires an implementation of the xFunc ** callback only; NULL pointers must be passed as the xStep and xFinal @@ -4516,15 +5761,24 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); ** SQL function or aggregate, pass NULL pointers for all three function ** callbacks. ** -** ^(If the ninth parameter to sqlite3_create_function_v2() is not NULL, -** then it is destructor for the application data pointer. -** The destructor is invoked when the function is deleted, either by being -** overloaded or when the database connection closes.)^ -** ^The destructor is also invoked if the call to -** sqlite3_create_function_v2() fails. -** ^When the destructor callback of the tenth parameter is invoked, it -** is passed a single argument which is a copy of the application data -** pointer which was the fifth parameter to sqlite3_create_function_v2(). +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). ** ** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of @@ -4534,7 +5788,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); ** nArg parameter is a better match than a function implementation with ** a negative nArg. ^A function where the preferred text encoding ** matches the database encoding is a better -** match than a function where the encoding is different. +** match than a function where the encoding is different. ** ^A function where the encoding difference is between UTF16le and UTF16be ** is a closer match than a function where the encoding difference is ** between UTF8 and UTF16. @@ -4546,7 +5800,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt); ** close the database connection nor finalize or reset the prepared ** statement in which the function is running. */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_function( +SQLITE_API int sqlite3_create_function( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4556,7 +5810,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( +SQLITE_API int sqlite3_create_function16( sqlite3 *db, const void *zFunctionName, int nArg, @@ -4566,7 +5820,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( +SQLITE_API int sqlite3_create_function_v2( sqlite3 *db, const char *zFunctionName, int nArg, @@ -4577,6 +5831,18 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( void (*xFinal)(sqlite3_context*), void(*xDestroy)(void*) ); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -4594,30 +5860,126 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( /* ** CAPI3REF: Function Flags ** -** These constants may be ORed together with the +** These constants may be ORed together with the ** [SQLITE_UTF8 | preferred text encoding] as the fourth argument ** to [sqlite3_create_function()], [sqlite3_create_function16()], or ** [sqlite3_create_function_v2()]. +** +**
    +** [[SQLITE_DETERMINISTIC]]
    SQLITE_DETERMINISTIC
    +** The SQLITE_DETERMINISTIC flag means that the new function always gives +** the same output when the input parameters are the same. +** The [abs|abs() function] is deterministic, for example, but +** [randomblob|randomblob()] is not. Functions must +** be deterministic in order to be used in certain contexts such as +** with the WHERE clause of [partial indexes] or in [generated columns]. +** SQLite might also optimize deterministic functions by factoring them +** out of inner loops. +**
    +** +** [[SQLITE_DIRECTONLY]]
    SQLITE_DIRECTONLY
    +** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], or [generated columns]. +**

    +** The SQLITE_DIRECTONLY flag is recommended for any +** [application-defined SQL function] +** that has side-effects or that could potentially leak sensitive information. +** This will prevent attacks in which an application is tricked +** into using a database file that has had its schema surreptitiously +** modified to invoke the application-defined function in ways that are +** harmful. +**

    +** Some people say it is good practice to set SQLITE_DIRECTONLY on all +** [application-defined SQL functions], regardless of whether or not they +** are security sensitive, as doing so prevents those functions from being used +** inside of the database schema, and thus ensures that the database +** can be inspected and modified using generic tools (such as the [CLI]) +** that do not have access to the application-defined functions. +**

    +** +** [[SQLITE_INNOCUOUS]]
    SQLITE_INNOCUOUS
    +** The SQLITE_INNOCUOUS flag means that the function is unlikely +** to cause problems even if misused. An innocuous function should have +** no side effects and should not depend on any values other than its +** input parameters. The [abs|abs() function] is an example of an +** innocuous function. +** The [load_extension() SQL function] is not innocuous because of its +** side effects. +**

    SQLITE_INNOCUOUS is similar to SQLITE_DETERMINISTIC, but is not +** exactly the same. The [random|random() function] is an example of a +** function that is innocuous but not deterministic. +**

    Some heightened security settings +** ([SQLITE_DBCONFIG_TRUSTED_SCHEMA] and [PRAGMA trusted_schema=OFF]) +** disable the use of SQL functions inside views and triggers and in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], and [generated columns] unless +** the function is tagged with SQLITE_INNOCUOUS. Most built-in functions +** are innocuous. Developers are advised to avoid using the +** SQLITE_INNOCUOUS flag for application-defined functions unless the +** function has been carefully audited and found to be free of potentially +** security-adverse side-effects and information-leaks. +**

    +** +** [[SQLITE_SUBTYPE]]
    SQLITE_SUBTYPE
    +** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** This flag instructs SQLite to omit some corner-case optimizations that +** might disrupt the operation of the [sqlite3_value_subtype()] function, +** causing it to return zero rather than the correct subtype(). +** All SQL functions that invoke [sqlite3_value_subtype()] should have this +** property. If the SQLITE_SUBTYPE property is omitted, then the return +** value from [sqlite3_value_subtype()] might sometimes be zero even though +** a non-zero subtype was specified by the function argument expression. +** +** [[SQLITE_RESULT_SUBTYPE]]
    SQLITE_RESULT_SUBTYPE
    +** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_result_subtype()] to cause a sub-type to be associated with its +** result. +** Every function that invokes [sqlite3_result_subtype()] should have this +** property. If it does not, then the call to [sqlite3_result_subtype()] +** might become a no-op if the function is used as term in an +** [expression index]. On the other hand, SQL functions that never invoke +** [sqlite3_result_subtype()] should avoid setting this property, as the +** purpose of this property is to disable certain optimizations that are +** incompatible with subtypes. +** +** [[SQLITE_SELFORDER1]]
    SQLITE_SELFORDER1
    +** The SQLITE_SELFORDER1 flag indicates that the function is an aggregate +** that internally orders the values provided to the first argument. The +** ordered-set aggregate SQL notation with a single ORDER BY term can be +** used to invoke this function. If the ordered-set aggregate notation is +** used on a function that lacks this flag, then an error is raised. Note +** that the ordered-set aggregate syntax is only available if SQLite is +** built using the -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES compile-time option. +**
    +**
    */ -#define SQLITE_DETERMINISTIC 0x800 +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 +#define SQLITE_INNOCUOUS 0x000200000 +#define SQLITE_RESULT_SUBTYPE 0x001000000 +#define SQLITE_SELFORDER1 0x002000000 /* ** CAPI3REF: Deprecated Functions ** DEPRECATED ** ** These functions are [deprecated]. In order to maintain -** backwards compatibility with older code, these functions continue +** backwards compatibility with older code, these functions continue ** to be supported. However, new applications should avoid ** the use of these functions. To encourage programmers to avoid ** these functions, we will not explain what they do. */ #ifndef SQLITE_OMIT_DEPRECATED -SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context*); -SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_expired(sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); -SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_global_recover(void); -SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_thread_cleanup(void); -SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), +SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), void*,sqlite3_int64); #endif @@ -4625,21 +5987,45 @@ SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(voi ** CAPI3REF: Obtaining SQL Values ** METHOD: sqlite3_value ** -** The C-language implementation of SQL functions and aggregates uses -** this set of interface routines to access the parameter values on -** the function or aggregate. +** Summary: +**
    +**
    sqlite3_value_blob→BLOB value +**
    sqlite3_value_double→REAL value +**
    sqlite3_value_int→32-bit INTEGER value +**
    sqlite3_value_int64→64-bit INTEGER value +**
    sqlite3_value_pointer→Pointer value +**
    sqlite3_value_text→UTF-8 TEXT value +**
    sqlite3_value_text16→UTF-16 TEXT value in +** the native byteorder +**
    sqlite3_value_text16be→UTF-16be TEXT value +**
    sqlite3_value_text16le→UTF-16le TEXT value +**
        +**
    sqlite3_value_bytes→Size of a BLOB +** or a UTF-8 TEXT in bytes +**
    sqlite3_value_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
    sqlite3_value_type→Default +** datatype of the value +**
    sqlite3_value_numeric_type   +** →  Best numeric datatype of the value +**
    sqlite3_value_nochange   +** →  True if the column is unchanged in an UPDATE +** against a virtual table. +**
    sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] +**
    ** -** The xFunc (for scalar functions) or xStep (for aggregates) parameters -** to [sqlite3_create_function()] and [sqlite3_create_function16()] -** define callbacks that implement the SQL functions and aggregates. -** The 3rd parameter to these callbacks is an array of pointers to -** [protected sqlite3_value] objects. There is one [sqlite3_value] object for -** each parameter to the SQL function. These routines are used to -** extract values from the [sqlite3_value] objects. +** Details: +** +** These routines extract type, size, and content information from +** [protected sqlite3_value] objects. Protected sqlite3_value objects +** are used to pass parameter information into the functions that +** implement [application-defined SQL functions] and [virtual tables]. ** ** These routines work only with [protected sqlite3_value] objects. ** Any attempt to use these routines on an [unprotected sqlite3_value] -** object results in undefined behavior. +** is not threadsafe. ** ** ^These routines work just like the corresponding [column access functions] ** except that these routines take a single [protected sqlite3_value] object @@ -4650,6 +6036,24 @@ SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(voi ** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces ** extract UTF-16 strings as big-endian and little-endian respectively. ** +** ^If [sqlite3_value] object V was initialized +** using [sqlite3_bind_pointer(S,I,P,X,D)] or [sqlite3_result_pointer(C,P,X,D)] +** and if X and Y are strings that compare equal according to strcmp(X,Y), +** then sqlite3_value_pointer(V,Y) will return the pointer P. ^Otherwise, +** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** ^(The sqlite3_value_type(V) interface returns the +** [SQLITE_INTEGER | datatype code] for the initial datatype of the +** [sqlite3_value] object V. The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].)^ +** Other interfaces might change the datatype for an sqlite3_value object. +** For example, if the datatype is initially SQLITE_INTEGER and +** sqlite3_value_text(V) is called to extract a text value for that +** integer, then subsequent calls to sqlite3_value_type(V) might return +** SQLITE_TEXT. Whether or not a persistent internal datatype conversion +** occurs is undefined and may change from one release of SQLite to the next. +** ** ^(The sqlite3_value_numeric_type() interface attempts to apply ** numeric affinity to the value. This means that an attempt is ** made to convert the value to an integer or floating point. If @@ -4658,6 +6062,24 @@ SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(voi ** then the conversion is performed. Otherwise no conversion occurs. ** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** +** ^Within the [xUpdate] method of a [virtual table], the +** sqlite3_value_nochange(X) interface returns true if and only if +** the column corresponding to X is unchanged by the UPDATE operation +** that the xUpdate method call was invoked to implement and if +** and the prior [xColumn] method call that was invoked to extracted +** the value for that column returned without setting a result (probably +** because it queried [sqlite3_vtab_nochange()] and found that the column +** was unchanging). ^Within an [xUpdate] method, any value for which +** sqlite3_value_nochange(X) is true will in all other respects appear +** to be a NULL value. If sqlite3_value_nochange(X) is invoked anywhere other +** than within an [xUpdate] method call for an UPDATE statement, then +** the return value is arbitrary and meaningless. +** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** or an expression, then sqlite3_value_frombind(X) returns zero. +** ** Please pay particular attention to the fact that the pointer returned ** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to @@ -4666,19 +6088,66 @@ SQLITE_API SQLITE_DEPRECATED int SQLITE_STDCALL sqlite3_memory_alarm(void(*)(voi ** ** These routines must be called from the same thread as ** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
      +**
    • sqlite3_value_blob() +**
    • sqlite3_value_text() +**
    • sqlite3_value_text16() +**
    • sqlite3_value_text16le() +**
    • sqlite3_value_text16be() +**
    • sqlite3_value_bytes() +**
    • sqlite3_value_bytes16() +**
    +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value*); -SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value*); -SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value*); -SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value*); -SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value*); -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value*); -SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value*); -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value*); -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value*); -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value*); -SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value*); -SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); +SQLITE_API double sqlite3_value_double(sqlite3_value*); +SQLITE_API int sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); +SQLITE_API void *sqlite3_value_pointer(sqlite3_value*, const char*); +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API int sqlite3_value_type(sqlite3_value*); +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); + +/* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); /* ** CAPI3REF: Finding The Subtype Of SQL Values @@ -4690,11 +6159,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value*); ** one SQL function to another. Use the [sqlite3_result_subtype()] ** routine to set the subtype for the return value of an SQL function. ** -** SQLite makes no use of subtype itself. It merely passes the subtype -** from the result of one [application-defined SQL function] into the -** input of another. +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_SUBTYPE] property in the text +** encoding argument when the function is [sqlite3_create_function|registered]. +** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() +** might return zero instead of the upstream subtype in some corner cases. */ -SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value*); +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); /* ** CAPI3REF: Copy And Free SQL Values @@ -4704,14 +6175,15 @@ SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value*); ** object D and returns a pointer to that copy. ^The [sqlite3_value] returned ** is a [protected sqlite3_value] object even if the input is not. ** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a -** memory allocation fails. +** memory allocation fails. ^If V is a [pointer value], then the result +** of sqlite3_value_dup(V) is a NULL value. ** ** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object ** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer ** then sqlite3_value_free(V) is a harmless no-op. */ -SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value*); -SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value*); +SQLITE_API void sqlite3_value_free(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context @@ -4720,9 +6192,9 @@ SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); ** Implementations of aggregate SQL functions use this ** routine to allocate memory for storing their state. ** -** ^The first time the sqlite3_aggregate_context(C,N) routine is called -** for a particular aggregate function, SQLite -** allocates N of memory, zeroes out that memory, and returns a pointer +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite allocates +** N bytes of memory, zeroes out that memory, and returns a pointer ** to the new memory. ^On second and subsequent calls to ** sqlite3_aggregate_context() for the same aggregate function instance, ** the same buffer is returned. Sqlite3_aggregate_context() is normally @@ -4733,19 +6205,19 @@ SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); ** In those cases, sqlite3_aggregate_context() might be called for the ** first time from within xFinal().)^ ** -** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer ** when first called if N is less than or equal to zero or if a memory -** allocate error occurs. +** allocation error occurs. ** ** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is ** determined by the N parameter on first successful call. Changing the -** value of N in subsequent call to sqlite3_aggregate_context() within +** value of N in any subsequent call to sqlite3_aggregate_context() within ** the same aggregate function instance will not resize the memory ** allocation.)^ Within the xFinal callback, it is customary to set -** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no ** pointless memory allocations occur. ** -** ^SQLite automatically frees the memory allocated by +** ^SQLite automatically frees the memory allocated by ** sqlite3_aggregate_context() when the aggregate query concludes. ** ** The first parameter must be a copy of the @@ -4756,7 +6228,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value*); ** This routine must be called from the same thread in which ** the aggregate SQL function is running. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context*, int nBytes); +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* ** CAPI3REF: User Data For Functions @@ -4771,7 +6243,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context*, int ** This routine must be called from the same thread in which ** the application-defined function is running. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context*); +SQLITE_API void *sqlite3_user_data(sqlite3_context*); /* ** CAPI3REF: Database Connection For Functions @@ -4783,62 +6255,133 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context*); ** and [sqlite3_create_function16()] routines that originally ** registered the application defined function. */ -SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context*); +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data ** METHOD: sqlite3_context ** ** These functions may be used by (non-aggregate) SQL functions to -** associate metadata with argument values. If the same value is passed to -** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated metadata may be preserved. An example -** of where this might be useful is in a regular-expression matching -** function. The compiled version of the regular expression can be stored as -** metadata associated with the pattern string. +** associate auxiliary data with argument values. If the same argument +** value is passed to multiple invocations of the same SQL function during +** query execution, under some circumstances the associated auxiliary data +** might be preserved. An example of where this might be useful is in a +** regular-expression matching function. The compiled version of the regular +** expression can be stored as auxiliary data associated with the pattern string. ** Then as long as the pattern string remains the same, ** the compiled regular expression can be reused on multiple ** invocations of the same function. ** -** ^The sqlite3_get_auxdata() interface returns a pointer to the metadata -** associated by the sqlite3_set_auxdata() function with the Nth argument -** value to the application-defined function. ^If there is no metadata -** associated with the function argument, this sqlite3_get_auxdata() interface +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data +** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument +** value to the application-defined function. ^N is zero for the left-most +** function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) interface ** returns a NULL pointer. ** -** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as metadata for the N-th -** argument of the application-defined function. ^Subsequent +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the +** N-th argument of the application-defined function. ^Subsequent ** calls to sqlite3_get_auxdata(C,N) return P from the most recent -** sqlite3_set_auxdata(C,N,P,X) call if the metadata is still valid or -** NULL if the metadata has been discarded. +** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or +** NULL if the auxiliary data has been discarded. ** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, ** SQLite will invoke the destructor function X with parameter P exactly -** once, when the metadata is discarded. -** SQLite is free to discard the metadata at any time, including:
      -**
    • when the corresponding function parameter changes, or -**
    • when [sqlite3_reset()] or [sqlite3_finalize()] is called for the -** SQL statement, or -**
    • when sqlite3_set_auxdata() is invoked again on the same parameter, or -**
    • during the original sqlite3_set_auxdata() call when a memory -** allocation error occurs.
    )^ +** once, when the auxiliary data is discarded. +** SQLite is free to discard the auxiliary data at any time, including:
      +**
    • ^(when the corresponding function parameter changes)^, or +**
    • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
    • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
    • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^ +**
    • ^(during the original sqlite3_set_auxdata() call if the function +** is evaluated during query planning instead of during query execution, +** as sometimes happens with [SQLITE_ENABLE_STAT4].)^
    ** -** Note the last bullet in particular. The destructor X in +** Note the last two bullets in particular. The destructor X in ** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the ** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() ** should be called near the end of the function implementation and the ** function implementation should not make any use of P after -** sqlite3_set_auxdata() has been called. +** sqlite3_set_auxdata() has been called. Furthermore, a call to +** sqlite3_get_auxdata() that occurs immediately after a corresponding call +** to sqlite3_set_auxdata() might still return NULL if an out-of-memory +** condition occurred during the sqlite3_set_auxdata() call or if the +** function is being evaluated during query planning rather than during +** query execution. ** -** ^(In practice, metadata is preserved between function calls for +** ^(In practice, auxiliary data is preserved between function calls for ** function parameters that are compile-time constants, including literal ** values and [parameters] and expressions composed from the same.)^ ** +** The value of the N parameter to these interfaces should be non-negative. +** Future enhancements may make use of negative N values to define new +** kinds of function caching behavior. +** ** These routines must be called from the same thread in which ** the SQL function is running. +** +** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()]. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context*, int N); -SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +/* +** CAPI3REF: Database Connection Client Data +** METHOD: sqlite3 +** +** These functions are used to associate one or more named pointers +** with a [database connection]. +** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P +** to be attached to [database connection] D using name N. Subsequent +** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P +** or a NULL pointer if there were no prior calls to +** sqlite3_set_clientdata() with the same values of D and N. +** Names are compared using strcmp() and are thus case sensitive. +** +** If P and X are both non-NULL, then the destructor X is invoked with +** argument P on the first of the following occurrences: +**
      +**
    • An out-of-memory error occurs during the call to +** sqlite3_set_clientdata() which attempts to register pointer P. +**
    • A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made +** with the same D and N parameters. +**
    • The database connection closes. SQLite does not make any guarantees +** about the order in which destructors are called, only that all +** destructors will be called exactly once at some point during the +** database connection closing process. +**
    +** +** SQLite does not do anything with client data other than invoke +** destructors on the client data at the appropriate time. The intended +** use for client data is to provide a mechanism for wrapper libraries +** to store additional information about an SQLite database connection. +** +** There is no limit (other than available memory) on the number of different +** client data pointers (with different names) that can be attached to a +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. +** +** There is no way to enumerate the client data pointers +** associated with a database connection. The N parameter can be thought +** of as a secret key such that only code that knows the secret key is able +** to access the associated data. +** +** Security Warning: These interfaces should not be exposed in scripting +** languages or in other circumstances where it might be possible for an +** an attacker to invoke them. Any agent that can invoke these interfaces +** can probably also take control of the process. +** +** Database connection client data is only available for SQLite +** version 3.44.0 ([dateof:3.44.0]) and later. +** +** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()]. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*); +SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*)); /* ** CAPI3REF: Constants Defining Special Destructor Behavior @@ -4890,8 +6433,9 @@ typedef void (*sqlite3_destructor_type)(void*); ** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() ** as the text of an error message. ^SQLite interprets the error ** message string from sqlite3_result_error() as UTF-8. ^SQLite -** interprets the string from sqlite3_result_error16() as UTF-16 in native -** byte order. ^If the third parameter to sqlite3_result_error() +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() ** or sqlite3_result_error16() is negative then SQLite takes as the error ** message all text up through the first zero character. ** ^If the third parameter to sqlite3_result_error() or @@ -4933,9 +6477,10 @@ typedef void (*sqlite3_destructor_type)(void*); ** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. ** ^SQLite takes the text result from the application from ** the 2nd parameter of the sqlite3_result_text* interfaces. -** ^If the 3rd parameter to the sqlite3_result_text* interfaces -** is negative, then SQLite takes result text from the 2nd parameter -** through the first zero character. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. ** ^If the 3rd parameter to the sqlite3_result_text* interfaces ** is non-negative, then as many bytes (not characters) of the text ** pointed to by the 2nd parameter are taken as the application-defined @@ -4956,9 +6501,28 @@ typedef void (*sqlite3_destructor_type)(void*); ** when it has finished using that result. ** ^If the 4th parameter to the sqlite3_result_text* interfaces ** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT -** then SQLite makes a copy of the result into space obtained from +** then SQLite makes a copy of the result into space obtained ** from [sqlite3_malloc()] before it returns. ** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** ** ^The sqlite3_result_value() interface sets the result of ** the application-defined function to be a copy of the ** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The @@ -4969,31 +6533,43 @@ typedef void (*sqlite3_destructor_type)(void*); ** [unprotected sqlite3_value] object is required, so either ** kind of [sqlite3_value] object can be used with this interface. ** +** ^The sqlite3_result_pointer(C,P,T,D) interface sets the result to an +** SQL NULL value, just like [sqlite3_result_null(C)], except that it +** also associates the host-language pointer P or type T with that +** NULL value such that the pointer can be retrieved within an +** [application-defined SQL function] using [sqlite3_value_pointer()]. +** ^If the D parameter is not NULL, then it is a pointer to a destructor +** for the P parameter. ^SQLite invokes D with P as its only argument +** when SQLite is finished with P. The T parameter should be a static +** string and preferably a string literal. The sqlite3_result_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** ** If these routines are called from within the different thread ** than the one containing the application-defined function that received ** the [sqlite3_context] pointer, the results are undefined. */ -SQLITE_API void SQLITE_STDCALL sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64(sqlite3_context*,const void*, +SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*, sqlite3_uint64,void(*)(void*)); -SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context*, double); -SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context*, const char*, int); -SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context*, const void*, int); -SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context*); -SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context*); -SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context*, int); -SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context*, int); -SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context*, sqlite3_int64); -SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context*); -SQLITE_API void SQLITE_STDCALL sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -SQLITE_API void SQLITE_STDCALL sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, +SQLITE_API void sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void sqlite3_result_null(sqlite3_context*); +SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, void(*)(void*), unsigned char encoding); -SQLITE_API void SQLITE_STDCALL sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); -SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); -SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context*, sqlite3_value*); -SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context*, int n); -SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); +SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void sqlite3_result_pointer(sqlite3_context*, void*,const char*,void(*)(void*)); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); /* @@ -5001,14 +6577,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context*, sqlite ** METHOD: sqlite3_context ** ** The sqlite3_result_subtype(C,T) function causes the subtype of -** the result from the [application-defined SQL function] with -** [sqlite3_context] C to be the value T. Only the lower 8 bits +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits ** of the subtype T are preserved in current versions of SQLite; ** higher order bits are discarded. ** The number of subtype bytes preserved by SQLite might increase ** in future releases of SQLite. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_RESULT_SUBTYPE] property in its +** text encoding argument when the SQL function is +** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE] +** property is omitted from the function that invokes sqlite3_result_subtype(), +** then in some cases the sqlite3_result_subtype() might fail to set +** the result subtype. +** +** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any +** SQL function that invokes the sqlite3_result_subtype() interface +** and that does not have the SQLITE_RESULT_SUBTYPE property will raise +** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1 +** by default. */ -SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned int); +SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); /* ** CAPI3REF: Define New Collating Sequences @@ -5032,7 +6622,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned **
  • [SQLITE_UTF16_ALIGNED]. ** )^ ** ^The eTextRep argument determines the encoding of strings passed -** to the collating function callback, xCallback. +** to the collating function callback, xCompare. ** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep ** force strings to be UTF16 with native byte order. ** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin @@ -5041,18 +6631,19 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned ** ^The fourth argument, pArg, is an application data pointer that is passed ** through as the first argument to the collating function callback. ** -** ^The fifth argument, xCallback, is a pointer to the collating function. +** ^The fifth argument, xCompare, is a pointer to the collating function. ** ^Multiple collating functions can be registered using the same name but ** with different eTextRep parameters and SQLite will use whichever ** function requires the least amount of data transformation. -** ^If the xCallback argument is NULL then the collating function is +** ^If the xCompare argument is NULL then the collating function is ** deleted. ^When all collating functions having the same name are deleted, ** that collation is no longer usable. ** -** ^The collating function callback is invoked with a copy of the pArg +** ^The collating function callback is invoked with a copy of the pArg ** application data pointer and with two strings in the encoding specified -** by the eTextRep argument. The collating function must return an -** integer that is negative, zero, or positive +** by the eTextRep argument. The two integer parameters to the collating +** function callback are the length of the two strings, in bytes. The collating +** function must return an integer that is negative, zero, or positive ** if the first string is less than, equal to, or greater than the second, ** respectively. A collating function must always return the same answer ** given the same inputs. If two or more collating functions are registered @@ -5069,7 +6660,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned ** ** ** If a collating function fails any of the above constraints and that -** collating function is registered and used, then the behavior of SQLite +** collating function is registered and used, then the behavior of SQLite ** is undefined. ** ** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() @@ -5079,36 +6670,36 @@ SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context*,unsigned ** calls to the collation creation functions or when the ** [database connection] is closed using [sqlite3_close()]. ** -** ^The xDestroy callback is not called if the +** ^The xDestroy callback is not called if the ** sqlite3_create_collation_v2() function fails. Applications that invoke -** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should ** check the return code and dispose of the application data pointer ** themselves rather than expecting SQLite to deal with it for them. -** This is different from every other SQLite interface. The inconsistency -** is unfortunate but cannot be changed without breaking backwards +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards ** compatibility. ** ** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_collation( - sqlite3*, - const char *zName, - int eTextRep, +SQLITE_API int sqlite3_create_collation( + sqlite3*, + const char *zName, + int eTextRep, void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); -SQLITE_API int SQLITE_STDCALL sqlite3_create_collation_v2( - sqlite3*, - const char *zName, - int eTextRep, +SQLITE_API int sqlite3_create_collation_v2( + sqlite3*, + const char *zName, + int eTextRep, void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); -SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( - sqlite3*, +SQLITE_API int sqlite3_create_collation16( + sqlite3*, const void *zName, - int eTextRep, + int eTextRep, void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); @@ -5140,68 +6731,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_collation16( ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed( - sqlite3*, - void*, +SQLITE_API int sqlite3_collation_needed( + sqlite3*, + void*, void(*)(void*,sqlite3*,int eTextRep,const char*) ); -SQLITE_API int SQLITE_STDCALL sqlite3_collation_needed16( - sqlite3*, +SQLITE_API int sqlite3_collation_needed16( + sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const void*) ); -#ifdef SQLITE_HAS_CODEC -/* -** Specify the key for an encrypted database. This routine should be -** called right after sqlite3_open(). -** -** The code to implement this API is not available in the public release -** of SQLite. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_key( - sqlite3 *db, /* Database to be rekeyed */ - const void *pKey, int nKey /* The key */ -); -SQLITE_API int SQLITE_STDCALL sqlite3_key_v2( - sqlite3 *db, /* Database to be rekeyed */ - const char *zDbName, /* Name of the database */ - const void *pKey, int nKey /* The key */ -); - -/* -** Change the key on an open database. If the current database is not -** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the -** database is decrypted. -** -** The code to implement this API is not available in the public release -** of SQLite. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_rekey( - sqlite3 *db, /* Database to be rekeyed */ - const void *pKey, int nKey /* The new key */ -); -SQLITE_API int SQLITE_STDCALL sqlite3_rekey_v2( - sqlite3 *db, /* Database to be rekeyed */ - const char *zDbName, /* Name of the database */ - const void *pKey, int nKey /* The new key */ -); - -/* -** Specify the activation key for a SEE database. Unless -** activated, none of the SEE routines will work. -*/ -SQLITE_API void SQLITE_STDCALL sqlite3_activate_see( - const char *zPassPhrase /* Activation phrase */ -); -#endif - #ifdef SQLITE_ENABLE_CEROD /* -** Specify the activation key for a CEROD database. Unless +** Specify the activation key for a CEROD database. Unless ** activated, none of the CEROD routines will work. */ -SQLITE_API void SQLITE_STDCALL sqlite3_activate_cerod( +SQLITE_API void sqlite3_activate_cerod( const char *zPassPhrase /* Activation phrase */ ); #endif @@ -5222,8 +6768,15 @@ SQLITE_API void SQLITE_STDCALL sqlite3_activate_cerod( ** of the default VFS is not implemented correctly, or not implemented at ** all, then the behavior of sqlite3_sleep() may deviate from the description ** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ -SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int); +SQLITE_API int sqlite3_sleep(int); /* ** CAPI3REF: Name Of The Folder Holding Temporary Files @@ -5254,7 +6807,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int); ** ^The [temp_store_directory pragma] may modify this variable and cause ** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, ** the [temp_store_directory pragma] always assumes that any string -** that this variable points to is held in memory obtained from +** that this variable points to is held in memory obtained from ** [sqlite3_malloc] and the pragma may attempt to free that memory ** using [sqlite3_free]. ** Hence, if this variable is modified directly, either it should be @@ -5311,7 +6864,7 @@ SQLITE_API char *sqlite3_temp_directory; ** ^The [data_store_directory pragma] may modify this variable and cause ** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, ** the [data_store_directory pragma] always assumes that any string -** that this variable points to is held in memory obtained from +** that this variable points to is held in memory obtained from ** [sqlite3_malloc] and the pragma may attempt to free that memory ** using [sqlite3_free]. ** Hence, if this variable is modified directly, either it should be @@ -5320,6 +6873,41 @@ SQLITE_API char *sqlite3_temp_directory; */ SQLITE_API char *sqlite3_data_directory; +/* +** CAPI3REF: Win32 Specific Interface +** +** These interfaces are available only on Windows. The +** [sqlite3_win32_set_directory] interface is used to set the value associated +** with the [sqlite3_temp_directory] or [sqlite3_data_directory] variable, to +** zValue, depending on the value of the type parameter. The zValue parameter +** should be NULL to cause the previous value to be freed via [sqlite3_free]; +** a non-NULL value will be copied into memory obtained from [sqlite3_malloc] +** prior to being used. The [sqlite3_win32_set_directory] interface returns +** [SQLITE_OK] to indicate success, [SQLITE_ERROR] if the type is unsupported, +** or [SQLITE_NOMEM] if memory could not be allocated. The value of the +** [sqlite3_data_directory] variable is intended to act as a replacement for +** the current directory on the sub-platforms of Win32 where that concept is +** not present, e.g. WinRT and UWP. The [sqlite3_win32_set_directory8] and +** [sqlite3_win32_set_directory16] interfaces behave exactly the same as the +** sqlite3_win32_set_directory interface except the string parameter must be +** UTF-8 or UTF-16, respectively. +*/ +SQLITE_API int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +); +SQLITE_API int sqlite3_win32_set_directory8(unsigned long type, const char *zValue); +SQLITE_API int sqlite3_win32_set_directory16(unsigned long type, const void *zValue); + +/* +** CAPI3REF: Win32 Directory Types +** +** These macros are only available on Windows. They define the allowed values +** for the type argument to the [sqlite3_win32_set_directory] interface. +*/ +#define SQLITE_WIN32_DATA_DIRECTORY_TYPE 1 +#define SQLITE_WIN32_TEMP_DIRECTORY_TYPE 2 + /* ** CAPI3REF: Test For Auto-Commit Mode ** KEYWORDS: {autocommit mode} @@ -5342,7 +6930,7 @@ SQLITE_API char *sqlite3_data_directory; ** connection while this routine is running, then the return value ** is undefined. */ -SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3*); +SQLITE_API int sqlite3_get_autocommit(sqlite3*); /* ** CAPI3REF: Find The Database Handle Of A Prepared Statement @@ -5355,24 +6943,61 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3*); ** to the [sqlite3_prepare_v2()] call (or its variants) that was used to ** create the statement in the first place. */ -SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt*); +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); + +/* +** CAPI3REF: Return The Schema Name For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name +** for the N-th database on database connection D, or a NULL pointer of N is +** out of range. An N value of 0 means the main database file. An N of 1 is +** the "temp" schema. Larger values of N correspond to various ATTACH-ed +** databases. +** +** Space to hold the string that is returned by sqlite3_db_name() is managed +** by SQLite itself. The string might be deallocated by any operation that +** changes the schema, including [ATTACH] or [DETACH] or calls to +** [sqlite3_serialize()] or [sqlite3_deserialize()], even operations that +** occur on a different thread. Applications that need to +** remember the string long-term should make their own copy. Applications that +** are accessing the same database connection simultaneously on multiple +** threads should mutex-protect calls to this API and should make their own +** private copy of the result prior to releasing the mutex. +*/ +SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); /* ** CAPI3REF: Return The Filename For A Database Connection ** METHOD: sqlite3 ** -** ^The sqlite3_db_filename(D,N) interface returns a pointer to a filename -** associated with database N of connection D. ^The main database file -** has the name "main". If there is no attached database N on the database +** ^The sqlite3_db_filename(D,N) interface returns a pointer to the filename +** associated with database N of connection D. +** ^If there is no attached database N on the database ** connection D, or if database N is a temporary or in-memory database, then -** a NULL pointer is returned. +** this function will return either a NULL pointer or an empty string. +** +** ^The string value returned by this routine is owned and managed by +** the database connection. ^The value will be valid until the database N +** is [DETACH]-ed or until the database connection closes. ** ** ^The filename returned by this function is the output of the ** xFullPathname method of the [VFS]. ^In other words, the filename ** will be an absolute pathname, even if the filename used ** to open the database originally was a URI or relative pathname. +** +** If the filename pointer returned by this routine is not NULL, then it +** can be used as the filename input parameter to these routines: +**
      +**
    • [sqlite3_uri_parameter()] +**
    • [sqlite3_uri_boolean()] +**
    • [sqlite3_uri_int64()] +**
    • [sqlite3_filename_database()] +**
    • [sqlite3_filename_journal()] +**
    • [sqlite3_filename_wal()] +**
    */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName); +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); /* ** CAPI3REF: Determine if a database is read-only @@ -5382,7 +7007,58 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const cha ** of connection D is read-only, 0 if it is read/write, or -1 if N is not ** the name of a database on connection D. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName); +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine the transaction state of a database +** METHOD: sqlite3 +** +** ^The sqlite3_txn_state(D,S) interface returns the current +** [transaction state] of schema S in database connection D. ^If S is NULL, +** then the highest transaction state of any schema on database connection D +** is returned. Transaction states are (in order of lowest to highest): +**
      +**
    1. SQLITE_TXN_NONE +**
    2. SQLITE_TXN_READ +**
    3. SQLITE_TXN_WRITE +**
    +** ^If the S argument to sqlite3_txn_state(D,S) is not the name of +** a valid schema, then -1 is returned. +*/ +SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); + +/* +** CAPI3REF: Allowed return values from sqlite3_txn_state() +** KEYWORDS: {transaction state} +** +** These constants define the current transaction state of a database file. +** ^The [sqlite3_txn_state(D,S)] interface returns one of these +** constants in order to describe the transaction state of schema S +** in [database connection] D. +** +**
    +** [[SQLITE_TXN_NONE]]
    SQLITE_TXN_NONE
    +**
    The SQLITE_TXN_NONE state means that no transaction is currently +** pending.
    +** +** [[SQLITE_TXN_READ]]
    SQLITE_TXN_READ
    +**
    The SQLITE_TXN_READ state means that the database is currently +** in a read transaction. Content has been read from the database file +** but nothing in the database file has changed. The transaction state +** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** no other conflicting concurrent write transactions. The transaction +** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or +** [COMMIT].
    +** +** [[SQLITE_TXN_WRITE]]
    SQLITE_TXN_WRITE
    +**
    The SQLITE_TXN_WRITE state means that the database is currently +** in a write transaction. Content has been written to the database file +** but has not yet committed. The transaction state will change to +** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
    +*/ +#define SQLITE_TXN_NONE 0 +#define SQLITE_TXN_READ 1 +#define SQLITE_TXN_WRITE 2 /* ** CAPI3REF: Find the next prepared statement @@ -5398,7 +7074,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbNa ** [sqlite3_next_stmt(D,S)] must refer to an open database ** connection and in particular must not be a NULL pointer. */ -SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* ** CAPI3REF: Commit And Rollback Notification Callbacks @@ -5447,8 +7123,74 @@ SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_ ** ** See also the [sqlite3_update_hook()] interface. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); -SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); +SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); + +/* +** CAPI3REF: Autovacuum Compaction Amount Callback +** METHOD: sqlite3 +** +** ^The sqlite3_autovacuum_pages(D,C,P,X) interface registers a callback +** function C that is invoked prior to each autovacuum of the database +** file. ^The callback is passed a copy of the generic data pointer (P), +** the schema-name of the attached database that is being autovacuumed, +** the size of the database file in pages, the number of free pages, +** and the number of bytes per page, respectively. The callback should +** return the number of free pages that should be removed by the +** autovacuum. ^If the callback returns zero, then no autovacuum happens. +** ^If the value returned is greater than or equal to the number of +** free pages, then a complete autovacuum happens. +** +**

    ^If there are multiple ATTACH-ed database files that are being +** modified as part of a transaction commit, then the autovacuum pages +** callback is invoked separately for each file. +** +**

    The callback is not reentrant. The callback function should +** not attempt to invoke any other SQLite interface. If it does, bad +** things may happen, including segmentation faults and corrupt database +** files. The callback function should be a simple function that +** does some arithmetic on its input parameters and returns a result. +** +** ^The X parameter to sqlite3_autovacuum_pages(D,C,P,X) is an optional +** destructor for the P parameter. ^If X is not NULL, then X(P) is +** invoked whenever the database connection closes or when the callback +** is overwritten by another invocation of sqlite3_autovacuum_pages(). +** +**

    ^There is only one autovacuum pages callback per database connection. +** ^Each call to the sqlite3_autovacuum_pages() interface overrides all +** previous invocations for that database connection. ^If the callback +** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer, +** then the autovacuum steps callback is canceled. The return value +** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might +** be some other error code if something goes wrong. The current +** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other +** return codes might be added in future releases. +** +**

    If no autovacuum pages callback is specified (the usual case) or +** a NULL pointer is provided for the callback, +** then the default behavior is to vacuum all free pages. So, in other +** words, the default behavior is the same as if the callback function +** were something like this: +** +** 

    +**     unsigned int demonstration_autovac_pages_callback(
+**       void *pClientData,
+**       const char *zSchema,
+**       unsigned int nDbPage,
+**       unsigned int nFreePage,
+**       unsigned int nBytePerPage
+**     ){
+**       return nFreePage;
+**     }
+**
    +*/ +SQLITE_API int sqlite3_autovacuum_pages( + sqlite3 *db, + unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int), + void*, + void(*)(void*) +); + /* ** CAPI3REF: Data Change Notification Callbacks @@ -5474,16 +7216,22 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(sqlite3*, void(*)(void *), ** ^In the case of an update, this is the [rowid] after the update takes place. ** ** ^(The update hook is not invoked when internal system tables are -** modified (i.e. sqlite_master and sqlite_sequence).)^ +** modified (i.e. sqlite_sequence).)^ ** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** ** ^In the current implementation, the update hook -** is not invoked when duplication rows are deleted because of an +** is not invoked when conflicting rows are deleted because of an ** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook ** invoked when rows are deleted using the [truncate optimization]. ** The exceptions defined in this paragraph might change in a future ** release of SQLite. ** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** ** The update hook implementation must not do anything that will modify ** the database connection that invoked the update hook. Any actions ** to modify the database connection must be deferred until after the @@ -5499,8 +7247,8 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook(sqlite3*, void(*)(void *), ** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], ** and [sqlite3_preupdate_hook()] interfaces. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( - sqlite3*, +SQLITE_API void *sqlite3_update_hook( + sqlite3*, void(*)(void *,int ,char const *,char const *,sqlite3_int64), void* ); @@ -5513,25 +7261,35 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( ** to the same database. Sharing is enabled if the argument is true ** and disabled if the argument is false.)^ ** +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. +** ** ^Cache sharing is enabled and disabled for an entire process. -** This is a change as of SQLite version 3.5.0. In prior versions of SQLite, +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, ** sharing was enabled or disabled for each thread separately. ** ** ^(The cache sharing mode set by this interface effects all subsequent ** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. -** Existing database connections continue use the sharing mode +** Existing database connections continue to use the sharing mode ** that was in effect at the time they were opened.)^ ** ** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled ** successfully. An [error code] is returned otherwise.)^ ** -** ^Shared cache is disabled by default. But this might change in -** future releases of SQLite. Applications that care about shared -** cache setting should set it explicitly. +** ^Shared cache is disabled by default. It is recommended that it stay +** that way. In other words, do not use this routine. This interface +** continues to be provided for historical compatibility, but its use is +** discouraged. Any use of shared cache is discouraged. If shared cache +** must be used, it is recommended that shared cache only be enabled for +** individual database connections using the [sqlite3_open_v2()] interface +** with the [SQLITE_OPEN_SHAREDCACHE] flag. ** ** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 -** and will always return SQLITE_MISUSE. On those systems, -** shared cache mode should be enabled per-database connection via +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via ** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. ** ** This interface is threadsafe on processors where writing a @@ -5539,7 +7297,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( ** ** See Also: [SQLite Shared-Cache Mode] */ -SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int); +SQLITE_API int sqlite3_enable_shared_cache(int); /* ** CAPI3REF: Attempt To Free Heap Memory @@ -5555,7 +7313,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int); ** ** See also: [sqlite3_db_release_memory()] */ -SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int); +SQLITE_API int sqlite3_release_memory(int); /* ** CAPI3REF: Free Memory Used By A Database Connection @@ -5569,11 +7327,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int); ** ** See also: [sqlite3_release_memory()] */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3*); +SQLITE_API int sqlite3_db_release_memory(sqlite3*); /* ** CAPI3REF: Impose A Limit On Heap Size ** +** These interfaces impose limits on the amount of heap memory that will be +** by all database connections within a single process. +** ** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the ** soft limit on the amount of heap memory that may be allocated by SQLite. ** ^SQLite strives to keep heap memory utilization below the soft heap @@ -5581,23 +7342,44 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3*); ** as heap memory usages approaches the limit. ** ^The soft heap limit is "soft" because even though SQLite strives to stay ** below the limit, it will exceed the limit rather than generate -** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** an [SQLITE_NOMEM] error. In other words, the soft heap limit ** is advisory only. ** -** ^The return value from sqlite3_soft_heap_limit64() is the size of -** the soft heap limit prior to the call, or negative in the case of an +** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of +** N bytes on the amount of memory that will be allocated. ^The +** sqlite3_hard_heap_limit64(N) interface is similar to +** sqlite3_soft_heap_limit64(N) except that memory allocations will fail +** when the hard heap limit is reached. +** +** ^The return value from both sqlite3_soft_heap_limit64() and +** sqlite3_hard_heap_limit64() is the size of +** the heap limit prior to the call, or negative in the case of an ** error. ^If the argument N is negative -** then no change is made to the soft heap limit. Hence, the current -** size of the soft heap limit can be determined by invoking -** sqlite3_soft_heap_limit64() with a negative argument. +** then no change is made to the heap limit. Hence, the current +** size of heap limits can be determined by invoking +** sqlite3_soft_heap_limit64(-1) or sqlite3_hard_heap_limit(-1). ** -** ^If the argument N is zero then the soft heap limit is disabled. +** ^Setting the heap limits to zero disables the heap limiter mechanism. ** -** ^(The soft heap limit is not enforced in the current implementation +** ^The soft heap limit may not be greater than the hard heap limit. +** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) +** is invoked with a value of N that is greater than the hard heap limit, +** the soft heap limit is set to the value of the hard heap limit. +** ^The soft heap limit is automatically enabled whenever the hard heap +** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and +** the soft heap limit is outside the range of 1..N, then the soft heap +** limit is set to N. ^Invoking sqlite3_soft_heap_limit64(0) when the +** hard heap limit is enabled makes the soft heap limit equal to the +** hard heap limit. +** +** The memory allocation limits can also be adjusted using +** [PRAGMA soft_heap_limit] and [PRAGMA hard_heap_limit]. +** +** ^(The heap limits are not enforced in the current implementation ** if one or more of following conditions are true: ** **
      -**
    • The soft heap limit is set to zero. +**
    • The limit value is set to zero. **
    • Memory accounting is disabled using a combination of the ** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and ** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. @@ -5608,20 +7390,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3*); ** from the heap. **
    )^ ** -** Beginning with SQLite version 3.7.3, the soft heap limit is enforced -** regardless of whether or not the [SQLITE_ENABLE_MEMORY_MANAGEMENT] -** compile-time option is invoked. With [SQLITE_ENABLE_MEMORY_MANAGEMENT], -** the soft heap limit is enforced on every memory allocation. Without -** [SQLITE_ENABLE_MEMORY_MANAGEMENT], the soft heap limit is only enforced -** when memory is allocated by the page cache. Testing suggests that because -** the page cache is the predominate memory user in SQLite, most -** applications will achieve adequate soft heap limit enforcement without -** the use of [SQLITE_ENABLE_MEMORY_MANAGEMENT]. -** -** The circumstances under which SQLite will enforce the soft heap limit may +** The circumstances under which SQLite will enforce the heap limits may ** changes in future releases of SQLite. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); /* ** CAPI3REF: Deprecated Soft Heap Limit Interface @@ -5632,7 +7405,7 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 ** only. All new applications should use the ** [sqlite3_soft_heap_limit64()] interface rather than this one. */ -SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); /* @@ -5645,11 +7418,13 @@ SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); ** interface returns SQLITE_OK and fills in the non-NULL pointers in ** the final five arguments with appropriate values if the specified ** column exists. ^The sqlite3_table_column_metadata() interface returns -** SQLITE_ERROR and if the specified column does not exist. +** SQLITE_ERROR if the specified column does not exist. ** ^If the column-name parameter to sqlite3_table_column_metadata() is a -** NULL pointer, then this routine simply checks for the existance of the +** NULL pointer, then this routine simply checks for the existence of the ** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it -** does not. +** does not. If the table name parameter T in a call to +** sqlite3_table_column_metadata(X,D,T,C,...) is NULL then the result is +** undefined behavior. ** ** ^The column is identified by the second, third and fourth parameters to ** this function. ^(The second parameter is either the name of the database @@ -5683,7 +7458,7 @@ SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); ** ** ^If the specified table is actually a view, an [error code] is returned. ** -** ^If the specified column is "rowid", "oid" or "_rowid_" and the table +** ^If the specified column is "rowid", "oid" or "_rowid_" and the table ** is not a [WITHOUT ROWID] table and an ** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output ** parameters are set for the explicitly declared column. ^(If there is no @@ -5702,7 +7477,7 @@ SQLITE_API SQLITE_DEPRECATED void SQLITE_STDCALL sqlite3_soft_heap_limit(int N); ** parsed, if that has not already been done, and returns an error if ** any errors are encountered while loading the schema. */ -SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( +SQLITE_API int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ const char *zTableName, /* Table name */ @@ -5749,7 +7524,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( ** prior to calling this API, ** otherwise an error will be returned. ** -** Security warning: It is recommended that the +** Security warning: It is recommended that the ** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this ** interface. The use of the [sqlite3_enable_load_extension()] interface ** should be avoided. This will keep the SQL function [load_extension()] @@ -5758,7 +7533,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( ** ** See also the [load_extension() SQL function]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( +SQLITE_API int sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Derived from zFile if 0 */ @@ -5781,16 +7556,16 @@ SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( ** ** ^This interface enables or disables both the C-API ** [sqlite3_load_extension()] and the SQL function [load_extension()]. -** Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) -** to enable or disable only the C-API. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ ** ** Security warning: It is recommended that extension loading -** be disabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** be enabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method ** rather than this interface, so the [load_extension()] SQL function ** remains disabled. This will prevent SQL injections from giving attackers ** access to extension loading capabilities. */ -SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff); +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* ** CAPI3REF: Automatically Load Statically Linked Extensions @@ -5802,7 +7577,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int ono ** ** ^(Even though the function prototype shows that xEntryPoint() takes ** no arguments and returns void, SQLite invokes xEntryPoint() with three -** arguments and expects and integer result as if the signature of the +** arguments and expects an integer result as if the signature of the ** entry point where as follows: ** ** 
    @@ -5828,7 +7603,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int ono
 ** See also: [sqlite3_reset_auto_extension()]
 ** and [sqlite3_cancel_auto_extension()]
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xEntryPoint)(void));
+SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Cancel Automatic Extension Loading
@@ -5836,11 +7611,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xEntryPoint)(void));
 ** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the
 ** initialization routine X that was registered using a prior call to
 ** [sqlite3_auto_extension(X)].  ^The [sqlite3_cancel_auto_extension(X)]
-** routine returns 1 if initialization routine X was successfully 
+** routine returns 1 if initialization routine X was successfully
 ** unregistered and it returns 0 if X was not on the list of initialization
 ** routines.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xEntryPoint)(void));
+SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void));
 
 /*
 ** CAPI3REF: Reset Automatic Extension Loading
@@ -5848,16 +7623,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xEntryPoint)(
 ** ^This interface disables all automatic extensions previously
 ** registered using [sqlite3_auto_extension()].
 */
-SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void);
-
-/*
-** The interface to the virtual-table mechanism is currently considered
-** to be experimental.  The interface might change in incompatible ways.
-** If this is a problem for you, do not use the interface at this time.
-**
-** When the virtual-table mechanism stabilizes, we will declare the
-** interface fixed, support it indefinitely, and remove this comment.
-*/
+SQLITE_API void sqlite3_reset_auto_extension(void);
 
 /*
 ** Structures used by the virtual table interface
@@ -5871,8 +7637,8 @@ typedef struct sqlite3_module sqlite3_module;
 ** CAPI3REF: Virtual Table Object
 ** KEYWORDS: sqlite3_module {virtual table module}
 **
-** This structure, sometimes called a "virtual table module", 
-** defines the implementation of a [virtual tables].  
+** This structure, sometimes called a "virtual table module",
+** defines the implementation of a [virtual table].
 ** This structure consists mostly of methods for the module.
 **
 ** ^A virtual table module is created by filling in a persistent
@@ -5911,11 +7677,18 @@ struct sqlite3_module {
                        void (**pxFunc)(sqlite3_context*,int,sqlite3_value**),
                        void **ppArg);
   int (*xRename)(sqlite3_vtab *pVtab, const char *zNew);
-  /* The methods above are in version 1 of the sqlite_module object. Those 
+  /* The methods above are in version 1 of the sqlite_module object. Those
   ** below are for version 2 and greater. */
   int (*xSavepoint)(sqlite3_vtab *pVTab, int);
   int (*xRelease)(sqlite3_vtab *pVTab, int);
   int (*xRollbackTo)(sqlite3_vtab *pVTab, int);
+  /* The methods above are in versions 1 and 2 of the sqlite_module object.
+  ** Those below are for version 3 and greater. */
+  int (*xShadowName)(const char*);
+  /* The methods above are in versions 1 through 3 of the sqlite_module object.
+  ** Those below are for version 4 and greater. */
+  int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema,
+                    const char *zTabName, int mFlags, char **pzErr);
 };
 
 /*
@@ -5958,7 +7731,7 @@ struct sqlite3_module {
 ** required by SQLite. If the table has at least 64 columns and any column
 ** to the right of the first 63 is required, then bit 63 of colUsed is also
 ** set. In other words, column iCol may be required if the expression
-** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to 
+** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to
 ** non-zero.
 **
 ** The [xBestIndex] method must fill aConstraintUsage[] with information
@@ -5966,12 +7739,18 @@ struct sqlite3_module {
 ** the right-hand side of the corresponding aConstraint[] is evaluated
 ** and becomes the argvIndex-th entry in argv.  ^(If aConstraintUsage[].omit
 ** is true, then the constraint is assumed to be fully handled by the
-** virtual table and is not checked again by SQLite.)^
+** virtual table and might not be checked again by the byte code.)^ ^(The
+** aConstraintUsage[].omit flag is an optimization hint. When the omit flag
+** is left in its default setting of false, the constraint will always be
+** checked separately in byte code.  If the omit flag is change to true, then
+** the constraint may or may not be checked in byte code.  In other words,
+** when the omit flag is true there is no guarantee that the constraint will
+** not be checked again using byte code.)^
 **
-** ^The idxNum and idxPtr values are recorded and passed into the
+** ^The idxNum and idxStr values are recorded and passed into the
 ** [xFilter] method.
-** ^[sqlite3_free()] is used to free idxPtr if and only if
-** needToFreeIdxPtr is true.
+** ^[sqlite3_free()] is used to free idxStr if and only if
+** needToFreeIdxStr is true.
 **
 ** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in
 ** the correct order to satisfy the ORDER BY clause so that no separate
@@ -5979,17 +7758,19 @@ struct sqlite3_module {
 **
 ** ^The estimatedCost value is an estimate of the cost of a particular
 ** strategy. A cost of N indicates that the cost of the strategy is similar
-** to a linear scan of an SQLite table with N rows. A cost of log(N) 
+** to a linear scan of an SQLite table with N rows. A cost of log(N)
 ** indicates that the expense of the operation is similar to that of a
 ** binary search on a unique indexed field of an SQLite table with N rows.
 **
 ** ^The estimatedRows value is an estimate of the number of rows that
 ** will be returned by the strategy.
 **
-** The xBestIndex method may optionally populate the idxFlags field with a 
-** mask of SQLITE_INDEX_SCAN_* flags. Currently there is only one such flag -
-** SQLITE_INDEX_SCAN_UNIQUE. If the xBestIndex method sets this flag, SQLite
-** assumes that the strategy may visit at most one row. 
+** The xBestIndex method may optionally populate the idxFlags field with a
+** mask of SQLITE_INDEX_SCAN_* flags. One such flag is
+** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN]
+** output to show the idxNum has hex instead of as decimal.  Another flag is
+** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will
+** return at most one row.
 **
 ** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then
 ** SQLite also assumes that if a call to the xUpdate() method is made as
@@ -6002,13 +7783,15 @@ struct sqlite3_module {
 ** the xUpdate method are automatically rolled back by SQLite.
 **
 ** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info
-** structure for SQLite version 3.8.2. If a virtual table extension is
-** used with an SQLite version earlier than 3.8.2, the results of attempting 
-** to read or write the estimatedRows field are undefined (but are likely 
-** to included crashing the application). The estimatedRows field should
+** structure for SQLite [version 3.8.2] ([dateof:3.8.2]).
+** If a virtual table extension is
+** used with an SQLite version earlier than 3.8.2, the results of attempting
+** to read or write the estimatedRows field are undefined (but are likely
+** to include crashing the application). The estimatedRows field should
 ** therefore only be used if [sqlite3_libversion_number()] returns a
 ** value greater than or equal to 3008002. Similarly, the idxFlags field
-** was added for version 3.9.0. It may therefore only be used if
+** was added for [version 3.9.0] ([dateof:3.9.0]).
+** It may therefore only be used if
 ** sqlite3_libversion_number() returns a value greater than or equal to
 ** 3009000.
 */
@@ -6046,26 +7829,70 @@ struct sqlite3_index_info {
 
 /*
 ** CAPI3REF: Virtual Table Scan Flags
+**
+** Virtual table implementations are allowed to set the
+** [sqlite3_index_info].idxFlags field to some combination of
+** these bits.
 */
-#define SQLITE_INDEX_SCAN_UNIQUE      1     /* Scan visits at most 1 row */
+#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */
+#define SQLITE_INDEX_SCAN_HEX    0x00000002 /* Display idxNum as hex */
+                                            /* in EXPLAIN QUERY PLAN */
 
 /*
 ** CAPI3REF: Virtual Table Constraint Operator Codes
 **
-** These macros defined the allowed values for the
+** These macros define the allowed values for the
 ** [sqlite3_index_info].aConstraint[].op field.  Each value represents
-** an operator that is part of a constraint term in the wHERE clause of
+** an operator that is part of a constraint term in the WHERE clause of
 ** a query that uses a [virtual table].
+**
+** ^The left-hand operand of the operator is given by the corresponding
+** aConstraint[].iColumn field.  ^An iColumn of -1 indicates the left-hand
+** operand is the rowid.
+** The SQLITE_INDEX_CONSTRAINT_LIMIT and SQLITE_INDEX_CONSTRAINT_OFFSET
+** operators have no left-hand operand, and so for those operators the
+** corresponding aConstraint[].iColumn is meaningless and should not be
+** used.
+**
+** All operator values from SQLITE_INDEX_CONSTRAINT_FUNCTION through
+** value 255 are reserved to represent functions that are overloaded
+** by the [xFindFunction|xFindFunction method] of the virtual table
+** implementation.
+**
+** The right-hand operands for each constraint might be accessible using
+** the [sqlite3_vtab_rhs_value()] interface.  Usually the right-hand
+** operand is only available if it appears as a single constant literal
+** in the input SQL.  If the right-hand operand is another column or an
+** expression (even a constant expression) or a parameter, then the
+** sqlite3_vtab_rhs_value() probably will not be able to extract it.
+** ^The SQLITE_INDEX_CONSTRAINT_ISNULL and
+** SQLITE_INDEX_CONSTRAINT_ISNOTNULL operators have no right-hand operand
+** and hence calls to sqlite3_vtab_rhs_value() for those operators will
+** always return SQLITE_NOTFOUND.
+**
+** The collating sequence to be used for comparison can be found using
+** the [sqlite3_vtab_collation()] interface.  For most real-world virtual
+** tables, the collating sequence of constraints does not matter (for example
+** because the constraints are numeric) and so the sqlite3_vtab_collation()
+** interface is not commonly needed.
 */
-#define SQLITE_INDEX_CONSTRAINT_EQ      2
-#define SQLITE_INDEX_CONSTRAINT_GT      4
-#define SQLITE_INDEX_CONSTRAINT_LE      8
-#define SQLITE_INDEX_CONSTRAINT_LT     16
-#define SQLITE_INDEX_CONSTRAINT_GE     32
-#define SQLITE_INDEX_CONSTRAINT_MATCH  64
-#define SQLITE_INDEX_CONSTRAINT_LIKE   65
-#define SQLITE_INDEX_CONSTRAINT_GLOB   66
-#define SQLITE_INDEX_CONSTRAINT_REGEXP 67
+#define SQLITE_INDEX_CONSTRAINT_EQ          2
+#define SQLITE_INDEX_CONSTRAINT_GT          4
+#define SQLITE_INDEX_CONSTRAINT_LE          8
+#define SQLITE_INDEX_CONSTRAINT_LT         16
+#define SQLITE_INDEX_CONSTRAINT_GE         32
+#define SQLITE_INDEX_CONSTRAINT_MATCH      64
+#define SQLITE_INDEX_CONSTRAINT_LIKE       65
+#define SQLITE_INDEX_CONSTRAINT_GLOB       66
+#define SQLITE_INDEX_CONSTRAINT_REGEXP     67
+#define SQLITE_INDEX_CONSTRAINT_NE         68
+#define SQLITE_INDEX_CONSTRAINT_ISNOT      69
+#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL  70
+#define SQLITE_INDEX_CONSTRAINT_ISNULL     71
+#define SQLITE_INDEX_CONSTRAINT_IS         72
+#define SQLITE_INDEX_CONSTRAINT_LIMIT      73
+#define SQLITE_INDEX_CONSTRAINT_OFFSET     74
+#define SQLITE_INDEX_CONSTRAINT_FUNCTION  150
 
 /*
 ** CAPI3REF: Register A Virtual Table Implementation
@@ -6077,7 +7904,7 @@ struct sqlite3_index_info {
 ** preexisting [virtual table] for the module.
 **
 ** ^The module name is registered on the [database connection] specified
-** by the first parameter.  ^The name of the module is given by the 
+** by the first parameter.  ^The name of the module is given by the
 ** second parameter.  ^The third parameter is a pointer to
 ** the implementation of the [virtual table module].   ^The fourth
 ** parameter is an arbitrary client data pointer that is passed through
@@ -6092,14 +7919,20 @@ struct sqlite3_index_info {
 ** ^The sqlite3_create_module()
 ** interface is equivalent to sqlite3_create_module_v2() with a NULL
 ** destructor.
+**
+** ^If the third parameter (the pointer to the sqlite3_module object) is
+** NULL then no new module is created and any existing modules with the
+** same name are dropped.
+**
+** See also: [sqlite3_drop_modules()]
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_create_module(
+SQLITE_API int sqlite3_create_module(
   sqlite3 *db,               /* SQLite connection to register module with */
   const char *zName,         /* Name of the module */
   const sqlite3_module *p,   /* Methods for the module */
   void *pClientData          /* Client data for xCreate/xConnect */
 );
-SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2(
+SQLITE_API int sqlite3_create_module_v2(
   sqlite3 *db,               /* SQLite connection to register module with */
   const char *zName,         /* Name of the module */
   const sqlite3_module *p,   /* Methods for the module */
@@ -6107,6 +7940,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2(
   void(*xDestroy)(void*)     /* Module destructor function */
 );
 
+/*
+** CAPI3REF: Remove Unnecessary Virtual Table Implementations
+** METHOD: sqlite3
+**
+** ^The sqlite3_drop_modules(D,L) interface removes all virtual
+** table modules from database connection D except those named on list L.
+** The L parameter must be either NULL or a pointer to an array of pointers
+** to strings where the array is terminated by a single NULL pointer.
+** ^If the L parameter is NULL, then all virtual table modules are removed.
+**
+** See also: [sqlite3_create_module()]
+*/
+SQLITE_API int sqlite3_drop_modules(
+  sqlite3 *db,                /* Remove modules from this connection */
+  const char **azKeep         /* Except, do not remove the ones named here */
+);
+
 /*
 ** CAPI3REF: Virtual Table Instance Object
 ** KEYWORDS: sqlite3_vtab
@@ -6162,14 +8012,14 @@ struct sqlite3_vtab_cursor {
 ** to declare the format (the names and datatypes of the columns) of
 ** the virtual tables they implement.
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3*, const char *zSQL);
+SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL);
 
 /*
 ** CAPI3REF: Overload A Function For A Virtual Table
 ** METHOD: sqlite3
 **
 ** ^(Virtual tables can provide alternative implementations of functions
-** using the [xFindFunction] method of the [virtual table module].  
+** using the [xFindFunction] method of the [virtual table module].
 ** But global versions of those functions
 ** must exist in order to be overloaded.)^
 **
@@ -6181,17 +8031,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3*, const char *zSQL);
 ** purpose is to be a placeholder function that can be overloaded
 ** by a [virtual table].
 */
-SQLITE_API int SQLITE_STDCALL sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
-
-/*
-** The interface to the virtual-table mechanism defined above (back up
-** to a comment remarkably similar to this one) is currently considered
-** to be experimental.  The interface might change in incompatible ways.
-** If this is a problem for you, do not use the interface at this time.
-**
-** When the virtual-table mechanism stabilizes, we will declare the
-** interface fixed, support it indefinitely, and remove this comment.
-*/
+SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg);
 
 /*
 ** CAPI3REF: A Handle To An Open BLOB
@@ -6220,7 +8060,7 @@ typedef struct sqlite3_blob sqlite3_blob;
 **     SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow;
 **
    )^ ** -** ^(Parameter zDb is not the filename that contains the database, but +** ^(Parameter zDb is not the filename that contains the database, but ** rather the symbolic name of the database. For attached databases, this is ** the name that appears after the AS keyword in the [ATTACH] statement. ** For the main database file, the database name is "main". For TEMP @@ -6233,29 +8073,35 @@ typedef struct sqlite3_blob sqlite3_blob; ** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored ** in *ppBlob. Otherwise an [error code] is returned and, unless the error ** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided -** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** the API is not misused, it is always safe to call [sqlite3_blob_close()] ** on *ppBlob after this function it returns. ** ** This function fails with SQLITE_ERROR if any of the following are true: **
      -**
    • ^(Database zDb does not exist)^, -**
    • ^(Table zTable does not exist within database zDb)^, -**
    • ^(Table zTable is a WITHOUT ROWID table)^, +**
    • ^(Database zDb does not exist)^, +**
    • ^(Table zTable does not exist within database zDb)^, +**
    • ^(Table zTable is a WITHOUT ROWID table)^, **
    • ^(Column zColumn does not exist)^, **
    • ^(Row iRow is not present in the table)^, **
    • ^(The specified column of row iRow contains a value that is not ** a TEXT or BLOB value)^, -**
    • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +**
    • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE ** constraint and the blob is being opened for read/write access)^, -**
    • ^([foreign key constraints | Foreign key constraints] are enabled, +**
    • ^([foreign key constraints | Foreign key constraints] are enabled, ** column zColumn is part of a [child key] definition and the blob is ** being opened for read/write access)^. **
    ** -** ^Unless it returns SQLITE_MISUSE, this function sets the -** [database connection] error code and message accessible via -** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** +** A BLOB referenced by sqlite3_blob_open() may be read using the +** [sqlite3_blob_read()] interface and modified by using +** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a +** different row of the same table using the [sqlite3_blob_reopen()] +** interface. However, the column, table, or database of a [BLOB handle] +** cannot be changed after the [BLOB handle] is opened. ** ** ^(If the row that a BLOB handle points to is modified by an ** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects @@ -6274,13 +8120,17 @@ typedef struct sqlite3_blob sqlite3_blob; ** blob. ** ** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces -** and the built-in [zeroblob] SQL function may be used to create a +** and the built-in [zeroblob] SQL function may be used to create a ** zero-filled blob to read or write using the incremental-blob interface. ** ** To avoid a resource leak, every open [BLOB handle] should eventually ** be released by a call to [sqlite3_blob_close()]. +** +** See also: [sqlite3_blob_close()], +** [sqlite3_blob_reopen()], [sqlite3_blob_read()], +** [sqlite3_blob_bytes()], [sqlite3_blob_write()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( +SQLITE_API int sqlite3_blob_open( sqlite3*, const char *zDb, const char *zTable, @@ -6294,11 +8144,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( ** CAPI3REF: Move a BLOB Handle to a New Row ** METHOD: sqlite3_blob ** -** ^This function is used to move an existing blob handle so that it points +** ^This function is used to move an existing [BLOB handle] so that it points ** to a different row of the same database table. ^The new row is identified ** by the rowid value passed as the second argument. Only the row can be ** changed. ^The database, table and column on which the blob handle is open -** remain the same. Moving an existing blob handle to a new row can be +** remain the same. Moving an existing [BLOB handle] to a new row is ** faster than closing the existing handle and opening a new one. ** ** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - @@ -6313,14 +8163,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( ** ** ^This function sets the database handle error code and message. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* ** CAPI3REF: Close A BLOB Handle ** DESTRUCTOR: sqlite3_blob ** ** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed -** unconditionally. Even if this routine returns an error code, the +** unconditionally. Even if this routine returns an error code, the ** handle is still closed.)^ ** ** ^If the blob handle being closed was opened for read-write access, and if @@ -6330,19 +8180,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64) ** code is returned and the transaction rolled back. ** ** Calling this function with an argument that is not a NULL pointer or an -** open blob handle results in undefined behaviour. ^Calling this routine -** with a null pointer (such as would be returned by a failed call to +** open blob handle results in undefined behavior. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to ** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function -** is passed a valid open blob handle, the values returned by the +** is passed a valid open blob handle, the values returned by the ** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *); +SQLITE_API int sqlite3_blob_close(sqlite3_blob *); /* ** CAPI3REF: Return The Size Of An Open BLOB ** METHOD: sqlite3_blob ** -** ^Returns the size in bytes of the BLOB accessible via the +** ^Returns the size in bytes of the BLOB accessible via the ** successfully opened [BLOB handle] in its only argument. ^The ** incremental blob I/O routines can only read or overwriting existing ** blob content; they cannot change the size of a blob. @@ -6352,7 +8202,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *); ** been closed by [sqlite3_blob_close()]. Passing any other pointer in ** to this routine results in undefined and probably undesirable behavior. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *); +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); /* ** CAPI3REF: Read Data From A BLOB Incrementally @@ -6381,7 +8231,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *); ** ** See also: [sqlite3_blob_write()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); +SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); /* ** CAPI3REF: Write Data Into A BLOB Incrementally @@ -6393,9 +8243,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, ** ** ^(On success, sqlite3_blob_write() returns SQLITE_OK. ** Otherwise, an [error code] or an [extended error code] is returned.)^ -** ^Unless SQLITE_MISUSE is returned, this function sets the -** [database connection] error code and message accessible via -** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. ** ** ^If the [BLOB handle] passed as the first argument was not opened for ** writing (the flags parameter to [sqlite3_blob_open()] was zero), @@ -6404,9 +8254,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, ** This function may only modify the contents of the BLOB; it is ** not possible to increase the size of a BLOB using this API. ** ^If offset iOffset is less than N bytes from the end of the BLOB, -** [SQLITE_ERROR] is returned and no data is written. The size of the -** BLOB (and hence the maximum value of N+iOffset) can be determined -** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less ** than zero [SQLITE_ERROR] is returned and no data is written. ** ** ^An attempt to write to an expired [BLOB handle] fails with an @@ -6423,7 +8273,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *, void *Z, int N, ** ** See also: [sqlite3_blob_read()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); +SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); /* ** CAPI3REF: Virtual File System Objects @@ -6454,9 +8304,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *, const void *z, ** ^(If the default VFS is unregistered, another VFS is chosen as ** the default. The choice for the new VFS is arbitrary.)^ */ -SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfsName); -SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); -SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*); +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); /* ** CAPI3REF: Mutexes @@ -6500,7 +8350,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*); **
      **
    • SQLITE_MUTEX_FAST **
    • SQLITE_MUTEX_RECURSIVE -**
    • SQLITE_MUTEX_STATIC_MASTER +**
    • SQLITE_MUTEX_STATIC_MAIN **
    • SQLITE_MUTEX_STATIC_MEM **
    • SQLITE_MUTEX_STATIC_OPEN **
    • SQLITE_MUTEX_STATIC_PRNG @@ -6557,26 +8407,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs*); ** ** ^(Some systems (for example, Windows 95) do not support the operation ** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() -** will always return SQLITE_BUSY. The SQLite core only ever uses -** sqlite3_mutex_try() as an optimization so this is acceptable -** behavior.)^ +** will always return SQLITE_BUSY. In most cases the SQLite core only uses +** sqlite3_mutex_try() as an optimization, so this is acceptable +** behavior. The exceptions are unix builds that set the +** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working +** sqlite3_mutex_try() is required.)^ ** ** ^The sqlite3_mutex_leave() routine exits a mutex that was ** previously entered by the same thread. The behavior ** is undefined if the mutex is not currently entered by the ** calling thread or is not currently allocated. ** -** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), or -** sqlite3_mutex_leave() is a NULL pointer, then all three routines -** behave as no-ops. +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. ** ** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. */ -SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int); -SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex*); -SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex*); -SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex*); -SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex*); +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); /* ** CAPI3REF: Mutex Methods Object @@ -6623,7 +8475,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex*); ** The only difference is that the public sqlite3_XXX functions enumerated ** above silently ignore any invocations that pass a NULL pointer instead ** of a valid mutex handle. The implementations of the methods defined -** by this structure are not required to handle this case, the results +** by this structure are not required to handle this case. The results ** of passing a NULL pointer instead of a valid mutex handle are undefined ** (i.e. it is acceptable to provide an implementation that segfaults if ** it is passed a NULL pointer). @@ -6686,8 +8538,8 @@ struct sqlite3_mutex_methods { ** interface should also return 1 when given a NULL pointer. */ #ifndef NDEBUG -SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex*); -SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); #endif /* @@ -6702,11 +8554,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); */ #define SQLITE_MUTEX_FAST 0 #define SQLITE_MUTEX_RECURSIVE 1 -#define SQLITE_MUTEX_STATIC_MASTER 2 +#define SQLITE_MUTEX_STATIC_MAIN 2 #define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ #define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ #define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ -#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_random() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_randomness() */ #define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ #define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ #define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ @@ -6717,21 +8569,26 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex*); #define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ #define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ +/* Legacy compatibility: */ +#define SQLITE_MUTEX_STATIC_MASTER 2 + + /* ** CAPI3REF: Retrieve the mutex for a database connection ** METHOD: sqlite3 ** -** ^This interface returns a pointer the [sqlite3_mutex] object that +** ^This interface returns a pointer the [sqlite3_mutex] object that ** serializes access to the [database connection] given in the argument ** when the [threading mode] is Serialized. ** ^If the [threading mode] is Single-thread or Multi-thread then this ** routine returns a NULL pointer. */ -SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3*); +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); /* ** CAPI3REF: Low-Level Control Of Database Files ** METHOD: sqlite3 +** KEYWORDS: {file control} ** ** ^The [sqlite3_file_control()] interface makes a direct call to the ** xFileControl method for the [sqlite3_io_methods] object associated @@ -6746,11 +8603,18 @@ SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3*); ** the xFileControl method. ^The return value of the xFileControl ** method becomes the return value of this routine. ** -** ^The SQLITE_FCNTL_FILE_POINTER value for the op parameter causes +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. +** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes ** a pointer to the underlying [sqlite3_file] object to be written into -** the space pointed to by the 4th parameter. ^The SQLITE_FCNTL_FILE_POINTER -** case is a short-circuit path which does not actually invoke the -** underlying sqlite3_io_methods.xFileControl method. +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. ** ** ^If the second parameter (zDbName) does not match the name of any ** open database file, then SQLITE_ERROR is returned. ^This error @@ -6760,9 +8624,9 @@ SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3*); ** an incorrect zDbName and an SQLITE_ERROR return from the underlying ** xFileControl method. ** -** See also: [SQLITE_FCNTL_LOCKSTATE] +** See also: [file control opcodes] */ -SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); +SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); /* ** CAPI3REF: Testing Interface @@ -6781,7 +8645,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3*, const char *zDbName ** Unlike most of the SQLite API, this function is not guaranteed to ** operate consistently from one release to the next. */ -SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...); +SQLITE_API int sqlite3_test_control(int op, ...); /* ** CAPI3REF: Testing Interface Operation Codes @@ -6797,26 +8661,223 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...); #define SQLITE_TESTCTRL_FIRST 5 #define SQLITE_TESTCTRL_PRNG_SAVE 5 #define SQLITE_TESTCTRL_PRNG_RESTORE 6 -#define SQLITE_TESTCTRL_PRNG_RESET 7 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_FK_NO_ACTION 7 #define SQLITE_TESTCTRL_BITVEC_TEST 8 #define SQLITE_TESTCTRL_FAULT_INSTALL 9 #define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 #define SQLITE_TESTCTRL_PENDING_BYTE 11 #define SQLITE_TESTCTRL_ASSERT 12 #define SQLITE_TESTCTRL_ALWAYS 13 -#define SQLITE_TESTCTRL_RESERVE 14 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_JSON_SELFCHECK 14 #define SQLITE_TESTCTRL_OPTIMIZATIONS 15 -#define SQLITE_TESTCTRL_ISKEYWORD 16 -#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 +#define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_GETOPT 16 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +#define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 #define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 #define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 #define SQLITE_TESTCTRL_NEVER_CORRUPT 20 #define SQLITE_TESTCTRL_VDBE_COVERAGE 21 #define SQLITE_TESTCTRL_BYTEORDER 22 #define SQLITE_TESTCTRL_ISINIT 23 #define SQLITE_TESTCTRL_SORTER_MMAP 24 #define SQLITE_TESTCTRL_IMPOSTER 25 -#define SQLITE_TESTCTRL_LAST 25 +#define SQLITE_TESTCTRL_PARSER_COVERAGE 26 +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_SEEK_COUNT 30 +#define SQLITE_TESTCTRL_TRACEFLAGS 31 +#define SQLITE_TESTCTRL_TUNE 32 +#define SQLITE_TESTCTRL_LOGEST 33 +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ + +/* +** CAPI3REF: SQL Keyword Checking +** +** These routines provide access to the set of SQL language keywords +** recognized by SQLite. Applications can uses these routines to determine +** whether or not a specific identifier needs to be escaped (for example, +** by enclosing in double-quotes) so as not to confuse the parser. +** +** The sqlite3_keyword_count() interface returns the number of distinct +** keywords understood by SQLite. +** +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and +** makes *Z point to that keyword expressed as UTF8 and writes the number +** of bytes in the keyword into *L. The string that *Z points to is not +** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns +** SQLITE_OK if N is within bounds and SQLITE_ERROR if not. If either Z +** or L are NULL or invalid pointers then calls to +** sqlite3_keyword_name(N,Z,L) result in undefined behavior. +** +** The sqlite3_keyword_check(Z,L) interface checks to see whether or not +** the L-byte UTF8 identifier that Z points to is a keyword, returning non-zero +** if it is and zero if not. +** +** The parser used by SQLite is forgiving. It is often possible to use +** a keyword as an identifier as long as such use does not result in a +** parsing ambiguity. For example, the statement +** "CREATE TABLE BEGIN(REPLACE,PRAGMA,END);" is accepted by SQLite, and +** creates a new table named "BEGIN" with three columns named +** "REPLACE", "PRAGMA", and "END". Nevertheless, best practice is to avoid +** using keywords as identifiers. Common techniques used to avoid keyword +** name collisions include: +**
        +**
      • Put all identifier names inside double-quotes. This is the official +** SQL way to escape identifier names. +**
      • Put identifier names inside [...]. This is not standard SQL, +** but it is what SQL Server does and so lots of programmers use this +** technique. +**
      • Begin every identifier with the letter "Z" as no SQL keywords start +** with "Z". +**
      • Include a digit somewhere in every identifier name. +**
      +** +** Note that the number of keywords understood by SQLite can depend on +** compile-time options. For example, "VACUUM" is not a keyword if +** SQLite is compiled with the [-DSQLITE_OMIT_VACUUM] option. Also, +** new keywords may be added to future releases of SQLite. +*/ +SQLITE_API int sqlite3_keyword_count(void); +SQLITE_API int sqlite3_keyword_name(int,const char**,int*); +SQLITE_API int sqlite3_keyword_check(const char*,int); + +/* +** CAPI3REF: Dynamic String Object +** KEYWORDS: {dynamic string} +** +** An instance of the sqlite3_str object contains a dynamically-sized +** string under construction. +** +** The lifecycle of an sqlite3_str object is as follows: +**
        +**
      1. ^The sqlite3_str object is created using [sqlite3_str_new()]. +**
      2. ^Text is appended to the sqlite3_str object using various +** methods, such as [sqlite3_str_appendf()]. +**
      3. ^The sqlite3_str object is destroyed and the string it created +** is returned using the [sqlite3_str_finish()] interface. +**
      +*/ +typedef struct sqlite3_str sqlite3_str; + +/* +** CAPI3REF: Create A New Dynamic String Object +** CONSTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_new(D)] interface allocates and initializes +** a new [sqlite3_str] object. To avoid memory leaks, the object returned by +** [sqlite3_str_new()] must be freed by a subsequent call to +** [sqlite3_str_finish(X)]. +** +** ^The [sqlite3_str_new(D)] interface always returns a pointer to a +** valid [sqlite3_str] object, though in the event of an out-of-memory +** error the returned object might be a special singleton that will +** silently reject new text, always return SQLITE_NOMEM from +** [sqlite3_str_errcode()], always return 0 for +** [sqlite3_str_length()], and always return NULL from +** [sqlite3_str_finish(X)]. It is always safe to use the value +** returned by [sqlite3_str_new(D)] as the sqlite3_str parameter +** to any of the other [sqlite3_str] methods. +** +** The D parameter to [sqlite3_str_new(D)] may be NULL. If the +** D parameter in [sqlite3_str_new(D)] is not NULL, then the maximum +** length of the string contained in the [sqlite3_str] object will be +** the value set for [sqlite3_limit](D,[SQLITE_LIMIT_LENGTH]) instead +** of [SQLITE_MAX_LENGTH]. +*/ +SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3*); + +/* +** CAPI3REF: Finalize A Dynamic String +** DESTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_finish(X)] interface destroys the sqlite3_str object X +** and returns a pointer to a memory buffer obtained from [sqlite3_malloc64()] +** that contains the constructed string. The calling application should +** pass the returned value to [sqlite3_free()] to avoid a memory leak. +** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any +** errors were encountered during construction of the string. ^The +** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** string in [sqlite3_str] object X is zero bytes long. +*/ +SQLITE_API char *sqlite3_str_finish(sqlite3_str*); + +/* +** CAPI3REF: Add Content To A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces add content to an sqlite3_str object previously obtained +** from [sqlite3_str_new()]. +** +** ^The [sqlite3_str_appendf(X,F,...)] and +** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] +** functionality of SQLite to append formatted text onto the end of +** [sqlite3_str] object X. +** +** ^The [sqlite3_str_append(X,S,N)] method appends exactly N bytes from string S +** onto the end of the [sqlite3_str] object X. N must be non-negative. +** S must contain at least N non-zero bytes of content. To append a +** zero-terminated string in its entirety, use the [sqlite3_str_appendall()] +** method instead. +** +** ^The [sqlite3_str_appendall(X,S)] method appends the complete content of +** zero-terminated string S onto the end of [sqlite3_str] object X. +** +** ^The [sqlite3_str_appendchar(X,N,C)] method appends N copies of the +** single-byte character C onto the end of [sqlite3_str] object X. +** ^This method can be used, for example, to add whitespace indentation. +** +** ^The [sqlite3_str_reset(X)] method resets the string under construction +** inside [sqlite3_str] object X back to zero bytes in length. +** +** These methods do not return a result code. ^If an error occurs, that fact +** is recorded in the [sqlite3_str] object and can be recovered by a +** subsequent call to [sqlite3_str_errcode(X)]. +*/ +SQLITE_API void sqlite3_str_appendf(sqlite3_str*, const char *zFormat, ...); +SQLITE_API void sqlite3_str_vappendf(sqlite3_str*, const char *zFormat, va_list); +SQLITE_API void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); +SQLITE_API void sqlite3_str_appendall(sqlite3_str*, const char *zIn); +SQLITE_API void sqlite3_str_appendchar(sqlite3_str*, int N, char C); +SQLITE_API void sqlite3_str_reset(sqlite3_str*); + +/* +** CAPI3REF: Status Of A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces return the current status of an [sqlite3_str] object. +** +** ^If any prior errors have occurred while constructing the dynamic string +** in sqlite3_str X, then the [sqlite3_str_errcode(X)] method will return +** an appropriate error code. ^The [sqlite3_str_errcode(X)] method returns +** [SQLITE_NOMEM] following any out-of-memory error, or +** [SQLITE_TOOBIG] if the size of the dynamic string exceeds +** [SQLITE_MAX_LENGTH], or [SQLITE_OK] if there have been no errors. +** +** ^The [sqlite3_str_length(X)] method returns the current length, in bytes, +** of the dynamic string under construction in [sqlite3_str] object X. +** ^The length returned by [sqlite3_str_length(X)] does not include the +** zero-termination byte. +** +** ^The [sqlite3_str_value(X)] method returns a pointer to the current +** content of the dynamic string under construction in X. The value +** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X +** and might be freed or altered by any subsequent method on the same +** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str_value(X)] after any subsequent method call on the same +** object. ^Applications may change the content of the string returned +** by [sqlite3_str_value(X)] as long as they do not write into any bytes +** outside the range of 0 to [sqlite3_str_length(X)] and do not read or +** write any byte after any subsequent sqlite3_str method call. +*/ +SQLITE_API int sqlite3_str_errcode(sqlite3_str*); +SQLITE_API int sqlite3_str_length(sqlite3_str*); +SQLITE_API char *sqlite3_str_value(sqlite3_str*); /* ** CAPI3REF: SQLite Runtime Status @@ -6844,8 +8905,8 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...); ** ** See also: [sqlite3_db_status()] */ -SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); -SQLITE_API int SQLITE_STDCALL sqlite3_status64( +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int sqlite3_status64( int op, sqlite3_int64 *pCurrent, sqlite3_int64 *pHighwater, @@ -6865,8 +8926,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( **
      This parameter is the current amount of memory checked out ** using [sqlite3_malloc()], either directly or indirectly. The ** figure includes calls made to [sqlite3_malloc()] by the application -** and internal memory usage by the SQLite library. Scratch memory -** controlled by [SQLITE_CONFIG_SCRATCH] and auxiliary page-cache +** and internal memory usage by the SQLite library. Auxiliary page-cache ** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in ** this parameter. The amount returned is the sum of the allocation ** sizes as reported by the xSize method in [sqlite3_mem_methods].
      )^ @@ -6875,7 +8935,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( **
      This parameter records the largest memory allocation request ** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their ** internal equivalents). Only the value returned in the -** *pHighwater parameter to [sqlite3_status()] is of interest. +** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
      )^ ** ** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
      SQLITE_STATUS_MALLOC_COUNT
      @@ -6884,11 +8944,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( ** ** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
      SQLITE_STATUS_PAGECACHE_USED
      **
      This parameter returns the number of pages used out of the -** [pagecache memory allocator] that was configured using +** [pagecache memory allocator] that was configured using ** [SQLITE_CONFIG_PAGECACHE]. The ** value returned is in pages, not in bytes.
      )^ ** -** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] ** ^(
      SQLITE_STATUS_PAGECACHE_OVERFLOW
      **
      This parameter returns the number of bytes of page cache ** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] @@ -6900,36 +8960,21 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( ** ** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
      SQLITE_STATUS_PAGECACHE_SIZE
      **
      This parameter records the largest memory allocation request -** handed to [pagecache memory allocator]. Only the value returned in the -** *pHighwater parameter to [sqlite3_status()] is of interest. +** handed to the [pagecache memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. ** The value written into the *pCurrent parameter is undefined.
      )^ ** -** [[SQLITE_STATUS_SCRATCH_USED]] ^(
      SQLITE_STATUS_SCRATCH_USED
      -**
      This parameter returns the number of allocations used out of the -** [scratch memory allocator] configured using -** [SQLITE_CONFIG_SCRATCH]. The value returned is in allocations, not -** in bytes. Since a single thread may only have one scratch allocation -** outstanding at time, this parameter also reports the number of threads -** using scratch memory at the same time.
      )^ +** [[SQLITE_STATUS_SCRATCH_USED]]
      SQLITE_STATUS_SCRATCH_USED
      +**
      No longer used.
      ** ** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
      SQLITE_STATUS_SCRATCH_OVERFLOW
      -**
      This parameter returns the number of bytes of scratch memory -** allocation which could not be satisfied by the [SQLITE_CONFIG_SCRATCH] -** buffer and where forced to overflow to [sqlite3_malloc()]. The values -** returned include overflows because the requested allocation was too -** larger (that is, because the requested allocation was larger than the -** "sz" parameter to [SQLITE_CONFIG_SCRATCH]) and because no scratch buffer -** slots were available. -**
      )^ +**
      No longer used.
      ** -** [[SQLITE_STATUS_SCRATCH_SIZE]] ^(
      SQLITE_STATUS_SCRATCH_SIZE
      -**
      This parameter records the largest memory allocation request -** handed to [scratch memory allocator]. Only the value returned in the -** *pHighwater parameter to [sqlite3_status()] is of interest. -** The value written into the *pCurrent parameter is undefined.
      )^ +** [[SQLITE_STATUS_SCRATCH_SIZE]]
      SQLITE_STATUS_SCRATCH_SIZE
      +**
      No longer used.
      ** ** [[SQLITE_STATUS_PARSER_STACK]] ^(
      SQLITE_STATUS_PARSER_STACK
      -**
      The *pHighwater parameter records the deepest parser stack. +**
      The *pHighwater parameter records the deepest parser stack. ** The *pCurrent value is undefined. The *pHighwater value is only ** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
      )^ **
    @@ -6939,24 +8984,24 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( #define SQLITE_STATUS_MEMORY_USED 0 #define SQLITE_STATUS_PAGECACHE_USED 1 #define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 -#define SQLITE_STATUS_SCRATCH_USED 3 -#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 +#define SQLITE_STATUS_SCRATCH_USED 3 /* NOT USED */ +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 /* NOT USED */ #define SQLITE_STATUS_MALLOC_SIZE 5 #define SQLITE_STATUS_PARSER_STACK 6 #define SQLITE_STATUS_PAGECACHE_SIZE 7 -#define SQLITE_STATUS_SCRATCH_SIZE 8 +#define SQLITE_STATUS_SCRATCH_SIZE 8 /* NOT USED */ #define SQLITE_STATUS_MALLOC_COUNT 9 /* ** CAPI3REF: Database Connection Status ** METHOD: sqlite3 ** -** ^This interface is used to retrieve runtime status information +** ^This interface is used to retrieve runtime status information ** about a single [database connection]. ^The first argument is the ** database connection object to be interrogated. ^The second argument ** is an integer constant, taken from the set of ** [SQLITE_DBSTATUS options], that -** determines the parameter to interrogate. The set of +** determines the parameter to interrogate. The set of ** [SQLITE_DBSTATUS options] is likely ** to grow in future releases of SQLite. ** @@ -6970,7 +9015,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( ** ** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); +SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); /* ** CAPI3REF: Status Parameters for database connections @@ -6991,7 +9036,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int ** checked out.)^ ** ** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
    SQLITE_DBSTATUS_LOOKASIDE_HIT
    -**
    This parameter returns the number malloc attempts that were +**
    This parameter returns the number of malloc attempts that were ** satisfied using lookaside memory. Only the high-water value is meaningful; ** the current value is always zero.)^ ** @@ -7016,10 +9061,22 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int ** memory used by all pager caches associated with the database connection.)^ ** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. ** +** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] +** ^(
    SQLITE_DBSTATUS_CACHE_USED_SHARED
    +**
    This parameter is similar to DBSTATUS_CACHE_USED, except that if a +** pager cache is shared between two or more connections the bytes of heap +** memory used by that pager cache is divided evenly between the attached +** connections.)^ In other words, if none of the pager caches associated +** with the database connection are shared, this request returns the same +** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** shared, the value returned by this call will be smaller than that returned +** by DBSTATUS_CACHE_USED. ^The highwater mark associated with +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** ** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
    SQLITE_DBSTATUS_SCHEMA_USED
    **
    This parameter returns the approximate number of bytes of heap ** memory used to store the schema for all databases associated -** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ ** ^The full amount of memory used by the schemas is reported, even if the ** schema memory is shared with other database connections due to ** [shared cache mode] being enabled. @@ -7034,13 +9091,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int ** ** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
    SQLITE_DBSTATUS_CACHE_HIT
    **
    This parameter returns the number of pager cache hits that have -** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT ** is always 0. **
    ** ** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
    SQLITE_DBSTATUS_CACHE_MISS
    **
    This parameter returns the number of pager cache misses that have -** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS ** is always 0. **
    ** @@ -7055,6 +9112,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int ** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. ** ** +** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
    SQLITE_DBSTATUS_CACHE_SPILL
    +**
    This parameter returns the number of dirty cache entries that have +** been written to disk in the middle of a transaction due to the page +** cache overflowing. Transactions are more efficient if they are written +** to disk all at once. When pages spill mid-transaction, that introduces +** additional overhead. This parameter can be used help identify +** inefficiencies that can be resolved by increasing the cache size. +**
    +** ** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
    SQLITE_DBSTATUS_DEFERRED_FKS
    **
    This parameter returns zero for the current value if and only if ** all foreign key constraints (deferred or immediate) have been @@ -7073,7 +9139,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int #define SQLITE_DBSTATUS_CACHE_MISS 8 #define SQLITE_DBSTATUS_CACHE_WRITE 9 #define SQLITE_DBSTATUS_DEFERRED_FKS 10 -#define SQLITE_DBSTATUS_MAX 10 /* Largest defined DBSTATUS */ +#define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 +#define SQLITE_DBSTATUS_CACHE_SPILL 12 +#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ /* @@ -7087,7 +9155,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int ** statements. For example, if the number of table steps greatly exceeds ** the number of table searches or result rows, that would tend to indicate ** that the prepared statement is using a full table scan rather than -** an index. +** an index. ** ** ^(This interface is used to retrieve and reset counter values from ** a [prepared statement]. The first argument is the prepared statement @@ -7100,7 +9168,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status(sqlite3*, int op, int *pCur, int ** ** See also: [sqlite3_status()] and [sqlite3_db_status()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); /* ** CAPI3REF: Status Parameters for prepared statements @@ -7114,7 +9182,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int rese ** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
    SQLITE_STMTSTATUS_FULLSCAN_STEP
    **
    ^This is the number of times that SQLite has stepped forward in ** a table as part of a full table scan. Large numbers for this counter -** may indicate opportunities for performance improvement through +** may indicate opportunities for performance improvement through ** careful use of indices.
    ** ** [[SQLITE_STMTSTATUS_SORT]]
    SQLITE_STMTSTATUS_SORT
    @@ -7132,10 +9200,38 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int rese ** [[SQLITE_STMTSTATUS_VM_STEP]]
    SQLITE_STMTSTATUS_VM_STEP
    **
    ^This is the number of virtual machine operations executed ** by the prepared statement if that number is less than or equal -** to 2147483647. The number of virtual machine operations can be +** to 2147483647. The number of virtual machine operations can be ** used as a proxy for the total work done by the prepared statement. ** If the number of virtual machine operations exceeds 2147483647 ** then the value returned by this statement status code is undefined. +** +** [[SQLITE_STMTSTATUS_REPREPARE]]
    SQLITE_STMTSTATUS_REPREPARE
    +**
    ^This is the number of times that the prepare statement has been +** automatically regenerated due to schema changes or changes to +** [bound parameters] that might affect the query plan. +** +** [[SQLITE_STMTSTATUS_RUN]]
    SQLITE_STMTSTATUS_RUN
    +**
    ^This is the number of times that the prepared statement has +** been run. A single "run" for the purposes of this counter is one +** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. +** The counter is incremented on the first [sqlite3_step()] call of each +** cycle. +** +** [[SQLITE_STMTSTATUS_FILTER_MISS]] +** [[SQLITE_STMTSTATUS_FILTER HIT]] +**
    SQLITE_STMTSTATUS_FILTER_HIT
    +** SQLITE_STMTSTATUS_FILTER_MISS
    +**
    ^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join +** step was bypassed because a Bloom filter returned not-found. The +** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of +** times that the Bloom filter returned a find, and thus the join step +** had to be processed as normal. +** +** [[SQLITE_STMTSTATUS_MEMUSED]]
    SQLITE_STMTSTATUS_MEMUSED
    +**
    ^This is the approximate number of bytes of heap memory +** used to store the prepared statement. ^This value is not actually +** a counter, and so the resetFlg parameter to sqlite3_stmt_status() +** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. **
    ** */ @@ -7143,6 +9239,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt*, int op,int rese #define SQLITE_STMTSTATUS_SORT 2 #define SQLITE_STMTSTATUS_AUTOINDEX 3 #define SQLITE_STMTSTATUS_VM_STEP 4 +#define SQLITE_STMTSTATUS_REPREPARE 5 +#define SQLITE_STMTSTATUS_RUN 6 +#define SQLITE_STMTSTATUS_FILTER_MISS 7 +#define SQLITE_STMTSTATUS_FILTER_HIT 8 +#define SQLITE_STMTSTATUS_MEMUSED 99 /* ** CAPI3REF: Custom Page Cache Object @@ -7178,15 +9279,15 @@ struct sqlite3_pcache_page { ** KEYWORDS: {page cache} ** ** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can -** register an alternative page cache implementation by passing in an +** register an alternative page cache implementation by passing in an ** instance of the sqlite3_pcache_methods2 structure.)^ -** In many applications, most of the heap memory allocated by +** In many applications, most of the heap memory allocated by ** SQLite is used for the page cache. -** By implementing a +** By implementing a ** custom page cache using this API, an application can better control -** the amount of memory consumed by SQLite, the way in which -** that memory is allocated and released, and the policies used to -** determine exactly which parts of a database file are cached and for +** the amount of memory consumed by SQLite, the way in which +** that memory is allocated and released, and the policies used to +** determine exactly which parts of a database file are cached and for ** how long. ** ** The alternative page cache mechanism is an @@ -7199,19 +9300,19 @@ struct sqlite3_pcache_page { ** [sqlite3_config()] returns.)^ ** ** [[the xInit() page cache method]] -** ^(The xInit() method is called once for each effective +** ^(The xInit() method is called once for each effective ** call to [sqlite3_initialize()])^ ** (usually only once during the lifetime of the process). ^(The xInit() ** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ -** The intent of the xInit() method is to set up global data structures -** required by the custom page cache implementation. -** ^(If the xInit() method is NULL, then the +** The intent of the xInit() method is to set up global data structures +** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the ** built-in default page cache is used instead of the application defined ** page cache.)^ ** ** [[the xShutdown() page cache method]] ** ^The xShutdown() method is called by [sqlite3_shutdown()]. -** It can be used to clean up +** It can be used to clean up ** any outstanding resources before process shutdown, if required. ** ^The xShutdown() method may be NULL. ** @@ -7230,7 +9331,7 @@ struct sqlite3_pcache_page { ** though this is not guaranteed. ^The ** first parameter, szPage, is the size in bytes of the pages that must ** be allocated by the cache. ^szPage will always a power of two. ^The -** second parameter szExtra is a number of bytes of extra storage +** second parameter szExtra is a number of bytes of extra storage ** associated with each page cache entry. ^The szExtra parameter will ** a number less than 250. SQLite will use the ** extra szExtra bytes on each page to store metadata about the underlying @@ -7243,7 +9344,7 @@ struct sqlite3_pcache_page { ** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will ** never invoke xUnpin() except to deliberately delete a page. ** ^In other words, calls to xUnpin() on a cache with bPurgeable set to -** false will always have the "discard" flag set to true. +** false will always have the "discard" flag set to true. ** ^Hence, a cache created with bPurgeable false will ** never contain any unpinned pages. ** @@ -7258,12 +9359,12 @@ struct sqlite3_pcache_page { ** [[the xPagecount() page cache methods]] ** The xPagecount() method must return the number of pages currently ** stored in the cache, both pinned and unpinned. -** +** ** [[the xFetch() page cache methods]] -** The xFetch() method locates a page in the cache and returns a pointer to +** The xFetch() method locates a page in the cache and returns a pointer to ** an sqlite3_pcache_page object associated with that page, or a NULL pointer. ** The pBuf element of the returned sqlite3_pcache_page object will be a -** pointer to a buffer of szPage bytes used to store the content of a +** pointer to a buffer of szPage bytes used to store the content of a ** single database page. The pExtra element of sqlite3_pcache_page will be ** a pointer to the szExtra bytes of extra storage that SQLite has requested ** for each entry in the page cache. @@ -7289,7 +9390,7 @@ struct sqlite3_pcache_page { ** ** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite ** will only use a createFlag of 2 after a prior call with a createFlag of 1 -** failed.)^ In between the to xFetch() calls, SQLite may +** failed.)^ In between the xFetch() calls, SQLite may ** attempt to unpin one or more cache pages by spilling the content of ** pinned pages to disk and synching the operating system disk cache. ** @@ -7302,8 +9403,8 @@ struct sqlite3_pcache_page { ** page cache implementation. ^The page cache implementation ** may choose to evict unpinned pages at any time. ** -** The cache must not perform any reference counting. A single -** call to xUnpin() unpins the page regardless of the number of prior calls +** The cache must not perform any reference counting. A single +** call to xUnpin() unpins the page regardless of the number of prior calls ** to xFetch(). ** ** [[the xRekey() page cache methods]] @@ -7343,7 +9444,7 @@ struct sqlite3_pcache_methods2 { int (*xPagecount)(sqlite3_pcache*); sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); - void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, unsigned oldKey, unsigned newKey); void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); void (*xDestroy)(sqlite3_pcache*); @@ -7388,7 +9489,7 @@ typedef struct sqlite3_backup sqlite3_backup; ** ** The backup API copies the content of one database into another. ** It is useful either for creating backups of databases or -** for copying in-memory databases to or from persistent files. +** for copying in-memory databases to or from persistent files. ** ** See Also: [Using the SQLite Online Backup API] ** @@ -7399,36 +9500,36 @@ typedef struct sqlite3_backup sqlite3_backup; ** ^Thus, the backup may be performed on a live source database without ** preventing other database connections from ** reading or writing to the source database while the backup is underway. -** -** ^(To perform a backup operation: +** +** ^(To perform a backup operation: **
      **
    1. sqlite3_backup_init() is called once to initialize the -** backup, -**
    2. sqlite3_backup_step() is called one or more times to transfer +** backup, +**
    3. sqlite3_backup_step() is called one or more times to transfer ** the data between the two databases, and finally -**
    4. sqlite3_backup_finish() is called to release all resources -** associated with the backup operation. +**
    5. sqlite3_backup_finish() is called to release all resources +** associated with the backup operation. **
    )^ ** There should be exactly one call to sqlite3_backup_finish() for each ** successful call to sqlite3_backup_init(). ** ** [[sqlite3_backup_init()]] sqlite3_backup_init() ** -** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the -** [database connection] associated with the destination database +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database ** and the database name, respectively. ** ^The database name is "main" for the main database, "temp" for the ** temporary database, or the name specified after the AS keyword in ** an [ATTACH] statement for an attached database. -** ^The S and M arguments passed to +** ^The S and M arguments passed to ** sqlite3_backup_init(D,N,S,M) identify the [database connection] ** and database name of the source database, respectively. ** ^The source and destination [database connections] (parameters S and D) ** must be different or else sqlite3_backup_init(D,N,S,M) will fail with ** an error. ** -** ^A call to sqlite3_backup_init() will fail, returning NULL, if -** there is already a read or read-write transaction open on the +** ^A call to sqlite3_backup_init() will fail, returning NULL, if +** there is already a read or read-write transaction open on the ** destination database. ** ** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is @@ -7440,14 +9541,14 @@ typedef struct sqlite3_backup sqlite3_backup; ** ^A successful call to sqlite3_backup_init() returns a pointer to an ** [sqlite3_backup] object. ** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and -** sqlite3_backup_finish() functions to perform the specified backup +** sqlite3_backup_finish() functions to perform the specified backup ** operation. ** ** [[sqlite3_backup_step()]] sqlite3_backup_step() ** -** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between ** the source and destination databases specified by [sqlite3_backup] object B. -** ^If N is negative, all remaining source pages are copied. +** ^If N is negative, all remaining source pages are copied. ** ^If sqlite3_backup_step(B,N) successfully copies N pages and there ** are still more pages to be copied, then the function returns [SQLITE_OK]. ** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages @@ -7469,8 +9570,8 @@ typedef struct sqlite3_backup sqlite3_backup; ** ** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then ** the [sqlite3_busy_handler | busy-handler function] -** is invoked (if one is specified). ^If the -** busy-handler returns non-zero before the lock is available, then +** is invoked (if one is specified). ^If the +** busy-handler returns non-zero before the lock is available, then ** [SQLITE_BUSY] is returned to the caller. ^In this case the call to ** sqlite3_backup_step() can be retried later. ^If the source ** [database connection] @@ -7478,15 +9579,15 @@ typedef struct sqlite3_backup sqlite3_backup; ** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this ** case the call to sqlite3_backup_step() can be retried later on. ^(If ** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or -** [SQLITE_READONLY] is returned, then -** there is no point in retrying the call to sqlite3_backup_step(). These -** errors are considered fatal.)^ The application must accept -** that the backup operation has failed and pass the backup operation handle +** [SQLITE_READONLY] is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal.)^ The application must accept +** that the backup operation has failed and pass the backup operation handle ** to the sqlite3_backup_finish() to release associated resources. ** ** ^The first call to sqlite3_backup_step() obtains an exclusive lock -** on the destination file. ^The exclusive lock is not released until either -** sqlite3_backup_finish() is called or the backup operation is complete +** on the destination file. ^The exclusive lock is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete ** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to ** sqlite3_backup_step() obtains a [shared lock] on the source database that ** lasts for the duration of the sqlite3_backup_step() call. @@ -7495,18 +9596,18 @@ typedef struct sqlite3_backup sqlite3_backup; ** through the backup process. ^If the source database is modified by an ** external process or via a database connection other than the one being ** used by the backup operation, then the backup will be automatically -** restarted by the next call to sqlite3_backup_step(). ^If the source +** restarted by the next call to sqlite3_backup_step(). ^If the source ** database is modified by the using the same database connection as is used ** by the backup operation, then the backup database is automatically ** updated at the same time. ** ** [[sqlite3_backup_finish()]] sqlite3_backup_finish() ** -** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the ** application wishes to abandon the backup operation, the application ** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). ** ^The sqlite3_backup_finish() interfaces releases all -** resources associated with the [sqlite3_backup] object. +** resources associated with the [sqlite3_backup] object. ** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any ** active write-transaction on the destination database is rolled back. ** The [sqlite3_backup] object is invalid @@ -7546,39 +9647,49 @@ typedef struct sqlite3_backup sqlite3_backup; ** connections, then the source database connection may be used concurrently ** from within other threads. ** -** However, the application must guarantee that the destination -** [database connection] is not passed to any other API (by any thread) after +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after ** sqlite3_backup_init() is called and before the corresponding call to ** sqlite3_backup_finish(). SQLite does not currently check to see ** if the application incorrectly accesses the destination [database connection] ** and so no error code is reported, but the operations may malfunction ** nevertheless. Use of the destination database connection while a -** backup is in progress might also also cause a mutex deadlock. +** backup is in progress might also cause a mutex deadlock. ** ** If running in [shared cache mode], the application must ** guarantee that the shared cache used by the destination database ** is not accessed while the backup is running. In practice this means -** that the application must guarantee that the disk file being +** that the application must guarantee that the disk file being ** backed up to is not accessed by any connection within the process, ** not just the specific connection that was passed to sqlite3_backup_init(). ** -** The [sqlite3_backup] object itself is partially threadsafe. Multiple +** The [sqlite3_backup] object itself is partially threadsafe. Multiple ** threads may safely make multiple concurrent calls to sqlite3_backup_step(). ** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() ** APIs are not strictly speaking threadsafe. If they are invoked at the ** same time as another thread is invoking sqlite3_backup_step() it is ** possible that they return invalid values. +** +** Alternatives To Using The Backup API +** +** Other techniques for safely creating a consistent backup of an SQLite +** database include: +** +**
      +**
    • The [VACUUM INTO] command. +**
    • The [sqlite3_rsync] utility program. +**
    */ -SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( +SQLITE_API sqlite3_backup *sqlite3_backup_init( sqlite3 *pDest, /* Destination database handle */ const char *zDestName, /* Destination database name */ sqlite3 *pSource, /* Source database handle */ const char *zSourceName /* Source database name */ ); -SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage); -SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p); -SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p); -SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); /* ** CAPI3REF: Unlock Notification @@ -7587,8 +9698,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); ** ^When running in shared-cache mode, a database operation may fail with ** an [SQLITE_LOCKED] error if the required locks on the shared-cache or ** individual tables within the shared-cache cannot be obtained. See -** [SQLite Shared-Cache Mode] for a description of shared-cache locking. -** ^This API may be used to register a callback that SQLite will invoke +** [SQLite Shared-Cache Mode] for a description of shared-cache locking. +** ^This API may be used to register a callback that SQLite will invoke ** when the connection currently holding the required lock relinquishes it. ** ^This API is only available if the library was compiled with the ** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. @@ -7596,18 +9707,18 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); ** See Also: [Using the SQLite Unlock Notification Feature]. ** ** ^Shared-cache locks are released when a database connection concludes -** its current transaction, either by committing it or rolling it back. +** its current transaction, either by committing it or rolling it back. ** ** ^When a connection (known as the blocked connection) fails to obtain a ** shared-cache lock and SQLITE_LOCKED is returned to the caller, the ** identity of the database connection (the blocking connection) that -** has locked the required resource is stored internally. ^After an +** has locked the required resource is stored internally. ^After an ** application receives an SQLITE_LOCKED error, it may call the -** sqlite3_unlock_notify() method with the blocked connection handle as +** sqlite3_unlock_notify() method with the blocked connection handle as ** the first argument to register for a callback that will be invoked ** when the blocking connections current transaction is concluded. ^The ** callback is invoked from within the [sqlite3_step] or [sqlite3_close] -** call that concludes the blocking connections transaction. +** call that concludes the blocking connection's transaction. ** ** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, ** there is a chance that the blocking connection will have already @@ -7617,15 +9728,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); ** ** ^If the blocked connection is attempting to obtain a write-lock on a ** shared-cache table, and more than one other connection currently holds -** a read-lock on the same table, then SQLite arbitrarily selects one of +** a read-lock on the same table, then SQLite arbitrarily selects one of ** the other connections to use as the blocking connection. ** -** ^(There may be at most one unlock-notify callback registered by a +** ^(There may be at most one unlock-notify callback registered by a ** blocked connection. If sqlite3_unlock_notify() is called when the ** blocked connection already has a registered unlock-notify callback, ** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is ** called with a NULL pointer as its second argument, then any existing -** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback is canceled. ^The blocked connections ** unlock-notify callback may also be canceled by closing the blocked ** connection using [sqlite3_close()]. ** @@ -7638,25 +9749,25 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); ** ** Callback Invocation Details ** -** When an unlock-notify callback is registered, the application provides a +** When an unlock-notify callback is registered, the application provides a ** single void* pointer that is passed to the callback when it is invoked. ** However, the signature of the callback function allows SQLite to pass ** it an array of void* context pointers. The first argument passed to ** an unlock-notify callback is a pointer to an array of void* pointers, ** and the second is the number of entries in the array. ** -** When a blocking connections transaction is concluded, there may be +** When a blocking connection's transaction is concluded, there may be ** more than one blocked connection that has registered for an unlock-notify ** callback. ^If two or more such blocked connections have specified the ** same callback function, then instead of invoking the callback function ** multiple times, it is invoked once with the set of void* context pointers ** specified by the blocked connections bundled together into an array. -** This gives the application an opportunity to prioritize any actions +** This gives the application an opportunity to prioritize any actions ** related to the set of unblocked database connections. ** ** Deadlock Detection ** -** Assuming that after registering for an unlock-notify callback a +** Assuming that after registering for an unlock-notify callback a ** database waits for the callback to be issued before taking any further ** action (a reasonable assumption), then using this API may cause the ** application to deadlock. For example, if connection X is waiting for @@ -7679,7 +9790,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); ** ** The "DROP TABLE" Exception ** -** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost +** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost ** always appropriate to call sqlite3_unlock_notify(). There is however, ** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, ** SQLite checks if there are any currently executing SELECT statements @@ -7692,10 +9803,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p); ** One way around this problem is to check the extended error code returned ** by an sqlite3_step() call. ^(If there is a blocking connection, then the ** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in -** the special "DROP TABLE/INDEX" case, the extended error code is just +** the special "DROP TABLE/INDEX" case, the extended error code is just ** SQLITE_LOCKED.)^ */ -SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( +SQLITE_API int sqlite3_unlock_notify( sqlite3 *pBlocked, /* Waiting connection */ void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ void *pNotifyArg /* Argument to pass to xNotify */ @@ -7710,8 +9821,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( ** strings in a case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *, const char *); -SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *, const char *, int); +SQLITE_API int sqlite3_stricmp(const char *, const char *); +SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); /* ** CAPI3REF: String Globbing @@ -7728,7 +9839,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *, const char *, int); ** ** See also: [sqlite3_strlike()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlob, const char *zStr); +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); /* ** CAPI3REF: String LIKE Matching @@ -7751,7 +9862,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlob, const char *zSt ** ** See also: [sqlite3_strglob()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); +SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); /* ** CAPI3REF: Error Logging Interface @@ -7774,7 +9885,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strlike(const char *zGlob, const char *zSt ** a few hundred characters, it will be truncated to the length of the ** buffer. */ -SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...); +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); /* ** CAPI3REF: Write-Ahead Log Commit Hook @@ -7783,8 +9894,8 @@ SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...) ** ^The [sqlite3_wal_hook()] function is used to register a callback that ** is invoked each time data is committed to a database in wal mode. ** -** ^(The callback is invoked by SQLite after the commit has taken place and -** the associated write-lock on the database released)^, so the implementation +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation ** may read, write or [checkpoint] the database as required. ** ** ^The first parameter passed to the callback function when it is invoked @@ -7803,15 +9914,16 @@ SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...) ** that does not correspond to any valid SQLite error code, the results ** are undefined. ** -** A single database handle may have at most a single write-ahead log callback +** A single database handle may have at most a single write-ahead log callback ** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any -** previously registered write-ahead log callback. ^Note that the -** [sqlite3_wal_autocheckpoint()] interface and the +** previously registered write-ahead log callback. ^The return value is +** a copy of the third parameter from the previous call, if any, or 0. +** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the ** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will ** overwrite any prior [sqlite3_wal_hook()] settings. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( - sqlite3*, +SQLITE_API void *sqlite3_wal_hook( + sqlite3*, int(*)(void *,sqlite3*,const char*,int), void* ); @@ -7824,7 +9936,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( ** [sqlite3_wal_hook()] that causes any database on [database connection] D ** to automatically [checkpoint] ** after committing a transaction if there are N or -** more frames in the [write-ahead log] file. ^Passing zero or +** more frames in the [write-ahead log] file. ^Passing zero or ** a negative value as the nFrame parameter disables automatic ** checkpoints entirely. ** @@ -7845,7 +9957,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( ** is only necessary if the default setting is found to be suboptimal ** for a particular application. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int N); +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); /* ** CAPI3REF: Checkpoint a database @@ -7854,7 +9966,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to ** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ ** -** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the ** [write-ahead log] for database X on [database connection] D to be ** transferred into the database file and for the write-ahead log to ** be reset. See the [checkpointing] documentation for addition @@ -7867,7 +9979,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int N); ** start a callback but which do not need the full power (and corresponding ** complication) of [sqlite3_wal_checkpoint_v2()]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); /* ** CAPI3REF: Checkpoint a database @@ -7880,10 +9992,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zD ** **
    **
    SQLITE_CHECKPOINT_PASSIVE
    -** ^Checkpoint as many frames as possible without waiting for any database -** readers or writers to finish, then sync the database file if all frames +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames ** in the log were checkpointed. ^The [busy-handler callback] -** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. ** ^On the other hand, passive mode might leave the checkpoint unfinished ** if there are concurrent readers or writers. ** @@ -7897,9 +10009,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zD ** **
    SQLITE_CHECKPOINT_RESTART
    ** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition -** that after checkpointing the log file it blocks (calls the +** that after checkpointing the log file it blocks (calls the ** [busy-handler callback]) -** until all readers are reading from the database file only. ^This ensures +** until all readers are reading from the database file only. ^This ensures ** that the next writer will restart the log file from the beginning. ** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new ** database writer attempts while it is pending, but does not impede readers. @@ -7921,31 +10033,31 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zD ** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. ** ** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If -** any other process is running a checkpoint operation at the same time, the -** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a ** busy-handler configured, it will not be invoked in this case. ** -** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the ** exclusive "writer" lock on the database file. ^If the writer lock cannot be ** obtained immediately, and a busy-handler is configured, it is invoked and ** the writer lock retried until either the busy-handler returns 0 or the lock ** is successfully obtained. ^The busy-handler is also invoked while waiting for ** database readers as described above. ^If the busy-handler returns 0 before ** the writer lock is obtained or while waiting for database readers, the -** checkpoint operation proceeds from that point in the same way as -** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible ** without blocking any further. ^SQLITE_BUSY is returned in this case. ** ** ^If parameter zDb is NULL or points to a zero length string, then the -** specified operation is attempted on all WAL databases [attached] to +** specified operation is attempted on all WAL databases [attached] to ** [database connection] db. In this case the -** values written to output parameters *pnLog and *pnCkpt are undefined. ^If -** an SQLITE_BUSY error is encountered when processing one or more of the -** attached WAL databases, the operation is still attempted on any remaining -** attached databases and SQLITE_BUSY is returned at the end. ^If any other -** error occurs while processing an attached database, processing is abandoned -** and the error code is returned to the caller immediately. ^If no error -** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned at the end. ^If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code is returned to the caller immediately. ^If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached ** databases, SQLITE_OK is returned. ** ** ^If database zDb is the name of an attached database that is not in WAL @@ -7961,7 +10073,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zD ** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface ** from SQL. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( +SQLITE_API int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of attached database (or NULL) */ int eMode, /* SQLITE_CHECKPOINT_* value */ @@ -7980,7 +10092,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( */ #define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ #define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ -#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for for readers */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ #define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ /* @@ -7993,21 +10105,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( ** If this interface is invoked outside the context of an xConnect or ** xCreate virtual table method then the behavior is undefined. ** -** At present, there is only one option that may be configured using -** this function. (See [SQLITE_VTAB_CONSTRAINT_SUPPORT].) Further options -** may be added in the future. +** In the call sqlite3_vtab_config(D,C,...) the D parameter is the +** [database connection] in which the virtual table is being created and +** which is passed in as the first argument to the [xConnect] or [xCreate] +** method that is invoking sqlite3_vtab_config(). The C parameter is one +** of the [virtual table configuration options]. The presence and meaning +** of parameters after C depend on which [virtual table configuration option] +** is used. */ -SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3*, int op, ...); +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); /* ** CAPI3REF: Virtual Table Configuration Options +** KEYWORDS: {virtual table configuration options} +** KEYWORDS: {virtual table configuration option} ** ** These macros define the various options to the ** [sqlite3_vtab_config()] interface that [virtual table] implementations ** can use to customize and optimize their behavior. ** **
    -**
    SQLITE_VTAB_CONSTRAINT_SUPPORT +** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] +**
    SQLITE_VTAB_CONSTRAINT_SUPPORT
    **
    Calls of the form ** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, ** where X is an integer. If X is zero, then the [virtual table] whose @@ -8021,24 +10140,56 @@ SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3*, int op, ...); ** If X is non-zero, then the virtual table implementation guarantees ** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before ** any modifications to internal or persistent data structures have been made. -** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite ** is able to roll back a statement or database transaction, and abandon -** or continue processing the current SQL statement as appropriate. +** or continue processing the current SQL statement as appropriate. ** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns ** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode ** had been ABORT. ** ** Virtual table implementations that are required to handle OR REPLACE -** must do so within the [xUpdate] method. If a call to the -** [sqlite3_vtab_on_conflict()] function indicates that the current ON -** CONFLICT policy is REPLACE, the virtual table implementation should +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should ** silently replace the appropriate rows within the xUpdate callback and ** return SQLITE_OK. Or, if this is not possible, it may return -** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT ** constraint handling. +**
    +** +** [[SQLITE_VTAB_DIRECTONLY]]
    SQLITE_VTAB_DIRECTONLY
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** prohibits that virtual table from being used from within triggers and +** views. +**
    +** +** [[SQLITE_VTAB_INNOCUOUS]]
    SQLITE_VTAB_INNOCUOUS
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** identify that virtual table as being safe to use from within triggers +** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the +** virtual table can do no serious harm even if it is controlled by a +** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS +** flag unless absolutely necessary. +**
    +** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
    SQLITE_VTAB_USES_ALL_SCHEMAS
    +**
    Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
    **
    */ #define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 +#define SQLITE_VTAB_INNOCUOUS 2 +#define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 /* ** CAPI3REF: Determine The Virtual Table Conflict Policy @@ -8050,7 +10201,325 @@ SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3*, int op, ...); ** of the SQL statement that triggered the call to the [xUpdate] method of the ** [virtual table]. */ -SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE +** +** If the sqlite3_vtab_nochange(X) routine is called within the [xColumn] +** method of a [virtual table], then it might return true if the +** column is being fetched as part of an UPDATE operation during which the +** column value will not change. The virtual table implementation can use +** this hint as permission to substitute a return value that is less +** expensive to compute and that the corresponding +** [xUpdate] method understands as a "no-change" value. +** +** If the [xColumn] method calls sqlite3_vtab_nochange() and finds that +** the column is not changed by the UPDATE statement, then the xColumn +** method can optionally return without setting a result, without calling +** any of the [sqlite3_result_int|sqlite3_result_xxxxx() interfaces]. +** In that case, [sqlite3_value_nochange(X)] will return true for the +** same column in the [xUpdate] method. +** +** The sqlite3_vtab_nochange() routine is an optimization. Virtual table +** implementations should continue to give a correct answer even if the +** sqlite3_vtab_nochange() interface were to always return false. In the +** current implementation, the sqlite3_vtab_nochange() interface does always +** returns false for the enhanced [UPDATE FROM] statement. +*/ +SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); + +/* +** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** METHOD: sqlite3_index_info +** +** This function may only be called from within a call to the [xBestIndex] +** method of a [virtual table]. This function returns a pointer to a string +** that is the name of the appropriate collation sequence to use for text +** comparisons on the constraint identified by its arguments. +** +** The first argument must be the pointer to the [sqlite3_index_info] object +** that is the first parameter to the xBestIndex() method. The second argument +** must be an index into the aConstraint[] array belonging to the +** sqlite3_index_info structure passed to xBestIndex. +** +** Important: +** The first parameter must be the same pointer that is passed into the +** xBestMethod() method. The first parameter may not be a pointer to a +** different [sqlite3_index_info] object, even an exact copy. +** +** The return value is computed as follows: +** +**
      +**

    1. If the constraint comes from a WHERE clause expression that contains +** a [COLLATE operator], then the name of the collation specified by +** that COLLATE operator is returned. +**

    2. If there is no COLLATE operator, but the column that is the subject +** of the constraint specifies an alternative collating sequence via +** a [COLLATE clause] on the column definition within the CREATE TABLE +** statement that was passed into [sqlite3_declare_vtab()], then the +** name of that alternative collating sequence is returned. +**

    3. Otherwise, "BINARY" is returned. +**

    +*/ +SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); + +/* +** CAPI3REF: Determine if a virtual table query is DISTINCT +** METHOD: sqlite3_index_info +** +** This API may only be used from within an [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this +** interface from outside of xBestIndex() is undefined and probably harmful. +** +** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and +** 3. The integer returned by sqlite3_vtab_distinct() +** gives the virtual table additional information about how the query +** planner wants the output to be ordered. As long as the virtual table +** can meet the ordering requirements of the query planner, it may set +** the "orderByConsumed" flag. +** +**

    1. +** ^If the sqlite3_vtab_distinct() interface returns 0, that means +** that the query planner needs the virtual table to return all rows in the +** sort order defined by the "nOrderBy" and "aOrderBy" fields of the +** [sqlite3_index_info] object. This is the default expectation. If the +** virtual table outputs all rows in sorted order, then it is always safe for +** the xBestIndex method to set the "orderByConsumed" flag, regardless of +** the return value from sqlite3_vtab_distinct(). +**

    2. +** ^(If the sqlite3_vtab_distinct() interface returns 1, that means +** that the query planner does not need the rows to be returned in sorted order +** as long as all rows with the same values in all columns identified by the +** "aOrderBy" field are adjacent.)^ This mode is used when the query planner +** is doing a GROUP BY. +**

    3. +** ^(If the sqlite3_vtab_distinct() interface returns 2, that means +** that the query planner does not need the rows returned in any particular +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for a DISTINCT query. +**

    4. +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries +** that have both DISTINCT and ORDER BY clauses. +**

    +** +**

    The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
    sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
    0yesyesno +**
    1noyesno +**
    2noyesyes +**
    3yesyesyes +**
    +** +** ^For the purposes of comparing virtual table output values to see if the +** values are same value for sorting purposes, two NULL values are considered +** to be the same. In other words, the comparison operator is "IS" +** (or "IS NOT DISTINCT FROM") and not "==". +** +** If a virtual table implementation is unable to meet the requirements +** specified above, then it must not set the "orderByConsumed" flag in the +** [sqlite3_index_info] object or an incorrect answer may result. +** +** ^A virtual table implementation is always free to return rows in any order +** it wants, as long as the "orderByConsumed" flag is not set. ^When the +** the "orderByConsumed" flag is unset, the query planner will add extra +** [bytecode] to ensure that the final results returned by the SQL query are +** ordered correctly. The use of the "orderByConsumed" flag and the +** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful +** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed" +** flag might help queries against a virtual table to run faster. Being +** overly aggressive and setting the "orderByConsumed" flag when it is not +** valid to do so, on the other hand, might cause SQLite to return incorrect +** results. +*/ +SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*); + +/* +** CAPI3REF: Identify and handle IN constraints in xBestIndex +** +** This interface may only be used from within an +** [xBestIndex|xBestIndex() method] of a [virtual table] implementation. +** The result of invoking this interface from any other context is +** undefined and probably harmful. +** +** ^(A constraint on a virtual table of the form +** "[IN operator|column IN (...)]" is +** communicated to the xBestIndex method as a +** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^ If xBestIndex wants to use +** this constraint, it must set the corresponding +** aConstraintUsage[].argvIndex to a positive integer. ^(Then, under +** the usual mode of handling IN operators, SQLite generates [bytecode] +** that invokes the [xFilter|xFilter() method] once for each value +** on the right-hand side of the IN operator.)^ Thus the virtual table +** only sees a single value from the right-hand side of the IN operator +** at a time. +** +** In some cases, however, it would be advantageous for the virtual +** table to see all values on the right-hand of the IN operator all at +** once. The sqlite3_vtab_in() interfaces facilitates this in two ways: +** +**

      +**

    1. +** ^A call to sqlite3_vtab_in(P,N,-1) will return true (non-zero) +** if and only if the [sqlite3_index_info|P->aConstraint][N] constraint +** is an [IN operator] that can be processed all at once. ^In other words, +** sqlite3_vtab_in() with -1 in the third argument is a mechanism +** by which the virtual table can ask SQLite if all-at-once processing +** of the IN operator is even possible. +** +**

    2. +** ^A call to sqlite3_vtab_in(P,N,F) with F==1 or F==0 indicates +** to SQLite that the virtual table does or does not want to process +** the IN operator all-at-once, respectively. ^Thus when the third +** parameter (F) is non-negative, this interface is the mechanism by +** which the virtual table tells SQLite how it wants to process the +** IN operator. +**

    +** +** ^The sqlite3_vtab_in(P,N,F) interface can be invoked multiple times +** within the same xBestIndex method call. ^For any given P,N pair, +** the return value from sqlite3_vtab_in(P,N,F) will always be the same +** within the same xBestIndex call. ^If the interface returns true +** (non-zero), that means that the constraint is an IN operator +** that can be processed all-at-once. ^If the constraint is not an IN +** operator or cannot be processed all-at-once, then the interface returns +** false. +** +** ^(All-at-once processing of the IN operator is selected if both of the +** following conditions are met: +** +**
      +**

    1. The P->aConstraintUsage[N].argvIndex value is set to a positive +** integer. This is how the virtual table tells SQLite that it wants to +** use the N-th constraint. +** +**

    2. The last call to sqlite3_vtab_in(P,N,F) for which F was +** non-negative had F>=1. +**

    )^ +** +** ^If either or both of the conditions above are false, then SQLite uses +** the traditional one-at-a-time processing strategy for the IN constraint. +** ^If both conditions are true, then the argvIndex-th parameter to the +** xFilter method will be an [sqlite3_value] that appears to be NULL, +** but which can be passed to [sqlite3_vtab_in_first()] and +** [sqlite3_vtab_in_next()] to find all values on the right-hand side +** of the IN constraint. +*/ +SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); + +/* +** CAPI3REF: Find all elements on the right-hand side of an IN constraint. +** +** These interfaces are only useful from within the +** [xFilter|xFilter() method] of a [virtual table] implementation. +** The result of invoking these interfaces from any other context +** is undefined and probably harmful. +** +** The X parameter in a call to sqlite3_vtab_in_first(X,P) or +** sqlite3_vtab_in_next(X,P) should be one of the parameters to the +** xFilter method which invokes these routines, and specifically +** a parameter that was previously selected for all-at-once IN constraint +** processing use the [sqlite3_vtab_in()] interface in the +** [xBestIndex|xBestIndex method]. ^(If the X parameter is not +** an xFilter argument that was selected for all-at-once IN constraint +** processing, then these routines return [SQLITE_ERROR].)^ +** +** ^(Use these routines to access all values on the right-hand side +** of the IN constraint using code like the following: +** +** 
    +**    for(rc=sqlite3_vtab_in_first(pList, &pVal);
+**        rc==SQLITE_OK && pVal;
+**        rc=sqlite3_vtab_in_next(pList, &pVal)
+**    ){
+**      // do something with pVal
+**    }
+**    if( rc!=SQLITE_OK ){
+**      // an error has occurred
+**    }
+**
    )^ +** +** ^On success, the sqlite3_vtab_in_first(X,P) and sqlite3_vtab_in_next(X,P) +** routines return SQLITE_OK and set *P to point to the first or next value +** on the RHS of the IN constraint. ^If there are no more values on the +** right hand side of the IN constraint, then *P is set to NULL and these +** routines return [SQLITE_DONE]. ^The return value might be +** some other value, such as SQLITE_NOMEM, in the event of a malfunction. +** +** The *ppOut values returned by these routines are only valid until the +** next call to either of these routines or until the end of the xFilter +** method from which these routines were called. If the virtual table +** implementation needs to retain the *ppOut values for longer, it must make +** copies. The *ppOut values are [protected sqlite3_value|protected]. +*/ +SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut); +SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); + +/* +** CAPI3REF: Constraint values in xBestIndex() +** METHOD: sqlite3_index_info +** +** This API may only be used from within the [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this interface +** from outside of an xBestIndex method are undefined and probably harmful. +** +** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within +** the [xBestIndex] method of a [virtual table] implementation, with P being +** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and +** J being a 0-based index into P->aConstraint[], then this routine +** attempts to set *V to the value of the right-hand operand of +** that constraint if the right-hand operand is known. ^If the +** right-hand operand is not known, then *V is set to a NULL pointer. +** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if +** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) +** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th +** constraint is not available. ^The sqlite3_vtab_rhs_value() interface +** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** something goes wrong. +** +** The sqlite3_vtab_rhs_value() interface is usually only successful if +** the right-hand operand of a constraint is a literal value in the original +** SQL statement. If the right-hand operand is an expression or a reference +** to some other column or a [host parameter], then sqlite3_vtab_rhs_value() +** will probably return [SQLITE_NOTFOUND]. +** +** ^(Some constraints, such as [SQLITE_INDEX_CONSTRAINT_ISNULL] and +** [SQLITE_INDEX_CONSTRAINT_ISNOTNULL], have no right-hand operand. For such +** constraints, sqlite3_vtab_rhs_value() always returns SQLITE_NOTFOUND.)^ +** +** ^The [sqlite3_value] object returned in *V is a protected sqlite3_value +** and remains valid for the duration of the xBestIndex method call. +** ^When xBestIndex returns, the sqlite3_value object returned by +** sqlite3_vtab_rhs_value() is automatically deallocated. +** +** The "_rhs_" in the name of this routine is an abbreviation for +** "Right-Hand Side". +*/ +SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); /* ** CAPI3REF: Conflict resolution modes @@ -8082,17 +10551,21 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); ** managed by the prepared statement S and will be automatically freed when ** S is finalized. ** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** **
    ** [[SQLITE_SCANSTAT_NLOOP]]
    SQLITE_SCANSTAT_NLOOP
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be +**
    ^The [sqlite3_int64] variable pointed to by the V parameter will be ** set to the total number of times that the X-th loop has run.
    ** ** [[SQLITE_SCANSTAT_NVISIT]]
    SQLITE_SCANSTAT_NVISIT
    -**
    ^The [sqlite3_int64] variable pointed to by the T parameter will be set +**
    ^The [sqlite3_int64] variable pointed to by the V parameter will be set ** to the total number of rows examined by all iterations of the X-th loop.
    ** ** [[SQLITE_SCANSTAT_EST]]
    SQLITE_SCANSTAT_EST
    -**
    ^The "double" variable pointed to by the T parameter will be set to the +**
    ^The "double" variable pointed to by the V parameter will be set to the ** query planner's estimate for the average number of rows output from each ** iteration of the X-th loop. If the query planner's estimates was accurate, ** then this value will approximate the quotient NVISIT/NLOOP and the @@ -8100,21 +10573,33 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); ** be the NLOOP value for the current loop. ** ** [[SQLITE_SCANSTAT_NAME]]
    SQLITE_SCANSTAT_NAME
    -**
    ^The "const char *" variable pointed to by the T parameter will be set +**
    ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the name of the index or table ** used for the X-th loop. ** ** [[SQLITE_SCANSTAT_EXPLAIN]]
    SQLITE_SCANSTAT_EXPLAIN
    -**
    ^The "const char *" variable pointed to by the T parameter will be set +**
    ^The "const char *" variable pointed to by the V parameter will be set ** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] ** description for the X-th loop. ** -** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECT
    -**
    ^The "int" variable pointed to by the T parameter will be set to the -** "select-id" for the X-th loop. The select-id identifies which query or -** subquery the loop is part of. The main query has a select-id of zero. -** The select-id is the same value as is output in the first column -** of an [EXPLAIN QUERY PLAN] query. +** [[SQLITE_SCANSTAT_SELECTID]]
    SQLITE_SCANSTAT_SELECTID
    +**
    ^The "int" variable pointed to by the V parameter will be set to the +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_PARENTID]]
    SQLITE_SCANSTAT_PARENTID
    +**
    The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]]
    SQLITE_SCANSTAT_NCYCLE
    +**
    The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. **
    */ #define SQLITE_SCANSTAT_NLOOP 0 @@ -8123,12 +10608,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); #define SQLITE_SCANSTAT_NAME 3 #define SQLITE_SCANSTAT_EXPLAIN 4 #define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 /* ** CAPI3REF: Prepared Statement Scan Status ** METHOD: sqlite3_stmt ** -** This interface returns information about the predicted and measured +** These interfaces return information about the predicted and measured ** performance for pStmt. Advanced applications can use this ** interface to compare the predicted and the measured performance and ** issue warnings and/or rerun [ANALYZE] if discrepancies are found. @@ -8139,28 +10626,47 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *); ** ** The "iScanStatusOp" parameter determines which status information to return. ** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior -** of this interface is undefined. -** ^The requested measurement is written into a variable pointed to by -** the "pOut" parameter. -** Parameter "idx" identifies the specific loop to retrieve statistics for. -** Loops are numbered starting from zero. ^If idx is out of range - less than -** zero or greater than or equal to the total number of loops used to implement -** the statement - a non-zero value is returned and the variable that pOut -** points to is unchanged. +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. ** -** ^Statistics might not be available for all loops in all statements. ^In cases -** where there exist loops with no available statistics, this function behaves -** as if the loop did not exist - it returns non-zero and leave the variable -** that pOut points to unchanged. +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. ** ** See also: [sqlite3_stmt_scanstatus_reset()] */ -SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( +SQLITE_API int sqlite3_stmt_scanstatus( sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ int idx, /* Index of loop to report on */ int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ void *pOut /* Result written here */ -); +); +SQLITE_API int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ +#define SQLITE_SCANSTAT_COMPLEX 0x0001 /* ** CAPI3REF: Zero Scan-Status Counters @@ -8171,22 +10677,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( ** This API is only available if the library is built with pre-processor ** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. */ -SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); /* ** CAPI3REF: Flush caches to disk mid-transaction +** METHOD: sqlite3 ** ** ^If a write-transaction is open on [database connection] D when the ** [sqlite3_db_cacheflush(D)] interface invoked, any dirty -** pages in the pager-cache that are not currently in use are written out +** pages in the pager-cache that are not currently in use are written out ** to disk. A dirty page may be in use if a database cursor created by an ** active SQL statement is reading from it, or if it is page 1 of a database ** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] ** interface flushes caches for all schemas - "main", "temp", and ** any [attached] databases. ** -** ^If this function needs to obtain extra database locks before dirty pages -** can be flushed to disk, it does so. ^If those locks cannot be obtained +** ^If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. ^If those locks cannot be obtained ** immediately and there is a busy-handler callback configured, it is invoked ** in the usual manner. ^If the required lock still cannot be obtained, then ** the database is skipped and an attempt made to flush any dirty pages @@ -8203,17 +10710,18 @@ SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); ** ^This function does not set the database handle error code or message ** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3*); +SQLITE_API int sqlite3_db_cacheflush(sqlite3*); /* ** CAPI3REF: The pre-update hook. +** METHOD: sqlite3 ** ** ^These interfaces are only available if SQLite is compiled using the ** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option. ** ** ^The [sqlite3_preupdate_hook()] interface registers a callback function ** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation -** on a [rowid table]. +** on a database table. ** ^At most one preupdate hook may be registered at a time on a single ** [database connection]; each call to [sqlite3_preupdate_hook()] overrides ** the previous setting. @@ -8222,28 +10730,36 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3*); ** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as ** the first parameter to callbacks. ** -** ^The preupdate hook only fires for changes to [rowid tables]; the preupdate -** hook is not invoked for changes to [virtual tables] or [WITHOUT ROWID] -** tables. +** ^The preupdate hook only fires for changes to real database tables; the +** preupdate hook is not invoked for changes to [virtual tables] or to +** system tables like sqlite_sequence or sqlite_stat1. ** ** ^The second parameter to the preupdate callback is a pointer to ** the [database connection] that registered the preupdate hook. ** ^The third parameter to the preupdate callback is one of the constants -** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to indentify the +** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the ** kind of update operation that is about to occur. ** ^(The fourth parameter to the preupdate callback is the name of the ** database within the database connection that is being modified. This -** will be "main" for the main database or "temp" for TEMP tables or +** will be "main" for the main database or "temp" for TEMP tables or ** the name given after the AS keyword in the [ATTACH] statement for attached ** databases.)^ ** ^The fifth parameter to the preupdate callback is the name of the ** table that is being modified. -** ^The sixth parameter to the preupdate callback is the initial [rowid] of the -** row being changes for SQLITE_UPDATE and SQLITE_DELETE changes and is -** undefined for SQLITE_INSERT changes. -** ^The seventh parameter to the preupdate callback is the final [rowid] of -** the row being changed for SQLITE_UPDATE and SQLITE_INSERT changes and is -** undefined for SQLITE_DELETE changes. +** +** For an UPDATE or DELETE operation on a [rowid table], the sixth +** parameter passed to the preupdate callback is the initial [rowid] of the +** row being modified or deleted. For an INSERT operation on a rowid table, +** or any operation on a WITHOUT ROWID table, the value of the sixth +** parameter is undefined. For an INSERT or UPDATE on a rowid table the +** seventh parameter is the final rowid value of the row being inserted +** or updated. The value of the seventh parameter passed to the callback +** function is not defined for operations on WITHOUT ROWID tables, or for +** DELETE operations on rowid tables. +** +** ^The sqlite3_preupdate_hook(D,C,P) function returns the P argument from +** the previous call on the same [database connection] D, or NULL for +** the first call on D. ** ** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], ** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces @@ -8277,13 +10793,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3*); ** ** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate ** callback was invoked as a result of a direct insert, update, or delete -** operation; or 1 for inserts, updates, or deletes invoked by top-level +** operation; or 1 for inserts, updates, or deletes invoked by top-level ** triggers; or 2 for changes resulting from triggers called by top-level ** triggers; and so forth. ** +** When the [sqlite3_blob_write()] API is used to update a blob column, +** the pre-update hook is invoked with SQLITE_DELETE. This is because the +** in this case the new values are not available. In this case, when a +** callback made with op==SQLITE_DELETE is actually a write using the +** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns +** the index of the column being written. In other cases, where the +** pre-update hook is being invoked for some other reason, including a +** regular DELETE, sqlite3_preupdate_blobwrite() returns -1. +** ** See also: [sqlite3_update_hook()] */ -SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_preupdate_hook( +#if defined(SQLITE_ENABLE_PREUPDATE_HOOK) +SQLITE_API void *sqlite3_preupdate_hook( sqlite3 *db, void(*xPreUpdate)( void *pCtx, /* Copy of third arg to preupdate_hook() */ @@ -8296,27 +10822,29 @@ SQLITE_API SQLITE_EXPERIMENTAL void *SQLITE_STDCALL sqlite3_preupdate_hook( ), void* ); -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_preupdate_count(sqlite3 *); -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_preupdate_depth(sqlite3 *); -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_count(sqlite3 *); +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *); +SQLITE_API int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *); +#endif /* ** CAPI3REF: Low-level system error code +** METHOD: sqlite3 ** ** ^Attempt to return the underlying operating system error code or error ** number that caused the most recent I/O error or failure to open a file. ** The return value is OS-dependent. For example, on unix systems, after ** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be ** called to get back the underlying "errno" that caused the problem, such -** as ENOSPC, EAUTH, EISDIR, and so forth. +** as ENOSPC, EAUTH, EISDIR, and so forth. */ -SQLITE_API int SQLITE_STDCALL sqlite3_system_errno(sqlite3*); +SQLITE_API int sqlite3_system_errno(sqlite3*); /* ** CAPI3REF: Database Snapshot -** KEYWORDS: {snapshot} -** EXPERIMENTAL +** KEYWORDS: {snapshot} {sqlite3_snapshot} ** ** An instance of the snapshot object records the state of a [WAL mode] ** database for some specific point in history. @@ -8333,35 +10861,63 @@ SQLITE_API int SQLITE_STDCALL sqlite3_system_errno(sqlite3*); ** version of the database file so that it is possible to later open a new read ** transaction that sees that historical version of the database rather than ** the most recent version. -** -** The constructor for this object is [sqlite3_snapshot_get()]. The -** [sqlite3_snapshot_open()] method causes a fresh read transaction to refer -** to an historical snapshot (if possible). The destructor for -** sqlite3_snapshot objects is [sqlite3_snapshot_free()]. */ -typedef struct sqlite3_snapshot sqlite3_snapshot; +typedef struct sqlite3_snapshot { + unsigned char hidden[48]; +} sqlite3_snapshot; /* ** CAPI3REF: Record A Database Snapshot -** EXPERIMENTAL +** CONSTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a ** new [sqlite3_snapshot] object that records the current state of ** schema S in database connection D. ^On success, the ** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly ** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. -** ^If schema S of [database connection] D is not a [WAL mode] database -** that is in a read transaction, then [sqlite3_snapshot_get(D,S,P)] -** leaves the *P value unchanged and returns an appropriate [error code]. +** If there is not already a read-transaction open on schema S when +** this function is called, one is opened automatically. +** +** If a read-transaction is opened by this function, then it is guaranteed +** that the returned snapshot object may not be invalidated by a database +** writer or checkpointer until after the read-transaction is closed. This +** is not guaranteed if a read-transaction is already open when this +** function is called. In that case, any subsequent write or checkpoint +** operation on the database may invalidate the returned snapshot handle, +** even while the read-transaction remains open. +** +** The following must be true for this function to succeed. If any of +** the following statements are false when sqlite3_snapshot_get() is +** called, SQLITE_ERROR is returned. The final value of *P is undefined +** in this case. +** +**
      +**
    • The database handle must not be in [autocommit mode]. +** +**
    • Schema S of [database connection] D must be a [WAL mode] database. +** +**
    • There must not be a write transaction open on schema S of database +** connection D. +** +**
    • One or more transactions must have been written to the current wal +** file since it was created on disk (by any connection). This means +** that a snapshot cannot be taken on a wal mode database with no wal +** file immediately after it is first opened. At least one transaction +** must be written to it first. +**
    +** +** This function may also return SQLITE_NOMEM. If it is called with the +** database handle in autocommit mode but fails for some other reason, +** whether or not a read transaction is opened on schema S is undefined. ** ** The [sqlite3_snapshot] object returned from a successful call to ** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] ** to avoid a memory leak. ** ** The [sqlite3_snapshot_get()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_get( +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( sqlite3 *db, const char *zSchema, sqlite3_snapshot **ppSnapshot @@ -8369,37 +10925,48 @@ SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_get( /* ** CAPI3REF: Start a read transaction on an historical snapshot -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** -** ^The [sqlite3_snapshot_open(D,S,P)] interface starts a -** read transaction for schema S of -** [database connection] D such that the read transaction -** refers to historical [snapshot] P, rather than the most -** recent change to the database. -** ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK on success -** or an appropriate [error code] if it fails. +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. ** -** ^In order to succeed, a call to [sqlite3_snapshot_open(D,S,P)] must be -** the first operation following the [BEGIN] that takes the schema S -** out of [autocommit mode]. -** ^In other words, schema S must not currently be in -** a transaction for [sqlite3_snapshot_open(D,S,P)] to work, but the -** database connection D must be out of [autocommit mode]. -** ^A [snapshot] will fail to open if it has been overwritten by a -** [checkpoint]. ** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the ** database connection D does not know that the database file for ** schema S is in [WAL mode]. A database connection might not know ** that the database file is in [WAL mode] if there has been no prior -** I/O on that database connection, or if the database entered [WAL mode] +** I/O on that database connection, or if the database entered [WAL mode] ** after the most recent I/O on the database connection.)^ ** (Hint: Run "[PRAGMA application_id]" against a newly opened ** database connection in order to make it ready to use snapshots.) ** ** The [sqlite3_snapshot_open()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_open( +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( sqlite3 *db, const char *zSchema, sqlite3_snapshot *pSnapshot @@ -8407,44 +10974,215 @@ SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_open( /* ** CAPI3REF: Destroy a snapshot -** EXPERIMENTAL +** DESTRUCTOR: sqlite3_snapshot ** ** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. ** The application must eventually free every [sqlite3_snapshot] object ** using this routine to avoid a memory leak. ** ** The [sqlite3_snapshot_free()] interface is only available when the -** SQLITE_ENABLE_SNAPSHOT compile-time option is used. +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. */ -SQLITE_API SQLITE_EXPERIMENTAL void SQLITE_STDCALL sqlite3_snapshot_free(sqlite3_snapshot*); +SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); /* ** CAPI3REF: Compare the ages of two snapshot handles. -** EXPERIMENTAL +** METHOD: sqlite3_snapshot ** ** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages -** of two valid snapshot handles. +** of two valid snapshot handles. ** -** If the two snapshot handles are not associated with the same database -** file, the result of the comparison is undefined. +** If the two snapshot handles are not associated with the same database +** file, the result of the comparison is undefined. ** ** Additionally, the result of the comparison is only valid if both of the ** snapshot handles were obtained by calling sqlite3_snapshot_get() since the ** last time the wal file was deleted. The wal file is deleted when the ** database is changed back to rollback mode or when the number of database -** clients drops to zero. If either snapshot handle was obtained before the -** wal file was last deleted, the value returned by this function +** clients drops to zero. If either snapshot handle was obtained before the +** wal file was last deleted, the value returned by this function ** is undefined. ** ** Otherwise, this API returns a negative value if P1 refers to an older ** snapshot than P2, zero if the two handles refer to the same database ** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. */ -SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_cmp( +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( sqlite3_snapshot *p1, sqlite3_snapshot *p2 ); +/* +** CAPI3REF: Recover snapshots from a wal file +** METHOD: sqlite3_snapshot +** +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. +** +** This function attempts to scan the WAL file associated with database zDb +** of database handle db and make all valid snapshots available to +** sqlite3_snapshot_open(). It is an error if there is already a read +** transaction open on the database, or if the database is not a WAL mode +** database. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Serialize a database +** +** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory +** that is a serialization of the S database on [database connection] D. +** If P is not a NULL pointer, then the size of the database in bytes +** is written into *P. +** +** For an ordinary on-disk database file, the serialization is just a +** copy of the disk file. For an in-memory database or a "TEMP" database, +** the serialization is the same sequence of bytes which would be written +** to disk if that database where backed up to disk. +** +** The usual case is that sqlite3_serialize() copies the serialization of +** the database into memory obtained from [sqlite3_malloc64()] and returns +** a pointer to that memory. The caller is responsible for freeing the +** returned value to avoid a memory leak. However, if the F argument +** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations +** are made, and the sqlite3_serialize() function will return a pointer +** to the contiguous memory representation of the database that SQLite +** is currently using for that database, or NULL if the no such contiguous +** memory representation of the database exists. A contiguous memory +** representation of the database will usually only exist if there has +** been a prior call to [sqlite3_deserialize(D,S,...)] with the same +** values of D and S. +** The size of the database is written into *P even if the +** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy +** of the database exists. +** +** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set, +** the returned buffer content will remain accessible and unchanged +** until either the next write operation on the connection or when +** the connection is closed, and applications must not modify the +** buffer. If the bit had been clear, the returned buffer will not +** be accessed by SQLite after the call. +** +** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the +** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory +** allocation error occurs. +** +** This interface is omitted if SQLite is compiled with the +** [SQLITE_OMIT_DESERIALIZE] option. +*/ +SQLITE_API unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to serialize. ex: "main", "temp", ... */ + sqlite3_int64 *piSize, /* Write size of the DB here, if not NULL */ + unsigned int mFlags /* Zero or more SQLITE_SERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_serialize +** +** Zero or more of the following constants can be OR-ed together for +** the F argument to [sqlite3_serialize(D,S,P,F)]. +** +** SQLITE_SERIALIZE_NOCOPY means that [sqlite3_serialize()] will return +** a pointer to contiguous in-memory database that it is currently using, +** without making a copy of the database. If SQLite is not currently using +** a contiguous in-memory database, then this option causes +** [sqlite3_serialize()] to return a NULL pointer. SQLite will only be +** using a contiguous in-memory database if it has been initialized by a +** prior call to [sqlite3_deserialize()]. +*/ +#define SQLITE_SERIALIZE_NOCOPY 0x001 /* Do no memory allocations */ + +/* +** CAPI3REF: Deserialize a database +** +** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the +** [database connection] D to disconnect from database S and then +** reopen S as an in-memory database based on the serialization contained +** in P. The serialized database P is N bytes in size. M is the size of +** the buffer P, which might be larger than N. If M is larger than N, and +** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is +** permitted to add content to the in-memory database as long as the total +** size does not exceed M bytes. +** +** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will +** invoke sqlite3_free() on the serialization buffer when the database +** connection closes. If the SQLITE_DESERIALIZE_RESIZEABLE bit is set, then +** SQLite will try to increase the buffer size using sqlite3_realloc64() +** if writes on the database cause it to grow larger than M bytes. +** +** Applications must not modify the buffer P or invalidate it before +** the database connection D is closed. +** +** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the +** database is currently in a read transaction or is involved in a backup +** operation. +** +** It is not possible to deserialized into the TEMP database. If the +** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the +** function returns SQLITE_ERROR. +** +** The deserialized database should not be in [WAL mode]. If the database +** is in WAL mode, then any attempt to use the database file will result +** in an [SQLITE_CANTOPEN] error. The application can set the +** [file format version numbers] (bytes 18 and 19) of the input database P +** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the +** database file into rollback mode and work around this limitation. +** +** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the +** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then +** [sqlite3_free()] is invoked on argument P prior to returning. +** +** This interface is omitted if SQLite is compiled with the +** [SQLITE_OMIT_DESERIALIZE] option. +*/ +SQLITE_API int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_deserialize() +** +** The following are allowed values for 6th argument (the F argument) to +** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. +** +** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization +** in the P argument is held in memory obtained from [sqlite3_malloc64()] +** and that SQLite should take ownership of this memory and automatically +** free it when it has finished using it. Without this flag, the caller +** is responsible for freeing any dynamically allocated memory. +** +** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to +** grow the size of the database using calls to [sqlite3_realloc64()]. This +** flag should only be used if SQLITE_DESERIALIZE_FREEONCLOSE is also used. +** Without this flag, the deserialized database cannot increase in size beyond +** the number of bytes specified by the M parameter. +** +** The SQLITE_DESERIALIZE_READONLY flag means that the deserialized database +** should be treated as read-only. +*/ +#define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */ +#define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ +#define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ + /* ** Undo the hack that converts floating point types to integer for ** builds on processors without floating point support. @@ -8453,10 +11191,21 @@ SQLITE_API SQLITE_EXPERIMENTAL int SQLITE_STDCALL sqlite3_snapshot_cmp( # undef double #endif +#if defined(__wasi__) +# undef SQLITE_WASI +# define SQLITE_WASI 1 +# ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION +# endif +# ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +# endif +#endif + #if 0 } /* End of the 'extern "C"' block */ #endif -#endif /* _SQLITE3_H_ */ +#endif /* SQLITE3_H */ /******** Begin file sqlite3rtree.h *********/ /* @@ -8498,7 +11247,7 @@ typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; ** ** SELECT ... FROM WHERE MATCH $zGeom(... params ...) */ -SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( +SQLITE_API int sqlite3_rtree_geometry_callback( sqlite3 *db, const char *zGeom, int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), @@ -8519,12 +11268,12 @@ struct sqlite3_rtree_geometry { }; /* -** Register a 2nd-generation geometry callback named zScore that can be +** Register a 2nd-generation geometry callback named zScore that can be ** used as part of an R-Tree geometry query as follows: ** ** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) */ -SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( +SQLITE_API int sqlite3_rtree_query_callback( sqlite3 *db, const char *zQueryFunc, int (*xQueryFunc)(sqlite3_rtree_query_info*), @@ -8534,7 +11283,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( /* -** A pointer to a structure of the following type is passed as the +** A pointer to a structure of the following type is passed as the ** argument to scored geometry callback registered using ** sqlite3_rtree_query_callback(). ** @@ -8556,7 +11305,7 @@ struct sqlite3_rtree_query_info { sqlite3_int64 iRowid; /* Rowid for current entry */ sqlite3_rtree_dbl rParentScore; /* Score of parent node */ int eParentWithin; /* Visibility of parent node */ - int eWithin; /* OUT: Visiblity */ + int eWithin; /* OUT: Visibility */ sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ /* The following fields are only available in 3.8.11 and later */ sqlite3_value **apSqlParam; /* Original SQL values of parameters */ @@ -8592,16 +11341,23 @@ extern "C" { /* ** CAPI3REF: Session Object Handle +** +** An instance of this object is a [session] that can be used to +** record changes to a database. */ typedef struct sqlite3_session sqlite3_session; /* ** CAPI3REF: Changeset Iterator Handle +** +** An instance of this object acts as a cursor for iterating +** over the elements of a [changeset] or [patchset]. */ typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; /* ** CAPI3REF: Create A New Session Object +** CONSTRUCTOR: sqlite3_session ** ** Create a new session object attached to database handle db. If successful, ** a pointer to the new object is written to *ppSession and SQLITE_OK is @@ -8622,7 +11378,7 @@ typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; ** is not possible for an application to register a pre-update hook on a ** database handle that has one or more session objects attached. Nor is ** it possible to create a session object attached to a database handle for -** which a pre-update hook is already defined. The results of attempting +** which a pre-update hook is already defined. The results of attempting ** either of these things are undefined. ** ** The session object will be used to create changesets for tables in @@ -8630,7 +11386,7 @@ typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; ** attached database. It is not an error if database zDb is not attached ** to the database when the session object is created. */ -int sqlite3session_create( +SQLITE_API int sqlite3session_create( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of db (e.g. "main") */ sqlite3_session **ppSession /* OUT: New session object */ @@ -8638,21 +11394,68 @@ int sqlite3session_create( /* ** CAPI3REF: Delete A Session Object +** DESTRUCTOR: sqlite3_session ** -** Delete a session object previously allocated using +** Delete a session object previously allocated using ** [sqlite3session_create()]. Once a session object has been deleted, the ** results of attempting to use pSession with any other session module ** function are undefined. ** ** Session objects must be deleted before the database handle to which they -** are attached is closed. Refer to the documentation for +** are attached is closed. Refer to the documentation for ** [sqlite3session_create()] for details. */ -void sqlite3session_delete(sqlite3_session *pSession); +SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); +/* +** CAPI3REF: Configure a Session Object +** METHOD: sqlite3_session +** +** This method is used to configure a session object after it has been +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. +** +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config +** +** The following values may passed as the the 2nd parameter to +** sqlite3session_object_config(). +** +**
    SQLITE_SESSION_OBJCONFIG_SIZE
    +** This option is used to set, clear or query the flag that enables +** the [sqlite3session_changeset_size()] API. Because it imposes some +** computational overhead, this API is disabled by default. Argument +** pArg must point to a value of type (int). If the value is initially +** 0, then the sqlite3session_changeset_size() API is disabled. If it +** is greater than 0, then the same API is enabled. Or, if the initial +** value is less than zero, no change is made. In all cases the (int) +** variable is set to 1 if the sqlite3session_changeset_size() API is +** enabled following the current call, or 0 otherwise. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. +** +**
    SQLITE_SESSION_OBJCONFIG_ROWID
    +** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. +*/ +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 /* ** CAPI3REF: Enable Or Disable A Session Object +** METHOD: sqlite3_session ** ** Enable or disable the recording of changes by a session object. When ** enabled, a session object records changes made to the database. When @@ -8662,16 +11465,17 @@ void sqlite3session_delete(sqlite3_session *pSession); ** the eventual changesets. ** ** Passing zero to this function disables the session. Passing a value -** greater than zero enables it. Passing a value less than zero is a +** greater than zero enables it. Passing a value less than zero is a ** no-op, and may be used to query the current state of the session. ** -** The return value indicates the final state of the session object: 0 if +** The return value indicates the final state of the session object: 0 if ** the session is disabled, or 1 if it is enabled. */ -int sqlite3session_enable(sqlite3_session *pSession, int bEnable); +SQLITE_API int sqlite3session_enable(sqlite3_session *pSession, int bEnable); /* ** CAPI3REF: Set Or Clear the Indirect Change Flag +** METHOD: sqlite3_session ** ** Each change recorded by a session object is marked as either direct or ** indirect. A change is marked as indirect if either: @@ -8679,7 +11483,7 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable); **
      **
    • The session object "indirect" flag is set when the change is ** made, or -**
    • The change is made by an SQL trigger or foreign key action +**
    • The change is made by an SQL trigger or foreign key action ** instead of directly as a result of a users SQL statement. **
    ** @@ -8691,32 +11495,33 @@ int sqlite3session_enable(sqlite3_session *pSession, int bEnable); ** flag. If the second argument passed to this function is zero, then the ** indirect flag is cleared. If it is greater than zero, the indirect flag ** is set. Passing a value less than zero does not modify the current value -** of the indirect flag, and may be used to query the current state of the +** of the indirect flag, and may be used to query the current state of the ** indirect flag for the specified session object. ** -** The return value indicates the final state of the indirect flag: 0 if +** The return value indicates the final state of the indirect flag: 0 if ** it is clear, or 1 if it is set. */ -int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); +SQLITE_API int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); /* ** CAPI3REF: Attach A Table To A Session Object +** METHOD: sqlite3_session ** ** If argument zTab is not NULL, then it is the name of a table to attach -** to the session object passed as the first argument. All subsequent changes -** made to the table while the session object is enabled will be recorded. See +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See ** documentation for [sqlite3session_changeset()] for further details. ** ** Or, if argument zTab is NULL, then changes are recorded for all tables -** in the database. If additional tables are added to the database (by -** executing "CREATE TABLE" statements) after this call is made, changes for +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for ** the new tables are also recorded. ** ** Changes can only be recorded for tables that have a PRIMARY KEY explicitly -** defined as part of their CREATE TABLE statement. It does not matter if the +** defined as part of their CREATE TABLE statement. It does not matter if the ** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY ** KEY may consist of a single column, or may be a composite key. -** +** ** It is not an error if the named table does not exist in the database. Nor ** is it an error if the named table does not have a PRIMARY KEY. However, ** no changes will be recorded in either of these scenarios. @@ -8724,24 +11529,54 @@ int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); ** Changes are not recorded for individual rows that have NULL values stored ** in one or more of their PRIMARY KEY columns. ** -** SQLITE_OK is returned if the call completes without error. Or, if an error +** SQLITE_OK is returned if the call completes without error. Or, if an error ** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +** +**

    Special sqlite_stat1 Handling

    +** +** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to +** some of the rules above. In SQLite, the schema of sqlite_stat1 is: +** 
    +**        CREATE TABLE sqlite_stat1(tbl,idx,stat)
+**
    +** +** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are +** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes +** are recorded for rows for which (idx IS NULL) is true. However, for such +** rows a zero-length blob (SQL value X'') is stored in the changeset or +** patchset instead of a NULL value. This allows such changesets to be +** manipulated by legacy implementations of sqlite3changeset_invert(), +** concat() and similar. +** +** The sqlite3changeset_apply() function automatically converts the +** zero-length blob back to a NULL value when updating the sqlite_stat1 +** table. However, if the application calls sqlite3changeset_new(), +** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset +** iterator directly (including on a changeset iterator passed to a +** conflict-handler callback) then the X'' value is returned. The application +** must translate X'' to NULL itself if required. +** +** Legacy (older than 3.22.0) versions of the sessions module cannot capture +** changes made to the sqlite_stat1 table. Legacy versions of the +** sqlite3changeset_apply() function silently ignore any modifications to the +** sqlite_stat1 table that are part of a changeset or patchset. */ -int sqlite3session_attach( +SQLITE_API int sqlite3session_attach( sqlite3_session *pSession, /* Session object */ const char *zTab /* Table name */ ); /* ** CAPI3REF: Set a table filter on a Session Object. +** METHOD: sqlite3_session ** -** The second argument (xFilter) is the "filter callback". For changes to rows -** in tables that are not attached to the Session oject, the filter is called -** to determine whether changes to the table's rows should be tracked or not. -** If xFilter returns 0, changes is not tracked. Note that once a table is +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes are not tracked. Note that once a table is ** attached, xFilter will not be called again. */ -void sqlite3session_table_filter( +SQLITE_API void sqlite3session_table_filter( sqlite3_session *pSession, /* Session object */ int(*xFilter)( void *pCtx, /* Copy of third arg to _filter_table() */ @@ -8752,10 +11587,11 @@ void sqlite3session_table_filter( /* ** CAPI3REF: Generate A Changeset From A Session Object +** METHOD: sqlite3_session ** -** Obtain a changeset containing changes to the tables attached to the -** session object passed as the first argument. If successful, -** set *ppChangeset to point to a buffer containing the changeset +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset ** and *pnChangeset to the size of the changeset in bytes before returning ** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to ** zero and return an SQLite error code. @@ -8770,7 +11606,7 @@ void sqlite3session_table_filter( ** modifies the values of primary key columns. If such a change is made, it ** is represented in a changeset as a DELETE followed by an INSERT. ** -** Changes are not recorded for rows that have NULL values stored in one or +** Changes are not recorded for rows that have NULL values stored in one or ** more of their PRIMARY KEY columns. If such a row is inserted or deleted, ** no corresponding change is present in the changesets returned by this ** function. If an existing row with one or more NULL values stored in @@ -8823,14 +11659,14 @@ void sqlite3session_table_filter( **
      **
    • For each record generated by an insert, the database is queried ** for a row with a matching primary key. If one is found, an INSERT -** change is added to the changeset. If no such row is found, no change +** change is added to the changeset. If no such row is found, no change ** is added to the changeset. ** -**
    • For each record generated by an update or delete, the database is +**
    • For each record generated by an update or delete, the database is ** queried for a row with a matching primary key. If such a row is ** found and one or more of the non-primary key fields have been -** modified from their original values, an UPDATE change is added to -** the changeset. Or, if no such row is found in the table, a DELETE +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE ** change is added to the changeset. If there is a row with a matching ** primary key in the database, but all fields contain their original ** values, no change is added to the changeset. @@ -8838,7 +11674,7 @@ void sqlite3session_table_filter( ** ** This means, amongst other things, that if a row is inserted and then later ** deleted while a session object is active, neither the insert nor the delete -** will be present in the changeset. Or if a row is deleted and then later a +** will be present in the changeset. Or if a row is deleted and then later a ** row with the same primary key values inserted while a session object is ** active, the resulting changeset will contain an UPDATE change instead of ** a DELETE and an INSERT. @@ -8847,21 +11683,38 @@ void sqlite3session_table_filter( ** it does not accumulate records when rows are inserted, updated or deleted. ** This may appear to have some counter-intuitive effects if a single row ** is written to more than once during a session. For example, if a row -** is inserted while a session object is enabled, then later deleted while +** is inserted while a session object is enabled, then later deleted while ** the same session object is disabled, no INSERT record will appear in the ** changeset, even though the delete took place while the session was disabled. -** Or, if one field of a row is updated while a session is disabled, and +** Or, if one field of a row is updated while a session is disabled, and ** another field of the same row is updated while the session is enabled, the ** resulting changeset will contain an UPDATE change that updates both fields. */ -int sqlite3session_changeset( +SQLITE_API int sqlite3session_changeset( sqlite3_session *pSession, /* Session object */ int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ void **ppChangeset /* OUT: Buffer containing changeset */ ); /* -** CAPI3REF: Load The Difference Between Tables Into A Session +** CAPI3REF: Return An Upper-limit For The Size Of The Changeset +** METHOD: sqlite3_session +** +** By default, this function always returns 0. For it to return +** a useful result, the sqlite3_session object must have been configured +** to enable this API using sqlite3session_object_config() with the +** SQLITE_SESSION_OBJCONFIG_SIZE verb. +** +** When enabled, this function returns an upper limit, in bytes, for the size +** of the changeset that might be produced if sqlite3session_changeset() were +** called. The final changeset size might be equal to or smaller than the +** size in bytes returned by this function. +*/ +SQLITE_API sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** METHOD: sqlite3_session ** ** If it is not already attached to the session object passed as the first ** argument, this function attaches table zTbl in the same manner as the @@ -8870,7 +11723,7 @@ int sqlite3session_changeset( ** an error). ** ** Argument zFromDb must be the name of a database ("main", "temp" etc.) -** attached to the same database handle as the session object that contains +** attached to the same database handle as the session object that contains ** a table compatible with the table attached to the session by this function. ** A table is considered compatible if it: ** @@ -8886,36 +11739,37 @@ int sqlite3session_changeset( ** APIs, tables without PRIMARY KEYs are simply ignored. ** ** This function adds a set of changes to the session object that could be -** used to update the table in database zFrom (call this the "from-table") -** so that its content is the same as the table attached to the session +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session ** object (call this the "to-table"). Specifically: ** **
        -**
      • For each row (primary key) that exists in the to-table but not in +**
      • For each row (primary key) that exists in the to-table but not in ** the from-table, an INSERT record is added to the session object. ** -**
      • For each row (primary key) that exists in the to-table but not in +**
      • For each row (primary key) that exists in the to-table but not in ** the from-table, a DELETE record is added to the session object. ** -**
      • For each row (primary key) that exists in both tables, but features -** different in each, an UPDATE record is added to the session. +**
      • For each row (primary key) that exists in both tables, but features +** different non-PK values in each, an UPDATE record is added to the +** session. **
      ** ** To clarify, if this function is called and then a changeset constructed -** using [sqlite3session_changeset()], then after applying that changeset to -** database zFrom the contents of the two compatible tables would be +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be ** identical. ** ** It an error if database zFrom does not exist or does not contain the ** required compatible table. ** -** If the operation successful, SQLITE_OK is returned. Otherwise, an SQLite +** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite ** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg -** may be set to point to a buffer containing an English language error +** may be set to point to a buffer containing an English language error ** message. It is the responsibility of the caller to free this buffer using ** sqlite3_free(). */ -int sqlite3session_diff( +SQLITE_API int sqlite3session_diff( sqlite3_session *pSession, const char *zFromDb, const char *zTbl, @@ -8925,23 +11779,24 @@ int sqlite3session_diff( /* ** CAPI3REF: Generate A Patchset From A Session Object +** METHOD: sqlite3_session ** ** The differences between a patchset and a changeset are that: ** **
        -**
      • DELETE records consist of the primary key fields only. The +**
      • DELETE records consist of the primary key fields only. The ** original values of other fields are omitted. -**
      • The original values of any modified fields are omitted from +**
      • The original values of any modified fields are omitted from ** UPDATE records. **
      ** -** A patchset blob may be used with up to date versions of all -** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), ** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, ** attempting to use a patchset blob with old versions of the -** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. ** -** Because the non-primary key "old.*" fields are omitted, no +** Because the non-primary key "old.*" fields are omitted, no ** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset ** is passed to the sqlite3changeset_apply() API. Other conflict types work ** in the same way as for changesets. @@ -8951,38 +11806,47 @@ int sqlite3session_diff( ** a single table are grouped together, tables appear in the order in which ** they were attached to the session object). */ -int sqlite3session_patchset( +SQLITE_API int sqlite3session_patchset( sqlite3_session *pSession, /* Session object */ - int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ - void **ppPatchset /* OUT: Buffer containing changeset */ + int *pnPatchset, /* OUT: Size of buffer at *ppPatchset */ + void **ppPatchset /* OUT: Buffer containing patchset */ ); /* ** CAPI3REF: Test if a changeset has recorded any changes. ** -** Return non-zero if no changes to attached tables have been recorded by -** the session object passed as the first argument. Otherwise, if one or +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or ** more changes have been recorded, return zero. ** ** Even if this function returns zero, it is possible that calling ** [sqlite3session_changeset()] on the session handle may still return a -** changeset that contains no changes. This can happen when a row in -** an attached table is modified and then later on the original values +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values ** are restored. However, if this function returns non-zero, then it is -** guaranteed that a call to sqlite3session_changeset() will return a +** guaranteed that a call to sqlite3session_changeset() will return a ** changeset containing zero changes. */ -int sqlite3session_isempty(sqlite3_session *pSession); +SQLITE_API int sqlite3session_isempty(sqlite3_session *pSession); /* -** CAPI3REF: Create An Iterator To Traverse A Changeset +** CAPI3REF: Query for the amount of heap memory used by a session object. +** +** This API returns the total amount of heap memory in bytes currently +** used by the session object passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3session_memory_used(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** CONSTRUCTOR: sqlite3_changeset_iter ** ** Create an iterator used to iterate through the contents of a changeset. ** If successful, *pp is set to point to the iterator handle and SQLITE_OK ** is returned. Otherwise, if an error occurs, *pp is set to zero and an ** SQLite error code is returned. ** -** The following functions can be used to advance and query a changeset +** The following functions can be used to advance and query a changeset ** iterator created by this function: ** **
        @@ -8999,25 +11863,52 @@ int sqlite3session_isempty(sqlite3_session *pSession); ** ** Assuming the changeset blob was created by one of the ** [sqlite3session_changeset()], [sqlite3changeset_concat()] or -** [sqlite3changeset_invert()] functions, all changes within the changeset -** that apply to a single table are grouped together. This means that when -** an application iterates through a changeset using an iterator created by -** this function, all changes that relate to a single table are visted -** consecutively. There is no chance that the iterator will visit a change -** the applies to table X, then one for table Y, and then later on visit +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit ** another change for table X. +** +** The behavior of sqlite3changeset_start_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter. +** +** Note that the sqlite3changeset_start_v2() API is still experimental +** and therefore subject to change. */ -int sqlite3changeset_start( +SQLITE_API int sqlite3changeset_start( sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ int nChangeset, /* Size of changeset blob in bytes */ void *pChangeset /* Pointer to blob containing changeset */ ); +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset, /* Pointer to blob containing changeset */ + int flags /* SESSION_CHANGESETSTART_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_start_v2 +** +** The following flags may passed via the 4th parameter to +** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: +** +**
        SQLITE_CHANGESETAPPLY_INVERT
        +** Invert the changeset while iterating through it. This is equivalent to +** inverting a changeset using sqlite3changeset_invert() before applying it. +** It is an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETSTART_INVERT 0x0002 /* ** CAPI3REF: Advance A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** -** This function may only be used with iterators created by function +** This function may only be used with iterators created by the function ** [sqlite3changeset_start()]. If it is called on an iterator passed to ** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE ** is returned and the call has no effect. @@ -9028,18 +11919,19 @@ int sqlite3changeset_start( ** point to the first change in the changeset. Each subsequent call advances ** the iterator to point to the next change in the changeset (if any). If ** no error occurs and the iterator points to a valid change after a call -** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. ** Otherwise, if all changes in the changeset have already been visited, ** SQLITE_DONE is returned. ** -** If an error occurs, an SQLite error code is returned. Possible error -** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or ** SQLITE_NOMEM. */ -int sqlite3changeset_next(sqlite3_changeset_iter *pIter); +SQLITE_API int sqlite3changeset_next(sqlite3_changeset_iter *pIter); /* ** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator @@ -9047,24 +11939,29 @@ int sqlite3changeset_next(sqlite3_changeset_iter *pIter); ** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this ** is not the case, this function returns [SQLITE_MISUSE]. ** -** If argument pzTab is not NULL, then *pzTab is set to point to a -** nul-terminated utf-8 encoded string containing the name of the table -** affected by the current change. The buffer remains valid until either -** sqlite3changeset_next() is called on the iterator or until the -** conflict-handler function returns. If pnCol is not NULL, then *pnCol is -** set to the number of columns in the table affected by the change. If -** pbIncorrect is not NULL, then *pbIndirect is set to true (1) if the change +** Arguments pOp, pnCol and pzTab may not be NULL. Upon return, three +** outputs are set through these pointers: +** +** *pOp is set to one of [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], +** depending on the type of change that the iterator currently points to; +** +** *pnCol is set to the number of columns in the table affected by the change; and +** +** *pzTab is set to point to a nul-terminated utf-8 encoded string containing +** the name of the table affected by the current change. The buffer remains +** valid until either sqlite3changeset_next() is called on the iterator +** or until the conflict-handler function returns. +** +** If pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change ** is an indirect change, or false (0) otherwise. See the documentation for ** [sqlite3session_indirect()] for a description of direct and indirect -** changes. Finally, if pOp is not NULL, then *pOp is set to one of -** [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], depending on the -** type of change that the iterator currently points to. +** changes. ** ** If no error occurs, SQLITE_OK is returned. If an error does occur, an ** SQLite error code is returned. The values of the output variables may not ** be trusted in this case. */ -int sqlite3changeset_op( +SQLITE_API int sqlite3changeset_op( sqlite3_changeset_iter *pIter, /* Iterator object */ const char **pzTab, /* OUT: Pointer to table name */ int *pnCol, /* OUT: Number of columns in table */ @@ -9074,6 +11971,7 @@ int sqlite3changeset_op( /* ** CAPI3REF: Obtain The Primary Key Definition Of A Table +** METHOD: sqlite3_changeset_iter ** ** For each modified table, a changeset includes the following: ** @@ -9089,7 +11987,7 @@ int sqlite3changeset_op( ** 0x01 if the corresponding column is part of the tables primary key, or ** 0x00 if it is not. ** -** If argumet pnCol is not NULL, then *pnCol is set to the number of columns +** If argument pnCol is not NULL, then *pnCol is set to the number of columns ** in the table. ** ** If this function is called when the iterator does not point to a valid @@ -9097,7 +11995,7 @@ int sqlite3changeset_op( ** SQLITE_OK is returned and the output variables populated as described ** above. */ -int sqlite3changeset_pk( +SQLITE_API int sqlite3changeset_pk( sqlite3_changeset_iter *pIter, /* Iterator object */ unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ int *pnCol /* OUT: Number of entries in output array */ @@ -9105,11 +12003,12 @@ int sqlite3changeset_pk( /* ** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator ** created by [sqlite3changeset_start()]. In the latter case, the most recent -** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. ** Furthermore, it may only be called if the type of change that the iterator ** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, ** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. @@ -9119,15 +12018,15 @@ int sqlite3changeset_pk( ** [SQLITE_RANGE] is returned and *ppValue is set to NULL. ** ** If successful, this function sets *ppValue to point to a protected -** sqlite3_value object containing the iVal'th value from the vector of +** sqlite3_value object containing the iVal'th value from the vector of ** original row values stored as part of the UPDATE or DELETE change and -** returns SQLITE_OK. The name of the function comes from the fact that this +** returns SQLITE_OK. The name of the function comes from the fact that this ** is similar to the "old.*" columns available to update or delete triggers. ** ** If some other error occurs (e.g. an OOM condition), an SQLite error code ** is returned and *ppValue is set to NULL. */ -int sqlite3changeset_old( +SQLITE_API int sqlite3changeset_old( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int iVal, /* Column number */ sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ @@ -9135,11 +12034,12 @@ int sqlite3changeset_old( /* ** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** The pIter argument passed to this function may either be an iterator ** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator ** created by [sqlite3changeset_start()]. In the latter case, the most recent -** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. ** Furthermore, it may only be called if the type of change that the iterator ** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, ** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. @@ -9149,18 +12049,18 @@ int sqlite3changeset_old( ** [SQLITE_RANGE] is returned and *ppValue is set to NULL. ** ** If successful, this function sets *ppValue to point to a protected -** sqlite3_value object containing the iVal'th value from the vector of +** sqlite3_value object containing the iVal'th value from the vector of ** new row values stored as part of the UPDATE or INSERT change and ** returns SQLITE_OK. If the change is an UPDATE and does not include -** a new value for the requested column, *ppValue is set to NULL and -** SQLITE_OK returned. The name of the function comes from the fact that -** this is similar to the "new.*" columns available to update or delete +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete ** triggers. ** ** If some other error occurs (e.g. an OOM condition), an SQLite error code ** is returned and *ppValue is set to NULL. */ -int sqlite3changeset_new( +SQLITE_API int sqlite3changeset_new( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int iVal, /* Column number */ sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ @@ -9168,6 +12068,7 @@ int sqlite3changeset_new( /* ** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function should only be used with iterator objects passed to a ** conflict-handler callback by [sqlite3changeset_apply()] with either @@ -9180,14 +12081,14 @@ int sqlite3changeset_new( ** [SQLITE_RANGE] is returned and *ppValue is set to NULL. ** ** If successful, this function sets *ppValue to point to a protected -** sqlite3_value object containing the iVal'th value from the +** sqlite3_value object containing the iVal'th value from the ** "conflicting row" associated with the current conflict-handler callback ** and returns SQLITE_OK. ** ** If some other error occurs (e.g. an OOM condition), an SQLite error code ** is returned and *ppValue is set to NULL. */ -int sqlite3changeset_conflict( +SQLITE_API int sqlite3changeset_conflict( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int iVal, /* Column number */ sqlite3_value **ppValue /* OUT: Value from conflicting row */ @@ -9195,6 +12096,7 @@ int sqlite3changeset_conflict( /* ** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** METHOD: sqlite3_changeset_iter ** ** This function may only be called with an iterator passed to an ** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case @@ -9203,7 +12105,7 @@ int sqlite3changeset_conflict( ** ** In all other cases this function returns SQLITE_MISUSE. */ -int sqlite3changeset_fk_conflicts( +SQLITE_API int sqlite3changeset_fk_conflicts( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int *pnOut /* OUT: Number of FK violations */ ); @@ -9211,6 +12113,7 @@ int sqlite3changeset_fk_conflicts( /* ** CAPI3REF: Finalize A Changeset Iterator +** METHOD: sqlite3_changeset_iter ** ** This function is used to finalize an iterator allocated with ** [sqlite3changeset_start()]. @@ -9222,21 +12125,23 @@ int sqlite3changeset_fk_conflicts( ** call has no effect. ** ** If an error was encountered within a call to an sqlite3changeset_xxx() -** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an ** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding ** to that error is returned by this function. Otherwise, SQLITE_OK is ** returned. This is to allow the following pattern (pseudo-code): ** +** 
         **   sqlite3changeset_start();
 **   while( SQLITE_ROW==sqlite3changeset_next() ){
 **     // Do something with change.
 **   }
 **   rc = sqlite3changeset_finalize();
 **   if( rc!=SQLITE_OK ){
-**     // An error has occurred 
+**     // An error has occurred
 **   }
+**
        */ -int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); +SQLITE_API int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); /* ** CAPI3REF: Invert A Changeset @@ -9260,13 +12165,13 @@ int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); ** zeroed and an SQLite error code returned. ** ** It is the responsibility of the caller to eventually call sqlite3_free() -** on the *ppOut pointer to free the buffer allocation following a successful +** on the *ppOut pointer to free the buffer allocation following a successful ** call to this function. ** ** WARNING/TODO: This function currently assumes that the input is a valid ** changeset. If it is not, the results are undefined. */ -int sqlite3changeset_invert( +SQLITE_API int sqlite3changeset_invert( int nIn, const void *pIn, /* Input changeset */ int *pnOut, void **ppOut /* OUT: Inverse of input */ ); @@ -9274,14 +12179,15 @@ int sqlite3changeset_invert( /* ** CAPI3REF: Concatenate Two Changeset Objects ** -** This function is used to concatenate two changesets, A and B, into a +** This function is used to concatenate two changesets, A and B, into a ** single changeset. The result is a changeset equivalent to applying -** changeset A followed by changeset B. +** changeset A followed by changeset B. ** -** This function combines the two input changesets using an +** This function combines the two input changesets using an ** sqlite3_changegroup object. Calling it produces similar results as the ** following code fragment: ** +** 
         **   sqlite3_changegroup *pGrp;
 **   rc = sqlite3_changegroup_new(&pGrp);
 **   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
@@ -9292,10 +12198,11 @@ int sqlite3changeset_invert(
 **     *ppOut = 0;
 **     *pnOut = 0;
 **   }
+**
        ** ** Refer to the sqlite3_changegroup documentation below for details. */ -int sqlite3changeset_concat( +SQLITE_API int sqlite3changeset_concat( int nA, /* Number of bytes in buffer pA */ void *pA, /* Pointer to buffer containing changeset A */ int nB, /* Number of bytes in buffer pB */ @@ -9306,12 +12213,28 @@ int sqlite3changeset_concat( /* -** Changegroup handle. +** CAPI3REF: Upgrade the Schema of a Changeset/Patchset +*/ +SQLITE_API int sqlite3changeset_upgrade( + sqlite3 *db, + const char *zDb, + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + + + +/* +** CAPI3REF: Changegroup Handle +** +** A changegroup is an object used to combine two or more +** [changesets] or [patchsets] */ typedef struct sqlite3_changegroup sqlite3_changegroup; /* -** CAPI3REF: Combine two or more changesets into a single changeset. +** CAPI3REF: Create A New Changegroup Object +** CONSTRUCTOR: sqlite3_changegroup ** ** An sqlite3_changegroup object is used to combine two or more changesets ** (or patchsets) into a single changeset (or patchset). A single changegroup @@ -9320,7 +12243,7 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; ** ** If successful, this function returns SQLITE_OK and populates (*pp) with ** a pointer to a new sqlite3_changegroup object before returning. The caller -** should eventually free the returned object using a call to +** should eventually free the returned object using a call to ** sqlite3changegroup_delete(). If an error occurs, an SQLite error code ** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. ** @@ -9332,7 +12255,7 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; **
      • Zero or more changesets (or patchsets) are added to the object ** by calling sqlite3changegroup_add(). ** -**
      • The result of combining all input changesets together is obtained +**
      • The result of combining all input changesets together is obtained ** by the application via a call to sqlite3changegroup_output(). ** **
      • The object is deleted using a call to sqlite3changegroup_delete(). @@ -9341,15 +12264,50 @@ typedef struct sqlite3_changegroup sqlite3_changegroup; ** Any number of calls to add() and output() may be made between the calls to ** new() and delete(), and in any order. ** -** As well as the regular sqlite3changegroup_add() and +** As well as the regular sqlite3changegroup_add() and ** sqlite3changegroup_output() functions, also available are the streaming ** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). */ -int sqlite3changegroup_new(sqlite3_changegroup **pp); +SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); /* +** CAPI3REF: Add a Schema to a Changegroup +** METHOD: sqlite3_changegroup_schema +** +** This method may be used to optionally enforce the rule that the changesets +** added to the changegroup handle must match the schema of database zDb +** ("main", "temp", or the name of an attached database). If +** sqlite3changegroup_add() is called to add a changeset that is not compatible +** with the configured schema, SQLITE_SCHEMA is returned and the changegroup +** object is left in an undefined state. +** +** A changeset schema is considered compatible with the database schema in +** the same way as for sqlite3changeset_apply(). Specifically, for each +** table in the changeset, there exists a database table with: +** +**
          +**
        • The name identified by the changeset, and +**
        • at least as many columns as recorded in the changeset, and +**
        • the primary key columns in the same position as recorded in +** the changeset. +**
        +** +** The output of the changegroup object always has the same schema as the +** database nominated using this function. In cases where changesets passed +** to sqlite3changegroup_add() have fewer columns than the corresponding table +** in the database schema, these are filled in using the default column +** values from the database schema. This makes it possible to combined +** changesets that have different numbers of columns for a single table +** within a changegroup, provided that they are otherwise compatible. +*/ +SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** METHOD: sqlite3_changegroup +** ** Add all changes within the changeset (or patchset) in buffer pData (size -** nData bytes) to the changegroup. +** nData bytes) to the changegroup. ** ** If the buffer contains a patchset, then all prior calls to this function ** on the same changegroup object must also have specified patchsets. Or, if @@ -9362,7 +12320,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp); ** apply to the same row as a change already present in the changegroup if ** the two rows have the same primary key. ** -** Changes to rows that that do not already appear in the changegroup are +** Changes to rows that do not already appear in the changegroup are ** simply copied into it. Or, if both the new changeset and the changegroup ** contain changes that apply to a single row, the final contents of the ** changegroup depends on the type of each change, as follows: @@ -9376,7 +12334,7 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp); ** changeset was recorded immediately after the changesets already ** added to the changegroup. ** INSERT UPDATE -** The INSERT change remains in the changegroup. The values in the +** The INSERT change remains in the changegroup. The values in the ** INSERT change are modified as if the row was inserted by the ** existing change and then updated according to the new change. ** INSERT DELETE @@ -9387,17 +12345,17 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp); ** changeset was recorded immediately after the changesets already ** added to the changegroup. ** UPDATE UPDATE -** The existing UPDATE remains within the changegroup. It is amended -** so that the accompanying values are as if the row was updated once +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once ** by the existing change and then again by the new change. ** UPDATE DELETE ** The existing UPDATE is replaced by the new DELETE within the ** changegroup. ** DELETE INSERT ** If one or more of the column values in the row inserted by the -** new change differ from those in the row deleted by the existing +** new change differ from those in the row deleted by the existing ** change, the existing DELETE is replaced by an UPDATE within the -** changegroup. Otherwise, if the inserted row is exactly the same +** changegroup. Otherwise, if the inserted row is exactly the same ** as the deleted row, the existing DELETE is simply discarded. ** DELETE UPDATE ** The new change is ignored. This case does not occur if the new @@ -9412,17 +12370,49 @@ int sqlite3changegroup_new(sqlite3_changegroup **pp); ** If the new changeset contains changes to a table that is already present ** in the changegroup, then the number of columns and the position of the ** primary key columns for the table must be consistent. If this is not the -** case, this function fails with SQLITE_SCHEMA. If the input changeset -** appears to be corrupt and the corruption is detected, SQLITE_CORRUPT is -** returned. Or, if an out-of-memory condition occurs during processing, this -** function returns SQLITE_NOMEM. In all cases, if an error occurs the -** final contents of the changegroup is undefined. +** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup +** object has been configured with a database schema using the +** sqlite3changegroup_schema() API, then it is possible to combine changesets +** with different numbers of columns for a single table, provided that +** they are otherwise compatible. ** -** If no error occurs, SQLITE_OK is returned. +** If the input changeset appears to be corrupt and the corruption is +** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition +** occurs during processing, this function returns SQLITE_NOMEM. +** +** In all cases, if an error occurs the state of the final contents of the +** changegroup is undefined. If no error occurs, SQLITE_OK is returned. */ -int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); +SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); /* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** METHOD: sqlite3_changegroup +** ** Obtain a buffer containing a changeset (or patchset) representing the ** current contents of the changegroup. If the inputs to the changegroup ** were themselves changesets, the output is a changeset. Or, if the @@ -9439,49 +12429,49 @@ int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); ** ** If an error occurs, an SQLite error code is returned and the output ** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK -** is returned and the output variables are set to the size of and a +** is returned and the output variables are set to the size of and a ** pointer to the output buffer, respectively. In this case it is the ** responsibility of the caller to eventually free the buffer using a ** call to sqlite3_free(). */ -int sqlite3changegroup_output( +SQLITE_API int sqlite3changegroup_output( sqlite3_changegroup*, int *pnData, /* OUT: Size of output buffer in bytes */ void **ppData /* OUT: Pointer to output buffer */ ); /* -** Delete a changegroup object. +** CAPI3REF: Delete A Changegroup Object +** DESTRUCTOR: sqlite3_changegroup */ -void sqlite3changegroup_delete(sqlite3_changegroup*); +SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); /* ** CAPI3REF: Apply A Changeset To A Database ** -** Apply a changeset to a database. This function attempts to update the -** "main" database attached to handle db with the changes found in the -** changeset passed via the second and third arguments. +** Apply a changeset or patchset to a database. These functions attempt to +** update the "main" database attached to handle db with the changes found in +** the changeset passed via the second and third arguments. ** -** The fourth argument (xFilter) passed to this function is the "filter +** The fourth argument (xFilter) passed to these functions is the "filter ** callback". If it is not NULL, then for each table affected by at least one ** change in the changeset, the filter callback is invoked with ** the table name as the second argument, and a copy of the context pointer -** passed as the sixth argument to this function as the first. If the "filter -** callback" returns zero, then no attempt is made to apply any changes to -** the table. Otherwise, if the return value is non-zero or the xFilter -** argument to this function is NULL, all changes related to the table are -** attempted. +** passed as the sixth argument as the first. If the "filter callback" +** returns zero, then no attempt is made to apply any changes to the table. +** Otherwise, if the return value is non-zero or the xFilter argument to +** is NULL, all changes related to the table are attempted. ** -** For each table that is not excluded by the filter callback, this function -** tests that the target database contains a compatible table. A table is +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is ** considered compatible if all of the following are true: ** **
          -**
        • The table has the same name as the name recorded in the +**
        • The table has the same name as the name recorded in the ** changeset, and -**
        • The table has the same number of columns as recorded in the +**
        • The table has at least as many columns as recorded in the ** changeset, and -**
        • The table has primary key columns in the same position as +**
        • The table has primary key columns in the same position as ** recorded in the changeset. **
        ** @@ -9490,11 +12480,11 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most ** one such warning is issued for each table in the changeset. ** -** For each change for which there is a compatible table, an attempt is made -** to modify the table contents according to the UPDATE, INSERT or DELETE -** change. If a change cannot be applied cleanly, the conflict handler -** function passed as the fifth argument to sqlite3changeset_apply() may be -** invoked. A description of exactly when the conflict handler is invoked for +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for ** each type of change is below. ** ** Unlike the xFilter argument, xConflict may not be passed NULL. The results @@ -9502,29 +12492,33 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** argument are undefined. ** ** Each time the conflict handler function is invoked, it must return one -** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or ** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned ** if the second argument passed to the conflict handler is either ** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler ** returns an illegal value, any changes already made are rolled back and -** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different ** actions are taken by sqlite3changeset_apply() depending on the value ** returned by each invocation of the conflict-handler function. Refer to -** the documentation for the three +** the documentation for the three ** [SQLITE_CHANGESET_OMIT|available return values] for details. ** **
        **
        DELETE Changes
        -** For each DELETE change, this function checks if the target database -** contains a row with the same primary key value (or values) as the -** original row values stored in the changeset. If it does, and the values -** stored in all non-primary key columns also match the values stored in +** For each DELETE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in ** the changeset the row is deleted from the target database. ** ** If a row with matching primary key values is found, but one or more of ** the non-primary key fields contains a value different from the original ** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. ** ** If no row with matching primary key values is found in the database, ** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] @@ -9539,31 +12533,33 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** **
        INSERT Changes
        ** For each INSERT change, an attempt is made to insert the new row into -** the database. +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. ** -** If the attempt to insert the row fails because the database already +** If the attempt to insert the row fails because the database already ** contains a row with the same primary key values, the conflict handler -** function is invoked with the second argument set to +** function is invoked with the second argument set to ** [SQLITE_CHANGESET_CONFLICT]. ** ** If the attempt to insert the row fails because of some other constraint -** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is ** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. -** This includes the case where the INSERT operation is re-attempted because -** an earlier call to the conflict handler function returned +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned ** [SQLITE_CHANGESET_REPLACE]. ** **
        UPDATE Changes
        -** For each UPDATE change, this function checks if the target database -** contains a row with the same primary key value (or values) as the -** original row values stored in the changeset. If it does, and the values -** stored in all non-primary key columns also match the values stored in -** the changeset the row is updated within the target database. +** For each UPDATE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. ** ** If a row with matching primary key values is found, but one or more of -** the non-primary key fields contains a value different from an original -** row value stored in the changeset, the conflict-handler function is -** invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since ** UPDATE changes only contain values for non-primary key fields that are ** to be modified, only those fields need to match the original values to ** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. @@ -9572,26 +12568,43 @@ void sqlite3changegroup_delete(sqlite3_changegroup*); ** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] ** passed as the second argument. ** -** If the UPDATE operation is attempted, but SQLite returns -** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with ** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. -** This includes the case where the UPDATE operation is attempted after +** This includes the case where the UPDATE operation is attempted after ** an earlier call to the conflict handler function returned -** [SQLITE_CHANGESET_REPLACE]. +** [SQLITE_CHANGESET_REPLACE]. **
        ** ** It is safe to execute SQL statements, including those that write to the ** table that the callback related to, from within the xConflict callback. -** This can be used to further customize the applications conflict +** This can be used to further customize the application's conflict ** resolution strategy. ** -** All changes made by this function are enclosed in a savepoint transaction. +** All changes made by these functions are enclosed in a savepoint transaction. ** If any other error (aside from a constraint failure when attempting to ** write to the target database) occurs, then the savepoint transaction is -** rolled back, restoring the target database to its original state, and an +** rolled back, restoring the target database to its original state, and an ** SQLite error code returned. +** +** If the output parameters (ppRebase) and (pnRebase) are non-NULL and +** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() +** may set (*ppRebase) to point to a "rebase" that may be used with the +** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase) +** is set to the size of the buffer in bytes. It is the responsibility of the +** caller to eventually free any such buffer using sqlite3_free(). The buffer +** is only allocated and populated if one or more conflicts were encountered +** while applying the patchset. See comments surrounding the sqlite3_rebaser +** APIs for further details. +** +** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter. +** +** Note that the sqlite3changeset_apply_v2() API is still experimental +** and therefore subject to change. */ -int sqlite3changeset_apply( +SQLITE_API int sqlite3changeset_apply( sqlite3 *db, /* Apply change to "main" db of this handle */ int nChangeset, /* Size of changeset in bytes */ void *pChangeset, /* Changeset blob */ @@ -9606,8 +12619,70 @@ int sqlite3changeset_apply( ), void *pCtx /* First argument passed to xConflict */ ); +SQLITE_API int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); -/* +/* +** CAPI3REF: Flags for sqlite3changeset_apply_v2 +** +** The following flags may passed via the 9th parameter to +** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]: +** +**
        +**
        SQLITE_CHANGESETAPPLY_NOSAVEPOINT
        +** Usually, the sessions module encloses all operations performed by +** a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The +** SAVEPOINT is committed if the changeset or patchset is successfully +** applied, or rolled back if an error occurs. Specifying this flag +** causes the sessions module to omit this savepoint. In this case, if the +** caller has an open transaction or savepoint when apply_v2() is called, +** it may revert the partially applied changeset by rolling it back. +** +**
        SQLITE_CHANGESETAPPLY_INVERT
        +** Invert the changeset before applying it. This is equivalent to inverting +** a changeset using sqlite3changeset_invert() before applying it. It is +** an error to specify this flag with a patchset. +** +**
        SQLITE_CHANGESETAPPLY_IGNORENOOP
        +** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
          +**
        • a delete change if the row being deleted cannot be found, +**
        • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
        • an insert change if all fields of the conflicting row match +** the row being inserted. +**
        +** +**
        SQLITE_CHANGESETAPPLY_FKNOACTION
        +** If this flag it set, then all foreign key constraints in the target +** database behave as if they were declared with "ON UPDATE NO ACTION ON +** DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL +** or SET DEFAULT. +*/ +#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 +#define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 +#define SQLITE_CHANGESETAPPLY_FKNOACTION 0x0008 + +/* ** CAPI3REF: Constants Passed To The Conflict Handler ** ** Values that may be passed as the second argument to a conflict-handler. @@ -9616,32 +12691,32 @@ int sqlite3changeset_apply( **
        SQLITE_CHANGESET_DATA
        ** The conflict handler is invoked with CHANGESET_DATA as the second argument ** when processing a DELETE or UPDATE change if a row with the required -** PRIMARY KEY fields is present in the database, but one or more other -** (non primary-key) fields modified by the update do not contain the +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the ** expected "before" values. -** +** ** The conflicting row, in this case, is the database row with the matching ** primary key. -** +** **
        SQLITE_CHANGESET_NOTFOUND
        ** The conflict handler is invoked with CHANGESET_NOTFOUND as the second ** argument when processing a DELETE or UPDATE change if a row with the ** required PRIMARY KEY fields is not present in the database. -** +** ** There is no conflicting row in this case. The results of invoking the ** sqlite3changeset_conflict() API are undefined. -** +** **
        SQLITE_CHANGESET_CONFLICT
        ** CHANGESET_CONFLICT is passed as the second argument to the conflict -** handler while processing an INSERT change if the operation would result +** handler while processing an INSERT change if the operation would result ** in duplicate primary key values. -** +** ** The conflicting row in this case is the database row with the matching ** primary key. ** **
        SQLITE_CHANGESET_FOREIGN_KEY
        ** If foreign key handling is enabled, and applying a changeset leaves the -** database in a state containing foreign key violations, the conflict +** database in a state containing foreign key violations, the conflict ** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument ** exactly once before the changeset is committed. If the conflict handler ** returns CHANGESET_OMIT, the changes, including those that caused the @@ -9651,12 +12726,12 @@ int sqlite3changeset_apply( ** No current or conflicting row information is provided. The only function ** it is possible to call on the supplied sqlite3_changeset_iter handle ** is sqlite3changeset_fk_conflicts(). -** +** **
        SQLITE_CHANGESET_CONSTRAINT
        -** If any other constraint violation occurs while applying a change (i.e. -** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is ** invoked with CHANGESET_CONSTRAINT as the second argument. -** +** ** There is no conflicting row in this case. The results of invoking the ** sqlite3changeset_conflict() API are undefined. ** @@ -9668,7 +12743,7 @@ int sqlite3changeset_apply( #define SQLITE_CHANGESET_CONSTRAINT 4 #define SQLITE_CHANGESET_FOREIGN_KEY 5 -/* +/* ** CAPI3REF: Constants Returned By The Conflict Handler ** ** A conflict handler callback must return one of the following three values. @@ -9676,13 +12751,13 @@ int sqlite3changeset_apply( **
        **
        SQLITE_CHANGESET_OMIT
        ** If a conflict handler returns this value no special action is taken. The -** change that caused the conflict is not applied. The session module +** change that caused the conflict is not applied. The session module ** continues to the next change in the changeset. ** **
        SQLITE_CHANGESET_REPLACE
        ** This value may only be returned if the second argument to the conflict ** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this -** is not the case, any changes applied so far are rolled back and the +** is not the case, any changes applied so far are rolled back and the ** call to sqlite3changeset_apply() returns SQLITE_MISUSE. ** ** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict @@ -9695,7 +12770,7 @@ int sqlite3changeset_apply( ** the original row is restored to the database before continuing. ** **
        SQLITE_CHANGESET_ABORT
        -** If this value is returned, any changes applied so far are rolled back +** If this value is returned, any changes applied so far are rolled back ** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. **
        */ @@ -9703,27 +12778,183 @@ int sqlite3changeset_apply( #define SQLITE_CHANGESET_REPLACE 1 #define SQLITE_CHANGESET_ABORT 2 +/* +** CAPI3REF: Rebasing changesets +** EXPERIMENTAL +** +** Suppose there is a site hosting a database in state S0. And that +** modifications are made that move that database to state S1 and a +** changeset recorded (the "local" changeset). Then, a changeset based +** on S0 is received from another site (the "remote" changeset) and +** applied to the database. The database is then in state +** (S1+"remote"), where the exact state depends on any conflict +** resolution decisions (OMIT or REPLACE) made while applying "remote". +** Rebasing a changeset is to update it to take those conflict +** resolution decisions into account, so that the same conflicts +** do not have to be resolved elsewhere in the network. +** +** For example, if both the local and remote changesets contain an +** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)": +** +** local: INSERT INTO t1 VALUES(1, 'v1'); +** remote: INSERT INTO t1 VALUES(1, 'v2'); +** +** and the conflict resolution is REPLACE, then the INSERT change is +** removed from the local changeset (it was overridden). Or, if the +** conflict resolution was "OMIT", then the local changeset is modified +** to instead contain: +** +** UPDATE t1 SET b = 'v2' WHERE a=1; +** +** Changes within the local changeset are rebased as follows: +** +**
        +**
        Local INSERT
        +** This may only conflict with a remote INSERT. If the conflict +** resolution was OMIT, then add an UPDATE change to the rebased +** changeset. Or, if the conflict resolution was REPLACE, add +** nothing to the rebased changeset. +** +**
        Local DELETE
        +** This may conflict with a remote UPDATE or DELETE. In both cases the +** only possible resolution is OMIT. If the remote operation was a +** DELETE, then add no change to the rebased changeset. If the remote +** operation was an UPDATE, then the old.* fields of change are updated +** to reflect the new.* values in the UPDATE. +** +**
        Local UPDATE
        +** This may conflict with a remote UPDATE or DELETE. If it conflicts +** with a DELETE, and the conflict resolution was OMIT, then the update +** is changed into an INSERT. Any undefined values in the new.* record +** from the update change are filled in using the old.* values from +** the conflicting DELETE. Or, if the conflict resolution was REPLACE, +** the UPDATE change is simply omitted from the rebased changeset. +** +** If conflict is with a remote UPDATE and the resolution is OMIT, then +** the old.* values are rebased using the new.* values in the remote +** change. Or, if the resolution is REPLACE, then the change is copied +** into the rebased changeset with updates to columns also updated by +** the conflicting remote UPDATE removed. If this means no columns would +** be updated, the change is omitted. +**
        +** +** A local change may be rebased against multiple remote changes +** simultaneously. If a single key is modified by multiple remote +** changesets, they are combined as follows before the local changeset +** is rebased: +** +**
          +**
        • If there has been one or more REPLACE resolutions on a +** key, it is rebased according to a REPLACE. +** +**
        • If there have been no REPLACE resolutions on a key, then +** the local changeset is rebased according to the most recent +** of the OMIT resolutions. +**
        +** +** Note that conflict resolutions from multiple remote changesets are +** combined on a per-field basis, not per-row. This means that in the +** case of multiple remote UPDATE operations, some fields of a single +** local change may be rebased for REPLACE while others are rebased for +** OMIT. +** +** In order to rebase a local changeset, the remote changeset must first +** be applied to the local database using sqlite3changeset_apply_v2() and +** the buffer of rebase information captured. Then: +** +**
          +**
        1. An sqlite3_rebaser object is created by calling +** sqlite3rebaser_create(). +**
        2. The new object is configured with the rebase buffer obtained from +** sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure(). +** If the local changeset is to be rebased against multiple remote +** changesets, then sqlite3rebaser_configure() should be called +** multiple times, in the same order that the multiple +** sqlite3changeset_apply_v2() calls were made. +**
        3. Each local changeset is rebased by calling sqlite3rebaser_rebase(). +**
        4. The sqlite3_rebaser object is deleted by calling +** sqlite3rebaser_delete(). +**
        +*/ +typedef struct sqlite3_rebaser sqlite3_rebaser; + +/* +** CAPI3REF: Create a changeset rebaser object. +** EXPERIMENTAL +** +** Allocate a new changeset rebaser object. If successful, set (*ppNew) to +** point to the new object and return SQLITE_OK. Otherwise, if an error +** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew) +** to NULL. +*/ +SQLITE_API int sqlite3rebaser_create(sqlite3_rebaser **ppNew); + +/* +** CAPI3REF: Configure a changeset rebaser object. +** EXPERIMENTAL +** +** Configure the changeset rebaser object to rebase changesets according +** to the conflict resolutions described by buffer pRebase (size nRebase +** bytes), which must have been obtained from a previous call to +** sqlite3changeset_apply_v2(). +*/ +SQLITE_API int sqlite3rebaser_configure( + sqlite3_rebaser*, + int nRebase, const void *pRebase +); + +/* +** CAPI3REF: Rebase a changeset +** EXPERIMENTAL +** +** Argument pIn must point to a buffer containing a changeset nIn bytes +** in size. This function allocates and populates a buffer with a copy +** of the changeset rebased according to the configuration of the +** rebaser object passed as the first argument. If successful, (*ppOut) +** is set to point to the new buffer containing the rebased changeset and +** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the +** responsibility of the caller to eventually free the new buffer using +** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) +** are set to zero and an SQLite error code returned. +*/ +SQLITE_API int sqlite3rebaser_rebase( + sqlite3_rebaser*, + int nIn, const void *pIn, + int *pnOut, void **ppOut +); + +/* +** CAPI3REF: Delete a changeset rebaser object. +** EXPERIMENTAL +** +** Delete the changeset rebaser object and all associated resources. There +** should be one call to this function for each successful invocation +** of sqlite3rebaser_create(). +*/ +SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p); + /* ** CAPI3REF: Streaming Versions of API functions. ** -** The six streaming API xxx_strm() functions serve similar purposes to the +** The six streaming API xxx_strm() functions serve similar purposes to the ** corresponding non-streaming API functions: ** ** ** -**
        Streaming functionNon-streaming equivalent
        sqlite3changeset_apply_str[sqlite3changeset_apply] -**
        sqlite3changeset_concat_str[sqlite3changeset_concat] -**
        sqlite3changeset_invert_str[sqlite3changeset_invert] -**
        sqlite3changeset_start_str[sqlite3changeset_start] -**
        sqlite3session_changeset_str[sqlite3session_changeset] -**
        sqlite3session_patchset_str[sqlite3session_patchset] +**
        sqlite3changeset_apply_strm[sqlite3changeset_apply] +**
        sqlite3changeset_apply_strm_v2[sqlite3changeset_apply_v2] +**
        sqlite3changeset_concat_strm[sqlite3changeset_concat] +**
        sqlite3changeset_invert_strm[sqlite3changeset_invert] +**
        sqlite3changeset_start_strm[sqlite3changeset_start] +**
        sqlite3session_changeset_strm[sqlite3session_changeset] +**
        sqlite3session_patchset_strm[sqlite3session_patchset] **
        ** ** Non-streaming functions that accept changesets (or patchsets) as input -** require that the entire changeset be stored in a single buffer in memory. -** Similarly, those that return a changeset or patchset do so by returning -** a pointer to a single large buffer allocated using sqlite3_malloc(). -** Normally this is convenient. However, if an application running in a +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a ** low-memory environment is required to handle very large changesets, the ** large contiguous memory allocations required can become onerous. ** @@ -9745,12 +12976,12 @@ int sqlite3changeset_apply( ** ** ** Each time the xInput callback is invoked by the sessions module, the first -** argument passed is a copy of the supplied pIn context pointer. The second -** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no -** error occurs the xInput method should copy up to (*pnData) bytes of data -** into the buffer and set (*pnData) to the actual number of bytes copied -** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) -** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite ** error code should be returned. In all cases, if an xInput callback returns ** an error, all processing is abandoned and the streaming API function ** returns a copy of the error code to the caller. @@ -9758,7 +12989,7 @@ int sqlite3changeset_apply( ** In the case of sqlite3changeset_start_strm(), the xInput callback may be ** invoked by the sessions module at any point during the lifetime of the ** iterator. If such an xInput callback returns an error, the iterator enters -** an error state, whereby all subsequent calls to iterator functions +** an error state, whereby all subsequent calls to iterator functions ** immediately fail with the same error code as returned by xInput. ** ** Similarly, streaming API functions that return changesets (or patchsets) @@ -9788,11 +13019,11 @@ int sqlite3changeset_apply( ** is immediately abandoned and the streaming API function returns a copy ** of the xOutput error code to the application. ** -** The sessions module never invokes an xOutput callback with the third +** The sessions module never invokes an xOutput callback with the third ** parameter set to a value less than or equal to zero. Other than this, ** no guarantees are made as to the size of the chunks of data returned. */ -int sqlite3changeset_apply_strm( +SQLITE_API int sqlite3changeset_apply_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ void *pIn, /* First arg for xInput */ @@ -9807,7 +13038,24 @@ int sqlite3changeset_apply_strm( ), void *pCtx /* First argument passed to xConflict */ ); -int sqlite3changeset_concat_strm( +SQLITE_API int sqlite3changeset_apply_v2_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); +SQLITE_API int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, int (*xInputB)(void *pIn, void *pData, int *pnData), @@ -9815,36 +13063,88 @@ int sqlite3changeset_concat_strm( int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ); -int sqlite3changeset_invert_strm( +SQLITE_API int sqlite3changeset_invert_strm( int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ); -int sqlite3changeset_start_strm( +SQLITE_API int sqlite3changeset_start_strm( sqlite3_changeset_iter **pp, int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn ); -int sqlite3session_changeset_strm( +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +); +SQLITE_API int sqlite3session_changeset_strm( sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ); -int sqlite3session_patchset_strm( +SQLITE_API int sqlite3session_patchset_strm( sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ); -int sqlite3changegroup_add_strm(sqlite3_changegroup*, +SQLITE_API int sqlite3changegroup_add_strm(sqlite3_changegroup*, int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn ); -int sqlite3changegroup_output_strm(sqlite3_changegroup*, - int (*xOutput)(void *pOut, const void *pData, int nData), +SQLITE_API int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ); +SQLITE_API int sqlite3rebaser_rebase_strm( + sqlite3_rebaser *pRebaser, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +**
        +**
        SQLITE_SESSION_CONFIG_STRMSIZE
        +** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +**
        +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 /* ** Make sure we can call this stuff from C++. @@ -9869,7 +13169,7 @@ int sqlite3changegroup_output_strm(sqlite3_changegroup*, ** ****************************************************************************** ** -** Interfaces to extend FTS5. Using the interfaces defined in this file, +** Interfaces to extend FTS5. Using the interfaces defined in this file, ** FTS5 may be extended with: ** ** * custom tokenizers, and @@ -9913,19 +13213,19 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken ** to the total number of tokens in the FTS5 table. Or, if iCol is ** non-negative but less than the number of columns in the table, return -** the total number of tokens in column iCol, considering all rows in +** the total number of tokens in column iCol, considering all rows in ** the FTS5 table. ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. -** an OOM condition or IO error), an appropriate SQLite error code is +** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** xColumnCount(pFts): @@ -9939,15 +13239,18 @@ struct Fts5PhraseIter { ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. -** an OOM condition or IO error), an appropriate SQLite error code is +** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** This function may be quite inefficient if used with an FTS5 table ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -9957,8 +13260,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -9966,27 +13271,24 @@ struct Fts5PhraseIter { ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. If the FTS5 table is created -** with either "detail=none" or "detail=column" and "content=" option +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always returns 0. ** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. The exception is if the table was created -** with the offsets=0 option specified. In this case *piOff is always -** set to -1. -** -** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) -** if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. +** "detail=none" or "detail=column" option. ** ** xRowid: ** Returns the rowid of the current row. @@ -10002,13 +13304,17 @@ struct Fts5PhraseIter { ** ** with $p set to a phrase equivalent to the phrase iPhrase of the ** current query is executed. Any column filter that applies to -** phrase iPhrase of the current query is included in $p. For each -** row visited, the callback function passed as the fourth argument -** is invoked. The context and API objects passed to the callback +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback ** function may be used to access the properties of each matched row. -** Invoking Api.xUserData() returns a copy of the pointer passed as +** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -10021,14 +13327,14 @@ struct Fts5PhraseIter { ** ** xSetAuxdata(pFts5, pAux, xDelete) ** -** Save the pointer passed as the second argument as the extension functions +** Save the pointer passed as the second argument as the extension function's ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for -** each FTS query (MATCH expression). If the extension function is invoked -** more than once for a single FTS query, then all invocations share a +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a ** single auxiliary data context. ** ** If there is already an auxiliary data pointer when this function is @@ -10039,7 +13345,7 @@ struct Fts5PhraseIter { ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning. @@ -10047,7 +13353,7 @@ struct Fts5PhraseIter { ** ** xGetAuxdata(pFts5, bClear) ** -** Returns the current auxiliary data pointer for the fts5 extension +** Returns the current auxiliary data pointer for the fts5 extension ** function. See the xSetAuxdata() method for details. ** ** If the bClear argument is non-zero, then the auxiliary data is cleared @@ -10067,7 +13373,7 @@ struct Fts5PhraseIter { ** method, to iterate through all instances of a single query phrase within ** the current row. This is the same information as is accessible via the ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient -** to use, this API may be faster under some circumstances. To iterate +** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; @@ -10085,11 +13391,15 @@ struct Fts5PhraseIter { ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. If the FTS5 table is created -** with either "detail=none" or "detail=column" and "content=" option +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -10110,22 +13420,78 @@ struct Fts5PhraseIter { ** } ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" option. If the FTS5 table is created with either -** "detail=none" "content=" option (i.e. if it is a contentless table), -** then this API always iterates through an empty set (all calls to +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to ** xPhraseFirstColumn() set iCol to -1). ** ** The information accessed using this API and its companion ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext ** (or xInst/xInstCount). The chief advantage of this API is that it is ** significantly more efficient than those alternatives when used with -** "detail=column" tables. +** "detail=column" tables. ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -10133,7 +13499,7 @@ struct Fts5ExtensionApi { int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); - int (*xTokenize)(Fts5Context*, + int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ @@ -10160,37 +13526,53 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; -/* +/* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* ** CUSTOM TOKENIZERS ** -** Applications may also register custom tokenizer types. A tokenizer -** is registered by providing fts5 with a populated instance of the +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the ** following structure. All structure methods must be defined, setting ** any member of the fts5_tokenizer struct to NULL leads to undefined ** behaviour. The structure methods are expected to function as follows: ** ** xCreate: -** This function is used to allocate and inititalize a tokenizer instance. +** This function is used to allocate and initialize a tokenizer instance. ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object -** was registered with FTS5 (the third argument to xCreateTokenizer()). +** pointer provided by the application when the fts5_tokenizer_v2 object +** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the ** tokenizer name as part of the CREATE VIRTUAL TABLE statement used ** to create the FTS5 table. ** -** The final argument is an output variable. If successful, (*ppOut) +** The final argument is an output variable. If successful, (*ppOut) ** should be set to point to the new tokenizer handle and SQLITE_OK ** returned. If an error occurs, some value other than SQLITE_OK should -** be returned. In this case, fts5 assumes that the final value of *ppOut +** be returned. In this case, fts5 assumes that the final value of *ppOut ** is undefined. ** ** xDelete: @@ -10199,12 +13581,12 @@ struct Fts5ExtensionApi { ** be invoked exactly once for each successful call to xCreate(). ** ** xTokenize: -** This function is expected to tokenize the nText byte string indicated +** This function is expected to tokenize the nText byte string indicated ** by argument pText. pText may or may not be nul-terminated. The first ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -10213,8 +13595,8 @@ struct Fts5ExtensionApi { ** determine the set of tokens to add to (or delete from) the ** FTS index. ** -**
      • FTS5_TOKENIZE_QUERY - A MATCH query is being executed -** against the FTS index. The tokenizer is being called to tokenize +**
      • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize ** a bareword or quoted string specified as part of the query. ** **
      • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as @@ -10222,12 +13604,19 @@ struct Fts5ExtensionApi { ** followed by a "*" character, indicating that the last token ** returned by the tokenizer will be treated as a token prefix. ** -**
      • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +**
      • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to ** satisfy an fts5_api.xTokenize() request made by an auxiliary ** function. Or an fts5_api.xColumnSize() request made by the same -** on a columnsize=0 database. +** on a columnsize=0 database. **
      ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -10237,10 +13626,10 @@ struct Fts5ExtensionApi { ** which the token is derived within the input. ** ** The second argument passed to the xToken() callback ("tflags") should -** normally be set to 0. The exception is if the tokenizer supports +** normally be set to 0. The exception is if the tokenizer supports ** synonyms. In this case see the discussion below for details. ** -** FTS5 assumes the xToken() callback is invoked for each token in the +** FTS5 assumes the xToken() callback is invoked for each token in the ** order that they occur within the input text. ** ** If an xToken() callback returns any value other than SQLITE_OK, then @@ -10251,10 +13640,34 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
        +**
      • There is no "iVersion" field, and +**
      • The xTokenize() method does not take a locale argument. +**
      +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a -** user wishes to query for a phrase such as "first place". Using the +** user wishes to query for a phrase such as "first place". Using the ** built-in tokenizers, the FTS5 query 'first + place' will match instances ** of "first place" within the document set, but not alternative forms ** such as "1st place". In some applications, it would be better to match @@ -10263,8 +13676,8 @@ struct Fts5ExtensionApi { ** ** There are several ways to approach this in FTS5: ** -**
      1. By mapping all synonyms to a single token. In this case, the -** In the above example, this means that the tokenizer returns the +**
        1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the ** same token for inputs "first" and "1st". Say that token is in ** fact "first", so that when the user inserts the document "I won ** 1st place" entries are added to the index for tokens "i", "won", @@ -10272,37 +13685,37 @@ struct Fts5ExtensionApi { ** the tokenizer substitutes "first" for "1st" and the query works ** as expected. ** -**
        2. By adding multiple synonyms for a single term to the FTS index. -** In this case, when tokenizing query text, the tokenizer may -** provide multiple synonyms for a single term within the document. -** FTS5 then queries the index for each synonym individually. For -** example, faced with the query: +**
        3. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: ** ** ** ... MATCH 'first place' ** ** the tokenizer offers both "1st" and "first" as synonyms for the -** first token in the MATCH query and FTS5 effectively runs a query +** first token in the MATCH query and FTS5 effectively runs a query ** similar to: ** ** ** ... MATCH '(first OR 1st) place' ** ** except that, for the purposes of auxiliary functions, the query -** still appears to contain just two phrases - "(first OR 1st)" +** still appears to contain just two phrases - "(first OR 1st)" ** being treated as a single phrase. ** **
        4. By adding multiple synonyms for a single term to the FTS index. ** Using this method, when tokenizing document text, the tokenizer -** provides multiple synonyms for each token. So that when a +** provides multiple synonyms for each token. So that when a ** document such as "I won first place" is tokenized, entries are ** added to the FTS index for "i", "won", "first", "1st" and ** "place". ** ** This way, even if the tokenizer does not provide synonyms -** when tokenizing query text (it should not - to do would be -** inefficient), it doesn't matter if the user queries for -** 'first + place' or '1st + place', as there are entires in the +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the ** FTS index corresponding to both forms of the first token. **
        ** @@ -10322,15 +13735,15 @@ struct Fts5ExtensionApi { ** ** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time ** xToken() is called. Multiple synonyms may be specified for a single token -** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. ** There is no limit to the number of synonyms that may be provided for a ** single token. ** -** In many cases, method (1) above is the best approach. It does not add +** In many cases, method (1) above is the best approach. It does not add ** extra data to the FTS index or require FTS5 to query for multiple terms, ** so it is efficient in terms of disk space and query speed. However, it ** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: +** token "first" is substituted for "1st" by the tokenizer, then the query: ** ** ** ... MATCH '1s*' @@ -10338,35 +13751,38 @@ struct Fts5ExtensionApi { ** will not match documents that contain the token "1st" (as the tokenizer ** will probably not map "1s" to any prefix of "first"). ** -** For full prefix support, method (3) may be preferred. In this case, +** For full prefix support, method (3) may be preferred. In this case, ** because the index contains entries for both "first" and "1st", prefix ** queries such as 'fi*' or '1s*' will match correctly. However, because ** extra entries are added to the FTS index, this method uses more space ** within the database. ** ** Method (2) offers a midpoint between (1) and (3). Using this method, -** a query such as '1s*' will match documents that contain the literal +** a query such as '1s*' will match documents that contain the literal ** token "1st", but not "first" (assuming the tokenizer is not able to ** provide synonyms for prefixes). However, a non-prefix query like '1st' ** will match against "1st" and "first". This method does not require -** extra disk space, as no extra entries are added to the FTS index. +** extra disk space, as no extra entries are added to the FTS index. ** On the other hand, it may require more CPU cycles to run MATCH queries, ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; -typedef struct fts5_tokenizer fts5_tokenizer; -struct fts5_tokenizer { +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); void (*xDelete)(Fts5Tokenizer*); - int (*xTokenize)(Fts5Tokenizer*, + int (*xTokenize)(Fts5Tokenizer*, void *pCtx, int flags, /* Mask of FTS5_TOKENIZE_* flags */ - const char *pText, int nText, + const char *pText, int nText, + const char *pLocale, int nLocale, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ int tflags, /* Mask of FTS5_TOKEN_* flags */ @@ -10378,6 +13794,31 @@ struct fts5_tokenizer { ); }; +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -10397,13 +13838,13 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -10412,7 +13853,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -10420,10 +13861,29 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* @@ -10436,18 +13896,23 @@ struct fts5_api { #endif /* _FTS5_H */ - /******** End of fts5.h *********/ /************** End of sqlite3.h *********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/* +** Reuse the STATIC_LRU for mutex access to sqlite3_temp_directory. +*/ +#define SQLITE_MUTEX_STATIC_TEMPDIR SQLITE_MUTEX_STATIC_VFS1 + /* ** Include the configuration header output by 'configure' if we're using the ** autoconf-based build */ -#ifdef _HAVE_SQLITE_CONFIG_H -#include "config.h" +#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) +#include "sqlite_cfg.h" +#define SQLITECONFIG_H 1 #endif /************** Include sqliteLimit.h in the middle of sqliteInt.h ***********/ @@ -10463,7 +13928,7 @@ struct fts5_api { ** May you share freely, never taking more than you give. ** ************************************************************************* -** +** ** This file defines various limits of what SQLite can process. */ @@ -10511,14 +13976,10 @@ struct fts5_api { #endif /* -** The maximum depth of an expression tree. This is limited to -** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might -** want to place more severe limits on the complexity of an -** expression. -** -** A value of 0 used to mean that the limit was not enforced. -** But that is no longer true. The limit is now strictly enforced -** at all times. +** The maximum depth of an expression tree. This is limited to +** some extent by SQLITE_MAX_SQL_LENGTH. But sometime you might +** want to place more severe limits on the complexity of an +** expression. A value of 0 means that there is no limit. */ #ifndef SQLITE_MAX_EXPR_DEPTH # define SQLITE_MAX_EXPR_DEPTH 1000 @@ -10530,7 +13991,7 @@ struct fts5_api { ** level of recursion for each term. A stack overflow can result ** if the number of terms is too large. In practice, most SQL ** never has more than 3 or 4 terms. Use a value of 0 to disable -** any limit on the number of terms in a compount SELECT. +** any limit on the number of terms in a compound SELECT. */ #ifndef SQLITE_MAX_COMPOUND_SELECT # define SQLITE_MAX_COMPOUND_SELECT 500 @@ -10541,7 +14002,7 @@ struct fts5_api { ** Not currently enforced. */ #ifndef SQLITE_MAX_VDBE_OP -# define SQLITE_MAX_VDBE_OP 25000 +# define SQLITE_MAX_VDBE_OP 250000000 #endif /* @@ -10585,9 +14046,12 @@ struct fts5_api { /* ** The maximum value of a ?nnn wildcard that the parser will accept. +** If the value exceeds 32767 then extra space is required for the Expr +** structure. But otherwise, we believe that the number can be as large +** as a signed 32-bit integer can hold. */ #ifndef SQLITE_MAX_VARIABLE_NUMBER -# define SQLITE_MAX_VARIABLE_NUMBER 999 +# define SQLITE_MAX_VARIABLE_NUMBER 32766 #endif /* Maximum page size. The upper bound on this value is 65536. This a limit @@ -10595,10 +14059,10 @@ struct fts5_api { ** ** Earlier versions of SQLite allowed the user to change this value at ** compile time. This is no longer permitted, on the grounds that it creates -** a library that is technically incompatible with an SQLite library -** compiled with a different limit. If a process operating on a database -** with a page-size of 65536 bytes crashes, then an instance of SQLite -** compiled with the default page-size limit will not be able to rollback +** a library that is technically incompatible with an SQLite library +** compiled with a different limit. If a process operating on a database +** with a page-size of 65536 bytes crashes, then an instance of SQLite +** compiled with the default page-size limit will not be able to rollback ** the aborted transaction. This could lead to database corruption. */ #ifdef SQLITE_MAX_PAGE_SIZE @@ -10642,7 +14106,7 @@ struct fts5_api { ** max_page_count macro. */ #ifndef SQLITE_MAX_PAGE_COUNT -# define SQLITE_MAX_PAGE_COUNT 1073741823 +# define SQLITE_MAX_PAGE_COUNT 0xfffffffe /* 4294967294 */ #endif /* @@ -10657,7 +14121,7 @@ struct fts5_api { ** Maximum depth of recursion for triggers. ** ** A value of 1 means that a trigger program will not be able to itself -** fire any triggers. A value of 0 means that no trigger programs at all +** fire any triggers. A value of 0 means that no trigger programs at all ** may be executed. */ #ifndef SQLITE_MAX_TRIGGER_DEPTH @@ -10676,6 +14140,23 @@ struct fts5_api { #pragma warn -spa /* Suspicious pointer arithmetic */ #endif +/* +** A few places in the code require atomic load/store of aligned +** integer values. +*/ +#ifndef __has_extension +# define __has_extension(x) 0 /* compatibility with non-clang compilers */ +#endif +#if GCC_VERSION>=4007000 || __has_extension(c_atomic) +# define SQLITE_ATOMIC_INTRINSICS 1 +# define AtomicLoad(PTR) __atomic_load_n((PTR),__ATOMIC_RELAXED) +# define AtomicStore(PTR,VAL) __atomic_store_n((PTR),(VAL),__ATOMIC_RELAXED) +#else +# define SQLITE_ATOMIC_INTRINSICS 0 +# define AtomicLoad(PTR) (*(PTR)) +# define AtomicStore(PTR,VAL) (*(PTR) = (VAL)) +#endif + /* ** Include standard header files as necessary */ @@ -10702,30 +14183,37 @@ struct fts5_api { ** So we have to define the macros in different ways depending on the ** compiler. */ -#if defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ +#if defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ +# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) +# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) +#elif defined(__PTRDIFF_TYPE__) /* This case should work for GCC */ # define SQLITE_INT_TO_PTR(X) ((void*)(__PTRDIFF_TYPE__)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(__PTRDIFF_TYPE__)(X)) #elif !defined(__GNUC__) /* Works for compilers other than LLVM */ # define SQLITE_INT_TO_PTR(X) ((void*)&((char*)0)[X]) # define SQLITE_PTR_TO_INT(X) ((int)(((char*)X)-(char*)0)) -#elif defined(HAVE_STDINT_H) /* Use this case if we have ANSI headers */ -# define SQLITE_INT_TO_PTR(X) ((void*)(intptr_t)(X)) -# define SQLITE_PTR_TO_INT(X) ((int)(intptr_t)(X)) #else /* Generates a warning - but it always works */ # define SQLITE_INT_TO_PTR(X) ((void*)(X)) # define SQLITE_PTR_TO_INT(X) ((int)(X)) #endif /* -** A macro to hint to the compiler that a function should not be +** Macros to hint to the compiler that a function should or should not be ** inlined. */ #if defined(__GNUC__) # define SQLITE_NOINLINE __attribute__((noinline)) +# define SQLITE_INLINE __attribute__((always_inline)) inline #elif defined(_MSC_VER) && _MSC_VER>=1310 # define SQLITE_NOINLINE __declspec(noinline) +# define SQLITE_INLINE __forceinline #else # define SQLITE_NOINLINE +# define SQLITE_INLINE +#endif +#if defined(SQLITE_COVERAGE_TEST) || defined(__STRICT_ANSI__) +# undef SQLITE_INLINE +# define SQLITE_INLINE #endif /* @@ -10734,11 +14222,12 @@ struct fts5_api { ** the SQLITE_DISABLE_INTRINSIC define. */ #if !defined(SQLITE_DISABLE_INTRINSIC) -# if defined(_MSC_VER) && _MSC_VER>=1300 +# if defined(_MSC_VER) && _MSC_VER>=1400 # if !defined(_WIN32_WCE) # include # pragma intrinsic(_byteswap_ushort) # pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) # pragma intrinsic(_ReadWriteBarrier) # else # include @@ -10746,6 +14235,29 @@ struct fts5_api { # endif #endif +/* +** Enable SQLITE_USE_SEH by default on MSVC builds. Only omit +** SEH support if the -DSQLITE_OMIT_SEH option is given. +*/ +#if defined(_MSC_VER) && !defined(SQLITE_OMIT_SEH) +# define SQLITE_USE_SEH 1 +#else +# undef SQLITE_USE_SEH +#endif + +/* +** Enable SQLITE_DIRECT_OVERFLOW_READ, unless the build explicitly +** disables it using -DSQLITE_DIRECT_OVERFLOW_READ=0 +*/ +#if defined(SQLITE_DIRECT_OVERFLOW_READ) && SQLITE_DIRECT_OVERFLOW_READ+1==1 + /* Disable if -DSQLITE_DIRECT_OVERFLOW_READ=0 */ +# undef SQLITE_DIRECT_OVERFLOW_READ +#else + /* In all other cases, enable */ +# define SQLITE_DIRECT_OVERFLOW_READ 1 +#endif + + /* ** The SQLITE_THREADSAFE macro must be defined as 0, 1, or 2. ** 0 means mutexes are permanently disable and the library is never @@ -10756,6 +14268,11 @@ struct fts5_api { ** ** Older versions of SQLite used an optional THREADSAFE macro. ** We support that for legacy. +** +** To ensure that the correct value of "THREADSAFE" is reported when querying +** for compile-time options at runtime (e.g. "PRAGMA compile_options"), this +** logic is partially replicated in ctime.c. If it is updated here, it should +** also be updated there. */ #if !defined(SQLITE_THREADSAFE) # if defined(THREADSAFE) @@ -10870,11 +14387,12 @@ struct fts5_api { ** is significant and used at least once. On switch statements ** where multiple cases go to the same block of code, testcase() ** can insure that all cases are evaluated. -** */ -#ifdef SQLITE_COVERAGE_TEST -SQLITE_PRIVATE void sqlite3Coverage(int); -# define testcase(X) if( X ){ sqlite3Coverage(__LINE__); } +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) +# ifndef SQLITE_AMALGAMATION + extern unsigned int sqlite3CoverageCounter; +# endif +# define testcase(X) if( X ){ sqlite3CoverageCounter += (unsigned)__LINE__; } #else # define testcase(X) #endif @@ -10904,6 +14422,14 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # define VVA_ONLY(X) #endif +/* +** Disable ALWAYS() and NEVER() (make them pass-throughs) for coverage +** and mutation testing +*/ +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif + /* ** The ALWAYS and NEVER macros surround boolean expressions which ** are intended to always be true or false, respectively. Such @@ -10919,7 +14445,7 @@ SQLITE_PRIVATE void sqlite3Coverage(int); ** be true and false so that the unreachable code they specify will ** not be counted as untested code. */ -#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) # define ALWAYS(X) (1) # define NEVER(X) (0) #elif !defined(NDEBUG) @@ -10930,6 +14456,21 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # define NEVER(X) (X) #endif +/* +** Some conditionals are optimizations only. In other words, if the +** conditionals are replaced with a constant 1 (true) or 0 (false) then +** the correct answer is still obtained, though perhaps not as quickly. +** +** The following macros mark these optimizations conditionals. +*/ +#if defined(SQLITE_MUTATION_TEST) +# define OK_IF_ALWAYS_TRUE(X) (1) +# define OK_IF_ALWAYS_FALSE(X) (0) +#else +# define OK_IF_ALWAYS_TRUE(X) (X) +# define OK_IF_ALWAYS_FALSE(X) (X) +#endif + /* ** Some malloc failures are only possible if SQLITE_TEST_REALLOC_STRESS is ** defined. We need to defend against those failures when testing with @@ -10978,6 +14519,15 @@ SQLITE_PRIVATE void sqlite3Coverage(int); # undef SQLITE_ENABLE_EXPLAIN_COMMENTS #endif +/* +** SQLITE_OMIT_VIRTUALTABLE implies SQLITE_OMIT_ALTERTABLE +*/ +#if defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_ALTERTABLE) +# define SQLITE_OMIT_ALTERTABLE +#endif + +#define SQLITE_DIGIT_SEPARATOR '_' + /* ** Return true (non-zero) if the input is an integer that is too large ** to fit in 32-bits. This macro is used inside of various testcase() @@ -11011,8 +14561,8 @@ SQLITE_PRIVATE void sqlite3Coverage(int); ** This is the header file for the generic hash-table implementation ** used in SQLite. */ -#ifndef _SQLITE_HASH_H_ -#define _SQLITE_HASH_H_ +#ifndef SQLITE_HASH_H +#define SQLITE_HASH_H /* Forward declarations of structures. */ typedef struct Hash Hash; @@ -11034,7 +14584,7 @@ typedef struct HashElem HashElem; ** element pointed to plus the next _ht.count-1 elements in the list. ** ** Hash.htsize and Hash.ht may be zero. In that case lookup is done -** by a linear search of the global list. For small tables, the +** by a linear search of the global list. For small tables, the ** Hash.ht table is never allocated because if there are few elements ** in the table, it is faster to do a linear search than to manage ** the hash table. @@ -11044,12 +14594,12 @@ struct Hash { unsigned int count; /* Number of entries in this table */ HashElem *first; /* The first element of the array */ struct _ht { /* the hash table */ - int count; /* Number of entries with this hash */ + unsigned int count; /* Number of entries with this hash */ HashElem *chain; /* Pointer to first entry with this hash */ } *ht; }; -/* Each element in the hash table is an instance of the following +/* Each element in the hash table is an instance of the following ** structure. All elements are stored on a single doubly-linked list. ** ** Again, this structure is intended to be opaque, but it can't really @@ -11090,9 +14640,9 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); /* ** Number of entries in a hash table */ -/* #define sqliteHashCount(H) ((H)->count) // NOT USED */ +#define sqliteHashCount(H) ((H)->count) -#endif /* _SQLITE_HASH_H_ */ +#endif /* SQLITE_HASH_H */ /************** End of hash.h ************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -11122,150 +14672,167 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #define TK_LP 22 #define TK_RP 23 #define TK_AS 24 -#define TK_WITHOUT 25 -#define TK_COMMA 26 -#define TK_OR 27 -#define TK_AND 28 -#define TK_IS 29 -#define TK_MATCH 30 -#define TK_LIKE_KW 31 -#define TK_BETWEEN 32 -#define TK_IN 33 -#define TK_ISNULL 34 -#define TK_NOTNULL 35 -#define TK_NE 36 -#define TK_EQ 37 -#define TK_GT 38 -#define TK_LE 39 -#define TK_LT 40 -#define TK_GE 41 -#define TK_ESCAPE 42 -#define TK_BITAND 43 -#define TK_BITOR 44 -#define TK_LSHIFT 45 -#define TK_RSHIFT 46 -#define TK_PLUS 47 -#define TK_MINUS 48 -#define TK_STAR 49 -#define TK_SLASH 50 -#define TK_REM 51 -#define TK_CONCAT 52 -#define TK_COLLATE 53 -#define TK_BITNOT 54 -#define TK_ID 55 -#define TK_INDEXED 56 -#define TK_ABORT 57 -#define TK_ACTION 58 -#define TK_AFTER 59 -#define TK_ANALYZE 60 -#define TK_ASC 61 -#define TK_ATTACH 62 -#define TK_BEFORE 63 -#define TK_BY 64 -#define TK_CASCADE 65 -#define TK_CAST 66 -#define TK_COLUMNKW 67 -#define TK_CONFLICT 68 -#define TK_DATABASE 69 -#define TK_DESC 70 -#define TK_DETACH 71 -#define TK_EACH 72 -#define TK_FAIL 73 -#define TK_FOR 74 -#define TK_IGNORE 75 -#define TK_INITIALLY 76 -#define TK_INSTEAD 77 -#define TK_NO 78 -#define TK_KEY 79 -#define TK_OF 80 -#define TK_OFFSET 81 -#define TK_PRAGMA 82 -#define TK_RAISE 83 -#define TK_RECURSIVE 84 -#define TK_REPLACE 85 -#define TK_RESTRICT 86 -#define TK_ROW 87 -#define TK_TRIGGER 88 -#define TK_VACUUM 89 -#define TK_VIEW 90 -#define TK_VIRTUAL 91 -#define TK_WITH 92 -#define TK_REINDEX 93 -#define TK_RENAME 94 -#define TK_CTIME_KW 95 -#define TK_ANY 96 -#define TK_STRING 97 -#define TK_JOIN_KW 98 -#define TK_CONSTRAINT 99 -#define TK_DEFAULT 100 -#define TK_NULL 101 -#define TK_PRIMARY 102 -#define TK_UNIQUE 103 -#define TK_CHECK 104 -#define TK_REFERENCES 105 -#define TK_AUTOINCR 106 -#define TK_ON 107 -#define TK_INSERT 108 -#define TK_DELETE 109 -#define TK_UPDATE 110 -#define TK_SET 111 -#define TK_DEFERRABLE 112 -#define TK_FOREIGN 113 -#define TK_DROP 114 -#define TK_UNION 115 -#define TK_ALL 116 -#define TK_EXCEPT 117 -#define TK_INTERSECT 118 -#define TK_SELECT 119 -#define TK_VALUES 120 -#define TK_DISTINCT 121 -#define TK_DOT 122 -#define TK_FROM 123 -#define TK_JOIN 124 -#define TK_USING 125 -#define TK_ORDER 126 -#define TK_GROUP 127 -#define TK_HAVING 128 -#define TK_LIMIT 129 -#define TK_WHERE 130 -#define TK_INTO 131 -#define TK_INTEGER 132 -#define TK_FLOAT 133 -#define TK_BLOB 134 -#define TK_VARIABLE 135 -#define TK_CASE 136 -#define TK_WHEN 137 -#define TK_THEN 138 -#define TK_ELSE 139 -#define TK_INDEX 140 -#define TK_ALTER 141 -#define TK_ADD 142 -#define TK_TO_TEXT 143 -#define TK_TO_BLOB 144 -#define TK_TO_NUMERIC 145 -#define TK_TO_INT 146 -#define TK_TO_REAL 147 -#define TK_ISNOT 148 -#define TK_END_OF_FILE 149 -#define TK_UNCLOSED_STRING 150 -#define TK_FUNCTION 151 -#define TK_COLUMN 152 -#define TK_AGG_FUNCTION 153 -#define TK_AGG_COLUMN 154 -#define TK_UMINUS 155 -#define TK_UPLUS 156 -#define TK_REGISTER 157 -#define TK_ASTERISK 158 -#define TK_SPAN 159 -#define TK_SPACE 160 -#define TK_ILLEGAL 161 - -/* The token codes above must all fit in 8 bits */ -#define TKFLG_MASK 0xff - -/* Flags that can be added to a token code when it is not -** being stored in a u8: */ -#define TKFLG_DONTFOLD 0x100 /* Omit constant folding optimizations */ +#define TK_COMMA 25 +#define TK_WITHOUT 26 +#define TK_ABORT 27 +#define TK_ACTION 28 +#define TK_AFTER 29 +#define TK_ANALYZE 30 +#define TK_ASC 31 +#define TK_ATTACH 32 +#define TK_BEFORE 33 +#define TK_BY 34 +#define TK_CASCADE 35 +#define TK_CAST 36 +#define TK_CONFLICT 37 +#define TK_DATABASE 38 +#define TK_DESC 39 +#define TK_DETACH 40 +#define TK_EACH 41 +#define TK_FAIL 42 +#define TK_OR 43 +#define TK_AND 44 +#define TK_IS 45 +#define TK_ISNOT 46 +#define TK_MATCH 47 +#define TK_LIKE_KW 48 +#define TK_BETWEEN 49 +#define TK_IN 50 +#define TK_ISNULL 51 +#define TK_NOTNULL 52 +#define TK_NE 53 +#define TK_EQ 54 +#define TK_GT 55 +#define TK_LE 56 +#define TK_LT 57 +#define TK_GE 58 +#define TK_ESCAPE 59 +#define TK_ID 60 +#define TK_COLUMNKW 61 +#define TK_DO 62 +#define TK_FOR 63 +#define TK_IGNORE 64 +#define TK_INITIALLY 65 +#define TK_INSTEAD 66 +#define TK_NO 67 +#define TK_KEY 68 +#define TK_OF 69 +#define TK_OFFSET 70 +#define TK_PRAGMA 71 +#define TK_RAISE 72 +#define TK_RECURSIVE 73 +#define TK_REPLACE 74 +#define TK_RESTRICT 75 +#define TK_ROW 76 +#define TK_ROWS 77 +#define TK_TRIGGER 78 +#define TK_VACUUM 79 +#define TK_VIEW 80 +#define TK_VIRTUAL 81 +#define TK_WITH 82 +#define TK_NULLS 83 +#define TK_FIRST 84 +#define TK_LAST 85 +#define TK_CURRENT 86 +#define TK_FOLLOWING 87 +#define TK_PARTITION 88 +#define TK_PRECEDING 89 +#define TK_RANGE 90 +#define TK_UNBOUNDED 91 +#define TK_EXCLUDE 92 +#define TK_GROUPS 93 +#define TK_OTHERS 94 +#define TK_TIES 95 +#define TK_GENERATED 96 +#define TK_ALWAYS 97 +#define TK_MATERIALIZED 98 +#define TK_REINDEX 99 +#define TK_RENAME 100 +#define TK_CTIME_KW 101 +#define TK_ANY 102 +#define TK_BITAND 103 +#define TK_BITOR 104 +#define TK_LSHIFT 105 +#define TK_RSHIFT 106 +#define TK_PLUS 107 +#define TK_MINUS 108 +#define TK_STAR 109 +#define TK_SLASH 110 +#define TK_REM 111 +#define TK_CONCAT 112 +#define TK_PTR 113 +#define TK_COLLATE 114 +#define TK_BITNOT 115 +#define TK_ON 116 +#define TK_INDEXED 117 +#define TK_STRING 118 +#define TK_JOIN_KW 119 +#define TK_CONSTRAINT 120 +#define TK_DEFAULT 121 +#define TK_NULL 122 +#define TK_PRIMARY 123 +#define TK_UNIQUE 124 +#define TK_CHECK 125 +#define TK_REFERENCES 126 +#define TK_AUTOINCR 127 +#define TK_INSERT 128 +#define TK_DELETE 129 +#define TK_UPDATE 130 +#define TK_SET 131 +#define TK_DEFERRABLE 132 +#define TK_FOREIGN 133 +#define TK_DROP 134 +#define TK_UNION 135 +#define TK_ALL 136 +#define TK_EXCEPT 137 +#define TK_INTERSECT 138 +#define TK_SELECT 139 +#define TK_VALUES 140 +#define TK_DISTINCT 141 +#define TK_DOT 142 +#define TK_FROM 143 +#define TK_JOIN 144 +#define TK_USING 145 +#define TK_ORDER 146 +#define TK_GROUP 147 +#define TK_HAVING 148 +#define TK_LIMIT 149 +#define TK_WHERE 150 +#define TK_RETURNING 151 +#define TK_INTO 152 +#define TK_NOTHING 153 +#define TK_FLOAT 154 +#define TK_BLOB 155 +#define TK_INTEGER 156 +#define TK_VARIABLE 157 +#define TK_CASE 158 +#define TK_WHEN 159 +#define TK_THEN 160 +#define TK_ELSE 161 +#define TK_INDEX 162 +#define TK_ALTER 163 +#define TK_ADD 164 +#define TK_WINDOW 165 +#define TK_OVER 166 +#define TK_FILTER 167 +#define TK_COLUMN 168 +#define TK_AGG_FUNCTION 169 +#define TK_AGG_COLUMN 170 +#define TK_TRUEFALSE 171 +#define TK_FUNCTION 172 +#define TK_UPLUS 173 +#define TK_UMINUS 174 +#define TK_TRUTH 175 +#define TK_REGISTER 176 +#define TK_VECTOR 177 +#define TK_SELECT_COLUMN 178 +#define TK_IF_NULL_ROW 179 +#define TK_ASTERISK 180 +#define TK_SPAN 181 +#define TK_ERROR 182 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 /************** End of parse.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ @@ -11274,6 +14841,19 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #include #include #include +#include + +/* +** Use a macro to replace memcpy() if compiled with SQLITE_INLINE_MEMCPY. +** This allows better measurements of where memcpy() is used when running +** cachegrind. But this macro version of memcpy() is very slow so it +** should not be used in production. This is a performance measurement +** hack only. +*/ +#ifdef SQLITE_INLINE_MEMCPY +# define memcpy(D,S,N) {char*xxd=(char*)(D);const char*xxs=(const char*)(S);\ + int xxn=(N);while(xxn-->0)*(xxd++)=*(xxs++);} +#endif /* ** If compiling for a processor that lacks floating point support, @@ -11282,7 +14862,8 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); #ifdef SQLITE_OMIT_FLOATING_POINT # define double sqlite_int64 # define float sqlite_int64 -# define LONGDOUBLE_TYPE sqlite_int64 +# define fabs(X) ((X)<0?-(X):(X)) +# define sqlite3IsOverflow(X) 0 # ifndef SQLITE_BIG_DBL # define SQLITE_BIG_DBL (((sqlite3_int64)1)<<50) # endif @@ -11331,7 +14912,6 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); */ #ifndef SQLITE_TEMP_STORE # define SQLITE_TEMP_STORE 1 -# define SQLITE_TEMP_STORE_xc 1 /* Exclude from ctime.c */ #endif /* @@ -11359,9 +14939,28 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); ** pagecaches for each database connection. A positive number is the ** number of pages. A negative number N translations means that a buffer ** of -1024*N bytes is allocated and used for as many pages as it will hold. +** +** The default value of "20" was chosen to minimize the run-time of the +** speedtest1 test program with options: --shrink-memory --reprepare */ #ifndef SQLITE_DEFAULT_PCACHE_INITSZ -# define SQLITE_DEFAULT_PCACHE_INITSZ 100 +# define SQLITE_DEFAULT_PCACHE_INITSZ 20 +#endif + +/* +** Default value for the SQLITE_CONFIG_SORTERREF_SIZE option. +*/ +#ifndef SQLITE_DEFAULT_SORTERREF_SIZE +# define SQLITE_DEFAULT_SORTERREF_SIZE 0x7fffffff +#endif + +/* +** The compile-time options SQLITE_MMAP_READWRITE and +** SQLITE_ENABLE_BATCH_ATOMIC_WRITE are not compatible with one another. +** You must choose one or the other (or neither) but not both. +*/ +#if defined(SQLITE_MMAP_READWRITE) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +#error Cannot use both SQLITE_MMAP_READWRITE and SQLITE_ENABLE_BATCH_ATOMIC_WRITE #endif /* @@ -11439,9 +15038,6 @@ SQLITE_PRIVATE void sqlite3HashClear(Hash*); # define INT8_TYPE signed char # endif #endif -#ifndef LONGDOUBLE_TYPE -# define LONGDOUBLE_TYPE long double -#endif typedef sqlite_int64 i64; /* 8-byte signed integer */ typedef sqlite_uint64 u64; /* 8-byte unsigned integer */ typedef UINT32_TYPE u32; /* 4-byte unsigned integer */ @@ -11460,15 +15056,9 @@ typedef INT8_TYPE i8; /* 1-byte signed integer */ /* ** The datatype used to store estimates of the number of rows in a -** table or index. This is an unsigned integer type. For 99.9% of -** the world, a 32-bit integer is sufficient. But a 64-bit integer -** can be used at compile-time if desired. +** table or index. */ -#ifdef SQLITE_64BIT_STATS - typedef u64 tRowcnt; /* 64-bit only if requested at compile-time */ -#else - typedef u32 tRowcnt; /* 32-bit is the default */ -#endif +typedef u64 tRowcnt; /* ** Estimated quantities used for query planning are stored as 16-bit @@ -11502,7 +15092,9 @@ typedef INT16_TYPE LogEst; # if defined(__SIZEOF_POINTER__) # define SQLITE_PTRSIZE __SIZEOF_POINTER__ # elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(_M_ARM) || defined(__arm__) || defined(__x86) + defined(_M_ARM) || defined(__arm__) || defined(__x86) || \ + (defined(__APPLE__) && defined(__ppc__)) || \ + (defined(__TOS_AIX__) && !defined(__64BIT__)) # define SQLITE_PTRSIZE 4 # else # define SQLITE_PTRSIZE 8 @@ -11527,8 +15119,31 @@ typedef INT16_TYPE LogEst; ** the end of buffer S. This macro returns true if P points to something ** contained within the buffer S. */ -#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) +#define SQLITE_WITHIN(P,S,E) (((uptr)(P)>=(uptr)(S))&&((uptr)(P)<(uptr)(E))) +/* +** P is one byte past the end of a large buffer. Return true if a span of bytes +** between S..E crosses the end of that buffer. In other words, return true +** if the sub-buffer S..E-1 overflows the buffer whose last byte is P-1. +** +** S is the start of the span. E is one byte past the end of end of span. +** +** P +** |-----------------| FALSE +** |-------| +** S E +** +** P +** |-----------------| +** |-------| TRUE +** S E +** +** P +** |-----------------| +** |-------| FALSE +** S E +*/ +#define SQLITE_OVERFLOW(P,S,E) (((uptr)(S)<(uptr)(P))&&((uptr)(E)>(uptr)(P))) /* ** Macros to determine whether the machine is big or little endian, @@ -11536,32 +15151,53 @@ typedef INT16_TYPE LogEst; ** ** For best performance, an attempt is made to guess at the byte-order ** using C-preprocessor macros. If that is unsuccessful, or if -** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** -DSQLITE_BYTEORDER=0 is set, then byte-order is determined ** at run-time. +** +** If you are building SQLite on some obscure platform for which the +** following ifdef magic does not work, you can always include either: +** +** -DSQLITE_BYTEORDER=1234 +** +** or +** +** -DSQLITE_BYTEORDER=4321 +** +** to cause the build to work for little-endian or big-endian processors, +** respectively. */ -#if (defined(i386) || defined(__i386__) || defined(_M_IX86) || \ - defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ - defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ - defined(__arm__)) && !defined(SQLITE_RUNTIME_BYTEORDER) -# define SQLITE_BYTEORDER 1234 -# define SQLITE_BIGENDIAN 0 -# define SQLITE_LITTLEENDIAN 1 -# define SQLITE_UTF16NATIVE SQLITE_UTF16LE +#ifndef SQLITE_BYTEORDER /* Replicate changes at tag-20230904a */ +# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define SQLITE_BYTEORDER 4321 +# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define SQLITE_BYTEORDER 1234 +# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 +# define SQLITE_BYTEORDER 4321 +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif #endif -#if (defined(sparc) || defined(__ppc__)) \ - && !defined(SQLITE_RUNTIME_BYTEORDER) -# define SQLITE_BYTEORDER 4321 +#if SQLITE_BYTEORDER==4321 # define SQLITE_BIGENDIAN 1 # define SQLITE_LITTLEENDIAN 0 # define SQLITE_UTF16NATIVE SQLITE_UTF16BE -#endif -#if !defined(SQLITE_BYTEORDER) +#elif SQLITE_BYTEORDER==1234 +# define SQLITE_BIGENDIAN 0 +# define SQLITE_LITTLEENDIAN 1 +# define SQLITE_UTF16NATIVE SQLITE_UTF16LE +#else # ifdef SQLITE_AMALGAMATION const int sqlite3one = 1; # else extern const int sqlite3one; # endif -# define SQLITE_BYTEORDER 0 /* 0 means "unknown at compile-time" */ # define SQLITE_BIGENDIAN (*(char *)(&sqlite3one)==0) # define SQLITE_LITTLEENDIAN (*(char *)(&sqlite3one)==1) # define SQLITE_UTF16NATIVE (SQLITE_BIGENDIAN?SQLITE_UTF16BE:SQLITE_UTF16LE) @@ -11573,13 +15209,25 @@ typedef INT16_TYPE LogEst; ** compilers. */ #define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define LARGEST_UINT64 (0xffffffff|(((u64)0xffffffff)<<32)) #define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) /* ** Round up a number to the next larger multiple of 8. This is used ** to force 8-byte alignment on 64-bit architectures. +** +** ROUND8() always does the rounding, for any argument. +** +** ROUND8P() assumes that the argument is already an integer number of +** pointers in size, and so it is a no-op on systems where the pointer +** size is 8. */ #define ROUND8(x) (((x)+7)&~7) +#if SQLITE_PTRSIZE==8 +# define ROUND8P(x) (x) +#else +# define ROUND8P(x) (((x)+7)&~7) +#endif /* ** Round down to the nearest multiple of 8 @@ -11596,9 +15244,9 @@ typedef INT16_TYPE LogEst; ** pointers. In that case, only verify 4-byte alignment. */ #ifdef SQLITE_4_BYTE_ALIGNED_MALLOC -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&3)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) #else -# define EIGHT_BYTE_ALIGNMENT(X) ((((char*)(X) - (char*)0)&7)==0) +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) #endif /* @@ -11626,7 +15274,6 @@ typedef INT16_TYPE LogEst; # else # define SQLITE_MAX_MMAP_SIZE 0 # endif -# define SQLITE_MAX_MMAP_SIZE_xc 1 /* exclude from ctime.c */ #endif /* @@ -11636,7 +15283,6 @@ typedef INT16_TYPE LogEst; */ #ifndef SQLITE_DEFAULT_MMAP_SIZE # define SQLITE_DEFAULT_MMAP_SIZE 0 -# define SQLITE_DEFAULT_MMAP_SIZE_xc 1 /* Exclude from ctime.c */ #endif #if SQLITE_DEFAULT_MMAP_SIZE>SQLITE_MAX_MMAP_SIZE # undef SQLITE_DEFAULT_MMAP_SIZE @@ -11644,28 +15290,91 @@ typedef INT16_TYPE LogEst; #endif /* -** Only one of SQLITE_ENABLE_STAT3 or SQLITE_ENABLE_STAT4 can be defined. -** Priority is given to SQLITE_ENABLE_STAT4. If either are defined, also -** define SQLITE_ENABLE_STAT3_OR_STAT4 +** TREETRACE_ENABLED will be either 1 or 0 depending on whether or not +** the Abstract Syntax Tree tracing logic is turned on. */ -#ifdef SQLITE_ENABLE_STAT4 -# undef SQLITE_ENABLE_STAT3 -# define SQLITE_ENABLE_STAT3_OR_STAT4 1 -#elif SQLITE_ENABLE_STAT3 -# define SQLITE_ENABLE_STAT3_OR_STAT4 1 -#elif SQLITE_ENABLE_STAT3_OR_STAT4 -# undef SQLITE_ENABLE_STAT3_OR_STAT4 +#if !defined(SQLITE_AMALGAMATION) +SQLITE_PRIVATE u32 sqlite3TreeTrace; +#endif +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_SELECTTRACE) \ + || defined(SQLITE_ENABLE_TREETRACE)) +# define TREETRACE_ENABLED 1 +# define TREETRACE(K,P,S,X) \ + if(sqlite3TreeTrace&(K)) \ + sqlite3DebugPrintf("%u/%d/%p: ",(S)->selId,(P)->addrExplain,(S)),\ + sqlite3DebugPrintf X +#else +# define TREETRACE(K,P,S,X) +# define TREETRACE_ENABLED 0 +#endif + +/* TREETRACE flag meanings: +** +** 0x00000001 Beginning and end of SELECT processing +** 0x00000002 WHERE clause processing +** 0x00000004 Query flattener +** 0x00000008 Result-set wildcard expansion +** 0x00000010 Query name resolution +** 0x00000020 Aggregate analysis +** 0x00000040 Window functions +** 0x00000080 Generated column names +** 0x00000100 Move HAVING terms into WHERE +** 0x00000200 Count-of-view optimization +** 0x00000400 Compound SELECT processing +** 0x00000800 Drop superfluous ORDER BY +** 0x00001000 LEFT JOIN simplifies to JOIN +** 0x00002000 Constant propagation +** 0x00004000 Push-down optimization +** 0x00008000 After all FROM-clause analysis +** 0x00010000 Beginning of DELETE/INSERT/UPDATE processing +** 0x00020000 Transform DISTINCT into GROUP BY +** 0x00040000 SELECT tree dump after all code has been generated +** 0x00080000 NOT NULL strength reduction +*/ + +/* +** Macros for "wheretrace" +*/ +SQLITE_PRIVATE u32 sqlite3WhereTrace; +#if defined(SQLITE_DEBUG) \ + && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) +# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X +# define WHERETRACE_ENABLED 1 +#else +# define WHERETRACE(K,X) #endif /* -** SELECTTRACE_ENABLED will be either 1 or 0 depending on whether or not -** the Select query generator tracing logic is turned on. +** Bits for the sqlite3WhereTrace mask: +** +** (---any--) Top-level block structure +** 0x-------F High-level debug messages +** 0x----FFF- More detail +** 0xFFFF---- Low-level debug messages +** +** 0x00000001 Code generation +** 0x00000002 Solver +** 0x00000004 Solver costs +** 0x00000008 WhereLoop inserts +** +** 0x00000010 Display sqlite3_index_info xBestIndex calls +** 0x00000020 Range an equality scan metrics +** 0x00000040 IN operator decisions +** 0x00000080 WhereLoop cost adjustments +** 0x00000100 +** 0x00000200 Covering index decisions +** 0x00000400 OR optimization +** 0x00000800 Index scanner +** 0x00001000 More details associated with code generation +** 0x00002000 +** 0x00004000 Show all WHERE terms at key points +** 0x00008000 Show the full SELECT statement at key places +** +** 0x00010000 Show more detail when printing WHERE terms +** 0x00020000 Show WHERE terms returned from whereScanNext() */ -#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_SELECTTRACE) -# define SELECTTRACE_ENABLED 1 -#else -# define SELECTTRACE_ENABLED 0 -#endif + /* ** An instance of the following structure is used to store the busy-handler @@ -11678,28 +15387,44 @@ typedef INT16_TYPE LogEst; */ typedef struct BusyHandler BusyHandler; struct BusyHandler { - int (*xFunc)(void *,int); /* The busy callback */ - void *pArg; /* First arg to busy callback */ - int nBusy; /* Incremented with each busy call */ + int (*xBusyHandler)(void *,int); /* The busy callback */ + void *pBusyArg; /* First arg to busy callback */ + int nBusy; /* Incremented with each busy call */ }; /* -** Name of the master database table. The master database table -** is a special table that holds the names and attributes of all -** user tables and indices. +** Name of table that holds the database schema. +** +** The PREFERRED names are used wherever possible. But LEGACY is also +** used for backwards compatibility. +** +** 1. Queries can use either the PREFERRED or the LEGACY names +** 2. The sqlite3_set_authorizer() callback uses the LEGACY name +** 3. The PRAGMA table_list statement uses the PREFERRED name +** +** The LEGACY names are stored in the internal symbol hash table +** in support of (2). Names are translated using sqlite3PreferredTableName() +** for (3). The sqlite3FindTable() function takes care of translating +** names for (1). +** +** Note that "sqlite_temp_schema" can also be called "temp.sqlite_schema". */ -#define MASTER_NAME "sqlite_master" -#define TEMP_MASTER_NAME "sqlite_temp_master" +#define LEGACY_SCHEMA_TABLE "sqlite_master" +#define LEGACY_TEMP_SCHEMA_TABLE "sqlite_temp_master" +#define PREFERRED_SCHEMA_TABLE "sqlite_schema" +#define PREFERRED_TEMP_SCHEMA_TABLE "sqlite_temp_schema" + /* -** The root-page of the master database table. +** The root-page of the schema table. */ -#define MASTER_ROOT 1 +#define SCHEMA_ROOT 1 /* -** The name of the schema table. +** The name of the schema table. The name is different for TEMP. */ -#define SCHEMA_TABLE(x) ((!OMIT_TEMPDB)&&(x==1)?TEMP_MASTER_NAME:MASTER_NAME) +#define SCHEMA_TABLE(x) \ + ((!OMIT_TEMPDB)&&(x==1)?LEGACY_TEMP_SCHEMA_TABLE:LEGACY_SCHEMA_TABLE) /* ** A convenience macro that returns the number of elements in @@ -11720,7 +15445,7 @@ struct BusyHandler { ** pointer will work here as long as it is distinct from SQLITE_STATIC ** and SQLITE_TRANSIENT. */ -#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3MallocSize) +#define SQLITE_DYNAMIC ((sqlite3_destructor_type)sqlite3OomClear) /* ** When SQLITE_OMIT_WSD is defined, it means that the target platform does @@ -11740,8 +15465,8 @@ struct BusyHandler { #define SQLITE_WSD const #define GLOBAL(t,v) (*(t*)sqlite3_wsd_find((void*)&(v), sizeof(v))) #define sqlite3GlobalConfig GLOBAL(struct Sqlite3Config, sqlite3Config) -SQLITE_API int SQLITE_STDCALL sqlite3_wsd_init(int N, int J); -SQLITE_API void *SQLITE_STDCALL sqlite3_wsd_find(void *K, int L); +SQLITE_API int sqlite3_wsd_init(int N, int J); +SQLITE_API void *sqlite3_wsd_find(void *K, int L); #else #define SQLITE_WSD #define GLOBAL(t,v) v @@ -11776,17 +15501,22 @@ typedef struct AutoincInfo AutoincInfo; typedef struct Bitvec Bitvec; typedef struct CollSeq CollSeq; typedef struct Column Column; +typedef struct Cte Cte; +typedef struct CteUse CteUse; typedef struct Db Db; +typedef struct DbClientData DbClientData; +typedef struct DbFixer DbFixer; typedef struct Schema Schema; typedef struct Expr Expr; typedef struct ExprList ExprList; -typedef struct ExprSpan ExprSpan; typedef struct FKey FKey; +typedef struct FpDecode FpDecode; typedef struct FuncDestructor FuncDestructor; typedef struct FuncDef FuncDef; typedef struct FuncDefHash FuncDefHash; typedef struct IdList IdList; typedef struct Index Index; +typedef struct IndexedExpr IndexedExpr; typedef struct IndexSample IndexSample; typedef struct KeyClass KeyClass; typedef struct KeyInfo KeyInfo; @@ -11794,16 +15524,23 @@ typedef struct Lookaside Lookaside; typedef struct LookasideSlot LookasideSlot; typedef struct Module Module; typedef struct NameContext NameContext; +typedef struct OnOrUsing OnOrUsing; typedef struct Parse Parse; +typedef struct ParseCleanup ParseCleanup; typedef struct PreUpdate PreUpdate; typedef struct PrintfArguments PrintfArguments; +typedef struct RCStr RCStr; +typedef struct RenameToken RenameToken; +typedef struct Returning Returning; typedef struct RowSet RowSet; typedef struct Savepoint Savepoint; typedef struct Select Select; typedef struct SQLiteThread SQLiteThread; typedef struct SelectDest SelectDest; +typedef struct Subquery Subquery; +typedef struct SrcItem SrcItem; typedef struct SrcList SrcList; -typedef struct StrAccum StrAccum; +typedef struct sqlite3_str StrAccum; /* Internal alias for sqlite3_str */ typedef struct Table Table; typedef struct TableLock TableLock; typedef struct Token Token; @@ -11812,21 +15549,60 @@ typedef struct Trigger Trigger; typedef struct TriggerPrg TriggerPrg; typedef struct TriggerStep TriggerStep; typedef struct UnpackedRecord UnpackedRecord; +typedef struct Upsert Upsert; typedef struct VTable VTable; typedef struct VtabCtx VtabCtx; typedef struct Walker Walker; typedef struct WhereInfo WhereInfo; +typedef struct Window Window; typedef struct With With; + +/* +** The bitmask datatype defined below is used for various optimizations. +** +** Changing this from a 64-bit to a 32-bit type limits the number of +** tables in a join to 32 instead of 64. But it also reduces the size +** of the library by 738 bytes on ix86. +*/ +#ifdef SQLITE_BITMASK_TYPE + typedef SQLITE_BITMASK_TYPE Bitmask; +#else + typedef u64 Bitmask; +#endif + +/* +** The number of bits in a Bitmask. "BMS" means "BitMask Size". +*/ +#define BMS ((int)(sizeof(Bitmask)*8)) + +/* +** A bit in a Bitmask +*/ +#define MASKBIT(n) (((Bitmask)1)<<(n)) +#define MASKBIT64(n) (((u64)1)<<(n)) +#define MASKBIT32(n) (((unsigned int)1)<<(n)) +#define SMASKBIT32(n) ((n)<=31?((unsigned int)1)<<(n):0) +#define ALLBITS ((Bitmask)-1) +#define TOPBIT (((Bitmask)1)<<(BMS-1)) + +/* A VList object records a mapping between parameters/variables/wildcards +** in the SQL statement (such as $abc, @pqr, or :xyz) and the integer +** variable number associated with that parameter. See the format description +** on the sqlite3VListAdd() routine for more information. A VList is really +** just an array of integers. +*/ +typedef int VList; + /* ** Defer sourcing vdbe.h and btree.h until after the "u8" and ** "BusyHandler" typedefs. vdbe.h also requires a few of the opaque ** pointer types (i.e. FuncDef) defined above. */ -/************** Include btree.h in the middle of sqliteInt.h *****************/ -/************** Begin file btree.h *******************************************/ +/************** Include os.h in the middle of sqliteInt.h ********************/ +/************** Begin file os.h **********************************************/ /* -** 2001 September 15 +** 2001 September 16 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -11835,858 +15611,318 @@ typedef struct With With; ** May you find forgiveness for yourself and forgive others. ** May you share freely, never taking more than you give. ** -************************************************************************* -** This header file defines the interface that the sqlite B-Tree file -** subsystem. See comments in the source code for a detailed description -** of what each interface routine does. +****************************************************************************** +** +** This header file (together with is companion C source-code file +** "os.c") attempt to abstract the underlying operating system so that +** the SQLite library will work on both POSIX and windows systems. +** +** This header file is #include-ed by sqliteInt.h and thus ends up +** being included by every source file. */ -#ifndef _BTREE_H_ -#define _BTREE_H_ - -/* TODO: This definition is just included so other modules compile. It -** needs to be revisited. -*/ -#define SQLITE_N_BTREE_META 16 +#ifndef _SQLITE_OS_H_ +#define _SQLITE_OS_H_ /* -** If defined as non-zero, auto-vacuum is enabled by default. Otherwise -** it must be turned on for each database using "PRAGMA auto_vacuum = 1". +** Attempt to automatically detect the operating system and setup the +** necessary pre-processor macros for it. */ -#ifndef SQLITE_DEFAULT_AUTOVACUUM - #define SQLITE_DEFAULT_AUTOVACUUM 0 -#endif - -#define BTREE_AUTOVACUUM_NONE 0 /* Do not do auto-vacuum */ -#define BTREE_AUTOVACUUM_FULL 1 /* Do full auto-vacuum */ -#define BTREE_AUTOVACUUM_INCR 2 /* Incremental vacuum */ +/************** Include os_setup.h in the middle of os.h *********************/ +/************** Begin file os_setup.h ****************************************/ +/* +** 2013 November 25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains pre-processor directives related to operating system +** detection and/or setup. +*/ +#ifndef SQLITE_OS_SETUP_H +#define SQLITE_OS_SETUP_H /* -** Forward declarations of structure -*/ -typedef struct Btree Btree; -typedef struct BtCursor BtCursor; -typedef struct BtShared BtShared; - - -SQLITE_PRIVATE int sqlite3BtreeOpen( - sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ - const char *zFilename, /* Name of database file to open */ - sqlite3 *db, /* Associated database connection */ - Btree **ppBtree, /* Return open Btree* here */ - int flags, /* Flags */ - int vfsFlags /* Flags passed through to VFS open */ -); - -/* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the -** following values. +** Figure out if we are dealing with Unix, Windows, or some other operating +** system. ** -** NOTE: These values must match the corresponding PAGER_ values in -** pager.h. +** After the following block of preprocess macros, all of +** +** SQLITE_OS_KV +** SQLITE_OS_OTHER +** SQLITE_OS_UNIX +** SQLITE_OS_WIN +** +** will defined to either 1 or 0. One of them will be 1. The others will be 0. +** If none of the macros are initially defined, then select either +** SQLITE_OS_UNIX or SQLITE_OS_WIN depending on the target platform. +** +** If SQLITE_OS_OTHER=1 is specified at compile-time, then the application +** must provide its own VFS implementation together with sqlite3_os_init() +** and sqlite3_os_end() routines. */ -#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ -#define BTREE_MEMORY 2 /* This is an in-memory DB */ -#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ -#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ - -SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); -SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree*,int); -#if SQLITE_MAX_MMAP_SIZE>0 -SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#if !defined(SQLITE_OS_KV) && !defined(SQLITE_OS_OTHER) && \ + !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_WIN) +# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ + defined(__MINGW32__) || defined(__BORLANDC__) +# define SQLITE_OS_WIN 1 +# define SQLITE_OS_UNIX 0 +# else +# define SQLITE_OS_WIN 0 +# define SQLITE_OS_UNIX 1 +# endif +#endif +#if SQLITE_OS_OTHER+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_KV+1>1 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +# define SQLITE_OMIT_LOAD_EXTENSION 1 +# define SQLITE_OMIT_WAL 1 +# define SQLITE_OMIT_DEPRECATED 1 +# undef SQLITE_TEMP_STORE +# define SQLITE_TEMP_STORE 3 /* Always use memory for temporary storage */ +# define SQLITE_DQS 0 +# define SQLITE_OMIT_SHARED_CACHE 1 +# define SQLITE_OMIT_AUTOINIT 1 +#endif +#if SQLITE_OS_UNIX+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_WIN +# define SQLITE_OS_WIN 0 +#endif +#if SQLITE_OS_WIN+1>1 +# undef SQLITE_OS_KV +# define SQLITE_OS_KV 0 +# undef SQLITE_OS_OTHER +# define SQLITE_OS_OTHER 0 +# undef SQLITE_OS_UNIX +# define SQLITE_OS_UNIX 0 #endif -SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); -SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); -SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); -SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree*,int); -SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree*); -SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree*); -SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); -SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); -SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char *zMaster); -SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); -SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); -SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int,int); -SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); -SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, int*, int flags); -SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree*); -SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree*); -SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree*); -SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); -SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *pBtree); -SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); -SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int); -SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *); -SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *); -SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); -SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); +#endif /* SQLITE_OS_SETUP_H */ -/* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR -** of the flags shown below. -** -** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. -** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data -** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With -** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored -** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL -** indices.) +/************** End of os_setup.h ********************************************/ +/************** Continuing where we left off in os.h *************************/ + +/* If the SET_FULLSYNC macro is not defined above, then make it +** a no-op */ -#define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ -#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ +#ifndef SET_FULLSYNC +# define SET_FULLSYNC(x,y) +#endif -SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); -SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, int*); -SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); -SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree*, int, int); +/* Maximum pathname length. Note: FILENAME_MAX defined by stdio.h +*/ +#ifndef SQLITE_MAX_PATHLEN +# define SQLITE_MAX_PATHLEN FILENAME_MAX +#endif -SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); -SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); - -SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); +/* Maximum number of symlinks that will be resolved while trying to +** expand a filename in xFullPathname() in the VFS. +*/ +#ifndef SQLITE_MAX_SYMLINK +# define SQLITE_MAX_SYMLINK 200 +#endif /* -** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta -** should be one of the following values. The integer values are assigned -** to constants so that the offset of the corresponding field in an -** SQLite database header may be found using the following formula: -** -** offset = 36 + (idx * 4) -** -** For example, the free-page-count field is located at byte offset 36 of -** the database file header. The incr-vacuum-flag field is located at -** byte offset 64 (== 36+4*7). -** -** The BTREE_DATA_VERSION value is not really a value stored in the header. -** It is a read-only number computed by the pager. But we merge it with -** the header value access routines since its access pattern is the same. -** Call it a "virtual meta value". +** The default size of a disk sector */ -#define BTREE_FREE_PAGE_COUNT 0 -#define BTREE_SCHEMA_VERSION 1 -#define BTREE_FILE_FORMAT 2 -#define BTREE_DEFAULT_CACHE_SIZE 3 -#define BTREE_LARGEST_ROOT_PAGE 4 -#define BTREE_TEXT_ENCODING 5 -#define BTREE_USER_VERSION 6 -#define BTREE_INCR_VACUUM 7 -#define BTREE_APPLICATION_ID 8 -#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ +#ifndef SQLITE_DEFAULT_SECTOR_SIZE +# define SQLITE_DEFAULT_SECTOR_SIZE 4096 +#endif /* -** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() -** interface. +** Temporary files are named starting with this prefix followed by 16 random +** alphanumeric characters, and no file extension. They are stored in the +** OS's standard temporary file directory, and are deleted prior to exit. +** If sqlite is being embedded in another program, you may wish to change the +** prefix to reflect your program's name, so that if your program exits +** prematurely, old temporary files can be easily identified. This can be done +** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. ** -** BTREE_HINT_RANGE (arguments: Expr*, Mem*) -** -** The first argument is an Expr* (which is guaranteed to be constant for -** the lifetime of the cursor) that defines constraints on which rows -** might be fetched with this cursor. The Expr* tree may contain -** TK_REGISTER nodes that refer to values stored in the array of registers -** passed as the second parameter. In other words, if Expr.op==TK_REGISTER -** then the value of the node is the value in Mem[pExpr.iTable]. Any -** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th -** column of the b-tree of the cursor. The Expr tree will not contain -** any function calls nor subqueries nor references to b-trees other than -** the cursor being hinted. -** -** The design of the _RANGE hint is aid b-tree implementations that try -** to prefetch content from remote machines - to provide those -** implementations with limits on what needs to be prefetched and thereby -** reduce network bandwidth. -** -** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by -** standard SQLite. The other hints are provided for extentions that use -** the SQLite parser and code generator but substitute their own storage -** engine. +** 2006-10-31: The default prefix used to be "sqlite_". But then +** Mcafee started using SQLite in their anti-virus product and it +** started putting files with the "sqlite" name in the c:/temp folder. +** This annoyed many windows users. Those users would then do a +** Google search for "sqlite", find the telephone numbers of the +** developers and call to wake them up at night and complain. +** For this reason, the default name prefix is changed to be "sqlite" +** spelled backwards. So the temp files are still identified, but +** anybody smart enough to figure out the code is also likely smart +** enough to know that calling the developer will not help get rid +** of the file. */ -#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ +#ifndef SQLITE_TEMP_FILE_PREFIX +# define SQLITE_TEMP_FILE_PREFIX "etilqs_" +#endif /* -** Values that may be OR'd together to form the argument to the -** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): +** The following values may be passed as the second argument to +** sqlite3OsLock(). The various locks exhibit the following semantics: ** -** The BTREE_BULKLOAD flag is set on index cursors when the index is going -** to be filled with content that is already in sorted order. +** SHARED: Any number of processes may hold a SHARED lock simultaneously. +** RESERVED: A single process may hold a RESERVED lock on a file at +** any time. Other processes may hold and obtain new SHARED locks. +** PENDING: A single process may hold a PENDING lock on a file at +** any one time. Existing SHARED locks may persist, but no new +** SHARED locks may be obtained by other processes. +** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. ** -** The BTREE_SEEK_EQ flag is set on cursors that will get OP_SeekGE or -** OP_SeekLE opcodes for a range search, but where the range of entries -** selected will all have the same key. In other words, the cursor will -** be used only for equality key searches. +** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a +** process that requests an EXCLUSIVE lock may actually obtain a PENDING +** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to +** sqlite3OsLock(). +*/ +#define NO_LOCK 0 +#define SHARED_LOCK 1 +#define RESERVED_LOCK 2 +#define PENDING_LOCK 3 +#define EXCLUSIVE_LOCK 4 + +/* +** File Locking Notes: (Mostly about windows but also some info for Unix) +** +** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because +** those functions are not available. So we use only LockFile() and +** UnlockFile(). +** +** LockFile() prevents not just writing but also reading by other processes. +** A SHARED_LOCK is obtained by locking a single randomly-chosen +** byte out of a specific range of bytes. The lock byte is obtained at +** random so two separate readers can probably access the file at the +** same time, unless they are unlucky and choose the same lock byte. +** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. +** There can only be one writer. A RESERVED_LOCK is obtained by locking +** a single byte of the file that is designated as the reserved lock byte. +** A PENDING_LOCK is obtained by locking a designated byte different from +** the RESERVED_LOCK byte. +** +** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, +** which means we can use reader/writer locks. When reader/writer locks +** are used, the lock is placed on the same range of bytes that is used +** for probabilistic locking in Win95/98/ME. Hence, the locking scheme +** will support two or more Win95 readers or two or more WinNT readers. +** But a single Win95 reader will lock out all WinNT readers and a single +** WinNT reader will lock out all other Win95 readers. +** +** The following #defines specify the range of bytes used for locking. +** SHARED_SIZE is the number of bytes available in the pool from which +** a random byte is selected for a shared lock. The pool of bytes for +** shared locks begins at SHARED_FIRST. +** +** The same locking strategy and +** byte ranges are used for Unix. This leaves open the possibility of having +** clients on win95, winNT, and unix all talking to the same shared file +** and all locking correctly. To do so would require that samba (or whatever +** tool is being used for file sharing) implements locks correctly between +** windows and unix. I'm guessing that isn't likely to happen, but by +** using the same locking range we are at least open to the possibility. +** +** Locking in windows is manditory. For this reason, we cannot store +** actual data in the bytes used for locking. The pager never allocates +** the pages involved in locking therefore. SHARED_SIZE is selected so +** that all locks will fit on a single page even at the minimum page size. +** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE +** is set high so that we don't have to allocate an unused page except +** for very large databases. But one should test the page skipping logic +** by setting PENDING_BYTE low and running the entire regression suite. +** +** Changing the value of PENDING_BYTE results in a subtly incompatible +** file format. Depending on how it is changed, you might not notice +** the incompatibility right away, even running a full regression test. +** The default location of PENDING_BYTE is the first byte past the +** 1GB boundary. ** */ -#define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ -#define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ +#ifdef SQLITE_OMIT_WSD +# define PENDING_BYTE (0x40000000) +#else +# define PENDING_BYTE sqlite3PendingByte +#endif +#define RESERVED_BYTE (PENDING_BYTE+1) +#define SHARED_FIRST (PENDING_BYTE+2) +#define SHARED_SIZE 510 -/* -** Flags passed as the third argument to sqlite3BtreeCursor(). -** -** For read-only cursors the wrFlag argument is always zero. For read-write -** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just -** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will -** only be used by SQLite for the following: -** -** * to seek to and then delete specific entries, and/or -** -** * to read values that will be used to create keys that other -** BTREE_FORDELETE cursors will seek to and delete. -** -** The BTREE_FORDELETE flag is an optimization hint. It is not used by -** by this, the native b-tree engine of SQLite, but it is available to -** alternative storage engines that might be substituted in place of this -** b-tree system. For alternative storage engines in which a delete of -** the main table row automatically deletes corresponding index rows, -** the FORDELETE flag hint allows those alternative storage engines to -** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK -** and DELETE operations as no-ops, and any READ operation against a -** FORDELETE cursor may return a null row: 0x01 0x00. +/* +** Wrapper around OS specific sqlite3_os_init() function. */ -#define BTREE_WRCSR 0x00000004 /* read-write cursor */ -#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ - -SQLITE_PRIVATE int sqlite3BtreeCursor( - Btree*, /* BTree containing table to open */ - int iTable, /* Index of root page */ - int wrFlag, /* 1 for writing. 0 for read-only */ - struct KeyInfo*, /* First argument to compare function */ - BtCursor *pCursor /* Space to write cursor structure */ -); -SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); -SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); -SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); -#ifdef SQLITE_ENABLE_CURSOR_HINTS -SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor*, int, ...); -#endif - -SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); -SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( - BtCursor*, - UnpackedRecord *pUnKey, - i64 intKey, - int bias, - int *pRes -); -SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); -SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); -SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, u8 flags); - -/* Allowed flags for the 2nd argument to sqlite3BtreeDelete() */ -#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ -#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ - -SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const void *pKey, i64 nKey, - const void *pData, int nData, - int nZero, int bias, int seekResult); -SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); -SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); -SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int *pRes); -SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); -SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int *pRes); -SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor*, i64 *pSize); -SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor*, u32 *pAmt); -SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor*, u32 *pAmt); -SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor*, u32 *pSize); -SQLITE_PRIVATE int sqlite3BtreeData(BtCursor*, u32 offset, u32 amt, void*); - -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck(Btree*, int *aRoot, int nRoot, int, int*); -SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); - -SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); -SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); -SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); -SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); -SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); -SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); -SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); - -#ifndef NDEBUG -SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); -#endif - -#ifndef SQLITE_OMIT_BTREECOUNT -SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *, i64 *); -#endif - -#ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); -SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); -#endif +SQLITE_PRIVATE int sqlite3OsInit(void); +/* +** Functions for accessing sqlite3_file methods +*/ +SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); +SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); +SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); +SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); +SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); +SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); +SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); +#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 +SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); #ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); -#endif - -/* -** If we are not using shared cache, then there is no need to -** use mutexes to access the BtShared structures. So make the -** Enter and Leave procedures no-ops. -*/ -#ifndef SQLITE_OMIT_SHARED_CACHE -SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); -SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); -SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); -SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); -#else -# define sqlite3BtreeEnter(X) -# define sqlite3BtreeEnterAll(X) -# define sqlite3BtreeSharable(X) 0 -# define sqlite3BtreeEnterCursor(X) -#endif - -#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE -SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); -SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); -SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); -#ifndef NDEBUG - /* These routines are used inside assert() statements only. */ -SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree*); -SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3*); -SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); -#endif -#else - -# define sqlite3BtreeLeave(X) -# define sqlite3BtreeLeaveCursor(X) -# define sqlite3BtreeLeaveAll(X) - -# define sqlite3BtreeHoldsMutex(X) 1 -# define sqlite3BtreeHoldsAllMutexes(X) 1 -# define sqlite3SchemaMutexHeld(X,Y,Z) 1 -#endif - - -#endif /* _BTREE_H_ */ - -/************** End of btree.h ***********************************************/ -/************** Continuing where we left off in sqliteInt.h ******************/ -/************** Include vdbe.h in the middle of sqliteInt.h ******************/ -/************** Begin file vdbe.h ********************************************/ -/* -** 2001 September 15 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -************************************************************************* -** Header file for the Virtual DataBase Engine (VDBE) -** -** This header defines the interface to the virtual database engine -** or VDBE. The VDBE implements an abstract machine that runs a -** simple program to access and modify the underlying database. -*/ -#ifndef _SQLITE_VDBE_H_ -#define _SQLITE_VDBE_H_ -/* #include */ - -/* -** A single VDBE is an opaque structure named "Vdbe". Only routines -** in the source file sqliteVdbe.c are allowed to see the insides -** of this structure. -*/ -typedef struct Vdbe Vdbe; - -/* -** The names of the following types declared in vdbeInt.h are required -** for the VdbeOp definition. -*/ -typedef struct Mem Mem; -typedef struct SubProgram SubProgram; - -/* -** A single instruction of the virtual machine has an opcode -** and as many as three operands. The instruction is recorded -** as an instance of the following structure: -*/ -struct VdbeOp { - u8 opcode; /* What operation to perform */ - signed char p4type; /* One of the P4_xxx constants for p4 */ - u8 notUsed1; - u8 p5; /* Fifth parameter is an unsigned character */ - int p1; /* First operand */ - int p2; /* Second parameter (often the jump destination) */ - int p3; /* The third parameter */ - union p4union { /* fourth parameter */ - int i; /* Integer value if p4type==P4_INT32 */ - void *p; /* Generic pointer */ - char *z; /* Pointer to data for string (char array) types */ - i64 *pI64; /* Used when p4type is P4_INT64 */ - double *pReal; /* Used when p4type is P4_REAL */ - FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ - sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ - CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ - Mem *pMem; /* Used when p4type is P4_MEM */ - VTable *pVtab; /* Used when p4type is P4_VTAB */ - KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ - int *ai; /* Used when p4type is P4_INTARRAY */ - SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ - Table *pTab; /* Used when p4type is P4_TABLE */ -#ifdef SQLITE_ENABLE_CURSOR_HINTS - Expr *pExpr; /* Used when p4type is P4_EXPR */ -#endif - int (*xAdvance)(BtCursor *, int *); - } p4; -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - char *zComment; /* Comment to improve readability */ -#endif -#ifdef VDBE_PROFILE - u32 cnt; /* Number of times this instruction was executed */ - u64 cycles; /* Total time spent executing this instruction */ -#endif -#ifdef SQLITE_VDBE_COVERAGE - int iSrcLine; /* Source-code line that generated this opcode */ -#endif -}; -typedef struct VdbeOp VdbeOp; +SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); +SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); +SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); +SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); +#endif /* SQLITE_OMIT_WAL */ +SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); +SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); /* -** A sub-routine used to implement a trigger program. +** Functions for accessing sqlite3_vfs methods */ -struct SubProgram { - VdbeOp *aOp; /* Array of opcodes for sub-program */ - int nOp; /* Elements in aOp[] */ - int nMem; /* Number of memory cells required */ - int nCsr; /* Number of cursors required */ - int nOnce; /* Number of OP_Once instructions */ - void *token; /* id that may be used to recursive triggers */ - SubProgram *pNext; /* Next sub-program already visited */ -}; +SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); +SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); +SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); +SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); +#ifndef SQLITE_OMIT_LOAD_EXTENSION +SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); +SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); +SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); +#endif /* SQLITE_OMIT_LOAD_EXTENSION */ +SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); +SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); +SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); +SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); /* -** A smaller version of VdbeOp used for the VdbeAddOpList() function because -** it takes up less space. +** Convenience functions for opening and closing files using +** sqlite3_malloc() to obtain space for the file-handle structure. */ -struct VdbeOpList { - u8 opcode; /* What operation to perform */ - signed char p1; /* First operand */ - signed char p2; /* Second parameter (often the jump destination) */ - signed char p3; /* Third parameter */ -}; -typedef struct VdbeOpList VdbeOpList; +SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); +SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); -/* -** Allowed values of VdbeOp.p4type -*/ -#define P4_NOTUSED 0 /* The P4 parameter is not used */ -#define P4_DYNAMIC (-1) /* Pointer to a string obtained from sqliteMalloc() */ -#define P4_STATIC (-2) /* Pointer to a static string */ -#define P4_COLLSEQ (-4) /* P4 is a pointer to a CollSeq structure */ -#define P4_FUNCDEF (-5) /* P4 is a pointer to a FuncDef structure */ -#define P4_KEYINFO (-6) /* P4 is a pointer to a KeyInfo structure */ -#define P4_EXPR (-7) /* P4 is a pointer to an Expr tree */ -#define P4_MEM (-8) /* P4 is a pointer to a Mem* structure */ -#define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ -#define P4_VTAB (-10) /* P4 is a pointer to an sqlite3_vtab structure */ -#define P4_MPRINTF (-11) /* P4 is a string obtained from sqlite3_mprintf() */ -#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ -#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ -#define P4_INT32 (-14) /* P4 is a 32-bit signed integer */ -#define P4_INTARRAY (-15) /* P4 is a vector of 32-bit integers */ -#define P4_SUBPROGRAM (-18) /* P4 is a pointer to a SubProgram structure */ -#define P4_ADVANCE (-19) /* P4 is a pointer to BtreeNext() or BtreePrev() */ -#define P4_TABLE (-20) /* P4 is a pointer to a Table structure */ -#define P4_FUNCCTX (-21) /* P4 is a pointer to an sqlite3_context object */ +#endif /* _SQLITE_OS_H_ */ -/* Error message codes for OP_Halt */ -#define P5_ConstraintNotNull 1 -#define P5_ConstraintUnique 2 -#define P5_ConstraintCheck 3 -#define P5_ConstraintFK 4 - -/* -** The Vdbe.aColName array contains 5n Mem structures, where n is the -** number of columns of data returned by the statement. -*/ -#define COLNAME_NAME 0 -#define COLNAME_DECLTYPE 1 -#define COLNAME_DATABASE 2 -#define COLNAME_TABLE 3 -#define COLNAME_COLUMN 4 -#ifdef SQLITE_ENABLE_COLUMN_METADATA -# define COLNAME_N 5 /* Number of COLNAME_xxx symbols */ -#else -# ifdef SQLITE_OMIT_DECLTYPE -# define COLNAME_N 1 /* Store only the name */ -# else -# define COLNAME_N 2 /* Store the name and decltype */ -# endif -#endif - -/* -** The following macro converts a relative address in the p2 field -** of a VdbeOp structure into a negative number so that -** sqlite3VdbeAddOpList() knows that the address is relative. Calling -** the macro again restores the address. -*/ -#define ADDR(X) (-1-(X)) - -/* -** The makefile scans the vdbe.c source file and creates the "opcodes.h" -** header file that defines a number for each opcode used by the VDBE. -*/ -/************** Include opcodes.h in the middle of vdbe.h ********************/ -/************** Begin file opcodes.h *****************************************/ -/* Automatically generated. Do not edit */ -/* See the tool/mkopcodeh.tcl script for details */ -#define OP_Savepoint 0 -#define OP_AutoCommit 1 -#define OP_Transaction 2 -#define OP_SorterNext 3 -#define OP_PrevIfOpen 4 -#define OP_NextIfOpen 5 -#define OP_Prev 6 -#define OP_Next 7 -#define OP_Checkpoint 8 -#define OP_JournalMode 9 -#define OP_Vacuum 10 -#define OP_VFilter 11 /* synopsis: iplan=r[P3] zplan='P4' */ -#define OP_VUpdate 12 /* synopsis: data=r[P3@P2] */ -#define OP_Goto 13 -#define OP_Gosub 14 -#define OP_InitCoroutine 15 -#define OP_Yield 16 -#define OP_MustBeInt 17 -#define OP_Jump 18 -#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ -#define OP_Once 20 -#define OP_If 21 -#define OP_IfNot 22 -#define OP_SeekLT 23 /* synopsis: key=r[P3@P4] */ -#define OP_SeekLE 24 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGE 25 /* synopsis: key=r[P3@P4] */ -#define OP_SeekGT 26 /* synopsis: key=r[P3@P4] */ -#define OP_Or 27 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ -#define OP_And 28 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ -#define OP_NoConflict 29 /* synopsis: key=r[P3@P4] */ -#define OP_NotFound 30 /* synopsis: key=r[P3@P4] */ -#define OP_Found 31 /* synopsis: key=r[P3@P4] */ -#define OP_NotExists 32 /* synopsis: intkey=r[P3] */ -#define OP_Last 33 -#define OP_IsNull 34 /* same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ -#define OP_NotNull 35 /* same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ -#define OP_Ne 36 /* same as TK_NE, synopsis: if r[P1]!=r[P3] goto P2 */ -#define OP_Eq 37 /* same as TK_EQ, synopsis: if r[P1]==r[P3] goto P2 */ -#define OP_Gt 38 /* same as TK_GT, synopsis: if r[P1]>r[P3] goto P2 */ -#define OP_Le 39 /* same as TK_LE, synopsis: if r[P1]<=r[P3] goto P2 */ -#define OP_Lt 40 /* same as TK_LT, synopsis: if r[P1]=r[P3] goto P2 */ -#define OP_SorterSort 42 -#define OP_BitAnd 43 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ -#define OP_BitOr 44 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ -#define OP_ShiftLeft 45 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ -#define OP_Add 47 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ -#define OP_Subtract 48 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ -#define OP_Multiply 49 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ -#define OP_Divide 50 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ -#define OP_Remainder 51 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ -#define OP_Concat 52 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ -#define OP_Sort 53 -#define OP_BitNot 54 /* same as TK_BITNOT, synopsis: r[P1]= ~r[P1] */ -#define OP_Rewind 55 -#define OP_IdxLE 56 /* synopsis: key=r[P3@P4] */ -#define OP_IdxGT 57 /* synopsis: key=r[P3@P4] */ -#define OP_IdxLT 58 /* synopsis: key=r[P3@P4] */ -#define OP_IdxGE 59 /* synopsis: key=r[P3@P4] */ -#define OP_RowSetRead 60 /* synopsis: r[P3]=rowset(P1) */ -#define OP_RowSetTest 61 /* synopsis: if r[P3] in rowset(P1) goto P2 */ -#define OP_Program 62 -#define OP_FkIfZero 63 /* synopsis: if fkctr[P1]==0 goto P2 */ -#define OP_IfPos 64 /* synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ -#define OP_IfNotZero 65 /* synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 */ -#define OP_DecrJumpZero 66 /* synopsis: if (--r[P1])==0 goto P2 */ -#define OP_IncrVacuum 67 -#define OP_VNext 68 -#define OP_Init 69 /* synopsis: Start at P2 */ -#define OP_Return 70 -#define OP_EndCoroutine 71 -#define OP_HaltIfNull 72 /* synopsis: if r[P3]=null halt */ -#define OP_Halt 73 -#define OP_Integer 74 /* synopsis: r[P2]=P1 */ -#define OP_Int64 75 /* synopsis: r[P2]=P4 */ -#define OP_String 76 /* synopsis: r[P2]='P4' (len=P1) */ -#define OP_Null 77 /* synopsis: r[P2..P3]=NULL */ -#define OP_SoftNull 78 /* synopsis: r[P1]=NULL */ -#define OP_Blob 79 /* synopsis: r[P2]=P4 (len=P1) */ -#define OP_Variable 80 /* synopsis: r[P2]=parameter(P1,P4) */ -#define OP_Move 81 /* synopsis: r[P2@P3]=r[P1@P3] */ -#define OP_Copy 82 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ -#define OP_SCopy 83 /* synopsis: r[P2]=r[P1] */ -#define OP_IntCopy 84 /* synopsis: r[P2]=r[P1] */ -#define OP_ResultRow 85 /* synopsis: output=r[P1@P2] */ -#define OP_CollSeq 86 -#define OP_Function0 87 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_Function 88 /* synopsis: r[P3]=func(r[P2@P5]) */ -#define OP_AddImm 89 /* synopsis: r[P1]=r[P1]+P2 */ -#define OP_RealAffinity 90 -#define OP_Cast 91 /* synopsis: affinity(r[P1]) */ -#define OP_Permutation 92 -#define OP_Compare 93 /* synopsis: r[P1@P3] <-> r[P2@P3] */ -#define OP_Column 94 /* synopsis: r[P3]=PX */ -#define OP_Affinity 95 /* synopsis: affinity(r[P1@P2]) */ -#define OP_MakeRecord 96 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ -#define OP_String8 97 /* same as TK_STRING, synopsis: r[P2]='P4' */ -#define OP_Count 98 /* synopsis: r[P2]=count() */ -#define OP_ReadCookie 99 -#define OP_SetCookie 100 -#define OP_ReopenIdx 101 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenRead 102 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenWrite 103 /* synopsis: root=P2 iDb=P3 */ -#define OP_OpenAutoindex 104 /* synopsis: nColumn=P2 */ -#define OP_OpenEphemeral 105 /* synopsis: nColumn=P2 */ -#define OP_SorterOpen 106 -#define OP_SequenceTest 107 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ -#define OP_OpenPseudo 108 /* synopsis: P3 columns in r[P2] */ -#define OP_Close 109 -#define OP_ColumnsUsed 110 -#define OP_Sequence 111 /* synopsis: r[P2]=cursor[P1].ctr++ */ -#define OP_NewRowid 112 /* synopsis: r[P2]=rowid */ -#define OP_Insert 113 /* synopsis: intkey=r[P3] data=r[P2] */ -#define OP_InsertInt 114 /* synopsis: intkey=P3 data=r[P2] */ -#define OP_Delete 115 -#define OP_ResetCount 116 -#define OP_SorterCompare 117 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ -#define OP_SorterData 118 /* synopsis: r[P2]=data */ -#define OP_RowKey 119 /* synopsis: r[P2]=key */ -#define OP_RowData 120 /* synopsis: r[P2]=data */ -#define OP_Rowid 121 /* synopsis: r[P2]=rowid */ -#define OP_NullRow 122 -#define OP_SorterInsert 123 -#define OP_IdxInsert 124 /* synopsis: key=r[P2] */ -#define OP_IdxDelete 125 /* synopsis: key=r[P2@P3] */ -#define OP_Seek 126 /* synopsis: Move P3 to P1.rowid */ -#define OP_IdxRowid 127 /* synopsis: r[P2]=rowid */ -#define OP_Destroy 128 -#define OP_Clear 129 -#define OP_ResetSorter 130 -#define OP_CreateIndex 131 /* synopsis: r[P2]=root iDb=P1 */ -#define OP_CreateTable 132 /* synopsis: r[P2]=root iDb=P1 */ -#define OP_Real 133 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ -#define OP_ParseSchema 134 -#define OP_LoadAnalysis 135 -#define OP_DropTable 136 -#define OP_DropIndex 137 -#define OP_DropTrigger 138 -#define OP_IntegrityCk 139 -#define OP_RowSetAdd 140 /* synopsis: rowset(P1)=r[P2] */ -#define OP_Param 141 -#define OP_FkCounter 142 /* synopsis: fkctr[P1]+=P2 */ -#define OP_MemMax 143 /* synopsis: r[P1]=max(r[P1],r[P2]) */ -#define OP_OffsetLimit 144 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ -#define OP_AggStep0 145 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggStep 146 /* synopsis: accum=r[P3] step(r[P2@P5]) */ -#define OP_AggFinal 147 /* synopsis: accum=r[P1] N=P2 */ -#define OP_Expire 148 -#define OP_TableLock 149 /* synopsis: iDb=P1 root=P2 write=P3 */ -#define OP_VBegin 150 -#define OP_VCreate 151 -#define OP_VDestroy 152 -#define OP_VOpen 153 -#define OP_VColumn 154 /* synopsis: r[P3]=vcolumn(P2) */ -#define OP_VRename 155 -#define OP_Pagecount 156 -#define OP_MaxPgcnt 157 -#define OP_CursorHint 158 -#define OP_Noop 159 -#define OP_Explain 160 - -/* Properties such as "out2" or "jump" that are specified in -** comments following the "case" for each opcode in the vdbe.c -** are encoded into bitvectors as follows: -*/ -#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ -#define OPFLG_IN1 0x02 /* in1: P1 is an input */ -#define OPFLG_IN2 0x04 /* in2: P2 is an input */ -#define OPFLG_IN3 0x08 /* in3: P3 is an input */ -#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ -#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ -#define OPFLG_INITIALIZER {\ -/* 0 */ 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, 0x01, 0x01,\ -/* 8 */ 0x00, 0x10, 0x00, 0x01, 0x00, 0x01, 0x01, 0x01,\ -/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0x03, 0x03, 0x09,\ -/* 24 */ 0x09, 0x09, 0x09, 0x26, 0x26, 0x09, 0x09, 0x09,\ -/* 32 */ 0x09, 0x01, 0x03, 0x03, 0x0b, 0x0b, 0x0b, 0x0b,\ -/* 40 */ 0x0b, 0x0b, 0x01, 0x26, 0x26, 0x26, 0x26, 0x26,\ -/* 48 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x01, 0x12, 0x01,\ -/* 56 */ 0x01, 0x01, 0x01, 0x01, 0x23, 0x0b, 0x01, 0x01,\ -/* 64 */ 0x03, 0x03, 0x03, 0x01, 0x01, 0x01, 0x02, 0x02,\ -/* 72 */ 0x08, 0x00, 0x10, 0x10, 0x10, 0x10, 0x00, 0x10,\ -/* 80 */ 0x10, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00, 0x00,\ -/* 88 */ 0x00, 0x02, 0x02, 0x02, 0x00, 0x00, 0x00, 0x00,\ -/* 96 */ 0x00, 0x10, 0x10, 0x10, 0x00, 0x00, 0x00, 0x00,\ -/* 104 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10,\ -/* 112 */ 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 120 */ 0x00, 0x10, 0x00, 0x04, 0x04, 0x00, 0x00, 0x10,\ -/* 128 */ 0x10, 0x00, 0x00, 0x10, 0x10, 0x10, 0x00, 0x00,\ -/* 136 */ 0x00, 0x00, 0x00, 0x00, 0x06, 0x10, 0x00, 0x04,\ -/* 144 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ -/* 152 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x10, 0x00, 0x00,\ -/* 160 */ 0x00,} - -/* The sqlite3P2Values() routine is able to run faster if it knows -** the value of the largest JUMP opcode. The smaller the maximum -** JUMP opcode the better, so the mkopcodeh.tcl script that -** generated this include file strives to group all JUMP opcodes -** together near the beginning of the list. -*/ -#define SQLITE_MX_JUMP_OPCODE 69 /* Maximum JUMP opcode */ - -/************** End of opcodes.h *********************************************/ -/************** Continuing where we left off in vdbe.h ***********************/ - -/* -** Prototypes for the VDBE interface. See comments on the implementation -** for a description of what each of these routines does. -*/ -SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); -SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); -SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); -SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); -SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); -SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); -#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) -SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); -#else -# define sqlite3VdbeVerifyNoMallocRequired(A,B) -#endif -SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp, int iLineno); -SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*,int,char*); -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, u32 addr, u8); -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, u32 addr, int P1); -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, u32 addr, int P2); -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, u32 addr, int P3); -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u8 P5); -SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); -SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); -SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); -SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); -SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); -SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); -SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3*,Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); -SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); -SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); -#ifdef SQLITE_DEBUG -SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); -#endif -SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); -SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); -SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*); -SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, int); -SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); -SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); -SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); -SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); -#ifndef SQLITE_OMIT_TRACE -SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); -#endif -SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); - -SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); -SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); -SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); -SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo *, char *, int, char **); - -typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); -SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); - -#ifndef SQLITE_OMIT_TRIGGER -SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); -#endif - -/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on -** each VDBE opcode. -** -** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op -** comments in VDBE programs that show key decision points in the code -** generator. -*/ -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS -SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); -# define VdbeComment(X) sqlite3VdbeComment X -SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); -# define VdbeNoopComment(X) sqlite3VdbeNoopComment X -# ifdef SQLITE_ENABLE_MODULE_COMMENTS -# define VdbeModuleComment(X) sqlite3VdbeNoopComment X -# else -# define VdbeModuleComment(X) -# endif -#else -# define VdbeComment(X) -# define VdbeNoopComment(X) -# define VdbeModuleComment(X) -#endif - -/* -** The VdbeCoverage macros are used to set a coverage testing point -** for VDBE branch instructions. The coverage testing points are line -** numbers in the sqlite3.c source file. VDBE branch coverage testing -** only works with an amalagmation build. That's ok since a VDBE branch -** coverage build designed for testing the test suite only. No application -** should ever ship with VDBE branch coverage measuring turned on. -** -** VdbeCoverage(v) // Mark the previously coded instruction -** // as a branch -** -** VdbeCoverageIf(v, conditional) // Mark previous if conditional true -** -** VdbeCoverageAlwaysTaken(v) // Previous branch is always taken -** -** VdbeCoverageNeverTaken(v) // Previous branch is never taken -** -** Every VDBE branch operation must be tagged with one of the macros above. -** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and -** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() -** routine in vdbe.c, alerting the developer to the missed tag. -*/ -#ifdef SQLITE_VDBE_COVERAGE -SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); -# define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) -# define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) -# define VdbeCoverageAlwaysTaken(v) sqlite3VdbeSetLineNumber(v,2); -# define VdbeCoverageNeverTaken(v) sqlite3VdbeSetLineNumber(v,1); -# define VDBE_OFFSET_LINENO(x) (__LINE__+x) -#else -# define VdbeCoverage(v) -# define VdbeCoverageIf(v,x) -# define VdbeCoverageAlwaysTaken(v) -# define VdbeCoverageNeverTaken(v) -# define VDBE_OFFSET_LINENO(x) 0 -#endif - -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS -SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); -#else -# define sqlite3VdbeScanStatus(a,b,c,d,e) -#endif - -#endif - -/************** End of vdbe.h ************************************************/ +/************** End of os.h **************************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ /************** Include pager.h in the middle of sqliteInt.h *****************/ /************** Begin file pager.h *******************************************/ @@ -12706,12 +15942,12 @@ SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const ch ** at a time and provides a journal for rollback. */ -#ifndef _PAGER_H_ -#define _PAGER_H_ +#ifndef SQLITE_PAGER_H +#define SQLITE_PAGER_H /* -** Default maximum size for persistent journal files. A negative -** value means no limit. This value may be overridden using the +** Default maximum size for persistent journal files. A negative +** value means no limit. This value may be overridden using the ** sqlite3PagerJournalSizeLimit() API. See also "PRAGMA journal_size_limit". */ #ifndef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT @@ -12735,14 +15971,15 @@ typedef struct Pager Pager; typedef struct PgHdr DbPage; /* -** Page number PAGER_MJ_PGNO is never used in an SQLite database (it is +** Page number PAGER_SJ_PGNO is never used in an SQLite database (it is ** reserved for working around a windows/posix incompatibility). It is -** used in the journal to signify that the remainder of the journal file -** is devoted to storing a master journal name - there are no more pages to -** roll back. See comments for function writeMasterJournal() in pager.c +** used in the journal to signify that the remainder of the journal file +** is devoted to storing a super-journal name - there are no more pages to +** roll back. See comments for function writeSuperJournal() in pager.c ** for details. */ -#define PAGER_MJ_PGNO(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) +#define PAGER_SJ_PGNO_COMPUTED(x) ((Pgno)((PENDING_BYTE/((x)->pageSize))+1)) +#define PAGER_SJ_PGNO(x) ((x)->lckPgno) /* ** Allowed values for the flags parameter to sqlite3PagerOpen(). @@ -12800,11 +16037,11 @@ typedef struct PgHdr DbPage; /* ** The remainder of this file contains the declarations of the functions -** that make up the Pager sub-system API. See source code comments for +** that make up the Pager sub-system API. See source code comments for ** a detailed description of each routine. */ -/* Open and close a Pager connection. */ +/* Open and close a Pager connection. */ SQLITE_PRIVATE int sqlite3PagerOpen( sqlite3_vfs*, Pager **ppPager, @@ -12814,16 +16051,13 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int, void(*)(DbPage*) ); -SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3*); SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager*, int, unsigned char*); /* Functions used to configure a Pager object. */ -SQLITE_PRIVATE void sqlite3PagerSetBusyhandler(Pager*, int(*)(void *), void *); +SQLITE_PRIVATE void sqlite3PagerSetBusyHandler(Pager*, int(*)(void *), void *); SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager*, u32*, int); -#ifdef SQLITE_HAS_CODEC -SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager*,Pager*); -#endif -SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager*, int); +SQLITE_PRIVATE Pgno sqlite3PagerMaxPageCount(Pager*, Pgno); SQLITE_PRIVATE void sqlite3PagerSetCachesize(Pager*, int); SQLITE_PRIVATE int sqlite3PagerSetSpillsize(Pager*, int); SQLITE_PRIVATE void sqlite3PagerSetMmapLimit(Pager *, sqlite3_int64); @@ -12837,27 +16071,28 @@ SQLITE_PRIVATE i64 sqlite3PagerJournalSizeLimit(Pager *, i64); SQLITE_PRIVATE sqlite3_backup **sqlite3PagerBackupPtr(Pager*); SQLITE_PRIVATE int sqlite3PagerFlush(Pager*); -/* Functions used to obtain and release page references. */ +/* Functions used to obtain and release page references. */ SQLITE_PRIVATE int sqlite3PagerGet(Pager *pPager, Pgno pgno, DbPage **ppPage, int clrFlag); SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno); SQLITE_PRIVATE void sqlite3PagerRef(DbPage*); SQLITE_PRIVATE void sqlite3PagerUnref(DbPage*); SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage*); +SQLITE_PRIVATE void sqlite3PagerUnrefPageOne(DbPage*); /* Operations on page references. */ SQLITE_PRIVATE int sqlite3PagerWrite(DbPage*); SQLITE_PRIVATE void sqlite3PagerDontWrite(DbPage*); SQLITE_PRIVATE int sqlite3PagerMovepage(Pager*,DbPage*,Pgno,int); SQLITE_PRIVATE int sqlite3PagerPageRefcount(DbPage*); -SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); -SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); +SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *); +SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *); /* Functions used to manage pager transactions and savepoints. */ SQLITE_PRIVATE void sqlite3PagerPagecount(Pager*, int*); SQLITE_PRIVATE int sqlite3PagerBegin(Pager*, int exFlag, int); -SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zMaster, int); +SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne(Pager*,const char *zSuper, int); SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager*); -SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster); +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zSuper); SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager*); SQLITE_PRIVATE int sqlite3PagerRollback(Pager*); SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int n); @@ -12865,17 +16100,32 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint); SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager); #ifndef SQLITE_OMIT_WAL -SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int, int*, int*); +SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, sqlite3*, int, int*, int*); SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerWalCallback(Pager *pPager); SQLITE_PRIVATE int sqlite3PagerOpenWal(Pager *pPager, int *pisOpen); -SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3*); # ifdef SQLITE_ENABLE_SNAPSHOT -SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppSnapshot); -SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager*, sqlite3_snapshot **ppSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager*, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager); +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager); # endif #endif +#if !defined(SQLITE_OMIT_WAL) && defined(SQLITE_ENABLE_SETLK_TIMEOUT) +SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager*, int); +SQLITE_PRIVATE void sqlite3PagerWalDb(Pager*, sqlite3*); +#else +# define sqlite3PagerWalWriteLock(y,z) SQLITE_OK +# define sqlite3PagerWalDb(x,y) +#endif + +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno); +#endif + #ifdef SQLITE_ENABLE_ZIPVFS SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager); #endif @@ -12887,15 +16137,15 @@ SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager*); SQLITE_PRIVATE int sqlite3PagerRefcount(Pager*); #endif SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager*); -SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager*, int); +SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager*, int); SQLITE_PRIVATE sqlite3_vfs *sqlite3PagerVfs(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager*); SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager*); SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager*); SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager*); SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager*); -SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, int *); -SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *); +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *, int, int, u64*); +SQLITE_PRIVATE void sqlite3PagerClearCache(Pager*); SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *); /* Functions used to truncate the database file. */ @@ -12903,10 +16153,6 @@ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager*,Pgno); SQLITE_PRIVATE void sqlite3PagerRekey(DbPage*, Pgno, u16); -#if defined(SQLITE_HAS_CODEC) && !defined(SQLITE_OMIT_WAL) -SQLITE_PRIVATE void *sqlite3PagerCodec(DbPage *); -#endif - /* Functions to support testing and debugging. */ #if !defined(NDEBUG) || defined(SQLITE_TEST) SQLITE_PRIVATE Pgno sqlite3PagerPagenumber(DbPage*); @@ -12922,10 +16168,1118 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); # define enable_simulated_io_errors() #endif -#endif /* _PAGER_H_ */ +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager*); +#endif + +#endif /* SQLITE_PAGER_H */ /************** End of pager.h ***********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include btree.h in the middle of sqliteInt.h *****************/ +/************** Begin file btree.h *******************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This header file defines the interface that the sqlite B-Tree file +** subsystem. See comments in the source code for a detailed description +** of what each interface routine does. +*/ +#ifndef SQLITE_BTREE_H +#define SQLITE_BTREE_H + +/* TODO: This definition is just included so other modules compile. It +** needs to be revisited. +*/ +#define SQLITE_N_BTREE_META 16 + +/* +** If defined as non-zero, auto-vacuum is enabled by default. Otherwise +** it must be turned on for each database using "PRAGMA auto_vacuum = 1". +*/ +#ifndef SQLITE_DEFAULT_AUTOVACUUM + #define SQLITE_DEFAULT_AUTOVACUUM 0 +#endif + +#define BTREE_AUTOVACUUM_NONE 0 /* Do not do auto-vacuum */ +#define BTREE_AUTOVACUUM_FULL 1 /* Do full auto-vacuum */ +#define BTREE_AUTOVACUUM_INCR 2 /* Incremental vacuum */ + +/* +** Forward declarations of structure +*/ +typedef struct Btree Btree; +typedef struct BtCursor BtCursor; +typedef struct BtShared BtShared; +typedef struct BtreePayload BtreePayload; + + +SQLITE_PRIVATE int sqlite3BtreeOpen( + sqlite3_vfs *pVfs, /* VFS to use with this b-tree */ + const char *zFilename, /* Name of database file to open */ + sqlite3 *db, /* Associated database connection */ + Btree **ppBtree, /* Return open Btree* here */ + int flags, /* Flags */ + int vfsFlags /* Flags passed through to VFS open */ +); + +/* The flags parameter to sqlite3BtreeOpen can be the bitwise or of the +** following values. +** +** NOTE: These values must match the corresponding PAGER_ values in +** pager.h. +*/ +#define BTREE_OMIT_JOURNAL 1 /* Do not create or use a rollback journal */ +#define BTREE_MEMORY 2 /* This is an in-memory DB */ +#define BTREE_SINGLE 4 /* The file contains at most 1 b-tree */ +#define BTREE_UNORDERED 8 /* Use of a hash implementation is OK */ + +SQLITE_PRIVATE int sqlite3BtreeClose(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSetCacheSize(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeSetSpillSize(Btree*,int); +#if SQLITE_MAX_MMAP_SIZE>0 +SQLITE_PRIVATE int sqlite3BtreeSetMmapLimit(Btree*,sqlite3_int64); +#endif +SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags(Btree*,unsigned); +SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int nPagesize, int nReserve, int eFix); +SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree*); +SQLITE_PRIVATE Pgno sqlite3BtreeMaxPageCount(Btree*,Pgno); +SQLITE_PRIVATE Pgno sqlite3BtreeLastPage(Btree*); +SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree*); +SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p); +SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *, int); +SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *); +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree*,int,int*); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree*, const char*); +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree*, int); +SQLITE_PRIVATE int sqlite3BtreeCommit(Btree*); +SQLITE_PRIVATE int sqlite3BtreeRollback(Btree*,int,int); +SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree*,int); +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree*, Pgno*, int flags); +SQLITE_PRIVATE int sqlite3BtreeTxnState(Btree*); +SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree*); + +SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *, int, void(*)(void *)); +SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *pBtree); +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *pBtree, int iTab, u8 isWriteLock); +#endif + +/* Savepoints are named, nestable SQL transactions mostly implemented */ +/* in vdbe.c and pager.c See https://sqlite.org/lang_savepoint.html */ +SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *, int, int); + +/* "Checkpoint" only refers to WAL. See https://sqlite.org/wal.html#ckpt */ +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); +#endif + +SQLITE_PRIVATE const char *sqlite3BtreeGetFilename(Btree *); +SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *); +SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *, Btree *); + +SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *); + +/* The flags parameter to sqlite3BtreeCreateTable can be the bitwise OR +** of the flags shown below. +** +** Every SQLite table must have either BTREE_INTKEY or BTREE_BLOBKEY set. +** With BTREE_INTKEY, the table key is a 64-bit integer and arbitrary data +** is stored in the leaves. (BTREE_INTKEY is used for SQL tables.) With +** BTREE_BLOBKEY, the key is an arbitrary BLOB and no content is stored +** anywhere - the key is the content. (BTREE_BLOBKEY is used for SQL +** indices.) +*/ +#define BTREE_INTKEY 1 /* Table has only 64-bit signed integer keys */ +#define BTREE_BLOBKEY 2 /* Table has keys only - no data */ + +SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree*, int, int*); +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree*, int, i64*); +SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree*, int, int); + +SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *pBtree, int idx, u32 *pValue); +SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree*, int idx, u32 value); + +SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p); + +/* +** The second parameter to sqlite3BtreeGetMeta or sqlite3BtreeUpdateMeta +** should be one of the following values. The integer values are assigned +** to constants so that the offset of the corresponding field in an +** SQLite database header may be found using the following formula: +** +** offset = 36 + (idx * 4) +** +** For example, the free-page-count field is located at byte offset 36 of +** the database file header. The incr-vacuum-flag field is located at +** byte offset 64 (== 36+4*7). +** +** The BTREE_DATA_VERSION value is not really a value stored in the header. +** It is a read-only number computed by the pager. But we merge it with +** the header value access routines since its access pattern is the same. +** Call it a "virtual meta value". +*/ +#define BTREE_FREE_PAGE_COUNT 0 +#define BTREE_SCHEMA_VERSION 1 +#define BTREE_FILE_FORMAT 2 +#define BTREE_DEFAULT_CACHE_SIZE 3 +#define BTREE_LARGEST_ROOT_PAGE 4 +#define BTREE_TEXT_ENCODING 5 +#define BTREE_USER_VERSION 6 +#define BTREE_INCR_VACUUM 7 +#define BTREE_APPLICATION_ID 8 +#define BTREE_DATA_VERSION 15 /* A virtual meta-value */ + +/* +** Kinds of hints that can be passed into the sqlite3BtreeCursorHint() +** interface. +** +** BTREE_HINT_RANGE (arguments: Expr*, Mem*) +** +** The first argument is an Expr* (which is guaranteed to be constant for +** the lifetime of the cursor) that defines constraints on which rows +** might be fetched with this cursor. The Expr* tree may contain +** TK_REGISTER nodes that refer to values stored in the array of registers +** passed as the second parameter. In other words, if Expr.op==TK_REGISTER +** then the value of the node is the value in Mem[pExpr.iTable]. Any +** TK_COLUMN node in the expression tree refers to the Expr.iColumn-th +** column of the b-tree of the cursor. The Expr tree will not contain +** any function calls nor subqueries nor references to b-trees other than +** the cursor being hinted. +** +** The design of the _RANGE hint is aid b-tree implementations that try +** to prefetch content from remote machines - to provide those +** implementations with limits on what needs to be prefetched and thereby +** reduce network bandwidth. +** +** Note that BTREE_HINT_FLAGS with BTREE_BULKLOAD is the only hint used by +** standard SQLite. The other hints are provided for extensions that use +** the SQLite parser and code generator but substitute their own storage +** engine. +*/ +#define BTREE_HINT_RANGE 0 /* Range constraints on queries */ + +/* +** Values that may be OR'd together to form the argument to the +** BTREE_HINT_FLAGS hint for sqlite3BtreeCursorHint(): +** +** The BTREE_BULKLOAD flag is set on index cursors when the index is going +** to be filled with content that is already in sorted order. +** +** The BTREE_SEEK_EQ flag is set on cursors that will get OP_SeekGE or +** OP_SeekLE opcodes for a range search, but where the range of entries +** selected will all have the same key. In other words, the cursor will +** be used only for equality key searches. +** +*/ +#define BTREE_BULKLOAD 0x00000001 /* Used to full index in sorted order */ +#define BTREE_SEEK_EQ 0x00000002 /* EQ seeks only - no range seeks */ + +/* +** Flags passed as the third argument to sqlite3BtreeCursor(). +** +** For read-only cursors the wrFlag argument is always zero. For read-write +** cursors it may be set to either (BTREE_WRCSR|BTREE_FORDELETE) or just +** (BTREE_WRCSR). If the BTREE_FORDELETE bit is set, then the cursor will +** only be used by SQLite for the following: +** +** * to seek to and then delete specific entries, and/or +** +** * to read values that will be used to create keys that other +** BTREE_FORDELETE cursors will seek to and delete. +** +** The BTREE_FORDELETE flag is an optimization hint. It is not used by +** by this, the native b-tree engine of SQLite, but it is available to +** alternative storage engines that might be substituted in place of this +** b-tree system. For alternative storage engines in which a delete of +** the main table row automatically deletes corresponding index rows, +** the FORDELETE flag hint allows those alternative storage engines to +** skip a lot of work. Namely: FORDELETE cursors may treat all SEEK +** and DELETE operations as no-ops, and any READ operation against a +** FORDELETE cursor may return a null row: 0x01 0x00. +*/ +#define BTREE_WRCSR 0x00000004 /* read-write cursor */ +#define BTREE_FORDELETE 0x00000008 /* Cursor is for seek/delete only */ + +SQLITE_PRIVATE int sqlite3BtreeCursor( + Btree*, /* BTree containing table to open */ + Pgno iTable, /* Index of root page */ + int wrFlag, /* 1 for writing. 0 for read-only */ + struct KeyInfo*, /* First argument to compare function */ + BtCursor *pCursor /* Space to write cursor structure */ +); +SQLITE_PRIVATE BtCursor *sqlite3BtreeFakeValidCursor(void); +SQLITE_PRIVATE int sqlite3BtreeCursorSize(void); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3BtreeClosesWithCursor(Btree*,BtCursor*); +#endif +SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorHintFlags(BtCursor*, unsigned); +#ifdef SQLITE_ENABLE_CURSOR_HINTS +SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor*, int, ...); +#endif + +SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeTableMoveto( + BtCursor*, + i64 intKey, + int bias, + int *pRes +); +SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( + BtCursor*, + UnpackedRecord *pUnKey, + int *pRes +); +SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor*, int*); +SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor*, u8 flags); + +/* Allowed flags for sqlite3BtreeDelete() and sqlite3BtreeInsert() */ +#define BTREE_SAVEPOSITION 0x02 /* Leave cursor pointing at NEXT or PREV */ +#define BTREE_AUXDELETE 0x04 /* not the primary delete operation */ +#define BTREE_APPEND 0x08 /* Insert is likely an append */ +#define BTREE_PREFORMAT 0x80 /* Inserted data is a preformated cell */ + +/* An instance of the BtreePayload object describes the content of a single +** entry in either an index or table btree. +** +** Index btrees (used for indexes and also WITHOUT ROWID tables) contain +** an arbitrary key and no data. These btrees have pKey,nKey set to the +** key and the pData,nData,nZero fields are uninitialized. The aMem,nMem +** fields give an array of Mem objects that are a decomposition of the key. +** The nMem field might be zero, indicating that no decomposition is available. +** +** Table btrees (used for rowid tables) contain an integer rowid used as +** the key and passed in the nKey field. The pKey field is zero. +** pData,nData hold the content of the new entry. nZero extra zero bytes +** are appended to the end of the content when constructing the entry. +** The aMem,nMem fields are uninitialized for table btrees. +** +** Field usage summary: +** +** Table BTrees Index Btrees +** +** pKey always NULL encoded key +** nKey the ROWID length of pKey +** pData data not used +** aMem not used decomposed key value +** nMem not used entries in aMem +** nData length of pData not used +** nZero extra zeros after pData not used +** +** This object is used to pass information into sqlite3BtreeInsert(). The +** same information used to be passed as five separate parameters. But placing +** the information into this object helps to keep the interface more +** organized and understandable, and it also helps the resulting code to +** run a little faster by using fewer registers for parameter passing. +*/ +struct BtreePayload { + const void *pKey; /* Key content for indexes. NULL for tables */ + sqlite3_int64 nKey; /* Size of pKey for indexes. PRIMARY KEY for tabs */ + const void *pData; /* Data for tables. */ + sqlite3_value *aMem; /* First of nMem value in the unpacked pKey */ + u16 nMem; /* Number of aMem[] value. Might be zero */ + int nData; /* Size of pData. 0 if none. */ + int nZero; /* Extra zero data appended after pData,nData */ +}; + +SQLITE_PRIVATE int sqlite3BtreeInsert(BtCursor*, const BtreePayload *pPayload, + int flags, int seekResult); +SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor*, int *pRes); +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor*, int flags); +SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor*, int flags); +SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor*); +SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor*, u32 *pAmt); +SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor*); +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor*); + +SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ + Btree *p, /* The btree to be checked */ + Pgno *aRoot, /* An array of root pages numbers for individual trees */ + sqlite3_value *aCnt, /* OUT: entry counts for each btree in aRoot[] */ + int nRoot, /* Number of entries in aRoot[] */ + int mxErr, /* Stop reporting errors after this many */ + int *pnErr, /* OUT: Write number of errors seen to this variable */ + char **pzOut /* OUT: Write the error message string here */ +); +SQLITE_PRIVATE struct Pager *sqlite3BtreePager(Btree*); +SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor*); + +#ifndef SQLITE_OMIT_INCRBLOB +SQLITE_PRIVATE int sqlite3BtreePayloadChecked(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor*, u32 offset, u32 amt, void*); +SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *); +#endif +SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *); +SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBt, int iVersion); +SQLITE_PRIVATE int sqlite3BtreeCursorHasHint(BtCursor*, unsigned int mask); +SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *pBt); +SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree*); +#else +# define sqlite3BtreeSeekCount(X) 0 +#endif + +#ifndef NDEBUG +SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor*); +#endif +SQLITE_PRIVATE int sqlite3BtreeCursorIsValidNN(BtCursor*); + +SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3*, BtCursor*, i64*); + +#ifdef SQLITE_TEST +SQLITE_PRIVATE int sqlite3BtreeCursorInfo(BtCursor*, int*, int); +SQLITE_PRIVATE void sqlite3BtreeCursorList(Btree*); +#endif + +#ifndef SQLITE_OMIT_WAL +SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree*, int, int *, int *); +#endif + +SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor*, BtCursor*, i64); + +SQLITE_PRIVATE void sqlite3BtreeClearCache(Btree*); + +/* +** If we are not using shared cache, then there is no need to +** use mutexes to access the BtShared structures. So make the +** Enter and Leave procedures no-ops. +*/ +#ifndef SQLITE_OMIT_SHARED_CACHE +SQLITE_PRIVATE void sqlite3BtreeEnter(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3*); +SQLITE_PRIVATE int sqlite3BtreeSharable(Btree*); +SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor*); +SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree*); +#else +# define sqlite3BtreeEnter(X) +# define sqlite3BtreeEnterAll(X) +# define sqlite3BtreeSharable(X) 0 +# define sqlite3BtreeEnterCursor(X) +# define sqlite3BtreeConnectionCount(X) 1 +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) && SQLITE_THREADSAFE +SQLITE_PRIVATE void sqlite3BtreeLeave(Btree*); +SQLITE_PRIVATE void sqlite3BtreeLeaveCursor(BtCursor*); +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3*); +#ifndef NDEBUG + /* These routines are used inside assert() statements only. */ +SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree*); +SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3*); +SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3*,int,Schema*); +#endif +#else + +# define sqlite3BtreeLeave(X) +# define sqlite3BtreeLeaveCursor(X) +# define sqlite3BtreeLeaveAll(X) + +# define sqlite3BtreeHoldsMutex(X) 1 +# define sqlite3BtreeHoldsAllMutexes(X) 1 +# define sqlite3SchemaMutexHeld(X,Y,Z) 1 +#endif + + +#endif /* SQLITE_BTREE_H */ + +/************** End of btree.h ***********************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ +/************** Include vdbe.h in the middle of sqliteInt.h ******************/ +/************** Begin file vdbe.h ********************************************/ +/* +** 2001 September 15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** Header file for the Virtual DataBase Engine (VDBE) +** +** This header defines the interface to the virtual database engine +** or VDBE. The VDBE implements an abstract machine that runs a +** simple program to access and modify the underlying database. +*/ +#ifndef SQLITE_VDBE_H +#define SQLITE_VDBE_H +/* #include */ + +/* +** A single VDBE is an opaque structure named "Vdbe". Only routines +** in the source file sqliteVdbe.c are allowed to see the insides +** of this structure. +*/ +typedef struct Vdbe Vdbe; + +/* +** The names of the following types declared in vdbeInt.h are required +** for the VdbeOp definition. +*/ +typedef struct sqlite3_value Mem; +typedef struct SubProgram SubProgram; +typedef struct SubrtnSig SubrtnSig; + +/* +** A signature for a reusable subroutine that materializes the RHS of +** an IN operator. +*/ +struct SubrtnSig { + int selId; /* SELECT-id for the SELECT statement on the RHS */ + char *zAff; /* Affinity of the overall IN expression */ + int iTable; /* Ephemeral table generated by the subroutine */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ +}; + +/* +** A single instruction of the virtual machine has an opcode +** and as many as three operands. The instruction is recorded +** as an instance of the following structure: +*/ +struct VdbeOp { + u8 opcode; /* What operation to perform */ + signed char p4type; /* One of the P4_xxx constants for p4 */ + u16 p5; /* Fifth parameter is an unsigned 16-bit integer */ + int p1; /* First operand */ + int p2; /* Second parameter (often the jump destination) */ + int p3; /* The third parameter */ + union p4union { /* fourth parameter */ + int i; /* Integer value if p4type==P4_INT32 */ + void *p; /* Generic pointer */ + char *z; /* Pointer to data for string (char array) types */ + i64 *pI64; /* Used when p4type is P4_INT64 */ + double *pReal; /* Used when p4type is P4_REAL */ + FuncDef *pFunc; /* Used when p4type is P4_FUNCDEF */ + sqlite3_context *pCtx; /* Used when p4type is P4_FUNCCTX */ + CollSeq *pColl; /* Used when p4type is P4_COLLSEQ */ + Mem *pMem; /* Used when p4type is P4_MEM */ + VTable *pVtab; /* Used when p4type is P4_VTAB */ + KeyInfo *pKeyInfo; /* Used when p4type is P4_KEYINFO */ + u32 *ai; /* Used when p4type is P4_INTARRAY */ + SubProgram *pProgram; /* Used when p4type is P4_SUBPROGRAM */ + Table *pTab; /* Used when p4type is P4_TABLE */ + SubrtnSig *pSubrtnSig; /* Used when p4type is P4_SUBRTNSIG */ +#ifdef SQLITE_ENABLE_CURSOR_HINTS + Expr *pExpr; /* Used when p4type is P4_EXPR */ +#endif + } p4; +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + char *zComment; /* Comment to improve readability */ +#endif +#ifdef SQLITE_VDBE_COVERAGE + u32 iSrcLine; /* Source-code line that generated this opcode + ** with flags in the upper 8 bits */ +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 nExec; + u64 nCycle; +#endif +}; +typedef struct VdbeOp VdbeOp; + + +/* +** A sub-routine used to implement a trigger program. +*/ +struct SubProgram { + VdbeOp *aOp; /* Array of opcodes for sub-program */ + int nOp; /* Elements in aOp[] */ + int nMem; /* Number of memory cells required */ + int nCsr; /* Number of cursors required */ + u8 *aOnce; /* Array of OP_Once flags */ + void *token; /* id that may be used to recursive triggers */ + SubProgram *pNext; /* Next sub-program already visited */ +}; + +/* +** A smaller version of VdbeOp used for the VdbeAddOpList() function because +** it takes up less space. +*/ +struct VdbeOpList { + u8 opcode; /* What operation to perform */ + signed char p1; /* First operand */ + signed char p2; /* Second parameter (often the jump destination) */ + signed char p3; /* Third parameter */ +}; +typedef struct VdbeOpList VdbeOpList; + +/* +** Allowed values of VdbeOp.p4type +*/ +#define P4_NOTUSED 0 /* The P4 parameter is not used */ +#define P4_TRANSIENT 0 /* P4 is a pointer to a transient string */ +#define P4_STATIC (-1) /* Pointer to a static string */ +#define P4_COLLSEQ (-2) /* P4 is a pointer to a CollSeq structure */ +#define P4_INT32 (-3) /* P4 is a 32-bit signed integer */ +#define P4_SUBPROGRAM (-4) /* P4 is a pointer to a SubProgram structure */ +#define P4_TABLE (-5) /* P4 is a pointer to a Table structure */ +/* Above do not own any resources. Must free those below */ +#define P4_FREE_IF_LE (-6) +#define P4_DYNAMIC (-6) /* Pointer to memory from sqliteMalloc() */ +#define P4_FUNCDEF (-7) /* P4 is a pointer to a FuncDef structure */ +#define P4_KEYINFO (-8) /* P4 is a pointer to a KeyInfo structure */ +#define P4_EXPR (-9) /* P4 is a pointer to an Expr tree */ +#define P4_MEM (-10) /* P4 is a pointer to a Mem* structure */ +#define P4_VTAB (-11) /* P4 is a pointer to an sqlite3_vtab structure */ +#define P4_REAL (-12) /* P4 is a 64-bit floating point value */ +#define P4_INT64 (-13) /* P4 is a 64-bit signed integer */ +#define P4_INTARRAY (-14) /* P4 is a vector of 32-bit integers */ +#define P4_FUNCCTX (-15) /* P4 is a pointer to an sqlite3_context object */ +#define P4_TABLEREF (-16) /* Like P4_TABLE, but reference counted */ +#define P4_SUBRTNSIG (-17) /* P4 is a SubrtnSig pointer */ + +/* Error message codes for OP_Halt */ +#define P5_ConstraintNotNull 1 +#define P5_ConstraintUnique 2 +#define P5_ConstraintCheck 3 +#define P5_ConstraintFK 4 + +/* +** The Vdbe.aColName array contains 5n Mem structures, where n is the +** number of columns of data returned by the statement. +*/ +#define COLNAME_NAME 0 +#define COLNAME_DECLTYPE 1 +#define COLNAME_DATABASE 2 +#define COLNAME_TABLE 3 +#define COLNAME_COLUMN 4 +#ifdef SQLITE_ENABLE_COLUMN_METADATA +# define COLNAME_N 5 /* Number of COLNAME_xxx symbols */ +#else +# ifdef SQLITE_OMIT_DECLTYPE +# define COLNAME_N 1 /* Store only the name */ +# else +# define COLNAME_N 2 /* Store the name and decltype */ +# endif +#endif + +/* +** The following macro converts a label returned by sqlite3VdbeMakeLabel() +** into an index into the Parse.aLabel[] array that contains the resolved +** address of that label. +*/ +#define ADDR(X) (~(X)) + +/* +** The makefile scans the vdbe.c source file and creates the "opcodes.h" +** header file that defines a number for each opcode used by the VDBE. +*/ +/************** Include opcodes.h in the middle of vdbe.h ********************/ +/************** Begin file opcodes.h *****************************************/ +/* Automatically generated. Do not edit */ +/* See the tool/mkopcodeh.tcl script for details */ +#define OP_Savepoint 0 +#define OP_AutoCommit 1 +#define OP_Transaction 2 +#define OP_Checkpoint 3 +#define OP_JournalMode 4 +#define OP_Vacuum 5 +#define OP_VFilter 6 /* jump, synopsis: iplan=r[P3] zplan='P4' */ +#define OP_VUpdate 7 /* synopsis: data=r[P3@P2] */ +#define OP_Init 8 /* jump0, synopsis: Start at P2 */ +#define OP_Goto 9 /* jump */ +#define OP_Gosub 10 /* jump */ +#define OP_InitCoroutine 11 /* jump0 */ +#define OP_Yield 12 /* jump0 */ +#define OP_MustBeInt 13 /* jump0 */ +#define OP_Jump 14 /* jump */ +#define OP_Once 15 /* jump */ +#define OP_If 16 /* jump */ +#define OP_IfNot 17 /* jump */ +#define OP_IsType 18 /* jump, synopsis: if typeof(P1.P3) in P5 goto P2 */ +#define OP_Not 19 /* same as TK_NOT, synopsis: r[P2]= !r[P1] */ +#define OP_IfNullRow 20 /* jump, synopsis: if P1.nullRow then r[P3]=NULL, goto P2 */ +#define OP_SeekLT 21 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekLE 22 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGE 23 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_SeekGT 24 /* jump0, synopsis: key=r[P3@P4] */ +#define OP_IfNotOpen 25 /* jump, synopsis: if( !csr[P1] ) goto P2 */ +#define OP_IfNoHope 26 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NoConflict 27 /* jump, synopsis: key=r[P3@P4] */ +#define OP_NotFound 28 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Found 29 /* jump, synopsis: key=r[P3@P4] */ +#define OP_SeekRowid 30 /* jump0, synopsis: intkey=r[P3] */ +#define OP_NotExists 31 /* jump, synopsis: intkey=r[P3] */ +#define OP_Last 32 /* jump0 */ +#define OP_IfSizeBetween 33 /* jump */ +#define OP_SorterSort 34 /* jump */ +#define OP_Sort 35 /* jump */ +#define OP_Rewind 36 /* jump0 */ +#define OP_SorterNext 37 /* jump */ +#define OP_Prev 38 /* jump */ +#define OP_Next 39 /* jump */ +#define OP_IdxLE 40 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxGT 41 /* jump, synopsis: key=r[P3@P4] */ +#define OP_IdxLT 42 /* jump, synopsis: key=r[P3@P4] */ +#define OP_Or 43 /* same as TK_OR, synopsis: r[P3]=(r[P1] || r[P2]) */ +#define OP_And 44 /* same as TK_AND, synopsis: r[P3]=(r[P1] && r[P2]) */ +#define OP_IdxGE 45 /* jump, synopsis: key=r[P3@P4] */ +#define OP_RowSetRead 46 /* jump, synopsis: r[P3]=rowset(P1) */ +#define OP_RowSetTest 47 /* jump, synopsis: if r[P3] in rowset(P1) goto P2 */ +#define OP_Program 48 /* jump0 */ +#define OP_FkIfZero 49 /* jump, synopsis: if fkctr[P1]==0 goto P2 */ +#define OP_IfPos 50 /* jump, synopsis: if r[P1]>0 then r[P1]-=P3, goto P2 */ +#define OP_IsNull 51 /* jump, same as TK_ISNULL, synopsis: if r[P1]==NULL goto P2 */ +#define OP_NotNull 52 /* jump, same as TK_NOTNULL, synopsis: if r[P1]!=NULL goto P2 */ +#define OP_Ne 53 /* jump, same as TK_NE, synopsis: IF r[P3]!=r[P1] */ +#define OP_Eq 54 /* jump, same as TK_EQ, synopsis: IF r[P3]==r[P1] */ +#define OP_Gt 55 /* jump, same as TK_GT, synopsis: IF r[P3]>r[P1] */ +#define OP_Le 56 /* jump, same as TK_LE, synopsis: IF r[P3]<=r[P1] */ +#define OP_Lt 57 /* jump, same as TK_LT, synopsis: IF r[P3]=r[P1] */ +#define OP_ElseEq 59 /* jump, same as TK_ESCAPE */ +#define OP_IfNotZero 60 /* jump, synopsis: if r[P1]!=0 then r[P1]--, goto P2 */ +#define OP_DecrJumpZero 61 /* jump, synopsis: if (--r[P1])==0 goto P2 */ +#define OP_IncrVacuum 62 /* jump */ +#define OP_VNext 63 /* jump */ +#define OP_Filter 64 /* jump, synopsis: if key(P3@P4) not in filter(P1) goto P2 */ +#define OP_PureFunc 65 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Function 66 /* synopsis: r[P3]=func(r[P2@NP]) */ +#define OP_Return 67 +#define OP_EndCoroutine 68 +#define OP_HaltIfNull 69 /* synopsis: if r[P3]=null halt */ +#define OP_Halt 70 +#define OP_Integer 71 /* synopsis: r[P2]=P1 */ +#define OP_Int64 72 /* synopsis: r[P2]=P4 */ +#define OP_String 73 /* synopsis: r[P2]='P4' (len=P1) */ +#define OP_BeginSubrtn 74 /* synopsis: r[P2]=NULL */ +#define OP_Null 75 /* synopsis: r[P2..P3]=NULL */ +#define OP_SoftNull 76 /* synopsis: r[P1]=NULL */ +#define OP_Blob 77 /* synopsis: r[P2]=P4 (len=P1) */ +#define OP_Variable 78 /* synopsis: r[P2]=parameter(P1) */ +#define OP_Move 79 /* synopsis: r[P2@P3]=r[P1@P3] */ +#define OP_Copy 80 /* synopsis: r[P2@P3+1]=r[P1@P3+1] */ +#define OP_SCopy 81 /* synopsis: r[P2]=r[P1] */ +#define OP_IntCopy 82 /* synopsis: r[P2]=r[P1] */ +#define OP_FkCheck 83 +#define OP_ResultRow 84 /* synopsis: output=r[P1@P2] */ +#define OP_CollSeq 85 +#define OP_AddImm 86 /* synopsis: r[P1]=r[P1]+P2 */ +#define OP_RealAffinity 87 +#define OP_Cast 88 /* synopsis: affinity(r[P1]) */ +#define OP_Permutation 89 +#define OP_Compare 90 /* synopsis: r[P1@P3] <-> r[P2@P3] */ +#define OP_IsTrue 91 /* synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 */ +#define OP_ZeroOrNull 92 /* synopsis: r[P2] = 0 OR NULL */ +#define OP_Offset 93 /* synopsis: r[P3] = sqlite_offset(P1) */ +#define OP_Column 94 /* synopsis: r[P3]=PX cursor P1 column P2 */ +#define OP_TypeCheck 95 /* synopsis: typecheck(r[P1@P2]) */ +#define OP_Affinity 96 /* synopsis: affinity(r[P1@P2]) */ +#define OP_MakeRecord 97 /* synopsis: r[P3]=mkrec(r[P1@P2]) */ +#define OP_Count 98 /* synopsis: r[P2]=count() */ +#define OP_ReadCookie 99 +#define OP_SetCookie 100 +#define OP_ReopenIdx 101 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenRead 102 /* synopsis: root=P2 iDb=P3 */ +#define OP_BitAnd 103 /* same as TK_BITAND, synopsis: r[P3]=r[P1]&r[P2] */ +#define OP_BitOr 104 /* same as TK_BITOR, synopsis: r[P3]=r[P1]|r[P2] */ +#define OP_ShiftLeft 105 /* same as TK_LSHIFT, synopsis: r[P3]=r[P2]<>r[P1] */ +#define OP_Add 107 /* same as TK_PLUS, synopsis: r[P3]=r[P1]+r[P2] */ +#define OP_Subtract 108 /* same as TK_MINUS, synopsis: r[P3]=r[P2]-r[P1] */ +#define OP_Multiply 109 /* same as TK_STAR, synopsis: r[P3]=r[P1]*r[P2] */ +#define OP_Divide 110 /* same as TK_SLASH, synopsis: r[P3]=r[P2]/r[P1] */ +#define OP_Remainder 111 /* same as TK_REM, synopsis: r[P3]=r[P2]%r[P1] */ +#define OP_Concat 112 /* same as TK_CONCAT, synopsis: r[P3]=r[P2]+r[P1] */ +#define OP_OpenWrite 113 /* synopsis: root=P2 iDb=P3 */ +#define OP_OpenDup 114 +#define OP_BitNot 115 /* same as TK_BITNOT, synopsis: r[P2]= ~r[P1] */ +#define OP_OpenAutoindex 116 /* synopsis: nColumn=P2 */ +#define OP_OpenEphemeral 117 /* synopsis: nColumn=P2 */ +#define OP_String8 118 /* same as TK_STRING, synopsis: r[P2]='P4' */ +#define OP_SorterOpen 119 +#define OP_SequenceTest 120 /* synopsis: if( cursor[P1].ctr++ ) pc = P2 */ +#define OP_OpenPseudo 121 /* synopsis: P3 columns in r[P2] */ +#define OP_Close 122 +#define OP_ColumnsUsed 123 +#define OP_SeekScan 124 /* synopsis: Scan-ahead up to P1 rows */ +#define OP_SeekHit 125 /* synopsis: set P2<=seekHit<=P3 */ +#define OP_Sequence 126 /* synopsis: r[P2]=cursor[P1].ctr++ */ +#define OP_NewRowid 127 /* synopsis: r[P2]=rowid */ +#define OP_Insert 128 /* synopsis: intkey=r[P3] data=r[P2] */ +#define OP_RowCell 129 +#define OP_Delete 130 +#define OP_ResetCount 131 +#define OP_SorterCompare 132 /* synopsis: if key(P1)!=trim(r[P3],P4) goto P2 */ +#define OP_SorterData 133 /* synopsis: r[P2]=data */ +#define OP_RowData 134 /* synopsis: r[P2]=data */ +#define OP_Rowid 135 /* synopsis: r[P2]=PX rowid of P1 */ +#define OP_NullRow 136 +#define OP_SeekEnd 137 +#define OP_IdxInsert 138 /* synopsis: key=r[P2] */ +#define OP_SorterInsert 139 /* synopsis: key=r[P2] */ +#define OP_IdxDelete 140 /* synopsis: key=r[P2@P3] */ +#define OP_DeferredSeek 141 /* synopsis: Move P3 to P1.rowid if needed */ +#define OP_IdxRowid 142 /* synopsis: r[P2]=rowid */ +#define OP_FinishSeek 143 +#define OP_Destroy 144 +#define OP_Clear 145 +#define OP_ResetSorter 146 +#define OP_CreateBtree 147 /* synopsis: r[P2]=root iDb=P1 flags=P3 */ +#define OP_SqlExec 148 +#define OP_ParseSchema 149 +#define OP_LoadAnalysis 150 +#define OP_DropTable 151 +#define OP_DropIndex 152 +#define OP_DropTrigger 153 +#define OP_Real 154 /* same as TK_FLOAT, synopsis: r[P2]=P4 */ +#define OP_IntegrityCk 155 +#define OP_RowSetAdd 156 /* synopsis: rowset(P1)=r[P2] */ +#define OP_Param 157 +#define OP_FkCounter 158 /* synopsis: fkctr[P1]+=P2 */ +#define OP_MemMax 159 /* synopsis: r[P1]=max(r[P1],r[P2]) */ +#define OP_OffsetLimit 160 /* synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) */ +#define OP_AggInverse 161 /* synopsis: accum=r[P3] inverse(r[P2@P5]) */ +#define OP_AggStep 162 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggStep1 163 /* synopsis: accum=r[P3] step(r[P2@P5]) */ +#define OP_AggValue 164 /* synopsis: r[P3]=value N=P2 */ +#define OP_AggFinal 165 /* synopsis: accum=r[P1] N=P2 */ +#define OP_Expire 166 +#define OP_CursorLock 167 +#define OP_CursorUnlock 168 +#define OP_TableLock 169 /* synopsis: iDb=P1 root=P2 write=P3 */ +#define OP_VBegin 170 +#define OP_VCreate 171 +#define OP_VDestroy 172 +#define OP_VOpen 173 +#define OP_VCheck 174 +#define OP_VInitIn 175 /* synopsis: r[P2]=ValueList(P1,P3) */ +#define OP_VColumn 176 /* synopsis: r[P3]=vcolumn(P2) */ +#define OP_VRename 177 +#define OP_Pagecount 178 +#define OP_MaxPgcnt 179 +#define OP_ClrSubtype 180 /* synopsis: r[P1].subtype = 0 */ +#define OP_GetSubtype 181 /* synopsis: r[P2] = r[P1].subtype */ +#define OP_SetSubtype 182 /* synopsis: r[P2].subtype = r[P1] */ +#define OP_FilterAdd 183 /* synopsis: filter(P1) += key(P3@P4) */ +#define OP_Trace 184 +#define OP_CursorHint 185 +#define OP_ReleaseReg 186 /* synopsis: release r[P1@P2] mask P3 */ +#define OP_Noop 187 +#define OP_Explain 188 +#define OP_Abortable 189 + +/* Properties such as "out2" or "jump" that are specified in +** comments following the "case" for each opcode in the vdbe.c +** are encoded into bitvectors as follows: +*/ +#define OPFLG_JUMP 0x01 /* jump: P2 holds jmp target */ +#define OPFLG_IN1 0x02 /* in1: P1 is an input */ +#define OPFLG_IN2 0x04 /* in2: P2 is an input */ +#define OPFLG_IN3 0x08 /* in3: P3 is an input */ +#define OPFLG_OUT2 0x10 /* out2: P2 is an output */ +#define OPFLG_OUT3 0x20 /* out3: P3 is an output */ +#define OPFLG_NCYCLE 0x40 /* ncycle:Cycles count against P1 */ +#define OPFLG_JUMP0 0x80 /* jump0: P2 might be zero */ +#define OPFLG_INITIALIZER {\ +/* 0 */ 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x41, 0x00,\ +/* 8 */ 0x81, 0x01, 0x01, 0x81, 0x83, 0x83, 0x01, 0x01,\ +/* 16 */ 0x03, 0x03, 0x01, 0x12, 0x01, 0xc9, 0xc9, 0xc9,\ +/* 24 */ 0xc9, 0x01, 0x49, 0x49, 0x49, 0x49, 0xc9, 0x49,\ +/* 32 */ 0xc1, 0x01, 0x41, 0x41, 0xc1, 0x01, 0x41, 0x41,\ +/* 40 */ 0x41, 0x41, 0x41, 0x26, 0x26, 0x41, 0x23, 0x0b,\ +/* 48 */ 0x81, 0x01, 0x03, 0x03, 0x03, 0x0b, 0x0b, 0x0b,\ +/* 56 */ 0x0b, 0x0b, 0x0b, 0x01, 0x03, 0x03, 0x01, 0x41,\ +/* 64 */ 0x01, 0x00, 0x00, 0x02, 0x02, 0x08, 0x00, 0x10,\ +/* 72 */ 0x10, 0x10, 0x00, 0x10, 0x00, 0x10, 0x10, 0x00,\ +/* 80 */ 0x00, 0x10, 0x10, 0x00, 0x00, 0x00, 0x02, 0x02,\ +/* 88 */ 0x02, 0x00, 0x00, 0x12, 0x1e, 0x20, 0x40, 0x00,\ +/* 96 */ 0x00, 0x00, 0x10, 0x10, 0x00, 0x40, 0x40, 0x26,\ +/* 104 */ 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26, 0x26,\ +/* 112 */ 0x26, 0x00, 0x40, 0x12, 0x40, 0x40, 0x10, 0x00,\ +/* 120 */ 0x00, 0x00, 0x40, 0x00, 0x40, 0x40, 0x10, 0x10,\ +/* 128 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x50,\ +/* 136 */ 0x00, 0x40, 0x04, 0x04, 0x00, 0x40, 0x50, 0x40,\ +/* 144 */ 0x10, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00,\ +/* 152 */ 0x00, 0x00, 0x10, 0x00, 0x06, 0x10, 0x00, 0x04,\ +/* 160 */ 0x1a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,\ +/* 168 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x10, 0x50,\ +/* 176 */ 0x40, 0x00, 0x10, 0x10, 0x02, 0x12, 0x12, 0x00,\ +/* 184 */ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,} + +/* The resolve3P2Values() routine is able to run faster if it knows +** the value of the largest JUMP opcode. The smaller the maximum +** JUMP opcode the better, so the mkopcodeh.tcl script that +** generated this include file strives to group all JUMP opcodes +** together near the beginning of the list. +*/ +#define SQLITE_MX_JUMP_OPCODE 64 /* Maximum JUMP opcode */ + +/************** End of opcodes.h *********************************************/ +/************** Continuing where we left off in vdbe.h ***********************/ + +/* +** Additional non-public SQLITE_PREPARE_* flags +*/ +#define SQLITE_PREPARE_SAVESQL 0x80 /* Preserve SQL text */ +#define SQLITE_PREPARE_MASK 0x0f /* Mask of public flags */ + +/* +** Prototypes for the VDBE interface. See comments on the implementation +** for a description of what each of these routines does. +*/ +SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse*); +SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe*,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe*,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeGoto(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe*,int,const char*,...); +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe*,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4(Vdbe*,int,int,int,int,const char *zP4,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8(Vdbe*,int,int,int,int,const u8*,int); +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int(Vdbe*,int,int,int,int,int); +SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall(Parse*,int,int,int,int,const FuncDef*,int); +SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe*,int); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N); +SQLITE_PRIVATE void sqlite3VdbeVerifyNoResultRow(Vdbe *p); +#else +# define sqlite3VdbeVerifyNoMallocRequired(A,B) +# define sqlite3VdbeVerifyNoResultRow(A) +#endif +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3VdbeVerifyAbortable(Vdbe *p, int); +SQLITE_PRIVATE void sqlite3VdbeNoJumpsOutsideSubrtn(Vdbe*,int,int,int); +#else +# define sqlite3VdbeVerifyAbortable(A,B) +# define sqlite3VdbeNoJumpsOutsideSubrtn(A,B,C,D) +#endif +SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList(Vdbe*, int nOp, VdbeOpList const *aOp,int iLineno); +#ifndef SQLITE_OMIT_EXPLAIN +SQLITE_PRIVATE int sqlite3VdbeExplain(Parse*,u8,const char*,...); +SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse*); +SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse*); +# define ExplainQueryPlan(P) sqlite3VdbeExplain P +# ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define ExplainQueryPlan2(V,P) (V = sqlite3VdbeExplain P) +# else +# define ExplainQueryPlan2(V,P) ExplainQueryPlan(P) +# endif +# define ExplainQueryPlanPop(P) sqlite3VdbeExplainPop(P) +# define ExplainQueryPlanParent(P) sqlite3VdbeExplainParent(P) +#else +# define ExplainQueryPlan(P) +# define ExplainQueryPlan2(V,P) +# define ExplainQueryPlanPop(P) +# define ExplainQueryPlanParent(P) 0 +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ +#endif +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_EXPLAIN) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char*,const char*); +#else +# define sqlite3ExplainBreakpoint(A,B) /*no-op*/ +#endif +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe*, int, char*, u16); +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe*, int addr, u8); +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe*, int addr, int P1); +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe*, int addr, int P2); +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe*, int addr, int P3); +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe*, u16 P5); +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe*, int); +SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe*, int addr); +SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe*, int addr); +SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe*, u8 op); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters(Parse*,int addr, int n, u32 mask, int); +#else +# define sqlite3VdbeReleaseRegisters(P,A,N,M,F) +#endif +SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe*, int addr, const char *zP4, int N); +SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe*, void *pP4, int p4type); +SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse*, Index*); +SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe*, int); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse*); +SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeMakeReady(Vdbe*,Parse*); +SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe*, int); +SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *, int); +#endif +SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe*); +SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe*,int); +SQLITE_PRIVATE int sqlite3VdbeSetColName(Vdbe*, int, int, const char *, void(*)(void*)); +SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe*); +SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe*); +SQLITE_PRIVATE u8 sqlite3VdbePrepareFlags(Vdbe*); +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe*, const char *z, int n, u8); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3*,Vdbe*,const char*); +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString(Vdbe*,const char*); +#endif +SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe*,Vdbe*); +SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe*, int*, int*); +SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe*, int, u8); +SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe*, int); +#ifndef SQLITE_OMIT_TRACE +SQLITE_PRIVATE char *sqlite3VdbeExpandSql(Vdbe*, const char*); +#endif +SQLITE_PRIVATE int sqlite3MemCompare(const Mem*, const Mem*, const CollSeq*); +SQLITE_PRIVATE int sqlite3BlobCompare(const Mem*, const Mem*); + +SQLITE_PRIVATE void sqlite3VdbeRecordUnpack(KeyInfo*,int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompare(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip(int, const void *, UnpackedRecord *, int); +SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord(KeyInfo*); + +typedef int (*RecordCompare)(int,const void*,UnpackedRecord*); +SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord*); + +SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *, SubProgram *); +SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe*); + +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val); + +SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context*); +#ifdef SQLITE_ENABLE_BYTECODE_VTAB +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3*); +#endif + +/* Use SQLITE_ENABLE_COMMENTS to enable generation of extra comments on +** each VDBE opcode. +** +** Use the SQLITE_ENABLE_MODULE_COMMENTS macro to see some extra no-op +** comments in VDBE programs that show key decision points in the code +** generator. +*/ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +SQLITE_PRIVATE void sqlite3VdbeComment(Vdbe*, const char*, ...); +# define VdbeComment(X) sqlite3VdbeComment X +SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe*, const char*, ...); +# define VdbeNoopComment(X) sqlite3VdbeNoopComment X +# ifdef SQLITE_ENABLE_MODULE_COMMENTS +# define VdbeModuleComment(X) sqlite3VdbeNoopComment X +# else +# define VdbeModuleComment(X) +# endif +#else +# define VdbeComment(X) +# define VdbeNoopComment(X) +# define VdbeModuleComment(X) +#endif + +/* +** The VdbeCoverage macros are used to set a coverage testing point +** for VDBE branch instructions. The coverage testing points are line +** numbers in the sqlite3.c source file. VDBE branch coverage testing +** only works with an amalgamation build. That's ok since a VDBE branch +** coverage build designed for testing the test suite only. No application +** should ever ship with VDBE branch coverage measuring turned on. +** +** VdbeCoverage(v) // Mark the previously coded instruction +** // as a branch +** +** VdbeCoverageIf(v, conditional) // Mark previous if conditional true +** +** VdbeCoverageAlwaysTaken(v) // Previous branch is always taken +** +** VdbeCoverageNeverTaken(v) // Previous branch is never taken +** +** VdbeCoverageNeverNull(v) // Previous three-way branch is only +** // taken on the first two ways. The +** // NULL option is not possible +** +** VdbeCoverageEqNe(v) // Previous OP_Jump is only interested +** // in distinguishing equal and not-equal. +** +** Every VDBE branch operation must be tagged with one of the macros above. +** If not, then when "make test" is run with -DSQLITE_VDBE_COVERAGE and +** -DSQLITE_DEBUG then an ALWAYS() will fail in the vdbeTakeBranch() +** routine in vdbe.c, alerting the developer to the missed tag. +** +** During testing, the test application will invoke +** sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE,...) to set a callback +** routine that is invoked as each bytecode branch is taken. The callback +** contains the sqlite3.c source line number of the VdbeCoverage macro and +** flags to indicate whether or not the branch was taken. The test application +** is responsible for keeping track of this and reporting byte-code branches +** that are never taken. +** +** See the VdbeBranchTaken() macro and vdbeTakeBranch() function in the +** vdbe.c source file for additional information. +*/ +#ifdef SQLITE_VDBE_COVERAGE +SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe*,int); +# define VdbeCoverage(v) sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageIf(v,x) if(x)sqlite3VdbeSetLineNumber(v,__LINE__) +# define VdbeCoverageAlwaysTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x5000000); +# define VdbeCoverageNeverTaken(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x6000000); +# define VdbeCoverageNeverNull(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageNeverNullIf(v,x) \ + if(x)sqlite3VdbeSetLineNumber(v,__LINE__|0x4000000); +# define VdbeCoverageEqNe(v) \ + sqlite3VdbeSetLineNumber(v,__LINE__|0x8000000); +# define VDBE_OFFSET_LINENO(x) (__LINE__+x) +#else +# define VdbeCoverage(v) +# define VdbeCoverageIf(v,x) +# define VdbeCoverageAlwaysTaken(v) +# define VdbeCoverageNeverTaken(v) +# define VdbeCoverageNeverNull(v) +# define VdbeCoverageNeverNullIf(v,x) +# define VdbeCoverageEqNe(v) +# define VDBE_OFFSET_LINENO(x) 0 +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +SQLITE_PRIVATE void sqlite3VdbeScanStatus(Vdbe*, int, int, int, LogEst, const char*); +SQLITE_PRIVATE void sqlite3VdbeScanStatusRange(Vdbe*, int, int, int); +SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters(Vdbe*, int, int, int); +#else +# define sqlite3VdbeScanStatus(a,b,c,d,e,f) +# define sqlite3VdbeScanStatusRange(a,b,c,d) +# define sqlite3VdbeScanStatusCounters(a,b,c,d) +#endif + +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, VdbeOp*); +#endif + +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr); +#endif + +#endif /* SQLITE_VDBE_H */ + +/************** End of vdbe.h ************************************************/ +/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include pcache.h in the middle of sqliteInt.h ****************/ /************** Begin file pcache.h ******************************************/ /* @@ -12940,7 +17294,7 @@ SQLITE_PRIVATE void sqlite3PagerRefdump(Pager*); ** ************************************************************************* ** This header file defines the interface that the sqlite page cache -** subsystem. +** subsystem. */ #ifndef _PCACHE_H_ @@ -12956,6 +17310,7 @@ struct PgHdr { sqlite3_pcache_page *pPage; /* Pcache object page handle */ void *pData; /* Page data */ void *pExtra; /* Extra content */ + PCache *pCache; /* PRIVATE: Cache that owns this page */ PgHdr *pDirty; /* Transient list of dirty sorted by pgno */ Pager *pPager; /* The pager this page is part of */ Pgno pgno; /* Page number for this page */ @@ -12965,14 +17320,15 @@ struct PgHdr { u16 flags; /* PGHDR flags defined below */ /********************************************************************** - ** Elements above are public. All that follows is private to pcache.c - ** and should not be accessed by other modules. + ** Elements above, except pCache, are public. All that follow are + ** private to pcache.c and should not be accessed by other modules. + ** pCache is grouped with the public elements for efficiency. */ - i16 nRef; /* Number of users of this page */ - PCache *pCache; /* Cache that owns this page */ - + i64 nRef; /* Number of users of this page */ PgHdr *pDirtyNext; /* Next element in list of dirty pages */ PgHdr *pDirtyPrev; /* Previous element in list of dirty pages */ + /* NB: pDirtyNext and pDirtyPrev are undefined if the + ** PgHdr object is not dirty */ }; /* Bit values for PgHdr.flags */ @@ -13017,7 +17373,7 @@ SQLITE_PRIVATE int sqlite3PcacheSetPageSize(PCache *, int); SQLITE_PRIVATE int sqlite3PcacheSize(void); /* One release per successful fetch. Page is pinned until released. -** Reference counted. +** Reference counted. */ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch(PCache*, Pgno, int createFlag); SQLITE_PRIVATE int sqlite3PcacheFetchStress(PCache*, Pgno, sqlite3_pcache_page**); @@ -13049,19 +17405,19 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *); SQLITE_PRIVATE void sqlite3PcacheClear(PCache*); /* Return the total number of outstanding page references */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache*); +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache*); /* Increment the reference count of an existing page */ SQLITE_PRIVATE void sqlite3PcacheRef(PgHdr*); -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr*); +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr*); /* Return the total number of pages stored in the cache */ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache*); #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* Iterate through all dirty pages currently stored in the cache. This -** interface is only available if SQLITE_CHECK_PAGES is defined when the +** interface is only available if SQLITE_CHECK_PAGES is defined when the ** library is built. */ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHdr *)); @@ -13111,286 +17467,14 @@ SQLITE_PRIVATE int sqlite3HeaderSizePcache1(void); /* Number of dirty pages as a percentage of the configured cache size */ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache*); +#ifdef SQLITE_DIRECT_OVERFLOW_READ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache); +#endif + #endif /* _PCACHE_H_ */ /************** End of pcache.h **********************************************/ /************** Continuing where we left off in sqliteInt.h ******************/ -/************** Include os.h in the middle of sqliteInt.h ********************/ -/************** Begin file os.h **********************************************/ -/* -** 2001 September 16 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This header file (together with is companion C source-code file -** "os.c") attempt to abstract the underlying operating system so that -** the SQLite library will work on both POSIX and windows systems. -** -** This header file is #include-ed by sqliteInt.h and thus ends up -** being included by every source file. -*/ -#ifndef _SQLITE_OS_H_ -#define _SQLITE_OS_H_ - -/* -** Attempt to automatically detect the operating system and setup the -** necessary pre-processor macros for it. -*/ -/************** Include os_setup.h in the middle of os.h *********************/ -/************** Begin file os_setup.h ****************************************/ -/* -** 2013 November 25 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains pre-processor directives related to operating system -** detection and/or setup. -*/ -#ifndef _OS_SETUP_H_ -#define _OS_SETUP_H_ - -/* -** Figure out if we are dealing with Unix, Windows, or some other operating -** system. -** -** After the following block of preprocess macros, all of SQLITE_OS_UNIX, -** SQLITE_OS_WIN, and SQLITE_OS_OTHER will defined to either 1 or 0. One of -** the three will be 1. The other two will be 0. -*/ -#if defined(SQLITE_OS_OTHER) -# if SQLITE_OS_OTHER==1 -# undef SQLITE_OS_UNIX -# define SQLITE_OS_UNIX 0 -# undef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# else -# undef SQLITE_OS_OTHER -# endif -#endif -#if !defined(SQLITE_OS_UNIX) && !defined(SQLITE_OS_OTHER) -# define SQLITE_OS_OTHER 0 -# ifndef SQLITE_OS_WIN -# if defined(_WIN32) || defined(WIN32) || defined(__CYGWIN__) || \ - defined(__MINGW32__) || defined(__BORLANDC__) -# define SQLITE_OS_WIN 1 -# define SQLITE_OS_UNIX 0 -# else -# define SQLITE_OS_WIN 0 -# define SQLITE_OS_UNIX 1 -# endif -# else -# define SQLITE_OS_UNIX 0 -# endif -#else -# ifndef SQLITE_OS_WIN -# define SQLITE_OS_WIN 0 -# endif -#endif - -#endif /* _OS_SETUP_H_ */ - -/************** End of os_setup.h ********************************************/ -/************** Continuing where we left off in os.h *************************/ - -/* If the SET_FULLSYNC macro is not defined above, then make it -** a no-op -*/ -#ifndef SET_FULLSYNC -# define SET_FULLSYNC(x,y) -#endif - -/* -** The default size of a disk sector -*/ -#ifndef SQLITE_DEFAULT_SECTOR_SIZE -# define SQLITE_DEFAULT_SECTOR_SIZE 4096 -#endif - -/* -** Temporary files are named starting with this prefix followed by 16 random -** alphanumeric characters, and no file extension. They are stored in the -** OS's standard temporary file directory, and are deleted prior to exit. -** If sqlite is being embedded in another program, you may wish to change the -** prefix to reflect your program's name, so that if your program exits -** prematurely, old temporary files can be easily identified. This can be done -** using -DSQLITE_TEMP_FILE_PREFIX=myprefix_ on the compiler command line. -** -** 2006-10-31: The default prefix used to be "sqlite_". But then -** Mcafee started using SQLite in their anti-virus product and it -** started putting files with the "sqlite" name in the c:/temp folder. -** This annoyed many windows users. Those users would then do a -** Google search for "sqlite", find the telephone numbers of the -** developers and call to wake them up at night and complain. -** For this reason, the default name prefix is changed to be "sqlite" -** spelled backwards. So the temp files are still identified, but -** anybody smart enough to figure out the code is also likely smart -** enough to know that calling the developer will not help get rid -** of the file. -*/ -#ifndef SQLITE_TEMP_FILE_PREFIX -# define SQLITE_TEMP_FILE_PREFIX "etilqs_" -#endif - -/* -** The following values may be passed as the second argument to -** sqlite3OsLock(). The various locks exhibit the following semantics: -** -** SHARED: Any number of processes may hold a SHARED lock simultaneously. -** RESERVED: A single process may hold a RESERVED lock on a file at -** any time. Other processes may hold and obtain new SHARED locks. -** PENDING: A single process may hold a PENDING lock on a file at -** any one time. Existing SHARED locks may persist, but no new -** SHARED locks may be obtained by other processes. -** EXCLUSIVE: An EXCLUSIVE lock precludes all other locks. -** -** PENDING_LOCK may not be passed directly to sqlite3OsLock(). Instead, a -** process that requests an EXCLUSIVE lock may actually obtain a PENDING -** lock. This can be upgraded to an EXCLUSIVE lock by a subsequent call to -** sqlite3OsLock(). -*/ -#define NO_LOCK 0 -#define SHARED_LOCK 1 -#define RESERVED_LOCK 2 -#define PENDING_LOCK 3 -#define EXCLUSIVE_LOCK 4 - -/* -** File Locking Notes: (Mostly about windows but also some info for Unix) -** -** We cannot use LockFileEx() or UnlockFileEx() on Win95/98/ME because -** those functions are not available. So we use only LockFile() and -** UnlockFile(). -** -** LockFile() prevents not just writing but also reading by other processes. -** A SHARED_LOCK is obtained by locking a single randomly-chosen -** byte out of a specific range of bytes. The lock byte is obtained at -** random so two separate readers can probably access the file at the -** same time, unless they are unlucky and choose the same lock byte. -** An EXCLUSIVE_LOCK is obtained by locking all bytes in the range. -** There can only be one writer. A RESERVED_LOCK is obtained by locking -** a single byte of the file that is designated as the reserved lock byte. -** A PENDING_LOCK is obtained by locking a designated byte different from -** the RESERVED_LOCK byte. -** -** On WinNT/2K/XP systems, LockFileEx() and UnlockFileEx() are available, -** which means we can use reader/writer locks. When reader/writer locks -** are used, the lock is placed on the same range of bytes that is used -** for probabilistic locking in Win95/98/ME. Hence, the locking scheme -** will support two or more Win95 readers or two or more WinNT readers. -** But a single Win95 reader will lock out all WinNT readers and a single -** WinNT reader will lock out all other Win95 readers. -** -** The following #defines specify the range of bytes used for locking. -** SHARED_SIZE is the number of bytes available in the pool from which -** a random byte is selected for a shared lock. The pool of bytes for -** shared locks begins at SHARED_FIRST. -** -** The same locking strategy and -** byte ranges are used for Unix. This leaves open the possibility of having -** clients on win95, winNT, and unix all talking to the same shared file -** and all locking correctly. To do so would require that samba (or whatever -** tool is being used for file sharing) implements locks correctly between -** windows and unix. I'm guessing that isn't likely to happen, but by -** using the same locking range we are at least open to the possibility. -** -** Locking in windows is manditory. For this reason, we cannot store -** actual data in the bytes used for locking. The pager never allocates -** the pages involved in locking therefore. SHARED_SIZE is selected so -** that all locks will fit on a single page even at the minimum page size. -** PENDING_BYTE defines the beginning of the locks. By default PENDING_BYTE -** is set high so that we don't have to allocate an unused page except -** for very large databases. But one should test the page skipping logic -** by setting PENDING_BYTE low and running the entire regression suite. -** -** Changing the value of PENDING_BYTE results in a subtly incompatible -** file format. Depending on how it is changed, you might not notice -** the incompatibility right away, even running a full regression test. -** The default location of PENDING_BYTE is the first byte past the -** 1GB boundary. -** -*/ -#ifdef SQLITE_OMIT_WSD -# define PENDING_BYTE (0x40000000) -#else -# define PENDING_BYTE sqlite3PendingByte -#endif -#define RESERVED_BYTE (PENDING_BYTE+1) -#define SHARED_FIRST (PENDING_BYTE+2) -#define SHARED_SIZE 510 - -/* -** Wrapper around OS specific sqlite3_os_init() function. -*/ -SQLITE_PRIVATE int sqlite3OsInit(void); - -/* -** Functions for accessing sqlite3_file methods -*/ -SQLITE_PRIVATE void sqlite3OsClose(sqlite3_file*); -SQLITE_PRIVATE int sqlite3OsRead(sqlite3_file*, void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsWrite(sqlite3_file*, const void*, int amt, i64 offset); -SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file*, i64 size); -SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file*, i64 *pSize); -SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file*, int); -SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file*,int,void*); -SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file*,int,void*); -#define SQLITE_FCNTL_DB_UNCHANGED 0xca093fa0 -SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmMap(sqlite3_file *,int,int,int,void volatile **); -SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int, int, int); -SQLITE_PRIVATE void sqlite3OsShmBarrier(sqlite3_file *id); -SQLITE_PRIVATE int sqlite3OsShmUnmap(sqlite3_file *id, int); -SQLITE_PRIVATE int sqlite3OsFetch(sqlite3_file *id, i64, int, void **); -SQLITE_PRIVATE int sqlite3OsUnfetch(sqlite3_file *, i64, void *); - - -/* -** Functions for accessing sqlite3_vfs methods -*/ -SQLITE_PRIVATE int sqlite3OsOpen(sqlite3_vfs *, const char *, sqlite3_file*, int, int *); -SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *, const char *, int); -SQLITE_PRIVATE int sqlite3OsAccess(sqlite3_vfs *, const char *, int, int *pResOut); -SQLITE_PRIVATE int sqlite3OsFullPathname(sqlite3_vfs *, const char *, int, char *); -#ifndef SQLITE_OMIT_LOAD_EXTENSION -SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *, const char *); -SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE void (*sqlite3OsDlSym(sqlite3_vfs *, void *, const char *))(void); -SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *, void *); -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ -SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *, int, char *); -SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *, int); -SQLITE_PRIVATE int sqlite3OsGetLastError(sqlite3_vfs*); -SQLITE_PRIVATE int sqlite3OsCurrentTimeInt64(sqlite3_vfs *, sqlite3_int64*); - -/* -** Convenience functions for opening and closing files using -** sqlite3_malloc() to obtain space for the file-handle structure. -*/ -SQLITE_PRIVATE int sqlite3OsOpenMalloc(sqlite3_vfs *, const char *, sqlite3_file **, int,int*); -SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); - -#endif /* _SQLITE_OS_H_ */ - -/************** End of os.h **************************************************/ -/************** Continuing where we left off in sqliteInt.h ******************/ /************** Include mutex.h in the middle of sqliteInt.h *****************/ /************** Begin file mutex.h *******************************************/ /* @@ -13451,9 +17535,9 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); */ #define sqlite3_mutex_alloc(X) ((sqlite3_mutex*)8) #define sqlite3_mutex_free(X) -#define sqlite3_mutex_enter(X) +#define sqlite3_mutex_enter(X) #define sqlite3_mutex_try(X) SQLITE_OK -#define sqlite3_mutex_leave(X) +#define sqlite3_mutex_leave(X) #define sqlite3_mutex_held(X) ((void)(X),1) #define sqlite3_mutex_notheld(X) ((void)(X),1) #define sqlite3MutexAlloc(X) ((sqlite3_mutex*)8) @@ -13462,6 +17546,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); #define MUTEX_LOGIC(X) #else #define MUTEX_LOGIC(X) X +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); #endif /* defined(SQLITE_MUTEX_OMIT) */ /************** End of mutex.h ***********************************************/ @@ -13478,7 +17563,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); /* ** Default synchronous levels. ** -** Note that (for historcal reasons) the PAGER_SYNCHRONOUS_* macros differ +** Note that (for historical reasons) the PAGER_SYNCHRONOUS_* macros differ ** from the SQLITE_DEFAULT_SYNCHRONOUS value by 1. ** ** PAGER_SYNCHRONOUS DEFAULT_SYNCHRONOUS @@ -13492,7 +17577,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); ** and the one-based values are used internally. */ #ifndef SQLITE_DEFAULT_SYNCHRONOUS -# define SQLITE_DEFAULT_SYNCHRONOUS (PAGER_SYNCHRONOUS_FULL-1) +# define SQLITE_DEFAULT_SYNCHRONOUS 2 #endif #ifndef SQLITE_DEFAULT_WAL_SYNCHRONOUS # define SQLITE_DEFAULT_WAL_SYNCHRONOUS SQLITE_DEFAULT_SYNCHRONOUS @@ -13506,7 +17591,7 @@ SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *); ** databases may be attached. */ struct Db { - char *zName; /* Name of this database */ + char *zDbSName; /* Name of this database. (schema name, not filename) */ Btree *pBt; /* The B*Tree structure for this database file */ u8 safety_level; /* How aggressive at syncing data to disk */ u8 bSyncSet; /* True if "PRAGMA synchronous=N" has been run */ @@ -13517,7 +17602,7 @@ struct Db { ** An instance of the following structure stores a database schema. ** ** Most Schema objects are associated with a Btree. The exception is -** the Schema for the TEMP databaes (sqlite3.aDb[1]) which is free-standing. +** the Schema for the TEMP database (sqlite3.aDb[1]) which is free-standing. ** In shared cache mode, a single Schema object can be shared by multiple ** Btrees that refer to the same underlying BtShared object. ** @@ -13565,7 +17650,7 @@ struct Schema { */ #define DB_SchemaLoaded 0x0001 /* The schema has been loaded */ #define DB_UnresetViews 0x0002 /* Some views have defined column names */ -#define DB_Empty 0x0004 /* The file is empty (length 0 bytes) */ +#define DB_ResetWanted 0x0008 /* Reset the schema when nSchemaLock==0 */ /* ** The number of different kinds of things that can be limited @@ -13592,34 +17677,84 @@ struct Schema { ** is shared by multiple database connections. Therefore, while parsing ** schema information, the Lookaside.bEnabled flag is cleared so that ** lookaside allocations are not used to construct the schema objects. +** +** New lookaside allocations are only allowed if bDisable==0. When +** bDisable is greater than zero, sz is set to zero which effectively +** disables lookaside without adding a new test for the bDisable flag +** in a performance-critical path. sz should be set by to szTrue whenever +** bDisable changes back to zero. +** +** Lookaside buffers are initially held on the pInit list. As they are +** used and freed, they are added back to the pFree list. New allocations +** come off of pFree first, then pInit as a fallback. This dual-list +** allows use to compute a high-water mark - the maximum number of allocations +** outstanding at any point in the past - by subtracting the number of +** allocations on the pInit list from the total number of allocations. +** +** Enhancement on 2019-12-12: Two-size-lookaside +** The default lookaside configuration is 100 slots of 1200 bytes each. +** The larger slot sizes are important for performance, but they waste +** a lot of space, as most lookaside allocations are less than 128 bytes. +** The two-size-lookaside enhancement breaks up the lookaside allocation +** into two pools: One of 128-byte slots and the other of the default size +** (1200-byte) slots. Allocations are filled from the small-pool first, +** failing over to the full-size pool if that does not work. Thus more +** lookaside slots are available while also using less memory. +** This enhancement can be omitted by compiling with +** SQLITE_OMIT_TWOSIZE_LOOKASIDE. */ struct Lookaside { u32 bDisable; /* Only operate the lookaside when zero */ u16 sz; /* Size of each buffer in bytes */ + u16 szTrue; /* True value of sz, even if disabled */ u8 bMalloced; /* True if pStart obtained from sqlite3_malloc() */ - int nOut; /* Number of buffers currently checked out */ - int mxOut; /* Highwater mark for nOut */ - int anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + u32 nSlot; /* Number of lookaside slots allocated */ + u32 anStat[3]; /* 0: hits. 1: size misses. 2: full misses */ + LookasideSlot *pInit; /* List of buffers not previously used */ LookasideSlot *pFree; /* List of available buffers */ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + LookasideSlot *pSmallInit; /* List of small buffers not previously used */ + LookasideSlot *pSmallFree; /* List of available small buffers */ + void *pMiddle; /* First byte past end of full-size buffers and + ** the first byte of LOOKASIDE_SMALL buffers */ +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ void *pStart; /* First byte of available memory space */ void *pEnd; /* First byte past end of available space */ + void *pTrueEnd; /* True value of pEnd, when db->pnBytesFreed!=0 */ }; struct LookasideSlot { LookasideSlot *pNext; /* Next buffer in the list of free buffers */ }; +#define DisableLookaside db->lookaside.bDisable++;db->lookaside.sz=0 +#define EnableLookaside db->lookaside.bDisable--;\ + db->lookaside.sz=db->lookaside.bDisable?0:db->lookaside.szTrue + +/* Size of the smaller allocations in two-size lookaside */ +#ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define LOOKASIDE_SMALL 0 +#else +# define LOOKASIDE_SMALL 128 +#endif + /* ** A hash table for built-in function definitions. (Application-defined ** functions use a regular table table from hash.h.) ** ** Hash each FuncDef structure into one of the FuncDefHash.a[] slots. -** Collisions are on the FuncDef.u.pHash chain. +** Collisions are on the FuncDef.u.pHash chain. Use the SQLITE_FUNC_HASH() +** macro to compute a hash on the function name. */ #define SQLITE_FUNC_HASH_SZ 23 struct FuncDefHash { FuncDef *a[SQLITE_FUNC_HASH_SZ]; /* Hash table for functions */ }; +#define SQLITE_FUNC_HASH(C,L) (((C)+(L))%SQLITE_FUNC_HASH_SZ) +#if defined(SQLITE_USER_AUTHENTICATION) +# warning "The SQLITE_USER_AUTHENTICATION extension is deprecated. \ + See ext/userauth/user-auth.txt for details." +#endif #ifdef SQLITE_USER_AUTHENTICATION /* ** Information held in the "sqlite3" database connection object and used @@ -13658,6 +17793,23 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); const char*); #endif +#ifndef SQLITE_OMIT_DEPRECATED +/* This is an extra SQLITE_TRACE macro that indicates "legacy" tracing +** in the style of sqlite3_trace() +*/ +#define SQLITE_TRACE_LEGACY 0x40 /* Use the legacy xTrace */ +#define SQLITE_TRACE_XPROFILE 0x80 /* Use the legacy xProfile */ +#else +#define SQLITE_TRACE_LEGACY 0 +#define SQLITE_TRACE_XPROFILE 0 +#endif /* SQLITE_OMIT_DEPRECATED */ +#define SQLITE_TRACE_NONLEGACY_MASK 0x0f /* Normal flags */ + +/* +** Maximum number of sqlite3.aDb[] entries. This is the number of attached +** databases plus 2 for "main" and "temp". +*/ +#define SQLITE_MAX_DB (SQLITE_MAX_ATTACHED+2) /* ** Each database connection is an instance of the following structure. @@ -13665,18 +17817,21 @@ SQLITE_PRIVATE void sqlite3CryptFunc(sqlite3_context*,int,sqlite3_value**); struct sqlite3 { sqlite3_vfs *pVfs; /* OS Interface */ struct Vdbe *pVdbe; /* List of active virtual machines */ - CollSeq *pDfltColl; /* The default collating sequence (BINARY) */ + CollSeq *pDfltColl; /* BINARY collseq for the database encoding */ sqlite3_mutex *mutex; /* Connection mutex */ Db *aDb; /* All backends */ int nDb; /* Number of backends currently in use */ - int flags; /* Miscellaneous flags. See below */ + u32 mDbFlags; /* flags recording internal state */ + u64 flags; /* flags settable by pragmas. See below */ i64 lastRowid; /* ROWID of most recent insert (see above) */ i64 szMmap; /* Default mmap_size setting */ + u32 nSchemaLock; /* Do not reset the schema when non-zero */ unsigned int openFlags; /* Flags passed to sqlite3_vfs.xOpen() */ int errCode; /* Most recent error code (SQLITE_*) */ + int errByteOffset; /* Byte offset of error in SQL statement */ int errMask; /* & result codes with this before returning */ int iSysErrno; /* Errno value from last system error */ - u16 dbOptFlags; /* Flags to enable/disable optimizations */ + u32 dbOptFlags; /* Flags to enable/disable optimizations */ u8 enc; /* Text encoding */ u8 autoCommit; /* The auto-commit flag. */ u8 temp_store; /* 1: file 2: memory 0: default */ @@ -13687,18 +17842,23 @@ struct sqlite3 { u8 suppressErr; /* Do not issue error messages if true */ u8 vtabOnConflict; /* Value to return for s3_vtab_on_conflict() */ u8 isTransactionSavepoint; /* True if the outermost savepoint is a TS */ + u8 mTrace; /* zero or more SQLITE_TRACE flags */ + u8 noSharedCache; /* True if no shared-cache backends */ + u8 nSqlExec; /* Number of pending OP_SqlExec opcodes */ + u8 eOpenState; /* Current condition of the connection */ int nextPagesize; /* Pagesize after VACUUM if >0 */ - u32 magic; /* Magic number for detect library misuse */ - int nChange; /* Value returned by sqlite3_changes() */ - int nTotalChange; /* Value returned by sqlite3_total_changes() */ + i64 nChange; /* Value returned by sqlite3_changes() */ + i64 nTotalChange; /* Value returned by sqlite3_total_changes() */ int aLimit[SQLITE_N_LIMIT]; /* Limits */ int nMaxSorterMmap; /* Maximum size of regions mapped by sorter */ struct sqlite3InitInfo { /* Information used during initialization */ - int newTnum; /* Rootpage of table being initialized */ + Pgno newTnum; /* Rootpage of table being initialized */ u8 iDb; /* Which db file is being initialized */ u8 busy; /* TRUE if currently initializing */ - u8 orphanTrigger; /* Last statement is orphaned TEMP trigger */ - u8 imposterTable; /* Building an imposter table */ + unsigned orphanTrigger : 1; /* Last statement is orphaned TEMP trigger */ + unsigned imposterTable : 1; /* Building an imposter table */ + unsigned reopenMemdb : 1; /* ATTACH is really a reopen using MemDB */ + const char **azInit; /* "type", "name", and "tbl_name" columns */ } init; int nVdbeActive; /* Number of VDBEs currently running */ int nVdbeRead; /* Number of active VDBEs that read or write */ @@ -13707,16 +17867,25 @@ struct sqlite3 { int nVDestroy; /* Number of active OP_VDestroy operations */ int nExtension; /* Number of loaded extensions */ void **aExtension; /* Array of shared library handles */ - void (*xTrace)(void*,const char*); /* Trace function */ - void *pTraceArg; /* Argument to the trace function */ + union { + void (*xLegacy)(void*,const char*); /* mTrace==SQLITE_TRACE_LEGACY */ + int (*xV2)(u32,void*,void*,void*); /* All other mTrace values */ + } trace; + void *pTraceArg; /* Argument to the trace function */ +#ifndef SQLITE_OMIT_DEPRECATED void (*xProfile)(void*,const char*,u64); /* Profiling function */ void *pProfileArg; /* Argument to profile function */ +#endif void *pCommitArg; /* Argument to xCommitCallback() */ int (*xCommitCallback)(void*); /* Invoked at every commit. */ void *pRollbackArg; /* Argument to xRollbackCallback() */ void (*xRollbackCallback)(void*); /* Invoked at every commit. */ void *pUpdateArg; void (*xUpdateCallback)(void*,int, const char*,const char*,sqlite_int64); + void *pAutovacPagesArg; /* Client argument to autovac_pages */ + void (*xAutovacDestr)(void*); /* Destructor for pAutovacPAgesArg */ + unsigned int (*xAutovacPages)(void*,const char*,u32,u32,u32); + Parse *pParse; /* Current parse */ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK void *pPreUpdateArg; /* First argument to xPreUpdateCallback */ void (*xPreUpdateCallback)( /* Registered using sqlite3_preupdate_hook() */ @@ -13751,21 +17920,23 @@ struct sqlite3 { Hash aModule; /* populated by sqlite3_create_module() */ VtabCtx *pVtabCtx; /* Context for active vtab connect/create */ VTable **aVTrans; /* Virtual tables with open transactions */ - VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ + VTable *pDisconnect; /* Disconnect these in next sqlite3_prepare() */ #endif Hash aFunc; /* Hash table of connection functions */ Hash aCollSeq; /* All collating sequences */ BusyHandler busyHandler; /* Busy callback */ Db aDbStatic[2]; /* Static space for the 2 default backends */ Savepoint *pSavepoint; /* List of active savepoints */ + int nAnalysisLimit; /* Number of index rows to ANALYZE */ int busyTimeout; /* Busy handler timeout, in msec */ int nSavepoint; /* Number of non-transaction savepoints */ int nStatement; /* Number of nested statement-transactions */ i64 nDeferredCons; /* Net deferred constraints this transaction. */ i64 nDeferredImmCons; /* Net deferred immediate constraints */ int *pnBytesFreed; /* If not NULL, increment this in DbFree() */ + DbClientData *pDbData; /* sqlite3_set_clientdata() content */ #ifdef SQLITE_ENABLE_UNLOCK_NOTIFY - /* The following variables are all protected by the STATIC_MASTER + /* The following variables are all protected by the STATIC_MAIN ** mutex, not by sqlite3.mutex. They are used by code in notify.c. ** ** When X.pUnlockConnection==Y, that means that X is waiting for Y to @@ -13792,6 +17963,13 @@ struct sqlite3 { #define SCHEMA_ENC(db) ((db)->aDb[0].pSchema->enc) #define ENC(db) ((db)->enc) +/* +** A u64 constant where the lower 32 bits are all zeros. Only the +** upper 32 bits are included in the argument. Necessary because some +** C-compilers still do not accept LL integer literals. +*/ +#define HI(X) ((u64)(X)<<32) + /* ** Possible values for the sqlite3.flags. ** @@ -13800,72 +17978,112 @@ struct sqlite3 { ** SQLITE_CkptFullFSync == PAGER_CKPT_FULLFSYNC ** SQLITE_CacheSpill == PAGER_CACHE_SPILL */ -#define SQLITE_VdbeTrace 0x00000001 /* True to trace VDBE execution */ -#define SQLITE_InternChanges 0x00000002 /* Uncommitted Hash table changes */ +#define SQLITE_WriteSchema 0x00000001 /* OK to update SQLITE_SCHEMA */ +#define SQLITE_LegacyFileFmt 0x00000002 /* Create new databases in format 1 */ #define SQLITE_FullColNames 0x00000004 /* Show full column names on SELECT */ #define SQLITE_FullFSync 0x00000008 /* Use full fsync on the backend */ #define SQLITE_CkptFullFSync 0x00000010 /* Use full fsync for checkpoint */ #define SQLITE_CacheSpill 0x00000020 /* OK to spill pager cache */ #define SQLITE_ShortColNames 0x00000040 /* Show short columns names */ -#define SQLITE_CountRows 0x00000080 /* Count rows changed by INSERT, */ - /* DELETE, or UPDATE and return */ - /* the count using a callback. */ +#define SQLITE_TrustedSchema 0x00000080 /* Allow unsafe functions and + ** vtabs in the schema definition */ #define SQLITE_NullCallback 0x00000100 /* Invoke the callback once if the */ /* result set is empty */ -#define SQLITE_SqlTrace 0x00000200 /* Debug print SQL as it executes */ -#define SQLITE_VdbeListing 0x00000400 /* Debug listings of VDBE programs */ -#define SQLITE_WriteSchema 0x00000800 /* OK to update SQLITE_MASTER */ -#define SQLITE_VdbeAddopTrace 0x00001000 /* Trace sqlite3VdbeAddOp() calls */ -#define SQLITE_IgnoreChecks 0x00002000 /* Do not enforce check constraints */ -#define SQLITE_ReadUncommitted 0x0004000 /* For shared-cache mode */ -#define SQLITE_LegacyFileFmt 0x00008000 /* Create new databases in format 1 */ -#define SQLITE_RecoveryMode 0x00010000 /* Ignore schema errors */ -#define SQLITE_ReverseOrder 0x00020000 /* Reverse unordered SELECTs */ -#define SQLITE_RecTriggers 0x00040000 /* Enable recursive triggers */ -#define SQLITE_ForeignKeys 0x00080000 /* Enforce foreign key constraints */ -#define SQLITE_AutoIndex 0x00100000 /* Enable automatic indexes */ -#define SQLITE_PreferBuiltin 0x00200000 /* Preference to built-in funcs */ -#define SQLITE_LoadExtension 0x00400000 /* Enable load_extension */ -#define SQLITE_LoadExtFunc 0x00800000 /* Enable load_extension() SQL func */ -#define SQLITE_EnableTrigger 0x01000000 /* True to enable triggers */ -#define SQLITE_DeferFKs 0x02000000 /* Defer all FK constraints */ -#define SQLITE_QueryOnly 0x04000000 /* Disable database changes */ -#define SQLITE_VdbeEQP 0x08000000 /* Debug EXPLAIN QUERY PLAN */ -#define SQLITE_Vacuum 0x10000000 /* Currently in a VACUUM */ -#define SQLITE_CellSizeCk 0x20000000 /* Check btree cell sizes on load */ -#define SQLITE_Fts3Tokenizer 0x40000000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_IgnoreChecks 0x00000200 /* Do not enforce check constraints */ +#define SQLITE_StmtScanStatus 0x00000400 /* Enable stmt_scanstats() counters */ +#define SQLITE_NoCkptOnClose 0x00000800 /* No checkpoint on close()/DETACH */ +#define SQLITE_ReverseOrder 0x00001000 /* Reverse unordered SELECTs */ +#define SQLITE_RecTriggers 0x00002000 /* Enable recursive triggers */ +#define SQLITE_ForeignKeys 0x00004000 /* Enforce foreign key constraints */ +#define SQLITE_AutoIndex 0x00008000 /* Enable automatic indexes */ +#define SQLITE_LoadExtension 0x00010000 /* Enable load_extension */ +#define SQLITE_LoadExtFunc 0x00020000 /* Enable load_extension() SQL func */ +#define SQLITE_EnableTrigger 0x00040000 /* True to enable triggers */ +#define SQLITE_DeferFKs 0x00080000 /* Defer all FK constraints */ +#define SQLITE_QueryOnly 0x00100000 /* Disable database changes */ +#define SQLITE_CellSizeCk 0x00200000 /* Check btree cell sizes on load */ +#define SQLITE_Fts3Tokenizer 0x00400000 /* Enable fts3_tokenizer(2) */ +#define SQLITE_EnableQPSG 0x00800000 /* Query Planner Stability Guarantee*/ +#define SQLITE_TriggerEQP 0x01000000 /* Show trigger EXPLAIN QUERY PLAN */ +#define SQLITE_ResetDatabase 0x02000000 /* Reset the database */ +#define SQLITE_LegacyAlter 0x04000000 /* Legacy ALTER TABLE behaviour */ +#define SQLITE_NoSchemaError 0x08000000 /* Do not report schema parse errors*/ +#define SQLITE_Defensive 0x10000000 /* Input SQL is likely hostile */ +#define SQLITE_DqsDDL 0x20000000 /* dbl-quoted strings allowed in DDL*/ +#define SQLITE_DqsDML 0x40000000 /* dbl-quoted strings allowed in DML*/ +#define SQLITE_EnableView 0x80000000 /* Enable the use of views */ +#define SQLITE_CountRows HI(0x00001) /* Count rows changed by INSERT, */ + /* DELETE, or UPDATE and return */ + /* the count using a callback. */ +#define SQLITE_CorruptRdOnly HI(0x00002) /* Prohibit writes due to error */ +#define SQLITE_ReadUncommit HI(0x00004) /* READ UNCOMMITTED in shared-cache */ +#define SQLITE_FkNoAction HI(0x00008) /* Treat all FK as NO ACTION */ +/* Flags used only if debugging */ +#ifdef SQLITE_DEBUG +#define SQLITE_SqlTrace HI(0x0100000) /* Debug print SQL as it executes */ +#define SQLITE_VdbeListing HI(0x0200000) /* Debug listings of VDBE progs */ +#define SQLITE_VdbeTrace HI(0x0400000) /* True to trace VDBE execution */ +#define SQLITE_VdbeAddopTrace HI(0x0800000) /* Trace sqlite3VdbeAddOp() calls */ +#define SQLITE_VdbeEQP HI(0x1000000) /* Debug EXPLAIN QUERY PLAN */ +#define SQLITE_ParserTrace HI(0x2000000) /* PRAGMA parser_trace=ON */ +#endif + +/* +** Allowed values for sqlite3.mDbFlags +*/ +#define DBFLAG_SchemaChange 0x0001 /* Uncommitted Hash table changes */ +#define DBFLAG_PreferBuiltin 0x0002 /* Preference to built-in funcs */ +#define DBFLAG_Vacuum 0x0004 /* Currently in a VACUUM */ +#define DBFLAG_VacuumInto 0x0008 /* Currently running VACUUM INTO */ +#define DBFLAG_SchemaKnownOk 0x0010 /* Schema is known to be valid */ +#define DBFLAG_InternalFunc 0x0020 /* Allow use of internal functions */ +#define DBFLAG_EncodingFixed 0x0040 /* No longer possible to change enc. */ /* ** Bits of the sqlite3.dbOptFlags field that are used by the ** sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS,...) interface to ** selectively disable various optimizations. */ -#define SQLITE_QueryFlattener 0x0001 /* Query flattening */ -#define SQLITE_ColumnCache 0x0002 /* Column cache */ -#define SQLITE_GroupByOrder 0x0004 /* GROUPBY cover of ORDERBY */ -#define SQLITE_FactorOutConst 0x0008 /* Constant factoring */ -/* not used 0x0010 // Was: SQLITE_IdxRealAsInt */ -#define SQLITE_DistinctOpt 0x0020 /* DISTINCT using indexes */ -#define SQLITE_CoverIdxScan 0x0040 /* Covering index scans */ -#define SQLITE_OrderByIdxJoin 0x0080 /* ORDER BY of joins via index */ -#define SQLITE_SubqCoroutine 0x0100 /* Evaluate subqueries as coroutines */ -#define SQLITE_Transitive 0x0200 /* Transitive constraints */ -#define SQLITE_OmitNoopJoin 0x0400 /* Omit unused tables in joins */ -#define SQLITE_Stat34 0x0800 /* Use STAT3 or STAT4 data */ -#define SQLITE_CursorHints 0x2000 /* Add OP_CursorHint opcodes */ -#define SQLITE_AllOpts 0xffff /* All optimizations */ +#define SQLITE_QueryFlattener 0x00000001 /* Query flattening */ +#define SQLITE_WindowFunc 0x00000002 /* Use xInverse for window functions */ +#define SQLITE_GroupByOrder 0x00000004 /* GROUPBY cover of ORDERBY */ +#define SQLITE_FactorOutConst 0x00000008 /* Constant factoring */ +#define SQLITE_DistinctOpt 0x00000010 /* DISTINCT using indexes */ +#define SQLITE_CoverIdxScan 0x00000020 /* Covering index scans */ +#define SQLITE_OrderByIdxJoin 0x00000040 /* ORDER BY of joins via index */ +#define SQLITE_Transitive 0x00000080 /* Transitive constraints */ +#define SQLITE_OmitNoopJoin 0x00000100 /* Omit unused tables in joins */ +#define SQLITE_CountOfView 0x00000200 /* The count-of-view optimization */ +#define SQLITE_CursorHints 0x00000400 /* Add OP_CursorHint opcodes */ +#define SQLITE_Stat4 0x00000800 /* Use STAT4 data */ + /* TH3 expects this value ^^^^^^^^^^ to be 0x0000800. Don't change it */ +#define SQLITE_PushDown 0x00001000 /* WHERE-clause push-down opt */ +#define SQLITE_SimplifyJoin 0x00002000 /* Convert LEFT JOIN to JOIN */ +#define SQLITE_SkipScan 0x00004000 /* Skip-scans */ +#define SQLITE_PropagateConst 0x00008000 /* The constant propagation opt */ +#define SQLITE_MinMaxOpt 0x00010000 /* The min/max optimization */ +#define SQLITE_SeekScan 0x00020000 /* The OP_SeekScan optimization */ +#define SQLITE_OmitOrderBy 0x00040000 /* Omit pointless ORDER BY */ + /* TH3 expects this value ^^^^^^^^^^ to be 0x40000. Coordinate any change */ +#define SQLITE_BloomFilter 0x00080000 /* Use a Bloom filter on searches */ +#define SQLITE_BloomPulldown 0x00100000 /* Run Bloom filters early */ +#define SQLITE_BalancedMerge 0x00200000 /* Balance multi-way merges */ +#define SQLITE_ReleaseReg 0x00400000 /* Use OP_ReleaseReg for testing */ +#define SQLITE_FlttnUnionAll 0x00800000 /* Disable the UNION ALL flattener */ + /* TH3 expects this value ^^^^^^^^^^ See flatten04.test */ +#define SQLITE_IndexedExpr 0x01000000 /* Pull exprs from index when able */ +#define SQLITE_Coroutines 0x02000000 /* Co-routines for subqueries */ +#define SQLITE_NullUnusedCols 0x04000000 /* NULL unused columns in subqueries */ +#define SQLITE_OnePass 0x08000000 /* Single-pass DELETE and UPDATE */ +#define SQLITE_OrderBySubq 0x10000000 /* ORDER BY in subquery helps outer */ +#define SQLITE_AllOpts 0xffffffff /* All optimizations */ /* ** Macros for testing whether or not optimizations are enabled or disabled. */ -#ifndef SQLITE_OMIT_BUILTIN_TEST #define OptimizationDisabled(db, mask) (((db)->dbOptFlags&(mask))!=0) #define OptimizationEnabled(db, mask) (((db)->dbOptFlags&(mask))==0) -#else -#define OptimizationDisabled(db, mask) 0 -#define OptimizationEnabled(db, mask) 1 -#endif /* ** Return true if it OK to factor constant expressions into the initialization @@ -13873,17 +18091,16 @@ struct sqlite3 { */ #define ConstFactorOk(P) ((P)->okConstFactor) -/* -** Possible values for the sqlite.magic field. -** The numbers are obtained at random and have no special meaning, other -** than being distinct from one another. +/* Possible values for the sqlite3.eOpenState field. +** The numbers are randomly selected such that a minimum of three bits must +** change to convert any number to another or to zero */ -#define SQLITE_MAGIC_OPEN 0xa029a697 /* Database is open */ -#define SQLITE_MAGIC_CLOSED 0x9f3c2d33 /* Database is closed */ -#define SQLITE_MAGIC_SICK 0x4b771290 /* Error and awaiting close */ -#define SQLITE_MAGIC_BUSY 0xf03b7906 /* Database currently in use */ -#define SQLITE_MAGIC_ERROR 0xb5357930 /* An SQLITE_MISUSE error occurred */ -#define SQLITE_MAGIC_ZOMBIE 0x64cffc7f /* Close with last statement close */ +#define SQLITE_STATE_OPEN 0x76 /* Database is open */ +#define SQLITE_STATE_CLOSED 0xce /* Database is closed */ +#define SQLITE_STATE_SICK 0xba /* Error and awaiting close */ +#define SQLITE_STATE_BUSY 0x6d /* Database currently in use */ +#define SQLITE_STATE_ERROR 0xd5 /* An SQLITE_MISUSE error occurred */ +#define SQLITE_STATE_ZOMBIE 0xa7 /* Close with last statement close */ /* ** Each SQL function is defined by an instance of the following @@ -13897,16 +18114,18 @@ struct sqlite3 { */ struct FuncDef { i8 nArg; /* Number of arguments. -1 means unlimited */ - u16 funcFlags; /* Some combination of SQLITE_FUNC_* */ + u32 funcFlags; /* Some combination of SQLITE_FUNC_* */ void *pUserData; /* User data parameter */ FuncDef *pNext; /* Next function with same name */ void (*xSFunc)(sqlite3_context*,int,sqlite3_value**); /* func or agg-step */ void (*xFinalize)(sqlite3_context*); /* Agg finalizer */ + void (*xValue)(sqlite3_context*); /* Current agg value */ + void (*xInverse)(sqlite3_context*,int,sqlite3_value**); /* inverse agg-step */ const char *zName; /* SQL name of the function. */ union { FuncDef *pHash; /* Next with a different name but the same hash */ FuncDestructor *pDestructor; /* Reference counted destructor function */ - } u; + } u; /* pHash if SQLITE_FUNC_BUILTIN, pDestructor otherwise */ }; /* @@ -13936,11 +18155,21 @@ struct FuncDestructor { ** are assert() statements in the code to verify this. ** ** Value constraints (enforced via assert()): -** SQLITE_FUNC_MINMAX == NC_MinMaxAgg == SF_MinMaxAgg -** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG -** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG -** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API +** SQLITE_FUNC_MINMAX == NC_MinMaxAgg == SF_MinMaxAgg +** SQLITE_FUNC_ANYORDER == NC_OrderAgg == SF_OrderByReqd +** SQLITE_FUNC_LENGTH == OPFLAG_LENGTHARG +** SQLITE_FUNC_TYPEOF == OPFLAG_TYPEOFARG +** SQLITE_FUNC_BYTELEN == OPFLAG_BYTELENARG +** SQLITE_FUNC_CONSTANT == SQLITE_DETERMINISTIC from the API +** SQLITE_FUNC_DIRECT == SQLITE_DIRECTONLY from the API +** SQLITE_FUNC_UNSAFE == SQLITE_INNOCUOUS -- opposite meanings!!! ** SQLITE_FUNC_ENCMASK depends on SQLITE_UTF* macros in the API +** +** Note that even though SQLITE_FUNC_UNSAFE and SQLITE_INNOCUOUS have the +** same bit value, their meanings are inverted. SQLITE_FUNC_UNSAFE is +** used internally and if set means that the function has side effects. +** SQLITE_INNOCUOUS is used by application code and means "not unsafe". +** See multiple instances of tag-20230109-1. */ #define SQLITE_FUNC_ENCMASK 0x0003 /* SQLITE_UTF8, SQLITE_UTF16BE or UTF16LE */ #define SQLITE_FUNC_LIKE 0x0004 /* Candidate for the LIKE optimization */ @@ -13949,13 +18178,35 @@ struct FuncDestructor { #define SQLITE_FUNC_NEEDCOLL 0x0020 /* sqlite3GetFuncCollSeq() might be called*/ #define SQLITE_FUNC_LENGTH 0x0040 /* Built-in length() function */ #define SQLITE_FUNC_TYPEOF 0x0080 /* Built-in typeof() function */ +#define SQLITE_FUNC_BYTELEN 0x00c0 /* Built-in octet_length() function */ #define SQLITE_FUNC_COUNT 0x0100 /* Built-in count(*) aggregate */ -#define SQLITE_FUNC_COALESCE 0x0200 /* Built-in coalesce() or ifnull() */ +/* 0x0200 -- available for reuse */ #define SQLITE_FUNC_UNLIKELY 0x0400 /* Built-in unlikely() function */ #define SQLITE_FUNC_CONSTANT 0x0800 /* Constant inputs give a constant output */ #define SQLITE_FUNC_MINMAX 0x1000 /* True for min() and max() aggregates */ #define SQLITE_FUNC_SLOCHNG 0x2000 /* "Slow Change". Value constant during a ** single query - might change over time */ +#define SQLITE_FUNC_TEST 0x4000 /* Built-in testing functions */ +#define SQLITE_FUNC_RUNONLY 0x8000 /* Cannot be used by valueFromFunction */ +#define SQLITE_FUNC_WINDOW 0x00010000 /* Built-in window-only function */ +#define SQLITE_FUNC_INTERNAL 0x00040000 /* For use by NestedParse() only */ +#define SQLITE_FUNC_DIRECT 0x00080000 /* Not for use in TRIGGERs or VIEWs */ +/* SQLITE_SUBTYPE 0x00100000 // Consumer of subtypes */ +#define SQLITE_FUNC_UNSAFE 0x00200000 /* Function has side effects */ +#define SQLITE_FUNC_INLINE 0x00400000 /* Functions implemented in-line */ +#define SQLITE_FUNC_BUILTIN 0x00800000 /* This is a built-in function */ +/* SQLITE_RESULT_SUBTYPE 0x01000000 // Generator of subtypes */ +#define SQLITE_FUNC_ANYORDER 0x08000000 /* count/min/max aggregate */ + +/* Identifier numbers for each in-line function */ +#define INLINEFUNC_coalesce 0 +#define INLINEFUNC_implies_nonnull_row 1 +#define INLINEFUNC_expr_implies_expr 2 +#define INLINEFUNC_expr_compare 3 +#define INLINEFUNC_affinity 4 +#define INLINEFUNC_iif 5 +#define INLINEFUNC_sqlite_offset 6 +#define INLINEFUNC_unlikely 99 /* Default case */ /* ** The following three macros, FUNCTION(), LIKEFUNC() and AGGREGATE() are @@ -13971,11 +18222,37 @@ struct FuncDestructor { ** VFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag. ** +** SFUNCTION(zName, nArg, iArg, bNC, xFunc) +** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and +** adds the SQLITE_DIRECTONLY flag. +** +** INLINE_FUNC(zName, nArg, iFuncId, mFlags) +** zName is the name of a function that is implemented by in-line +** byte code rather than by the usual callbacks. The iFuncId +** parameter determines the function id. The mFlags parameter is +** optional SQLITE_FUNC_ flags for this function. +** +** TEST_FUNC(zName, nArg, iFuncId, mFlags) +** zName is the name of a test-only function implemented by in-line +** byte code rather than by the usual callbacks. The iFuncId +** parameter determines the function id. The mFlags parameter is +** optional SQLITE_FUNC_ flags for this function. +** ** DFUNCTION(zName, nArg, iArg, bNC, xFunc) ** Like FUNCTION except it omits the SQLITE_FUNC_CONSTANT flag and ** adds the SQLITE_FUNC_SLOCHNG flag. Used for date & time functions ** and functions like sqlite_version() that can change, but not during -** a single query. +** a single query. The iArg is ignored. The user-data is always set +** to a NULL pointer. The bNC parameter is not used. +** +** MFUNCTION(zName, nArg, xPtr, xFunc) +** For math-library functions. xPtr is an arbitrary pointer. +** +** PURE_DATE(zName, nArg, iArg, bNC, xFunc) +** Used for "pure" date/time functions, this macro is like DFUNCTION +** except that it does set the SQLITE_FUNC_CONSTANT flags. iArg is +** ignored and the user-data for these functions is set to an +** arbitrary non-NULL pointer. The bNC parameter is not used. ** ** AGGREGATE(zName, nArg, iArg, bNC, xStep, xFinal) ** Used to create an aggregate function definition implemented by @@ -13983,6 +18260,12 @@ struct FuncDestructor { ** are interpreted in the same way as the first 4 parameters to ** FUNCTION(). ** +** WAGGREGATE(zName, nArg, iArg, xStep, xFinal, xValue, xInverse) +** Used to create an aggregate function definition implemented by +** the C functions xStep and xFinal. The first four parameters +** are interpreted in the same way as the first 4 parameters to +** FUNCTION(). +** ** LIKEFUNC(zName, nArg, pArg, flags) ** Used to create a scalar function definition of a function zName ** that accepts nArg arguments and is implemented by a call to C @@ -13992,29 +18275,58 @@ struct FuncDestructor { ** parameter. */ #define FUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define VFUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define SFUNCTION(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_UTF8|SQLITE_DIRECTONLY|SQLITE_FUNC_UNSAFE, \ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } +#define MFUNCTION(zName, nArg, xPtr, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8, \ + xPtr, 0, xFunc, 0, 0, 0, #zName, {0} } +#define JFUNCTION(zName, nArg, bUseCache, bWS, bRS, bJsonB, iArg, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_DETERMINISTIC|SQLITE_FUNC_CONSTANT|\ + SQLITE_UTF8|((bUseCache)*SQLITE_FUNC_RUNONLY)|\ + ((bRS)*SQLITE_SUBTYPE)|((bWS)*SQLITE_RESULT_SUBTYPE), \ + SQLITE_INT_TO_PTR(iArg|((bJsonB)*JSON_BLOB)),0,xFunc,0, 0, 0, #zName, {0} } +#define INLINE_FUNC(zName, nArg, iArg, mFlags) \ + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_UTF8|SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ + SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } +#define TEST_FUNC(zName, nArg, iArg, mFlags) \ + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_UTF8|SQLITE_FUNC_INTERNAL|SQLITE_FUNC_TEST| \ + SQLITE_FUNC_INLINE|SQLITE_FUNC_CONSTANT|(mFlags), \ + SQLITE_INT_TO_PTR(iArg), 0, noopFunc, 0, 0, 0, #zName, {0} } #define DFUNCTION(zName, nArg, iArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_SLOCHNG|SQLITE_UTF8, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } +#define PURE_DATE(zName, nArg, iArg, bNC, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + (void*)&sqlite3Config, 0, xFunc, 0, 0, 0, #zName, {0} } #define FUNCTION2(zName, nArg, iArg, bNC, xFunc, extraFlags) \ - {nArg,SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ - SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, #zName, {0} } + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_FUNC_CONSTANT|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL)|extraFlags,\ + SQLITE_INT_TO_PTR(iArg), 0, xFunc, 0, 0, 0, #zName, {0} } #define STR_FUNCTION(zName, nArg, pArg, bNC, xFunc) \ - {nArg, SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ - pArg, 0, xFunc, 0, #zName, } + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_FUNC_SLOCHNG|SQLITE_UTF8|(bNC*SQLITE_FUNC_NEEDCOLL), \ + pArg, 0, xFunc, 0, 0, 0, #zName, } #define LIKEFUNC(zName, nArg, arg, flags) \ - {nArg, SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ - (void *)arg, 0, likeFunc, 0, #zName, {0} } -#define AGGREGATE(zName, nArg, arg, nc, xStep, xFinal) \ - {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL), \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} -#define AGGREGATE2(zName, nArg, arg, nc, xStep, xFinal, extraFlags) \ - {nArg, SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|extraFlags, \ - SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,#zName, {0}} + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_FUNC_CONSTANT|SQLITE_UTF8|flags, \ + (void *)arg, 0, likeFunc, 0, 0, 0, #zName, {0} } +#define WAGGREGATE(zName, nArg, arg, nc, xStep, xFinal, xValue, xInverse, f) \ + {nArg, SQLITE_FUNC_BUILTIN|SQLITE_UTF8|(nc*SQLITE_FUNC_NEEDCOLL)|f, \ + SQLITE_INT_TO_PTR(arg), 0, xStep,xFinal,xValue,xInverse,#zName, {0}} +#define INTERNAL_FUNCTION(zName, nArg, xFunc) \ + {nArg, SQLITE_FUNC_BUILTIN|\ + SQLITE_FUNC_INTERNAL|SQLITE_UTF8|SQLITE_FUNC_CONSTANT, \ + 0, 0, xFunc, 0, 0, 0, #zName, {0} } + /* ** All current savepoints are stored in a linked list starting at @@ -14046,30 +18358,84 @@ struct Savepoint { struct Module { const sqlite3_module *pModule; /* Callback pointers */ const char *zName; /* Name passed to create_module() */ + int nRefModule; /* Number of pointers to this object */ void *pAux; /* pAux passed to create_module() */ void (*xDestroy)(void *); /* Module destructor function */ Table *pEpoTab; /* Eponymous table for this module */ }; /* -** information about each column of an SQL table is held in an instance -** of this structure. +** Information about each column of an SQL table is held in an instance +** of the Column structure, in the Table.aCol[] array. +** +** Definitions: +** +** "table column index" This is the index of the column in the +** Table.aCol[] array, and also the index of +** the column in the original CREATE TABLE stmt. +** +** "storage column index" This is the index of the column in the +** record BLOB generated by the OP_MakeRecord +** opcode. The storage column index is less than +** or equal to the table column index. It is +** equal if and only if there are no VIRTUAL +** columns to the left. +** +** Notes on zCnName: +** The zCnName field stores the name of the column, the datatype of the +** column, and the collating sequence for the column, in that order, all in +** a single allocation. Each string is 0x00 terminated. The datatype +** is only included if the COLFLAG_HASTYPE bit of colFlags is set and the +** collating sequence name is only included if the COLFLAG_HASCOLL bit is +** set. */ struct Column { - char *zName; /* Name of this column, \000, then the type */ - Expr *pDflt; /* Default value of this column */ - char *zColl; /* Collating sequence. If NULL, use the default */ - u8 notNull; /* An OE_ code for handling a NOT NULL constraint */ - char affinity; /* One of the SQLITE_AFF_... values */ - u8 szEst; /* Estimated size of value in this column. sizeof(INT)==1 */ - u8 colFlags; /* Boolean properties. See COLFLAG_ defines below */ + char *zCnName; /* Name of this column */ + unsigned notNull :4; /* An OE_ code for handling a NOT NULL constraint */ + unsigned eCType :4; /* One of the standard types */ + char affinity; /* One of the SQLITE_AFF_... values */ + u8 szEst; /* Est size of value in this column. sizeof(INT)==1 */ + u8 hName; /* Column name hash for faster lookup */ + u16 iDflt; /* 1-based index of DEFAULT. 0 means "none" */ + u16 colFlags; /* Boolean properties. See COLFLAG_ defines below */ }; -/* Allowed values for Column.colFlags: +/* Allowed values for Column.eCType. +** +** Values must match entries in the global constant arrays +** sqlite3StdTypeLen[] and sqlite3StdType[]. Each value is one more +** than the offset into these arrays for the corresponding name. +** Adjust the SQLITE_N_STDTYPE value if adding or removing entries. */ -#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ -#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ -#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLTYPE_CUSTOM 0 /* Type appended to zName */ +#define COLTYPE_ANY 1 +#define COLTYPE_BLOB 2 +#define COLTYPE_INT 3 +#define COLTYPE_INTEGER 4 +#define COLTYPE_REAL 5 +#define COLTYPE_TEXT 6 +#define SQLITE_N_STDTYPE 6 /* Number of standard types */ + +/* Allowed values for Column.colFlags. +** +** Constraints: +** TF_HasVirtual == COLFLAG_VIRTUAL +** TF_HasStored == COLFLAG_STORED +** TF_HasHidden == COLFLAG_HIDDEN +*/ +#define COLFLAG_PRIMKEY 0x0001 /* Column is part of the primary key */ +#define COLFLAG_HIDDEN 0x0002 /* A hidden column in a virtual table */ +#define COLFLAG_HASTYPE 0x0004 /* Type name follows column name */ +#define COLFLAG_UNIQUE 0x0008 /* Column def contains "UNIQUE" or "PK" */ +#define COLFLAG_SORTERREF 0x0010 /* Use sorter-refs with this column */ +#define COLFLAG_VIRTUAL 0x0020 /* GENERATED ALWAYS AS ... VIRTUAL */ +#define COLFLAG_STORED 0x0040 /* GENERATED ALWAYS AS ... STORED */ +#define COLFLAG_NOTAVAIL 0x0080 /* STORED column not yet calculated */ +#define COLFLAG_BUSY 0x0100 /* Blocks recursion on GENERATED columns */ +#define COLFLAG_HASCOLL 0x0200 /* Has collating sequence name in zCnName */ +#define COLFLAG_NOEXPAND 0x0400 /* Omit this column when expanding "*" */ +#define COLFLAG_GENERATED 0x0060 /* Combo: _STORED, _VIRTUAL */ +#define COLFLAG_NOINSERT 0x0062 /* Combo: _HIDDEN, _STORED, _VIRTUAL */ /* ** A "Collating Sequence" is defined by an instance of the following @@ -14109,11 +18475,13 @@ struct CollSeq { ** Note also that the numeric types are grouped together so that testing ** for a numeric type is a single comparison. And the BLOB type is first. */ -#define SQLITE_AFF_BLOB 'A' -#define SQLITE_AFF_TEXT 'B' -#define SQLITE_AFF_NUMERIC 'C' -#define SQLITE_AFF_INTEGER 'D' -#define SQLITE_AFF_REAL 'E' +#define SQLITE_AFF_NONE 0x40 /* '@' */ +#define SQLITE_AFF_BLOB 0x41 /* 'A' */ +#define SQLITE_AFF_TEXT 0x42 /* 'B' */ +#define SQLITE_AFF_NUMERIC 0x43 /* 'C' */ +#define SQLITE_AFF_INTEGER 0x44 /* 'D' */ +#define SQLITE_AFF_REAL 0x45 /* 'E' */ +#define SQLITE_AFF_FLEXNUM 0x46 /* 'F' */ #define sqlite3IsNumericAffinity(X) ((X)>=SQLITE_AFF_NUMERIC) @@ -14133,7 +18501,6 @@ struct CollSeq { ** prove that the operands are always NOT NULL. */ #define SQLITE_JUMPIFNULL 0x10 /* jumps if either operand is NULL */ -#define SQLITE_STOREP2 0x20 /* Store result in reg[P2] rather than jump */ #define SQLITE_NULLEQ 0x80 /* NULL=NULL */ #define SQLITE_NOTNULL 0x90 /* Assert that operands are never NULL */ @@ -14185,45 +18552,60 @@ struct VTable { sqlite3_vtab *pVtab; /* Pointer to vtab instance */ int nRef; /* Number of pointers to this structure */ u8 bConstraint; /* True if constraints are supported */ + u8 bAllSchemas; /* True if might use any attached schema */ + u8 eVtabRisk; /* Riskiness of allowing hacker access */ int iSavepoint; /* Depth of the SAVEPOINT stack */ VTable *pNext; /* Next in linked list (see above) */ }; +/* Allowed values for VTable.eVtabRisk +*/ +#define SQLITE_VTABRISK_Low 0 +#define SQLITE_VTABRISK_Normal 1 +#define SQLITE_VTABRISK_High 2 + /* -** The schema for each SQL table and view is represented in memory -** by an instance of the following structure. +** The schema for each SQL table, virtual table, and view is represented +** in memory by an instance of the following structure. */ struct Table { char *zName; /* Name of the table or view */ Column *aCol; /* Information about each column */ Index *pIndex; /* List of SQL indexes on this table. */ - Select *pSelect; /* NULL for tables. Points to definition if a view. */ - FKey *pFKey; /* Linked list of all foreign keys in this table */ char *zColAff; /* String defining the affinity of each column */ ExprList *pCheck; /* All CHECK constraints */ /* ... also used as column name list in a VIEW */ - int tnum; /* Root BTree page for this table */ + Pgno tnum; /* Root BTree page for this table */ + u32 nTabRef; /* Number of pointers to this Table */ + u32 tabFlags; /* Mask of TF_* values */ i16 iPKey; /* If not negative, use aCol[iPKey] as the rowid */ i16 nCol; /* Number of columns in this table */ - u16 nRef; /* Number of pointers to this Table */ + i16 nNVCol; /* Number of columns that are not VIRTUAL */ LogEst nRowLogEst; /* Estimated rows in table - from sqlite_stat1 table */ LogEst szTabRow; /* Estimated size of each table row in bytes */ #ifdef SQLITE_ENABLE_COSTMULT LogEst costMult; /* Cost multiplier for using this table */ #endif - u8 tabFlags; /* Mask of TF_* values */ u8 keyConf; /* What to do in case of uniqueness conflict on iPKey */ -#ifndef SQLITE_OMIT_ALTERTABLE - int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ -#endif -#ifndef SQLITE_OMIT_VIRTUALTABLE - int nModuleArg; /* Number of arguments to the module */ - char **azModuleArg; /* 0: module 1: schema 2: vtab name 3...: args */ - VTable *pVTable; /* List of VTable objects. */ -#endif - Trigger *pTrigger; /* List of triggers stored in pSchema */ + u8 eTabType; /* 0: normal, 1: virtual, 2: view */ + union { + struct { /* Used by ordinary tables: */ + int addColOffset; /* Offset in CREATE TABLE stmt to add a new column */ + FKey *pFKey; /* Linked list of all foreign keys in this table */ + ExprList *pDfltList; /* DEFAULT clauses on various columns. + ** Or the AS clause for generated columns. */ + } tab; + struct { /* Used by views: */ + Select *pSelect; /* View definition */ + } view; + struct { /* Used by virtual tables only: */ + int nArg; /* Number of arguments to the module */ + char **azArg; /* 0: module 1: schema 2: vtab name 3...: args */ + VTable *p; /* List of VTable objects. */ + } vtab; + } u; + Trigger *pTrigger; /* List of triggers on this object */ Schema *pSchema; /* Schema that contains this table */ - Table *pNextZombie; /* Next on the Parse.pZombieTab list */ }; /* @@ -14233,17 +18615,42 @@ struct Table { ** followed by non-hidden columns. Example: "CREATE VIRTUAL TABLE x USING ** vtab1(a HIDDEN, b);". Since "b" is a non-hidden column but "a" is hidden, ** the TF_OOOHidden attribute would apply in this case. Such tables require -** special handling during INSERT processing. +** special handling during INSERT processing. The "OOO" means "Out Of Order". +** +** Constraints: +** +** TF_HasVirtual == COLFLAG_VIRTUAL +** TF_HasStored == COLFLAG_STORED +** TF_HasHidden == COLFLAG_HIDDEN */ -#define TF_Readonly 0x01 /* Read-only system table */ -#define TF_Ephemeral 0x02 /* An ephemeral table */ -#define TF_HasPrimaryKey 0x04 /* Table has a primary key */ -#define TF_Autoincrement 0x08 /* Integer primary key is autoincrement */ -#define TF_Virtual 0x10 /* Is a virtual table */ -#define TF_WithoutRowid 0x20 /* No rowid. PRIMARY KEY is the key */ -#define TF_NoVisibleRowid 0x40 /* No user-visible "rowid" column */ -#define TF_OOOHidden 0x80 /* Out-of-Order hidden columns */ +#define TF_Readonly 0x00000001 /* Read-only system table */ +#define TF_HasHidden 0x00000002 /* Has one or more hidden columns */ +#define TF_HasPrimaryKey 0x00000004 /* Table has a primary key */ +#define TF_Autoincrement 0x00000008 /* Integer primary key is autoincrement */ +#define TF_HasStat1 0x00000010 /* nRowLogEst set from sqlite_stat1 */ +#define TF_HasVirtual 0x00000020 /* Has one or more VIRTUAL columns */ +#define TF_HasStored 0x00000040 /* Has one or more STORED columns */ +#define TF_HasGenerated 0x00000060 /* Combo: HasVirtual + HasStored */ +#define TF_WithoutRowid 0x00000080 /* No rowid. PRIMARY KEY is the key */ +#define TF_MaybeReanalyze 0x00000100 /* Maybe run ANALYZE on this table */ +#define TF_NoVisibleRowid 0x00000200 /* No user-visible "rowid" column */ +#define TF_OOOHidden 0x00000400 /* Out-of-Order hidden columns */ +#define TF_HasNotNull 0x00000800 /* Contains NOT NULL constraints */ +#define TF_Shadow 0x00001000 /* True for a shadow table */ +#define TF_HasStat4 0x00002000 /* STAT4 info available for this table */ +#define TF_Ephemeral 0x00004000 /* An ephemeral table */ +#define TF_Eponymous 0x00008000 /* An eponymous virtual table */ +#define TF_Strict 0x00010000 /* STRICT mode */ +/* +** Allowed values for Table.eTabType +*/ +#define TABTYP_NORM 0 /* Ordinary table */ +#define TABTYP_VTAB 1 /* Virtual table */ +#define TABTYP_VIEW 2 /* A view */ + +#define IsView(X) ((X)->eTabType==TABTYP_VIEW) +#define IsOrdinaryTable(X) ((X)->eTabType==TABTYP_NORM) /* ** Test to see whether or not a table is a virtual table. This is @@ -14251,9 +18658,12 @@ struct Table { ** table support is omitted from the build. */ #ifndef SQLITE_OMIT_VIRTUALTABLE -# define IsVirtual(X) (((X)->tabFlags & TF_Virtual)!=0) +# define IsVirtual(X) ((X)->eTabType==TABTYP_VTAB) +# define ExprIsVtab(X) \ + ((X)->op==TK_COLUMN && (X)->y.pTab->eTabType==TABTYP_VTAB) #else # define IsVirtual(X) 0 +# define ExprIsVtab(X) 0 #endif /* @@ -14278,6 +18688,15 @@ struct Table { #define HasRowid(X) (((X)->tabFlags & TF_WithoutRowid)==0) #define VisibleRowid(X) (((X)->tabFlags & TF_NoVisibleRowid)==0) +/* Macro is true if the SQLITE_ALLOW_ROWID_IN_VIEW (mis-)feature is +** available. By default, this macro is false +*/ +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW +# define ViewCanHaveRowid 0 +#else +# define ViewCanHaveRowid (sqlite3Config.mNoVisibleRowid==0) +#endif + /* ** Each foreign key constraint is an instance of the following structure. ** @@ -14337,16 +18756,22 @@ struct FKey { ** is returned. REPLACE means that preexisting database rows that caused ** a UNIQUE constraint violation are removed so that the new insert or ** update can proceed. Processing continues and no error is reported. +** UPDATE applies to insert operations only and means that the insert +** is omitted and the DO UPDATE clause of an upsert is run instead. ** -** RESTRICT, SETNULL, and CASCADE actions apply only to foreign keys. +** RESTRICT, SETNULL, SETDFLT, and CASCADE actions apply only to foreign keys. ** RESTRICT is the same as ABORT for IMMEDIATE foreign keys and the ** same as ROLLBACK for DEFERRED keys. SETNULL means that the foreign -** key is set to NULL. CASCADE means that a DELETE or UPDATE of the +** key is set to NULL. SETDFLT means that the foreign key is set +** to its default value. CASCADE means that a DELETE or UPDATE of the ** referenced table row is propagated into the row that holds the ** foreign key. ** +** The OE_Default value is a place holder that means to use whatever +** conflict resolution algorithm is required from context. +** ** The following symbolic values are used to record which type -** of action to take. +** of conflict resolution action to take. */ #define OE_None 0 /* There is no constraint to check */ #define OE_Rollback 1 /* Fail the operation and rollback the transaction */ @@ -14354,13 +18779,12 @@ struct FKey { #define OE_Fail 3 /* Stop the operation but leave all prior changes */ #define OE_Ignore 4 /* Ignore the error. Do not do the INSERT or UPDATE */ #define OE_Replace 5 /* Delete existing record, then do INSERT or UPDATE */ - -#define OE_Restrict 6 /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */ -#define OE_SetNull 7 /* Set the foreign key value to NULL */ -#define OE_SetDflt 8 /* Set the foreign key value to its default */ -#define OE_Cascade 9 /* Cascade the changes */ - -#define OE_Default 10 /* Do whatever the default action is */ +#define OE_Update 6 /* Process as a DO UPDATE in an upsert */ +#define OE_Restrict 7 /* OE_Abort for IMMEDIATE, OE_Rollback for DEFERRED */ +#define OE_SetNull 8 /* Set the foreign key value to NULL */ +#define OE_SetDflt 9 /* Set the foreign key value to its default */ +#define OE_Cascade 10 /* Cascade the changes */ +#define OE_Default 11 /* Do whatever the default action is */ /* @@ -14375,13 +18799,19 @@ struct FKey { struct KeyInfo { u32 nRef; /* Number of references to this KeyInfo object */ u8 enc; /* Text encoding - one of the SQLITE_UTF* values */ - u16 nField; /* Number of key columns in the index */ - u16 nXField; /* Number of columns beyond the key columns */ + u16 nKeyField; /* Number of key columns in the index */ + u16 nAllField; /* Total columns, including key plus others */ sqlite3 *db; /* The database connection */ - u8 *aSortOrder; /* Sort order for each column. */ + u8 *aSortFlags; /* Sort order for each column. */ CollSeq *aColl[1]; /* Collating sequence for each term of the key */ }; +/* +** Allowed bit values for entries in the KeyInfo.aSortFlags[] array. +*/ +#define KEYINFO_ORDER_DESC 0x01 /* DESC sort order */ +#define KEYINFO_ORDER_BIGNULL 0x02 /* NULL is larger than any other value */ + /* ** This object holds a record which has been parsed out into individual ** fields, for the purposes of doing a comparison. @@ -14420,11 +18850,16 @@ struct KeyInfo { struct UnpackedRecord { KeyInfo *pKeyInfo; /* Collation and sort-order information */ Mem *aMem; /* Values */ + union { + char *z; /* Cache of aMem[0].z for vdbeRecordCompareString() */ + i64 i; /* Cache of aMem[0].u.i for vdbeRecordCompareInt() */ + } u; + int n; /* Cache of aMem[0].n used by vdbeRecordCompareString() */ u16 nField; /* Number of entries in apMem[] */ i8 default_rc; /* Comparison result if keys are equal */ u8 errCode; /* Error detected by xRecordCompare (CORRUPT or NOMEM) */ - i8 r1; /* Value to return if (lhs > rhs) */ - i8 r2; /* Value to return if (rhs < lhs) */ + i8 r1; /* Value to return if (lhs < rhs) */ + i8 r2; /* Value to return if (lhs > rhs) */ u8 eqSeen; /* True if an equality comparison has been seen */ }; @@ -14451,12 +18886,24 @@ struct UnpackedRecord { ** The Index.onError field determines whether or not the indexed columns ** must be unique and what to do if they are not. When Index.onError=OE_None, ** it means this is not a unique index. Otherwise it is a unique index -** and the value of Index.onError indicate the which conflict resolution -** algorithm to employ whenever an attempt is made to insert a non-unique +** and the value of Index.onError indicates which conflict resolution +** algorithm to employ when an attempt is made to insert a non-unique ** element. ** +** The colNotIdxed bitmask is used in combination with SrcItem.colUsed +** for a fast test to see if an index can serve as a covering index. +** colNotIdxed has a 1 bit for every column of the original table that +** is *not* available in the index. Thus the expression +** "colUsed & colNotIdxed" will be non-zero if the index is not a +** covering index. The most significant bit of of colNotIdxed will always +** be true (note-20221022-a). If a column beyond the 63rd column of the +** table is used, the "colUsed & colNotIdxed" test will always be non-zero +** and we have to assume either that the index is not covering, or use +** an alternative (slower) algorithm to determine whether or not +** the index is covering. +** ** While parsing a CREATE TABLE or CREATE INDEX statement in order to -** generate VDBE code (as opposed to parsing one read from an sqlite_master +** generate VDBE code (as opposed to parsing one read from an sqlite_schema ** table as part of parsing an existing database schema), transient instances ** of this structure may be created. In this case the Index.tnum variable is ** used to store the address of a VDBE instruction, not a database page @@ -14475,25 +18922,34 @@ struct Index { const char **azColl; /* Array of collation sequence names for index */ Expr *pPartIdxWhere; /* WHERE clause for partial indices */ ExprList *aColExpr; /* Column expressions */ - int tnum; /* DB Page containing root of this index */ + Pgno tnum; /* DB Page containing root of this index */ LogEst szIdxRow; /* Estimated average row size in bytes */ u16 nKeyCol; /* Number of columns forming the key */ u16 nColumn; /* Number of columns stored in the index */ u8 onError; /* OE_Abort, OE_Ignore, OE_Replace, or OE_None */ - unsigned idxType:2; /* 1==UNIQUE, 2==PRIMARY KEY, 0==CREATE INDEX */ + unsigned idxType:2; /* 0:Normal 1:UNIQUE, 2:PRIMARY KEY, 3:IPK */ unsigned bUnordered:1; /* Use this index for == or IN queries only */ unsigned uniqNotNull:1; /* True if UNIQUE and NOT NULL for all columns */ unsigned isResized:1; /* True if resizeIndexObject() has been called */ unsigned isCovering:1; /* True if this is a covering index */ unsigned noSkipScan:1; /* Do not try to use skip-scan if true */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + unsigned hasStat1:1; /* aiRowLogEst values come from sqlite_stat1 */ + unsigned bLowQual:1; /* sqlite_stat1 says this is a low-quality index */ + unsigned bNoQuery:1; /* Do not use this index to optimize queries */ + unsigned bAscKeyBug:1; /* True if the bba7b69f9849b5bf bug applies */ + unsigned bHasVCol:1; /* Index references one or more VIRTUAL columns */ + unsigned bHasExpr:1; /* Index contains an expression, either a literal + ** expression, or a reference to a VIRTUAL column */ +#ifdef SQLITE_ENABLE_STAT4 int nSample; /* Number of elements in aSample[] */ + int mxSample; /* Number of slots allocated to aSample[] */ int nSampleCol; /* Size of IndexSample.anEq[] and so on */ tRowcnt *aAvgEq; /* Average nEq values for keys not in aSample */ IndexSample *aSample; /* Samples of the left-most key */ tRowcnt *aiRowEst; /* Non-logarithmic stat1 data for this index */ tRowcnt nRowEst0; /* Non-logarithmic number of rows in the index */ #endif + Bitmask colNotIdxed; /* Unindexed columns in pTab */ }; /* @@ -14502,6 +18958,7 @@ struct Index { #define SQLITE_IDXTYPE_APPDEF 0 /* Created using CREATE INDEX */ #define SQLITE_IDXTYPE_UNIQUE 1 /* Implements a UNIQUE constraint */ #define SQLITE_IDXTYPE_PRIMARYKEY 2 /* Is the PRIMARY KEY for the table */ +#define SQLITE_IDXTYPE_IPK 3 /* INTEGER PRIMARY KEY index */ /* Return true if index X is a PRIMARY KEY index */ #define IsPrimaryKeyIndex(X) ((X)->idxType==SQLITE_IDXTYPE_PRIMARYKEY) @@ -14516,7 +18973,7 @@ struct Index { #define XN_EXPR (-2) /* Indexed column is an expression */ /* -** Each sample stored in the sqlite_stat3 table is represented in memory +** Each sample stored in the sqlite_stat4 table is represented in memory ** using a structure of this type. See documentation at the top of the ** analyze.c source file for additional information. */ @@ -14528,13 +18985,21 @@ struct IndexSample { tRowcnt *anDLt; /* Est. number of distinct keys less than this sample */ }; +/* +** Possible values to use within the flags argument to sqlite3GetToken(). +*/ +#define SQLITE_TOKEN_QUOTED 0x1 /* Token is a quoted identifier. */ +#define SQLITE_TOKEN_KEYWORD 0x2 /* Token is a keyword. */ + /* ** Each token coming out of the lexer is an instance of ** this structure. Tokens are also used as part of an expression. ** -** Note if Token.z==0 then Token.dyn and Token.n are undefined and -** may contain random values. Do not make any assumptions about Token.dyn -** and Token.n when Token.z==0. +** The memory that "z" points to is owned by other objects. Take care +** that the owner of the "z" string does not deallocate the string before +** the Token goes out of scope! Very often, the "z" points to some place +** in the middle of the Parse.zSql text. But it might also point to a +** static string. */ struct Token { const char *z; /* Text of the token. Not NULL-terminated! */ @@ -14546,7 +19011,7 @@ struct Token { ** code for a SELECT that contains aggregate functions. ** ** If Expr.op==TK_AGG_COLUMN or TK_AGG_FUNCTION then Expr.pAggInfo is a -** pointer to this structure. The Expr.iColumn field is the index in +** pointer to this structure. The Expr.iAgg field is the index in ** AggInfo.aCol[] or AggInfo.aFunc[] of information needed to generate ** code for that node. ** @@ -14559,32 +19024,56 @@ struct AggInfo { ** from source tables rather than from accumulators */ u8 useSortingIdx; /* In direct mode, reference the sorting index rather ** than the source table */ + u16 nSortingColumn; /* Number of columns in the sorting index */ int sortingIdx; /* Cursor number of the sorting index */ int sortingIdxPTab; /* Cursor number of pseudo-table */ - int nSortingColumn; /* Number of columns in the sorting index */ - int mnReg, mxReg; /* Range of registers allocated for aCol and aFunc */ + int iFirstReg; /* First register in range for aCol[] and aFunc[] */ ExprList *pGroupBy; /* The group by clause */ struct AggInfo_col { /* For each column used in source tables */ Table *pTab; /* Source table */ + Expr *pCExpr; /* The original expression */ int iTable; /* Cursor number of the source table */ - int iColumn; /* Column number within the source table */ - int iSorterColumn; /* Column number in the sorting index */ - int iMem; /* Memory location that acts as accumulator */ - Expr *pExpr; /* The original expression */ + i16 iColumn; /* Column number within the source table */ + i16 iSorterColumn; /* Column number in the sorting index */ } *aCol; int nColumn; /* Number of used entries in aCol[] */ int nAccumulator; /* Number of columns that show through to the output. ** Additional columns are used only as parameters to ** aggregate functions */ struct AggInfo_func { /* For each aggregate function */ - Expr *pExpr; /* Expression encoding the function */ + Expr *pFExpr; /* Expression encoding the function */ FuncDef *pFunc; /* The aggregate function implementation */ - int iMem; /* Memory location that acts as accumulator */ int iDistinct; /* Ephemeral table used to enforce DISTINCT */ + int iDistAddr; /* Address of OP_OpenEphemeral */ + int iOBTab; /* Ephemeral table to implement ORDER BY */ + u8 bOBPayload; /* iOBTab has payload columns separate from key */ + u8 bOBUnique; /* Enforce uniqueness on iOBTab keys */ + u8 bUseSubtype; /* Transfer subtype info through sorter */ } *aFunc; int nFunc; /* Number of entries in aFunc[] */ + u32 selId; /* Select to which this AggInfo belongs */ +#ifdef SQLITE_DEBUG + Select *pSelect; /* SELECT statement that this AggInfo supports */ +#endif }; +/* +** Macros to compute aCol[] and aFunc[] register numbers. +** +** These macros should not be used prior to the call to +** assignAggregateRegisters() that computes the value of pAggInfo->iFirstReg. +** The assert()s that are part of this macro verify that constraint. +*/ +#ifndef NDEBUG +#define AggInfoColumnReg(A,I) (assert((A)->iFirstReg),(A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + (assert((A)->iFirstReg),(A)->iFirstReg+(A)->nColumn+(I)) +#else +#define AggInfoColumnReg(A,I) ((A)->iFirstReg+(I)) +#define AggInfoFuncReg(A,I) \ + ((A)->iFirstReg+(A)->nColumn+(I)) +#endif + /* ** The datatype ynVar is a signed integer, either 16-bit or 32-bit. ** Usually it is 16-bits. But if SQLITE_MAX_VARIABLE_NUMBER is greater @@ -14592,10 +19081,10 @@ struct AggInfo { ** it uses less memory in the Expr object, which is a big memory user ** in systems with lots of prepared statements. And few applications ** need more than about 10 or 20 variables. But some extreme users want -** to have prepared statements with over 32767 variables, and for them +** to have prepared statements with over 32766 variables, and for them ** the option is available (at compile-time). */ -#if SQLITE_MAX_VARIABLE_NUMBER<=32767 +#if SQLITE_MAX_VARIABLE_NUMBER<32767 typedef i16 ynVar; #else typedef int ynVar; @@ -14612,10 +19101,10 @@ typedef int ynVar; ** tree. ** ** If the expression is an SQL literal (TK_INTEGER, TK_FLOAT, TK_BLOB, -** or TK_STRING), then Expr.token contains the text of the SQL literal. If -** the expression is a variable (TK_VARIABLE), then Expr.token contains the +** or TK_STRING), then Expr.u.zToken contains the text of the SQL literal. If +** the expression is a variable (TK_VARIABLE), then Expr.u.zToken contains the ** variable name. Finally, if the expression is an SQL function (TK_FUNCTION), -** then Expr.token contains the name of the function. +** then Expr.u.zToken contains the name of the function. ** ** Expr.pRight and Expr.pLeft are the left and right subexpressions of a ** binary operator. Either or both may be NULL. @@ -14655,7 +19144,7 @@ typedef int ynVar; ** help reduce memory requirements, sometimes an Expr object will be ** truncated. And to reduce the number of memory allocations, sometimes ** two or more Expr objects will be stored in a single memory allocation, -** together with Expr.zToken strings. +** together with Expr.u.zToken strings. ** ** If the EP_Reduced and EP_TokenOnly flags are set when ** an Expr object is truncated. When EP_Reduced is set, then all @@ -14666,7 +19155,14 @@ typedef int ynVar; */ struct Expr { u8 op; /* Operation performed by this node */ - char affinity; /* The affinity of the column or 0 if not a column */ + char affExpr; /* affinity, or RAISE type */ + u8 op2; /* TK_REGISTER/TK_TRUTH: original value of Expr.op + ** TK_COLUMN: the value of p5 for OP_Column + ** TK_AGG_FUNCTION: nesting depth + ** TK_FUNCTION: NC_SelfRef flag if needs OP_PureFunc */ +#ifdef SQLITE_DEBUG + u8 vvaFlags; /* Verification flags. */ +#endif u32 flags; /* Various flags. EP_* See below */ union { char *zToken; /* Token value. Zero terminated and dequoted */ @@ -14696,67 +19192,115 @@ struct Expr { int iTable; /* TK_COLUMN: cursor number of table holding column ** TK_REGISTER: register number ** TK_TRIGGER: 1 -> new, 0 -> old - ** EP_Unlikely: 134217728 times likelihood */ + ** EP_Unlikely: 134217728 times likelihood + ** TK_IN: ephemeral table holding RHS + ** TK_SELECT_COLUMN: Number of columns on the LHS + ** TK_SELECT: 1st register of result vector */ ynVar iColumn; /* TK_COLUMN: column index. -1 for rowid. - ** TK_VARIABLE: variable number (always >= 1). */ + ** TK_VARIABLE: variable number (always >= 1). + ** TK_SELECT_COLUMN: column of the result vector */ i16 iAgg; /* Which entry in pAggInfo->aCol[] or ->aFunc[] */ - i16 iRightJoinTable; /* If EP_FromJoin, the right table of the join */ - u8 op2; /* TK_REGISTER: original value of Expr.op - ** TK_COLUMN: the value of p5 for OP_Column - ** TK_AGG_FUNCTION: nesting depth */ + union { + int iJoin; /* If EP_OuterON or EP_InnerON, the right table */ + int iOfst; /* else: start of token from start of statement */ + } w; AggInfo *pAggInfo; /* Used by TK_AGG_COLUMN and TK_AGG_FUNCTION */ - Table *pTab; /* Table for TK_COLUMN expressions. */ + union { + Table *pTab; /* TK_COLUMN: Table containing column. Can be NULL + ** for a column of an index on an expression */ + Window *pWin; /* EP_WinFunc: Window/Filter defn for a function */ + struct { /* TK_IN, TK_SELECT, and TK_EXISTS */ + int iAddr; /* Subroutine entry address */ + int regReturn; /* Register used to hold return address */ + } sub; + } y; }; -/* -** The following are the meanings of bits in the Expr.flags field. +/* The following are the meanings of bits in the Expr.flags field. +** Value restrictions: +** +** EP_Agg == NC_HasAgg == SF_HasAgg +** EP_Win == NC_HasWin */ -#define EP_FromJoin 0x000001 /* Originates in ON/USING clause of outer join */ -#define EP_Agg 0x000002 /* Contains one or more aggregate functions */ -#define EP_Resolved 0x000004 /* IDs have been resolved to COLUMNs */ -#define EP_Error 0x000008 /* Expression contains one or more errors */ -#define EP_Distinct 0x000010 /* Aggregate function with DISTINCT keyword */ -#define EP_VarSelect 0x000020 /* pSelect is correlated, not constant */ -#define EP_DblQuoted 0x000040 /* token.z was originally in "..." */ -#define EP_InfixFunc 0x000080 /* True for an infix function: LIKE, GLOB, etc */ -#define EP_Collate 0x000100 /* Tree contains a TK_COLLATE operator */ -#define EP_Generic 0x000200 /* Ignore COLLATE or affinity on this tree */ -#define EP_IntValue 0x000400 /* Integer value contained in u.iValue */ -#define EP_xIsSelect 0x000800 /* x.pSelect is valid (otherwise x.pList is) */ -#define EP_Skip 0x001000 /* COLLATE, AS, or UNLIKELY */ -#define EP_Reduced 0x002000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ -#define EP_TokenOnly 0x004000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ -#define EP_Static 0x008000 /* Held in memory not obtained from malloc() */ -#define EP_MemToken 0x010000 /* Need to sqlite3DbFree() Expr.zToken */ -#define EP_NoReduce 0x020000 /* Cannot EXPRDUP_REDUCE this Expr */ -#define EP_Unlikely 0x040000 /* unlikely() or likelihood() function */ -#define EP_ConstFunc 0x080000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ -#define EP_CanBeNull 0x100000 /* Can be null despite NOT NULL constraint */ -#define EP_Subquery 0x200000 /* Tree contains a TK_SELECT operator */ -#define EP_Alias 0x400000 /* Is an alias for a result set column */ +#define EP_OuterON 0x000001 /* Originates in ON/USING clause of outer join */ +#define EP_InnerON 0x000002 /* Originates in ON/USING of an inner join */ +#define EP_Distinct 0x000004 /* Aggregate function with DISTINCT keyword */ +#define EP_HasFunc 0x000008 /* Contains one or more functions of any kind */ +#define EP_Agg 0x000010 /* Contains one or more aggregate functions */ +#define EP_FixedCol 0x000020 /* TK_Column with a known fixed value */ +#define EP_VarSelect 0x000040 /* pSelect is correlated, not constant */ +#define EP_DblQuoted 0x000080 /* token.z was originally in "..." */ +#define EP_InfixFunc 0x000100 /* True for an infix function: LIKE, GLOB, etc */ +#define EP_Collate 0x000200 /* Tree contains a TK_COLLATE operator */ +#define EP_Commuted 0x000400 /* Comparison operator has been commuted */ +#define EP_IntValue 0x000800 /* Integer value contained in u.iValue */ +#define EP_xIsSelect 0x001000 /* x.pSelect is valid (otherwise x.pList is) */ +#define EP_Skip 0x002000 /* Operator does not contribute to affinity */ +#define EP_Reduced 0x004000 /* Expr struct EXPR_REDUCEDSIZE bytes only */ +#define EP_Win 0x008000 /* Contains window functions */ +#define EP_TokenOnly 0x010000 /* Expr struct EXPR_TOKENONLYSIZE bytes only */ +#define EP_FullSize 0x020000 /* Expr structure must remain full sized */ +#define EP_IfNullRow 0x040000 /* The TK_IF_NULL_ROW opcode */ +#define EP_Unlikely 0x080000 /* unlikely() or likelihood() function */ +#define EP_ConstFunc 0x100000 /* A SQLITE_FUNC_CONSTANT or _SLOCHNG function */ +#define EP_CanBeNull 0x200000 /* Can be null despite NOT NULL constraint */ +#define EP_Subquery 0x400000 /* Tree contains a TK_SELECT operator */ +#define EP_Leaf 0x800000 /* Expr.pLeft, .pRight, .u.pSelect all NULL */ +#define EP_WinFunc 0x1000000 /* TK_FUNCTION with Expr.y.pWin set */ +#define EP_Subrtn 0x2000000 /* Uses Expr.y.sub. TK_IN, _SELECT, or _EXISTS */ +#define EP_Quoted 0x4000000 /* TK_ID was originally quoted */ +#define EP_Static 0x8000000 /* Held in memory not obtained from malloc() */ +#define EP_IsTrue 0x10000000 /* Always has boolean value of TRUE */ +#define EP_IsFalse 0x20000000 /* Always has boolean value of FALSE */ +#define EP_FromDDL 0x40000000 /* Originates from sqlite_schema */ +#define EP_SubtArg 0x80000000 /* Is argument to SQLITE_SUBTYPE function */ -/* -** Combinations of two or more EP_* flags +/* The EP_Propagate mask is a set of properties that automatically propagate +** upwards into parent nodes. */ -#define EP_Propagate (EP_Collate|EP_Subquery) /* Propagate these bits up tree */ +#define EP_Propagate (EP_Collate|EP_Subquery|EP_HasFunc) -/* -** These macros can be used to test, set, or clear bits in the +/* Macros can be used to test, set, or clear bits in the ** Expr.flags field. */ #define ExprHasProperty(E,P) (((E)->flags&(P))!=0) #define ExprHasAllProperty(E,P) (((E)->flags&(P))==(P)) #define ExprSetProperty(E,P) (E)->flags|=(P) #define ExprClearProperty(E,P) (E)->flags&=~(P) +#define ExprAlwaysTrue(E) (((E)->flags&(EP_OuterON|EP_IsTrue))==EP_IsTrue) +#define ExprAlwaysFalse(E) (((E)->flags&(EP_OuterON|EP_IsFalse))==EP_IsFalse) +#define ExprIsFullSize(E) (((E)->flags&(EP_Reduced|EP_TokenOnly))==0) + +/* Macros used to ensure that the correct members of unions are accessed +** in Expr. +*/ +#define ExprUseUToken(E) (((E)->flags&EP_IntValue)==0) +#define ExprUseUValue(E) (((E)->flags&EP_IntValue)!=0) +#define ExprUseWOfst(E) (((E)->flags&(EP_InnerON|EP_OuterON))==0) +#define ExprUseWJoin(E) (((E)->flags&(EP_InnerON|EP_OuterON))!=0) +#define ExprUseXList(E) (((E)->flags&EP_xIsSelect)==0) +#define ExprUseXSelect(E) (((E)->flags&EP_xIsSelect)!=0) +#define ExprUseYTab(E) (((E)->flags&(EP_WinFunc|EP_Subrtn))==0) +#define ExprUseYWin(E) (((E)->flags&EP_WinFunc)!=0) +#define ExprUseYSub(E) (((E)->flags&EP_Subrtn)!=0) + +/* Flags for use with Expr.vvaFlags +*/ +#define EP_NoReduce 0x01 /* Cannot EXPRDUP_REDUCE this Expr */ +#define EP_Immutable 0x02 /* Do not change this Expr node */ /* The ExprSetVVAProperty() macro is used for Verification, Validation, ** and Accreditation only. It works like ExprSetProperty() during VVA ** processes but is a no-op for delivery. */ #ifdef SQLITE_DEBUG -# define ExprSetVVAProperty(E,P) (E)->flags|=(P) +# define ExprSetVVAProperty(E,P) (E)->vvaFlags|=(P) +# define ExprHasVVAProperty(E,P) (((E)->vvaFlags&(P))!=0) +# define ExprClearVVAProperties(E) (E)->vvaFlags = 0 #else # define ExprSetVVAProperty(E,P) +# define ExprHasVVAProperty(E,P) 0 +# define ExprClearVVAProperties(E) #endif /* @@ -14774,6 +19318,18 @@ struct Expr { */ #define EXPRDUP_REDUCE 0x0001 /* Used reduced-size Expr nodes */ +/* +** True if the expression passed as an argument was a function with +** an OVER() clause (a window function). +*/ +#ifdef SQLITE_OMIT_WINDOWFUNC +# define IsWindowFunc(p) 0 +#else +# define IsWindowFunc(p) ( \ + ExprHasProperty((p), EP_WinFunc) && p->y.pWin->eFrmType!=TK_FILTER \ + ) +#endif + /* ** A list of expressions. Each expression may optionally have a ** name. An expr/name combination can be used in several ways, such @@ -14782,44 +19338,56 @@ struct Expr { ** also be used as the argument to a function, in which case the a.zName ** field is not used. ** -** By default the Expr.zSpan field holds a human-readable description of -** the expression that is used in the generation of error messages and -** column labels. In this case, Expr.zSpan is typically the text of a -** column expression as it exists in a SELECT statement. However, if -** the bSpanIsTab flag is set, then zSpan is overloaded to mean the name -** of the result column in the form: DATABASE.TABLE.COLUMN. This later -** form is used for name resolution with nested FROM clauses. +** In order to try to keep memory usage down, the Expr.a.zEName field +** is used for multiple purposes: +** +** eEName Usage +** ---------- ------------------------- +** ENAME_NAME (1) the AS of result set column +** (2) COLUMN= of an UPDATE +** +** ENAME_TAB DB.TABLE.NAME used to resolve names +** of subqueries +** +** ENAME_SPAN Text of the original result set +** expression. */ struct ExprList { int nExpr; /* Number of expressions on the list */ + int nAlloc; /* Number of a[] slots allocated */ struct ExprList_item { /* For each expression in the list */ - Expr *pExpr; /* The list of expressions */ - char *zName; /* Token associated with this expression */ - char *zSpan; /* Original text of the expression */ - u8 sortOrder; /* 1 for DESC or 0 for ASC */ - unsigned done :1; /* A flag to indicate when processing is finished */ - unsigned bSpanIsTab :1; /* zSpan holds DB.TABLE.COLUMN */ - unsigned reusable :1; /* Constant expression is reusable */ + Expr *pExpr; /* The parse tree for this expression */ + char *zEName; /* Token associated with this expression */ + struct { + u8 sortFlags; /* Mask of KEYINFO_ORDER_* flags */ + unsigned eEName :2; /* Meaning of zEName */ + unsigned done :1; /* Indicates when processing is finished */ + unsigned reusable :1; /* Constant expression is reusable */ + unsigned bSorterRef :1; /* Defer evaluation until after sorting */ + unsigned bNulls :1; /* True if explicit "NULLS FIRST/LAST" */ + unsigned bUsed :1; /* This column used in a SF_NestedFrom subquery */ + unsigned bUsingTerm:1; /* Term from the USING clause of a NestedFrom */ + unsigned bNoExpand: 1; /* Term is an auxiliary in NestedFrom and should + ** not be expanded by "*" in parent queries */ + } fg; union { - struct { + struct { /* Used by any ExprList other than Parse.pConsExpr */ u16 iOrderByCol; /* For ORDER BY, column number in result set */ u16 iAlias; /* Index into Parse.aAlias[] for zName */ } x; - int iConstExprReg; /* Register in which Expr value is cached */ + int iConstExprReg; /* Register in which Expr value is cached. Used only + ** by Parse.pConstExpr */ } u; - } *a; /* Alloc a power of two greater or equal to nExpr */ + } a[1]; /* One slot for each expression in the list */ }; /* -** An instance of this structure is used by the parser to record both -** the parse tree for an expression and the span of input text for an -** expression. +** Allowed values for Expr.a.eEName */ -struct ExprSpan { - Expr *pExpr; /* The expression parse tree */ - const char *zStart; /* First character of input text */ - const char *zEnd; /* One character past the end of input text */ -}; +#define ENAME_NAME 0 /* The AS clause of a result set */ +#define ENAME_SPAN 1 /* Complete text of the result set expression */ +#define ENAME_TAB 2 /* "DB.TABLE.NAME" for the result set */ +#define ENAME_ROWID 3 /* "DB.TABLE._rowid_" for * expansion of rowid */ /* ** An instance of this structure can hold a simple list of identifiers, @@ -14837,48 +19405,38 @@ struct ExprSpan { ** If "a" is the k-th column of table "t", then IdList.a[0].idx==k. */ struct IdList { + int nId; /* Number of identifiers on the list */ + u8 eU4; /* Which element of a.u4 is valid */ struct IdList_item { char *zName; /* Name of the identifier */ - int idx; /* Index in some Table.aCol[] of a column named zName */ - } *a; - int nId; /* Number of identifiers on the list */ + union { + int idx; /* Index in some Table.aCol[] of a column named zName */ + Expr *pExpr; /* Expr to implement a USING variable -- NOT USED */ + } u4; + } a[1]; }; /* -** The bitmask datatype defined below is used for various optimizations. -** -** Changing this from a 64-bit to a 32-bit type limits the number of -** tables in a join to 32 instead of 64. But it also reduces the size -** of the library by 738 bytes on ix86. +** Allowed values for IdList.eType, which determines which value of the a.u4 +** is valid. */ -#ifdef SQLITE_BITMASK_TYPE - typedef SQLITE_BITMASK_TYPE Bitmask; -#else - typedef u64 Bitmask; -#endif +#define EU4_NONE 0 /* Does not use IdList.a.u4 */ +#define EU4_IDX 1 /* Uses IdList.a.u4.idx */ +#define EU4_EXPR 2 /* Uses IdList.a.u4.pExpr -- NOT CURRENTLY USED */ /* -** The number of bits in a Bitmask. "BMS" means "BitMask Size". +** Details of the implementation of a subquery. */ -#define BMS ((int)(sizeof(Bitmask)*8)) +struct Subquery { + Select *pSelect; /* A SELECT statement used in place of a table name */ + int addrFillSub; /* Address of subroutine to initialize a subquery */ + int regReturn; /* Register holding return address of addrFillSub */ + int regResult; /* Registers holding results of a co-routine */ +}; /* -** A bit in a Bitmask -*/ -#define MASKBIT(n) (((Bitmask)1)<<(n)) -#define MASKBIT32(n) (((unsigned int)1)<<(n)) -#define ALLBITS ((Bitmask)-1) - -/* -** The following structure describes the FROM clause of a SELECT statement. -** Each table or subquery in the FROM clause is a separate element of -** the SrcList.a[] array. -** -** With the addition of multiple database support, the following structure -** can also be used to describe a particular table such as the table that -** is modified by an INSERT, DELETE, or UPDATE statement. In standard SQL, -** such a table must be a simple name: ID. But in SQLite, the table can -** now be identified by a database name, a dot, then the table name: ID.ID. +** The SrcItem object represents a single term in the FROM clause of a query. +** The SrcList object is mostly an array of SrcItems. ** ** The jointype starts out showing the join type between the current table ** and the next table on the list. The parser builds the list this way. @@ -14887,55 +19445,113 @@ struct IdList { ** ** In the colUsed field, the high-order bit (bit 63) is set if the table ** contains more than 63 columns and the 64-th or later column is used. +** +** Aggressive use of "union" helps keep the size of the object small. This +** has been shown to boost performance, in addition to saving memory. +** Access to union elements is gated by the following rules which should +** always be checked, either by an if-statement or by an assert(). +** +** Field Only access if this is true +** --------------- ----------------------------------- +** u1.zIndexedBy fg.isIndexedBy +** u1.pFuncArg fg.isTabFunc +** u1.nRow !fg.isTabFunc && !fg.isIndexedBy +** +** u2.pIBIndex fg.isIndexedBy +** u2.pCteUse fg.isCte +** +** u3.pOn !fg.isUsing +** u3.pUsing fg.isUsing +** +** u4.zDatabase !fg.fixedSchema && !fg.isSubquery +** u4.pSchema fg.fixedSchema +** u4.pSubq fg.isSubquery +** +** See also the sqlite3SrcListDelete() routine for assert() statements that +** check invariants on the fields of this object, especially the flags +** inside the fg struct. +*/ +struct SrcItem { + char *zName; /* Name of the table */ + char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ + Table *pSTab; /* Table object for zName. Mnemonic: Srcitem-TABle */ + struct { + u8 jointype; /* Type of join between this table and the previous */ + unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ + unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ + unsigned isSubquery :1; /* True if this term is a subquery */ + unsigned isTabFunc :1; /* True if table-valued-function syntax */ + unsigned isCorrelated :1; /* True if sub-query is correlated */ + unsigned isMaterialized:1; /* This is a materialized view */ + unsigned viaCoroutine :1; /* Implemented as a co-routine */ + unsigned isRecursive :1; /* True for recursive reference in WITH */ + unsigned fromDDL :1; /* Comes from sqlite_schema */ + unsigned isCte :1; /* This is a CTE */ + unsigned notCte :1; /* This item may not match a CTE */ + unsigned isUsing :1; /* u3.pUsing is valid */ + unsigned isOn :1; /* u3.pOn was once valid and non-NULL */ + unsigned isSynthUsing :1; /* u3.pUsing is synthesized from NATURAL */ + unsigned isNestedFrom :1; /* pSelect is a SF_NestedFrom subquery */ + unsigned rowidUsed :1; /* The ROWID of this table is referenced */ + unsigned fixedSchema :1; /* Uses u4.pSchema, not u4.zDatabase */ + unsigned hadSchema :1; /* Had u4.zDatabase before u4.pSchema */ + } fg; + int iCursor; /* The VDBE cursor number used to access this table */ + Bitmask colUsed; /* Bit N set if column N used. Details above for N>62 */ + union { + char *zIndexedBy; /* Identifier from "INDEXED BY " clause */ + ExprList *pFuncArg; /* Arguments to table-valued-function */ + u32 nRow; /* Number of rows in a VALUES clause */ + } u1; + union { + Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ + CteUse *pCteUse; /* CTE Usage info when fg.isCte is true */ + } u2; + union { + Expr *pOn; /* fg.isUsing==0 => The ON clause of a join */ + IdList *pUsing; /* fg.isUsing==1 => The USING clause of a join */ + } u3; + union { + Schema *pSchema; /* Schema to which this item is fixed */ + char *zDatabase; /* Name of database holding this table */ + Subquery *pSubq; /* Description of a subquery */ + } u4; +}; + +/* +** The OnOrUsing object represents either an ON clause or a USING clause. +** It can never be both at the same time, but it can be neither. +*/ +struct OnOrUsing { + Expr *pOn; /* The ON clause of a join */ + IdList *pUsing; /* The USING clause of a join */ +}; + +/* +** This object represents one or more tables that are the source of +** content for an SQL statement. For example, a single SrcList object +** is used to hold the FROM clause of a SELECT statement. SrcList also +** represents the target tables for DELETE, INSERT, and UPDATE statements. +** */ struct SrcList { int nSrc; /* Number of tables or subqueries in the FROM clause */ u32 nAlloc; /* Number of entries allocated in a[] below */ - struct SrcList_item { - Schema *pSchema; /* Schema to which this item is fixed */ - char *zDatabase; /* Name of database holding this table */ - char *zName; /* Name of the table */ - char *zAlias; /* The "B" part of a "A AS B" phrase. zName is the "A" */ - Table *pTab; /* An SQL table corresponding to zName */ - Select *pSelect; /* A SELECT statement used in place of a table name */ - int addrFillSub; /* Address of subroutine to manifest a subquery */ - int regReturn; /* Register holding return address of addrFillSub */ - int regResult; /* Registers holding results of a co-routine */ - struct { - u8 jointype; /* Type of join between this able and the previous */ - unsigned notIndexed :1; /* True if there is a NOT INDEXED clause */ - unsigned isIndexedBy :1; /* True if there is an INDEXED BY clause */ - unsigned isTabFunc :1; /* True if table-valued-function syntax */ - unsigned isCorrelated :1; /* True if sub-query is correlated */ - unsigned viaCoroutine :1; /* Implemented as a co-routine */ - unsigned isRecursive :1; /* True for recursive reference in WITH */ - } fg; -#ifndef SQLITE_OMIT_EXPLAIN - u8 iSelectId; /* If pSelect!=0, the id of the sub-select in EQP */ -#endif - int iCursor; /* The VDBE cursor number used to access this table */ - Expr *pOn; /* The ON clause of a join */ - IdList *pUsing; /* The USING clause of a join */ - Bitmask colUsed; /* Bit N (1<" clause */ - ExprList *pFuncArg; /* Arguments to table-valued-function */ - } u1; - Index *pIBIndex; /* Index structure corresponding to u1.zIndexedBy */ - } a[1]; /* One entry for each identifier on the list */ + SrcItem a[1]; /* One entry for each identifier on the list */ }; /* ** Permitted values of the SrcList.a.jointype field */ -#define JT_INNER 0x0001 /* Any kind of inner or cross join */ -#define JT_CROSS 0x0002 /* Explicit use of the CROSS keyword */ -#define JT_NATURAL 0x0004 /* True for a "natural" join */ -#define JT_LEFT 0x0008 /* Left outer join */ -#define JT_RIGHT 0x0010 /* Right outer join */ -#define JT_OUTER 0x0020 /* The "OUTER" keyword is present */ -#define JT_ERROR 0x0040 /* unknown or unsupported join type */ - +#define JT_INNER 0x01 /* Any kind of inner or cross join */ +#define JT_CROSS 0x02 /* Explicit use of the CROSS keyword */ +#define JT_NATURAL 0x04 /* True for a "natural" join */ +#define JT_LEFT 0x08 /* Left outer join */ +#define JT_RIGHT 0x10 /* Right outer join */ +#define JT_OUTER 0x20 /* The "OUTER" keyword is present */ +#define JT_LTORJ 0x40 /* One of the LEFT operands of a RIGHT JOIN + ** Mnemonic: Left Table Of Right Join */ +#define JT_ERROR 0x80 /* unknown or unsupported join type */ /* ** Flags appropriate for the wctrlFlags parameter of sqlite3WhereBegin() @@ -14948,19 +19564,20 @@ struct SrcList { #define WHERE_ORDERBY_MIN 0x0001 /* ORDER BY processing for min() func */ #define WHERE_ORDERBY_MAX 0x0002 /* ORDER BY processing for max() func */ #define WHERE_ONEPASS_DESIRED 0x0004 /* Want to do one-pass UPDATE/DELETE */ -#define WHERE_DUPLICATES_OK 0x0008 /* Ok to return a row more than once */ -#define WHERE_OMIT_OPEN_CLOSE 0x0010 /* Table cursors are already open */ -#define WHERE_FORCE_TABLE 0x0020 /* Do not use an index-only search */ -#define WHERE_ONETABLE_ONLY 0x0040 /* Only code the 1st table in pTabList */ -#define WHERE_NO_AUTOINDEX 0x0080 /* Disallow automatic indexes */ -#define WHERE_GROUPBY 0x0100 /* pOrderBy is really a GROUP BY */ -#define WHERE_DISTINCTBY 0x0200 /* pOrderby is really a DISTINCT clause */ -#define WHERE_WANT_DISTINCT 0x0400 /* All output needs to be distinct */ -#define WHERE_SORTBYGROUP 0x0800 /* Support sqlite3WhereIsSorted() */ -#define WHERE_REOPEN_IDX 0x1000 /* Try to use OP_ReopenIdx */ -#define WHERE_ONEPASS_MULTIROW 0x2000 /* ONEPASS is ok with multiple rows */ -#define WHERE_USE_LIMIT 0x4000 /* There is a constant LIMIT clause */ -#define WHERE_SEEK_TABLE 0x8000 /* Do not defer seeks on main table */ +#define WHERE_ONEPASS_MULTIROW 0x0008 /* ONEPASS is ok with multiple rows */ +#define WHERE_DUPLICATES_OK 0x0010 /* Ok to return a row more than once */ +#define WHERE_OR_SUBCLAUSE 0x0020 /* Processing a sub-WHERE as part of + ** the OR optimization */ +#define WHERE_GROUPBY 0x0040 /* pOrderBy is really a GROUP BY */ +#define WHERE_DISTINCTBY 0x0080 /* pOrderby is really a DISTINCT clause */ +#define WHERE_WANT_DISTINCT 0x0100 /* All output needs to be distinct */ +#define WHERE_SORTBYGROUP 0x0200 /* Support sqlite3WhereIsSorted() */ +#define WHERE_AGG_DISTINCT 0x0400 /* Query is "SELECT agg(DISTINCT ...)" */ +#define WHERE_ORDERBY_LIMIT 0x0800 /* ORDERBY+LIMIT on the inner loop */ +#define WHERE_RIGHT_JOIN 0x1000 /* Processing a RIGHT JOIN */ +#define WHERE_KEEP_ALL_JOINS 0x2000 /* Do not do the omit-noop-join opt */ +#define WHERE_USE_LIMIT 0x4000 /* Use the LIMIT in cost estimates */ + /* 0x8000 not currently used */ /* Allowed return values from sqlite3WhereIsDistinct() */ @@ -14993,40 +19610,96 @@ struct SrcList { struct NameContext { Parse *pParse; /* The parser */ SrcList *pSrcList; /* One or more tables used to resolve names */ - ExprList *pEList; /* Optional list of result-set columns */ - AggInfo *pAggInfo; /* Information about aggregates at this level */ + union { + ExprList *pEList; /* Optional list of result-set columns */ + AggInfo *pAggInfo; /* Information about aggregates at this level */ + Upsert *pUpsert; /* ON CONFLICT clause information from an upsert */ + int iBaseReg; /* For TK_REGISTER when parsing RETURNING */ + } uNC; NameContext *pNext; /* Next outer name context. NULL for outermost */ int nRef; /* Number of names resolved by this context */ - int nErr; /* Number of errors encountered while resolving names */ - u16 ncFlags; /* Zero or more NC_* flags defined below */ + int nNcErr; /* Number of errors encountered while resolving names */ + int ncFlags; /* Zero or more NC_* flags defined below */ + u32 nNestedSelect; /* Number of nested selects using this NC */ + Select *pWinSelect; /* SELECT statement for any window functions */ }; /* ** Allowed values for the NameContext, ncFlags field. ** ** Value constraints (all checked via assert()): -** NC_HasAgg == SF_HasAgg -** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX +** NC_HasAgg == SF_HasAgg == EP_Agg +** NC_MinMaxAgg == SF_MinMaxAgg == SQLITE_FUNC_MINMAX +** NC_OrderAgg == SF_OrderByReqd == SQLITE_FUNC_ANYORDER +** NC_HasWin == EP_Win ** */ -#define NC_AllowAgg 0x0001 /* Aggregate functions are allowed here */ -#define NC_PartIdx 0x0002 /* True if resolving a partial index WHERE */ -#define NC_IsCheck 0x0004 /* True if resolving names in a CHECK constraint */ -#define NC_InAggFunc 0x0008 /* True if analyzing arguments to an agg func */ -#define NC_HasAgg 0x0010 /* One or more aggregate functions seen */ -#define NC_IdxExpr 0x0020 /* True if resolving columns of CREATE INDEX */ -#define NC_VarSelect 0x0040 /* A correlated subquery has been seen */ -#define NC_MinMaxAgg 0x1000 /* min/max aggregates seen. See note above */ +#define NC_AllowAgg 0x000001 /* Aggregate functions are allowed here */ +#define NC_PartIdx 0x000002 /* True if resolving a partial index WHERE */ +#define NC_IsCheck 0x000004 /* True if resolving a CHECK constraint */ +#define NC_GenCol 0x000008 /* True for a GENERATED ALWAYS AS clause */ +#define NC_HasAgg 0x000010 /* One or more aggregate functions seen */ +#define NC_IdxExpr 0x000020 /* True if resolving columns of CREATE INDEX */ +#define NC_SelfRef 0x00002e /* Combo: PartIdx, isCheck, GenCol, and IdxExpr */ +#define NC_Subquery 0x000040 /* A subquery has been seen */ +#define NC_UEList 0x000080 /* True if uNC.pEList is used */ +#define NC_UAggInfo 0x000100 /* True if uNC.pAggInfo is used */ +#define NC_UUpsert 0x000200 /* True if uNC.pUpsert is used */ +#define NC_UBaseReg 0x000400 /* True if uNC.iBaseReg is used */ +#define NC_MinMaxAgg 0x001000 /* min/max aggregates seen. See note above */ +/* 0x002000 // available for reuse */ +#define NC_AllowWin 0x004000 /* Window functions are allowed here */ +#define NC_HasWin 0x008000 /* One or more window functions seen */ +#define NC_IsDDL 0x010000 /* Resolving names in a CREATE statement */ +#define NC_InAggFunc 0x020000 /* True if analyzing arguments to an agg func */ +#define NC_FromDDL 0x040000 /* SQL text comes from sqlite_schema */ +#define NC_NoSelect 0x080000 /* Do not descend into sub-selects */ +#define NC_Where 0x100000 /* Processing WHERE clause of a SELECT */ +#define NC_OrderAgg 0x8000000 /* Has an aggregate other than count/min/max */ + +/* +** An instance of the following object describes a single ON CONFLICT +** clause in an upsert. +** +** The pUpsertTarget field is only set if the ON CONFLICT clause includes +** conflict-target clause. (In "ON CONFLICT(a,b)" the "(a,b)" is the +** conflict-target clause.) The pUpsertTargetWhere is the optional +** WHERE clause used to identify partial unique indexes. +** +** pUpsertSet is the list of column=expr terms of the UPDATE statement. +** The pUpsertSet field is NULL for a ON CONFLICT DO NOTHING. The +** pUpsertWhere is the WHERE clause for the UPDATE and is NULL if the +** WHERE clause is omitted. +*/ +struct Upsert { + ExprList *pUpsertTarget; /* Optional description of conflict target */ + Expr *pUpsertTargetWhere; /* WHERE clause for partial index targets */ + ExprList *pUpsertSet; /* The SET clause from an ON CONFLICT UPDATE */ + Expr *pUpsertWhere; /* WHERE clause for the ON CONFLICT UPDATE */ + Upsert *pNextUpsert; /* Next ON CONFLICT clause in the list */ + u8 isDoUpdate; /* True for DO UPDATE. False for DO NOTHING */ + u8 isDup; /* True if 2nd or later with same pUpsertIdx */ + /* Above this point is the parse tree for the ON CONFLICT clauses. + ** The next group of fields stores intermediate data. */ + void *pToFree; /* Free memory when deleting the Upsert object */ + /* All fields above are owned by the Upsert object and must be freed + ** when the Upsert is destroyed. The fields below are used to transfer + ** information from the INSERT processing down into the UPDATE processing + ** while generating code. The fields below are owned by the INSERT + ** statement and will be freed by INSERT processing. */ + Index *pUpsertIdx; /* UNIQUE constraint specified by pUpsertTarget */ + SrcList *pUpsertSrc; /* Table to be updated */ + int regData; /* First register holding array of VALUES */ + int iDataCur; /* Index of the data cursor */ + int iIdxCur; /* Index of the first index cursor */ +}; /* ** An instance of the following structure contains all information ** needed to generate code for a single SELECT statement. ** -** nLimit is set to -1 if there is no LIMIT clause. nOffset is set to 0. -** If there is a LIMIT clause, the parser sets nLimit to the value of the -** limit and nOffset to the value of the offset (or 0 if there is not -** offset). But later on, nLimit and nOffset become the memory locations -** in the VDBE that record the limit and offset counters. +** See the header comment on the computeLimitRegisters() routine for a +** detailed description of the meaning of the iLimit and iOffset fields. ** ** addrOpenEphm[] entries contain the address of OP_OpenEphemeral opcodes. ** These addresses must be stored so that we can go back and fill in @@ -15039,15 +19712,13 @@ struct NameContext { ** sequences for the ORDER BY clause. */ struct Select { - ExprList *pEList; /* The fields of the result */ u8 op; /* One of: TK_UNION TK_ALL TK_INTERSECT TK_EXCEPT */ LogEst nSelectRow; /* Estimated number of result rows */ u32 selFlags; /* Various SF_* values */ int iLimit, iOffset; /* Memory registers holding LIMIT & OFFSET counters */ -#if SELECTTRACE_ENABLED - char zSelName[12]; /* Symbolic name of this SELECT use for debugging */ -#endif + u32 selId; /* Unique identifier number for this SELECT */ int addrOpenEphm[2]; /* OP_OpenEphem opcodes related to this select */ + ExprList *pEList; /* The fields of the result */ SrcList *pSrc; /* The FROM clause */ Expr *pWhere; /* The WHERE clause */ ExprList *pGroupBy; /* The GROUP BY clause */ @@ -15056,8 +19727,11 @@ struct Select { Select *pPrior; /* Prior select in a compound select statement */ Select *pNext; /* Next select to the left in a compound */ Expr *pLimit; /* LIMIT expression. NULL means not used. */ - Expr *pOffset; /* OFFSET expression. NULL means not used. */ With *pWith; /* WITH clause attached to this select. Or NULL. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin; /* List of window functions */ + Window *pWinDefn; /* List of named window definitions */ +#endif }; /* @@ -15065,29 +19739,46 @@ struct Select { ** "Select Flag". ** ** Value constraints (all checked via assert()) -** SF_HasAgg == NC_HasAgg -** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX -** SF_FixedLimit == WHERE_USE_LIMIT +** SF_HasAgg == NC_HasAgg +** SF_MinMaxAgg == NC_MinMaxAgg == SQLITE_FUNC_MINMAX +** SF_OrderByReqd == NC_OrderAgg == SQLITE_FUNC_ANYORDER +** SF_FixedLimit == WHERE_USE_LIMIT */ -#define SF_Distinct 0x00001 /* Output should be DISTINCT */ -#define SF_All 0x00002 /* Includes the ALL keyword */ -#define SF_Resolved 0x00004 /* Identifiers have been resolved */ -#define SF_Aggregate 0x00008 /* Contains agg functions or a GROUP BY */ -#define SF_HasAgg 0x00010 /* Contains aggregate functions */ -#define SF_UsesEphemeral 0x00020 /* Uses the OpenEphemeral opcode */ -#define SF_Expanded 0x00040 /* sqlite3SelectExpand() called on this */ -#define SF_HasTypeInfo 0x00080 /* FROM subqueries have Table metadata */ -#define SF_Compound 0x00100 /* Part of a compound query */ -#define SF_Values 0x00200 /* Synthesized from VALUES clause */ -#define SF_MultiValue 0x00400 /* Single VALUES term with multiple rows */ -#define SF_NestedFrom 0x00800 /* Part of a parenthesized FROM clause */ -#define SF_MinMaxAgg 0x01000 /* Aggregate containing min() or max() */ -#define SF_Recursive 0x02000 /* The recursive part of a recursive CTE */ -#define SF_FixedLimit 0x04000 /* nSelectRow set by a constant LIMIT */ -#define SF_MaybeConvert 0x08000 /* Need convertCompoundSelectToSubquery() */ -#define SF_Converted 0x10000 /* By convertCompoundSelectToSubquery() */ -#define SF_IncludeHidden 0x20000 /* Include hidden columns in output */ +#define SF_Distinct 0x0000001 /* Output should be DISTINCT */ +#define SF_All 0x0000002 /* Includes the ALL keyword */ +#define SF_Resolved 0x0000004 /* Identifiers have been resolved */ +#define SF_Aggregate 0x0000008 /* Contains agg functions or a GROUP BY */ +#define SF_HasAgg 0x0000010 /* Contains aggregate functions */ +#define SF_UsesEphemeral 0x0000020 /* Uses the OpenEphemeral opcode */ +#define SF_Expanded 0x0000040 /* sqlite3SelectExpand() called on this */ +#define SF_HasTypeInfo 0x0000080 /* FROM subqueries have Table metadata */ +#define SF_Compound 0x0000100 /* Part of a compound query */ +#define SF_Values 0x0000200 /* Synthesized from VALUES clause */ +#define SF_MultiValue 0x0000400 /* Single VALUES term with multiple rows */ +#define SF_NestedFrom 0x0000800 /* Part of a parenthesized FROM clause */ +#define SF_MinMaxAgg 0x0001000 /* Aggregate containing min() or max() */ +#define SF_Recursive 0x0002000 /* The recursive part of a recursive CTE */ +#define SF_FixedLimit 0x0004000 /* nSelectRow set by a constant LIMIT */ +#define SF_MaybeConvert 0x0008000 /* Need convertCompoundSelectToSubquery() */ +#define SF_Converted 0x0010000 /* By convertCompoundSelectToSubquery() */ +#define SF_IncludeHidden 0x0020000 /* Include hidden columns in output */ +#define SF_ComplexResult 0x0040000 /* Result contains subquery or function */ +#define SF_WhereBegin 0x0080000 /* Really a WhereBegin() call. Debug Only */ +#define SF_WinRewrite 0x0100000 /* Window function rewrite accomplished */ +#define SF_View 0x0200000 /* SELECT statement is a view */ +#define SF_NoopOrderBy 0x0400000 /* ORDER BY is ignored for this query */ +#define SF_UFSrcCheck 0x0800000 /* Check pSrc as required by UPDATE...FROM */ +#define SF_PushDown 0x1000000 /* Modified by WHERE-clause push-down opt */ +#define SF_MultiPart 0x2000000 /* Has multiple incompatible PARTITIONs */ +#define SF_CopyCte 0x4000000 /* SELECT statement is a copy of a CTE */ +#define SF_OrderByReqd 0x8000000 /* The ORDER BY clause may not be omitted */ +#define SF_UpdateFrom 0x10000000 /* Query originates with UPDATE FROM */ +#define SF_Correlated 0x20000000 /* True if references the outer context */ +/* True if SrcItem X is a subquery that has SF_NestedFrom */ +#define IsNestedFrom(X) \ + ((X)->fg.isSubquery && \ + ((X)->u4.pSubq->pSelect->selFlags&SF_NestedFrom)!=0) /* ** The results of a SELECT can be distributed in several ways, as defined @@ -15106,9 +19797,6 @@ struct Select { ** statements within triggers whose only purpose is ** the side-effects of functions. ** -** All of the above are free to ignore their ORDER BY clause. Those that -** follow must honor the ORDER BY clause. -** ** SRT_Output Generate a row of output (using the OP_ResultRow ** opcode) for each row in the result set. ** @@ -15120,7 +19808,11 @@ struct Select { ** SRT_Set The result must be a single column. Store each ** row of result as the key in table pDest->iSDParm. ** Apply the affinity pDest->affSdst before storing -** results. Used to implement "IN (SELECT ...)". +** results. if pDest->iSDParm2 is positive, then it is +** a register holding a Bloom filter for the IN operator +** that should be populated in addition to the +** pDest->iSDParm table. This SRT is used to +** implement "IN (SELECT ...)". ** ** SRT_EphemTab Create an temporary table pDest->iSDParm and store ** the result there. The cursor is left open after @@ -15152,18 +19844,31 @@ struct Select { ** SRT_DistQueue Store results in priority queue pDest->iSDParm only if ** the same record has never been stored before. The ** index at pDest->iSDParm+1 hold all prior stores. +** +** SRT_Upfrom Store results in the temporary table already opened by +** pDest->iSDParm. If (pDest->iSDParm<0), then the temp +** table is an intkey table - in this case the first +** column returned by the SELECT is used as the integer +** key. If (pDest->iSDParm>0), then the table is an index +** table. (pDest->iSDParm) is the number of key columns in +** each index record in this case. */ #define SRT_Union 1 /* Store result as keys in an index */ #define SRT_Except 2 /* Remove result from a UNION index */ #define SRT_Exists 3 /* Store 1 if the result is not empty */ #define SRT_Discard 4 /* Do not save the results anywhere */ -#define SRT_Fifo 5 /* Store result as data with an automatic rowid */ -#define SRT_DistFifo 6 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistFifo 5 /* Like SRT_Fifo, but unique results only */ +#define SRT_DistQueue 6 /* Like SRT_Queue, but unique results only */ + +/* The DISTINCT clause is ignored for all of the above. Not that +** IgnorableDistinct() implies IgnorableOrderby() */ +#define IgnorableDistinct(X) ((X->eDest)<=SRT_DistQueue) + #define SRT_Queue 7 /* Store result in an queue */ -#define SRT_DistQueue 8 /* Like SRT_Queue, but unique results only */ +#define SRT_Fifo 8 /* Store result as data with an automatic rowid */ /* The ORDER BY clause is ignored for all of the above */ -#define IgnorableOrderby(X) ((X->eDest)<=SRT_DistQueue) +#define IgnorableOrderby(X) ((X->eDest)<=SRT_Fifo) #define SRT_Output 9 /* Output each row of result */ #define SRT_Mem 10 /* Store result in a memory cell */ @@ -15171,17 +19876,19 @@ struct Select { #define SRT_EphemTab 12 /* Create transient tab and store like SRT_Table */ #define SRT_Coroutine 13 /* Generate a single row of result */ #define SRT_Table 14 /* Store result as data with an automatic rowid */ +#define SRT_Upfrom 15 /* Store result as data with rowid */ /* ** An instance of this object describes where to put of the results of ** a SELECT statement. */ struct SelectDest { - u8 eDest; /* How to dispose of the results. On of SRT_* above. */ - char affSdst; /* Affinity used when eDest==SRT_Set */ + u8 eDest; /* How to dispose of the results. One of SRT_* above. */ int iSDParm; /* A parameter used by the eDest disposal method */ + int iSDParm2; /* A second parameter for the eDest disposal method */ int iSdst; /* Base register where results are written */ int nSdst; /* Number of registers allocated */ + char *zAffSdst; /* Affinity used for SRT_Set */ ExprList *pOrderBy; /* Key columns for SRT_Queue and SRT_DistQueue */ }; @@ -15201,13 +19908,6 @@ struct AutoincInfo { int regCtr; /* Memory register holding the rowid counter */ }; -/* -** Size of the column cache -*/ -#ifndef SQLITE_N_COLCACHE -# define SQLITE_N_COLCACHE 10 -#endif - /* ** At least one instance of the following structure is created for each ** trigger that may be fired while parsing an INSERT, UPDATE or DELETE @@ -15247,12 +19947,46 @@ struct TriggerPrg { #else typedef unsigned int yDbMask; # define DbMaskTest(M,I) (((M)&(((yDbMask)1)<<(I)))!=0) -# define DbMaskZero(M) (M)=0 -# define DbMaskSet(M,I) (M)|=(((yDbMask)1)<<(I)) -# define DbMaskAllZero(M) (M)==0 -# define DbMaskNonZero(M) (M)!=0 +# define DbMaskZero(M) ((M)=0) +# define DbMaskSet(M,I) ((M)|=(((yDbMask)1)<<(I))) +# define DbMaskAllZero(M) ((M)==0) +# define DbMaskNonZero(M) ((M)!=0) #endif +/* +** For each index X that has as one of its arguments either an expression +** or the name of a virtual generated column, and if X is in scope such that +** the value of the expression can simply be read from the index, then +** there is an instance of this object on the Parse.pIdxExpr list. +** +** During code generation, while generating code to evaluate expressions, +** this list is consulted and if a matching expression is found, the value +** is read from the index rather than being recomputed. +*/ +struct IndexedExpr { + Expr *pExpr; /* The expression contained in the index */ + int iDataCur; /* The data cursor associated with the index */ + int iIdxCur; /* The index cursor */ + int iIdxCol; /* The index column that contains value of pExpr */ + u8 bMaybeNullRow; /* True if we need an OP_IfNullRow check */ + u8 aff; /* Affinity of the pExpr expression */ + IndexedExpr *pIENext; /* Next in a list of all indexed expressions */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + const char *zIdxName; /* Name of index, used only for bytecode comments */ +#endif +}; + +/* +** An instance of the ParseCleanup object specifies an operation that +** should be performed after parsing to deallocation resources obtained +** during the parse and which are no longer needed. +*/ +struct ParseCleanup { + ParseCleanup *pNext; /* Next cleanup task */ + void *pPtr; /* Pointer to object to deallocate */ + void (*xCleanup)(sqlite3*,void*); /* Deallocation routine */ +}; + /* ** An SQL parser context. A copy of this structure is passed through ** the parser and down into all the parser action routine in order to @@ -15283,107 +20017,145 @@ struct Parse { u8 hasCompound; /* Need to invoke convertCompoundSelectToSubquery() */ u8 okConstFactor; /* OK to factor out constants */ u8 disableLookaside; /* Number of times lookaside has been disabled */ - u8 nColCache; /* Number of entries in aColCache[] */ - int aTempReg[8]; /* Holding area for temporary registers */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ + u8 withinRJSubrtn; /* Nesting level for RIGHT JOIN body subroutines */ + u8 bHasWith; /* True if statement contains WITH */ + u8 mSubrtnSig; /* mini Bloom filter on available SubrtnSig.selId */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + u8 earlyCleanup; /* OOM inside sqlite3ParserAddCleanup() */ +#endif +#ifdef SQLITE_DEBUG + u8 ifNotExists; /* Might be true if IF NOT EXISTS. Assert()s only */ +#endif int nRangeReg; /* Size of the temporary register block */ int iRangeReg; /* First register in temporary register block */ int nErr; /* Number of errors seen */ int nTab; /* Number of previously allocated VDBE cursors */ int nMem; /* Number of memory cells used so far */ - int nSet; /* Number of sets used so far */ - int nOnce; /* Number of OP_Once instructions so far */ - int nOpAlloc; /* Number of slots allocated for Vdbe.aOp[] */ int szOpAlloc; /* Bytes of memory space allocated for Vdbe.aOp[] */ - int iFixedOp; /* Never back out opcodes iFixedOp-1 or earlier */ - int ckBase; /* Base register of data during check constraints */ - int iSelfTab; /* Table of an index whose exprs are being coded */ - int iCacheLevel; /* ColCache valid when aColCache[].iLevel<=iCacheLevel */ - int iCacheCnt; /* Counter used to generate aColCache[].lru values */ - int nLabel; /* Number of labels used */ + int iSelfTab; /* Table associated with an index on expr, or negative + ** of the base register during check-constraint eval */ + int nLabel; /* The *negative* of the number of labels used */ + int nLabelAlloc; /* Number of slots in aLabel */ int *aLabel; /* Space to hold the labels */ - struct yColCache { - int iTable; /* Table cursor number */ - i16 iColumn; /* Table column number */ - u8 tempReg; /* iReg is a temp register that needs to be freed */ - int iLevel; /* Nesting level */ - int iReg; /* Reg with value of this column. 0 means none. */ - int lru; /* Least recently used entry has the smallest value */ - } aColCache[SQLITE_N_COLCACHE]; /* One for each column cache entry */ ExprList *pConstExpr;/* Constant expressions */ + IndexedExpr *pIdxEpr;/* List of expressions used by active indexes */ + IndexedExpr *pIdxPartExpr; /* Exprs constrained by index WHERE clauses */ Token constraintName;/* Name of the constraint currently being parsed */ yDbMask writeMask; /* Start a write transaction on these databases */ yDbMask cookieMask; /* Bitmask of schema verified databases */ - int cookieValue[SQLITE_MAX_ATTACHED+2]; /* Values of cookies to verify */ int regRowid; /* Register holding rowid of CREATE TABLE entry */ int regRoot; /* Register holding root page number for new objects */ int nMaxArg; /* Max args passed to user function by sub-program */ -#if SELECTTRACE_ENABLED - int nSelect; /* Number of SELECT statements seen */ - int nSelectIndent; /* How far to indent SELECTTRACE() output */ + int nSelect; /* Number of SELECT stmts. Counter for Select.selId */ +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + u32 nProgressSteps; /* xProgress steps taken during sqlite3_prepare() */ #endif #ifndef SQLITE_OMIT_SHARED_CACHE int nTableLock; /* Number of locks in aTableLock */ TableLock *aTableLock; /* Required table locks for shared-cache mode */ #endif AutoincInfo *pAinc; /* Information about AUTOINCREMENT counters */ - - /* Information used while coding trigger programs. */ Parse *pToplevel; /* Parse structure for main program (or NULL) */ Table *pTriggerTab; /* Table triggers are being coded for */ - int addrCrTab; /* Address of OP_CreateTable opcode on CREATE TABLE */ - u32 nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ + TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ + ParseCleanup *pCleanup; /* List of cleanup operations to run after parse */ + union { + int addrCrTab; /* Address of OP_CreateBtree on CREATE TABLE */ + Returning *pReturning; /* The RETURNING clause */ + } u1; u32 oldmask; /* Mask of old.* columns referenced */ u32 newmask; /* Mask of new.* columns referenced */ + LogEst nQueryLoop; /* Est number of iterations of a query (10*log2(N)) */ u8 eTriggerOp; /* TK_UPDATE, TK_INSERT or TK_DELETE */ + u8 bReturning; /* Coding a RETURNING trigger */ u8 eOrconf; /* Default ON CONFLICT policy for trigger steps */ u8 disableTriggers; /* True to disable triggers */ + /************************************************************************** + ** Fields above must be initialized to zero. The fields that follow, + ** down to the beginning of the recursive section, do not need to be + ** initialized as they will be set before being used. The boundary is + ** determined by offsetof(Parse,aTempReg). + **************************************************************************/ + + int aTempReg[8]; /* Holding area for temporary registers */ + Parse *pOuterParse; /* Outer Parse object when nested */ + Token sNameToken; /* Token with unqualified schema object name */ + /************************************************************************ ** Above is constant between recursions. Below is reset before and after ** each recursion. The boundary between these two regions is determined - ** using offsetof(Parse,nVar) so the nVar field must be the first field - ** in the recursive region. + ** using offsetof(Parse,sLastToken) so the sLastToken field must be the + ** first field in the recursive region. ************************************************************************/ + Token sLastToken; /* The last token parsed */ ynVar nVar; /* Number of '?' variables seen in the SQL so far */ - int nzVar; /* Number of available slots in azVar[] */ u8 iPkSortOrder; /* ASC or DESC for INTEGER PRIMARY KEY */ u8 explain; /* True if the EXPLAIN flag is found on the query */ + u8 eParseMode; /* PARSE_MODE_XXX constant */ #ifndef SQLITE_OMIT_VIRTUALTABLE - u8 declareVtab; /* True if inside sqlite3_declare_vtab() */ int nVtabLock; /* Number of virtual tables to lock */ #endif - int nAlias; /* Number of aliased result set columns */ int nHeight; /* Expression tree height of current sub-select */ #ifndef SQLITE_OMIT_EXPLAIN - int iSelectId; /* ID of current select for EXPLAIN output */ - int iNextSelectId; /* Next available select ID for EXPLAIN output */ + int addrExplain; /* Address of current OP_Explain opcode */ #endif - char **azVar; /* Pointers to names of parameters */ + VList *pVList; /* Mapping between variable names and numbers */ Vdbe *pReprepare; /* VM being reprepared (sqlite3Reprepare()) */ const char *zTail; /* All SQL text past the last semicolon parsed */ Table *pNewTable; /* A table being constructed by CREATE TABLE */ + Index *pNewIndex; /* An index being constructed by CREATE INDEX. + ** Also used to hold redundant UNIQUE constraints + ** during a RENAME COLUMN */ Trigger *pNewTrigger; /* Trigger under construct by a CREATE TRIGGER */ const char *zAuthContext; /* The 6th parameter to db->xAuth callbacks */ - Token sNameToken; /* Token with unqualified schema object name */ - Token sLastToken; /* The last token parsed */ #ifndef SQLITE_OMIT_VIRTUALTABLE Token sArg; /* Complete text of a module argument */ Table **apVtabLock; /* Pointer to virtual tables needing locking */ #endif - Table *pZombieTab; /* List of Table objects to delete after code gen */ - TriggerPrg *pTriggerPrg; /* Linked list of coded triggers */ With *pWith; /* Current WITH clause, or NULL */ - With *pWithToFree; /* Free this WITH object at the end of the parse */ +#ifndef SQLITE_OMIT_ALTERTABLE + RenameToken *pRename; /* Tokens subject to renaming by ALTER TABLE */ +#endif }; +/* Allowed values for Parse.eParseMode +*/ +#define PARSE_MODE_NORMAL 0 +#define PARSE_MODE_DECLARE_VTAB 1 +#define PARSE_MODE_RENAME 2 +#define PARSE_MODE_UNMAP 3 + +/* +** Sizes and pointers of various parts of the Parse object. +*/ +#define PARSE_HDR(X) (((char*)(X))+offsetof(Parse,zErrMsg)) +#define PARSE_HDR_SZ (offsetof(Parse,aTempReg)-offsetof(Parse,zErrMsg)) /* Recursive part w/o aColCache*/ +#define PARSE_RECURSE_SZ offsetof(Parse,sLastToken) /* Recursive part */ +#define PARSE_TAIL_SZ (sizeof(Parse)-PARSE_RECURSE_SZ) /* Non-recursive part */ +#define PARSE_TAIL(X) (((char*)(X))+PARSE_RECURSE_SZ) /* Pointer to tail */ + /* ** Return true if currently inside an sqlite3_declare_vtab() call. */ #ifdef SQLITE_OMIT_VIRTUALTABLE #define IN_DECLARE_VTAB 0 #else - #define IN_DECLARE_VTAB (pParse->declareVtab) + #define IN_DECLARE_VTAB (pParse->eParseMode==PARSE_MODE_DECLARE_VTAB) +#endif + +#if defined(SQLITE_OMIT_ALTERTABLE) + #define IN_RENAME_OBJECT 0 +#else + #define IN_RENAME_OBJECT (pParse->eParseMode>=PARSE_MODE_RENAME) +#endif + +#if defined(SQLITE_OMIT_VIRTUALTABLE) && defined(SQLITE_OMIT_ALTERTABLE) + #define IN_SPECIAL_PARSE 0 +#else + #define IN_SPECIAL_PARSE (pParse->eParseMode!=PARSE_MODE_NORMAL) #endif /* @@ -15409,44 +20181,47 @@ struct AuthContext { */ #define OPFLAG_NCHANGE 0x01 /* OP_Insert: Set to update db->nChange */ /* Also used in P2 (not P5) of OP_Delete */ +#define OPFLAG_NOCHNG 0x01 /* OP_VColumn nochange for UPDATE */ #define OPFLAG_EPHEM 0x01 /* OP_Column: Ephemeral output is ok */ -#define OPFLAG_LASTROWID 0x02 /* Set to update db->lastRowid */ +#define OPFLAG_LASTROWID 0x20 /* Set to update db->lastRowid */ #define OPFLAG_ISUPDATE 0x04 /* This OP_Insert is an sql UPDATE */ #define OPFLAG_APPEND 0x08 /* This is likely to be an append */ #define OPFLAG_USESEEKRESULT 0x10 /* Try to avoid a seek in BtreeInsert() */ -#ifdef SQLITE_ENABLE_PREUPDATE_HOOK #define OPFLAG_ISNOOP 0x40 /* OP_Delete does pre-update-hook only */ -#endif #define OPFLAG_LENGTHARG 0x40 /* OP_Column only used for length() */ #define OPFLAG_TYPEOFARG 0x80 /* OP_Column only used for typeof() */ +#define OPFLAG_BYTELENARG 0xc0 /* OP_Column only for octet_length() */ #define OPFLAG_BULKCSR 0x01 /* OP_Open** used to open bulk cursor */ #define OPFLAG_SEEKEQ 0x02 /* OP_Open** cursor uses EQ seek only */ #define OPFLAG_FORDELETE 0x08 /* OP_Open should use BTREE_FORDELETE */ #define OPFLAG_P2ISREG 0x10 /* P2 to OP_Open** is a register number */ #define OPFLAG_PERMUTE 0x01 /* OP_Compare: use the permutation */ -#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete: keep cursor position */ +#define OPFLAG_SAVEPOSITION 0x02 /* OP_Delete/Insert: save cursor pos */ #define OPFLAG_AUXDELETE 0x04 /* OP_Delete: index in a DELETE op */ +#define OPFLAG_NOCHNG_MAGIC 0x6d /* OP_MakeRecord: serialtype 10 is ok */ +#define OPFLAG_PREFORMAT 0x80 /* OP_Insert uses preformatted cell */ /* - * Each trigger present in the database schema is stored as an instance of - * struct Trigger. - * - * Pointers to instances of struct Trigger are stored in two ways. - * 1. In the "trigHash" hash table (part of the sqlite3* that represents the - * database). This allows Trigger structures to be retrieved by name. - * 2. All triggers associated with a single table form a linked list, using the - * pNext member of struct Trigger. A pointer to the first element of the - * linked list is stored as the "pTrigger" member of the associated - * struct Table. - * - * The "step_list" member points to the first element of a linked list - * containing the SQL statements specified as the trigger program. - */ +** Each trigger present in the database schema is stored as an instance of +** struct Trigger. +** +** Pointers to instances of struct Trigger are stored in two ways. +** 1. In the "trigHash" hash table (part of the sqlite3* that represents the +** database). This allows Trigger structures to be retrieved by name. +** 2. All triggers associated with a single table form a linked list, using the +** pNext member of struct Trigger. A pointer to the first element of the +** linked list is stored as the "pTrigger" member of the associated +** struct Table. +** +** The "step_list" member points to the first element of a linked list +** containing the SQL statements specified as the trigger program. +*/ struct Trigger { char *zName; /* The name of the trigger */ char *table; /* The table or view to which the trigger applies */ u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT */ u8 tr_tm; /* One of TRIGGER_BEFORE, TRIGGER_AFTER */ + u8 bReturning; /* This trigger implements a RETURNING clause */ Expr *pWhen; /* The WHEN clause of the expression (may be NULL) */ IdList *pColumns; /* If this is an UPDATE OF trigger, the is stored here */ @@ -15467,93 +20242,120 @@ struct Trigger { #define TRIGGER_AFTER 2 /* - * An instance of struct TriggerStep is used to store a single SQL statement - * that is a part of a trigger-program. - * - * Instances of struct TriggerStep are stored in a singly linked list (linked - * using the "pNext" member) referenced by the "step_list" member of the - * associated struct Trigger instance. The first element of the linked list is - * the first step of the trigger-program. - * - * The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or - * "SELECT" statement. The meanings of the other members is determined by the - * value of "op" as follows: - * - * (op == TK_INSERT) - * orconf -> stores the ON CONFLICT algorithm - * pSelect -> If this is an INSERT INTO ... SELECT ... statement, then - * this stores a pointer to the SELECT statement. Otherwise NULL. - * zTarget -> Dequoted name of the table to insert into. - * pExprList -> If this is an INSERT INTO ... VALUES ... statement, then - * this stores values to be inserted. Otherwise NULL. - * pIdList -> If this is an INSERT INTO ... () VALUES ... - * statement, then this stores the column-names to be - * inserted into. - * - * (op == TK_DELETE) - * zTarget -> Dequoted name of the table to delete from. - * pWhere -> The WHERE clause of the DELETE statement if one is specified. - * Otherwise NULL. - * - * (op == TK_UPDATE) - * zTarget -> Dequoted name of the table to update. - * pWhere -> The WHERE clause of the UPDATE statement if one is specified. - * Otherwise NULL. - * pExprList -> A list of the columns to update and the expressions to update - * them to. See sqlite3Update() documentation of "pChanges" - * argument. - * - */ +** An instance of struct TriggerStep is used to store a single SQL statement +** that is a part of a trigger-program. +** +** Instances of struct TriggerStep are stored in a singly linked list (linked +** using the "pNext" member) referenced by the "step_list" member of the +** associated struct Trigger instance. The first element of the linked list is +** the first step of the trigger-program. +** +** The "op" member indicates whether this is a "DELETE", "INSERT", "UPDATE" or +** "SELECT" statement. The meanings of the other members is determined by the +** value of "op" as follows: +** +** (op == TK_INSERT) +** orconf -> stores the ON CONFLICT algorithm +** pSelect -> The content to be inserted - either a SELECT statement or +** a VALUES clause. +** zTarget -> Dequoted name of the table to insert into. +** pIdList -> If this is an INSERT INTO ... () VALUES ... +** statement, then this stores the column-names to be +** inserted into. +** pUpsert -> The ON CONFLICT clauses for an Upsert +** +** (op == TK_DELETE) +** zTarget -> Dequoted name of the table to delete from. +** pWhere -> The WHERE clause of the DELETE statement if one is specified. +** Otherwise NULL. +** +** (op == TK_UPDATE) +** zTarget -> Dequoted name of the table to update. +** pWhere -> The WHERE clause of the UPDATE statement if one is specified. +** Otherwise NULL. +** pExprList -> A list of the columns to update and the expressions to update +** them to. See sqlite3Update() documentation of "pChanges" +** argument. +** +** (op == TK_SELECT) +** pSelect -> The SELECT statement +** +** (op == TK_RETURNING) +** pExprList -> The list of expressions that follow the RETURNING keyword. +** +*/ struct TriggerStep { - u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT */ + u8 op; /* One of TK_DELETE, TK_UPDATE, TK_INSERT, TK_SELECT, + ** or TK_RETURNING */ u8 orconf; /* OE_Rollback etc. */ Trigger *pTrig; /* The trigger that this step is a part of */ Select *pSelect; /* SELECT statement or RHS of INSERT INTO SELECT ... */ char *zTarget; /* Target table for DELETE, UPDATE, INSERT */ + SrcList *pFrom; /* FROM clause for UPDATE statement (if any) */ Expr *pWhere; /* The WHERE clause for DELETE or UPDATE steps */ - ExprList *pExprList; /* SET clause for UPDATE. */ + ExprList *pExprList; /* SET clause for UPDATE, or RETURNING clause */ IdList *pIdList; /* Column names for INSERT */ + Upsert *pUpsert; /* Upsert clauses on an INSERT */ + char *zSpan; /* Original SQL text of this command */ TriggerStep *pNext; /* Next in the link-list */ TriggerStep *pLast; /* Last element in link-list. Valid for 1st elem only */ }; /* -** The following structure contains information used by the sqliteFix... -** routines as they walk the parse tree to make database references -** explicit. +** Information about a RETURNING clause */ -typedef struct DbFixer DbFixer; -struct DbFixer { - Parse *pParse; /* The parsing context. Error messages written here */ - Schema *pSchema; /* Fix items to this schema */ - int bVarOnly; /* Check for variable references only */ - const char *zDb; /* Make sure all objects are contained in this database */ - const char *zType; /* Type of the container - used for error messages */ - const Token *pName; /* Name of the container - used for error messages */ +struct Returning { + Parse *pParse; /* The parse that includes the RETURNING clause */ + ExprList *pReturnEL; /* List of expressions to return */ + Trigger retTrig; /* The transient trigger that implements RETURNING */ + TriggerStep retTStep; /* The trigger step */ + int iRetCur; /* Transient table holding RETURNING results */ + int nRetCol; /* Number of in pReturnEL after expansion */ + int iRetReg; /* Register array for holding a row of RETURNING */ + char zName[40]; /* Name of trigger: "sqlite_returning_%p" */ }; /* -** An objected used to accumulate the text of a string where we +** An object used to accumulate the text of a string where we ** do not necessarily know how big the string will be in the end. */ -struct StrAccum { +struct sqlite3_str { sqlite3 *db; /* Optional database for lookaside. Can be NULL */ - char *zBase; /* A base allocation. Not from malloc. */ char *zText; /* The string collected so far */ - u32 nChar; /* Length of the string so far */ u32 nAlloc; /* Amount of space allocated in zText */ u32 mxAlloc; /* Maximum allowed allocation. 0 for no malloc usage */ - u8 accError; /* STRACCUM_NOMEM or STRACCUM_TOOBIG */ + u32 nChar; /* Length of the string so far */ + u8 accError; /* SQLITE_NOMEM or SQLITE_TOOBIG */ u8 printfFlags; /* SQLITE_PRINTF flags below */ }; -#define STRACCUM_NOMEM 1 -#define STRACCUM_TOOBIG 2 #define SQLITE_PRINTF_INTERNAL 0x01 /* Internal-use-only converters allowed */ #define SQLITE_PRINTF_SQLFUNC 0x02 /* SQL function arguments to VXPrintf */ -#define SQLITE_PRINTF_MALLOCED 0x04 /* True if xText is allocated space */ +#define SQLITE_PRINTF_MALLOCED 0x04 /* True if zText is allocated space */ #define isMalloced(X) (((X)->printfFlags & SQLITE_PRINTF_MALLOCED)!=0) +/* +** The following object is the header for an "RCStr" or "reference-counted +** string". An RCStr is passed around and used like any other char* +** that has been dynamically allocated. The important interface +** differences: +** +** 1. RCStr strings are reference counted. They are deallocated +** when the reference count reaches zero. +** +** 2. Use sqlite3RCStrUnref() to free an RCStr string rather than +** sqlite3_free() +** +** 3. Make a (read-only) copy of a read-only RCStr string using +** sqlite3RCStrRef(). +** +** "String" is in the name, but an RCStr object can also be used to hold +** binary data. +*/ +struct RCStr { + u64 nRCRef; /* Number of references */ + /* Total structure size should be a multiple of 8 bytes for alignment */ +}; /* ** A pointer to this structure is used to communicate information @@ -15564,8 +20366,35 @@ typedef struct { char **pzErrMsg; /* Error message stored here */ int iDb; /* 0 for main database. 1 for TEMP, 2.. for ATTACHed */ int rc; /* Result code stored here */ + u32 mInitFlags; /* Flags controlling error messages */ + u32 nInitRow; /* Number of rows processed */ + Pgno mxPage; /* Maximum page number. 0 for no limit. */ } InitData; +/* +** Allowed values for mInitFlags +*/ +#define INITFLAG_AlterMask 0x0003 /* Types of ALTER */ +#define INITFLAG_AlterRename 0x0001 /* Reparse after a RENAME */ +#define INITFLAG_AlterDrop 0x0002 /* Reparse after a DROP COLUMN */ +#define INITFLAG_AlterAdd 0x0003 /* Reparse after an ADD COLUMN */ + +/* Tuning parameters are set using SQLITE_TESTCTRL_TUNE and are controlled +** on debug-builds of the CLI using ".testctrl tune ID VALUE". Tuning +** parameters are for temporary use during development, to help find +** optimal values for parameters in the query planner. The should not +** be used on trunk check-ins. They are a temporary mechanism available +** for transient development builds only. +** +** Tuning parameters are numbered starting with 1. +*/ +#define SQLITE_NTUNE 6 /* Should be zero for all trunk check-ins */ +#ifdef SQLITE_DEBUG +# define Tuning(X) (sqlite3Config.aTune[(X)-1]) +#else +# define Tuning(X) 0 +#endif + /* ** Structure containing global configuration data for the SQLite library. ** @@ -15573,10 +20402,15 @@ typedef struct { */ struct Sqlite3Config { int bMemstat; /* True to enable memory status */ - int bCoreMutex; /* True to enable core mutexing */ - int bFullMutex; /* True to enable full mutexing */ - int bOpenUri; /* True to interpret filenames as URIs */ - int bUseCis; /* Use covering indices for full-scans */ + u8 bCoreMutex; /* True to enable core mutexing */ + u8 bFullMutex; /* True to enable full mutexing */ + u8 bOpenUri; /* True to interpret filenames as URIs */ + u8 bUseCis; /* Use covering indices for full-scans */ + u8 bSmallMalloc; /* Avoid large memory allocations if true */ + u8 bExtraSchemaChecks; /* Verify type,name,tbl_name in schema */ +#ifdef SQLITE_DEBUG + u8 bJsonSelfcheck; /* Double-check JSON parsing */ +#endif int mxStrlen; /* Maximum string length */ int neverCorrupt; /* Database is always well-formed */ int szLookaside; /* Default lookaside buffer size */ @@ -15590,9 +20424,6 @@ struct Sqlite3Config { int mnReq, mxReq; /* Min and max heap requests sizes */ sqlite3_int64 szMmap; /* mmap() space per open file */ sqlite3_int64 mxMmap; /* Maximum value for szMmap */ - void *pScratch; /* Scratch memory */ - int szScratch; /* Size of each scratch buffer */ - int nScratch; /* Number of scratch buffers */ void *pPage; /* Page cache memory */ int szPage; /* Size of each page in pPage[] */ int nPage; /* Number of pages in pPage[] */ @@ -15618,13 +20449,29 @@ struct Sqlite3Config { /* The following callback (if not NULL) is invoked on every VDBE branch ** operation. Set the callback using SQLITE_TESTCTRL_VDBE_COVERAGE. */ - void (*xVdbeBranch)(void*,int iSrcLine,u8 eThis,u8 eMx); /* Callback */ + void (*xVdbeBranch)(void*,unsigned iSrcLine,u8 eThis,u8 eMx); /* Callback */ void *pVdbeBranchArg; /* 1st argument */ #endif -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_OMIT_DESERIALIZE + sqlite3_int64 mxMemdbSize; /* Default max memdb size */ +#endif +#ifndef SQLITE_UNTESTABLE int (*xTestCallback)(int); /* Invoked by sqlite3FaultSim() */ +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + u32 mNoVisibleRowid; /* TF_NoVisibleRowid if the ROWID_IN_VIEW + ** feature is disabled. 0 if rowids can + ** occur in views. */ #endif int bLocaltimeFault; /* True to fail localtime() calls */ + int (*xAltLocaltime)(const void*,void*); /* Alternative localtime() routine */ + int iOnceResetThreshold; /* When to reset OP_Once counters */ + u32 szSorterRef; /* Min size in bytes to use sorter-refs */ + unsigned int iPrngSeed; /* Alternative fixed seed for the PRNG */ + /* vvvv--- must be last ---vvv */ +#ifdef SQLITE_DEBUG + sqlite3_int64 aTune[SQLITE_NTUNE]; /* Tuning parameters */ +#endif }; /* @@ -15654,25 +20501,68 @@ struct Walker { int (*xSelectCallback)(Walker*,Select*); /* Callback for SELECTs */ void (*xSelectCallback2)(Walker*,Select*);/* Second callback for SELECTs */ int walkerDepth; /* Number of subqueries */ - u8 eCode; /* A small processing code */ + u16 eCode; /* A small processing code */ + u16 mWFlags; /* Use-dependent flags */ union { /* Extra data for callback */ - NameContext *pNC; /* Naming context */ - int n; /* A counter */ - int iCur; /* A cursor number */ - SrcList *pSrcList; /* FROM clause */ - struct SrcCount *pSrcCount; /* Counting column references */ - struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ - int *aiCol; /* array of column indexes */ + NameContext *pNC; /* Naming context */ + int n; /* A counter */ + int iCur; /* A cursor number */ + SrcList *pSrcList; /* FROM clause */ + struct CCurHint *pCCurHint; /* Used by codeCursorHint() */ + struct RefSrcList *pRefSrcList; /* sqlite3ReferencesSrcList() */ + int *aiCol; /* array of column indexes */ + struct IdxCover *pIdxCover; /* Check for index coverage */ + ExprList *pGroupBy; /* GROUP BY clause */ + Select *pSelect; /* HAVING to WHERE clause ctx */ + struct WindowRewrite *pRewrite; /* Window rewrite context */ + struct WhereConst *pConst; /* WHERE clause constants */ + struct RenameCtx *pRename; /* RENAME COLUMN context */ + struct Table *pTab; /* Table of generated column */ + struct CoveringIndexCheck *pCovIdxCk; /* Check for covering index */ + SrcItem *pSrcItem; /* A single FROM clause item */ + DbFixer *pFix; /* See sqlite3FixSelect() */ + Mem *aMem; /* See sqlite3BtreeCursorHint() */ } u; }; +/* +** The following structure contains information used by the sqliteFix... +** routines as they walk the parse tree to make database references +** explicit. +*/ +struct DbFixer { + Parse *pParse; /* The parsing context. Error messages written here */ + Walker w; /* Walker object */ + Schema *pSchema; /* Fix items to this schema */ + u8 bTemp; /* True for TEMP schema entries */ + const char *zDb; /* Make sure all objects are contained in this database */ + const char *zType; /* Type of the container - used for error messages */ + const Token *pName; /* Name of the container - used for error messages */ +}; + /* Forward declarations */ SQLITE_PRIVATE int sqlite3WalkExpr(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3WalkExprNN(Walker*, Expr*); SQLITE_PRIVATE int sqlite3WalkExprList(Walker*, ExprList*); SQLITE_PRIVATE int sqlite3WalkSelect(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker*, Select*); SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker*, Select*); SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); +SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker*, Select*); +SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker*, Select*); +SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker*,Select*); +SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker*,Select*); +SQLITE_PRIVATE void sqlite3WalkWinDefnDummyCallback(Walker*,Select*); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker*, Select*); +#endif + +#ifndef SQLITE_OMIT_CTE +SQLITE_PRIVATE void sqlite3SelectPopWith(Walker*, Select*); +#else +# define sqlite3SelectPopWith 0 +#endif /* ** Return code from the parse-tree walking primitives and their @@ -15683,18 +20573,64 @@ SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker*, Expr*); #define WRC_Abort 2 /* Abandon the tree walk */ /* -** An instance of this structure represents a set of one or more CTEs -** (common table expressions) created by a single WITH clause. +** A single common table expression +*/ +struct Cte { + char *zName; /* Name of this CTE */ + ExprList *pCols; /* List of explicit column names, or NULL */ + Select *pSelect; /* The definition of this CTE */ + const char *zCteErr; /* Error message for circular references */ + CteUse *pUse; /* Usage information for this CTE */ + u8 eM10d; /* The MATERIALIZED flag */ +}; + +/* +** Allowed values for the materialized flag (eM10d): +*/ +#define M10d_Yes 0 /* AS MATERIALIZED */ +#define M10d_Any 1 /* Not specified. Query planner's choice */ +#define M10d_No 2 /* AS NOT MATERIALIZED */ + +/* +** An instance of the With object represents a WITH clause containing +** one or more CTEs (common table expressions). */ struct With { - int nCte; /* Number of CTEs in the WITH clause */ - With *pOuter; /* Containing WITH clause, or NULL */ - struct Cte { /* For each CTE in the WITH clause.... */ - char *zName; /* Name of this CTE */ - ExprList *pCols; /* List of explicit column names, or NULL */ - Select *pSelect; /* The definition of this CTE */ - const char *zCteErr; /* Error message for circular references */ - } a[1]; + int nCte; /* Number of CTEs in the WITH clause */ + int bView; /* Belongs to the outermost Select of a view */ + With *pOuter; /* Containing WITH clause, or NULL */ + Cte a[1]; /* For each CTE in the WITH clause.... */ +}; + +/* +** The Cte object is not guaranteed to persist for the entire duration +** of code generation. (The query flattener or other parser tree +** edits might delete it.) The following object records information +** about each Common Table Expression that must be preserved for the +** duration of the parse. +** +** The CteUse objects are freed using sqlite3ParserAddCleanup() rather +** than sqlite3SelectDelete(), which is what enables them to persist +** until the end of code generation. +*/ +struct CteUse { + int nUse; /* Number of users of this CTE */ + int addrM9e; /* Start of subroutine to compute materialization */ + int regRtn; /* Return address register for addrM9e subroutine */ + int iCur; /* Ephemeral table holding the materialization */ + LogEst nRowEst; /* Estimated number of rows in the table */ + u8 eM10d; /* The MATERIALIZED flag */ +}; + + +/* Client data associated with sqlite3_set_clientdata() and +** sqlite3_get_clientdata(). +*/ +struct DbClientData { + DbClientData *pNext; /* Next in a linked list */ + void *pData; /* The data */ + void (*xDestructor)(void*); /* Destructor. Might be NULL */ + char zName[1]; /* Name of this client data. MUST BE LAST */ }; #ifdef SQLITE_DEBUG @@ -15708,6 +20644,87 @@ struct TreeView { }; #endif /* SQLITE_DEBUG */ +/* +** This object is used in various ways, most (but not all) related to window +** functions. +** +** (1) A single instance of this structure is attached to the +** the Expr.y.pWin field for each window function in an expression tree. +** This object holds the information contained in the OVER clause, +** plus additional fields used during code generation. +** +** (2) All window functions in a single SELECT form a linked-list +** attached to Select.pWin. The Window.pFunc and Window.pExpr +** fields point back to the expression that is the window function. +** +** (3) The terms of the WINDOW clause of a SELECT are instances of this +** object on a linked list attached to Select.pWinDefn. +** +** (4) For an aggregate function with a FILTER clause, an instance +** of this object is stored in Expr.y.pWin with eFrmType set to +** TK_FILTER. In this case the only field used is Window.pFilter. +** +** The uses (1) and (2) are really the same Window object that just happens +** to be accessible in two different ways. Use case (3) are separate objects. +*/ +struct Window { + char *zName; /* Name of window (may be NULL) */ + char *zBase; /* Name of base window for chaining (may be NULL) */ + ExprList *pPartition; /* PARTITION BY clause */ + ExprList *pOrderBy; /* ORDER BY clause */ + u8 eFrmType; /* TK_RANGE, TK_GROUPS, TK_ROWS, or 0 */ + u8 eStart; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 eEnd; /* UNBOUNDED, CURRENT, PRECEDING or FOLLOWING */ + u8 bImplicitFrame; /* True if frame was implicitly specified */ + u8 eExclude; /* TK_NO, TK_CURRENT, TK_TIES, TK_GROUP, or 0 */ + Expr *pStart; /* Expression for " PRECEDING" */ + Expr *pEnd; /* Expression for " FOLLOWING" */ + Window **ppThis; /* Pointer to this object in Select.pWin list */ + Window *pNextWin; /* Next window function belonging to this SELECT */ + Expr *pFilter; /* The FILTER expression */ + FuncDef *pWFunc; /* The function */ + int iEphCsr; /* Partition buffer or Peer buffer */ + int regAccum; /* Accumulator */ + int regResult; /* Interim result */ + int csrApp; /* Function cursor (used by min/max) */ + int regApp; /* Function register (also used by min/max) */ + int regPart; /* Array of registers for PARTITION BY values */ + Expr *pOwner; /* Expression object this window is attached to */ + int nBufferCol; /* Number of columns in buffer table */ + int iArgCol; /* Offset of first argument for this function */ + int regOne; /* Register containing constant value 1 */ + int regStartRowid; + int regEndRowid; + u8 bExprArgs; /* Defer evaluation of window function arguments + ** due to the SQLITE_SUBTYPE flag */ +}; + +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow); +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal); + +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3*, Window*); +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window*); +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowAlloc(Parse*, int, int, Expr*, int , Expr*, u8); +SQLITE_PRIVATE void sqlite3WindowAttach(Parse*, Expr*, Window*); +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin); +SQLITE_PRIVATE int sqlite3WindowCompare(const Parse*, const Window*, const Window*, int); +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse*, Select*); +SQLITE_PRIVATE void sqlite3WindowCodeStep(Parse*, Select*, WhereInfo*, int, int); +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse*, Select*); +SQLITE_PRIVATE void sqlite3WindowUpdate(Parse*, Window*, Window*, FuncDef*); +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p); +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p); +SQLITE_PRIVATE void sqlite3WindowFunctions(void); +SQLITE_PRIVATE void sqlite3WindowChain(Parse*, Window*, Window*); +SQLITE_PRIVATE Window *sqlite3WindowAssemble(Parse*, Window*, ExprList*, ExprList*, Token*); +#else +# define sqlite3WindowDelete(a,b) +# define sqlite3WindowFunctions() +# define sqlite3WindowAttach(a,b,c) +#endif + /* ** Assuming zIn points to the first byte of a UTF-8 character, ** advance zIn to point to the first byte of the next UTF-8 character. @@ -15725,6 +20742,7 @@ struct TreeView { ** using sqlite3_log(). The routines also provide a convenient place ** to set a debugger breakpoint. */ +SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType); SQLITE_PRIVATE int sqlite3CorruptError(int); SQLITE_PRIVATE int sqlite3MisuseError(int); SQLITE_PRIVATE int sqlite3CantopenError(int); @@ -15740,6 +20758,12 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define SQLITE_NOMEM_BKPT SQLITE_NOMEM # define SQLITE_IOERR_NOMEM_BKPT SQLITE_IOERR_NOMEM #endif +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +SQLITE_PRIVATE int sqlite3CorruptPgnoError(int,Pgno); +# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptPgnoError(__LINE__,(P)) +#else +# define SQLITE_CORRUPT_PGNO(P) sqlite3CorruptError(__LINE__) +#endif /* ** FTS3 and FTS4 both require virtual table support @@ -15758,15 +20782,6 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define SQLITE_ENABLE_FTS3 1 #endif -/* -** The ctype.h header is needed for non-ASCII systems. It is also -** needed by FTS3 when FTS3 is included in the amalgamation. -*/ -#if !defined(SQLITE_ASCII) || \ - (defined(SQLITE_ENABLE_FTS3) && defined(SQLITE_AMALGAMATION)) -# include -#endif - /* ** The following macros mimic the standard library functions toupper(), ** isspace(), isalnum(), isdigit() and isxdigit(), respectively. The @@ -15781,6 +20796,8 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define sqlite3Isxdigit(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x08) # define sqlite3Tolower(x) (sqlite3UpperToLower[(unsigned char)(x)]) # define sqlite3Isquote(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x80) +# define sqlite3JsonId1(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x42) +# define sqlite3JsonId2(x) (sqlite3CtypeMap[(unsigned char)(x)]&0x46) #else # define sqlite3Toupper(x) toupper((unsigned char)(x)) # define sqlite3Isspace(x) isspace((unsigned char)(x)) @@ -15790,16 +20807,17 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int); # define sqlite3Isxdigit(x) isxdigit((unsigned char)(x)) # define sqlite3Tolower(x) tolower((unsigned char)(x)) # define sqlite3Isquote(x) ((x)=='"'||(x)=='\''||(x)=='['||(x)=='`') +# define sqlite3JsonId1(x) (sqlite3IsIdChar(x)&&(x)<'0') +# define sqlite3JsonId2(x) sqlite3IsIdChar(x) #endif -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS SQLITE_PRIVATE int sqlite3IsIdChar(u8); -#endif /* ** Internal function prototypes */ SQLITE_PRIVATE int sqlite3StrICmp(const char*,const char*); SQLITE_PRIVATE int sqlite3Strlen30(const char*); +#define sqlite3Strlen30NN(C) (strlen(C)&0x3fffffff) SQLITE_PRIVATE char *sqlite3ColumnType(Column*,char*); #define sqlite3StrNICmp sqlite3_strnicmp @@ -15812,18 +20830,19 @@ SQLITE_PRIVATE void *sqlite3DbMallocRaw(sqlite3*, u64); SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3*, u64); SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3*,const char*); SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3*,const char*, u64); +SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3*,const char*,const char*); SQLITE_PRIVATE void *sqlite3Realloc(void*, u64); SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *, void *, u64); SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *, void *, u64); SQLITE_PRIVATE void sqlite3DbFree(sqlite3*, void*); -SQLITE_PRIVATE int sqlite3MallocSize(void*); -SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, void*); -SQLITE_PRIVATE void *sqlite3ScratchMalloc(int); -SQLITE_PRIVATE void sqlite3ScratchFree(void*); +SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3*, void*); +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3*, void*); +SQLITE_PRIVATE int sqlite3MallocSize(const void*); +SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3*, const void*); SQLITE_PRIVATE void *sqlite3PageMalloc(int); SQLITE_PRIVATE void sqlite3PageFree(void*); SQLITE_PRIVATE void sqlite3MemSetDefault(void); -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE void sqlite3BenignMallocHooks(void (*)(void), void (*)(void)); #endif SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); @@ -15838,19 +20857,25 @@ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void); */ #ifdef SQLITE_USE_ALLOCA # define sqlite3StackAllocRaw(D,N) alloca(N) -# define sqlite3StackAllocZero(D,N) memset(alloca(N), 0, N) +# define sqlite3StackAllocRawNN(D,N) alloca(N) # define sqlite3StackFree(D,P) +# define sqlite3StackFreeNN(D,P) #else # define sqlite3StackAllocRaw(D,N) sqlite3DbMallocRaw(D,N) -# define sqlite3StackAllocZero(D,N) sqlite3DbMallocZero(D,N) +# define sqlite3StackAllocRawNN(D,N) sqlite3DbMallocRawNN(D,N) # define sqlite3StackFree(D,P) sqlite3DbFree(D,P) +# define sqlite3StackFreeNN(D,P) sqlite3DbFreeNN(D,P) #endif -#ifdef SQLITE_ENABLE_MEMSYS3 -SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); -#endif +/* Do not allow both MEMSYS5 and MEMSYS3 to be defined together. If they +** are, disable MEMSYS3 +*/ #ifdef SQLITE_ENABLE_MEMSYS5 SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys5(void); +#undef SQLITE_ENABLE_MEMSYS3 +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 +SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void); #endif @@ -15871,15 +20896,29 @@ SQLITE_PRIVATE sqlite3_int64 sqlite3StatusValue(int); SQLITE_PRIVATE void sqlite3StatusUp(int, int); SQLITE_PRIVATE void sqlite3StatusDown(int, int); SQLITE_PRIVATE void sqlite3StatusHighwater(int, int); +SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3*,int*); /* Access to mutexes used by sqlite3_status() */ SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void); SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void); -#ifndef SQLITE_OMIT_FLOATING_POINT -SQLITE_PRIVATE int sqlite3IsNaN(double); +#if defined(SQLITE_ENABLE_MULTITHREADED_CHECKS) && !defined(SQLITE_MUTEX_OMIT) +SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex*); #else -# define sqlite3IsNaN(X) 0 +# define sqlite3MutexWarnOnContention(x) +#endif + +#ifndef SQLITE_OMIT_FLOATING_POINT +# define EXP754 (((u64)0x7ff)<<52) +# define MAN754 ((((u64)1)<<52)-1) +# define IsNaN(X) (((X)&EXP754)==EXP754 && ((X)&MAN754)!=0) +# define IsOvfl(X) (((X)&EXP754)==EXP754) +SQLITE_PRIVATE int sqlite3IsNaN(double); +SQLITE_PRIVATE int sqlite3IsOverflow(double); +#else +# define IsNaN(X) 0 +# define sqlite3IsNaN(X) 0 +# define sqlite3IsOVerflow(X) 0 #endif /* @@ -15892,8 +20931,20 @@ struct PrintfArguments { sqlite3_value **apArg; /* The argument values */ }; -SQLITE_PRIVATE void sqlite3VXPrintf(StrAccum*, const char*, va_list); -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum*, const char*, ...); +/* +** An instance of this object receives the decoding of a floating point +** value into an approximate decimal representation. +*/ +struct FpDecode { + char sign; /* '+' or '-' */ + char isSpecial; /* 1: Infinity 2: NaN */ + int n; /* Significant digits in the decode */ + int iDP; /* Location of the decimal point */ + char *z; /* Start of significant digits */ + char zBuf[24]; /* Storage for significant digits */ +}; + +SQLITE_PRIVATE void sqlite3FpDecode(FpDecode*,double,int,int); SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3*,const char*, ...); SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3*,const char*, va_list); #if defined(SQLITE_DEBUG) || defined(SQLITE_HAVE_OS_TRACE) @@ -15904,76 +20955,155 @@ SQLITE_PRIVATE void *sqlite3TestTextToPtr(const char*); #endif #if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView*, const char *zFormat, ...); SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView*, const Expr*, u8); +SQLITE_PRIVATE void sqlite3TreeViewBareExprList(TreeView*, const ExprList*, const char*); SQLITE_PRIVATE void sqlite3TreeViewExprList(TreeView*, const ExprList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewBareIdList(TreeView*, const IdList*, const char*); +SQLITE_PRIVATE void sqlite3TreeViewIdList(TreeView*, const IdList*, u8, const char*); +SQLITE_PRIVATE void sqlite3TreeViewColumnList(TreeView*, const Column*, int, u8); +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView*, const SrcList*); SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView*, const Select*, u8); SQLITE_PRIVATE void sqlite3TreeViewWith(TreeView*, const With*, u8); +SQLITE_PRIVATE void sqlite3TreeViewUpsert(TreeView*, const Upsert*, u8); +#if TREETRACE_ENABLED +SQLITE_PRIVATE void sqlite3TreeViewDelete(const With*, const SrcList*, const Expr*, + const ExprList*,const Expr*, const Trigger*); +SQLITE_PRIVATE void sqlite3TreeViewInsert(const With*, const SrcList*, + const IdList*, const Select*, const ExprList*, + int, const Upsert*, const Trigger*); +SQLITE_PRIVATE void sqlite3TreeViewUpdate(const With*, const SrcList*, const ExprList*, + const Expr*, int, const ExprList*, const Expr*, + const Upsert*, const Trigger*); +#endif +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3TreeViewTriggerStep(TreeView*, const TriggerStep*, u8, u8); +SQLITE_PRIVATE void sqlite3TreeViewTrigger(TreeView*, const Trigger*, u8, u8); +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView*, const Window*, u8); +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView*, const Window*, u8); +#endif +SQLITE_PRIVATE void sqlite3ShowExpr(const Expr*); +SQLITE_PRIVATE void sqlite3ShowExprList(const ExprList*); +SQLITE_PRIVATE void sqlite3ShowIdList(const IdList*); +SQLITE_PRIVATE void sqlite3ShowSrcList(const SrcList*); +SQLITE_PRIVATE void sqlite3ShowSelect(const Select*); +SQLITE_PRIVATE void sqlite3ShowWith(const With*); +SQLITE_PRIVATE void sqlite3ShowUpsert(const Upsert*); +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3ShowTriggerStep(const TriggerStep*); +SQLITE_PRIVATE void sqlite3ShowTriggerStepList(const TriggerStep*); +SQLITE_PRIVATE void sqlite3ShowTrigger(const Trigger*); +SQLITE_PRIVATE void sqlite3ShowTriggerList(const Trigger*); +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3ShowWindow(const Window*); +SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window*); +#endif #endif - SQLITE_PRIVATE void sqlite3SetString(char **, sqlite3*, const char*); +SQLITE_PRIVATE void sqlite3ProgressCheck(Parse*); SQLITE_PRIVATE void sqlite3ErrorMsg(Parse*, const char*, ...); +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3*,int); SQLITE_PRIVATE void sqlite3Dequote(char*); +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr*); +SQLITE_PRIVATE void sqlite3DequoteToken(Token*); +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse*, Expr*); SQLITE_PRIVATE void sqlite3TokenInit(Token*,char*); SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char*, int); -SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*, char **); +SQLITE_PRIVATE int sqlite3RunParser(Parse*, const char*); SQLITE_PRIVATE void sqlite3FinishCoding(Parse*); SQLITE_PRIVATE int sqlite3GetTempReg(Parse*); SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse*,int); SQLITE_PRIVATE int sqlite3GetTempRange(Parse*,int); SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse*,int,int); SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse*); +SQLITE_PRIVATE void sqlite3TouchRegister(Parse*,int); +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse*,int); +#endif #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse*,int,int); #endif SQLITE_PRIVATE Expr *sqlite3ExprAlloc(sqlite3*,int,const Token*,int); SQLITE_PRIVATE Expr *sqlite3Expr(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3ExprAttachSubtrees(sqlite3*,Expr*,Expr*,Expr*); -SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*, const Token*); +SQLITE_PRIVATE Expr *sqlite3PExpr(Parse*, int, Expr*, Expr*); SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse*, Expr*, Select*); -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3*,Expr*, Expr*); -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, Token*); -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse*,Expr*, Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse*,ExprList*, const Token*, int); +SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy(Parse*,Expr*,ExprList*); +SQLITE_PRIVATE void sqlite3ExprOrderByAggregateError(Parse*,Expr*); +SQLITE_PRIVATE void sqlite3ExprFunctionUsable(Parse*,const Expr*,const FuncDef*); +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse*, Expr*, u32); SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3*, Expr*); +SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3*,void*); +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse*, Expr*); SQLITE_PRIVATE ExprList *sqlite3ExprListAppend(Parse*,ExprList*,Expr*); -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int); -SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,Token*,int); -SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,ExprSpan*); +SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector(Parse*,ExprList*,IdList*,Expr*); +SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse*, int, ExprList*); +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList*,int,int); +SQLITE_PRIVATE void sqlite3ExprListSetName(Parse*,ExprList*,const Token*,int); +SQLITE_PRIVATE void sqlite3ExprListSetSpan(Parse*,ExprList*,const char*,const char*); SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3*, ExprList*); +SQLITE_PRIVATE void sqlite3ExprListDeleteGeneric(sqlite3*,void*); SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList*); +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index*); SQLITE_PRIVATE int sqlite3Init(sqlite3*, char**); SQLITE_PRIVATE int sqlite3InitCallback(void*, int, char**, char**); +SQLITE_PRIVATE int sqlite3InitOne(sqlite3*, int, char**, u32); SQLITE_PRIVATE void sqlite3Pragma(Parse*,Token*,Token*,Token*,int); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3*,const char *zName); +#endif SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3*); SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3*,int); SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3*); SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3*); +SQLITE_PRIVATE void sqlite3ColumnSetExpr(Parse*,Table*,Column*,Expr*); +SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table*,Column*); +SQLITE_PRIVATE void sqlite3ColumnSetColl(sqlite3*,Column*,const char*zColl); +SQLITE_PRIVATE const char *sqlite3ColumnColl(Column*); SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3*,Table*); +SQLITE_PRIVATE void sqlite3GenerateColumnNames(Parse *pParse, Select *pSelect); SQLITE_PRIVATE int sqlite3ColumnsFromExprList(Parse*,ExprList*,i16*,Column**); -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation(Parse*,Table*,Select*); -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*); -SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *, int); +SQLITE_PRIVATE void sqlite3SubqueryColumnTypes(Parse*,Table*,Select*,char); +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse*,Select*,char); +SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *, int); SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table*); -SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index*, i16); +SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index*, i16); +#ifdef SQLITE_OMIT_GENERATED_COLUMNS +# define sqlite3TableColumnToStorage(T,X) (X) /* No-op pass-through */ +# define sqlite3StorageColumnToTable(T,X) (X) /* No-op pass-through */ +#else +SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table*, i16); +SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table*, i16); +#endif SQLITE_PRIVATE void sqlite3StartTable(Parse*,Token*,Token*,int,int,int,int); #if SQLITE_ENABLE_HIDDEN_COLUMNS SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table*, Column*); #else # define sqlite3ColumnPropertiesFromName(T,C) /* no-op */ #endif -SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token*,Token*); +SQLITE_PRIVATE void sqlite3AddColumn(Parse*,Token,Token); SQLITE_PRIVATE void sqlite3AddNotNull(Parse*, int); SQLITE_PRIVATE void sqlite3AddPrimaryKey(Parse*, ExprList*, int, int, int); -SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*); -SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,ExprSpan*); +SQLITE_PRIVATE void sqlite3AddCheckConstraint(Parse*, Expr*, const char*, const char*); +SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse*,Expr*,const char*,const char*); SQLITE_PRIVATE void sqlite3AddCollateType(Parse*, Token*); -SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u8,Select*); +SQLITE_PRIVATE void sqlite3AddGenerated(Parse*,Expr*,Token*); +SQLITE_PRIVATE void sqlite3EndTable(Parse*,Token*,Token*,u32,Select*); +SQLITE_PRIVATE void sqlite3AddReturning(Parse*,ExprList*); SQLITE_PRIVATE int sqlite3ParseUri(const char*,const char*,unsigned int*, sqlite3_vfs**,char**,char **); +#define sqlite3CodecQueryParameters(A,B,C) 0 SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3*,const char*); -SQLITE_PRIVATE int sqlite3CodeOnce(Parse *); -#ifdef SQLITE_OMIT_BUILTIN_TEST +#ifdef SQLITE_UNTESTABLE # define sqlite3FaultSim(X) SQLITE_OK #else SQLITE_PRIVATE int sqlite3FaultSim(int); @@ -15986,12 +21116,13 @@ SQLITE_PRIVATE int sqlite3BitvecSet(Bitvec*, u32); SQLITE_PRIVATE void sqlite3BitvecClear(Bitvec*, u32, void*); SQLITE_PRIVATE void sqlite3BitvecDestroy(Bitvec*); SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec*); -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE int sqlite3BitvecBuiltinTest(int,int*); #endif -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*, void*, unsigned int); -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet*); +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3*); +SQLITE_PRIVATE void sqlite3RowSetDelete(void*); +SQLITE_PRIVATE void sqlite3RowSetClear(void*); SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet*, i64); SQLITE_PRIVATE int sqlite3RowSetTest(RowSet*, int iBatch, i64); SQLITE_PRIVATE int sqlite3RowSetNext(RowSet*, i64*); @@ -16010,6 +21141,8 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask); SQLITE_PRIVATE void sqlite3DropTable(Parse*, SrcList*, int, int); SQLITE_PRIVATE void sqlite3CodeDropTable(Parse*, Table*, int, int); SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3*, Table*); +SQLITE_PRIVATE void sqlite3DeleteTableGeneric(sqlite3*, void*); +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3*, Index*); #ifndef SQLITE_OMIT_AUTOINCREMENT SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse); SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); @@ -16017,42 +21150,56 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse); # define sqlite3AutoincrementBegin(X) # define sqlite3AutoincrementEnd(X) #endif -SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int); +SQLITE_PRIVATE void sqlite3Insert(Parse*, SrcList*, Select*, IdList*, int, Upsert*); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns(Parse*, int, Table*); +#endif SQLITE_PRIVATE void *sqlite3ArrayAllocate(sqlite3*,void*,int,int*,int*); -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3*, IdList*, Token*); +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse*, IdList*, Token*); SQLITE_PRIVATE int sqlite3IdListIndex(IdList*,const char*); -SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(sqlite3*, SrcList*, int, int); -SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(sqlite3*, SrcList*, Token*, Token*); +SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge(Parse*, SrcList*, int, int); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2); +SQLITE_PRIVATE SrcList *sqlite3SrcListAppend(Parse*, SrcList*, Token*, Token*); +SQLITE_PRIVATE void sqlite3SubqueryDelete(sqlite3*,Subquery*); +SQLITE_PRIVATE Select *sqlite3SubqueryDetach(sqlite3*,SrcItem*); +SQLITE_PRIVATE int sqlite3SrcItemAttachSubquery(Parse*, SrcItem*, Select*, int); SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm(Parse*, SrcList*, Token*, Token*, - Token*, Select*, Expr*, IdList*); + Token*, Select*, OnOrUsing*); SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *, SrcList *, Token *); SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse*, SrcList*, ExprList*); -SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, struct SrcList_item *); -SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList*); +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *, SrcItem *); +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(Parse*,SrcList*); SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse*, SrcList*); SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3*, IdList*); +SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3*, OnOrUsing*); SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3*, SrcList*); SQLITE_PRIVATE Index *sqlite3AllocateIndexObject(sqlite3*,i16,int,char**); -SQLITE_PRIVATE Index *sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, - Expr*, int, int); +SQLITE_PRIVATE void sqlite3CreateIndex(Parse*,Token*,Token*,SrcList*,ExprList*,int,Token*, + Expr*, int, int, u8); SQLITE_PRIVATE void sqlite3DropIndex(Parse*, SrcList*, int); SQLITE_PRIVATE int sqlite3Select(Parse*, Select*, SelectDest*); SQLITE_PRIVATE Select *sqlite3SelectNew(Parse*,ExprList*,SrcList*,Expr*,ExprList*, - Expr*,ExprList*,u32,Expr*,Expr*); + Expr*,ExprList*,u32,Expr*); SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3*, Select*); +SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3*,void*); SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse*, SrcList*); -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, int); +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse*, Table*, Trigger*); SQLITE_PRIVATE void sqlite3OpenTable(Parse*, int iCur, int iDb, Table*, int); #if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) -SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,Expr*,char*); +SQLITE_PRIVATE Expr *sqlite3LimitWhere(Parse*,SrcList*,Expr*,ExprList*,Expr*,char*); #endif -SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*); -SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*, Expr*, int); -SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*,ExprList*,u16,int); +SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe*,int,const char*); +SQLITE_PRIVATE void sqlite3DeleteFrom(Parse*, SrcList*, Expr*, ExprList*, Expr*); +SQLITE_PRIVATE void sqlite3Update(Parse*, SrcList*, ExprList*,Expr*,int,ExprList*,Expr*, + Upsert*); +SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin(Parse*,SrcList*,Expr*,ExprList*, + ExprList*,Select*,u16,int); SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo*); SQLITE_PRIVATE LogEst sqlite3WhereOutputRowCount(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo*); +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo*); +SQLITE_PRIVATE void sqlite3WhereMinMaxOptEarlyOut(Vdbe*,WhereInfo*); SQLITE_PRIVATE int sqlite3WhereIsSorted(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereContinueLabel(WhereInfo*); SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo*); @@ -16060,48 +21207,53 @@ SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo*, int*); #define ONEPASS_OFF 0 /* Use of ONEPASS not allowed */ #define ONEPASS_SINGLE 1 /* ONEPASS valid for a single row update */ #define ONEPASS_MULTI 2 /* ONEPASS is valid for multiple rows */ +SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo*); SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn(Parse*, Index*, int, int, int); SQLITE_PRIVATE int sqlite3ExprCodeGetColumn(Parse*, Table*, int, int, int, u8); -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg(Parse*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable(Vdbe*, Table*, int, int, int); SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse*, int, int, int); -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse*, int, int, int); -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse*); -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse*); -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse*, int, int); -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse*); -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse*, int, int); +SQLITE_PRIVATE void sqlite3ExprToRegister(Expr *pExpr, int iReg); SQLITE_PRIVATE void sqlite3ExprCode(Parse*, Expr*, int); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn(Parse*, Table*, Column*, int); +#endif SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse*, Expr*, int); SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse*, Expr*, int); -SQLITE_PRIVATE void sqlite3ExprCodeAtInit(Parse*, Expr*, int, u8); +SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse*, Expr*, int*); SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse*, Expr*, int); -SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse*, Expr*, int); SQLITE_PRIVATE int sqlite3ExprCodeExprList(Parse*, ExprList*, int, int, u8); #define SQLITE_ECEL_DUP 0x01 /* Deep, not shallow copies */ #define SQLITE_ECEL_FACTOR 0x02 /* Factor out constant terms */ #define SQLITE_ECEL_REF 0x04 /* Use ExprList.u.x.iOrderByCol */ +#define SQLITE_ECEL_OMITREF 0x08 /* Omit if ExprList.u.x.iOrderByCol */ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse*, Expr*, int, int); SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse*, Expr*, int, int); SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3*,const char*, const char*); -SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,int isView,const char*, const char*); -SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,int isView,struct SrcList_item *); +#define LOCATE_VIEW 0x01 +#define LOCATE_NOERR 0x02 +SQLITE_PRIVATE Table *sqlite3LocateTable(Parse*,u32 flags,const char*, const char*); +SQLITE_PRIVATE const char *sqlite3PreferredTableName(const char*); +SQLITE_PRIVATE Table *sqlite3LocateTableItem(Parse*,u32 flags,SrcItem *); SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3*,const char*, const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3*,int,const char*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3*,int,const char*); -SQLITE_PRIVATE void sqlite3Vacuum(Parse*); -SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*); -SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, Token*); -SQLITE_PRIVATE int sqlite3ExprCompare(Expr*, Expr*, int); -SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList*, ExprList*, int); -SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr*, Expr*, int); +SQLITE_PRIVATE void sqlite3Vacuum(Parse*,Token*,Expr*); +SQLITE_PRIVATE int sqlite3RunVacuum(char**, sqlite3*, int, sqlite3_value*); +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3*, const Token*); +SQLITE_PRIVATE int sqlite3ExprCompare(const Parse*,const Expr*,const Expr*, int); +SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr*,Expr*,int); +SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList*,const ExprList*, int); +SQLITE_PRIVATE int sqlite3ExprImpliesExpr(const Parse*,const Expr*,const Expr*, int); +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr*,int,int); +SQLITE_PRIVATE void sqlite3AggInfoPersistWalkerInit(Walker*,Parse*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext*, Expr*); SQLITE_PRIVATE void sqlite3ExprAnalyzeAggList(NameContext*,ExprList*); -SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr*, SrcList*); +SQLITE_PRIVATE int sqlite3ExprCoveredByIndex(Expr*, int iCur, Index *pIdx); +SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse*, Expr*, SrcList*); SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse*); -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE void sqlite3PrngSaveState(void); SQLITE_PRIVATE void sqlite3PrngRestoreState(void); #endif @@ -16109,29 +21261,38 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3*,int); SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse*, int); SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse*, const char *zDb); SQLITE_PRIVATE void sqlite3BeginTransaction(Parse*, int); -SQLITE_PRIVATE void sqlite3CommitTransaction(Parse*); -SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse*); +SQLITE_PRIVATE void sqlite3EndTransaction(Parse*,int); SQLITE_PRIVATE void sqlite3Savepoint(Parse*, int, Token*); SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *); SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3*); -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr*); -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr*); +SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char*); +SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr*); +SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr*); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse*,Expr*); SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr*, u8); -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr*,int); +SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse*, Expr*, ExprList*); +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint(Expr*,const SrcList*,int,int); #ifdef SQLITE_ENABLE_CURSOR_HINTS SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr*); #endif -SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr*, int*); +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr*, int*, Parse*); SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr*); SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr*, char); SQLITE_PRIVATE int sqlite3IsRowid(const char*); +SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab); SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse*,Table*,Trigger*,int,int,int,i16,u8,u8,u8,int); SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete(Parse*, Table*, int, int, int*, int); SQLITE_PRIVATE int sqlite3GenerateIndexKey(Parse*, Index*, int, int, int, int*,Index*,int); SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse*,int); +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn(Expr*,int*,int); SQLITE_PRIVATE void sqlite3GenerateConstraintChecks(Parse*,Table*,int*,int,int,int,int, - u8,u8,int,int*,int*); + u8,u8,int,int*,int*,Upsert*); +#ifdef SQLITE_ENABLE_NULL_TRIM +SQLITE_PRIVATE void sqlite3SetMakeRecordP5(Vdbe*,Table*); +#else +# define sqlite3SetMakeRecordP5(A,B) +#endif SQLITE_PRIVATE void sqlite3CompleteInsertion(Parse*,Table*,int,int,int,int*,int,int,int); SQLITE_PRIVATE int sqlite3OpenTableAndIndices(Parse*, Table*, int, u8, int, u8*, int*, int*); SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse*, int, int); @@ -16140,27 +21301,29 @@ SQLITE_PRIVATE void sqlite3MayAbort(Parse*); SQLITE_PRIVATE void sqlite3HaltConstraint(Parse*, int, int, char*, i8, u8); SQLITE_PRIVATE void sqlite3UniqueConstraint(Parse*, int, Index*); SQLITE_PRIVATE void sqlite3RowidConstraint(Parse*, int, Table*); -SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3*,Expr*,int); -SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,ExprList*,int); -SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,SrcList*,int); -SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,IdList*); -SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,Select*,int); -#if SELECTTRACE_ENABLED -SQLITE_PRIVATE void sqlite3SelectSetName(Select*,const char*); -#else -# define sqlite3SelectSetName(A,B) -#endif +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3*,const Expr*,int); +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3*,const ExprList*,int); +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3*,const SrcList*,int); +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3*,const IdList*); +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3*,const Select*,int); +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch(int,const char*); SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs(FuncDef*,int); SQLITE_PRIVATE FuncDef *sqlite3FindFunction(sqlite3*,const char*,int,u8,u8); +SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum*,sqlite3_value*); SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void); SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void); +SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void); SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3*); +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) +SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3*); +#endif SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3*); SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3*); SQLITE_PRIVATE void sqlite3ChangeCookie(Parse*, int); +SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p); #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) -SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, int); +SQLITE_PRIVATE void sqlite3MaterializeView(Parse*, Table*, Expr*, ExprList*,Expr*,int); #endif #ifndef SQLITE_OMIT_TRIGGER @@ -16176,14 +21339,19 @@ SQLITE_PRIVATE void sqlite3CodeRowTrigger(Parse*, Trigger *, int, ExprList*, i SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect(Parse *, Trigger *, Table *, int, int, int); void sqliteViewTriggers(Parse*, Table*, Expr*, int, ExprList*); SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3*, TriggerStep*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(sqlite3*,Token*, IdList*, - Select*,u8); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(sqlite3*,Token*,ExprList*, Expr*, u8); -SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(sqlite3*,Token*, Expr*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3*,Select*, + const char*,const char*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep(Parse*,Token*, IdList*, + Select*,u8,Upsert*, + const char*,const char*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep(Parse*,Token*,SrcList*,ExprList*, + Expr*, u8, const char*,const char*); +SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep(Parse*,Token*, Expr*, + const char*,const char*); SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3*, Trigger*); SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3*,int,const char*); SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Table*,int); +SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc(Parse*, TriggerStep*); # define sqlite3ParseToplevel(p) ((p)->pToplevel ? (p)->pToplevel : (p)) # define sqlite3IsToplevel(p) ((p)->pToplevel==0) #else @@ -16197,9 +21365,13 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask(Parse*,Trigger*,ExprList*,int,int,Tab # define sqlite3ParseToplevel(p) p # define sqlite3IsToplevel(p) 1 # define sqlite3TriggerColmask(A,B,C,D,E,F,G) 0 +# define sqlite3TriggerStepSrc(A,B) 0 #endif SQLITE_PRIVATE int sqlite3JoinType(Parse*, Token*, Token*, Token*); +SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol); +SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem*,int); +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr*,int,u32); SQLITE_PRIVATE void sqlite3CreateForeignKey(Parse*, ExprList*, Token*, ExprList*, int); SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse*, int); #ifndef SQLITE_OMIT_AUTHORIZATION @@ -16214,30 +21386,35 @@ SQLITE_PRIVATE int sqlite3AuthReadCol(Parse*, const char *, const char *, int) # define sqlite3AuthContextPush(a,b,c) # define sqlite3AuthContextPop(a) ((void)(a)) #endif +SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName); SQLITE_PRIVATE void sqlite3Attach(Parse*, Expr*, Expr*, Expr*); SQLITE_PRIVATE void sqlite3Detach(Parse*, Expr*); SQLITE_PRIVATE void sqlite3FixInit(DbFixer*, Parse*, int, const char*, const Token*); SQLITE_PRIVATE int sqlite3FixSrcList(DbFixer*, SrcList*); SQLITE_PRIVATE int sqlite3FixSelect(DbFixer*, Select*); SQLITE_PRIVATE int sqlite3FixExpr(DbFixer*, Expr*); -SQLITE_PRIVATE int sqlite3FixExprList(DbFixer*, ExprList*); SQLITE_PRIVATE int sqlite3FixTriggerStep(DbFixer*, TriggerStep*); + +SQLITE_PRIVATE int sqlite3RealSameAsInt(double,sqlite3_int64); +SQLITE_PRIVATE i64 sqlite3RealToI64(double); +SQLITE_PRIVATE int sqlite3Int64ToText(i64,char*); SQLITE_PRIVATE int sqlite3AtoF(const char *z, double*, int, u8); SQLITE_PRIVATE int sqlite3GetInt32(const char *, int*); +SQLITE_PRIVATE int sqlite3GetUInt32(const char*, u32*); SQLITE_PRIVATE int sqlite3Atoi(const char*); -SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nChar); +#ifndef SQLITE_OMIT_UTF16 +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *pData, int nByte, int nChar); +#endif SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *pData, int nByte); SQLITE_PRIVATE u32 sqlite3Utf8Read(const u8**); +SQLITE_PRIVATE int sqlite3Utf8ReadLimited(const u8*, int, u32*); SQLITE_PRIVATE LogEst sqlite3LogEst(u64); SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst,LogEst); -#ifndef SQLITE_OMIT_VIRTUALTABLE SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double); -#endif -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst); -#endif +SQLITE_PRIVATE VList *sqlite3VListAdd(sqlite3*,VList*,const char*,int,int); +SQLITE_PRIVATE const char *sqlite3VListNumToName(VList*,int); +SQLITE_PRIVATE int sqlite3VListNameToNum(VList*,const char*,int); /* ** Routines to read and write variable-length integers. These used to @@ -16256,6 +21433,8 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); */ #define getVarint32(A,B) \ (u8)((*(A)<(u8)0x80)?((B)=(u32)*(A)),1:sqlite3GetVarint32((A),(u32 *)&(B))) +#define getVarint32NR(A,B) \ + B=(u32)*(A);if(B>=0x80)sqlite3GetVarint32((A),(u32*)&(B)) #define putVarint32(A,B) \ (u8)(((u32)(B)<(u32)0x80)?(*(A)=(unsigned char)(B)),1:\ sqlite3PutVarint((A),(B))) @@ -16264,16 +21443,22 @@ SQLITE_PRIVATE int sqlite3VarintLen(u64 v); SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3*, Index*); +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3*,const Table*); SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe*, Table*, int); -SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2); -SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity); -SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr); +SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2); +SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity); +SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table*,int); +SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr); SQLITE_PRIVATE int sqlite3Atoi64(const char*, i64*, int, u8); SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char*, i64*); SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3*, int, const char*,...); SQLITE_PRIVATE void sqlite3Error(sqlite3*,int); +SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3*); SQLITE_PRIVATE void sqlite3SystemError(sqlite3*,int); +#if !defined(SQLITE_OMIT_BLOB_LITERAL) SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3*, const char *z, int n); +#endif SQLITE_PRIVATE u8 sqlite3HexToInt(int h); SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); @@ -16281,17 +21466,30 @@ SQLITE_PRIVATE int sqlite3TwoPartName(Parse *, Token *, Token *, Token **); SQLITE_PRIVATE const char *sqlite3ErrName(int); #endif +#ifndef SQLITE_OMIT_DESERIALIZE +SQLITE_PRIVATE int sqlite3MemdbInit(void); +SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs*); +#else +# define sqlite3IsMemdb(X) 0 +#endif + SQLITE_PRIVATE const char *sqlite3ErrStr(int); SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse); SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq(sqlite3*,u8 enc, const char*,int); +SQLITE_PRIVATE int sqlite3IsBinary(const CollSeq*); SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char*zName); -SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(Parse *pParse, Expr*, const Token*, int); -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse*,Expr*,const char*); +SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8); +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr); +SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse*,const Expr*,const Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken(const Parse *pParse, Expr*, const Token*, int); +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(const Parse*,Expr*,const char*); SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr*); +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr*); SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *, CollSeq *); -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *, const char *); -SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, int); +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3*); +SQLITE_PRIVATE int sqlite3CheckObjectName(Parse*, const char*,const char*,const char*); +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *, i64); SQLITE_PRIVATE int sqlite3AddInt64(i64*,i64); SQLITE_PRIVATE int sqlite3SubInt64(i64*,i64); SQLITE_PRIVATE int sqlite3MulInt64(i64*,i64); @@ -16304,48 +21502,78 @@ SQLITE_PRIVATE void sqlite3FileSuffix3(const char*, char*); SQLITE_PRIVATE u8 sqlite3GetBoolean(const char *z,u8); SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value*, u8); +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value*, void(*)(void*)); SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value*, u8); SQLITE_PRIVATE void sqlite3ValueSetStr(sqlite3_value*, int, const void *,u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value*); SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value*); +#ifndef SQLITE_UNTESTABLE +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context*); +#endif SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *); +#ifndef SQLITE_OMIT_UTF16 SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *, const void*, int, u8); -SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, Expr *, u8, u8, sqlite3_value **); +#endif +SQLITE_PRIVATE int sqlite3ValueFromExpr(sqlite3 *, const Expr *, u8, u8, sqlite3_value **); SQLITE_PRIVATE void sqlite3ValueApplyAffinity(sqlite3_value *, u8, u8); #ifndef SQLITE_AMALGAMATION SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[]; SQLITE_PRIVATE const char sqlite3StrBINARY[]; +SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[]; +SQLITE_PRIVATE const char sqlite3StdTypeAffinity[]; +SQLITE_PRIVATE const char *sqlite3StdType[]; SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[]; +SQLITE_PRIVATE const unsigned char *sqlite3aLTb; +SQLITE_PRIVATE const unsigned char *sqlite3aEQb; +SQLITE_PRIVATE const unsigned char *sqlite3aGTb; SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[]; -SQLITE_PRIVATE const Token sqlite3IntTokens[]; SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config; SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; #ifndef SQLITE_OMIT_WSD SQLITE_PRIVATE int sqlite3PendingByte; #endif +#endif /* SQLITE_AMALGAMATION */ +#ifdef VDBE_PROFILE +SQLITE_PRIVATE sqlite3_uint64 sqlite3NProfileCnt; #endif -SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, int, int); +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3*, int, Pgno, Pgno); SQLITE_PRIVATE void sqlite3Reindex(Parse*, Token*, Token*); SQLITE_PRIVATE void sqlite3AlterFunctions(void); SQLITE_PRIVATE void sqlite3AlterRenameTable(Parse*, SrcList*, Token*); +SQLITE_PRIVATE void sqlite3AlterRenameColumn(Parse*, SrcList*, Token*, Token*); SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *, int *); SQLITE_PRIVATE void sqlite3NestedParse(Parse*, const char*, ...); -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*); -SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *, Expr *, int, int); +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3*, int); +SQLITE_PRIVATE void sqlite3CodeRhsOfIN(Parse*, Expr*, int); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse*, Expr*); SQLITE_PRIVATE void sqlite3SelectPrep(Parse*, Select*, NameContext*); +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse*, SrcItem*); SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p); -SQLITE_PRIVATE int sqlite3MatchSpanName(const char*, const char*, const char*, const char*); +SQLITE_PRIVATE int sqlite3MatchEName( + const struct ExprList_item*, + const char*, + const char*, + const char*, + int* +); +SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr*); +SQLITE_PRIVATE u8 sqlite3StrIHash(const char*); SQLITE_PRIVATE int sqlite3ResolveExprNames(NameContext*, Expr*); SQLITE_PRIVATE int sqlite3ResolveExprListNames(NameContext*, ExprList*); SQLITE_PRIVATE void sqlite3ResolveSelectNames(Parse*, Select*, NameContext*); -SQLITE_PRIVATE void sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); +SQLITE_PRIVATE int sqlite3ResolveSelfReference(Parse*,Table*,int,Expr*,ExprList*); SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy(Parse*, Select*, ExprList*, const char*); SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *, Table *, int, int); SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *, Token *); SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *, SrcList *); +SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse*, SrcList*, const Token*); +SQLITE_PRIVATE const void *sqlite3RenameTokenMap(Parse*, const void*, const Token*); +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse*, const void *pTo, const void *pFrom); +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse*, Expr*); +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse*, ExprList*); SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq(Parse*, u8, CollSeq *, const char*); -SQLITE_PRIVATE char sqlite3AffinityType(const char*, u8*); +SQLITE_PRIVATE char sqlite3AffinityType(const char*, Column*); SQLITE_PRIVATE void sqlite3Analyze(Parse*, Token*, Token*); SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler*); SQLITE_PRIVATE int sqlite3FindDb(sqlite3*, Token*); @@ -16362,45 +21590,69 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3*,int,int); SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoRef(KeyInfo*); SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse*, Index*); +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList(Parse*, ExprList*, int, int); +SQLITE_PRIVATE const char *sqlite3SelectOpName(int); +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse*, ExprList*); + #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo*); #endif SQLITE_PRIVATE int sqlite3CreateFunc(sqlite3 *, const char *, int, int, void *, void (*)(sqlite3_context*,int,sqlite3_value **), - void (*)(sqlite3_context*,int,sqlite3_value **), void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*), + void (*)(sqlite3_context*,int,sqlite3_value **), FuncDestructor *pDestructor ); -SQLITE_PRIVATE void sqlite3OomFault(sqlite3*); +SQLITE_PRIVATE void sqlite3NoopDestructor(void*); +SQLITE_PRIVATE void *sqlite3OomFault(sqlite3*); SQLITE_PRIVATE void sqlite3OomClear(sqlite3*); SQLITE_PRIVATE int sqlite3ApiExit(sqlite3 *db, int); SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *); +SQLITE_PRIVATE char *sqlite3RCStrRef(char*); +SQLITE_PRIVATE void sqlite3RCStrUnref(void*); +SQLITE_PRIVATE char *sqlite3RCStrNew(u64); +SQLITE_PRIVATE char *sqlite3RCStrResize(char*,u64); + SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum*, sqlite3*, char*, int, int); -SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum*,const char*,int); -SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum*,const char*); -SQLITE_PRIVATE void sqlite3AppendChar(StrAccum*,int,char); +SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum*, i64); SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum*); -SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum*); +SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum*, u8); +SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context*,StrAccum*); SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest*,int,int); SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *, SrcList *, int, int); +SQLITE_PRIVATE void sqlite3RecordErrorByteOffset(sqlite3*,const char*); +SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3*,const Expr*); SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *); SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *, Pgno, const u8 *); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void); -SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue(Parse*,Index*,UnpackedRecord**,Expr*,u8,int,int*); +#ifndef SQLITE_OMIT_SUBQUERY +SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse*, Expr*); +#else +# define sqlite3ExprCheckIN(x,y) SQLITE_OK +#endif + +#ifdef SQLITE_ENABLE_STAT4 +SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( + Parse*,Index*,UnpackedRecord**,Expr*,int,int,int*); SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr(Parse*, Expr*, u8, sqlite3_value**); SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord*); SQLITE_PRIVATE int sqlite3Stat4Column(sqlite3*, const void*, int, int, sqlite3_value**); +SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3*, Index*, int); #endif /* ** The interface to the LEMON-generated parser */ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(u64)); -SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); -SQLITE_PRIVATE void sqlite3Parser(void*, int, Token, Parse*); +#ifndef SQLITE_AMALGAMATION +SQLITE_PRIVATE void *sqlite3ParserAlloc(void*(*)(u64), Parse*); +SQLITE_PRIVATE void sqlite3ParserFree(void*, void(*)(void*)); +#endif +SQLITE_PRIVATE void sqlite3Parser(void*, int, Token); +SQLITE_PRIVATE int sqlite3ParserFallback(int); #ifdef YYTRACKMAXSTACKDEPTH SQLITE_PRIVATE int sqlite3ParserStackPeak(void*); #endif @@ -16413,7 +21665,7 @@ SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3*); #endif #ifndef SQLITE_OMIT_SHARED_CACHE -SQLITE_PRIVATE void sqlite3TableLock(Parse *, int, int, u8, const char *); +SQLITE_PRIVATE void sqlite3TableLock(Parse *, int, Pgno, u8, const char *); #else #define sqlite3TableLock(v,w,x,y,z) #endif @@ -16423,13 +21675,14 @@ SQLITE_PRIVATE int sqlite3Utf8To8(unsigned char*); #endif #ifdef SQLITE_OMIT_VIRTUALTABLE -# define sqlite3VtabClear(Y) +# define sqlite3VtabClear(D,T) # define sqlite3VtabSync(X,Y) SQLITE_OK # define sqlite3VtabRollback(X) # define sqlite3VtabCommit(X) # define sqlite3VtabInSync(db) 0 # define sqlite3VtabLock(X) # define sqlite3VtabUnlock(X) +# define sqlite3VtabModuleUnref(D,X) # define sqlite3VtabUnlockList(X) # define sqlite3VtabSavepoint(X, Y, Z) SQLITE_OK # define sqlite3GetVTable(X,Y) ((VTable*)0) @@ -16441,12 +21694,30 @@ SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db); SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db); SQLITE_PRIVATE void sqlite3VtabLock(VTable *); SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *); +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3*); SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *, int, int); SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe*, sqlite3_vtab*); SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3*, Table*); +SQLITE_PRIVATE Module *sqlite3VtabCreateModule( + sqlite3*, + const char*, + const sqlite3_module*, + void*, + void(*)(void*) + ); # define sqlite3VtabInSync(db) ((db)->nVTrans>0 && (db)->aVTrans==0) #endif +SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db); +#ifndef SQLITE_OMIT_VIRTUALTABLE +SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName); +SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3*,Table*,const char*); +SQLITE_PRIVATE void sqlite3MarkAllShadowTablesOf(sqlite3*, Table*); +#else +# define sqlite3ShadowTableName(A,B) 0 +# define sqlite3IsShadowTableOf(A,B,C) 0 +# define sqlite3MarkAllShadowTablesOf(A,B) +#endif SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse*,Module*); SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3*,Module*); SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse*,Table*); @@ -16458,15 +21729,22 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3*, int, const char *, char **); SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse*, Table*); SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3*, int, const char *); SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *, VTable *); + SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction(sqlite3 *,FuncDef*, int nArg, Expr*); -SQLITE_PRIVATE void sqlite3InvalidFunction(sqlite3_context*,int,sqlite3_value**); +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse*); SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context*); SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe*, const char*, int); SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *, sqlite3_stmt *); -SQLITE_PRIVATE void sqlite3ParserReset(Parse*); +SQLITE_PRIVATE void sqlite3ParseObjectInit(Parse*,sqlite3*); +SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse*); +SQLITE_PRIVATE void *sqlite3ParserAddCleanup(Parse*,void(*)(sqlite3*,void*),void*); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_PRIVATE char *sqlite3Normalize(Vdbe*, const char*); +#endif SQLITE_PRIVATE int sqlite3Reprepare(Vdbe*); SQLITE_PRIVATE void sqlite3ExprListCheckLength(Parse*, ExprList*, const char*); -SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, Expr *, Expr *); +SQLITE_PRIVATE CollSeq *sqlite3ExprCompareCollSeq(Parse*,const Expr*); +SQLITE_PRIVATE CollSeq *sqlite3BinaryCompareCollSeq(Parse *, const Expr*, const Expr*); SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3*); SQLITE_PRIVATE const char *sqlite3JournalModename(int); #ifndef SQLITE_OMIT_WAL @@ -16474,13 +21752,35 @@ SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3*, int, int, int*, int*); SQLITE_PRIVATE int sqlite3WalDefaultHook(void*,sqlite3*,const char*,int); #endif #ifndef SQLITE_OMIT_CTE -SQLITE_PRIVATE With *sqlite3WithAdd(Parse*,With*,Token*,ExprList*,Select*); +SQLITE_PRIVATE Cte *sqlite3CteNew(Parse*,Token*,ExprList*,Select*,u8); +SQLITE_PRIVATE void sqlite3CteDelete(sqlite3*,Cte*); +SQLITE_PRIVATE With *sqlite3WithAdd(Parse*,With*,Cte*); SQLITE_PRIVATE void sqlite3WithDelete(sqlite3*,With*); -SQLITE_PRIVATE void sqlite3WithPush(Parse*, With*, u8); +SQLITE_PRIVATE void sqlite3WithDeleteGeneric(sqlite3*,void*); +SQLITE_PRIVATE With *sqlite3WithPush(Parse*, With*, u8); #else -#define sqlite3WithPush(x,y,z) -#define sqlite3WithDelete(x,y) +# define sqlite3CteNew(P,T,E,S) ((void*)0) +# define sqlite3CteDelete(D,C) +# define sqlite3CteWithAdd(P,W,C) ((void*)0) +# define sqlite3WithDelete(x,y) +# define sqlite3WithPush(x,y,z) ((void*)0) #endif +#ifndef SQLITE_OMIT_UPSERT +SQLITE_PRIVATE Upsert *sqlite3UpsertNew(sqlite3*,ExprList*,Expr*,ExprList*,Expr*,Upsert*); +SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3*,Upsert*); +SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3*,Upsert*); +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget(Parse*,SrcList*,Upsert*,Upsert*); +SQLITE_PRIVATE void sqlite3UpsertDoUpdate(Parse*,Upsert*,Table*,Index*,int); +SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert*,Index*); +SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert*); +#else +#define sqlite3UpsertNew(u,v,w,x,y,z) ((Upsert*)0) +#define sqlite3UpsertDelete(x,y) +#define sqlite3UpsertDup(x,y) ((Upsert*)0) +#define sqlite3UpsertOfIndex(x,y) ((Upsert*)0) +#define sqlite3UpsertNextIsIPK(x) 0 +#endif + /* Declarations for functions in fkey.c. All of these are replaced by ** no-op macros if OMIT_FOREIGN_KEY is defined. In this case no foreign @@ -16496,12 +21796,15 @@ SQLITE_PRIVATE void sqlite3FkActions(Parse*, Table*, ExprList*, int, int*, int SQLITE_PRIVATE int sqlite3FkRequired(Parse*, Table*, int*, int); SQLITE_PRIVATE u32 sqlite3FkOldmask(Parse*, Table*); SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *); +SQLITE_PRIVATE void sqlite3FkClearTriggerCache(sqlite3*,int); #else #define sqlite3FkActions(a,b,c,d,e,f) #define sqlite3FkCheck(a,b,c,d,e,f) #define sqlite3FkDropTable(a,b,c) #define sqlite3FkOldmask(a,b) 0 #define sqlite3FkRequired(a,b,c,d) 0 + #define sqlite3FkReferences(a) 0 + #define sqlite3FkClearTriggerCache(a,b) #endif #ifndef SQLITE_OMIT_FOREIGN_KEY SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *, Table*); @@ -16520,10 +21823,10 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex(Parse*,Table*,FKey*,Index**,int**); /* ** The interface to the code in fault.c used for identifying "benign" -** malloc failures. This is only present if SQLITE_OMIT_BUILTIN_TEST +** malloc failures. This is only present if SQLITE_UNTESTABLE ** is not defined. */ -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE void sqlite3BeginBenignMalloc(void); SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); #else @@ -16545,11 +21848,12 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void); #define IN_INDEX_NOOP_OK 0x0001 /* OK to return IN_INDEX_NOOP */ #define IN_INDEX_MEMBERSHIP 0x0002 /* IN operator used for membership test */ #define IN_INDEX_LOOP 0x0004 /* IN operator used as a loop */ -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*); +SQLITE_PRIVATE int sqlite3FindInIndex(Parse *, Expr *, u32, int*, int*, int*); SQLITE_PRIVATE int sqlite3JournalOpen(sqlite3_vfs *, const char *, sqlite3_file *, int, int); SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *); #endif @@ -16558,12 +21862,13 @@ SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *); SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p); #if SQLITE_MAX_EXPR_DEPTH>0 -SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *); +SQLITE_PRIVATE int sqlite3SelectExprHeight(const Select *); SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse*, int); #else #define sqlite3SelectExprHeight(x) 0 #define sqlite3ExprCheckHeight(x,y) #endif +SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr*,int); SQLITE_PRIVATE u32 sqlite3Get4byte(const u8*); SQLITE_PRIVATE void sqlite3Put4byte(u8*, u32); @@ -16581,6 +21886,9 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db); #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3ParserTrace(FILE*, char *); #endif +#if defined(YYCOVERAGE) +SQLITE_PRIVATE int sqlite3ParserCoverage(FILE*); +#endif /* ** If the SQLITE_ENABLE IOTRACE exists then the global variable @@ -16626,8 +21934,8 @@ SQLITE_API SQLITE_EXTERN void (SQLITE_CDECL *sqlite3IoTrace)(const char*,...); */ #ifdef SQLITE_MEMDEBUG SQLITE_PRIVATE void sqlite3MemdebugSetType(void*,u8); -SQLITE_PRIVATE int sqlite3MemdebugHasType(void*,u8); -SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugHasType(const void*,u8); +SQLITE_PRIVATE int sqlite3MemdebugNoType(const void*,u8); #else # define sqlite3MemdebugSetType(X,Y) /* no-op */ # define sqlite3MemdebugHasType(X,Y) 1 @@ -16635,8 +21943,7 @@ SQLITE_PRIVATE int sqlite3MemdebugNoType(void*,u8); #endif #define MEMTYPE_HEAP 0x01 /* General heap allocations */ #define MEMTYPE_LOOKASIDE 0x02 /* Heap that might have been lookaside */ -#define MEMTYPE_SCRATCH 0x04 /* Scratch allocations */ -#define MEMTYPE_PCACHE 0x08 /* Page cache allocations */ +#define MEMTYPE_PCACHE 0x04 /* Page cache allocations */ /* ** Threading interface @@ -16646,13 +21953,946 @@ SQLITE_PRIVATE int sqlite3ThreadCreate(SQLiteThread**,void*(*)(void*),void*); SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread*, void**); #endif +#if defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST) +SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3*); +#endif #if defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST) SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); #endif -#endif /* _SQLITEINT_H_ */ +SQLITE_PRIVATE int sqlite3ExprVectorSize(const Expr *pExpr); +SQLITE_PRIVATE int sqlite3ExprIsVector(const Expr *pExpr); +SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr*, int); +SQLITE_PRIVATE Expr *sqlite3ExprForVectorField(Parse*,Expr*,int,int); +SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse*, Expr*); + +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt); +#endif + +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void); +#endif + +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +SQLITE_PRIVATE sqlite3_uint64 sqlite3Hwtime(void); +#endif + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +# define IS_STMT_SCANSTATUS(db) (db->flags & SQLITE_StmtScanStatus) +#else +# define IS_STMT_SCANSTATUS(db) 0 +#endif + +#endif /* SQLITEINT_H */ /************** End of sqliteInt.h *******************************************/ +/************** Begin file os_common.h ***************************************/ +/* +** 2004 May 22 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains macros and a little bit of code that is common to +** all of the platform-specific files (os_*.c) and is #included into those +** files. +** +** This file should be #included by the os_*.c files only. It is not a +** general purpose header file. +*/ +#ifndef _OS_COMMON_H_ +#define _OS_COMMON_H_ + +/* +** At least two bugs have slipped in because we changed the MEMORY_DEBUG +** macro to SQLITE_DEBUG and some older makefiles have not yet made the +** switch. The following code should catch this problem at compile-time. +*/ +#ifdef MEMORY_DEBUG +# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." +#endif + +/* +** Macros for performance tracing. Normally turned off. Only works +** on i486 hardware. +*/ +#ifdef SQLITE_PERFORMANCE_TRACE + +static sqlite_uint64 g_start; +static sqlite_uint64 g_elapsed; +#define TIMER_START g_start=sqlite3Hwtime() +#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start +#define TIMER_ELAPSED g_elapsed +#else +#define TIMER_START +#define TIMER_END +#define TIMER_ELAPSED ((sqlite_uint64)0) +#endif + +/* +** If we compile with the SQLITE_TEST macro set, then the following block +** of code will give us the ability to simulate a disk I/O error. This +** is used for testing the I/O recovery logic. +*/ +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_io_error_hit; +SQLITE_API extern int sqlite3_io_error_hardhit; +SQLITE_API extern int sqlite3_io_error_pending; +SQLITE_API extern int sqlite3_io_error_persist; +SQLITE_API extern int sqlite3_io_error_benign; +SQLITE_API extern int sqlite3_diskfull_pending; +SQLITE_API extern int sqlite3_diskfull; +#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) +#define SimulateIOError(CODE) \ + if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ + || sqlite3_io_error_pending-- == 1 ) \ + { local_ioerr(); CODE; } +static void local_ioerr(){ + IOTRACE(("IOERR\n")); + sqlite3_io_error_hit++; + if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; +} +#define SimulateDiskfullError(CODE) \ + if( sqlite3_diskfull_pending ){ \ + if( sqlite3_diskfull_pending == 1 ){ \ + local_ioerr(); \ + sqlite3_diskfull = 1; \ + sqlite3_io_error_hit = 1; \ + CODE; \ + }else{ \ + sqlite3_diskfull_pending--; \ + } \ + } +#else +#define SimulateIOErrorBenign(X) +#define SimulateIOError(A) +#define SimulateDiskfullError(A) +#endif /* defined(SQLITE_TEST) */ + +/* +** When testing, keep a count of the number of open files. +*/ +#if defined(SQLITE_TEST) +SQLITE_API extern int sqlite3_open_file_count; +#define OpenCounter(X) sqlite3_open_file_count+=(X) +#else +#define OpenCounter(X) +#endif /* defined(SQLITE_TEST) */ + +#endif /* !defined(_OS_COMMON_H_) */ + +/************** End of os_common.h *******************************************/ +/************** Begin file ctime.c *******************************************/ +/* DO NOT EDIT! +** This file is automatically generated by the script in the canonical +** SQLite source tree at tool/mkctimec.tcl. +** +** To modify this header, edit any of the various lists in that script +** which specify categories of generated conditionals in this file. +*/ + +/* +** 2010 February 23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements routines used to report what compile-time options +** SQLite was built with. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS /* IMP: R-16824-07538 */ + +/* +** Include the configuration header output by 'configure' if we're using the +** autoconf-based build +*/ +#if defined(_HAVE_SQLITE_CONFIG_H) && !defined(SQLITECONFIG_H) +/* #include "sqlite_cfg.h" */ +#define SQLITECONFIG_H 1 +#endif + +/* These macros are provided to "stringify" the value of the define +** for those options in which the value is meaningful. */ +#define CTIMEOPT_VAL_(opt) #opt +#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) + +/* Like CTIMEOPT_VAL, but especially for SQLITE_DEFAULT_LOOKASIDE. This +** option requires a separate macro because legal values contain a single +** comma. e.g. (-DSQLITE_DEFAULT_LOOKASIDE="100,100") */ +#define CTIMEOPT_VAL2_(opt1,opt2) #opt1 "," #opt2 +#define CTIMEOPT_VAL2(opt) CTIMEOPT_VAL2_(opt) +/* #include "sqliteInt.h" */ + +/* +** An array of names of all compile-time options. This array should +** be sorted A-Z. +** +** This array looks large, but in a typical installation actually uses +** only a handful of compile-time options, so most times this array is usually +** rather short and uses little memory space. +*/ +static const char * const sqlite3azCompileOpt[] = { + +#ifdef SQLITE_32BIT_ROWID + "32BIT_ROWID", +#endif +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC + "4_BYTE_ALIGNED_MALLOC", +#endif +#ifdef SQLITE_ALLOW_COVERING_INDEX_SCAN +# if SQLITE_ALLOW_COVERING_INDEX_SCAN != 1 + "ALLOW_COVERING_INDEX_SCAN=" CTIMEOPT_VAL(SQLITE_ALLOW_COVERING_INDEX_SCAN), +# endif +#endif +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + "ALLOW_ROWID_IN_VIEW", +#endif +#ifdef SQLITE_ALLOW_URI_AUTHORITY + "ALLOW_URI_AUTHORITY", +#endif +#ifdef SQLITE_ATOMIC_INTRINSICS + "ATOMIC_INTRINSICS=" CTIMEOPT_VAL(SQLITE_ATOMIC_INTRINSICS), +#endif +#ifdef SQLITE_BITMASK_TYPE + "BITMASK_TYPE=" CTIMEOPT_VAL(SQLITE_BITMASK_TYPE), +#endif +#ifdef SQLITE_BUG_COMPATIBLE_20160819 + "BUG_COMPATIBLE_20160819", +#endif +#ifdef SQLITE_CASE_SENSITIVE_LIKE + "CASE_SENSITIVE_LIKE", +#endif +#ifdef SQLITE_CHECK_PAGES + "CHECK_PAGES", +#endif +#if defined(__clang__) && defined(__clang_major__) + "COMPILER=clang-" CTIMEOPT_VAL(__clang_major__) "." + CTIMEOPT_VAL(__clang_minor__) "." + CTIMEOPT_VAL(__clang_patchlevel__), +#elif defined(_MSC_VER) + "COMPILER=msvc-" CTIMEOPT_VAL(_MSC_VER), +#elif defined(__GNUC__) && defined(__VERSION__) + "COMPILER=gcc-" __VERSION__, +#endif +#ifdef SQLITE_COVERAGE_TEST + "COVERAGE_TEST", +#endif +#ifdef SQLITE_DEBUG + "DEBUG", +#endif +#ifdef SQLITE_DEFAULT_AUTOMATIC_INDEX + "DEFAULT_AUTOMATIC_INDEX", +#endif +#ifdef SQLITE_DEFAULT_AUTOVACUUM + "DEFAULT_AUTOVACUUM", +#endif +#ifdef SQLITE_DEFAULT_CACHE_SIZE + "DEFAULT_CACHE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_CACHE_SIZE), +#endif +#ifdef SQLITE_DEFAULT_CKPTFULLFSYNC + "DEFAULT_CKPTFULLFSYNC", +#endif +#ifdef SQLITE_DEFAULT_FILE_FORMAT + "DEFAULT_FILE_FORMAT=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_FORMAT), +#endif +#ifdef SQLITE_DEFAULT_FILE_PERMISSIONS + "DEFAULT_FILE_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_FILE_PERMISSIONS), +#endif +#ifdef SQLITE_DEFAULT_FOREIGN_KEYS + "DEFAULT_FOREIGN_KEYS", +#endif +#ifdef SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT + "DEFAULT_JOURNAL_SIZE_LIMIT=" CTIMEOPT_VAL(SQLITE_DEFAULT_JOURNAL_SIZE_LIMIT), +#endif +#ifdef SQLITE_DEFAULT_LOCKING_MODE + "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), +#endif +#ifdef SQLITE_DEFAULT_LOOKASIDE + "DEFAULT_LOOKASIDE=" CTIMEOPT_VAL2(SQLITE_DEFAULT_LOOKASIDE), +#endif +#ifdef SQLITE_DEFAULT_MEMSTATUS +# if SQLITE_DEFAULT_MEMSTATUS != 1 + "DEFAULT_MEMSTATUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_MEMSTATUS), +# endif +#endif +#ifdef SQLITE_DEFAULT_MMAP_SIZE + "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), +#endif +#ifdef SQLITE_DEFAULT_PAGE_SIZE + "DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_PAGE_SIZE), +#endif +#ifdef SQLITE_DEFAULT_PCACHE_INITSZ + "DEFAULT_PCACHE_INITSZ=" CTIMEOPT_VAL(SQLITE_DEFAULT_PCACHE_INITSZ), +#endif +#ifdef SQLITE_DEFAULT_PROXYDIR_PERMISSIONS + "DEFAULT_PROXYDIR_PERMISSIONS=" CTIMEOPT_VAL(SQLITE_DEFAULT_PROXYDIR_PERMISSIONS), +#endif +#ifdef SQLITE_DEFAULT_RECURSIVE_TRIGGERS + "DEFAULT_RECURSIVE_TRIGGERS", +#endif +#ifdef SQLITE_DEFAULT_ROWEST + "DEFAULT_ROWEST=" CTIMEOPT_VAL(SQLITE_DEFAULT_ROWEST), +#endif +#ifdef SQLITE_DEFAULT_SECTOR_SIZE + "DEFAULT_SECTOR_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_SECTOR_SIZE), +#endif +#ifdef SQLITE_DEFAULT_SYNCHRONOUS + "DEFAULT_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_SYNCHRONOUS), +#endif +#ifdef SQLITE_DEFAULT_WAL_AUTOCHECKPOINT + "DEFAULT_WAL_AUTOCHECKPOINT=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_AUTOCHECKPOINT), +#endif +#ifdef SQLITE_DEFAULT_WAL_SYNCHRONOUS + "DEFAULT_WAL_SYNCHRONOUS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WAL_SYNCHRONOUS), +#endif +#ifdef SQLITE_DEFAULT_WORKER_THREADS + "DEFAULT_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_DEFAULT_WORKER_THREADS), +#endif +#ifdef SQLITE_DIRECT_OVERFLOW_READ + "DIRECT_OVERFLOW_READ", +#endif +#ifdef SQLITE_DISABLE_DIRSYNC + "DISABLE_DIRSYNC", +#endif +#ifdef SQLITE_DISABLE_FTS3_UNICODE + "DISABLE_FTS3_UNICODE", +#endif +#ifdef SQLITE_DISABLE_FTS4_DEFERRED + "DISABLE_FTS4_DEFERRED", +#endif +#ifdef SQLITE_DISABLE_INTRINSIC + "DISABLE_INTRINSIC", +#endif +#ifdef SQLITE_DISABLE_LFS + "DISABLE_LFS", +#endif +#ifdef SQLITE_DISABLE_PAGECACHE_OVERFLOW_STATS + "DISABLE_PAGECACHE_OVERFLOW_STATS", +#endif +#ifdef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + "DISABLE_SKIPAHEAD_DISTINCT", +#endif +#ifdef SQLITE_DQS + "DQS=" CTIMEOPT_VAL(SQLITE_DQS), +#endif +#ifdef SQLITE_ENABLE_8_3_NAMES + "ENABLE_8_3_NAMES=" CTIMEOPT_VAL(SQLITE_ENABLE_8_3_NAMES), +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + "ENABLE_API_ARMOR", +#endif +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + "ENABLE_ATOMIC_WRITE", +#endif +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + "ENABLE_BATCH_ATOMIC_WRITE", +#endif +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + "ENABLE_BYTECODE_VTAB", +#endif +#ifdef SQLITE_ENABLE_CEROD + "ENABLE_CEROD=" CTIMEOPT_VAL(SQLITE_ENABLE_CEROD), +#endif +#ifdef SQLITE_ENABLE_COLUMN_METADATA + "ENABLE_COLUMN_METADATA", +#endif +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + "ENABLE_COLUMN_USED_MASK", +#endif +#ifdef SQLITE_ENABLE_COSTMULT + "ENABLE_COSTMULT", +#endif +#ifdef SQLITE_ENABLE_CURSOR_HINTS + "ENABLE_CURSOR_HINTS", +#endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + "ENABLE_DBPAGE_VTAB", +#endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + "ENABLE_DBSTAT_VTAB", +#endif +#ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT + "ENABLE_EXPENSIVE_ASSERT", +#endif +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + "ENABLE_EXPLAIN_COMMENTS", +#endif +#ifdef SQLITE_ENABLE_FTS3 + "ENABLE_FTS3", +#endif +#ifdef SQLITE_ENABLE_FTS3_PARENTHESIS + "ENABLE_FTS3_PARENTHESIS", +#endif +#ifdef SQLITE_ENABLE_FTS3_TOKENIZER + "ENABLE_FTS3_TOKENIZER", +#endif +#ifdef SQLITE_ENABLE_FTS4 + "ENABLE_FTS4", +#endif +#ifdef SQLITE_ENABLE_FTS5 + "ENABLE_FTS5", +#endif +#ifdef SQLITE_ENABLE_GEOPOLY + "ENABLE_GEOPOLY", +#endif +#ifdef SQLITE_ENABLE_HIDDEN_COLUMNS + "ENABLE_HIDDEN_COLUMNS", +#endif +#ifdef SQLITE_ENABLE_ICU + "ENABLE_ICU", +#endif +#ifdef SQLITE_ENABLE_IOTRACE + "ENABLE_IOTRACE", +#endif +#ifdef SQLITE_ENABLE_LOAD_EXTENSION + "ENABLE_LOAD_EXTENSION", +#endif +#ifdef SQLITE_ENABLE_LOCKING_STYLE + "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), +#endif +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS + "ENABLE_MATH_FUNCTIONS", +#endif +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT + "ENABLE_MEMORY_MANAGEMENT", +#endif +#ifdef SQLITE_ENABLE_MEMSYS3 + "ENABLE_MEMSYS3", +#endif +#ifdef SQLITE_ENABLE_MEMSYS5 + "ENABLE_MEMSYS5", +#endif +#ifdef SQLITE_ENABLE_MULTIPLEX + "ENABLE_MULTIPLEX", +#endif +#ifdef SQLITE_ENABLE_NORMALIZE + "ENABLE_NORMALIZE", +#endif +#ifdef SQLITE_ENABLE_NULL_TRIM + "ENABLE_NULL_TRIM", +#endif +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + "ENABLE_OFFSET_SQL_FUNC", +#endif +#ifdef SQLITE_ENABLE_ORDERED_SET_AGGREGATES + "ENABLE_ORDERED_SET_AGGREGATES", +#endif +#ifdef SQLITE_ENABLE_OVERSIZE_CELL_CHECK + "ENABLE_OVERSIZE_CELL_CHECK", +#endif +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + "ENABLE_PREUPDATE_HOOK", +#endif +#ifdef SQLITE_ENABLE_QPSG + "ENABLE_QPSG", +#endif +#ifdef SQLITE_ENABLE_RBU + "ENABLE_RBU", +#endif +#ifdef SQLITE_ENABLE_RTREE + "ENABLE_RTREE", +#endif +#ifdef SQLITE_ENABLE_SESSION + "ENABLE_SESSION", +#endif +#ifdef SQLITE_ENABLE_SNAPSHOT + "ENABLE_SNAPSHOT", +#endif +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + "ENABLE_SORTER_REFERENCES", +#endif +#ifdef SQLITE_ENABLE_SQLLOG + "ENABLE_SQLLOG", +#endif +#ifdef SQLITE_ENABLE_STAT4 + "ENABLE_STAT4", +#endif +#ifdef SQLITE_ENABLE_STMTVTAB + "ENABLE_STMTVTAB", +#endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + "ENABLE_STMT_SCANSTATUS", +#endif +#ifdef SQLITE_ENABLE_TREETRACE + "ENABLE_TREETRACE", +#endif +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + "ENABLE_UNKNOWN_SQL_FUNCTION", +#endif +#ifdef SQLITE_ENABLE_UNLOCK_NOTIFY + "ENABLE_UNLOCK_NOTIFY", +#endif +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + "ENABLE_UPDATE_DELETE_LIMIT", +#endif +#ifdef SQLITE_ENABLE_URI_00_ERROR + "ENABLE_URI_00_ERROR", +#endif +#ifdef SQLITE_ENABLE_VFSTRACE + "ENABLE_VFSTRACE", +#endif +#ifdef SQLITE_ENABLE_WHERETRACE + "ENABLE_WHERETRACE", +#endif +#ifdef SQLITE_ENABLE_ZIPVFS + "ENABLE_ZIPVFS", +#endif +#ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS + "EXPLAIN_ESTIMATED_ROWS", +#endif +#ifdef SQLITE_EXTRA_AUTOEXT + "EXTRA_AUTOEXT=" CTIMEOPT_VAL(SQLITE_EXTRA_AUTOEXT), +#endif +#ifdef SQLITE_EXTRA_IFNULLROW + "EXTRA_IFNULLROW", +#endif +#ifdef SQLITE_EXTRA_INIT + "EXTRA_INIT=" CTIMEOPT_VAL(SQLITE_EXTRA_INIT), +#endif +#ifdef SQLITE_EXTRA_SHUTDOWN + "EXTRA_SHUTDOWN=" CTIMEOPT_VAL(SQLITE_EXTRA_SHUTDOWN), +#endif +#ifdef SQLITE_FTS3_MAX_EXPR_DEPTH + "FTS3_MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_FTS3_MAX_EXPR_DEPTH), +#endif +#ifdef SQLITE_FTS5_ENABLE_TEST_MI + "FTS5_ENABLE_TEST_MI", +#endif +#ifdef SQLITE_FTS5_NO_WITHOUT_ROWID + "FTS5_NO_WITHOUT_ROWID", +#endif +#if HAVE_ISNAN || SQLITE_HAVE_ISNAN + "HAVE_ISNAN", +#endif +#ifdef SQLITE_HOMEGROWN_RECURSIVE_MUTEX +# if SQLITE_HOMEGROWN_RECURSIVE_MUTEX != 1 + "HOMEGROWN_RECURSIVE_MUTEX=" CTIMEOPT_VAL(SQLITE_HOMEGROWN_RECURSIVE_MUTEX), +# endif +#endif +#ifdef SQLITE_IGNORE_AFP_LOCK_ERRORS + "IGNORE_AFP_LOCK_ERRORS", +#endif +#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS + "IGNORE_FLOCK_LOCK_ERRORS", +#endif +#ifdef SQLITE_INLINE_MEMCPY + "INLINE_MEMCPY", +#endif +#ifdef SQLITE_INT64_TYPE + "INT64_TYPE", +#endif +#ifdef SQLITE_INTEGRITY_CHECK_ERROR_MAX + "INTEGRITY_CHECK_ERROR_MAX=" CTIMEOPT_VAL(SQLITE_INTEGRITY_CHECK_ERROR_MAX), +#endif +#ifdef SQLITE_LEGACY_JSON_VALID + "LEGACY_JSON_VALID", +#endif +#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS + "LIKE_DOESNT_MATCH_BLOBS", +#endif +#ifdef SQLITE_LOCK_TRACE + "LOCK_TRACE", +#endif +#ifdef SQLITE_LOG_CACHE_SPILL + "LOG_CACHE_SPILL", +#endif +#ifdef SQLITE_MALLOC_SOFT_LIMIT + "MALLOC_SOFT_LIMIT=" CTIMEOPT_VAL(SQLITE_MALLOC_SOFT_LIMIT), +#endif +#ifdef SQLITE_MAX_ATTACHED + "MAX_ATTACHED=" CTIMEOPT_VAL(SQLITE_MAX_ATTACHED), +#endif +#ifdef SQLITE_MAX_COLUMN + "MAX_COLUMN=" CTIMEOPT_VAL(SQLITE_MAX_COLUMN), +#endif +#ifdef SQLITE_MAX_COMPOUND_SELECT + "MAX_COMPOUND_SELECT=" CTIMEOPT_VAL(SQLITE_MAX_COMPOUND_SELECT), +#endif +#ifdef SQLITE_MAX_DEFAULT_PAGE_SIZE + "MAX_DEFAULT_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_DEFAULT_PAGE_SIZE), +#endif +#ifdef SQLITE_MAX_EXPR_DEPTH + "MAX_EXPR_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_EXPR_DEPTH), +#endif +#ifdef SQLITE_MAX_FUNCTION_ARG + "MAX_FUNCTION_ARG=" CTIMEOPT_VAL(SQLITE_MAX_FUNCTION_ARG), +#endif +#ifdef SQLITE_MAX_LENGTH + "MAX_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LENGTH), +#endif +#ifdef SQLITE_MAX_LIKE_PATTERN_LENGTH + "MAX_LIKE_PATTERN_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_LIKE_PATTERN_LENGTH), +#endif +#ifdef SQLITE_MAX_MEMORY + "MAX_MEMORY=" CTIMEOPT_VAL(SQLITE_MAX_MEMORY), +#endif +#ifdef SQLITE_MAX_MMAP_SIZE + "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), +#endif +#ifdef SQLITE_MAX_MMAP_SIZE_ + "MAX_MMAP_SIZE_=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE_), +#endif +#ifdef SQLITE_MAX_PAGE_COUNT + "MAX_PAGE_COUNT=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_COUNT), +#endif +#ifdef SQLITE_MAX_PAGE_SIZE + "MAX_PAGE_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_PAGE_SIZE), +#endif +#ifdef SQLITE_MAX_SCHEMA_RETRY + "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), +#endif +#ifdef SQLITE_MAX_SQL_LENGTH + "MAX_SQL_LENGTH=" CTIMEOPT_VAL(SQLITE_MAX_SQL_LENGTH), +#endif +#ifdef SQLITE_MAX_TRIGGER_DEPTH + "MAX_TRIGGER_DEPTH=" CTIMEOPT_VAL(SQLITE_MAX_TRIGGER_DEPTH), +#endif +#ifdef SQLITE_MAX_VARIABLE_NUMBER + "MAX_VARIABLE_NUMBER=" CTIMEOPT_VAL(SQLITE_MAX_VARIABLE_NUMBER), +#endif +#ifdef SQLITE_MAX_VDBE_OP + "MAX_VDBE_OP=" CTIMEOPT_VAL(SQLITE_MAX_VDBE_OP), +#endif +#ifdef SQLITE_MAX_WORKER_THREADS + "MAX_WORKER_THREADS=" CTIMEOPT_VAL(SQLITE_MAX_WORKER_THREADS), +#endif +#ifdef SQLITE_MEMDEBUG + "MEMDEBUG", +#endif +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT + "MIXED_ENDIAN_64BIT_FLOAT", +#endif +#ifdef SQLITE_MMAP_READWRITE + "MMAP_READWRITE", +#endif +#ifdef SQLITE_MUTEX_NOOP + "MUTEX_NOOP", +#endif +#ifdef SQLITE_MUTEX_OMIT + "MUTEX_OMIT", +#endif +#ifdef SQLITE_MUTEX_PTHREADS + "MUTEX_PTHREADS", +#endif +#ifdef SQLITE_MUTEX_W32 + "MUTEX_W32", +#endif +#ifdef SQLITE_NEED_ERR_NAME + "NEED_ERR_NAME", +#endif +#ifdef SQLITE_NO_SYNC + "NO_SYNC", +#endif +#ifdef SQLITE_OMIT_ALTERTABLE + "OMIT_ALTERTABLE", +#endif +#ifdef SQLITE_OMIT_ANALYZE + "OMIT_ANALYZE", +#endif +#ifdef SQLITE_OMIT_ATTACH + "OMIT_ATTACH", +#endif +#ifdef SQLITE_OMIT_AUTHORIZATION + "OMIT_AUTHORIZATION", +#endif +#ifdef SQLITE_OMIT_AUTOINCREMENT + "OMIT_AUTOINCREMENT", +#endif +#ifdef SQLITE_OMIT_AUTOINIT + "OMIT_AUTOINIT", +#endif +#ifdef SQLITE_OMIT_AUTOMATIC_INDEX + "OMIT_AUTOMATIC_INDEX", +#endif +#ifdef SQLITE_OMIT_AUTORESET + "OMIT_AUTORESET", +#endif +#ifdef SQLITE_OMIT_AUTOVACUUM + "OMIT_AUTOVACUUM", +#endif +#ifdef SQLITE_OMIT_BETWEEN_OPTIMIZATION + "OMIT_BETWEEN_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_BLOB_LITERAL + "OMIT_BLOB_LITERAL", +#endif +#ifdef SQLITE_OMIT_CAST + "OMIT_CAST", +#endif +#ifdef SQLITE_OMIT_CHECK + "OMIT_CHECK", +#endif +#ifdef SQLITE_OMIT_COMPLETE + "OMIT_COMPLETE", +#endif +#ifdef SQLITE_OMIT_COMPOUND_SELECT + "OMIT_COMPOUND_SELECT", +#endif +#ifdef SQLITE_OMIT_CONFLICT_CLAUSE + "OMIT_CONFLICT_CLAUSE", +#endif +#ifdef SQLITE_OMIT_CTE + "OMIT_CTE", +#endif +#if defined(SQLITE_OMIT_DATETIME_FUNCS) || defined(SQLITE_OMIT_FLOATING_POINT) + "OMIT_DATETIME_FUNCS", +#endif +#ifdef SQLITE_OMIT_DECLTYPE + "OMIT_DECLTYPE", +#endif +#ifdef SQLITE_OMIT_DEPRECATED + "OMIT_DEPRECATED", +#endif +#ifdef SQLITE_OMIT_DESERIALIZE + "OMIT_DESERIALIZE", +#endif +#ifdef SQLITE_OMIT_DISKIO + "OMIT_DISKIO", +#endif +#ifdef SQLITE_OMIT_EXPLAIN + "OMIT_EXPLAIN", +#endif +#ifdef SQLITE_OMIT_FLAG_PRAGMAS + "OMIT_FLAG_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_FLOATING_POINT + "OMIT_FLOATING_POINT", +#endif +#ifdef SQLITE_OMIT_FOREIGN_KEY + "OMIT_FOREIGN_KEY", +#endif +#ifdef SQLITE_OMIT_GET_TABLE + "OMIT_GET_TABLE", +#endif +#ifdef SQLITE_OMIT_HEX_INTEGER + "OMIT_HEX_INTEGER", +#endif +#ifdef SQLITE_OMIT_INCRBLOB + "OMIT_INCRBLOB", +#endif +#ifdef SQLITE_OMIT_INTEGRITY_CHECK + "OMIT_INTEGRITY_CHECK", +#endif +#ifdef SQLITE_OMIT_INTROSPECTION_PRAGMAS + "OMIT_INTROSPECTION_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_JSON + "OMIT_JSON", +#endif +#ifdef SQLITE_OMIT_LIKE_OPTIMIZATION + "OMIT_LIKE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + "OMIT_LOAD_EXTENSION", +#endif +#ifdef SQLITE_OMIT_LOCALTIME + "OMIT_LOCALTIME", +#endif +#ifdef SQLITE_OMIT_LOOKASIDE + "OMIT_LOOKASIDE", +#endif +#ifdef SQLITE_OMIT_MEMORYDB + "OMIT_MEMORYDB", +#endif +#ifdef SQLITE_OMIT_OR_OPTIMIZATION + "OMIT_OR_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_PAGER_PRAGMAS + "OMIT_PAGER_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_PARSER_TRACE + "OMIT_PARSER_TRACE", +#endif +#ifdef SQLITE_OMIT_POPEN + "OMIT_POPEN", +#endif +#ifdef SQLITE_OMIT_PRAGMA + "OMIT_PRAGMA", +#endif +#ifdef SQLITE_OMIT_PROGRESS_CALLBACK + "OMIT_PROGRESS_CALLBACK", +#endif +#ifdef SQLITE_OMIT_QUICKBALANCE + "OMIT_QUICKBALANCE", +#endif +#ifdef SQLITE_OMIT_REINDEX + "OMIT_REINDEX", +#endif +#ifdef SQLITE_OMIT_SCHEMA_PRAGMAS + "OMIT_SCHEMA_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS + "OMIT_SCHEMA_VERSION_PRAGMAS", +#endif +#ifdef SQLITE_OMIT_SEH + "OMIT_SEH", +#endif +#ifdef SQLITE_OMIT_SHARED_CACHE + "OMIT_SHARED_CACHE", +#endif +#ifdef SQLITE_OMIT_SHUTDOWN_DIRECTORIES + "OMIT_SHUTDOWN_DIRECTORIES", +#endif +#ifdef SQLITE_OMIT_SUBQUERY + "OMIT_SUBQUERY", +#endif +#ifdef SQLITE_OMIT_TCL_VARIABLE + "OMIT_TCL_VARIABLE", +#endif +#ifdef SQLITE_OMIT_TEMPDB + "OMIT_TEMPDB", +#endif +#ifdef SQLITE_OMIT_TEST_CONTROL + "OMIT_TEST_CONTROL", +#endif +#ifdef SQLITE_OMIT_TRACE +# if SQLITE_OMIT_TRACE != 1 + "OMIT_TRACE=" CTIMEOPT_VAL(SQLITE_OMIT_TRACE), +# endif +#endif +#ifdef SQLITE_OMIT_TRIGGER + "OMIT_TRIGGER", +#endif +#ifdef SQLITE_OMIT_TRUNCATE_OPTIMIZATION + "OMIT_TRUNCATE_OPTIMIZATION", +#endif +#ifdef SQLITE_OMIT_UTF16 + "OMIT_UTF16", +#endif +#ifdef SQLITE_OMIT_VACUUM + "OMIT_VACUUM", +#endif +#ifdef SQLITE_OMIT_VIEW + "OMIT_VIEW", +#endif +#ifdef SQLITE_OMIT_VIRTUALTABLE + "OMIT_VIRTUALTABLE", +#endif +#ifdef SQLITE_OMIT_WAL + "OMIT_WAL", +#endif +#ifdef SQLITE_OMIT_WSD + "OMIT_WSD", +#endif +#ifdef SQLITE_OMIT_XFER_OPT + "OMIT_XFER_OPT", +#endif +#ifdef SQLITE_PERFORMANCE_TRACE + "PERFORMANCE_TRACE", +#endif +#ifdef SQLITE_POWERSAFE_OVERWRITE +# if SQLITE_POWERSAFE_OVERWRITE != 1 + "POWERSAFE_OVERWRITE=" CTIMEOPT_VAL(SQLITE_POWERSAFE_OVERWRITE), +# endif +#endif +#ifdef SQLITE_PREFER_PROXY_LOCKING + "PREFER_PROXY_LOCKING", +#endif +#ifdef SQLITE_PROXY_DEBUG + "PROXY_DEBUG", +#endif +#ifdef SQLITE_REVERSE_UNORDERED_SELECTS + "REVERSE_UNORDERED_SELECTS", +#endif +#ifdef SQLITE_RTREE_INT_ONLY + "RTREE_INT_ONLY", +#endif +#ifdef SQLITE_SECURE_DELETE + "SECURE_DELETE", +#endif +#ifdef SQLITE_SMALL_STACK + "SMALL_STACK", +#endif +#ifdef SQLITE_SORTER_PMASZ + "SORTER_PMASZ=" CTIMEOPT_VAL(SQLITE_SORTER_PMASZ), +#endif +#ifdef SQLITE_SOUNDEX + "SOUNDEX", +#endif +#ifdef SQLITE_STAT4_SAMPLES + "STAT4_SAMPLES=" CTIMEOPT_VAL(SQLITE_STAT4_SAMPLES), +#endif +#ifdef SQLITE_STMTJRNL_SPILL + "STMTJRNL_SPILL=" CTIMEOPT_VAL(SQLITE_STMTJRNL_SPILL), +#endif +#ifdef SQLITE_SUBSTR_COMPATIBILITY + "SUBSTR_COMPATIBILITY", +#endif +#if (!defined(SQLITE_WIN32_MALLOC) \ + && !defined(SQLITE_ZERO_MALLOC) \ + && !defined(SQLITE_MEMDEBUG) \ + ) || defined(SQLITE_SYSTEM_MALLOC) + "SYSTEM_MALLOC", +#endif +#ifdef SQLITE_TCL + "TCL", +#endif +#ifdef SQLITE_TEMP_STORE + "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), +#endif +#ifdef SQLITE_TEST + "TEST", +#endif +#if defined(SQLITE_THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), +#elif defined(THREADSAFE) + "THREADSAFE=" CTIMEOPT_VAL(THREADSAFE), +#else + "THREADSAFE=1", +#endif +#ifdef SQLITE_UNLINK_AFTER_CLOSE + "UNLINK_AFTER_CLOSE", +#endif +#ifdef SQLITE_UNTESTABLE + "UNTESTABLE", +#endif +#ifdef SQLITE_USER_AUTHENTICATION + "USER_AUTHENTICATION", +#endif +#ifdef SQLITE_USE_ALLOCA + "USE_ALLOCA", +#endif +#ifdef SQLITE_USE_FCNTL_TRACE + "USE_FCNTL_TRACE", +#endif +#ifdef SQLITE_USE_URI + "USE_URI", +#endif +#ifdef SQLITE_VDBE_COVERAGE + "VDBE_COVERAGE", +#endif +#ifdef SQLITE_WIN32_MALLOC + "WIN32_MALLOC", +#endif +#ifdef SQLITE_ZERO_MALLOC + "ZERO_MALLOC", +#endif + +} ; + +SQLITE_PRIVATE const char **sqlite3CompileOptions(int *pnOpt){ + *pnOpt = sizeof(sqlite3azCompileOpt) / sizeof(sqlite3azCompileOpt[0]); + return (const char**)sqlite3azCompileOpt; +} + +#endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ + +/************** End of ctime.c ***********************************************/ /************** Begin file global.c ******************************************/ /* ** 2008 June 13 @@ -16671,7 +22911,7 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3*); /* #include "sqliteInt.h" */ /* An array to map all upper-case characters into their corresponding -** lower-case character. +** lower-case character. ** ** SQLite only considers US-ASCII (or EBCDIC) characters. We do not ** handle case conversions for the UTF character set since the tables @@ -16693,7 +22933,7 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { 198,199,200,201,202,203,204,205,206,207,208,209,210,211,212,213,214,215, 216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233, 234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251, - 252,253,254,255 + 252,253,254,255, #endif #ifdef SQLITE_EBCDIC 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* 0x */ @@ -16713,7 +22953,35 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { 224,225,162,163,164,165,166,167,168,169,234,235,236,237,238,239, /* Ex */ 240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255, /* Fx */ #endif +/* All of the upper-to-lower conversion data is above. The following +** 18 integers are completely unrelated. They are appended to the +** sqlite3UpperToLower[] array to avoid UBSAN warnings. Here's what is +** going on: +** +** The SQL comparison operators (<>, =, >, <=, <, and >=) are implemented +** by invoking sqlite3MemCompare(A,B) which compares values A and B and +** returns negative, zero, or positive if A is less then, equal to, or +** greater than B, respectively. Then the true false results is found by +** consulting sqlite3aLTb[opcode], sqlite3aEQb[opcode], or +** sqlite3aGTb[opcode] depending on whether the result of compare(A,B) +** is negative, zero, or positive, where opcode is the specific opcode. +** The only works because the comparison opcodes are consecutive and in +** this order: NE EQ GT LE LT GE. Various assert()s throughout the code +** ensure that is the case. +** +** These elements must be appended to another array. Otherwise the +** index (here shown as [256-OP_Ne]) would be out-of-bounds and thus +** be undefined behavior. That's goofy, but the C-standards people thought +** it was a good idea, so here we are. +*/ +/* NE EQ GT LE LT GE */ + 1, 0, 0, 1, 1, 0, /* aLTb[]: Use when compare(A,B) less than zero */ + 0, 1, 0, 1, 0, 1, /* aEQb[]: Use when compare(A,B) equals zero */ + 1, 0, 1, 0, 0, 1 /* aGTb[]: Use when compare(A,B) greater than zero*/ }; +SQLITE_PRIVATE const unsigned char *sqlite3aLTb = &sqlite3UpperToLower[256-OP_Ne]; +SQLITE_PRIVATE const unsigned char *sqlite3aEQb = &sqlite3UpperToLower[256+6-OP_Ne]; +SQLITE_PRIVATE const unsigned char *sqlite3aGTb = &sqlite3UpperToLower[256+12-OP_Ne]; /* ** The following 256 byte lookup table is used to support SQLites built-in @@ -16725,7 +22993,7 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** isalnum() 0x06 ** isxdigit() 0x08 ** toupper() 0x20 -** SQLite identifier character 0x40 +** SQLite identifier character 0x40 $, _, or non-ascii ** Quote character 0x80 ** ** Bit 0x20 is set if the mapped character requires translation to upper @@ -16735,18 +23003,14 @@ SQLITE_PRIVATE const unsigned char sqlite3UpperToLower[] = { ** ** (x & ~(map[x]&0x20)) ** -** Standard function tolower() is implemented using the sqlite3UpperToLower[] +** The equivalent of tolower() is implemented using the sqlite3UpperToLower[] ** array. tolower() is used more often than toupper() by SQLite. ** -** Bit 0x40 is set if the character non-alphanumeric and can be used in an +** Bit 0x40 is set if the character is non-alphanumeric and can be used in an ** SQLite identifier. Identifiers are alphanumerics, "_", "$", and any ** non-ASCII UTF character. Hence the test for whether or not a character is ** part of an identifier is 0x46. -** -** SQLite's versions are identical to the standard versions assuming a -** locale of "C". They are implemented as macros in sqliteInt.h. */ -#ifdef SQLITE_ASCII SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /* 00..07 ........ */ 0x00, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, /* 08..0f ........ */ @@ -16784,7 +23048,6 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, /* f0..f7 ........ */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40 /* f8..ff ........ */ }; -#endif /* EVIDENCE-OF: R-02982-34736 In order to maintain full backwards ** compatibility for legacy applications, the URI filename capability is @@ -16798,15 +23061,22 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { ** SQLITE_USE_URI symbol defined. */ #ifndef SQLITE_USE_URI -# define SQLITE_USE_URI 0 +# define SQLITE_USE_URI 0 #endif /* EVIDENCE-OF: R-38720-18127 The default setting is determined by the ** SQLITE_ALLOW_COVERING_INDEX_SCAN compile-time option, or is "on" if ** that compile-time option is omitted. */ -#ifndef SQLITE_ALLOW_COVERING_INDEX_SCAN +#if !defined(SQLITE_ALLOW_COVERING_INDEX_SCAN) # define SQLITE_ALLOW_COVERING_INDEX_SCAN 1 +#else +# if !SQLITE_ALLOW_COVERING_INDEX_SCAN +# error "Compile-time disabling of covering index scan using the\ + -DSQLITE_ALLOW_COVERING_INDEX_SCAN=0 option is deprecated.\ + Contact SQLite developers if this is a problem for you, and\ + delete this #error macro to continue with your build." +# endif #endif /* The minimum PMA size is set to this value multiplied by the database @@ -16817,17 +23087,46 @@ SQLITE_PRIVATE const unsigned char sqlite3CtypeMap[256] = { #endif /* Statement journals spill to disk when their size exceeds the following -** threashold (in bytes). 0 means that statement journals are created and +** threshold (in bytes). 0 means that statement journals are created and ** written to disk immediately (the default behavior for SQLite versions ** before 3.12.0). -1 means always keep the entire statement journal in ** memory. (The statement journal is also always held entirely in memory ** if journal_mode=MEMORY or if temp_store=MEMORY, regardless of this ** setting.) */ -#ifndef SQLITE_STMTJRNL_SPILL +#ifndef SQLITE_STMTJRNL_SPILL # define SQLITE_STMTJRNL_SPILL (64*1024) #endif +/* +** The default lookaside-configuration, the format "SZ,N". SZ is the +** number of bytes in each lookaside slot (should be a multiple of 8) +** and N is the number of slots. The lookaside-configuration can be +** changed as start-time using sqlite3_config(SQLITE_CONFIG_LOOKASIDE) +** or at run-time for an individual database connection using +** sqlite3_db_config(db, SQLITE_DBCONFIG_LOOKASIDE); +** +** With the two-size-lookaside enhancement, less lookaside is required. +** The default configuration of 1200,40 actually provides 30 1200-byte slots +** and 93 128-byte slots, which is more lookaside than is available +** using the older 1200,100 configuration without two-size-lookaside. +*/ +#ifndef SQLITE_DEFAULT_LOOKASIDE +# ifdef SQLITE_OMIT_TWOSIZE_LOOKASIDE +# define SQLITE_DEFAULT_LOOKASIDE 1200,100 /* 120KB of memory */ +# else +# define SQLITE_DEFAULT_LOOKASIDE 1200,40 /* 48KB of memory */ +# endif +#endif + + +/* The default maximum size of an in-memory database created using +** sqlite3_deserialize() +*/ +#ifndef SQLITE_MEMDB_DEFAULT_MAXSIZE +# define SQLITE_MEMDB_DEFAULT_MAXSIZE 1073741824 +#endif + /* ** The following singleton contains the global configuration for ** the SQLite library. @@ -16838,10 +23137,14 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { SQLITE_THREADSAFE==1, /* bFullMutex */ SQLITE_USE_URI, /* bOpenUri */ SQLITE_ALLOW_COVERING_INDEX_SCAN, /* bUseCis */ + 0, /* bSmallMalloc */ + 1, /* bExtraSchemaChecks */ +#ifdef SQLITE_DEBUG + 0, /* bJsonSelfcheck */ +#endif 0x7ffffffe, /* mxStrlen */ 0, /* neverCorrupt */ - 128, /* szLookaside */ - 500, /* nLookaside */ + SQLITE_DEFAULT_LOOKASIDE, /* szLookaside, nLookaside */ SQLITE_STMTJRNL_SPILL, /* nStmtSpill */ {0,0,0,0,0,0,0,0}, /* m */ {0,0,0,0,0,0,0,0,0}, /* mutex */ @@ -16851,9 +23154,6 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, 0, /* mnHeap, mxHeap */ SQLITE_DEFAULT_MMAP_SIZE, /* szMmap */ SQLITE_MAX_MMAP_SIZE, /* mxMmap */ - (void*)0, /* pScratch */ - 0, /* szScratch */ - 0, /* nScratch */ (void*)0, /* pPage */ 0, /* szPage */ SQLITE_DEFAULT_PCACHE_INITSZ, /* nPage */ @@ -16878,10 +23178,23 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { 0, /* xVdbeBranch */ 0, /* pVbeBranchArg */ #endif -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_OMIT_DESERIALIZE + SQLITE_MEMDB_DEFAULT_MAXSIZE, /* mxMemdbSize */ +#endif +#ifndef SQLITE_UNTESTABLE 0, /* xTestCallback */ #endif - 0 /* bLocaltimeFault */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + 0, /* mNoVisibleRowid. 0 == allow rowid-in-view */ +#endif + 0, /* bLocaltimeFault */ + 0, /* xAltLocaltime */ + 0x7ffffffe, /* iOnceResetThreshold */ + SQLITE_DEFAULT_SORTERREF_SIZE, /* szSorterRef */ + 0, /* iPrngSeed */ +#ifdef SQLITE_DEBUG + {0,0,0,0,0,0}, /* aTune */ +#endif }; /* @@ -16891,20 +23204,31 @@ SQLITE_PRIVATE SQLITE_WSD struct Sqlite3Config sqlite3Config = { */ SQLITE_PRIVATE FuncDefHash sqlite3BuiltinFunctions; +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) /* -** Constant tokens for values 0 and 1. +** Counter used for coverage testing. Does not come into play for +** release builds. +** +** Access to this global variable is not mutex protected. This might +** result in TSAN warnings. But as the variable does not exist in +** release builds, that should not be a concern. */ -SQLITE_PRIVATE const Token sqlite3IntTokens[] = { - { "0", 1 }, - { "1", 1 } -}; +SQLITE_PRIVATE unsigned int sqlite3CoverageCounter; +#endif /* SQLITE_COVERAGE_TEST || SQLITE_DEBUG */ +#ifdef VDBE_PROFILE +/* +** The following performance counter can be used in place of +** sqlite3Hwtime() for profiling. This is a no-op on standard builds. +*/ +SQLITE_PRIVATE sqlite3_uint64 sqlite3NProfileCnt = 0; +#endif /* ** The value of the "pending" byte must be 0x40000000 (1 byte past the ** 1-gibabyte boundary) in a compatible database. SQLite never uses ** the database page that contains the pending byte. It never attempts -** to read or write that page. The pending byte page is set assign +** to read or write that page. The pending byte page is set aside ** for use by the VFS layers as space for managing file locks. ** ** During testing, it is often desirable to move the pending byte to @@ -16922,12 +23246,18 @@ SQLITE_PRIVATE const Token sqlite3IntTokens[] = { SQLITE_PRIVATE int sqlite3PendingByte = 0x40000000; #endif +/* +** Tracing flags set by SQLITE_TESTCTRL_TRACEFLAGS. +*/ +SQLITE_PRIVATE u32 sqlite3TreeTrace = 0; +SQLITE_PRIVATE u32 sqlite3WhereTrace = 0; + /* #include "opcodes.h" */ /* ** Properties of opcodes. The OPFLG_INITIALIZER macro is ** created by mkopcodeh.awk during compilation. Data is obtained ** from the comments following the "case OP_xxxx:" statements in -** the vdbe.c file. +** the vdbe.c file. */ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; @@ -16936,452 +23266,37 @@ SQLITE_PRIVATE const unsigned char sqlite3OpcodeProperty[] = OPFLG_INITIALIZER; */ SQLITE_PRIVATE const char sqlite3StrBINARY[] = "BINARY"; -/************** End of global.c **********************************************/ -/************** Begin file ctime.c *******************************************/ /* -** 2010 February 23 +** Standard typenames. These names must match the COLTYPE_* definitions. +** Adjust the SQLITE_N_STDTYPE value if adding or removing entries. ** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: +** sqlite3StdType[] The actual names of the datatypes. ** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. +** sqlite3StdTypeLen[] The length (in bytes) of each entry +** in sqlite3StdType[]. ** -************************************************************************* -** -** This file implements routines used to report what compile-time options -** SQLite was built with. +** sqlite3StdTypeAffinity[] The affinity associated with each entry +** in sqlite3StdType[]. */ - -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS - -/* #include "sqliteInt.h" */ - -/* -** An array of names of all compile-time options. This array should -** be sorted A-Z. -** -** This array looks large, but in a typical installation actually uses -** only a handful of compile-time options, so most times this array is usually -** rather short and uses little memory space. -*/ -static const char * const azCompileOpt[] = { - -/* These macros are provided to "stringify" the value of the define -** for those options in which the value is meaningful. */ -#define CTIMEOPT_VAL_(opt) #opt -#define CTIMEOPT_VAL(opt) CTIMEOPT_VAL_(opt) - -#if SQLITE_32BIT_ROWID - "32BIT_ROWID", -#endif -#if SQLITE_4_BYTE_ALIGNED_MALLOC - "4_BYTE_ALIGNED_MALLOC", -#endif -#if SQLITE_CASE_SENSITIVE_LIKE - "CASE_SENSITIVE_LIKE", -#endif -#if SQLITE_CHECK_PAGES - "CHECK_PAGES", -#endif -#if SQLITE_COVERAGE_TEST - "COVERAGE_TEST", -#endif -#if SQLITE_DEBUG - "DEBUG", -#endif -#if SQLITE_DEFAULT_LOCKING_MODE - "DEFAULT_LOCKING_MODE=" CTIMEOPT_VAL(SQLITE_DEFAULT_LOCKING_MODE), -#endif -#if defined(SQLITE_DEFAULT_MMAP_SIZE) && !defined(SQLITE_DEFAULT_MMAP_SIZE_xc) - "DEFAULT_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_DEFAULT_MMAP_SIZE), -#endif -#if SQLITE_DISABLE_DIRSYNC - "DISABLE_DIRSYNC", -#endif -#if SQLITE_DISABLE_LFS - "DISABLE_LFS", -#endif -#if SQLITE_ENABLE_8_3_NAMES - "ENABLE_8_3_NAMES", -#endif -#if SQLITE_ENABLE_API_ARMOR - "ENABLE_API_ARMOR", -#endif -#if SQLITE_ENABLE_ATOMIC_WRITE - "ENABLE_ATOMIC_WRITE", -#endif -#if SQLITE_ENABLE_CEROD - "ENABLE_CEROD", -#endif -#if SQLITE_ENABLE_COLUMN_METADATA - "ENABLE_COLUMN_METADATA", -#endif -#if SQLITE_ENABLE_DBSTAT_VTAB - "ENABLE_DBSTAT_VTAB", -#endif -#if SQLITE_ENABLE_EXPENSIVE_ASSERT - "ENABLE_EXPENSIVE_ASSERT", -#endif -#if SQLITE_ENABLE_FTS1 - "ENABLE_FTS1", -#endif -#if SQLITE_ENABLE_FTS2 - "ENABLE_FTS2", -#endif -#if SQLITE_ENABLE_FTS3 - "ENABLE_FTS3", -#endif -#if SQLITE_ENABLE_FTS3_PARENTHESIS - "ENABLE_FTS3_PARENTHESIS", -#endif -#if SQLITE_ENABLE_FTS4 - "ENABLE_FTS4", -#endif -#if SQLITE_ENABLE_FTS5 - "ENABLE_FTS5", -#endif -#if SQLITE_ENABLE_ICU - "ENABLE_ICU", -#endif -#if SQLITE_ENABLE_IOTRACE - "ENABLE_IOTRACE", -#endif -#if SQLITE_ENABLE_JSON1 - "ENABLE_JSON1", -#endif -#if SQLITE_ENABLE_LOAD_EXTENSION - "ENABLE_LOAD_EXTENSION", -#endif -#if SQLITE_ENABLE_LOCKING_STYLE - "ENABLE_LOCKING_STYLE=" CTIMEOPT_VAL(SQLITE_ENABLE_LOCKING_STYLE), -#endif -#if SQLITE_ENABLE_MEMORY_MANAGEMENT - "ENABLE_MEMORY_MANAGEMENT", -#endif -#if SQLITE_ENABLE_MEMSYS3 - "ENABLE_MEMSYS3", -#endif -#if SQLITE_ENABLE_MEMSYS5 - "ENABLE_MEMSYS5", -#endif -#if SQLITE_ENABLE_OVERSIZE_CELL_CHECK - "ENABLE_OVERSIZE_CELL_CHECK", -#endif -#if SQLITE_ENABLE_RTREE - "ENABLE_RTREE", -#endif -#if defined(SQLITE_ENABLE_STAT4) - "ENABLE_STAT4", -#elif defined(SQLITE_ENABLE_STAT3) - "ENABLE_STAT3", -#endif -#if SQLITE_ENABLE_UNLOCK_NOTIFY - "ENABLE_UNLOCK_NOTIFY", -#endif -#if SQLITE_ENABLE_UPDATE_DELETE_LIMIT - "ENABLE_UPDATE_DELETE_LIMIT", -#endif -#if SQLITE_HAS_CODEC - "HAS_CODEC", -#endif -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN - "HAVE_ISNAN", -#endif -#if SQLITE_HOMEGROWN_RECURSIVE_MUTEX - "HOMEGROWN_RECURSIVE_MUTEX", -#endif -#if SQLITE_IGNORE_AFP_LOCK_ERRORS - "IGNORE_AFP_LOCK_ERRORS", -#endif -#if SQLITE_IGNORE_FLOCK_LOCK_ERRORS - "IGNORE_FLOCK_LOCK_ERRORS", -#endif -#ifdef SQLITE_INT64_TYPE - "INT64_TYPE", -#endif -#ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - "LIKE_DOESNT_MATCH_BLOBS", -#endif -#if SQLITE_LOCK_TRACE - "LOCK_TRACE", -#endif -#if defined(SQLITE_MAX_MMAP_SIZE) && !defined(SQLITE_MAX_MMAP_SIZE_xc) - "MAX_MMAP_SIZE=" CTIMEOPT_VAL(SQLITE_MAX_MMAP_SIZE), -#endif -#ifdef SQLITE_MAX_SCHEMA_RETRY - "MAX_SCHEMA_RETRY=" CTIMEOPT_VAL(SQLITE_MAX_SCHEMA_RETRY), -#endif -#if SQLITE_MEMDEBUG - "MEMDEBUG", -#endif -#if SQLITE_MIXED_ENDIAN_64BIT_FLOAT - "MIXED_ENDIAN_64BIT_FLOAT", -#endif -#if SQLITE_NO_SYNC - "NO_SYNC", -#endif -#if SQLITE_OMIT_ALTERTABLE - "OMIT_ALTERTABLE", -#endif -#if SQLITE_OMIT_ANALYZE - "OMIT_ANALYZE", -#endif -#if SQLITE_OMIT_ATTACH - "OMIT_ATTACH", -#endif -#if SQLITE_OMIT_AUTHORIZATION - "OMIT_AUTHORIZATION", -#endif -#if SQLITE_OMIT_AUTOINCREMENT - "OMIT_AUTOINCREMENT", -#endif -#if SQLITE_OMIT_AUTOINIT - "OMIT_AUTOINIT", -#endif -#if SQLITE_OMIT_AUTOMATIC_INDEX - "OMIT_AUTOMATIC_INDEX", -#endif -#if SQLITE_OMIT_AUTORESET - "OMIT_AUTORESET", -#endif -#if SQLITE_OMIT_AUTOVACUUM - "OMIT_AUTOVACUUM", -#endif -#if SQLITE_OMIT_BETWEEN_OPTIMIZATION - "OMIT_BETWEEN_OPTIMIZATION", -#endif -#if SQLITE_OMIT_BLOB_LITERAL - "OMIT_BLOB_LITERAL", -#endif -#if SQLITE_OMIT_BTREECOUNT - "OMIT_BTREECOUNT", -#endif -#if SQLITE_OMIT_BUILTIN_TEST - "OMIT_BUILTIN_TEST", -#endif -#if SQLITE_OMIT_CAST - "OMIT_CAST", -#endif -#if SQLITE_OMIT_CHECK - "OMIT_CHECK", -#endif -#if SQLITE_OMIT_COMPLETE - "OMIT_COMPLETE", -#endif -#if SQLITE_OMIT_COMPOUND_SELECT - "OMIT_COMPOUND_SELECT", -#endif -#if SQLITE_OMIT_CTE - "OMIT_CTE", -#endif -#if SQLITE_OMIT_DATETIME_FUNCS - "OMIT_DATETIME_FUNCS", -#endif -#if SQLITE_OMIT_DECLTYPE - "OMIT_DECLTYPE", -#endif -#if SQLITE_OMIT_DEPRECATED - "OMIT_DEPRECATED", -#endif -#if SQLITE_OMIT_DISKIO - "OMIT_DISKIO", -#endif -#if SQLITE_OMIT_EXPLAIN - "OMIT_EXPLAIN", -#endif -#if SQLITE_OMIT_FLAG_PRAGMAS - "OMIT_FLAG_PRAGMAS", -#endif -#if SQLITE_OMIT_FLOATING_POINT - "OMIT_FLOATING_POINT", -#endif -#if SQLITE_OMIT_FOREIGN_KEY - "OMIT_FOREIGN_KEY", -#endif -#if SQLITE_OMIT_GET_TABLE - "OMIT_GET_TABLE", -#endif -#if SQLITE_OMIT_INCRBLOB - "OMIT_INCRBLOB", -#endif -#if SQLITE_OMIT_INTEGRITY_CHECK - "OMIT_INTEGRITY_CHECK", -#endif -#if SQLITE_OMIT_LIKE_OPTIMIZATION - "OMIT_LIKE_OPTIMIZATION", -#endif -#if SQLITE_OMIT_LOAD_EXTENSION - "OMIT_LOAD_EXTENSION", -#endif -#if SQLITE_OMIT_LOCALTIME - "OMIT_LOCALTIME", -#endif -#if SQLITE_OMIT_LOOKASIDE - "OMIT_LOOKASIDE", -#endif -#if SQLITE_OMIT_MEMORYDB - "OMIT_MEMORYDB", -#endif -#if SQLITE_OMIT_OR_OPTIMIZATION - "OMIT_OR_OPTIMIZATION", -#endif -#if SQLITE_OMIT_PAGER_PRAGMAS - "OMIT_PAGER_PRAGMAS", -#endif -#if SQLITE_OMIT_PRAGMA - "OMIT_PRAGMA", -#endif -#if SQLITE_OMIT_PROGRESS_CALLBACK - "OMIT_PROGRESS_CALLBACK", -#endif -#if SQLITE_OMIT_QUICKBALANCE - "OMIT_QUICKBALANCE", -#endif -#if SQLITE_OMIT_REINDEX - "OMIT_REINDEX", -#endif -#if SQLITE_OMIT_SCHEMA_PRAGMAS - "OMIT_SCHEMA_PRAGMAS", -#endif -#if SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS - "OMIT_SCHEMA_VERSION_PRAGMAS", -#endif -#if SQLITE_OMIT_SHARED_CACHE - "OMIT_SHARED_CACHE", -#endif -#if SQLITE_OMIT_SUBQUERY - "OMIT_SUBQUERY", -#endif -#if SQLITE_OMIT_TCL_VARIABLE - "OMIT_TCL_VARIABLE", -#endif -#if SQLITE_OMIT_TEMPDB - "OMIT_TEMPDB", -#endif -#if SQLITE_OMIT_TRACE - "OMIT_TRACE", -#endif -#if SQLITE_OMIT_TRIGGER - "OMIT_TRIGGER", -#endif -#if SQLITE_OMIT_TRUNCATE_OPTIMIZATION - "OMIT_TRUNCATE_OPTIMIZATION", -#endif -#if SQLITE_OMIT_UTF16 - "OMIT_UTF16", -#endif -#if SQLITE_OMIT_VACUUM - "OMIT_VACUUM", -#endif -#if SQLITE_OMIT_VIEW - "OMIT_VIEW", -#endif -#if SQLITE_OMIT_VIRTUALTABLE - "OMIT_VIRTUALTABLE", -#endif -#if SQLITE_OMIT_WAL - "OMIT_WAL", -#endif -#if SQLITE_OMIT_WSD - "OMIT_WSD", -#endif -#if SQLITE_OMIT_XFER_OPT - "OMIT_XFER_OPT", -#endif -#if SQLITE_PERFORMANCE_TRACE - "PERFORMANCE_TRACE", -#endif -#if SQLITE_PROXY_DEBUG - "PROXY_DEBUG", -#endif -#if SQLITE_RTREE_INT_ONLY - "RTREE_INT_ONLY", -#endif -#if SQLITE_SECURE_DELETE - "SECURE_DELETE", -#endif -#if SQLITE_SMALL_STACK - "SMALL_STACK", -#endif -#if SQLITE_SOUNDEX - "SOUNDEX", -#endif -#if SQLITE_SYSTEM_MALLOC - "SYSTEM_MALLOC", -#endif -#if SQLITE_TCL - "TCL", -#endif -#if defined(SQLITE_TEMP_STORE) && !defined(SQLITE_TEMP_STORE_xc) - "TEMP_STORE=" CTIMEOPT_VAL(SQLITE_TEMP_STORE), -#endif -#if SQLITE_TEST - "TEST", -#endif -#if defined(SQLITE_THREADSAFE) - "THREADSAFE=" CTIMEOPT_VAL(SQLITE_THREADSAFE), -#endif -#if SQLITE_USE_ALLOCA - "USE_ALLOCA", -#endif -#if SQLITE_USER_AUTHENTICATION - "USER_AUTHENTICATION", -#endif -#if SQLITE_WIN32_MALLOC - "WIN32_MALLOC", -#endif -#if SQLITE_ZERO_MALLOC - "ZERO_MALLOC" -#endif +SQLITE_PRIVATE const unsigned char sqlite3StdTypeLen[] = { 3, 4, 3, 7, 4, 4 }; +SQLITE_PRIVATE const char sqlite3StdTypeAffinity[] = { + SQLITE_AFF_NUMERIC, + SQLITE_AFF_BLOB, + SQLITE_AFF_INTEGER, + SQLITE_AFF_INTEGER, + SQLITE_AFF_REAL, + SQLITE_AFF_TEXT +}; +SQLITE_PRIVATE const char *sqlite3StdType[] = { + "ANY", + "BLOB", + "INT", + "INTEGER", + "REAL", + "TEXT" }; -/* -** Given the name of a compile-time option, return true if that option -** was used and false if not. -** -** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix -** is not required for a match. -*/ -SQLITE_API int SQLITE_STDCALL sqlite3_compileoption_used(const char *zOptName){ - int i, n; - -#if SQLITE_ENABLE_API_ARMOR - if( zOptName==0 ){ - (void)SQLITE_MISUSE_BKPT; - return 0; - } -#endif - if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; - n = sqlite3Strlen30(zOptName); - - /* Since ArraySize(azCompileOpt) is normally in single digits, a - ** linear search is adequate. No need for a binary search. */ - for(i=0; i=0 && NaDb[] (or -1) */ - u8 nullRow; /* True if pointing to a row with no data */ - u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ - u8 isTable; /* True for rowid tables. False for indexes */ + u8 eCurType; /* One of the CURTYPE_* values above */ + i8 iDb; /* Index of cursor database in db->aDb[] */ + u8 nullRow; /* True if pointing to a row with no data */ + u8 deferredMoveto; /* A call to sqlite3BtreeMoveto() is needed */ + u8 isTable; /* True for rowid tables. False for indexes */ #ifdef SQLITE_DEBUG - u8 seekOp; /* Most recent seek operation on this cursor */ - u8 wrFlag; /* The wrFlag argument to sqlite3BtreeCursor() */ -#endif - Bool isEphemeral:1; /* True for an ephemeral table */ - Bool useRandomRowid:1;/* Generate new record numbers semi-randomly */ - Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ - Pgno pgnoRoot; /* Root page of the open btree cursor */ - i16 nField; /* Number of fields in the header */ - u16 nHdrParsed; /* Number of header fields parsed so far */ - union { - BtCursor *pCursor; /* CURTYPE_BTREE. Btree cursor */ - sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ - int pseudoTableReg; /* CURTYPE_PSEUDO. Reg holding content. */ - VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ - } uc; - Btree *pBt; /* Separate file holding temporary table */ - KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ - int seekResult; /* Result of previous sqlite3BtreeMoveto() */ - i64 seqCount; /* Sequence counter */ - i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ - VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ - int *aAltMap; /* Mapping from table to index column numbers */ -#ifdef SQLITE_ENABLE_COLUMN_USED_MASK - u64 maskUsed; /* Mask of columns used by this cursor */ + u8 seekOp; /* Most recent seek operation on this cursor */ + u8 wrFlag; /* The wrFlag argument to sqlite3BtreeCursor() */ #endif + Bool isEphemeral:1; /* True for an ephemeral table */ + Bool useRandomRowid:1; /* Generate new record numbers semi-randomly */ + Bool isOrdered:1; /* True if the table is not BTREE_UNORDERED */ + Bool noReuse:1; /* OpenEphemeral may not reuse this cursor */ + Bool colCache:1; /* pCache pointer is initialized and non-NULL */ + u16 seekHit; /* See the OP_SeekHit and OP_IfNoHope opcodes */ + union { /* pBtx for isEphermeral. pAltMap otherwise */ + Btree *pBtx; /* Separate file holding temporary table */ + u32 *aAltMap; /* Mapping from table to index column numbers */ + } ub; + i64 seqCount; /* Sequence counter */ - /* Cached information about the header for the data record that the - ** cursor is currently pointing to. Only valid if cacheStatus matches + /* Cached OP_Column parse information is only valid if cacheStatus matches ** Vdbe.cacheCtr. Vdbe.cacheCtr will never take on the value of - ** CACHE_STALE and so setting cacheStatus=CACHE_STALE guarantees that - ** the cache is out of date. - ** - ** aRow might point to (ephemeral) data for the current row, or it might - ** be NULL. - */ - u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ - u32 payloadSize; /* Total number of bytes in the record */ - u32 szRow; /* Byte available in aRow */ - u32 iHdrOffset; /* Offset to next unparsed byte of the header */ - const u8 *aRow; /* Data for the current row, if all on one page */ - u32 *aOffset; /* Pointer to aType[nField] */ - u32 aType[1]; /* Type values for all entries in the record */ + ** CACHE_STALE (0) and so setting cacheStatus=CACHE_STALE guarantees that + ** the cache is out of date. */ + u32 cacheStatus; /* Cache is valid if this matches Vdbe.cacheCtr */ + int seekResult; /* Result of previous sqlite3BtreeMoveto() or 0 + ** if there have been no prior seeks on the cursor. */ + /* seekResult does not distinguish between "no seeks have ever occurred + ** on this cursor" and "the most recent seek was an exact match". + ** For CURTYPE_PSEUDO, seekResult is the register holding the record */ + + /* When a new VdbeCursor is allocated, only the fields above are zeroed. + ** The fields that follow are uninitialized, and must be individually + ** initialized prior to first use. */ + VdbeCursor *pAltCursor; /* Associated index cursor from which to read */ + union { + BtCursor *pCursor; /* CURTYPE_BTREE or _PSEUDO. Btree cursor */ + sqlite3_vtab_cursor *pVCur; /* CURTYPE_VTAB. Vtab cursor */ + VdbeSorter *pSorter; /* CURTYPE_SORTER. Sorter object */ + } uc; + KeyInfo *pKeyInfo; /* Info about index keys needed by index cursors */ + u32 iHdrOffset; /* Offset to next unparsed byte of the header */ + Pgno pgnoRoot; /* Root page of the open btree cursor */ + i16 nField; /* Number of fields in the header */ + u16 nHdrParsed; /* Number of header fields parsed so far */ + i64 movetoTarget; /* Argument to the deferred sqlite3BtreeMoveto() */ + u32 *aOffset; /* Pointer to aType[nField] */ + const u8 *aRow; /* Data for the current row, if all on one page */ + u32 payloadSize; /* Total number of bytes in the record */ + u32 szRow; /* Byte available in aRow */ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + u64 maskUsed; /* Mask of columns used by this cursor */ +#endif + VdbeTxtBlbCache *pCache; /* Cache of large TEXT or BLOB values */ + /* 2*nField extra array elements allocated for aType[], beyond the one ** static element declared in the structure. nField total array slots for ** aType[] and nField+1 array slots for aOffset[] */ + u32 aType[1]; /* Type values record decode. MUST BE LAST */ +}; + +/* Return true if P is a null-only cursor +*/ +#define IsNullCursor(P) \ + ((P)->eCurType==CURTYPE_PSEUDO && (P)->nullRow && (P)->seekResult==0) + +/* +** A value for VdbeCursor.cacheStatus that means the cache is always invalid. +*/ +#define CACHE_STALE 0 + +/* +** Large TEXT or BLOB values can be slow to load, so we want to avoid +** loading them more than once. For that reason, large TEXT and BLOB values +** can be stored in a cache defined by this object, and attached to the +** VdbeCursor using the pCache field. +*/ +struct VdbeTxtBlbCache { + char *pCValue; /* A RCStr buffer to hold the value */ + i64 iOffset; /* File offset of the row being cached */ + int iCol; /* Column for which the cache is valid */ + u32 cacheStatus; /* Vdbe.cacheCtr value */ + u32 colCacheCtr; /* Column cache counter */ }; /* ** When a sub-program is executed (OP_Program), a structure of this type ** is allocated to store the current value of the program counter, as ** well as the current memory cell array and various other frame specific -** values stored in the Vdbe struct. When the sub-program is finished, +** values stored in the Vdbe struct. When the sub-program is finished, ** these values are copied back to the Vdbe from the VdbeFrame structure, ** restoring the state of the VM to as it was before the sub-program ** began executing. @@ -17558,59 +23507,61 @@ struct VdbeFrame { Vdbe *v; /* VM this frame belongs to */ VdbeFrame *pParent; /* Parent of this frame, or NULL if parent is main */ Op *aOp; /* Program instructions for parent frame */ - i64 *anExec; /* Event counters from parent frame */ Mem *aMem; /* Array of memory cells for parent frame */ - u8 *aOnceFlag; /* Array of OP_Once flags for parent frame */ VdbeCursor **apCsr; /* Array of Vdbe cursors for parent frame */ + u8 *aOnce; /* Bitmask used by OP_Once */ void *token; /* Copy of SubProgram.token */ i64 lastRowid; /* Last insert rowid (sqlite3.lastRowid) */ AuxData *pAuxData; /* Linked list of auxdata allocations */ +#if SQLITE_DEBUG + u32 iFrameMagic; /* magic number for sanity checking */ +#endif int nCursor; /* Number of entries in apCsr */ int pc; /* Program Counter in parent (calling) frame */ int nOp; /* Size of aOp array */ int nMem; /* Number of entries in aMem */ - int nOnceFlag; /* Number of entries in aOnceFlag */ int nChildMem; /* Number of memory cells for child frame */ int nChildCsr; /* Number of cursors for child frame */ - int nChange; /* Statement changes (Vdbe.nChange) */ - int nDbChange; /* Value of db->nChange */ + i64 nChange; /* Statement changes (Vdbe.nChange) */ + i64 nDbChange; /* Value of db->nChange */ }; -#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) +/* Magic number for sanity checking on VdbeFrame objects */ +#define SQLITE_FRAME_MAGIC 0x879fb71e /* -** A value for VdbeCursor.cacheValid that means the cache is always invalid. +** Return a pointer to the array of registers allocated for use +** by a VdbeFrame. */ -#define CACHE_STALE 0 +#define VdbeFrameMem(p) ((Mem *)&((u8 *)p)[ROUND8(sizeof(VdbeFrame))]) /* ** Internally, the vdbe manipulates nearly all SQL values as Mem ** structures. Each Mem struct may cache multiple representations (string, ** integer etc.) of the same value. */ -struct Mem { +struct sqlite3_value { union MemValue { double r; /* Real value used when MEM_Real is set in flags */ i64 i; /* Integer value used when MEM_Int is set in flags */ - int nZero; /* Used when bit MEM_Zero is set in flags */ + int nZero; /* Extra zero bytes when MEM_Zero and MEM_Blob set */ + const char *zPType; /* Pointer type when MEM_Term|MEM_Subtype|MEM_Null */ FuncDef *pDef; /* Used only when flags==MEM_Agg */ - RowSet *pRowSet; /* Used only when flags==MEM_RowSet */ - VdbeFrame *pFrame; /* Used when flags==MEM_Frame */ } u; + char *z; /* String or BLOB value */ + int n; /* Number of characters in string value, excluding '\0' */ u16 flags; /* Some combination of MEM_Null, MEM_Str, MEM_Dyn, etc. */ u8 enc; /* SQLITE_UTF8, SQLITE_UTF16BE, SQLITE_UTF16LE */ u8 eSubtype; /* Subtype for this value */ - int n; /* Number of characters in string value, excluding '\0' */ - char *z; /* String or BLOB value */ /* ShallowCopy only needs to copy the information above */ - char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ + sqlite3 *db; /* The associated database connection */ int szMalloc; /* Size of the zMalloc allocation */ u32 uTemp; /* Transient storage for serial_type in OP_MakeRecord */ - sqlite3 *db; /* The associated database connection */ + char *zMalloc; /* Space to hold MEM_Str or MEM_Blob if szMalloc>0 */ void (*xDel)(void*);/* Destructor for Mem.z - only valid if MEM_Dyn */ #ifdef SQLITE_DEBUG Mem *pScopyFrom; /* This Mem is a shallow copy of pScopyFrom */ - void *pFiller; /* So that sizeof(Mem) is a multiple of 8 */ + u16 mScopyFlags; /* flags value immediately after the shallow copy */ #endif }; @@ -17618,55 +23569,85 @@ struct Mem { ** Size of struct Mem not including the Mem.zMalloc member or anything that ** follows. */ -#define MEMCELLSIZE offsetof(Mem,zMalloc) +#define MEMCELLSIZE offsetof(Mem,db) -/* One or more of the following flags are set to indicate the validOK +/* One or more of the following flags are set to indicate the ** representations of the value stored in the Mem struct. ** +** * MEM_Null An SQL NULL value +** +** * MEM_Null|MEM_Zero An SQL NULL with the virtual table +** UPDATE no-change flag set +** +** * MEM_Null|MEM_Term| An SQL NULL, but also contains a +** MEM_Subtype pointer accessible using +** sqlite3_value_pointer(). +** +** * MEM_Null|MEM_Cleared Special SQL NULL that compares non-equal +** to other NULLs even using the IS operator. +** +** * MEM_Str A string, stored in Mem.z with +** length Mem.n. Zero-terminated if +** MEM_Term is set. This flag is +** incompatible with MEM_Blob and +** MEM_Null, but can appear with MEM_Int, +** MEM_Real, and MEM_IntReal. +** +** * MEM_Blob A blob, stored in Mem.z length Mem.n. +** Incompatible with MEM_Str, MEM_Null, +** MEM_Int, MEM_Real, and MEM_IntReal. +** +** * MEM_Blob|MEM_Zero A blob in Mem.z of length Mem.n plus +** MEM.u.i extra 0x00 bytes at the end. +** +** * MEM_Int Integer stored in Mem.u.i. +** +** * MEM_Real Real stored in Mem.u.r. +** +** * MEM_IntReal Real stored as an integer in Mem.u.i. +** ** If the MEM_Null flag is set, then the value is an SQL NULL value. -** No other flags may be set in this case. +** For a pointer type created using sqlite3_bind_pointer() or +** sqlite3_result_pointer() the MEM_Term and MEM_Subtype flags are also set. ** ** If the MEM_Str flag is set then Mem.z points at a string representation. ** Usually this is encoded in the same unicode encoding as the main ** database (see below for exceptions). If the MEM_Term flag is also -** set, then the string is nul terminated. The MEM_Int and MEM_Real +** set, then the string is nul terminated. The MEM_Int and MEM_Real ** flags may coexist with the MEM_Str flag. */ -#define MEM_Null 0x0001 /* Value is NULL */ +#define MEM_Undefined 0x0000 /* Value is undefined */ +#define MEM_Null 0x0001 /* Value is NULL (or a pointer) */ #define MEM_Str 0x0002 /* Value is a string */ #define MEM_Int 0x0004 /* Value is an integer */ #define MEM_Real 0x0008 /* Value is a real number */ #define MEM_Blob 0x0010 /* Value is a BLOB */ -#define MEM_AffMask 0x001f /* Mask of affinity bits */ -#define MEM_RowSet 0x0020 /* Value is a RowSet object */ -#define MEM_Frame 0x0040 /* Value is a VdbeFrame object */ -#define MEM_Undefined 0x0080 /* Value is undefined */ -#define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ -#define MEM_TypeMask 0x81ff /* Mask of type bits */ +#define MEM_IntReal 0x0020 /* MEM_Int that stringifies like MEM_Real */ +#define MEM_AffMask 0x003f /* Mask of affinity bits */ - -/* Whenever Mem contains a valid string or blob representation, one of -** the following flags must be set to determine the memory management -** policy for Mem.z. The MEM_Term flag tells us whether or not the -** string is \000 or \u0000 terminated +/* Extra bits that modify the meanings of the core datatypes above */ -#define MEM_Term 0x0200 /* String rep is nul terminated */ -#define MEM_Dyn 0x0400 /* Need to call Mem.xDel() on Mem.z */ -#define MEM_Static 0x0800 /* Mem.z points to a static string */ -#define MEM_Ephem 0x1000 /* Mem.z points to an ephemeral string */ -#define MEM_Agg 0x2000 /* Mem.z points to an agg function context */ -#define MEM_Zero 0x4000 /* Mem.i contains count of 0s appended to blob */ -#define MEM_Subtype 0x8000 /* Mem.eSubtype is valid */ -#ifdef SQLITE_OMIT_INCRBLOB - #undef MEM_Zero - #define MEM_Zero 0x0000 -#endif +#define MEM_FromBind 0x0040 /* Value originates from sqlite3_bind() */ + /* 0x0080 // Available */ +#define MEM_Cleared 0x0100 /* NULL set by OP_Null, not from data */ +#define MEM_Term 0x0200 /* String in Mem.z is zero terminated */ +#define MEM_Zero 0x0400 /* Mem.i contains count of 0s appended to blob */ +#define MEM_Subtype 0x0800 /* Mem.eSubtype is valid */ +#define MEM_TypeMask 0x0dbf /* Mask of type bits */ + +/* Bits that determine the storage for Mem.z for a string or blob or +** aggregate accumulator. +*/ +#define MEM_Dyn 0x1000 /* Need to call Mem.xDel() on Mem.z */ +#define MEM_Static 0x2000 /* Mem.z points to a static string */ +#define MEM_Ephem 0x4000 /* Mem.z points to an ephemeral string */ +#define MEM_Agg 0x8000 /* Mem.z points to an agg function context */ /* Return TRUE if Mem X contains dynamically allocated content - anything ** that needs to be deallocated to avoid a leak. */ #define VdbeMemDynamic(X) \ - (((X)->flags&(MEM_Agg|MEM_Dyn|MEM_RowSet|MEM_Frame))!=0) + (((X)->flags&(MEM_Agg|MEM_Dyn))!=0) /* ** Clear any existing type flags from a Mem and replace them with f @@ -17675,26 +23656,37 @@ struct Mem { ((p)->flags = ((p)->flags&~(MEM_TypeMask|MEM_Zero))|f) /* -** Return true if a memory cell is not marked as invalid. This macro +** True if Mem X is a NULL-nochng type. +*/ +#define MemNullNochng(X) \ + (((X)->flags&MEM_TypeMask)==(MEM_Null|MEM_Zero) \ + && (X)->n==0 && (X)->u.nZero==0) + +/* +** Return true if a memory cell has been initialized and is valid. ** is for use inside assert() statements only. +** +** A Memory cell is initialized if at least one of the +** MEM_Null, MEM_Str, MEM_Int, MEM_Real, MEM_Blob, or MEM_IntReal bits +** is set. It is "undefined" if all those bits are zero. */ #ifdef SQLITE_DEBUG -#define memIsValid(M) ((M)->flags & MEM_Undefined)==0 +#define memIsValid(M) ((M)->flags & MEM_AffMask)!=0 #endif /* -** Each auxiliary data pointer stored by a user defined function +** Each auxiliary data pointer stored by a user defined function ** implementation calling sqlite3_set_auxdata() is stored in an instance ** of this structure. All such structures associated with a single VM ** are stored in a linked list headed at Vdbe.pAuxData. All are destroyed ** when the VM is halted (if not before). */ struct AuxData { - int iOp; /* Instruction number of OP_Function opcode */ - int iArg; /* Index of function argument. */ + int iAuxOp; /* Instruction number of OP_Function opcode */ + int iAuxArg; /* Index of function argument. */ void *pAux; /* Aux data pointer */ - void (*xDelete)(void *); /* Destructor for the aux data */ - AuxData *pNext; /* Next element in list */ + void (*xDeleteAux)(void*); /* Destructor for the aux data */ + AuxData *pNextAux; /* Next element in list */ }; /* @@ -17717,32 +23709,32 @@ struct sqlite3_context { Vdbe *pVdbe; /* The VM that owns this context */ int iOp; /* Instruction number of OP_Function */ int isError; /* Error code returned by the function. */ + u8 enc; /* Encoding to use for results */ u8 skipFlag; /* Skip accumulator loading if true */ - u8 fErrorOrAux; /* isError!=0 or pVdbe->pAuxData modified */ u8 argc; /* Number of arguments */ sqlite3_value *argv[1]; /* Argument set */ }; -/* -** An Explain object accumulates indented output which is helpful -** in describing recursive data structures. -*/ -struct Explain { - Vdbe *pVdbe; /* Attach the explanation to this Vdbe */ - StrAccum str; /* The string being accumulated */ - int nIndent; /* Number of elements in aIndent */ - u16 aIndent[100]; /* Levels of indentation */ - char zBase[100]; /* Initial space */ -}; - /* A bitfield type for use inside of structures. Always follow with :N where ** N is the number of bits. */ typedef unsigned bft; /* Bit Field Type */ +/* The ScanStatus object holds a single value for the +** sqlite3_stmt_scanstatus() interface. +** +** aAddrRange[]: +** This array is used by ScanStatus elements associated with EQP +** notes that make an SQLITE_SCANSTAT_NCYCLE value available. It is +** an array of up to 3 ranges of VM addresses for which the Vdbe.anCycle[] +** values should be summed to calculate the NCYCLE value. Each pair of +** integer addresses is a start and end address (both inclusive) for a range +** instructions. A start value of 0 indicates an empty range. +*/ typedef struct ScanStatus ScanStatus; struct ScanStatus { int addrExplain; /* OP_Explain for loop */ + int aAddrRange[6]; int addrLoop; /* Address of "loops" counter */ int addrVisit; /* Address of "rows visited" counter */ int iSelectID; /* The "Select-ID" for this loop */ @@ -17750,6 +23742,19 @@ struct ScanStatus { char *zName; /* Name of table or index */ }; +/* The DblquoteStr object holds the text of a double-quoted +** string for a prepared statement. A linked list of these objects +** is constructed during statement parsing and is held on Vdbe.pDblStr. +** When computing a normalized SQL statement for an SQL statement, that +** list is consulted for each double-quoted identifier to see if the +** identifier should really be a string literal. +*/ +typedef struct DblquoteStr DblquoteStr; +struct DblquoteStr { + DblquoteStr *pNextStr; /* Next string literal in the list */ + char z[8]; /* Dequoted value for the string */ +}; + /* ** An instance of the virtual machine. This structure contains the complete ** state of the virtual machine. @@ -17759,80 +23764,86 @@ struct ScanStatus { */ struct Vdbe { sqlite3 *db; /* The database connection that owns this statement */ - Op *aOp; /* Space to hold the virtual machine's program */ - Mem *aMem; /* The memory locations */ - Mem **apArg; /* Arguments to currently executing user function */ - Mem *aColName; /* Column names to return */ - Mem *pResultSet; /* Pointer to an array of results */ + Vdbe **ppVPrev,*pVNext; /* Linked list of VDBEs with the same Vdbe.db */ Parse *pParse; /* Parsing context used to create this Vdbe */ - int nMem; /* Number of memory locations currently allocated */ - int nOp; /* Number of instructions in the program */ - int nCursor; /* Number of slots in apCsr[] */ - u32 magic; /* Magic number for sanity checking */ - char *zErrMsg; /* Error message written here */ - Vdbe *pPrev,*pNext; /* Linked list of VDBEs with the same Vdbe.db */ - VdbeCursor **apCsr; /* One element of this array for each open cursor */ - Mem *aVar; /* Values for the OP_Variable opcode. */ - char **azVar; /* Name of variables */ ynVar nVar; /* Number of entries in aVar[] */ - ynVar nzVar; /* Number of entries in azVar[] */ + int nMem; /* Number of memory locations currently allocated */ + int nCursor; /* Number of slots in apCsr[] */ u32 cacheCtr; /* VdbeCursor row cache generation counter */ int pc; /* The program counter */ int rc; /* Value to return */ -#ifdef SQLITE_DEBUG - int rcApp; /* errcode set by sqlite3_result_error_code() */ -#endif - u16 nResColumn; /* Number of columns in one row of the result set */ - u8 errorAction; /* Recovery action to do in case of an error */ - bft expired:1; /* True if the VM needs to be recompiled */ - bft doingRerun:1; /* True if rerunning after an auto-reprepare */ - u8 minWriteFileFormat; /* Minimum file format for writable database files */ - bft explain:2; /* True if EXPLAIN present on SQL command */ - bft changeCntOn:1; /* True to update the change-counter */ - bft runOnlyOnce:1; /* Automatically expire on reset */ - bft usesStmtJournal:1; /* True if uses a statement journal */ - bft readOnly:1; /* True for statements that do not write */ - bft bIsReader:1; /* True for statements that read */ - bft isPrepareV2:1; /* True if prepared with prepare_v2() */ - int nChange; /* Number of db changes made since last reset */ - yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ - yDbMask lockMask; /* Subset of btreeMask that requires a lock */ - int iStatement; /* Statement number (or 0 if has not opened stmt) */ - u32 aCounter[5]; /* Counters used by sqlite3_stmt_status() */ -#ifndef SQLITE_OMIT_TRACE - i64 startTime; /* Time when query started - used for profiling */ -#endif + i64 nChange; /* Number of db changes made since last reset */ + int iStatement; /* Statement number (or 0 if has no opened stmt) */ i64 iCurrentTime; /* Value of julianday('now') for this statement */ i64 nFkConstraint; /* Number of imm. FK constraints this VM */ i64 nStmtDefCons; /* Number of def. constraints when stmt started */ i64 nStmtDefImmCons; /* Number of def. imm constraints when stmt started */ + Mem *aMem; /* The memory locations */ + Mem **apArg; /* Arguments to currently executing user function */ + VdbeCursor **apCsr; /* One element of this array for each open cursor */ + Mem *aVar; /* Values for the OP_Variable opcode. */ + + /* When allocating a new Vdbe object, all of the fields below should be + ** initialized to zero or NULL */ + + Op *aOp; /* Space to hold the virtual machine's program */ + int nOp; /* Number of instructions in the program */ + int nOpAlloc; /* Slots allocated for aOp[] */ + Mem *aColName; /* Column names to return */ + Mem *pResultRow; /* Current output row */ + char *zErrMsg; /* Error message written here */ + VList *pVList; /* Name of variables */ +#ifndef SQLITE_OMIT_TRACE + i64 startTime; /* Time when query started - used for profiling */ +#endif +#ifdef SQLITE_DEBUG + int rcApp; /* errcode set by sqlite3_result_error_code() */ + u32 nWrite; /* Number of write operations that have occurred */ +#endif + u16 nResColumn; /* Number of columns in one row of the result set */ + u16 nResAlloc; /* Column slots allocated to aColName[] */ + u8 errorAction; /* Recovery action to do in case of an error */ + u8 minWriteFileFormat; /* Minimum file format for writable database files */ + u8 prepFlags; /* SQLITE_PREPARE_* flags */ + u8 eVdbeState; /* On of the VDBE_*_STATE values */ + bft expired:2; /* 1: recompile VM immediately 2: when convenient */ + bft explain:2; /* 0: normal, 1: EXPLAIN, 2: EXPLAIN QUERY PLAN */ + bft changeCntOn:1; /* True to update the change-counter */ + bft usesStmtJournal:1; /* True if uses a statement journal */ + bft readOnly:1; /* True for statements that do not write */ + bft bIsReader:1; /* True for statements that read */ + bft haveEqpOps:1; /* Bytecode supports EXPLAIN QUERY PLAN */ + yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ + yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ char *zSql; /* Text of the SQL statement that generated this */ +#ifdef SQLITE_ENABLE_NORMALIZE + char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ +#endif void *pFree; /* Free this when deleting the vdbe */ VdbeFrame *pFrame; /* Parent frame */ VdbeFrame *pDelFrame; /* List of frame objects to free on VM reset */ int nFrame; /* Number of frames in pFrame list */ u32 expmask; /* Binding to these vars invalidates VM */ SubProgram *pProgram; /* Linked list of all sub-programs used by VM */ - int nOnceFlag; /* Size of array aOnceFlag[] */ - u8 *aOnceFlag; /* Flags for OP_Once */ AuxData *pAuxData; /* Linked list of auxdata allocations */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS - i64 *anExec; /* Number of times each op has been executed */ int nScan; /* Entries in aScan[] */ ScanStatus *aScan; /* Scan definitions for sqlite3_stmt_scanstatus() */ #endif }; /* -** The following are allowed values for Vdbe.magic +** The following are allowed values for Vdbe.eVdbeState */ -#define VDBE_MAGIC_INIT 0x26bceaa5 /* Building a VDBE program */ -#define VDBE_MAGIC_RUN 0xbdf20da3 /* VDBE is ready to execute */ -#define VDBE_MAGIC_HALT 0x519c2973 /* VDBE has completed execution */ -#define VDBE_MAGIC_DEAD 0xb606c3c8 /* The VDBE has been deallocated */ +#define VDBE_INIT_STATE 0 /* Prepared statement under construction */ +#define VDBE_READY_STATE 1 /* Ready to run but not yet started */ +#define VDBE_RUN_STATE 2 /* Run in progress */ +#define VDBE_HALT_STATE 3 /* Finished. Need reset() or finalize() */ /* -** Structure used to store the context required by the +** Structure used to store the context required by the ** sqlite3_preupdate_*() API functions. */ struct PreUpdate { @@ -17844,35 +23855,75 @@ struct PreUpdate { UnpackedRecord *pUnpacked; /* Unpacked version of aRecord[] */ UnpackedRecord *pNewUnpacked; /* Unpacked version of new.* record */ int iNewReg; /* Register for new.* values */ + int iBlobWrite; /* Value returned by preupdate_blobwrite() */ i64 iKey1; /* First key value passed to hook */ i64 iKey2; /* Second key value passed to hook */ - int iPKey; /* If not negative index of IPK column */ Mem *aNew; /* Array of new.* values */ + Table *pTab; /* Schema object being updated */ + Index *pPk; /* PK index if pTab is WITHOUT ROWID */ + sqlite3_value **apDflt; /* Array of default values, if required */ }; +/* +** An instance of this object is used to pass an vector of values into +** OP_VFilter, the xFilter method of a virtual table. The vector is the +** set of values on the right-hand side of an IN constraint. +** +** The value as passed into xFilter is an sqlite3_value with a "pointer" +** type, such as is generated by sqlite3_result_pointer() and read by +** sqlite3_value_pointer. Such values have MEM_Term|MEM_Subtype|MEM_Null +** and a subtype of 'p'. The sqlite3_vtab_in_first() and _next() interfaces +** know how to use this object to step through all the values in the +** right operand of the IN constraint. +*/ +typedef struct ValueList ValueList; +struct ValueList { + BtCursor *pCsr; /* An ephemeral table holding all values */ + sqlite3_value *pOut; /* Register to hold each decoded output value */ +}; + +/* Size of content associated with serial types that fit into a +** single-byte varint. +*/ +#ifndef SQLITE_AMALGAMATION +SQLITE_PRIVATE const u8 sqlite3SmallTypeSizes[]; +#endif + /* ** Function prototypes */ SQLITE_PRIVATE void sqlite3VdbeError(Vdbe*, const char *, ...); SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *, VdbeCursor*); +SQLITE_PRIVATE void sqlite3VdbeFreeCursorNN(Vdbe*,VdbeCursor*); void sqliteVdbePopStack(Vdbe*,int); -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor**, int*); +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeHandleMovedCursor(VdbeCursor *p); +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor*); SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor*); -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE*, int, Op*); -#endif SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32); SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8); -SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem*, int, u32*); -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(unsigned char*, Mem*, u32); -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); +#ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT +SQLITE_PRIVATE u64 sqlite3FloatSwap(u64 in); +# define swapMixedEndianFloat(X) X = sqlite3FloatSwap(X) +#else +# define swapMixedEndianFloat(X) +#endif +SQLITE_PRIVATE void sqlite3VdbeSerialGet(const unsigned char*, u32, Mem*); SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3*, AuxData**, int, int); int sqlite2BtreeKeyCompare(BtCursor *, const void *, int, int, int *); SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare(sqlite3*,VdbeCursor*,UnpackedRecord*,int*); SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3*, BtCursor*, i64*); SQLITE_PRIVATE int sqlite3VdbeExec(Vdbe*); +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE int sqlite3VdbeNextOpcode(Vdbe*,Mem*,int,int*,int*,Op**); +SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3*,Op*); +#endif +#if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) +SQLITE_PRIVATE char *sqlite3VdbeDisplayComment(sqlite3*,const Op*,const char*); +#endif +#if !defined(SQLITE_OMIT_EXPLAIN) SQLITE_PRIVATE int sqlite3VdbeList(Vdbe*); +#endif SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe*); SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *, int); SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem*); @@ -17880,37 +23931,60 @@ SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem*, const Mem*); SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem*, const Mem*, int); SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem*, Mem*); SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem*); -SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, int, u8, void(*)(void*)); +SQLITE_PRIVATE int sqlite3VdbeMemSetStr(Mem*, const char*, i64, u8, void(*)(void*)); SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem*, i64); #ifdef SQLITE_OMIT_FLOATING_POINT # define sqlite3VdbeMemSetDouble sqlite3VdbeMemSetInt64 #else SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem*, double); #endif +SQLITE_PRIVATE void sqlite3VdbeMemSetPointer(Mem*, void*, const char*, void(*)(void*)); SQLITE_PRIVATE void sqlite3VdbeMemInit(Mem*,sqlite3*,u16); SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem*); +#ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem*,int); -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem*); +#else +SQLITE_PRIVATE int sqlite3VdbeMemSetZeroBlob(Mem*,int); +#endif +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem*); +#endif +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem*); +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem*, u8, u8); -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem*); +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem*); SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem*); SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem*); +SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem*, int ifNull); SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem*); SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem*); -SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem*,u8,u8); -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,int,Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem*,u8,u8); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree(BtCursor*,u32,u32,Mem*); +SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset(BtCursor*,u32,Mem*); SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p); +SQLITE_PRIVATE void sqlite3VdbeMemReleaseMalloc(Mem*p); SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem*, FuncDef*); +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem*, Mem*, FuncDef*); +#endif +#if !defined(SQLITE_OMIT_EXPLAIN) || defined(SQLITE_ENABLE_BYTECODE_VTAB) SQLITE_PRIVATE const char *sqlite3OpcodeName(int); +#endif SQLITE_PRIVATE int sqlite3VdbeMemGrow(Mem *pMem, int n, int preserve); SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int n); SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *, int); -SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE int sqlite3VdbeFrameIsValid(VdbeFrame*); +#endif +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void*); /* Destructor on Mem */ +SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame*); /* Actually deletes the Frame */ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK -SQLITE_PRIVATE void sqlite3VdbePreUpdateHook(Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int); +SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( + Vdbe*,VdbeCursor*,int,const char*,Table*,i64,int,int); #endif SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p); @@ -17918,12 +23992,22 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit(sqlite3 *, int, VdbeCursor *); SQLITE_PRIVATE void sqlite3VdbeSorterReset(sqlite3 *, VdbeSorter *); SQLITE_PRIVATE void sqlite3VdbeSorterClose(sqlite3 *, VdbeCursor *); SQLITE_PRIVATE int sqlite3VdbeSorterRowkey(const VdbeCursor *, Mem *); -SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *, int *); +SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *, const VdbeCursor *); SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *, int *); SQLITE_PRIVATE int sqlite3VdbeSorterWrite(const VdbeCursor *, Mem *); SQLITE_PRIVATE int sqlite3VdbeSorterCompare(const VdbeCursor *, Mem *, int, int *); -#if !defined(SQLITE_OMIT_SHARED_CACHE) +SQLITE_PRIVATE void sqlite3VdbeValueListFree(void*); + +#ifdef SQLITE_DEBUG +SQLITE_PRIVATE void sqlite3VdbeIncrWriteCounter(Vdbe*, VdbeCursor*); +SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe*); +#else +# define sqlite3VdbeIncrWriteCounter(V,C) +# define sqlite3VdbeAssertAbortable(V) +#endif + +#if !defined(SQLITE_OMIT_SHARED_CACHE) SQLITE_PRIVATE void sqlite3VdbeEnter(Vdbe*); #else # define sqlite3VdbeEnter(X) @@ -17946,12 +24030,14 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *, int); # define sqlite3VdbeCheckFk(p,i) 0 #endif -SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); #ifdef SQLITE_DEBUG SQLITE_PRIVATE void sqlite3VdbePrintSql(Vdbe*); -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf); +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr); +#endif +#ifndef SQLITE_OMIT_UTF16 +SQLITE_PRIVATE int sqlite3VdbeMemTranslate(Mem*, u8); +SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); #endif -SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem); #ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); @@ -17961,7 +24047,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *); #define ExpandBlob(P) SQLITE_OK #endif -#endif /* !defined(_VDBEINT_H_) */ +#endif /* !defined(SQLITE_VDBEINT_H) */ /************** End of vdbeInt.h *********************************************/ /************** Continuing where we left off in status.c *********************/ @@ -18072,7 +24158,6 @@ SQLITE_PRIVATE void sqlite3StatusHighwater(int op, int X){ : sqlite3MallocMutex()) ); assert( op==SQLITE_STATUS_MALLOC_SIZE || op==SQLITE_STATUS_PAGECACHE_SIZE - || op==SQLITE_STATUS_SCRATCH_SIZE || op==SQLITE_STATUS_PARSER_STACK ); if( newValue>wsdStat.mxValue[op] ){ wsdStat.mxValue[op] = newValue; @@ -18082,7 +24167,7 @@ SQLITE_PRIVATE void sqlite3StatusHighwater(int op, int X){ /* ** Query status information. */ -SQLITE_API int SQLITE_STDCALL sqlite3_status64( +SQLITE_API int sqlite3_status64( int op, sqlite3_int64 *pCurrent, sqlite3_int64 *pHighwater, @@ -18107,8 +24192,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status64( (void)pMutex; /* Prevent warning when SQLITE_THREADSAFE=0 */ return SQLITE_OK; } -SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ - sqlite3_int64 iCur, iHwtr; +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag){ + sqlite3_int64 iCur = 0, iHwtr = 0; int rc; #ifdef SQLITE_ENABLE_API_ARMOR if( pCurrent==0 || pHighwater==0 ) return SQLITE_MISUSE_BKPT; @@ -18121,10 +24206,36 @@ SQLITE_API int SQLITE_STDCALL sqlite3_status(int op, int *pCurrent, int *pHighwa return rc; } +/* +** Return the number of LookasideSlot elements on the linked list +*/ +static u32 countLookasideSlots(LookasideSlot *p){ + u32 cnt = 0; + while( p ){ + p = p->pNext; + cnt++; + } + return cnt; +} + +/* +** Count the number of slots of lookaside memory that are outstanding +*/ +SQLITE_PRIVATE int sqlite3LookasideUsed(sqlite3 *db, int *pHighwater){ + u32 nInit = countLookasideSlots(db->lookaside.pInit); + u32 nFree = countLookasideSlots(db->lookaside.pFree); +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + nInit += countLookasideSlots(db->lookaside.pSmallInit); + nFree += countLookasideSlots(db->lookaside.pSmallFree); +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( pHighwater ) *pHighwater = db->lookaside.nSlot - nInit; + return db->lookaside.nSlot - (nInit+nFree); +} + /* ** Query status information for a single database connection */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_status( +SQLITE_API int sqlite3_db_status( sqlite3 *db, /* The database connection whose status is desired */ int op, /* Status verb */ int *pCurrent, /* Write current value here */ @@ -18140,10 +24251,24 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( sqlite3_mutex_enter(db->mutex); switch( op ){ case SQLITE_DBSTATUS_LOOKASIDE_USED: { - *pCurrent = db->lookaside.nOut; - *pHighwater = db->lookaside.mxOut; + *pCurrent = sqlite3LookasideUsed(db, pHighwater); if( resetFlag ){ - db->lookaside.mxOut = db->lookaside.nOut; + LookasideSlot *p = db->lookaside.pFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = db->lookaside.pInit; + db->lookaside.pInit = db->lookaside.pFree; + db->lookaside.pFree = 0; + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + p = db->lookaside.pSmallFree; + if( p ){ + while( p->pNext ) p = p->pNext; + p->pNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = 0; + } +#endif } break; } @@ -18164,11 +24289,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( break; } - /* + /* ** Return an approximation for the amount of memory currently used ** by all pagers associated with the given database connection. The ** highwater mark is meaningless and is returned as zero. */ + case SQLITE_DBSTATUS_CACHE_USED_SHARED: case SQLITE_DBSTATUS_CACHE_USED: { int totalUsed = 0; int i; @@ -18177,7 +24303,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( Btree *pBt = db->aDb[i].pBt; if( pBt ){ Pager *pPager = sqlite3BtreePager(pBt); - totalUsed += sqlite3PagerMemUsed(pPager); + int nByte = sqlite3PagerMemUsed(pPager); + if( op==SQLITE_DBSTATUS_CACHE_USED_SHARED ){ + nByte = nByte / sqlite3BtreeConnectionCount(pBt); + } + totalUsed += nByte; } } sqlite3BtreeLeaveAll(db); @@ -18197,13 +24327,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( sqlite3BtreeEnterAll(db); db->pnBytesFreed = &nByte; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; if( ALWAYS(pSchema!=0) ){ HashElem *p; nByte += sqlite3GlobalConfig.m.xRoundup(sizeof(HashElem)) * ( - pSchema->tblHash.count + pSchema->tblHash.count + pSchema->trigHash.count + pSchema->idxHash.count + pSchema->fkeyHash.count @@ -18222,6 +24354,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( } } db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; sqlite3BtreeLeaveAll(db); *pHighwater = 0; @@ -18239,10 +24372,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( int nByte = 0; /* Used to accumulate return value */ db->pnBytesFreed = &nByte; - for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pNext){ - sqlite3VdbeClearObject(db, pVdbe); - sqlite3DbFree(db, pVdbe); + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; + for(pVdbe=db->pVdbe; pVdbe; pVdbe=pVdbe->pVNext){ + sqlite3VdbeDelete(pVdbe); } + db->lookaside.pEnd = db->lookaside.pTrueEnd; db->pnBytesFreed = 0; *pHighwater = 0; /* IMP: R-64479-57858 */ @@ -18253,14 +24388,17 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( /* ** Set *pCurrent to the total cache hits or misses encountered by all - ** pagers the database handle is connected to. *pHighwater is always set + ** pagers the database handle is connected to. *pHighwater is always set ** to zero. */ + case SQLITE_DBSTATUS_CACHE_SPILL: + op = SQLITE_DBSTATUS_CACHE_WRITE+1; + /* no break */ deliberate_fall_through case SQLITE_DBSTATUS_CACHE_HIT: case SQLITE_DBSTATUS_CACHE_MISS: case SQLITE_DBSTATUS_CACHE_WRITE:{ int i; - int nRet = 0; + u64 nRet = 0; assert( SQLITE_DBSTATUS_CACHE_MISS==SQLITE_DBSTATUS_CACHE_HIT+1 ); assert( SQLITE_DBSTATUS_CACHE_WRITE==SQLITE_DBSTATUS_CACHE_HIT+2 ); @@ -18273,7 +24411,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( *pHighwater = 0; /* IMP: R-42420-56072 */ /* IMP: R-54100-20147 */ /* IMP: R-29431-39229 */ - *pCurrent = nRet; + *pCurrent = (int)nRet & 0x7fffffff; break; } @@ -18309,7 +24447,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( ** ************************************************************************* ** This file contains the C functions that implement date and time -** functions for SQLite. +** functions for SQLite. ** ** There is only one exported symbol in this file - the function ** sqlite3RegisterDateTimeFunctions() found at the bottom of the file. @@ -18318,7 +24456,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( ** SQLite processes all times and dates as julian day numbers. The ** dates and times are stored as the number of days since noon ** in Greenwich on November 24, 4714 B.C. according to the Gregorian -** calendar system. +** calendar system. ** ** 1970-01-01 00:00:00 is JD 2440587.5 ** 2000-01-01 00:00:00 is JD 2451544.5 @@ -18338,7 +24476,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_status( ** ** Jean Meeus ** Astronomical Algorithms, 2nd Edition, 1998 -** ISBM 0-943396-61-1 +** ISBN 0-943396-61-1 ** Willmann-Bell, Inc ** Richmond, Virginia (USA) */ @@ -18364,16 +24502,20 @@ struct tm *__cdecl localtime(const time_t *); */ typedef struct DateTime DateTime; struct DateTime { - sqlite3_int64 iJD; /* The julian day number times 86400000 */ - int Y, M, D; /* Year, month, and day */ - int h, m; /* Hour and minutes */ - int tz; /* Timezone offset in minutes */ - double s; /* Seconds */ - char validYMD; /* True (1) if Y,M,D are valid */ - char validHMS; /* True (1) if h,m,s are valid */ - char validJD; /* True (1) if iJD is valid */ - char validTZ; /* True (1) if tz is valid */ - char tzSet; /* Timezone was set explicitly */ + sqlite3_int64 iJD; /* The julian day number times 86400000 */ + int Y, M, D; /* Year, month, and day */ + int h, m; /* Hour and minutes */ + int tz; /* Timezone offset in minutes */ + double s; /* Seconds */ + char validJD; /* True (1) if iJD is valid */ + char validYMD; /* True (1) if Y,M,D are valid */ + char validHMS; /* True (1) if h,m,s are valid */ + char nFloor; /* Days to implement "floor" */ + unsigned rawS : 1; /* Raw numeric value stored in s */ + unsigned isError : 1; /* An overflow has occurred */ + unsigned useSubsec : 1; /* Display subsecond precision */ + unsigned isUtc : 1; /* Time is known to be UTC */ + unsigned isLocal : 1; /* Time is known to be localtime */ }; @@ -18406,8 +24548,8 @@ struct DateTime { */ static int getDigits(const char *zDate, const char *zFormat, ...){ /* The aMx[] array translates the 3rd character of each format - ** spec into a max size: a b c d e f */ - static const u16 aMx[] = { 12, 14, 24, 31, 59, 9999 }; + ** spec into a max size: a b c d e f */ + static const u16 aMx[] = { 12, 14, 24, 31, 59, 14712 }; va_list ap; int cnt = 0; char nextC; @@ -18471,6 +24613,8 @@ static int parseTimezone(const char *zDate, DateTime *p){ sgn = +1; }else if( c=='Z' || c=='z' ){ zDate++; + p->isLocal = 0; + p->isUtc = 1; goto zulu_time; }else{ return c!=0; @@ -18483,7 +24627,6 @@ static int parseTimezone(const char *zDate, DateTime *p){ p->tz = sgn*(nMn + nHr*60); zulu_time: while( sqlite3Isspace(*zDate) ){ zDate++; } - p->tzSet = 1; return *zDate!=0; } @@ -18521,15 +24664,23 @@ static int parseHhMmSs(const char *zDate, DateTime *p){ s = 0; } p->validJD = 0; + p->rawS = 0; p->validHMS = 1; p->h = h; p->m = m; p->s = s + ms; if( parseTimezone(zDate, p) ) return 1; - p->validTZ = (p->tz!=0)?1:0; return 0; } +/* +** Put the DateTime object into its error state. +*/ +static void datetimeError(DateTime *p){ + memset(p, 0, sizeof(*p)); + p->isError = 1; +} + /* ** Convert from YYYY-MM-DD HH:MM:SS to julian day. We always assume ** that the YYYY-MM-DD is according to the Gregorian calendar. @@ -18549,27 +24700,56 @@ static void computeJD(DateTime *p){ M = 1; D = 1; } + if( Y<-4713 || Y>9999 || p->rawS ){ + datetimeError(p); + return; + } if( M<=2 ){ Y--; M += 12; } - A = Y/100; - B = 2 - A + (A/4); + A = (Y+4800)/100; + B = 38 - A + (A/4); X1 = 36525*(Y+4716)/100; X2 = 306001*(M+1)/10000; p->iJD = (sqlite3_int64)((X1 + X2 + D + B - 1524.5 ) * 86400000); p->validJD = 1; if( p->validHMS ){ - p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000); - if( p->validTZ ){ + p->iJD += p->h*3600000 + p->m*60000 + (sqlite3_int64)(p->s*1000 + 0.5); + if( p->tz ){ p->iJD -= p->tz*60000; p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; + p->isUtc = 1; + p->isLocal = 0; } } } +/* +** Given the YYYY-MM-DD information current in p, determine if there +** is day-of-month overflow and set nFloor to the number of days that +** would need to be subtracted from the date in order to bring the +** date back to the end of the month. +*/ +static void computeFloor(DateTime *p){ + assert( p->validYMD || p->isError ); + assert( p->D>=0 && p->D<=31 ); + assert( p->M>=0 && p->M<=12 ); + if( p->D<=28 ){ + p->nFloor = 0; + }else if( (1<M) & 0x15aa ){ + p->nFloor = 0; + }else if( p->M!=2 ){ + p->nFloor = (p->D==31); + }else if( p->Y%4!=0 || (p->Y%100==0 && p->Y%400!=0) ){ + p->nFloor = p->D - 28; + }else{ + p->nFloor = p->D - 29; + } +} + /* ** Parse dates of the form ** @@ -18608,12 +24788,16 @@ static int parseYyyyMmDd(const char *zDate, DateTime *p){ p->Y = neg ? -Y : Y; p->M = M; p->D = D; - if( p->validTZ ){ + computeFloor(p); + if( p->tz ){ computeJD(p); } return 0; } + +static void clearYMD_HMS_TZ(DateTime *p); /* Forward declaration */ + /* ** Set the time to the current time reported by the VFS. ** @@ -18623,12 +24807,30 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ p->iJD = sqlite3StmtCurrentTime(context); if( p->iJD>0 ){ p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; + clearYMD_HMS_TZ(p); return 0; }else{ return 1; } } +/* +** Input "r" is a numeric quantity which might be a julian day number, +** or the number of seconds since 1970. If the value if r is within +** range of a julian day number, install it as such and set validJD. +** If the value is a valid unix timestamp, put it in p->s and set p->rawS. +*/ +static void setRawDateNumber(DateTime *p, double r){ + p->s = r; + p->rawS = 1; + if( r>=0.0 && r<5373484.5 ){ + p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); + p->validJD = 1; + } +} + /* ** Attempt to parse the given string into a julian day number. Return ** the number of errors. @@ -18636,7 +24838,7 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ ** The following are acceptable forms for the input string: ** ** YYYY-MM-DD HH:MM:SS.FFF +/-HH:MM -** DDDD.DD +** DDDD.DD ** now ** ** In the first form, the +/-HH:MM is always optional. The fractional @@ -18646,8 +24848,8 @@ static int setDateTimeToCurrent(sqlite3_context *context, DateTime *p){ ** as there is a year and date. */ static int parseDateOrTime( - sqlite3_context *context, - const char *zDate, + sqlite3_context *context, + const char *zDate, DateTime *p ){ double r; @@ -18655,30 +24857,55 @@ static int parseDateOrTime( return 0; }else if( parseHhMmSs(zDate, p)==0 ){ return 0; - }else if( sqlite3StrICmp(zDate,"now")==0){ + }else if( sqlite3StrICmp(zDate,"now")==0 && sqlite3NotPureFunc(context) ){ return setDateTimeToCurrent(context, p); - }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8) ){ - p->iJD = (sqlite3_int64)(r*86400000.0 + 0.5); - p->validJD = 1; + }else if( sqlite3AtoF(zDate, &r, sqlite3Strlen30(zDate), SQLITE_UTF8)>0 ){ + setRawDateNumber(p, r); return 0; + }else if( (sqlite3StrICmp(zDate,"subsec")==0 + || sqlite3StrICmp(zDate,"subsecond")==0) + && sqlite3NotPureFunc(context) ){ + p->useSubsec = 1; + return setDateTimeToCurrent(context, p); } return 1; } +/* The julian day number for 9999-12-31 23:59:59.999 is 5373484.4999999. +** Multiplying this by 86400000 gives 464269060799999 as the maximum value +** for DateTime.iJD. +** +** But some older compilers (ex: gcc 4.2.1 on older Macs) cannot deal with +** such a large integer literal, so we have to encode it. +*/ +#define INT_464269060799999 ((((i64)0x1a640)<<32)|0x1072fdff) + +/* +** Return TRUE if the given julian day number is within range. +** +** The input is the JulianDay times 86400000. +*/ +static int validJulianDay(sqlite3_int64 iJD){ + return iJD>=0 && iJD<=INT_464269060799999; +} + /* ** Compute the Year, Month, and Day from the julian day number. */ static void computeYMD(DateTime *p){ - int Z, A, B, C, D, E, X1; + int Z, alpha, A, B, C, D, E, X1; if( p->validYMD ) return; if( !p->validJD ){ p->Y = 2000; p->M = 1; p->D = 1; + }else if( !validJulianDay(p->iJD) ){ + datetimeError(p); + return; }else{ Z = (int)((p->iJD + 43200000)/86400000); - A = (int)((Z - 1867216.25)/36524.25); - A = Z + 1 + A - (A/4); + alpha = (int)((Z + 32044.75)/36524.25) - 52; + A = Z + 1 + alpha - ((alpha+100)/4) + 25; B = A + 1524; C = (int)((B - 122.1)/365.25); D = (36525*(C&32767))/100; @@ -18695,17 +24922,15 @@ static void computeYMD(DateTime *p){ ** Compute the Hour, Minute, and Seconds from the julian day number. */ static void computeHMS(DateTime *p){ - int s; + int day_ms, day_min; /* milliseconds, minutes into the day */ if( p->validHMS ) return; computeJD(p); - s = (int)((p->iJD + 43200000) % 86400000); - p->s = s/1000.0; - s = (int)p->s; - p->s -= s; - p->h = s/3600; - s -= p->h*3600; - p->m = s/60; - p->s += s - p->m*60; + day_ms = (int)((p->iJD + 43200000) % 86400000); + p->s = (day_ms % 60000)/1000.0; + day_min = day_ms/60000; + p->m = day_min % 60; + p->h = day_min / 60; + p->rawS = 0; p->validHMS = 1; } @@ -18723,20 +24948,20 @@ static void computeYMD_HMS(DateTime *p){ static void clearYMD_HMS_TZ(DateTime *p){ p->validYMD = 0; p->validHMS = 0; - p->validTZ = 0; + p->tz = 0; } #ifndef SQLITE_OMIT_LOCALTIME /* ** On recent Windows platforms, the localtime_s() function is available -** as part of the "Secure CRT". It is essentially equivalent to -** localtime_r() available under most POSIX platforms, except that the +** as part of the "Secure CRT". It is essentially equivalent to +** localtime_r() available under most POSIX platforms, except that the ** order of the parameters is reversed. ** ** See http://msdn.microsoft.com/en-us/library/a442x3ye(VS.80).aspx. ** ** If the user has not indicated to use localtime_r() or localtime_s() -** already, check for an MSVC build environment that provides +** already, check for an MSVC build environment that provides ** localtime_s(). */ #if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S \ @@ -18751,8 +24976,10 @@ static void clearYMD_HMS_TZ(DateTime *p){ ** is available. This routine returns 0 on success and ** non-zero on any kind of error. ** -** If the sqlite3GlobalConfig.bLocaltimeFault variable is true then this -** routine will always fail. +** If the sqlite3GlobalConfig.bLocaltimeFault variable is non-zero then this +** routine will always fail. If bLocaltimeFault is nonzero and +** sqlite3GlobalConfig.xAltLocaltime is not NULL, then xAltLocaltime() is +** invoked in place of the OS-defined localtime() function. ** ** EVIDENCE-OF: R-62172-00036 In this implementation, the standard C ** library function localtime_r() is used to assist in the calculation of @@ -18763,19 +24990,35 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #if !HAVE_LOCALTIME_R && !HAVE_LOCALTIME_S struct tm *pX; #if SQLITE_THREADSAFE>0 - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); #endif sqlite3_mutex_enter(mutex); pX = localtime(t); -#ifndef SQLITE_OMIT_BUILTIN_TEST - if( sqlite3GlobalConfig.bLocaltimeFault ) pX = 0; +#ifndef SQLITE_UNTESTABLE + if( sqlite3GlobalConfig.bLocaltimeFault ){ + if( sqlite3GlobalConfig.xAltLocaltime!=0 + && 0==sqlite3GlobalConfig.xAltLocaltime((const void*)t,(void*)pTm) + ){ + pX = pTm; + }else{ + pX = 0; + } + } #endif if( pX ) *pTm = *pX; +#if SQLITE_THREADSAFE>0 sqlite3_mutex_leave(mutex); +#endif rc = pX==0; #else -#ifndef SQLITE_OMIT_BUILTIN_TEST - if( sqlite3GlobalConfig.bLocaltimeFault ) return 1; +#ifndef SQLITE_UNTESTABLE + if( sqlite3GlobalConfig.bLocaltimeFault ){ + if( sqlite3GlobalConfig.xAltLocaltime!=0 ){ + return sqlite3GlobalConfig.xAltLocaltime((const void*)t,(void*)pTm); + }else{ + return 1; + } + } #endif #if HAVE_LOCALTIME_R rc = localtime_r(t, pTm)==0; @@ -18790,68 +25033,100 @@ static int osLocaltime(time_t *t, struct tm *pTm){ #ifndef SQLITE_OMIT_LOCALTIME /* -** Compute the difference (in milliseconds) between localtime and UTC -** (a.k.a. GMT) for the time value p where p is in UTC. If no error occurs, -** return this value and set *pRc to SQLITE_OK. -** -** Or, if an error does occur, set *pRc to SQLITE_ERROR. The returned value -** is undefined in this case. +** Assuming the input DateTime is UTC, move it to its localtime equivalent. */ -static sqlite3_int64 localtimeOffset( - DateTime *p, /* Date at which to calculate offset */ - sqlite3_context *pCtx, /* Write error here if one occurs */ - int *pRc /* OUT: Error code. SQLITE_OK or ERROR */ +static int toLocaltime( + DateTime *p, /* Date at which to calculate offset */ + sqlite3_context *pCtx /* Write error here if one occurs */ ){ - DateTime x, y; time_t t; struct tm sLocal; + int iYearDiff; /* Initialize the contents of sLocal to avoid a compiler warning. */ memset(&sLocal, 0, sizeof(sLocal)); - x = *p; - computeYMD_HMS(&x); - if( x.Y<1971 || x.Y>=2038 ){ + computeJD(p); + if( p->iJD<2108667600*(i64)100000 /* 1970-01-01 */ + || p->iJD>2130141456*(i64)100000 /* 2038-01-18 */ + ){ /* EVIDENCE-OF: R-55269-29598 The localtime_r() C function normally only ** works for years between 1970 and 2037. For dates outside this range, ** SQLite attempts to map the year into an equivalent year within this ** range, do the calculation, then map the year back. */ - x.Y = 2000; - x.M = 1; - x.D = 1; - x.h = 0; - x.m = 0; - x.s = 0.0; - } else { - int s = (int)(x.s + 0.5); - x.s = s; + DateTime x = *p; + computeYMD_HMS(&x); + iYearDiff = (2000 + x.Y%4) - x.Y; + x.Y += iYearDiff; + x.validJD = 0; + computeJD(&x); + t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); + }else{ + iYearDiff = 0; + t = (time_t)(p->iJD/1000 - 21086676*(i64)10000); } - x.tz = 0; - x.validJD = 0; - computeJD(&x); - t = (time_t)(x.iJD/1000 - 21086676*(i64)10000); if( osLocaltime(&t, &sLocal) ){ sqlite3_result_error(pCtx, "local time unavailable", -1); - *pRc = SQLITE_ERROR; - return 0; + return SQLITE_ERROR; } - y.Y = sLocal.tm_year + 1900; - y.M = sLocal.tm_mon + 1; - y.D = sLocal.tm_mday; - y.h = sLocal.tm_hour; - y.m = sLocal.tm_min; - y.s = sLocal.tm_sec; - y.validYMD = 1; - y.validHMS = 1; - y.validJD = 0; - y.validTZ = 0; - computeJD(&y); - *pRc = SQLITE_OK; - return y.iJD - x.iJD; + p->Y = sLocal.tm_year + 1900 - iYearDiff; + p->M = sLocal.tm_mon + 1; + p->D = sLocal.tm_mday; + p->h = sLocal.tm_hour; + p->m = sLocal.tm_min; + p->s = sLocal.tm_sec + (p->iJD%1000)*0.001; + p->validYMD = 1; + p->validHMS = 1; + p->validJD = 0; + p->rawS = 0; + p->tz = 0; + p->isError = 0; + return SQLITE_OK; } #endif /* SQLITE_OMIT_LOCALTIME */ +/* +** The following table defines various date transformations of the form +** +** 'NNN days' +** +** Where NNN is an arbitrary floating-point number and "days" can be one +** of several units of time. +*/ +static const struct { + u8 nName; /* Length of the name */ + char zName[7]; /* Name of the transformation */ + float rLimit; /* Maximum NNN value for this transform */ + float rXform; /* Constant used for this transform */ +} aXformType[] = { + /* 0 */ { 6, "second", 4.6427e+14, 1.0 }, + /* 1 */ { 6, "minute", 7.7379e+12, 60.0 }, + /* 2 */ { 4, "hour", 1.2897e+11, 3600.0 }, + /* 3 */ { 3, "day", 5373485.0, 86400.0 }, + /* 4 */ { 5, "month", 176546.0, 2592000.0 }, + /* 5 */ { 4, "year", 14713.0, 31536000.0 }, +}; + +/* +** If the DateTime p is raw number, try to figure out if it is +** a julian day number of a unix timestamp. Set the p value +** appropriately. +*/ +static void autoAdjustDate(DateTime *p){ + if( !p->rawS || p->validJD ){ + p->rawS = 0; + }else if( p->s>=-21086676*(i64)10000 /* -4713-11-24 12:00:00 */ + && p->s<=(25340230*(i64)10000)+799 /* 9999-12-31 23:59:59 */ + ){ + double r = p->s*1000.0 + 210866760000000.0; + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + } +} + /* ** Process a modifier to a date-time stamp. The modifiers are ** as follows: @@ -18862,31 +25137,99 @@ static sqlite3_int64 localtimeOffset( ** NNN.NNNN seconds ** NNN months ** NNN years +** +/-YYYY-MM-DD HH:MM:SS.SSS +** ceiling +** floor ** start of month ** start of year ** start of week ** start of day ** weekday N ** unixepoch +** auto ** localtime ** utc +** subsec +** subsecond ** ** Return 0 on success and 1 if there is any kind of error. If the error ** is in a system call (i.e. localtime()), then an error message is written ** to context pCtx. If the error is an unrecognized modifier, no error is ** written to pCtx. */ -static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ +static int parseModifier( + sqlite3_context *pCtx, /* Function context */ + const char *z, /* The text of the modifier */ + int n, /* Length of zMod in bytes */ + DateTime *p, /* The date/time value to be modified */ + int idx /* Parameter index of the modifier */ +){ int rc = 1; - int n; double r; - char *z, zBuf[30]; - z = zBuf; - for(n=0; n1 ) return 1; /* IMP: R-33611-57934 */ + autoAdjustDate(p); + rc = 0; + } + break; + } + case 'c': { + /* + ** ceiling + ** + ** Resolve day-of-month overflow by rolling forward into the next + ** month. As this is the default action, this modifier is really + ** a no-op that is only included for symmetry. See "floor". + */ + if( sqlite3_stricmp(z, "ceiling")==0 ){ + computeJD(p); + clearYMD_HMS_TZ(p); + rc = 0; + p->nFloor = 0; + } + break; + } + case 'f': { + /* + ** floor + ** + ** Resolve day-of-month overflow by rolling back to the end of the + ** previous month. + */ + if( sqlite3_stricmp(z, "floor")==0 ){ + computeJD(p); + p->iJD -= p->nFloor*86400000; + clearYMD_HMS_TZ(p); + rc = 0; + } + break; + } + case 'j': { + /* + ** julianday + ** + ** Always interpret the prior number as a julian-day value. If this + ** is not the first modifier, or if the prior argument is not a numeric + ** value in the allowed range of julian day numbers understood by + ** SQLite (0..5373484.5) then the result will be NULL. + */ + if( sqlite3_stricmp(z, "julianday")==0 ){ + if( idx>1 ) return 1; /* IMP: R-31176-64601 */ + if( p->validJD && p->rawS ){ + rc = 0; + p->rawS = 0; + } + } + break; + } #ifndef SQLITE_OMIT_LOCALTIME case 'l': { /* localtime @@ -18894,10 +25237,10 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ ** Assuming the current time value is UTC (a.k.a. GMT), shift it to ** show local time. */ - if( strcmp(z, "localtime")==0 ){ - computeJD(p); - p->iJD += localtimeOffset(p, pCtx, &rc); - clearYMD_HMS_TZ(p); + if( sqlite3_stricmp(z, "localtime")==0 && sqlite3NotPureFunc(pCtx) ){ + rc = p->isLocal ? SQLITE_OK : toLocaltime(p, pCtx); + p->isUtc = 0; + p->isLocal = 1; } break; } @@ -18906,29 +25249,49 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ /* ** unixepoch ** - ** Treat the current value of p->iJD as the number of + ** Treat the current value of p->s as the number of ** seconds since 1970. Convert to a real julian day number. */ - if( strcmp(z, "unixepoch")==0 && p->validJD ){ - p->iJD = (p->iJD + 43200)/86400 + 21086676*(i64)10000000; - clearYMD_HMS_TZ(p); - rc = 0; + if( sqlite3_stricmp(z, "unixepoch")==0 && p->rawS ){ + if( idx>1 ) return 1; /* IMP: R-49255-55373 */ + r = p->s*1000.0 + 210866760000000.0; + if( r>=0.0 && r<464269060800000.0 ){ + clearYMD_HMS_TZ(p); + p->iJD = (sqlite3_int64)(r + 0.5); + p->validJD = 1; + p->rawS = 0; + rc = 0; + } } #ifndef SQLITE_OMIT_LOCALTIME - else if( strcmp(z, "utc")==0 ){ - if( p->tzSet==0 ){ - sqlite3_int64 c1; + else if( sqlite3_stricmp(z, "utc")==0 && sqlite3NotPureFunc(pCtx) ){ + if( p->isUtc==0 ){ + i64 iOrigJD; /* Original localtime */ + i64 iGuess; /* Guess at the corresponding utc time */ + int cnt = 0; /* Safety to prevent infinite loop */ + i64 iErr; /* Guess is off by this much */ + computeJD(p); - c1 = localtimeOffset(p, pCtx, &rc); - if( rc==SQLITE_OK ){ - p->iJD -= c1; - clearYMD_HMS_TZ(p); - p->iJD += c1 - localtimeOffset(p, pCtx, &rc); - } - p->tzSet = 1; - }else{ - rc = SQLITE_OK; + iGuess = iOrigJD = p->iJD; + iErr = 0; + do{ + DateTime new; + memset(&new, 0, sizeof(new)); + iGuess -= iErr; + new.iJD = iGuess; + new.validJD = 1; + rc = toLocaltime(&new, pCtx); + if( rc ) return rc; + computeJD(&new); + iErr = new.iJD - iOrigJD; + }while( iErr && cnt++<3 ); + memset(p, 0, sizeof(*p)); + p->iJD = iGuess; + p->validJD = 1; + p->isUtc = 1; + p->isLocal = 0; } + rc = SQLITE_OK; } #endif break; @@ -18941,12 +25304,12 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ ** weekday N where 0==Sunday, 1==Monday, and so forth. If the ** date is already on the appropriate weekday, this is a no-op. */ - if( strncmp(z, "weekday ", 8)==0 - && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8) - && (n=(int)r)==r && n>=0 && r<7 ){ + if( sqlite3_strnicmp(z, "weekday ", 8)==0 + && sqlite3AtoF(&z[8], &r, sqlite3Strlen30(&z[8]), SQLITE_UTF8)>0 + && r>=0.0 && r<7.0 && (n=(int)r)==r ){ sqlite3_int64 Z; computeYMD_HMS(p); - p->validTZ = 0; + p->tz = 0; p->validJD = 0; computeJD(p); Z = ((p->iJD + 129600000)/86400000) % 7; @@ -18963,24 +25326,39 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ ** ** Move the date backwards to the beginning of the current day, ** or month or year. + ** + ** subsecond + ** subsec + ** + ** Show subsecond precision in the output of datetime() and + ** unixepoch() and strftime('%s'). */ - if( strncmp(z, "start of ", 9)!=0 ) break; + if( sqlite3_strnicmp(z, "start of ", 9)!=0 ){ + if( sqlite3_stricmp(z, "subsec")==0 + || sqlite3_stricmp(z, "subsecond")==0 + ){ + p->useSubsec = 1; + rc = 0; + } + break; + } + if( !p->validJD && !p->validYMD && !p->validHMS ) break; z += 9; computeYMD(p); p->validHMS = 1; p->h = p->m = 0; p->s = 0.0; - p->validTZ = 0; + p->rawS = 0; + p->tz = 0; p->validJD = 0; - if( strcmp(z,"month")==0 ){ + if( sqlite3_stricmp(z,"month")==0 ){ p->D = 1; rc = 0; - }else if( strcmp(z,"year")==0 ){ - computeYMD(p); + }else if( sqlite3_stricmp(z,"year")==0 ){ p->M = 1; p->D = 1; rc = 0; - }else if( strcmp(z,"day")==0 ){ + }else if( sqlite3_stricmp(z,"day")==0 ){ rc = 0; } break; @@ -18998,18 +25376,75 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ case '8': case '9': { double rRounder; - for(n=1; z[n] && z[n]!=':' && !sqlite3Isspace(z[n]); n++){} - if( !sqlite3AtoF(z, &r, n, SQLITE_UTF8) ){ - rc = 1; + int i; + int Y,M,D,h,m,x; + const char *z2 = z; + char z0 = z[0]; + for(n=1; z[n]; n++){ + if( z[n]==':' ) break; + if( sqlite3Isspace(z[n]) ) break; + if( z[n]=='-' ){ + if( n==5 && getDigits(&z[1], "40f", &Y)==1 ) break; + if( n==6 && getDigits(&z[1], "50f", &Y)==1 ) break; + } + } + if( sqlite3AtoF(z, &r, n, SQLITE_UTF8)<=0 ){ + assert( rc==1 ); break; } - if( z[n]==':' ){ + if( z[n]=='-' ){ + /* A modifier of the form (+|-)YYYY-MM-DD adds or subtracts the + ** specified number of years, months, and days. MM is limited to + ** the range 0-11 and DD is limited to 0-30. + */ + if( z0!='+' && z0!='-' ) break; /* Must start with +/- */ + if( n==5 ){ + if( getDigits(&z[1], "40f-20a-20d", &Y, &M, &D)!=3 ) break; + }else{ + assert( n==6 ); + if( getDigits(&z[1], "50f-20a-20d", &Y, &M, &D)!=3 ) break; + z++; + } + if( M>=12 ) break; /* M range 0..11 */ + if( D>=31 ) break; /* D range 0..30 */ + computeYMD_HMS(p); + p->validJD = 0; + if( z0=='-' ){ + p->Y -= Y; + p->M -= M; + D = -D; + }else{ + p->Y += Y; + p->M += M; + } + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + computeFloor(p); + computeJD(p); + p->validHMS = 0; + p->validYMD = 0; + p->iJD += (i64)D*86400000; + if( z[11]==0 ){ + rc = 0; + break; + } + if( sqlite3Isspace(z[11]) + && getDigits(&z[12], "20c:20e", &h, &m)==2 + ){ + z2 = &z[12]; + n = 2; + }else{ + break; + } + } + if( z2[n]==':' ){ /* A modifier of the form (+|-)HH:MM:SS.FFF adds (or subtracts) the ** specified number of hours, minutes, seconds, and fractional seconds ** to the time. The ".FFF" may be omitted. The ":SS.FFF" may be ** omitted. */ - const char *z2 = z; + DateTime tx; sqlite3_int64 day; if( !sqlite3Isdigit(*z2) ) z2++; @@ -19019,53 +25454,60 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ tx.iJD -= 43200000; day = tx.iJD/86400000; tx.iJD -= day*86400000; - if( z[0]=='-' ) tx.iJD = -tx.iJD; + if( z0=='-' ) tx.iJD = -tx.iJD; computeJD(p); clearYMD_HMS_TZ(p); p->iJD += tx.iJD; rc = 0; break; } + + /* If control reaches this point, it means the transformation is + ** one of the forms like "+NNN days". */ z += n; while( sqlite3Isspace(*z) ) z++; n = sqlite3Strlen30(z); - if( n>10 || n<3 ) break; - if( z[n-1]=='s' ){ z[n-1] = 0; n--; } + if( n<3 || n>10 ) break; + if( sqlite3UpperToLower[(u8)z[n-1]]=='s' ) n--; computeJD(p); - rc = 0; + assert( rc==1 ); rRounder = r<0 ? -0.5 : +0.5; - if( n==3 && strcmp(z,"day")==0 ){ - p->iJD += (sqlite3_int64)(r*86400000.0 + rRounder); - }else if( n==4 && strcmp(z,"hour")==0 ){ - p->iJD += (sqlite3_int64)(r*(86400000.0/24.0) + rRounder); - }else if( n==6 && strcmp(z,"minute")==0 ){ - p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0)) + rRounder); - }else if( n==6 && strcmp(z,"second")==0 ){ - p->iJD += (sqlite3_int64)(r*(86400000.0/(24.0*60.0*60.0)) + rRounder); - }else if( n==5 && strcmp(z,"month")==0 ){ - int x, y; - computeYMD_HMS(p); - p->M += (int)r; - x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; - p->Y += x; - p->M -= x*12; - p->validJD = 0; - computeJD(p); - y = (int)r; - if( y!=r ){ - p->iJD += (sqlite3_int64)((r - y)*30.0*86400000.0 + rRounder); + p->nFloor = 0; + for(i=0; i-aXformType[i].rLimit && rM += (int)r; + x = p->M>0 ? (p->M-1)/12 : (p->M-12)/12; + p->Y += x; + p->M -= x*12; + computeFloor(p); + p->validJD = 0; + r -= (int)r; + break; + } + case 5: { /* Special processing to add years */ + int y = (int)r; + assert( strcmp(aXformType[5].zName,"year")==0 ); + computeYMD_HMS(p); + assert( p->M>=0 && p->M<=12 ); + p->Y += y; + computeFloor(p); + p->validJD = 0; + r -= (int)r; + break; + } + } + computeJD(p); + p->iJD += (sqlite3_int64)(r*1000.0*aXformType[i].rXform + rRounder); + rc = 0; + break; } - }else if( n==4 && strcmp(z,"year")==0 ){ - int y = (int)r; - computeYMD_HMS(p); - p->Y += y; - p->validJD = 0; - computeJD(p); - if( y!=r ){ - p->iJD += (sqlite3_int64)((r - y)*365.0*86400000.0 + rRounder); - } - }else{ - rc = 1; } clearYMD_HMS_TZ(p); break; @@ -19087,22 +25529,22 @@ static int parseModifier(sqlite3_context *pCtx, const char *zMod, DateTime *p){ ** then assume a default value of "now" for argv[0]. */ static int isDate( - sqlite3_context *context, - int argc, - sqlite3_value **argv, + sqlite3_context *context, + int argc, + sqlite3_value **argv, DateTime *p ){ - int i; + int i, n; const unsigned char *z; int eType; memset(p, 0, sizeof(*p)); if( argc==0 ){ + if( !sqlite3NotPureFunc(context) ) return 1; return setDateTimeToCurrent(context, p); } if( (eType = sqlite3_value_type(argv[0]))==SQLITE_FLOAT || eType==SQLITE_INTEGER ){ - p->iJD = (sqlite3_int64)(sqlite3_value_double(argv[0])*86400000.0 + 0.5); - p->validJD = 1; + setRawDateNumber(p, sqlite3_value_double(argv[0])); }else{ z = sqlite3_value_text(argv[0]); if( !z || parseDateOrTime(context, (char*)z, p) ){ @@ -19111,7 +25553,16 @@ static int isDate( } for(i=1; iisError || !validJulianDay(p->iJD) ) return 1; + if( argc==1 && p->validYMD && p->D>28 ){ + /* Make sure a YYYY-MM-DD is normalized. + ** Example: 2023-02-31 -> 2023-03-03 */ + assert( p->validJD ); + p->validYMD = 0; } return 0; } @@ -19139,6 +25590,28 @@ static void juliandayFunc( } } +/* +** unixepoch( TIMESTRING, MOD, MOD, ...) +** +** Return the number of seconds (including fractional seconds) since +** the unix epoch of 1970-01-01 00:00:00 GMT. +*/ +static void unixepochFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + computeJD(&x); + if( x.useSubsec ){ + sqlite3_result_double(context, (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + sqlite3_result_int64(context, x.iJD/1000 - 21086676*(i64)10000); + } + } +} + /* ** datetime( TIMESTRING, MOD, MOD, ...) ** @@ -19151,11 +25624,51 @@ static void datetimeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - char zBuf[100]; + int Y, s, n; + char zBuf[32]; computeYMD_HMS(&x); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d %02d:%02d:%02d", - x.Y, x.M, x.D, x.h, x.m, (int)(x.s)); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + Y = x.Y; + if( Y<0 ) Y = -Y; + zBuf[1] = '0' + (Y/1000)%10; + zBuf[2] = '0' + (Y/100)%10; + zBuf[3] = '0' + (Y/10)%10; + zBuf[4] = '0' + (Y)%10; + zBuf[5] = '-'; + zBuf[6] = '0' + (x.M/10)%10; + zBuf[7] = '0' + (x.M)%10; + zBuf[8] = '-'; + zBuf[9] = '0' + (x.D/10)%10; + zBuf[10] = '0' + (x.D)%10; + zBuf[11] = ' '; + zBuf[12] = '0' + (x.h/10)%10; + zBuf[13] = '0' + (x.h)%10; + zBuf[14] = ':'; + zBuf[15] = '0' + (x.m/10)%10; + zBuf[16] = '0' + (x.m)%10; + zBuf[17] = ':'; + if( x.useSubsec ){ + s = (int)(1000.0*x.s + 0.5); + zBuf[18] = '0' + (s/10000)%10; + zBuf[19] = '0' + (s/1000)%10; + zBuf[20] = '.'; + zBuf[21] = '0' + (s/100)%10; + zBuf[22] = '0' + (s/10)%10; + zBuf[23] = '0' + (s)%10; + zBuf[24] = 0; + n = 24; + }else{ + s = (int)x.s; + zBuf[18] = '0' + (s/10)%10; + zBuf[19] = '0' + (s)%10; + zBuf[20] = 0; + n = 20; + } + if( x.Y<0 ){ + zBuf[0] = '-'; + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(context, &zBuf[1], n-1, SQLITE_TRANSIENT); + } } } @@ -19171,10 +25684,33 @@ static void timeFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - char zBuf[100]; + int s, n; + char zBuf[16]; computeHMS(&x); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%02d:%02d:%02d", x.h, x.m, (int)x.s); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + zBuf[0] = '0' + (x.h/10)%10; + zBuf[1] = '0' + (x.h)%10; + zBuf[2] = ':'; + zBuf[3] = '0' + (x.m/10)%10; + zBuf[4] = '0' + (x.m)%10; + zBuf[5] = ':'; + if( x.useSubsec ){ + s = (int)(1000.0*x.s + 0.5); + zBuf[6] = '0' + (s/10000)%10; + zBuf[7] = '0' + (s/1000)%10; + zBuf[8] = '.'; + zBuf[9] = '0' + (s/100)%10; + zBuf[10] = '0' + (s/10)%10; + zBuf[11] = '0' + (s)%10; + zBuf[12] = 0; + n = 12; + }else{ + s = (int)x.s; + zBuf[6] = '0' + (s/10)%10; + zBuf[7] = '0' + (s)%10; + zBuf[8] = 0; + n = 8; + } + sqlite3_result_text(context, zBuf, n, SQLITE_TRANSIENT); } } @@ -19190,29 +25726,108 @@ static void dateFunc( ){ DateTime x; if( isDate(context, argc, argv, &x)==0 ){ - char zBuf[100]; + int Y; + char zBuf[16]; computeYMD(&x); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%04d-%02d-%02d", x.Y, x.M, x.D); - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); + Y = x.Y; + if( Y<0 ) Y = -Y; + zBuf[1] = '0' + (Y/1000)%10; + zBuf[2] = '0' + (Y/100)%10; + zBuf[3] = '0' + (Y/10)%10; + zBuf[4] = '0' + (Y)%10; + zBuf[5] = '-'; + zBuf[6] = '0' + (x.M/10)%10; + zBuf[7] = '0' + (x.M)%10; + zBuf[8] = '-'; + zBuf[9] = '0' + (x.D/10)%10; + zBuf[10] = '0' + (x.D)%10; + zBuf[11] = 0; + if( x.Y<0 ){ + zBuf[0] = '-'; + sqlite3_result_text(context, zBuf, 11, SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(context, &zBuf[1], 10, SQLITE_TRANSIENT); + } } } +/* +** Compute the number of days after the most recent January 1. +** +** In other words, compute the zero-based day number for the +** current year: +** +** Jan01 = 0, Jan02 = 1, ..., Jan31 = 30, Feb01 = 31, ... +** Dec31 = 364 or 365. +*/ +static int daysAfterJan01(DateTime *pDate){ + DateTime jan01 = *pDate; + assert( jan01.validYMD ); + assert( jan01.validHMS ); + assert( pDate->validJD ); + jan01.validJD = 0; + jan01.M = 1; + jan01.D = 1; + computeJD(&jan01); + return (int)((pDate->iJD-jan01.iJD+43200000)/86400000); +} + +/* +** Return the number of days after the most recent Monday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Monday, 1=Tuesday, 2=Wednesday, ..., 6=Sunday. +*/ +static int daysAfterMonday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+43200000)/86400000) % 7; +} + +/* +** Return the number of days after the most recent Sunday. +** +** In other words, return the day of the week according +** to this code: +** +** 0=Sunday, 1=Monday, 2=Tues, ..., 6=Saturday +*/ +static int daysAfterSunday(DateTime *pDate){ + assert( pDate->validJD ); + return (int)((pDate->iJD+129600000)/86400000) % 7; +} + /* ** strftime( FORMAT, TIMESTRING, MOD, MOD, ...) ** ** Return a string described by FORMAT. Conversions as follows: ** -** %d day of month +** %d day of month 01-31 +** %e day of month 1-31 ** %f ** fractional seconds SS.SSS +** %F ISO date. YYYY-MM-DD +** %G ISO year corresponding to %V 0000-9999. +** %g 2-digit ISO year corresponding to %V 00-99 ** %H hour 00-24 -** %j day of year 000-366 +** %k hour 0-24 (leading zero converted to space) +** %I hour 01-12 +** %j day of year 001-366 ** %J ** julian day number +** %l hour 1-12 (leading zero converted to space) ** %m month 01-12 ** %M minute 00-59 +** %p "am" or "pm" +** %P "AM" or "PM" +** %R time as HH:MM ** %s seconds since 1970-01-01 ** %S seconds 00-59 -** %w day of week 0-6 sunday==0 -** %W week of year 00-53 +** %T time as HH:MM:SS +** %u day of week 1-7 Monday==1, Sunday==7 +** %w day of week 0-6 Sunday==0, Monday==1 +** %U week of year 00-53 (First Sunday is start of week 01) +** %V week of year 01-53 (First week containing Thursday is week 01) +** %W week of year 00-53 (First Monday is start of week 01) ** %Y year 0000-9999 ** %% % */ @@ -19222,131 +25837,161 @@ static void strftimeFunc( sqlite3_value **argv ){ DateTime x; - u64 n; size_t i,j; - char *z; sqlite3 *db; const char *zFmt; - char zBuf[100]; + sqlite3_str sRes; + + if( argc==0 ) return; zFmt = (const char*)sqlite3_value_text(argv[0]); if( zFmt==0 || isDate(context, argc-1, argv+1, &x) ) return; db = sqlite3_context_db_handle(context); - for(i=0, n=1; zFmt[i]; i++, n++){ - if( zFmt[i]=='%' ){ - switch( zFmt[i+1] ){ - case 'd': - case 'H': - case 'm': - case 'M': - case 'S': - case 'W': - n++; - /* fall thru */ - case 'w': - case '%': - break; - case 'f': - n += 8; - break; - case 'j': - n += 3; - break; - case 'Y': - n += 8; - break; - case 's': - case 'J': - n += 50; - break; - default: - return; /* ERROR. return a NULL */ - } - i++; - } - } - testcase( n==sizeof(zBuf)-1 ); - testcase( n==sizeof(zBuf) ); - testcase( n==(u64)db->aLimit[SQLITE_LIMIT_LENGTH]+1 ); - testcase( n==(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ); - if( n(u64)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - return; - }else{ - z = sqlite3DbMallocRawNN(db, (int)n); - if( z==0 ){ - sqlite3_result_error_nomem(context); - return; - } - } + sqlite3StrAccumInit(&sRes, 0, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + computeJD(&x); computeYMD_HMS(&x); for(i=j=0; zFmt[i]; i++){ - if( zFmt[i]!='%' ){ - z[j++] = zFmt[i]; - }else{ - i++; - switch( zFmt[i] ){ - case 'd': sqlite3_snprintf(3, &z[j],"%02d",x.D); j+=2; break; - case 'f': { - double s = x.s; - if( s>59.999 ) s = 59.999; - sqlite3_snprintf(7, &z[j],"%06.3f", s); - j += sqlite3Strlen30(&z[j]); - break; + char cf; + if( zFmt[i]!='%' ) continue; + if( j59.999 ) s = 59.999; + sqlite3_str_appendf(&sRes, "%06.3f", s); + break; + } + case 'F': { + sqlite3_str_appendf(&sRes, "%04d-%02d-%02d", x.Y, x.M, x.D); + break; + } + case 'G': /* Fall thru */ + case 'g': { + DateTime y = x; + assert( y.validJD ); + /* Move y so that it is the Thursday in the same week as x */ + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + if( cf=='g' ){ + sqlite3_str_appendf(&sRes, "%02d", y.Y%100); + }else{ + sqlite3_str_appendf(&sRes, "%04d", y.Y); } - case 'H': sqlite3_snprintf(3, &z[j],"%02d",x.h); j+=2; break; - case 'W': /* Fall thru */ - case 'j': { - int nDay; /* Number of days since 1st day of year */ - DateTime y = x; - y.validJD = 0; - y.M = 1; - y.D = 1; - computeJD(&y); - nDay = (int)((x.iJD-y.iJD+43200000)/86400000); - if( zFmt[i]=='W' ){ - int wd; /* 0=Monday, 1=Tuesday, ... 6=Sunday */ - wd = (int)(((x.iJD+43200000)/86400000)%7); - sqlite3_snprintf(3, &z[j],"%02d",(nDay+7-wd)/7); - j += 2; - }else{ - sqlite3_snprintf(4, &z[j],"%03d",nDay+1); - j += 3; - } - break; + break; + } + case 'H': + case 'k': { + sqlite3_str_appendf(&sRes, cf=='H' ? "%02d" : "%2d", x.h); + break; + } + case 'I': /* Fall thru */ + case 'l': { + int h = x.h; + if( h>12 ) h -= 12; + if( h==0 ) h = 12; + sqlite3_str_appendf(&sRes, cf=='I' ? "%02d" : "%2d", h); + break; + } + case 'j': { /* Day of year. Jan01==1, Jan02==2, and so forth */ + sqlite3_str_appendf(&sRes,"%03d",daysAfterJan01(&x)+1); + break; + } + case 'J': { /* Julian day number. (Non-standard) */ + sqlite3_str_appendf(&sRes,"%.16g",x.iJD/86400000.0); + break; + } + case 'm': { + sqlite3_str_appendf(&sRes,"%02d",x.M); + break; + } + case 'M': { + sqlite3_str_appendf(&sRes,"%02d",x.m); + break; + } + case 'p': /* Fall thru */ + case 'P': { + if( x.h>=12 ){ + sqlite3_str_append(&sRes, cf=='p' ? "PM" : "pm", 2); + }else{ + sqlite3_str_append(&sRes, cf=='p' ? "AM" : "am", 2); } - case 'J': { - sqlite3_snprintf(20, &z[j],"%.16g",x.iJD/86400000.0); - j+=sqlite3Strlen30(&z[j]); - break; + break; + } + case 'R': { + sqlite3_str_appendf(&sRes, "%02d:%02d", x.h, x.m); + break; + } + case 's': { + if( x.useSubsec ){ + sqlite3_str_appendf(&sRes,"%.3f", + (x.iJD - 21086676*(i64)10000000)/1000.0); + }else{ + i64 iS = (i64)(x.iJD/1000 - 21086676*(i64)10000); + sqlite3_str_appendf(&sRes,"%lld",iS); } - case 'm': sqlite3_snprintf(3, &z[j],"%02d",x.M); j+=2; break; - case 'M': sqlite3_snprintf(3, &z[j],"%02d",x.m); j+=2; break; - case 's': { - sqlite3_snprintf(30,&z[j],"%lld", - (i64)(x.iJD/1000 - 21086676*(i64)10000)); - j += sqlite3Strlen30(&z[j]); - break; - } - case 'S': sqlite3_snprintf(3,&z[j],"%02d",(int)x.s); j+=2; break; - case 'w': { - z[j++] = (char)(((x.iJD+129600000)/86400000) % 7) + '0'; - break; - } - case 'Y': { - sqlite3_snprintf(5,&z[j],"%04d",x.Y); j+=sqlite3Strlen30(&z[j]); - break; - } - default: z[j++] = '%'; break; + break; + } + case 'S': { + sqlite3_str_appendf(&sRes,"%02d",(int)x.s); + break; + } + case 'T': { + sqlite3_str_appendf(&sRes,"%02d:%02d:%02d", x.h, x.m, (int)x.s); + break; + } + case 'u': /* Day of week. 1 to 7. Monday==1, Sunday==7 */ + case 'w': { /* Day of week. 0 to 6. Sunday==0, Monday==1 */ + char c = (char)daysAfterSunday(&x) + '0'; + if( c=='0' && cf=='u' ) c = '7'; + sqlite3_str_appendchar(&sRes, 1, c); + break; + } + case 'U': { /* Week num. 00-53. First Sun of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterSunday(&x)+7)/7); + break; + } + case 'V': { /* Week num. 01-53. First week with a Thur is week 01 */ + DateTime y = x; + /* Adjust y so that is the Thursday in the same week as x */ + assert( y.validJD ); + y.iJD += (3 - daysAfterMonday(&x))*86400000; + y.validYMD = 0; + computeYMD(&y); + sqlite3_str_appendf(&sRes,"%02d", daysAfterJan01(&y)/7+1); + break; + } + case 'W': { /* Week num. 00-53. First Mon of the year is week 01 */ + sqlite3_str_appendf(&sRes,"%02d", + (daysAfterJan01(&x)-daysAfterMonday(&x)+7)/7); + break; + } + case 'Y': { + sqlite3_str_appendf(&sRes,"%04d",x.Y); + break; + } + case '%': { + sqlite3_str_appendchar(&sRes, 1, '%'); + break; + } + default: { + sqlite3_str_reset(&sRes); + return; } } } - z[j] = 0; - sqlite3_result_text(context, z, -1, - z==zBuf ? SQLITE_TRANSIENT : SQLITE_DYNAMIC); + if( j=d2.iJD ){ + sign = '+'; + Y = d1.Y - d2.Y; + if( Y ){ + d2.Y = d1.Y; + d2.validJD = 0; + computeJD(&d2); + } + M = d1.M - d2.M; + if( M<0 ){ + Y--; + M += 12; + } + if( M!=0 ){ + d2.M = d1.M; + d2.validJD = 0; + computeJD(&d2); + } + while( d1.iJDd2.iJD ){ + M--; + if( M<0 ){ + M = 11; + Y--; + } + d2.M++; + if( d2.M>12 ){ + d2.M = 1; + d2.Y++; + } + d2.validJD = 0; + computeJD(&d2); + } + d1.iJD = d2.iJD - d1.iJD; + d1.iJD += (u64)1486995408 * (u64)100000; + } + clearYMD_HMS_TZ(&d1); + computeYMD_HMS(&d1); + sqlite3StrAccumInit(&sRes, 0, 0, 0, 100); + sqlite3_str_appendf(&sRes, "%c%04d-%02d-%02d %02d:%02d:%06.3f", + sign, Y, M, d1.D-1, d1.h, d1.m, d1.s); + sqlite3ResultStrAccum(context, &sRes); +} + + /* ** current_timestamp() ** @@ -19411,7 +26165,6 @@ static void currentTimeFunc( ){ time_t t; char *zFormat = (char *)sqlite3_user_data(context); - sqlite3 *db; sqlite3_int64 iT; struct tm *pTm; struct tm sNow; @@ -19426,10 +26179,10 @@ static void currentTimeFunc( #if HAVE_GMTIME_R pTm = gmtime_r(&t, &sNow); #else - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); pTm = gmtime(&t); if( pTm ) memcpy(&sNow, pTm, sizeof(sNow)); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); #endif if( pTm ){ strftime(zBuf, 20, zFormat, &sNow); @@ -19438,6 +26191,36 @@ static void currentTimeFunc( } #endif +#if !defined(SQLITE_OMIT_DATETIME_FUNCS) && defined(SQLITE_DEBUG) +/* +** datedebug(...) +** +** This routine returns JSON that describes the internal DateTime object. +** Used for debugging and testing only. Subject to change. +*/ +static void datedebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + DateTime x; + if( isDate(context, argc, argv, &x)==0 ){ + char *zJson; + zJson = sqlite3_mprintf( + "{iJD:%lld,Y:%d,M:%d,D:%d,h:%d,m:%d,tz:%d," + "s:%.3f,validJD:%d,validYMS:%d,validHMS:%d," + "nFloor:%d,rawS:%d,isError:%d,useSubsec:%d," + "isUtc:%d,isLocal:%d}", + x.iJD, x.Y, x.M, x.D, x.h, x.m, x.tz, + x.s, x.validJD, x.validYMD, x.validHMS, + x.nFloor, x.rawS, x.isError, x.useSubsec, + x.isUtc, x.isLocal); + sqlite3_result_text(context, zJson, -1, sqlite3_free); + } +} +#endif /* !SQLITE_OMIT_DATETIME_FUNCS && SQLITE_DEBUG */ + + /* ** This function registered all of the above C functions as SQL ** functions. This should be the only routine in this file with @@ -19446,11 +26229,16 @@ static void currentTimeFunc( SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ static FuncDef aDateTimeFuncs[] = { #ifndef SQLITE_OMIT_DATETIME_FUNCS - DFUNCTION(julianday, -1, 0, 0, juliandayFunc ), - DFUNCTION(date, -1, 0, 0, dateFunc ), - DFUNCTION(time, -1, 0, 0, timeFunc ), - DFUNCTION(datetime, -1, 0, 0, datetimeFunc ), - DFUNCTION(strftime, -1, 0, 0, strftimeFunc ), + PURE_DATE(julianday, -1, 0, 0, juliandayFunc ), + PURE_DATE(unixepoch, -1, 0, 0, unixepochFunc ), + PURE_DATE(date, -1, 0, 0, dateFunc ), + PURE_DATE(time, -1, 0, 0, timeFunc ), + PURE_DATE(datetime, -1, 0, 0, datetimeFunc ), + PURE_DATE(strftime, -1, 0, 0, strftimeFunc ), + PURE_DATE(timediff, 2, 0, 0, timediffFunc ), +#ifdef SQLITE_DEBUG + PURE_DATE(datedebug, -1, 0, 0, datedebugFunc ), +#endif DFUNCTION(current_time, 0, 0, 0, ctimeFunc ), DFUNCTION(current_timestamp, 0, 0, 0, ctimestampFunc), DFUNCTION(current_date, 0, 0, 0, cdateFunc ), @@ -19480,9 +26268,7 @@ SQLITE_PRIVATE void sqlite3RegisterDateTimeFunctions(void){ ** This file contains OS interface code that is common to all ** architectures. */ -#define _SQLITE_OS_C_ 1 /* #include "sqliteInt.h" */ -#undef _SQLITE_OS_C_ /* ** If we compile with the SQLITE_TEST macro set, then the following block @@ -19567,7 +26353,7 @@ SQLITE_PRIVATE int sqlite3OsTruncate(sqlite3_file *id, i64 size){ } SQLITE_PRIVATE int sqlite3OsSync(sqlite3_file *id, int flags){ DO_OS_MALLOC_TEST(id); - return id->pMethods->xSync(id, flags); + return flags ? id->pMethods->xSync(id, flags) : SQLITE_OK; } SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ DO_OS_MALLOC_TEST(id); @@ -19575,9 +26361,11 @@ SQLITE_PRIVATE int sqlite3OsFileSize(sqlite3_file *id, i64 *pSize){ } SQLITE_PRIVATE int sqlite3OsLock(sqlite3_file *id, int lockType){ DO_OS_MALLOC_TEST(id); + assert( lockType>=SQLITE_LOCK_SHARED && lockType<=SQLITE_LOCK_EXCLUSIVE ); return id->pMethods->xLock(id, lockType); } SQLITE_PRIVATE int sqlite3OsUnlock(sqlite3_file *id, int lockType){ + assert( lockType==SQLITE_LOCK_NONE || lockType==SQLITE_LOCK_SHARED ); return id->pMethods->xUnlock(id, lockType); } SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ @@ -19594,25 +26382,35 @@ SQLITE_PRIVATE int sqlite3OsCheckReservedLock(sqlite3_file *id, int *pResOut){ ** routine has no return value since the return value would be meaningless. */ SQLITE_PRIVATE int sqlite3OsFileControl(sqlite3_file *id, int op, void *pArg){ + if( id->pMethods==0 ) return SQLITE_NOTFOUND; #ifdef SQLITE_TEST - if( op!=SQLITE_FCNTL_COMMIT_PHASETWO ){ + if( op!=SQLITE_FCNTL_COMMIT_PHASETWO + && op!=SQLITE_FCNTL_LOCK_TIMEOUT + && op!=SQLITE_FCNTL_CKPT_DONE + && op!=SQLITE_FCNTL_CKPT_START + ){ /* Faults are not injected into COMMIT_PHASETWO because, assuming SQLite ** is using a regular VFS, it is called after the corresponding ** transaction has been committed. Injecting a fault at this point - ** confuses the test scripts - the COMMIT comand returns SQLITE_NOMEM + ** confuses the test scripts - the COMMIT command returns SQLITE_NOMEM ** but the transaction is committed anyway. ** ** The core must call OsFileControl() though, not OsFileControlHint(), ** as if a custom VFS (e.g. zipvfs) returns an error here, it probably ** means the commit really has failed and an error should be returned - ** to the user. */ + ** to the user. + ** + ** The CKPT_DONE and CKPT_START file-controls are write-only signals + ** to the cksumvfs. Their return code is meaningless and is ignored + ** by the SQLite core, so there is no point in simulating OOMs for them. + */ DO_OS_MALLOC_TEST(id); } #endif return id->pMethods->xFileControl(id, op, pArg); } SQLITE_PRIVATE void sqlite3OsFileControlHint(sqlite3_file *id, int op, void *pArg){ - (void)id->pMethods->xFileControl(id, op, pArg); + if( id->pMethods ) (void)id->pMethods->xFileControl(id, op, pArg); } SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id){ @@ -19620,8 +26418,10 @@ SQLITE_PRIVATE int sqlite3OsSectorSize(sqlite3_file *id){ return (xSectorSize ? xSectorSize(id) : SQLITE_DEFAULT_SECTOR_SIZE); } SQLITE_PRIVATE int sqlite3OsDeviceCharacteristics(sqlite3_file *id){ + if( NEVER(id->pMethods==0) ) return 0; return id->pMethods->xDeviceCharacteristics(id); } +#ifndef SQLITE_OMIT_WAL SQLITE_PRIVATE int sqlite3OsShmLock(sqlite3_file *id, int offset, int n, int flags){ return id->pMethods->xShmLock(id, offset, n, flags); } @@ -19641,6 +26441,7 @@ SQLITE_PRIVATE int sqlite3OsShmMap( DO_OS_MALLOC_TEST(id); return id->pMethods->xShmMap(id, iPage, pgsz, bExtend, pp); } +#endif /* SQLITE_OMIT_WAL */ #if SQLITE_MAX_MMAP_SIZE>0 /* The real implementation of xFetch and xUnfetch */ @@ -19679,14 +26480,15 @@ SQLITE_PRIVATE int sqlite3OsOpen( ** down into the VFS layer. Some SQLITE_OPEN_ flags (for example, ** SQLITE_OPEN_FULLMUTEX or SQLITE_OPEN_SHAREDCACHE) are blocked before ** reaching the VFS. */ - rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x87f7f, pFlagsOut); + assert( zPath || (flags & SQLITE_OPEN_EXCLUSIVE) ); + rc = pVfs->xOpen(pVfs, zPath, pFile, flags & 0x1087f7f, pFlagsOut); assert( rc==SQLITE_OK || pFile->pMethods==0 ); return rc; } SQLITE_PRIVATE int sqlite3OsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ DO_OS_MALLOC_TEST(0); assert( dirSync==0 || dirSync==1 ); - return pVfs->xDelete(pVfs, zPath, dirSync); + return pVfs->xDelete!=0 ? pVfs->xDelete(pVfs, zPath, dirSync) : SQLITE_OK; } SQLITE_PRIVATE int sqlite3OsAccess( sqlite3_vfs *pVfs, @@ -19709,6 +26511,8 @@ SQLITE_PRIVATE int sqlite3OsFullPathname( } #ifndef SQLITE_OMIT_LOAD_EXTENSION SQLITE_PRIVATE void *sqlite3OsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + assert( zPath!=0 ); + assert( strlen(zPath)<=SQLITE_MAX_PATHLEN ); /* tag-20210611-1 */ return pVfs->xDlOpen(pVfs, zPath); } SQLITE_PRIVATE void sqlite3OsDlError(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ @@ -19722,7 +26526,15 @@ SQLITE_PRIVATE void sqlite3OsDlClose(sqlite3_vfs *pVfs, void *pHandle){ } #endif /* SQLITE_OMIT_LOAD_EXTENSION */ SQLITE_PRIVATE int sqlite3OsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ - return pVfs->xRandomness(pVfs, nByte, zBufOut); + if( sqlite3Config.iPrngSeed ){ + memset(zBufOut, 0, nByte); + if( ALWAYS(nByte>(signed)sizeof(unsigned)) ) nByte = sizeof(unsigned int); + memcpy(zBufOut, &sqlite3Config.iPrngSeed, nByte); + return SQLITE_OK; + }else{ + return pVfs->xRandomness(pVfs, nByte, zBufOut); + } + } SQLITE_PRIVATE int sqlite3OsSleep(sqlite3_vfs *pVfs, int nMicro){ return pVfs->xSleep(pVfs, nMicro); @@ -19762,12 +26574,15 @@ SQLITE_PRIVATE int sqlite3OsOpenMalloc( rc = sqlite3OsOpen(pVfs, zFile, pFile, flags, pOutFlags); if( rc!=SQLITE_OK ){ sqlite3_free(pFile); + *ppFile = 0; }else{ *ppFile = pFile; } }else{ + *ppFile = 0; rc = SQLITE_NOMEM_BKPT; } + assert( *ppFile!=0 || rc!=SQLITE_OK ); return rc; } SQLITE_PRIVATE void sqlite3OsCloseFree(sqlite3_file *pFile){ @@ -19799,7 +26614,7 @@ static sqlite3_vfs * SQLITE_WSD vfsList = 0; ** Locate a VFS by name. If no name is given, simply return the ** first VFS on the list. */ -SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfs){ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfs){ sqlite3_vfs *pVfs = 0; #if SQLITE_THREADSAFE sqlite3_mutex *mutex; @@ -19809,7 +26624,7 @@ SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfs){ if( rc ) return 0; #endif #if SQLITE_THREADSAFE - mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); #endif sqlite3_mutex_enter(mutex); for(pVfs = vfsList; pVfs; pVfs=pVfs->pNext){ @@ -19824,7 +26639,7 @@ SQLITE_API sqlite3_vfs *SQLITE_STDCALL sqlite3_vfs_find(const char *zVfs){ ** Unlink a VFS from the linked list */ static void vfsUnlink(sqlite3_vfs *pVfs){ - assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)) ); + assert( sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)) ); if( pVfs==0 ){ /* No-op */ }else if( vfsList==pVfs ){ @@ -19845,7 +26660,7 @@ static void vfsUnlink(sqlite3_vfs *pVfs){ ** VFS multiple times. The new VFS becomes the default if makeDflt is ** true. */ -SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDflt){ MUTEX_LOGIC(sqlite3_mutex *mutex;) #ifndef SQLITE_OMIT_AUTOINIT int rc = sqlite3_initialize(); @@ -19855,7 +26670,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDf if( pVfs==0 ) return SQLITE_MISUSE_BKPT; #endif - MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); if( makeDflt || vfsList==0 ){ @@ -19873,10 +26688,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_register(sqlite3_vfs *pVfs, int makeDf /* ** Unregister a VFS so that it is no longer accessible. */ -SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ -#if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ + MUTEX_LOGIC(sqlite3_mutex *mutex;) +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return rc; #endif + MUTEX_LOGIC( mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) sqlite3_mutex_enter(mutex); vfsUnlink(pVfs); sqlite3_mutex_leave(mutex); @@ -19897,23 +26715,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vfs_unregister(sqlite3_vfs *pVfs){ ** ************************************************************************* ** -** This file contains code to support the concept of "benign" +** This file contains code to support the concept of "benign" ** malloc failures (when the xMalloc() or xRealloc() method of the ** sqlite3_mem_methods structure fails to allocate a block of memory -** and returns 0). +** and returns 0). ** ** Most malloc failures are non-benign. After they occur, SQLite ** abandons the current operation and returns an error code (usually ** SQLITE_NOMEM) to the user. However, sometimes a fault is not necessarily -** fatal. For example, if a malloc fails while resizing a hash table, this -** is completely recoverable simply by not carrying out the resize. The -** hash table will continue to function normally. So a malloc failure +** fatal. For example, if a malloc fails while resizing a hash table, this +** is completely recoverable simply by not carrying out the resize. The +** hash table will continue to function normally. So a malloc failure ** during a hash table resize is a benign fault. */ /* #include "sqliteInt.h" */ -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE /* ** Global variables. @@ -19971,7 +26789,7 @@ SQLITE_PRIVATE void sqlite3EndBenignMalloc(void){ } } -#endif /* #ifndef SQLITE_OMIT_BUILTIN_TEST */ +#endif /* #ifndef SQLITE_UNTESTABLE */ /************** End of fault.c ***********************************************/ /************** Begin file mem0.c ********************************************/ @@ -20096,7 +26914,9 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ */ #include #include +#ifdef SQLITE_MIGHT_BE_SINGLE_CORE #include +#endif /* SQLITE_MIGHT_BE_SINGLE_CORE */ static malloc_zone_t* _sqliteZone_; #define SQLITE_MALLOC(x) malloc_zone_malloc(_sqliteZone_, (x)) #define SQLITE_FREE(x) malloc_zone_free(_sqliteZone_, (x)); @@ -20107,7 +26927,7 @@ static malloc_zone_t* _sqliteZone_; #else /* if not __APPLE__ */ /* -** Use standard C library malloc and free on non-Apple systems. +** Use standard C library malloc and free on non-Apple systems. ** Also used by Apple systems if SQLITE_WITHOUT_ZONEMALLOC is defined. */ #define SQLITE_MALLOC(x) malloc(x) @@ -20164,7 +26984,9 @@ static malloc_zone_t* _sqliteZone_; */ static void *sqlite3MemMalloc(int nByte){ #ifdef SQLITE_MALLOCSIZE - void *p = SQLITE_MALLOC( nByte ); + void *p; + testcase( ROUND8(nByte)==nByte ); + p = SQLITE_MALLOC( nByte ); if( p==0 ){ testcase( sqlite3GlobalConfig.xLog!=0 ); sqlite3_log(SQLITE_NOMEM, "failed to allocate %u bytes of memory", nByte); @@ -20173,7 +26995,7 @@ static void *sqlite3MemMalloc(int nByte){ #else sqlite3_int64 *p; assert( nByte>0 ); - nByte = ROUND8(nByte); + testcase( ROUND8(nByte)!=nByte ); p = SQLITE_MALLOC( nByte+8 ); if( p ){ p[0] = nByte; @@ -20191,7 +27013,7 @@ static void *sqlite3MemMalloc(int nByte){ ** or sqlite3MemRealloc(). ** ** For this low-level routine, we already know that pPrior!=0 since -** cases where pPrior==0 will have been intecepted and dealt with +** cases where pPrior==0 will have been intercepted and dealt with ** by higher-level routines. */ static void sqlite3MemFree(void *pPrior){ @@ -20279,27 +27101,18 @@ static int sqlite3MemInit(void *NotUsed){ return SQLITE_OK; } len = sizeof(cpuCount); - /* One usually wants to use hw.acctivecpu for MT decisions, but not here */ + /* One usually wants to use hw.activecpu for MT decisions, but not here */ sysctlbyname("hw.ncpu", &cpuCount, &len, NULL, 0); if( cpuCount>1 ){ /* defer MT decisions to system malloc */ _sqliteZone_ = malloc_default_zone(); }else{ - /* only 1 core, use our own zone to contention over global locks, + /* only 1 core, use our own zone to contention over global locks, ** e.g. we have our own dedicated locks */ - bool success; - malloc_zone_t* newzone = malloc_create_zone(4096, 0); - malloc_set_zone_name(newzone, "Sqlite_Heap"); - do{ - success = OSAtomicCompareAndSwapPtrBarrier(NULL, newzone, - (void * volatile *)&_sqliteZone_); - }while(!_sqliteZone_); - if( !success ){ - /* somebody registered a zone first */ - malloc_destroy_zone(newzone); - } + _sqliteZone_ = malloc_create_zone(4096, 0); + malloc_set_zone_name(_sqliteZone_, "Sqlite_Heap"); } -#endif +#endif /* defined(__APPLE__) && !defined(SQLITE_WITHOUT_ZONEMALLOC) */ UNUSED_PARAMETER(NotUsed); return SQLITE_OK; } @@ -20418,7 +27231,7 @@ struct MemBlockHdr { ** when this module is combined with other in the amalgamation. */ static struct { - + /* ** Mutex to control access to the memory allocation subsystem. */ @@ -20429,7 +27242,7 @@ static struct { */ struct MemBlockHdr *pFirst; struct MemBlockHdr *pLast; - + /* ** The number of levels of backtrace to save in new allocations. */ @@ -20442,7 +27255,7 @@ static struct { int nTitle; /* Bytes of zTitle to save. Includes '\0' and padding */ char zTitle[100]; /* The title text */ - /* + /* ** sqlite3MallocDisallow() increments the following counter. ** sqlite3MallocAllow() decrements it. */ @@ -20487,7 +27300,7 @@ static void adjustStats(int iSize, int increment){ ** This routine checks the guards at either end of the allocation and ** if they are incorrect it asserts. */ -static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){ +static struct MemBlockHdr *sqlite3MemsysGetHeader(const void *pAllocation){ struct MemBlockHdr *p; int *pInt; u8 *pU8; @@ -20501,7 +27314,7 @@ static struct MemBlockHdr *sqlite3MemsysGetHeader(void *pAllocation){ pU8 = (u8*)pAllocation; assert( pInt[nReserve/sizeof(int)]==(int)REARGUARD ); /* This checks any of the "extra" bytes allocated due - ** to rounding up to an 8 byte boundary to ensure + ** to rounding up to an 8 byte boundary to ensure ** they haven't been overwritten. */ while( nReserve-- > p->iSize ) assert( pU8[nReserve]==0x65 ); @@ -20630,7 +27443,7 @@ static void *sqlite3MemMalloc(int nByte){ p = (void*)pInt; } sqlite3_mutex_leave(mem.mutex); - return p; + return p; } /* @@ -20640,7 +27453,7 @@ static void sqlite3MemFree(void *pPrior){ struct MemBlockHdr *pHdr; void **pBt; char *z; - assert( sqlite3GlobalConfig.bMemstat || sqlite3GlobalConfig.bCoreMutex==0 + assert( sqlite3GlobalConfig.bMemstat || sqlite3GlobalConfig.bCoreMutex==0 || mem.mutex!=0 ); pHdr = sqlite3MemsysGetHeader(pPrior); pBt = (void**)pHdr; @@ -20666,15 +27479,15 @@ static void sqlite3MemFree(void *pPrior){ randomFill(z, sizeof(void*)*pHdr->nBacktraceSlots + sizeof(*pHdr) + (int)pHdr->iSize + sizeof(int) + pHdr->nTitle); free(z); - sqlite3_mutex_leave(mem.mutex); + sqlite3_mutex_leave(mem.mutex); } /* ** Change the size of an existing memory allocation. ** ** For this debugging implementation, we *always* make a copy of the -** allocation into a new place in memory. In this way, if the -** higher level code is using pointer to the old allocation, it is +** allocation into a new place in memory. In this way, if the +** higher level code is using pointer to the old allocation, it is ** much more likely to break and we are much more liking to find ** the error. */ @@ -20717,7 +27530,7 @@ SQLITE_PRIVATE void sqlite3MemSetDefault(void){ ** Set the "type" of an allocation. */ SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ - if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); @@ -20734,9 +27547,9 @@ SQLITE_PRIVATE void sqlite3MemdebugSetType(void *p, u8 eType){ ** ** assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); */ -SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ +SQLITE_PRIVATE int sqlite3MemdebugHasType(const void *p, u8 eType){ int rc = 1; - if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ @@ -20756,9 +27569,9 @@ SQLITE_PRIVATE int sqlite3MemdebugHasType(void *p, u8 eType){ ** ** assert( sqlite3MemdebugNoType(p, MEMTYPE_LOOKASIDE) ); */ -SQLITE_PRIVATE int sqlite3MemdebugNoType(void *p, u8 eType){ +SQLITE_PRIVATE int sqlite3MemdebugNoType(const void *p, u8 eType){ int rc = 1; - if( p && sqlite3GlobalConfig.m.xMalloc==sqlite3MemMalloc ){ + if( p && sqlite3GlobalConfig.m.xFree==sqlite3MemFree ){ struct MemBlockHdr *pHdr; pHdr = sqlite3MemsysGetHeader(p); assert( pHdr->iForeGuard==FOREGUARD ); /* Allocation is valid */ @@ -20808,7 +27621,7 @@ SQLITE_PRIVATE void sqlite3MemdebugSync(){ } /* -** Open the file indicated and write a log of all unfreed memory +** Open the file indicated and write a log of all unfreed memory ** allocations into that log. */ SQLITE_PRIVATE void sqlite3MemdebugDump(const char *zFilename){ @@ -20825,7 +27638,7 @@ SQLITE_PRIVATE void sqlite3MemdebugDump(const char *zFilename){ for(pHdr=mem.pFirst; pHdr; pHdr=pHdr->pNext){ char *z = (char*)pHdr; z -= pHdr->nBacktraceSlots*sizeof(void*) + pHdr->nTitle; - fprintf(out, "**** %lld bytes at %p from %s ****\n", + fprintf(out, "**** %lld bytes at %p from %s ****\n", pHdr->iSize, &pHdr[1], pHdr->nTitle ? z : "???"); if( pHdr->nBacktrace ){ fflush(out); @@ -20838,7 +27651,7 @@ SQLITE_PRIVATE void sqlite3MemdebugDump(const char *zFilename){ fprintf(out, "COUNTS:\n"); for(i=0; i=nBlock ); - if( nBlock>=mem3.szMaster-1 ){ - /* Use the entire master */ - void *p = memsys3Checkout(mem3.iMaster, mem3.szMaster); - mem3.iMaster = 0; - mem3.szMaster = 0; - mem3.mnMaster = 0; + assert( mem3.szKeyBlk>=nBlock ); + if( nBlock>=mem3.szKeyBlk-1 ){ + /* Use the entire key chunk */ + void *p = memsys3Checkout(mem3.iKeyBlk, mem3.szKeyBlk); + mem3.iKeyBlk = 0; + mem3.szKeyBlk = 0; + mem3.mnKeyBlk = 0; return p; }else{ - /* Split the master block. Return the tail. */ + /* Split the key block. Return the tail. */ u32 newi, x; - newi = mem3.iMaster + mem3.szMaster - nBlock; - assert( newi > mem3.iMaster+1 ); - mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = nBlock; - mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x |= 2; + newi = mem3.iKeyBlk + mem3.szKeyBlk - nBlock; + assert( newi > mem3.iKeyBlk+1 ); + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = nBlock; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x |= 2; mem3.aPool[newi-1].u.hdr.size4x = nBlock*4 + 1; - mem3.szMaster -= nBlock; - mem3.aPool[newi-1].u.hdr.prevSize = mem3.szMaster; - x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; - mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; - if( mem3.szMaster < mem3.mnMaster ){ - mem3.mnMaster = mem3.szMaster; + mem3.szKeyBlk -= nBlock; + mem3.aPool[newi-1].u.hdr.prevSize = mem3.szKeyBlk; + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + if( mem3.szKeyBlk < mem3.mnKeyBlk ){ + mem3.mnKeyBlk = mem3.szKeyBlk; } return (void*)&mem3.aPool[newi]; } @@ -21168,18 +27981,18 @@ static void *memsys3FromMaster(u32 nBlock){ /* ** *pRoot is the head of a list of free chunks of the same size ** or same size hash. In other words, *pRoot is an entry in either -** mem3.aiSmall[] or mem3.aiHash[]. +** mem3.aiSmall[] or mem3.aiHash[]. ** ** This routine examines all entries on the given list and tries -** to coalesce each entries with adjacent free chunks. +** to coalesce each entries with adjacent free chunks. ** -** If it sees a chunk that is larger than mem3.iMaster, it replaces -** the current mem3.iMaster with the new larger chunk. In order for -** this mem3.iMaster replacement to work, the master chunk must be +** If it sees a chunk that is larger than mem3.iKeyBlk, it replaces +** the current mem3.iKeyBlk with the new larger chunk. In order for +** this mem3.iKeyBlk replacement to work, the key chunk must be ** linked into the hash tables. That is not the normal state of -** affairs, of course. The calling routine must link the master +** affairs, of course. The calling routine must link the key ** chunk before invoking this routine, then must unlink the (possibly -** changed) master chunk once this routine has finished. +** changed) key chunk once this routine has finished. */ static void memsys3Merge(u32 *pRoot){ u32 iNext, prev, size, i, x; @@ -21206,9 +28019,9 @@ static void memsys3Merge(u32 *pRoot){ }else{ size /= 4; } - if( size>mem3.szMaster ){ - mem3.iMaster = i; - mem3.szMaster = size; + if( size>mem3.szKeyBlk ){ + mem3.iKeyBlk = i; + mem3.szKeyBlk = size; } } } @@ -21257,26 +28070,26 @@ static void *memsys3MallocUnsafe(int nByte){ /* STEP 2: ** Try to satisfy the allocation by carving a piece off of the end - ** of the master chunk. This step usually works if step 1 fails. + ** of the key chunk. This step usually works if step 1 fails. */ - if( mem3.szMaster>=nBlock ){ - return memsys3FromMaster(nBlock); + if( mem3.szKeyBlk>=nBlock ){ + return memsys3FromKeyBlk(nBlock); } - /* STEP 3: + /* STEP 3: ** Loop through the entire memory pool. Coalesce adjacent free - ** chunks. Recompute the master chunk as the largest free chunk. + ** chunks. Recompute the key chunk as the largest free chunk. ** Then try again to satisfy the allocation by carving a piece off - ** of the end of the master chunk. This step happens very + ** of the end of the key chunk. This step happens very ** rarely (we hope!) */ for(toFree=nBlock*16; toFree<(mem3.nPool*16); toFree *= 2){ memsys3OutOfMemory(toFree); - if( mem3.iMaster ){ - memsys3Link(mem3.iMaster); - mem3.iMaster = 0; - mem3.szMaster = 0; + if( mem3.iKeyBlk ){ + memsys3Link(mem3.iKeyBlk); + mem3.iKeyBlk = 0; + mem3.szKeyBlk = 0; } for(i=0; i=nBlock ){ - return memsys3FromMaster(nBlock); + if( mem3.szKeyBlk ){ + memsys3Unlink(mem3.iKeyBlk); + if( mem3.szKeyBlk>=nBlock ){ + return memsys3FromKeyBlk(nBlock); } } } @@ -21317,23 +28130,23 @@ static void memsys3FreeUnsafe(void *pOld){ mem3.aPool[i+size-1].u.hdr.size4x &= ~2; memsys3Link(i); - /* Try to expand the master using the newly freed chunk */ - if( mem3.iMaster ){ - while( (mem3.aPool[mem3.iMaster-1].u.hdr.size4x&2)==0 ){ - size = mem3.aPool[mem3.iMaster-1].u.hdr.prevSize; - mem3.iMaster -= size; - mem3.szMaster += size; - memsys3Unlink(mem3.iMaster); - x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; - mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; - mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = mem3.szMaster; + /* Try to expand the key using the newly freed chunk */ + if( mem3.iKeyBlk ){ + while( (mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x&2)==0 ){ + size = mem3.aPool[mem3.iKeyBlk-1].u.hdr.prevSize; + mem3.iKeyBlk -= size; + mem3.szKeyBlk += size; + memsys3Unlink(mem3.iKeyBlk); + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk; } - x = mem3.aPool[mem3.iMaster-1].u.hdr.size4x & 2; - while( (mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x&1)==0 ){ - memsys3Unlink(mem3.iMaster+mem3.szMaster); - mem3.szMaster += mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.size4x/4; - mem3.aPool[mem3.iMaster-1].u.hdr.size4x = mem3.szMaster*4 | x; - mem3.aPool[mem3.iMaster+mem3.szMaster-1].u.hdr.prevSize = mem3.szMaster; + x = mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x & 2; + while( (mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x&1)==0 ){ + memsys3Unlink(mem3.iKeyBlk+mem3.szKeyBlk); + mem3.szKeyBlk += mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.size4x/4; + mem3.aPool[mem3.iKeyBlk-1].u.hdr.size4x = mem3.szKeyBlk*4 | x; + mem3.aPool[mem3.iKeyBlk+mem3.szKeyBlk-1].u.hdr.prevSize = mem3.szKeyBlk; } } } @@ -21371,7 +28184,7 @@ static void *memsys3Malloc(int nBytes){ memsys3Enter(); p = memsys3MallocUnsafe(nBytes); memsys3Leave(); - return (void*)p; + return (void*)p; } /* @@ -21429,11 +28242,11 @@ static int memsys3Init(void *NotUsed){ mem3.aPool = (Mem3Block *)sqlite3GlobalConfig.pHeap; mem3.nPool = (sqlite3GlobalConfig.nHeap / sizeof(Mem3Block)) - 2; - /* Initialize the master block. */ - mem3.szMaster = mem3.nPool; - mem3.mnMaster = mem3.szMaster; - mem3.iMaster = 1; - mem3.aPool[0].u.hdr.size4x = (mem3.szMaster<<2) + 2; + /* Initialize the key block. */ + mem3.szKeyBlk = mem3.nPool; + mem3.mnKeyBlk = mem3.szKeyBlk; + mem3.iKeyBlk = 1; + mem3.aPool[0].u.hdr.size4x = (mem3.szKeyBlk<<2) + 2; mem3.aPool[mem3.nPool].u.hdr.prevSize = mem3.nPool; mem3.aPool[mem3.nPool].u.hdr.size4x = 1; @@ -21452,7 +28265,7 @@ static void memsys3Shutdown(void *NotUsed){ /* -** Open the file indicated and write a log of all unfreed memory +** Open the file indicated and write a log of all unfreed memory ** allocations into that log. */ SQLITE_PRIVATE void sqlite3Memsys3Dump(const char *zFilename){ @@ -21493,7 +28306,7 @@ SQLITE_PRIVATE void sqlite3Memsys3Dump(const char *zFilename){ fprintf(out, "%p %6d bytes checked out\n", &mem3.aPool[i], (size/4)*8-8); }else{ fprintf(out, "%p %6d bytes free%s\n", &mem3.aPool[i], (size/4)*8-8, - i==mem3.iMaster ? " **master**" : ""); + i==mem3.iKeyBlk ? " **key**" : ""); } } for(i=0; i= M*(1 + log2(n)/2) - n + 1 @@ -21610,7 +28423,7 @@ SQLITE_PRIVATE const sqlite3_mem_methods *sqlite3MemGetMemsys3(void){ /* #include "sqliteInt.h" */ /* -** This version of the memory allocator is used only when +** This version of the memory allocator is used only when ** SQLITE_ENABLE_MEMSYS5 is defined. */ #ifdef SQLITE_ENABLE_MEMSYS5 @@ -21655,7 +28468,7 @@ static SQLITE_WSD struct Mem5Global { int szAtom; /* Smallest possible allocation in bytes */ int nBlock; /* Number of szAtom sized blocks in zPool */ u8 *zPool; /* Memory available to be allocated */ - + /* ** Mutex to control access to the memory allocation subsystem. */ @@ -21674,7 +28487,7 @@ static SQLITE_WSD struct Mem5Global { u32 maxCount; /* Maximum instantaneous currentCount */ u32 maxRequest; /* Largest allocation (exclusive of internal frag) */ #endif - + /* ** Lists of free blocks. aiFreelist[0] is a list of free blocks of ** size mem5.szAtom. aiFreelist[1] holds blocks of size szAtom*2. @@ -21850,7 +28663,7 @@ static void memsys5FreeUnsafe(void *pOld){ u32 size, iLogsize; int iBlock; - /* Set iBlock to the index of the block pointed to by pOld in + /* Set iBlock to the index of the block pointed to by pOld in ** the array of mem5.szAtom byte blocks pointed to by mem5.zPool. */ iBlock = (int)(((u8 *)pOld-mem5.zPool)/mem5.szAtom); @@ -21919,7 +28732,7 @@ static void *memsys5Malloc(int nBytes){ p = memsys5MallocUnsafe(nBytes); memsys5Leave(); } - return (void*)p; + return (void*)p; } /* @@ -21932,14 +28745,14 @@ static void memsys5Free(void *pPrior){ assert( pPrior!=0 ); memsys5Enter(); memsys5FreeUnsafe(pPrior); - memsys5Leave(); + memsys5Leave(); } /* ** Change the size of an existing memory allocation. ** ** The outer layer memory allocator prevents this routine from -** being called with pPrior==0. +** being called with pPrior==0. ** ** nBytes is always a value obtained from a prior call to ** memsys5Round(). Hence nBytes is always a non-negative power @@ -21979,8 +28792,17 @@ static void *memsys5Realloc(void *pPrior, int nBytes){ */ static int memsys5Roundup(int n){ int iFullSz; - if( n > 0x40000000 ) return 0; - for(iFullSz=mem5.szAtom; iFullSz0x10000000 ){ + if( n>0x40000000 ) return 0; + if( n>0x20000000 ) return 0x40000000; + return 0x20000000; + } + for(iFullSz=mem5.szAtom*8; iFullSz=(i64)n ) return iFullSz/2; return iFullSz; } @@ -22072,7 +28894,7 @@ static void memsys5Shutdown(void *NotUsed){ #ifdef SQLITE_TEST /* -** Open the file indicated and write a log of all unfreed memory +** Open the file indicated and write a log of all unfreed memory ** allocations into that log. */ SQLITE_PRIVATE void sqlite3Memsys5Dump(const char *zFilename){ @@ -22114,7 +28936,7 @@ SQLITE_PRIVATE void sqlite3Memsys5Dump(const char *zFilename){ #endif /* -** This routine is the only routine in this file with external +** This routine is the only routine in this file with external ** linkage. It returns a pointer to a static sqlite3_mem_methods ** struct populated with the memsys5 methods. */ @@ -22164,14 +28986,201 @@ static SQLITE_WSD int mutexIsInit = 0; #ifndef SQLITE_MUTEX_OMIT + +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS +/* +** This block (enclosed by SQLITE_ENABLE_MULTITHREADED_CHECKS) contains +** the implementation of a wrapper around the system default mutex +** implementation (sqlite3DefaultMutex()). +** +** Most calls are passed directly through to the underlying default +** mutex implementation. Except, if a mutex is configured by calling +** sqlite3MutexWarnOnContention() on it, then if contention is ever +** encountered within xMutexEnter() a warning is emitted via sqlite3_log(). +** +** This type of mutex is used as the database handle mutex when testing +** apps that usually use SQLITE_CONFIG_MULTITHREAD mode. +*/ + +/* +** Type for all mutexes used when SQLITE_ENABLE_MULTITHREADED_CHECKS +** is defined. Variable CheckMutex.mutex is a pointer to the real mutex +** allocated by the system mutex implementation. Variable iType is usually set +** to the type of mutex requested - SQLITE_MUTEX_RECURSIVE, SQLITE_MUTEX_FAST +** or one of the static mutex identifiers. Or, if this is a recursive mutex +** that has been configured using sqlite3MutexWarnOnContention(), it is +** set to SQLITE_MUTEX_WARNONCONTENTION. +*/ +typedef struct CheckMutex CheckMutex; +struct CheckMutex { + int iType; + sqlite3_mutex *mutex; +}; + +#define SQLITE_MUTEX_WARNONCONTENTION (-1) + +/* +** Pointer to real mutex methods object used by the CheckMutex +** implementation. Set by checkMutexInit(). +*/ +static SQLITE_WSD const sqlite3_mutex_methods *pGlobalMutexMethods; + +#ifdef SQLITE_DEBUG +static int checkMutexHeld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexHeld(((CheckMutex*)p)->mutex); +} +static int checkMutexNotheld(sqlite3_mutex *p){ + return pGlobalMutexMethods->xMutexNotheld(((CheckMutex*)p)->mutex); +} +#endif + +/* +** Initialize and deinitialize the mutex subsystem. +*/ +static int checkMutexInit(void){ + pGlobalMutexMethods = sqlite3DefaultMutex(); + return SQLITE_OK; +} +static int checkMutexEnd(void){ + pGlobalMutexMethods = 0; + return SQLITE_OK; +} + +/* +** Allocate a mutex. +*/ +static sqlite3_mutex *checkMutexAlloc(int iType){ + static CheckMutex staticMutexes[] = { + {2, 0}, {3, 0}, {4, 0}, {5, 0}, + {6, 0}, {7, 0}, {8, 0}, {9, 0}, + {10, 0}, {11, 0}, {12, 0}, {13, 0} + }; + CheckMutex *p = 0; + + assert( SQLITE_MUTEX_RECURSIVE==1 && SQLITE_MUTEX_FAST==0 ); + if( iType<2 ){ + p = sqlite3MallocZero(sizeof(CheckMutex)); + if( p==0 ) return 0; + p->iType = iType; + }else{ +#ifdef SQLITE_ENABLE_API_ARMOR + if( iType-2>=ArraySize(staticMutexes) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + p = &staticMutexes[iType-2]; + } + + if( p->mutex==0 ){ + p->mutex = pGlobalMutexMethods->xMutexAlloc(iType); + if( p->mutex==0 ){ + if( iType<2 ){ + sqlite3_free(p); + } + p = 0; + } + } + + return (sqlite3_mutex*)p; +} + +/* +** Free a mutex. +*/ +static void checkMutexFree(sqlite3_mutex *p){ + assert( SQLITE_MUTEX_RECURSIVE<2 ); + assert( SQLITE_MUTEX_FAST<2 ); + assert( SQLITE_MUTEX_WARNONCONTENTION<2 ); + +#ifdef SQLITE_ENABLE_API_ARMOR + if( ((CheckMutex*)p)->iType<2 ) +#endif + { + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexFree(pCheck->mutex); + sqlite3_free(pCheck); + } +#ifdef SQLITE_ENABLE_API_ARMOR + else{ + (void)SQLITE_MISUSE_BKPT; + } +#endif +} + +/* +** Enter the mutex. +*/ +static void checkMutexEnter(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + if( pCheck->iType==SQLITE_MUTEX_WARNONCONTENTION ){ + if( SQLITE_OK==pGlobalMutexMethods->xMutexTry(pCheck->mutex) ){ + return; + } + sqlite3_log(SQLITE_MISUSE, + "illegal multi-threaded access to database connection" + ); + } + pGlobalMutexMethods->xMutexEnter(pCheck->mutex); +} + +/* +** Enter the mutex (do not block). +*/ +static int checkMutexTry(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + return pGlobalMutexMethods->xMutexTry(pCheck->mutex); +} + +/* +** Leave the mutex. +*/ +static void checkMutexLeave(sqlite3_mutex *p){ + CheckMutex *pCheck = (CheckMutex*)p; + pGlobalMutexMethods->xMutexLeave(pCheck->mutex); +} + +sqlite3_mutex_methods const *multiThreadedCheckMutex(void){ + static const sqlite3_mutex_methods sMutex = { + checkMutexInit, + checkMutexEnd, + checkMutexAlloc, + checkMutexFree, + checkMutexEnter, + checkMutexTry, + checkMutexLeave, +#ifdef SQLITE_DEBUG + checkMutexHeld, + checkMutexNotheld +#else + 0, + 0 +#endif + }; + return &sMutex; +} + +/* +** Mark the SQLITE_MUTEX_RECURSIVE mutex passed as the only argument as +** one on which there should be no contention. +*/ +SQLITE_PRIVATE void sqlite3MutexWarnOnContention(sqlite3_mutex *p){ + if( sqlite3GlobalConfig.mutex.xMutexAlloc==checkMutexAlloc ){ + CheckMutex *pCheck = (CheckMutex*)p; + assert( pCheck->iType==SQLITE_MUTEX_RECURSIVE ); + pCheck->iType = SQLITE_MUTEX_WARNONCONTENTION; + } +} +#endif /* ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS */ + /* ** Initialize the mutex system. */ -SQLITE_PRIVATE int sqlite3MutexInit(void){ +SQLITE_PRIVATE int sqlite3MutexInit(void){ int rc = SQLITE_OK; if( !sqlite3GlobalConfig.mutex.xMutexAlloc ){ /* If the xMutexAlloc method has not been set, then the user did not - ** install a mutex implementation via sqlite3_config() prior to + ** install a mutex implementation via sqlite3_config() prior to ** sqlite3_initialize() being called. This block copies pointers to ** the default implementation into the sqlite3GlobalConfig structure. */ @@ -22179,7 +29188,11 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ sqlite3_mutex_methods *pTo = &sqlite3GlobalConfig.mutex; if( sqlite3GlobalConfig.bCoreMutex ){ +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + pFrom = multiThreadedCheckMutex(); +#else pFrom = sqlite3DefaultMutex(); +#endif }else{ pFrom = sqlite3NoopMutex(); } @@ -22201,6 +29214,7 @@ SQLITE_PRIVATE int sqlite3MutexInit(void){ GLOBAL(int, mutexIsInit) = 1; #endif + sqlite3MemoryBarrier(); return rc; } @@ -22224,7 +29238,7 @@ SQLITE_PRIVATE int sqlite3MutexEnd(void){ /* ** Retrieve a pointer to a static mutex or allocate a new dynamic one. */ -SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_mutex_alloc(int id){ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int id){ #ifndef SQLITE_OMIT_AUTOINIT if( id<=SQLITE_MUTEX_RECURSIVE && sqlite3_initialize() ) return 0; if( id>SQLITE_MUTEX_RECURSIVE && sqlite3MutexInit() ) return 0; @@ -22245,7 +29259,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MutexAlloc(int id){ /* ** Free a dynamic mutex. */ -SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex *p){ if( p ){ assert( sqlite3GlobalConfig.mutex.xMutexFree ); sqlite3GlobalConfig.mutex.xMutexFree(p); @@ -22256,7 +29270,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_free(sqlite3_mutex *p){ ** Obtain the mutex p. If some other thread already has the mutex, block ** until it can be obtained. */ -SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex *p){ if( p ){ assert( sqlite3GlobalConfig.mutex.xMutexEnter ); sqlite3GlobalConfig.mutex.xMutexEnter(p); @@ -22267,7 +29281,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_enter(sqlite3_mutex *p){ ** Obtain the mutex p. If successful, return SQLITE_OK. Otherwise, if another ** thread holds the mutex and it cannot be obtained, return SQLITE_BUSY. */ -SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex *p){ int rc = SQLITE_OK; if( p ){ assert( sqlite3GlobalConfig.mutex.xMutexTry ); @@ -22278,11 +29292,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_try(sqlite3_mutex *p){ /* ** The sqlite3_mutex_leave() routine exits a mutex that was previously -** entered by the same thread. The behavior is undefined if the mutex +** entered by the same thread. The behavior is undefined if the mutex ** is not currently entered. If a NULL pointer is passed as an argument ** this function is a no-op. */ -SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex *p){ if( p ){ assert( sqlite3GlobalConfig.mutex.xMutexLeave ); sqlite3GlobalConfig.mutex.xMutexLeave(p); @@ -22293,16 +29307,29 @@ SQLITE_API void SQLITE_STDCALL sqlite3_mutex_leave(sqlite3_mutex *p){ /* ** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routine are ** intended for use inside assert() statements. +** +** Because these routines raise false-positive alerts in TSAN, disable +** them (make them always return 1) when compiling with TSAN. */ -SQLITE_API int SQLITE_STDCALL sqlite3_mutex_held(sqlite3_mutex *p){ +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexHeld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexHeld(p); } -SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex *p){ +# if defined(__has_feature) +# if __has_feature(thread_sanitizer) + p = 0; +# endif +# endif assert( p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld ); return p==0 || sqlite3GlobalConfig.mutex.xMutexNotheld(p); } -#endif +#endif /* NDEBUG */ #endif /* !defined(SQLITE_MUTEX_OMIT) */ @@ -22347,9 +29374,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_mutex_notheld(sqlite3_mutex *p){ */ static int noopMutexInit(void){ return SQLITE_OK; } static int noopMutexEnd(void){ return SQLITE_OK; } -static sqlite3_mutex *noopMutexAlloc(int id){ +static sqlite3_mutex *noopMutexAlloc(int id){ UNUSED_PARAMETER(id); - return (sqlite3_mutex*)8; + return (sqlite3_mutex*)8; } static void noopMutexFree(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } static void noopMutexEnter(sqlite3_mutex *p){ UNUSED_PARAMETER(p); return; } @@ -22414,7 +29441,7 @@ static int debugMutexEnd(void){ return SQLITE_OK; } /* ** The sqlite3_mutex_alloc() routine allocates a new ** mutex and returns a pointer to it. If it returns NULL -** that means that a mutex could not be allocated. +** that means that a mutex could not be allocated. */ static sqlite3_mutex *debugMutexAlloc(int id){ static sqlite3_debug_mutex aStatic[SQLITE_MUTEX_STATIC_VFS3 - 1]; @@ -22554,7 +29581,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ /* ** The sqlite3_mutex.id, sqlite3_mutex.nRef, and sqlite3_mutex.owner fields -** are necessary under two condidtions: (1) Debug builds and (2) using +** are necessary under two conditions: (1) Debug builds and (2) using ** home-grown mutexes. Encapsulate these conditions into a single #define. */ #if defined(SQLITE_DEBUG) || defined(SQLITE_HOMEGROWN_RECURSIVE_MUTEX) @@ -22578,11 +29605,12 @@ struct sqlite3_mutex { #endif }; #if SQLITE_MUTEX_NREF -#define SQLITE3_MUTEX_INITIALIZER {PTHREAD_MUTEX_INITIALIZER,0,0,(pthread_t)0,0} +# define SQLITE3_MUTEX_INITIALIZER(id) \ + {PTHREAD_MUTEX_INITIALIZER,id,0,(pthread_t)0,0} #elif defined(SQLITE_ENABLE_API_ARMOR) -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER, 0 } +# define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER, id } #else -#define SQLITE3_MUTEX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER } +#define SQLITE3_MUTEX_INITIALIZER(id) { PTHREAD_MUTEX_INITIALIZER } #endif /* @@ -22591,7 +29619,7 @@ struct sqlite3_mutex { ** there might be race conditions that can cause these routines to ** deliver incorrect results. In particular, if pthread_equal() is ** not an atomic operation, then these routines might delivery -** incorrect results. On most platforms, pthread_equal() is a +** incorrect results. On most platforms, pthread_equal() is a ** comparison of two integers and is therefore atomic. But we are ** told that HPUX is not such a platform. If so, then these routines ** will not always work correctly on HPUX. @@ -22639,7 +29667,7 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } **
          **
        • SQLITE_MUTEX_FAST **
        • SQLITE_MUTEX_RECURSIVE -**
        • SQLITE_MUTEX_STATIC_MASTER +**
        • SQLITE_MUTEX_STATIC_MAIN **
        • SQLITE_MUTEX_STATIC_MEM **
        • SQLITE_MUTEX_STATIC_OPEN **
        • SQLITE_MUTEX_STATIC_PRNG @@ -22673,24 +29701,24 @@ static int pthreadMutexEnd(void){ return SQLITE_OK; } ** ** Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST ** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() -** returns a different mutex on every call. But for the static +** returns a different mutex on every call. But for the static ** mutex types, the same mutex is returned on every call that has ** the same type number. */ static sqlite3_mutex *pthreadMutexAlloc(int iType){ static sqlite3_mutex staticMutexes[] = { - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) }; sqlite3_mutex *p; switch( iType ){ @@ -22708,6 +29736,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ pthread_mutexattr_settype(&recursiveAttr, PTHREAD_MUTEX_RECURSIVE); pthread_mutex_init(&p->mutex, &recursiveAttr); pthread_mutexattr_destroy(&recursiveAttr); +#endif +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_RECURSIVE; #endif } break; @@ -22716,6 +29747,9 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ p = sqlite3MallocZero( sizeof(*p) ); if( p ){ pthread_mutex_init(&p->mutex, 0); +#if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) + p->id = SQLITE_MUTEX_FAST; +#endif } break; } @@ -22731,7 +29765,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ } } #if SQLITE_MUTEX_NREF || defined(SQLITE_ENABLE_API_ARMOR) - if( p ) p->id = iType; + assert( p==0 || p->id==iType ); #endif return p; } @@ -22744,7 +29778,7 @@ static sqlite3_mutex *pthreadMutexAlloc(int iType){ */ static void pthreadMutexFree(sqlite3_mutex *p){ assert( p->nRef==0 ); -#if SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( p->id==SQLITE_MUTEX_FAST || p->id==SQLITE_MUTEX_RECURSIVE ) #endif { @@ -22778,7 +29812,7 @@ static void pthreadMutexEnter(sqlite3_mutex *p){ ** is atomic - that it cannot be deceived into thinking self ** and p->owner are equal if p->owner changes between two values ** that are not equal to self while the comparison is taking place. - ** This implementation also assumes a coherent cache - that + ** This implementation also assumes a coherent cache - that ** separate processes cannot read different values from the same ** address at the same time. If either of these two conditions ** are not met, then the mutexes will fail and problems will result. @@ -22821,7 +29855,7 @@ static int pthreadMutexTry(sqlite3_mutex *p){ ** is atomic - that it cannot be deceived into thinking self ** and p->owner are equal if p->owner changes between two values ** that are not equal to self while the comparison is taking place. - ** This implementation also assumes a coherent cache - that + ** This implementation also assumes a coherent cache - that ** separate processes cannot read different values from the same ** address at the same time. If either of these two conditions ** are not met, then the mutexes will fail and problems will result. @@ -22935,205 +29969,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ /* ** Include code that is common to all os_*.c files */ -/************** Include os_common.h in the middle of mutex_w32.c *************/ -/************** Begin file os_common.h ***************************************/ -/* -** 2004 May 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains macros and a little bit of code that is common to -** all of the platform-specific files (os_*.c) and is #included into those -** files. -** -** This file should be #included by the os_*.c files only. It is not a -** general purpose header file. -*/ -#ifndef _OS_COMMON_H_ -#define _OS_COMMON_H_ - -/* -** At least two bugs have slipped in because we changed the MEMORY_DEBUG -** macro to SQLITE_DEBUG and some older makefiles have not yet made the -** switch. The following code should catch this problem at compile-time. -*/ -#ifdef MEMORY_DEBUG -# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." -#endif - -/* -** Macros for performance tracing. Normally turned off. Only works -** on i486 hardware. -*/ -#ifdef SQLITE_PERFORMANCE_TRACE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - #error Need implementation of sqlite3Hwtime() for your platform. - - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(_HWTIME_H_) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ - -static sqlite_uint64 g_start; -static sqlite_uint64 g_elapsed; -#define TIMER_START g_start=sqlite3Hwtime() -#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start -#define TIMER_ELAPSED g_elapsed -#else -#define TIMER_START -#define TIMER_END -#define TIMER_ELAPSED ((sqlite_uint64)0) -#endif - -/* -** If we compile with the SQLITE_TEST macro set, then the following block -** of code will give us the ability to simulate a disk I/O error. This -** is used for testing the I/O recovery logic. -*/ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_io_error_hit; -SQLITE_API extern int sqlite3_io_error_hardhit; -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_persist; -SQLITE_API extern int sqlite3_io_error_benign; -SQLITE_API extern int sqlite3_diskfull_pending; -SQLITE_API extern int sqlite3_diskfull; -#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) -#define SimulateIOError(CODE) \ - if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ - || sqlite3_io_error_pending-- == 1 ) \ - { local_ioerr(); CODE; } -static void local_ioerr(){ - IOTRACE(("IOERR\n")); - sqlite3_io_error_hit++; - if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; -} -#define SimulateDiskfullError(CODE) \ - if( sqlite3_diskfull_pending ){ \ - if( sqlite3_diskfull_pending == 1 ){ \ - local_ioerr(); \ - sqlite3_diskfull = 1; \ - sqlite3_io_error_hit = 1; \ - CODE; \ - }else{ \ - sqlite3_diskfull_pending--; \ - } \ - } -#else -#define SimulateIOErrorBenign(X) -#define SimulateIOError(A) -#define SimulateDiskfullError(A) -#endif /* defined(SQLITE_TEST) */ - -/* -** When testing, keep a count of the number of open files. -*/ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_open_file_count; -#define OpenCounter(X) sqlite3_open_file_count+=(X) -#else -#define OpenCounter(X) -#endif /* defined(SQLITE_TEST) */ - -#endif /* !defined(_OS_COMMON_H_) */ - -/************** End of os_common.h *******************************************/ -/************** Continuing where we left off in mutex_w32.c ******************/ +/* #include "os_common.h" */ /* ** Include the header file for the Windows VFS. @@ -23154,8 +29990,8 @@ SQLITE_API extern int sqlite3_open_file_count; ** ** This file contains code that is specific to Windows. */ -#ifndef _OS_WIN_H_ -#define _OS_WIN_H_ +#ifndef SQLITE_OS_WIN_H +#define SQLITE_OS_WIN_H /* ** Include the primary Windows SDK header file. @@ -23227,7 +30063,7 @@ SQLITE_API extern int sqlite3_open_file_count; # define SQLITE_OS_WIN_THREADS 0 #endif -#endif /* _OS_WIN_H_ */ +#endif /* SQLITE_OS_WIN_H */ /************** End of os_win.h **********************************************/ /************** Continuing where we left off in mutex_w32.c ******************/ @@ -23246,9 +30082,9 @@ struct sqlite3_mutex { CRITICAL_SECTION mutex; /* Mutex controlling the lock */ int id; /* Mutex type */ #ifdef SQLITE_DEBUG - volatile int nRef; /* Number of enterances */ + volatile int nRef; /* Number of entrances */ volatile DWORD owner; /* Thread holding this mutex */ - volatile int trace; /* True to trace changes */ + volatile LONG trace; /* True to trace changes */ #endif }; @@ -23260,10 +30096,10 @@ struct sqlite3_mutex { #define SQLITE_W32_MUTEX_INITIALIZER { 0 } #ifdef SQLITE_DEBUG -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0, \ +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id, \ 0L, (DWORD)0, 0 } #else -#define SQLITE3_MUTEX_INITIALIZER { SQLITE_W32_MUTEX_INITIALIZER, 0 } +#define SQLITE3_MUTEX_INITIALIZER(id) { SQLITE_W32_MUTEX_INITIALIZER, id } #endif #ifdef SQLITE_DEBUG @@ -23295,8 +30131,7 @@ SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ SQLITE_MEMORY_BARRIER; #elif defined(__GNUC__) __sync_synchronize(); -#elif !defined(SQLITE_DISABLE_INTRINSIC) && \ - defined(_MSC_VER) && _MSC_VER>=1300 +#elif MSVC_VERSION>=1400 _ReadWriteBarrier(); #elif defined(MemoryBarrier) MemoryBarrier(); @@ -23307,18 +30142,18 @@ SQLITE_PRIVATE void sqlite3MemoryBarrier(void){ ** Initialize and deinitialize the mutex subsystem. */ static sqlite3_mutex winMutex_staticMutexes[] = { - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER, - SQLITE3_MUTEX_INITIALIZER + SQLITE3_MUTEX_INITIALIZER(2), + SQLITE3_MUTEX_INITIALIZER(3), + SQLITE3_MUTEX_INITIALIZER(4), + SQLITE3_MUTEX_INITIALIZER(5), + SQLITE3_MUTEX_INITIALIZER(6), + SQLITE3_MUTEX_INITIALIZER(7), + SQLITE3_MUTEX_INITIALIZER(8), + SQLITE3_MUTEX_INITIALIZER(9), + SQLITE3_MUTEX_INITIALIZER(10), + SQLITE3_MUTEX_INITIALIZER(11), + SQLITE3_MUTEX_INITIALIZER(12), + SQLITE3_MUTEX_INITIALIZER(13) }; static int winMutex_isInit = 0; @@ -23330,8 +30165,8 @@ static int winMutex_isNt = -1; /* <0 means "need to query" */ */ static LONG SQLITE_WIN32_VOLATILE winMutex_lock = 0; -SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void); /* os_win.c */ -SQLITE_API void SQLITE_STDCALL sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ +SQLITE_API int sqlite3_win32_is_nt(void); /* os_win.c */ +SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds); /* os_win.c */ static int winMutexInit(void){ /* The first to increment to 1 does actual initialization */ @@ -23380,7 +30215,7 @@ static int winMutexEnd(void){ **
            **
          • SQLITE_MUTEX_FAST **
          • SQLITE_MUTEX_RECURSIVE -**
          • SQLITE_MUTEX_STATIC_MASTER +**
          • SQLITE_MUTEX_STATIC_MAIN **
          • SQLITE_MUTEX_STATIC_MEM **
          • SQLITE_MUTEX_STATIC_OPEN **
          • SQLITE_MUTEX_STATIC_PRNG @@ -23448,15 +30283,15 @@ static sqlite3_mutex *winMutexAlloc(int iType){ } #endif p = &winMutex_staticMutexes[iType-2]; - p->id = iType; #ifdef SQLITE_DEBUG #ifdef SQLITE_WIN32_MUTEX_TRACE_STATIC - p->trace = 1; + InterlockedCompareExchange(&p->trace, 1, 0); #endif #endif break; } } + assert( p==0 || p->id==iType ); return p; } @@ -23507,8 +30342,8 @@ static void winMutexEnter(sqlite3_mutex *p){ p->owner = tid; p->nRef++; if( p->trace ){ - OSTRACE(("ENTER-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("ENTER-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } @@ -23550,8 +30385,8 @@ static int winMutexTry(sqlite3_mutex *p){ #endif #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("TRY-MUTEX tid=%lu, mutex=%p (%d), owner=%lu, nRef=%d, rc=%s\n", - tid, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); + OSTRACE(("TRY-MUTEX tid=%lu, mutex(%d)=%p (%d), owner=%lu, nRef=%d, rc=%s\n", + tid, p->id, p, p->trace, p->owner, p->nRef, sqlite3ErrName(rc))); } #endif return rc; @@ -23579,8 +30414,8 @@ static void winMutexLeave(sqlite3_mutex *p){ LeaveCriticalSection(&p->mutex); #ifdef SQLITE_DEBUG if( p->trace ){ - OSTRACE(("LEAVE-MUTEX tid=%lu, mutex=%p (%d), nRef=%d\n", - tid, p, p->trace, p->nRef)); + OSTRACE(("LEAVE-MUTEX tid=%lu, mutex(%d)=%p (%d), nRef=%d\n", + tid, p->id, p, p->trace, p->nRef)); } #endif } @@ -23631,7 +30466,7 @@ SQLITE_PRIVATE sqlite3_mutex_methods const *sqlite3DefaultMutex(void){ ** held by SQLite. An example of non-essential memory is memory used to ** cache database pages that are not currently in use. */ -SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int n){ +SQLITE_API int sqlite3_release_memory(int n){ #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT return sqlite3PcacheReleaseMemory(n); #else @@ -23644,12 +30479,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_release_memory(int n){ } /* -** An instance of the following object records the location of -** each unused scratch buffer. +** Default value of the hard heap limit. 0 means "no limit". */ -typedef struct ScratchFreeslot { - struct ScratchFreeslot *pNext; /* Next unused scratch buffer */ -} ScratchFreeslot; +#ifndef SQLITE_MAX_MEMORY +# define SQLITE_MAX_MEMORY 0 +#endif /* ** State information local to the memory allocation subsystem. @@ -23657,23 +30491,14 @@ typedef struct ScratchFreeslot { static SQLITE_WSD struct Mem0Global { sqlite3_mutex *mutex; /* Mutex to serialize access */ sqlite3_int64 alarmThreshold; /* The soft heap limit */ - - /* - ** Pointers to the end of sqlite3GlobalConfig.pScratch memory - ** (so that a range test can be used to determine if an allocation - ** being freed came from pScratch) and a pointer to the list of - ** unused scratch allocations. - */ - void *pScratchEnd; - ScratchFreeslot *pScratchFree; - u32 nScratchFree; + sqlite3_int64 hardLimit; /* The hard upper bound on memory */ /* ** True if heap is nearly "full" where "full" is defined by the ** sqlite3_soft_heap_limit() setting. */ int nearlyFull; -} mem0 = { 0, 0, 0, 0, 0, 0 }; +} mem0 = { 0, SQLITE_MAX_MEMORY, SQLITE_MAX_MEMORY, 0 }; #define mem0 GLOBAL(struct Mem0Global, mem0) @@ -23690,7 +30515,7 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3MallocMutex(void){ ** that was invoked when memory usage grew too large. Now it is a ** no-op. */ -SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( +SQLITE_API int sqlite3_memory_alarm( void(*xCallback)(void *pArg, sqlite3_int64 used,int N), void *pArg, sqlite3_int64 iThreshold @@ -23703,10 +30528,17 @@ SQLITE_API int SQLITE_STDCALL sqlite3_memory_alarm( #endif /* -** Set the soft heap-size limit for the library. Passing a zero or -** negative value indicates no limit. +** Set the soft heap-size limit for the library. An argument of +** zero disables the limit. A negative argument is a no-op used to +** obtain the return value. +** +** The return value is the value of the heap limit just before this +** interface was called. +** +** If the hard heap limit is enabled, then the soft heap limit cannot +** be disabled nor raised above the hard heap limit. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 n){ +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 n){ sqlite3_int64 priorLimit; sqlite3_int64 excess; sqlite3_int64 nUsed; @@ -23720,19 +30552,53 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_soft_heap_limit64(sqlite3_int64 sqlite3_mutex_leave(mem0.mutex); return priorLimit; } + if( mem0.hardLimit>0 && (n>mem0.hardLimit || n==0) ){ + n = mem0.hardLimit; + } mem0.alarmThreshold = n; nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); - mem0.nearlyFull = (n>0 && n<=nUsed); + AtomicStore(&mem0.nearlyFull, n>0 && n<=nUsed); sqlite3_mutex_leave(mem0.mutex); excess = sqlite3_memory_used() - n; if( excess>0 ) sqlite3_release_memory((int)(excess & 0x7fffffff)); return priorLimit; } -SQLITE_API void SQLITE_STDCALL sqlite3_soft_heap_limit(int n){ +SQLITE_API void sqlite3_soft_heap_limit(int n){ if( n<0 ) n = 0; sqlite3_soft_heap_limit64(n); } +/* +** Set the hard heap-size limit for the library. An argument of zero +** disables the hard heap limit. A negative argument is a no-op used +** to obtain the return value without affecting the hard heap limit. +** +** The return value is the value of the hard heap limit just prior to +** calling this interface. +** +** Setting the hard heap limit will also activate the soft heap limit +** and constrain the soft heap limit to be no more than the hard heap +** limit. +*/ +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 n){ + sqlite3_int64 priorLimit; +#ifndef SQLITE_OMIT_AUTOINIT + int rc = sqlite3_initialize(); + if( rc ) return -1; +#endif + sqlite3_mutex_enter(mem0.mutex); + priorLimit = mem0.hardLimit; + if( n>=0 ){ + mem0.hardLimit = n; + if( n=100 - && sqlite3GlobalConfig.nScratch>0 ){ - int i, n, sz; - ScratchFreeslot *pSlot; - sz = ROUNDDOWN8(sqlite3GlobalConfig.szScratch); - sqlite3GlobalConfig.szScratch = sz; - pSlot = (ScratchFreeslot*)sqlite3GlobalConfig.pScratch; - n = sqlite3GlobalConfig.nScratch; - mem0.pScratchFree = pSlot; - mem0.nScratchFree = n; - for(i=0; ipNext = (ScratchFreeslot*)(sz+(char*)pSlot); - pSlot = pSlot->pNext; - } - pSlot->pNext = 0; - mem0.pScratchEnd = (void*)&pSlot[1]; - }else{ - mem0.pScratchEnd = 0; - sqlite3GlobalConfig.pScratch = 0; - sqlite3GlobalConfig.szScratch = 0; - sqlite3GlobalConfig.nScratch = 0; - } if( sqlite3GlobalConfig.pPage==0 || sqlite3GlobalConfig.szPage<512 || sqlite3GlobalConfig.nPage<=0 ){ sqlite3GlobalConfig.pPage = 0; @@ -23781,7 +30624,7 @@ SQLITE_PRIVATE int sqlite3MallocInit(void){ ** sqlite3_soft_heap_limit(). */ SQLITE_PRIVATE int sqlite3HeapNearlyFull(void){ - return mem0.nearlyFull; + return AtomicLoad(&mem0.nearlyFull); } /* @@ -23797,7 +30640,7 @@ SQLITE_PRIVATE void sqlite3MallocEnd(void){ /* ** Return the amount of memory currently checked out. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void){ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void){ sqlite3_int64 res, mx; sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, 0); return res; @@ -23808,14 +30651,14 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_used(void){ ** checked out since either the beginning of this process ** or since the most recent reset. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_memory_highwater(int resetFlag){ +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag){ sqlite3_int64 res, mx; sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &res, &mx, resetFlag); return mx; } /* -** Trigger the alarm +** Trigger the alarm */ static void sqlite3MallocAlarm(int nByte){ if( mem0.alarmThreshold<=0 ) return; @@ -23824,23 +30667,57 @@ static void sqlite3MallocAlarm(int nByte){ sqlite3_mutex_enter(mem0.mutex); } +#ifdef SQLITE_DEBUG +/* +** This routine is called whenever an out-of-memory condition is seen, +** It's only purpose to to serve as a breakpoint for gdb or similar +** code debuggers when working on out-of-memory conditions, for example +** caused by PRAGMA hard_heap_limit=N. +*/ +static SQLITE_NOINLINE void test_oom_breakpoint(u64 n){ + static u64 nOomFault = 0; + nOomFault += n; + /* The assert() is never reached in a human lifetime. It is here mostly + ** to prevent code optimizers from optimizing out this function. */ + assert( (nOomFault>>32) < 0xffffffff ); +} +#else +# define test_oom_breakpoint(X) /* No-op for production builds */ +#endif + /* ** Do a memory allocation with statistics and alarms. Assume the ** lock is already held. */ -static int mallocWithAlarm(int n, void **pp){ - int nFull; +static void mallocWithAlarm(int n, void **pp){ void *p; + int nFull; assert( sqlite3_mutex_held(mem0.mutex) ); + assert( n>0 ); + + /* In Firefox (circa 2017-02-08), xRoundup() is remapped to an internal + ** implementation of malloc_good_size(), which must be called in debug + ** mode and specifically when the DMD "Dark Matter Detector" is enabled + ** or else a crash results. Hence, do not attempt to optimize out the + ** following xRoundup() call. */ nFull = sqlite3GlobalConfig.m.xRoundup(n); + sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, n); if( mem0.alarmThreshold>0 ){ sqlite3_int64 nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); if( nUsed >= mem0.alarmThreshold - nFull ){ - mem0.nearlyFull = 1; + AtomicStore(&mem0.nearlyFull, 1); sqlite3MallocAlarm(nFull); + if( mem0.hardLimit ){ + nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED); + if( nUsed >= mem0.hardLimit - nFull ){ + test_oom_breakpoint(1); + *pp = 0; + return; + } + } }else{ - mem0.nearlyFull = 0; + AtomicStore(&mem0.nearlyFull, 0); } } p = sqlite3GlobalConfig.m.xMalloc(nFull); @@ -23856,21 +30733,36 @@ static int mallocWithAlarm(int n, void **pp){ sqlite3StatusUp(SQLITE_STATUS_MALLOC_COUNT, 1); } *pp = p; - return nFull; } +/* +** Maximum size of any single memory allocation. +** +** This is not a limit on the total amount of memory used. This is +** a limit on the size parameter to sqlite3_malloc() and sqlite3_realloc(). +** +** The upper bound is slightly less than 2GiB: 0x7ffffeff == 2,147,483,391 +** This provides a 256-byte safety margin for defense against 32-bit +** signed integer overflow bugs when computing memory allocation sizes. +** Paranoid applications might want to reduce the maximum allocation size +** further for an even larger safety margin. 0x3fffffff or 0x0fffffff +** or even smaller would be reasonable upper bounds on the size of a memory +** allocations for most applications. +*/ +#ifndef SQLITE_MAX_ALLOCATION_SIZE +# define SQLITE_MAX_ALLOCATION_SIZE 2147483391 +#endif +#if SQLITE_MAX_ALLOCATION_SIZE>2147483391 +# error Maximum size for SQLITE_MAX_ALLOCATION_SIZE is 2147483391 +#endif + /* ** Allocate memory. This routine is like sqlite3_malloc() except that it ** assumes the memory subsystem has already been initialized. */ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ void *p; - if( n==0 || n>=0x7fffff00 ){ - /* A memory allocation of a number of bytes which is near the maximum - ** signed integer value might cause an integer overflow inside of the - ** xMalloc(). Hence we limit the maximum size to 0x7fffff00, giving - ** 255 bytes of overhead. SQLite itself will never use anything near - ** this amount. The only way to reach the limit is with sqlite3_malloc() */ + if( n==0 || n>SQLITE_MAX_ALLOCATION_SIZE ){ p = 0; }else if( sqlite3GlobalConfig.bMemstat ){ sqlite3_mutex_enter(mem0.mutex); @@ -23888,124 +30780,25 @@ SQLITE_PRIVATE void *sqlite3Malloc(u64 n){ ** First make sure the memory subsystem is initialized, then do the ** allocation. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_malloc(int n){ +SQLITE_API void *sqlite3_malloc(int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif return n<=0 ? 0 : sqlite3Malloc(n); } -SQLITE_API void *SQLITE_STDCALL sqlite3_malloc64(sqlite3_uint64 n){ +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif return sqlite3Malloc(n); } -/* -** Each thread may only have a single outstanding allocation from -** xScratchMalloc(). We verify this constraint in the single-threaded -** case by setting scratchAllocOut to 1 when an allocation -** is outstanding clearing it when the allocation is freed. -*/ -#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) -static int scratchAllocOut = 0; -#endif - - -/* -** Allocate memory that is to be used and released right away. -** This routine is similar to alloca() in that it is not intended -** for situations where the memory might be held long-term. This -** routine is intended to get memory to old large transient data -** structures that would not normally fit on the stack of an -** embedded processor. -*/ -SQLITE_PRIVATE void *sqlite3ScratchMalloc(int n){ - void *p; - assert( n>0 ); - - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusHighwater(SQLITE_STATUS_SCRATCH_SIZE, n); - if( mem0.nScratchFree && sqlite3GlobalConfig.szScratch>=n ){ - p = mem0.pScratchFree; - mem0.pScratchFree = mem0.pScratchFree->pNext; - mem0.nScratchFree--; - sqlite3StatusUp(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3_mutex_leave(mem0.mutex); - p = sqlite3Malloc(n); - if( sqlite3GlobalConfig.bMemstat && p ){ - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusUp(SQLITE_STATUS_SCRATCH_OVERFLOW, sqlite3MallocSize(p)); - sqlite3_mutex_leave(mem0.mutex); - } - sqlite3MemdebugSetType(p, MEMTYPE_SCRATCH); - } - assert( sqlite3_mutex_notheld(mem0.mutex) ); - - -#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* EVIDENCE-OF: R-12970-05880 SQLite will not use more than one scratch - ** buffers per thread. - ** - ** This can only be checked in single-threaded mode. - */ - assert( scratchAllocOut==0 ); - if( p ) scratchAllocOut++; -#endif - - return p; -} -SQLITE_PRIVATE void sqlite3ScratchFree(void *p){ - if( p ){ - -#if SQLITE_THREADSAFE==0 && !defined(NDEBUG) - /* Verify that no more than two scratch allocation per thread - ** is outstanding at one time. (This is only checked in the - ** single-threaded case since checking in the multi-threaded case - ** would be much more complicated.) */ - assert( scratchAllocOut>=1 && scratchAllocOut<=2 ); - scratchAllocOut--; -#endif - - if( SQLITE_WITHIN(p, sqlite3GlobalConfig.pScratch, mem0.pScratchEnd) ){ - /* Release memory from the SQLITE_CONFIG_SCRATCH allocation */ - ScratchFreeslot *pSlot; - pSlot = (ScratchFreeslot*)p; - sqlite3_mutex_enter(mem0.mutex); - pSlot->pNext = mem0.pScratchFree; - mem0.pScratchFree = pSlot; - mem0.nScratchFree++; - assert( mem0.nScratchFree <= (u32)sqlite3GlobalConfig.nScratch ); - sqlite3StatusDown(SQLITE_STATUS_SCRATCH_USED, 1); - sqlite3_mutex_leave(mem0.mutex); - }else{ - /* Release memory back to the heap */ - assert( sqlite3MemdebugHasType(p, MEMTYPE_SCRATCH) ); - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_SCRATCH) ); - sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - if( sqlite3GlobalConfig.bMemstat ){ - int iSize = sqlite3MallocSize(p); - sqlite3_mutex_enter(mem0.mutex); - sqlite3StatusDown(SQLITE_STATUS_SCRATCH_OVERFLOW, iSize); - sqlite3StatusDown(SQLITE_STATUS_MEMORY_USED, iSize); - sqlite3StatusDown(SQLITE_STATUS_MALLOC_COUNT, 1); - sqlite3GlobalConfig.m.xFree(p); - sqlite3_mutex_leave(mem0.mutex); - }else{ - sqlite3GlobalConfig.m.xFree(p); - } - } - } -} - /* ** TRUE if p is a lookaside memory allocation from db */ #ifndef SQLITE_OMIT_LOOKASIDE -static int isLookaside(sqlite3 *db, void *p){ - return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pEnd); +static int isLookaside(sqlite3 *db, const void *p){ + return SQLITE_WITHIN(p, db->lookaside.pStart, db->lookaside.pTrueEnd); } #else #define isLookaside(A,B) 0 @@ -24015,29 +30808,45 @@ static int isLookaside(sqlite3 *db, void *p){ ** Return the size of a memory allocation previously obtained from ** sqlite3Malloc() or sqlite3_malloc(). */ -SQLITE_PRIVATE int sqlite3MallocSize(void *p){ +SQLITE_PRIVATE int sqlite3MallocSize(const void *p){ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - return sqlite3GlobalConfig.m.xSize(p); + return sqlite3GlobalConfig.m.xSize((void*)p); } -SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, void *p){ - assert( p!=0 ); - if( db==0 || !isLookaside(db,p) ){ -#if SQLITE_DEBUG - if( db==0 ){ - assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); - assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); - }else{ - assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); - } +static int lookasideMallocSize(sqlite3 *db, const void *p){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + return plookaside.pMiddle ? db->lookaside.szTrue : LOOKASIDE_SMALL; +#else + return db->lookaside.szTrue; #endif - return sqlite3GlobalConfig.m.xSize(p); - }else{ - assert( sqlite3_mutex_held(db->mutex) ); - return db->lookaside.sz; - } } -SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void *p){ +SQLITE_PRIVATE int sqlite3DbMallocSize(sqlite3 *db, const void *p){ + assert( p!=0 ); +#ifdef SQLITE_DEBUG + if( db==0 ){ + assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); + assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); + }else if( !isLookaside(db,p) ){ + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + } +#endif + if( db ){ + if( ((uptr)p)<(uptr)(db->lookaside.pTrueEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return LOOKASIDE_SMALL; + } +#endif + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + assert( sqlite3_mutex_held(db->mutex) ); + return db->lookaside.szTrue; + } + } + } + return sqlite3GlobalConfig.m.xSize((void*)p); +} +SQLITE_API sqlite3_uint64 sqlite3_msize(void *p){ assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); return p ? sqlite3GlobalConfig.m.xSize(p) : 0; @@ -24046,7 +30855,7 @@ SQLITE_API sqlite3_uint64 SQLITE_STDCALL sqlite3_msize(void *p){ /* ** Free memory previously obtained from sqlite3Malloc(). */ -SQLITE_API void SQLITE_STDCALL sqlite3_free(void *p){ +SQLITE_API void sqlite3_free(void *p){ if( p==0 ) return; /* IMP: R-49053-54554 */ assert( sqlite3MemdebugHasType(p, MEMTYPE_HEAP) ); assert( sqlite3MemdebugNoType(p, (u8)~MEMTYPE_HEAP) ); @@ -24071,27 +30880,41 @@ static SQLITE_NOINLINE void measureAllocationSize(sqlite3 *db, void *p){ /* ** Free memory that might be associated with a particular database -** connection. +** connection. Calling sqlite3DbFree(D,X) for X==0 is a harmless no-op. +** The sqlite3DbFreeNN(D,X) version requires that X be non-NULL. */ -SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ +SQLITE_PRIVATE void sqlite3DbFreeNN(sqlite3 *db, void *p){ assert( db==0 || sqlite3_mutex_held(db->mutex) ); - if( p==0 ) return; + assert( p!=0 ); if( db ){ + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } if( db->pnBytesFreed ){ measureAllocationSize(db, p); return; } - if( isLookaside(db, p) ){ - LookasideSlot *pBuf = (LookasideSlot*)p; -#if SQLITE_DEBUG - /* Trash all content in the buffer being freed */ - memset(p, 0xaa, db->lookaside.sz); -#endif - pBuf->pNext = db->lookaside.pFree; - db->lookaside.pFree = pBuf; - db->lookaside.nOut--; - return; - } } assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); @@ -24099,6 +30922,47 @@ SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ sqlite3MemdebugSetType(p, MEMTYPE_HEAP); sqlite3_free(p); } +SQLITE_PRIVATE void sqlite3DbNNFreeNN(sqlite3 *db, void *p){ + assert( db!=0 ); + assert( sqlite3_mutex_held(db->mutex) ); + assert( p!=0 ); + if( ((uptr)p)<(uptr)(db->lookaside.pEnd) ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)(db->lookaside.pMiddle) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, LOOKASIDE_SMALL); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pSmallFree; + db->lookaside.pSmallFree = pBuf; + return; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( ((uptr)p)>=(uptr)(db->lookaside.pStart) ){ + LookasideSlot *pBuf = (LookasideSlot*)p; + assert( db->pnBytesFreed==0 ); +#ifdef SQLITE_DEBUG + memset(p, 0xaa, db->lookaside.szTrue); /* Trash freed content */ +#endif + pBuf->pNext = db->lookaside.pFree; + db->lookaside.pFree = pBuf; + return; + } + } + if( db->pnBytesFreed ){ + measureAllocationSize(db, p); + return; + } + assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); + sqlite3MemdebugSetType(p, MEMTYPE_HEAP); + sqlite3_free(p); +} +SQLITE_PRIVATE void sqlite3DbFree(sqlite3 *db, void *p){ + assert( db==0 || sqlite3_mutex_held(db->mutex) ); + if( p ) sqlite3DbFreeNN(db, p); +} /* ** Change the size of an existing memory allocation @@ -24127,18 +30991,26 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ if( nOld==nNew ){ pNew = pOld; }else if( sqlite3GlobalConfig.bMemstat ){ + sqlite3_int64 nUsed; sqlite3_mutex_enter(mem0.mutex); sqlite3StatusHighwater(SQLITE_STATUS_MALLOC_SIZE, (int)nBytes); nDiff = nNew - nOld; - if( sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED) >= + if( nDiff>0 && (nUsed = sqlite3StatusValue(SQLITE_STATUS_MEMORY_USED)) >= mem0.alarmThreshold-nDiff ){ sqlite3MallocAlarm(nDiff); + if( mem0.hardLimit>0 && nUsed >= mem0.hardLimit - nDiff ){ + sqlite3_mutex_leave(mem0.mutex); + test_oom_breakpoint(1); + return 0; + } } pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); +#ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT if( pNew==0 && mem0.alarmThreshold>0 ){ sqlite3MallocAlarm((int)nBytes); pNew = sqlite3GlobalConfig.m.xRealloc(pOld, nNew); } +#endif if( pNew ){ nNew = sqlite3MallocSize(pNew); sqlite3StatusUp(SQLITE_STATUS_MEMORY_USED, nNew-nOld); @@ -24155,14 +31027,14 @@ SQLITE_PRIVATE void *sqlite3Realloc(void *pOld, u64 nBytes){ ** The public interface to sqlite3Realloc. Make sure that the memory ** subsystem is initialized prior to invoking sqliteRealloc. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_realloc(void *pOld, int n){ +SQLITE_API void *sqlite3_realloc(void *pOld, int n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif if( n<0 ) n = 0; /* IMP: R-26507-47431 */ return sqlite3Realloc(pOld, n); } -SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ +SQLITE_API void *sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize() ) return 0; #endif @@ -24172,7 +31044,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_realloc64(void *pOld, sqlite3_uint64 n){ /* ** Allocate and zero memory. -*/ +*/ SQLITE_PRIVATE void *sqlite3MallocZero(u64 n){ void *p = sqlite3Malloc(n); if( p ){ @@ -24202,13 +31074,13 @@ static SQLITE_NOINLINE void *dbMallocRawFinish(sqlite3 *db, u64 n){ assert( db!=0 ); p = sqlite3Malloc(n); if( !p ) sqlite3OomFault(db); - sqlite3MemdebugSetType(p, + sqlite3MemdebugSetType(p, (db->lookaside.bDisable==0) ? MEMTYPE_LOOKASIDE : MEMTYPE_HEAP); return p; } /* -** Allocate memory, either lookaside (if possible) or heap. +** Allocate memory, either lookaside (if possible) or heap. ** If the allocation fails, set the mallocFailed flag in ** the connection pointer. ** @@ -24242,23 +31114,37 @@ SQLITE_PRIVATE void *sqlite3DbMallocRawNN(sqlite3 *db, u64 n){ assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); assert( db->pnBytesFreed==0 ); - if( db->lookaside.bDisable==0 ){ - assert( db->mallocFailed==0 ); - if( n>db->lookaside.sz ){ + if( n>db->lookaside.sz ){ + if( !db->lookaside.bDisable ){ db->lookaside.anStat[1]++; - }else if( (pBuf = db->lookaside.pFree)==0 ){ - db->lookaside.anStat[2]++; - }else{ - db->lookaside.pFree = pBuf->pNext; - db->lookaside.nOut++; + }else if( db->mallocFailed ){ + return 0; + } + return dbMallocRawFinish(db, n); + } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( n<=LOOKASIDE_SMALL ){ + if( (pBuf = db->lookaside.pSmallFree)!=0 ){ + db->lookaside.pSmallFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pSmallInit)!=0 ){ + db->lookaside.pSmallInit = pBuf->pNext; db->lookaside.anStat[0]++; - if( db->lookaside.nOut>db->lookaside.mxOut ){ - db->lookaside.mxOut = db->lookaside.nOut; - } return (void*)pBuf; } - }else if( db->mallocFailed ){ - return 0; + } +#endif + if( (pBuf = db->lookaside.pFree)!=0 ){ + db->lookaside.pFree = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else if( (pBuf = db->lookaside.pInit)!=0 ){ + db->lookaside.pInit = pBuf->pNext; + db->lookaside.anStat[0]++; + return (void*)pBuf; + }else{ + db->lookaside.anStat[2]++; } #else assert( db!=0 ); @@ -24282,7 +31168,16 @@ SQLITE_PRIVATE void *sqlite3DbRealloc(sqlite3 *db, void *p, u64 n){ assert( db!=0 ); if( p==0 ) return sqlite3DbMallocRawNN(db, n); assert( sqlite3_mutex_held(db->mutex) ); - if( isLookaside(db,p) && n<=db->lookaside.sz ) return p; + if( ((uptr)p)<(uptr)db->lookaside.pEnd ){ +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( ((uptr)p)>=(uptr)db->lookaside.pMiddle ){ + if( n<=LOOKASIDE_SMALL ) return p; + }else +#endif + if( ((uptr)p)>=(uptr)db->lookaside.pStart ){ + if( n<=db->lookaside.szTrue ) return p; + } + } return dbReallocFinish(db, p, n); } static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ @@ -24293,14 +31188,14 @@ static SQLITE_NOINLINE void *dbReallocFinish(sqlite3 *db, void *p, u64 n){ if( isLookaside(db, p) ){ pNew = sqlite3DbMallocRawNN(db, n); if( pNew ){ - memcpy(pNew, p, db->lookaside.sz); + memcpy(pNew, p, lookasideMallocSize(db, p)); sqlite3DbFree(db, p); } }else{ assert( sqlite3MemdebugHasType(p, (MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); assert( sqlite3MemdebugNoType(p, (u8)~(MEMTYPE_LOOKASIDE|MEMTYPE_HEAP)) ); sqlite3MemdebugSetType(p, MEMTYPE_HEAP); - pNew = sqlite3_realloc64(p, n); + pNew = sqlite3Realloc(p, n); if( !pNew ){ sqlite3OomFault(db); } @@ -24325,9 +31220,9 @@ SQLITE_PRIVATE void *sqlite3DbReallocOrFree(sqlite3 *db, void *p, u64 n){ } /* -** Make a copy of a string in memory obtained from sqliteMalloc(). These +** Make a copy of a string in memory obtained from sqliteMalloc(). These ** functions call sqlite3MallocRaw() directly instead of sqliteMalloc(). This -** is because when memory debugging is turned on, these two functions are +** is because when memory debugging is turned on, these two functions are ** called via macros that record the current file and line number in the ** ThreadData structure. */ @@ -24337,9 +31232,8 @@ SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ if( z==0 ){ return 0; } - n = sqlite3Strlen30(z) + 1; - assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRaw(db, (int)n); + n = strlen(z) + 1; + zNew = sqlite3DbMallocRaw(db, n); if( zNew ){ memcpy(zNew, z, n); } @@ -24348,11 +31242,9 @@ SQLITE_PRIVATE char *sqlite3DbStrDup(sqlite3 *db, const char *z){ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ char *zNew; assert( db!=0 ); - if( z==0 ){ - return 0; - } + assert( z!=0 || n==0 ); assert( (n&0x7fffffff)==n ); - zNew = sqlite3DbMallocRawNN(db, n+1); + zNew = z ? sqlite3DbMallocRawNN(db, n+1) : 0; if( zNew ){ memcpy(zNew, z, (size_t)n); zNew[n] = 0; @@ -24360,12 +31252,31 @@ SQLITE_PRIVATE char *sqlite3DbStrNDup(sqlite3 *db, const char *z, u64 n){ return zNew; } +/* +** The text between zStart and zEnd represents a phrase within a larger +** SQL statement. Make a copy of this phrase in space obtained form +** sqlite3DbMalloc(). Omit leading and trailing whitespace. +*/ +SQLITE_PRIVATE char *sqlite3DbSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ + int n; +#ifdef SQLITE_DEBUG + /* Because of the way the parser works, the span is guaranteed to contain + ** at least one non-space character */ + for(n=0; sqlite3Isspace(zStart[n]); n++){ assert( &zStart[n]mallocFailed, and also ** temporarily disable the lookaside memory allocator and interrupt ** any running VDBEs. +** +** Always return a NULL pointer so that this routine can be invoked using +** +** return sqlite3OomFault(db); +** +** and thereby avoid unnecessary stack frame allocations for the overwhelmingly +** common case where no OOM occurs. */ -SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ +SQLITE_PRIVATE void *sqlite3OomFault(sqlite3 *db){ if( db->mallocFailed==0 && db->bBenignMalloc==0 ){ db->mallocFailed = 1; if( db->nVdbeExec>0 ){ - db->u1.isInterrupted = 1; + AtomicStore(&db->u1.isInterrupted, 1); + } + DisableLookaside; + if( db->pParse ){ + Parse *pParse; + sqlite3ErrorMsg(db->pParse, "out of memory"); + db->pParse->rc = SQLITE_NOMEM_BKPT; + for(pParse=db->pParse->pOuterParse; pParse; pParse = pParse->pOuterParse){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } } - db->lookaside.bDisable++; } + return 0; } /* @@ -24394,51 +31322,54 @@ SQLITE_PRIVATE void sqlite3OomFault(sqlite3 *db){ SQLITE_PRIVATE void sqlite3OomClear(sqlite3 *db){ if( db->mallocFailed && db->nVdbeExec==0 ){ db->mallocFailed = 0; - db->u1.isInterrupted = 0; + AtomicStore(&db->u1.isInterrupted, 0); assert( db->lookaside.bDisable>0 ); - db->lookaside.bDisable--; + EnableLookaside; } } /* -** Take actions at the end of an API call to indicate an OOM error +** Take actions at the end of an API call to deal with error codes. */ -static SQLITE_NOINLINE int apiOomError(sqlite3 *db){ - sqlite3OomClear(db); - sqlite3Error(db, SQLITE_NOMEM); - return SQLITE_NOMEM_BKPT; +static SQLITE_NOINLINE int apiHandleError(sqlite3 *db, int rc){ + if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomClear(db); + sqlite3Error(db, SQLITE_NOMEM); + return SQLITE_NOMEM_BKPT; + } + return rc & db->errMask; } /* -** This function must be called before exiting any API function (i.e. +** This function must be called before exiting any API function (i.e. ** returning control to the user) that has called sqlite3_malloc or ** sqlite3_realloc. ** ** The returned value is normally a copy of the second argument to this ** function. However, if a malloc() failure has occurred since the previous -** invocation SQLITE_NOMEM is returned instead. +** invocation SQLITE_NOMEM is returned instead. ** ** If an OOM as occurred, then the connection error-code (the value ** returned by sqlite3_errcode()) is set to SQLITE_NOMEM. */ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ /* If the db handle must hold the connection handle mutex here. - ** Otherwise the read (and possible write) of db->mallocFailed + ** Otherwise the read (and possible write) of db->mallocFailed ** is unsafe, as is the call to sqlite3Error(). */ assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); - if( db->mallocFailed || rc==SQLITE_IOERR_NOMEM ){ - return apiOomError(db); + if( db->mallocFailed || rc ){ + return apiHandleError(db, rc); } - return rc & db->errMask; + return 0; } /************** End of malloc.c **********************************************/ /************** Begin file printf.c ******************************************/ /* ** The "printf" code that follows dates from the 1980's. It is in -** the public domain. +** the public domain. ** ************************************************************************** ** @@ -24453,7 +31384,7 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ ** Conversion types fall into various categories as defined by the ** following enumeration. */ -#define etRADIX 0 /* Integer types. %d, %x, %o, and so forth */ +#define etRADIX 0 /* non-decimal integer types. %x %o */ #define etFLOAT 1 /* Floating point. %f */ #define etEXP 2 /* Exponentional notation. %e and %E */ #define etGENERIC 3 /* Floating or exponential, depending on exponent. %g */ @@ -24467,12 +31398,13 @@ SQLITE_PRIVATE int sqlite3ApiExit(sqlite3* db, int rc){ #define etSQLESCAPE2 10 /* Strings with '\'' doubled and enclosed in '', NULL pointers replaced by SQL NULL. %Q */ #define etTOKEN 11 /* a pointer to a Token structure */ -#define etSRCLIST 12 /* a pointer to a SrcList */ +#define etSRCITEM 12 /* a pointer to a SrcItem */ #define etPOINTER 13 /* The %p conversion */ #define etSQLESCAPE3 14 /* %w -> Strings with '\"' doubled */ #define etORDINAL 15 /* %r -> 1st, 2nd, 3rd, 4th, etc. English only */ +#define etDECIMAL 16 /* %d or %u, but not %x, %o */ -#define etINVALID 16 /* Any unrecognized conversion type */ +#define etINVALID 17 /* Any unrecognized conversion type */ /* @@ -24496,9 +31428,8 @@ typedef struct et_info { /* Information about each format field */ /* ** Allowed values for et_info.flags */ -#define FLAG_SIGNED 1 /* True if the value to convert is signed */ -#define FLAG_INTERN 2 /* True if for internal use only */ -#define FLAG_STRING 4 /* Allow infinity precision */ +#define FLAG_SIGNED 1 /* True if the value to convert is signed */ +#define FLAG_STRING 4 /* Allow infinite precision */ /* @@ -24508,7 +31439,7 @@ typedef struct et_info { /* Information about each format field */ static const char aDigits[] = "0123456789ABCDEF0123456789abcdef"; static const char aPrefix[] = "-x0\000X0"; static const et_info fmtinfo[] = { - { 'd', 10, 1, etRADIX, 0, 0 }, + { 'd', 10, 1, etDECIMAL, 0, 0 }, { 's', 0, 4, etSTRING, 0, 0 }, { 'g', 0, 1, etGENERIC, 30, 0 }, { 'z', 0, 4, etDYNSTRING, 0, 0 }, @@ -24517,7 +31448,7 @@ static const et_info fmtinfo[] = { { 'w', 0, 4, etSQLESCAPE3, 0, 0 }, { 'c', 0, 0, etCHARX, 0, 0 }, { 'o', 8, 0, etRADIX, 0, 2 }, - { 'u', 10, 0, etRADIX, 0, 0 }, + { 'u', 10, 0, etDECIMAL, 0, 0 }, { 'x', 16, 0, etRADIX, 16, 1 }, { 'X', 16, 0, etRADIX, 0, 4 }, #ifndef SQLITE_OMIT_FLOATING_POINT @@ -24526,56 +31457,31 @@ static const et_info fmtinfo[] = { { 'E', 0, 1, etEXP, 14, 0 }, { 'G', 0, 1, etGENERIC, 14, 0 }, #endif - { 'i', 10, 1, etRADIX, 0, 0 }, + { 'i', 10, 1, etDECIMAL, 0, 0 }, { 'n', 0, 0, etSIZE, 0, 0 }, { '%', 0, 0, etPERCENT, 0, 0 }, { 'p', 16, 0, etPOINTER, 0, 1 }, -/* All the rest have the FLAG_INTERN bit set and are thus for internal -** use only */ - { 'T', 0, 2, etTOKEN, 0, 0 }, - { 'S', 0, 2, etSRCLIST, 0, 0 }, - { 'r', 10, 3, etORDINAL, 0, 0 }, + /* All the rest are undocumented and are for internal use only */ + { 'T', 0, 0, etTOKEN, 0, 0 }, + { 'S', 0, 0, etSRCITEM, 0, 0 }, + { 'r', 10, 1, etORDINAL, 0, 0 }, }; -/* -** If SQLITE_OMIT_FLOATING_POINT is defined, then none of the floating point -** conversions will work. -*/ -#ifndef SQLITE_OMIT_FLOATING_POINT -/* -** "*val" is a double such that 0.1 <= *val < 10.0 -** Return the ascii code for the leading digit of *val, then -** multiply "*val" by 10.0 to renormalize. +/* Notes: ** -** Example: -** input: *val = 3.14159 -** output: *val = 1.4159 function return = '3' -** -** The counter *cnt is incremented each time. After counter exceeds -** 16 (the number of significant digits in a 64-bit float) '0' is -** always returned. +** %S Takes a pointer to SrcItem. Shows name or database.name +** %!S Like %S but prefer the zName over the zAlias */ -static char et_getdigit(LONGDOUBLE_TYPE *val, int *cnt){ - int digit; - LONGDOUBLE_TYPE d; - if( (*cnt)<=0 ) return '0'; - (*cnt)--; - digit = (int)*val; - d = digit; - digit += '0'; - *val = (*val - d)*10.0; - return (char)digit; -} -#endif /* SQLITE_OMIT_FLOATING_POINT */ /* ** Set the StrAccum object to an error mode. */ -static void setStrAccumError(StrAccum *p, u8 eError){ - assert( eError==STRACCUM_NOMEM || eError==STRACCUM_TOOBIG ); +SQLITE_PRIVATE void sqlite3StrAccumSetError(StrAccum *p, u8 eError){ + assert( eError==SQLITE_NOMEM || eError==SQLITE_TOOBIG ); p->accError = eError; - p->nAlloc = 0; + if( p->mxAlloc ) sqlite3_str_reset(p); + if( eError==SQLITE_TOOBIG ) sqlite3ErrorToParser(p->db, eError); } /* @@ -24594,6 +31500,28 @@ static char *getTextArg(PrintfArguments *p){ return (char*)sqlite3_value_text(p->apArg[p->nUsed++]); } +/* +** Allocate memory for a temporary buffer needed for printf rendering. +** +** If the requested size of the temp buffer is larger than the size +** of the output buffer in pAccum, then cause an SQLITE_TOOBIG error. +** Do the size check before the memory allocation to prevent rogue +** SQL from requesting large allocations using the precision or width +** field of the printf() function. +*/ +static char *printfTempBuf(sqlite3_str *pAccum, sqlite3_int64 n){ + char *z; + if( pAccum->accError ) return 0; + if( n>pAccum->nAlloc && n>pAccum->mxAlloc ){ + sqlite3StrAccumSetError(pAccum, SQLITE_TOOBIG); + return 0; + } + z = sqlite3DbMallocRaw(pAccum->db, n); + if( z==0 ){ + sqlite3StrAccumSetError(pAccum, SQLITE_NOMEM); + } + return z; +} /* ** On machines with a small stack size, you can redefine the @@ -24604,11 +31532,18 @@ static char *getTextArg(PrintfArguments *p){ #endif #define etBUFSIZE SQLITE_PRINT_BUF_SIZE /* Size of the output buffer */ +/* +** Hard limit on the precision of floating-point conversions. +*/ +#ifndef SQLITE_PRINTF_PRECISION_LIMIT +# define SQLITE_FP_PRECISION_LIMIT 100000000 +#endif + /* ** Render a string given by "fmt" into the StrAccum object. */ -SQLITE_PRIVATE void sqlite3VXPrintf( - StrAccum *pAccum, /* Accumulate results here */ +SQLITE_API void sqlite3_str_vappendf( + sqlite3_str *pAccum, /* Accumulate results here */ const char *fmt, /* Format string */ va_list ap /* arguments */ ){ @@ -24619,42 +31554,40 @@ SQLITE_PRIVATE void sqlite3VXPrintf( int idx; /* A general purpose loop counter */ int width; /* Width of the current field */ etByte flag_leftjustify; /* True if "-" flag is present */ - etByte flag_plussign; /* True if "+" flag is present */ - etByte flag_blanksign; /* True if " " flag is present */ + etByte flag_prefix; /* '+' or ' ' or 0 for prefix */ etByte flag_alternateform; /* True if "#" flag is present */ etByte flag_altform2; /* True if "!" flag is present */ etByte flag_zeropad; /* True if field width constant starts with zero */ - etByte flag_long; /* True if "l" flag is present */ - etByte flag_longlong; /* True if the "ll" flag is present */ + etByte flag_long; /* 1 for the "l" flag, 2 for "ll", 0 by default */ etByte done; /* Loop termination flag */ + etByte cThousand; /* Thousands separator for %d and %u */ etByte xtype = etINVALID; /* Conversion paradigm */ u8 bArgList; /* True for SQLITE_PRINTF_SQLFUNC */ - u8 useIntern; /* Ok to use internal conversions (ex: %T) */ char prefix; /* Prefix character. "+" or "-" or " " or '\0'. */ sqlite_uint64 longvalue; /* Value for integer types */ - LONGDOUBLE_TYPE realvalue; /* Value for real types */ + double realvalue; /* Value for real types */ const et_info *infop; /* Pointer to the appropriate info structure */ char *zOut; /* Rendering buffer */ int nOut; /* Size of the rendering buffer */ char *zExtra = 0; /* Malloced memory used by some conversion */ -#ifndef SQLITE_OMIT_FLOATING_POINT - int exp, e2; /* exponent of real numbers */ - int nsd; /* Number of significant digits returned */ - double rounder; /* Used for rounding floating point values */ + int exp, e2; /* exponent of real numbers */ etByte flag_dp; /* True if decimal point should be shown */ etByte flag_rtz; /* True if trailing zeros should be removed */ -#endif + PrintfArguments *pArgList = 0; /* Arguments for SQLITE_PRINTF_SQLFUNC */ char buf[etBUFSIZE]; /* Conversion buffer */ + /* pAccum never starts out with an empty buffer that was obtained from + ** malloc(). This precondition is required by the mprintf("%z...") + ** optimization. */ + assert( pAccum->nChar>0 || (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + bufpt = 0; - if( pAccum->printfFlags ){ - if( (bArgList = (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC))!=0 ){ - pArgList = va_arg(ap, PrintfArguments*); - } - useIntern = pAccum->printfFlags & SQLITE_PRINTF_INTERNAL; + if( (pAccum->printfFlags & SQLITE_PRINTF_SQLFUNC)!=0 ){ + pArgList = va_arg(ap, PrintfArguments*); + bArgList = 1; }else{ - bArgList = useIntern = 0; + bArgList = 0; } for(; (c=(*fmt))!=0; ++fmt){ if( c!='%' ){ @@ -24664,113 +31597,124 @@ SQLITE_PRIVATE void sqlite3VXPrintf( #else do{ fmt++; }while( *fmt && *fmt != '%' ); #endif - sqlite3StrAccumAppend(pAccum, bufpt, (int)(fmt - bufpt)); + sqlite3_str_append(pAccum, bufpt, (int)(fmt - bufpt)); if( *fmt==0 ) break; } if( (c=(*++fmt))==0 ){ - sqlite3StrAccumAppend(pAccum, "%", 1); + sqlite3_str_append(pAccum, "%", 1); break; } /* Find out what flags are present */ - flag_leftjustify = flag_plussign = flag_blanksign = + flag_leftjustify = flag_prefix = cThousand = flag_alternateform = flag_altform2 = flag_zeropad = 0; done = 0; + width = 0; + flag_long = 0; + precision = -1; do{ switch( c ){ case '-': flag_leftjustify = 1; break; - case '+': flag_plussign = 1; break; - case ' ': flag_blanksign = 1; break; + case '+': flag_prefix = '+'; break; + case ' ': flag_prefix = ' '; break; case '#': flag_alternateform = 1; break; case '!': flag_altform2 = 1; break; case '0': flag_zeropad = 1; break; + case ',': cThousand = ','; break; default: done = 1; break; + case 'l': { + flag_long = 1; + c = *++fmt; + if( c=='l' ){ + c = *++fmt; + flag_long = 2; + } + done = 1; + break; + } + case '1': case '2': case '3': case '4': case '5': + case '6': case '7': case '8': case '9': { + unsigned wx = c - '0'; + while( (c = *++fmt)>='0' && c<='9' ){ + wx = wx*10 + c - '0'; + } + testcase( wx>0x7fffffff ); + width = wx & 0x7fffffff; +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( c!='.' && c!='l' ){ + done = 1; + }else{ + fmt--; + } + break; + } + case '*': { + if( bArgList ){ + width = (int)getIntArg(pArgList); + }else{ + width = va_arg(ap,int); + } + if( width<0 ){ + flag_leftjustify = 1; + width = width >= -2147483647 ? -width : 0; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ + width = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( (c = fmt[1])!='.' && c!='l' ){ + c = *++fmt; + done = 1; + } + break; + } + case '.': { + c = *++fmt; + if( c=='*' ){ + if( bArgList ){ + precision = (int)getIntArg(pArgList); + }else{ + precision = va_arg(ap,int); + } + if( precision<0 ){ + precision = precision >= -2147483647 ? -precision : -1; + } + c = *++fmt; + }else{ + unsigned px = 0; + while( c>='0' && c<='9' ){ + px = px*10 + c - '0'; + c = *++fmt; + } + testcase( px>0x7fffffff ); + precision = px & 0x7fffffff; + } +#ifdef SQLITE_PRINTF_PRECISION_LIMIT + if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ + precision = SQLITE_PRINTF_PRECISION_LIMIT; + } +#endif + if( c=='l' ){ + --fmt; + }else{ + done = 1; + } + break; + } } }while( !done && (c=(*++fmt))!=0 ); - /* Get the field width */ - if( c=='*' ){ - if( bArgList ){ - width = (int)getIntArg(pArgList); - }else{ - width = va_arg(ap,int); - } - if( width<0 ){ - flag_leftjustify = 1; - width = width >= -2147483647 ? -width : 0; - } - c = *++fmt; - }else{ - unsigned wx = 0; - while( c>='0' && c<='9' ){ - wx = wx*10 + c - '0'; - c = *++fmt; - } - testcase( wx>0x7fffffff ); - width = wx & 0x7fffffff; - } - assert( width>=0 ); -#ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( width>SQLITE_PRINTF_PRECISION_LIMIT ){ - width = SQLITE_PRINTF_PRECISION_LIMIT; - } -#endif - /* Get the precision */ - if( c=='.' ){ - c = *++fmt; - if( c=='*' ){ - if( bArgList ){ - precision = (int)getIntArg(pArgList); - }else{ - precision = va_arg(ap,int); - } - c = *++fmt; - if( precision<0 ){ - precision = precision >= -2147483647 ? -precision : -1; - } - }else{ - unsigned px = 0; - while( c>='0' && c<='9' ){ - px = px*10 + c - '0'; - c = *++fmt; - } - testcase( px>0x7fffffff ); - precision = px & 0x7fffffff; - } - }else{ - precision = -1; - } - assert( precision>=(-1) ); -#ifdef SQLITE_PRINTF_PRECISION_LIMIT - if( precision>SQLITE_PRINTF_PRECISION_LIMIT ){ - precision = SQLITE_PRINTF_PRECISION_LIMIT; - } -#endif - - - /* Get the conversion type modifier */ - if( c=='l' ){ - flag_long = 1; - c = *++fmt; - if( c=='l' ){ - flag_longlong = 1; - c = *++fmt; - }else{ - flag_longlong = 0; - } - }else{ - flag_long = flag_longlong = 0; - } /* Fetch the info entry for the field */ infop = &fmtinfo[0]; xtype = etINVALID; for(idx=0; idxflags & FLAG_INTERN)==0 ){ - xtype = infop->type; - }else{ - return; - } + xtype = infop->type; break; } } @@ -24780,15 +31724,11 @@ SQLITE_PRIVATE void sqlite3VXPrintf( ** ** flag_alternateform TRUE if a '#' is present. ** flag_altform2 TRUE if a '!' is present. - ** flag_plussign TRUE if a '+' is present. + ** flag_prefix '+' or ' ' or zero ** flag_leftjustify TRUE if a '-' is present or if the ** field width was negative. ** flag_zeropad TRUE if the width began with 0. - ** flag_long TRUE if the letter 'l' (ell) prefixed - ** the conversion character. - ** flag_longlong TRUE if the letter 'll' (ell ell) prefixed - ** the conversion character. - ** flag_blanksign TRUE if a ' ' is present. + ** flag_long 1 for "l", 2 for "ll" ** width The specified field width. This is ** always non-negative. Zero is the default. ** precision The specified precision. The default @@ -24796,44 +31736,50 @@ SQLITE_PRIVATE void sqlite3VXPrintf( ** xtype The class of the conversion. ** infop Pointer to the appropriate info struct. */ + assert( width>=0 ); + assert( precision>=(-1) ); switch( xtype ){ case etPOINTER: - flag_longlong = sizeof(char*)==sizeof(i64); - flag_long = sizeof(char*)==sizeof(long int); - /* Fall through into the next case */ + flag_long = sizeof(char*)==sizeof(i64) ? 2 : + sizeof(char*)==sizeof(long int) ? 1 : 0; + /* no break */ deliberate_fall_through case etORDINAL: case etRADIX: + cThousand = 0; + /* no break */ deliberate_fall_through + case etDECIMAL: if( infop->flags & FLAG_SIGNED ){ i64 v; if( bArgList ){ v = getIntArg(pArgList); - }else if( flag_longlong ){ - v = va_arg(ap,i64); }else if( flag_long ){ - v = va_arg(ap,long int); + if( flag_long==2 ){ + v = va_arg(ap,i64) ; + }else{ + v = va_arg(ap,long int); + } }else{ v = va_arg(ap,int); } if( v<0 ){ - if( v==SMALLEST_INT64 ){ - longvalue = ((u64)1)<<63; - }else{ - longvalue = -v; - } + testcase( v==SMALLEST_INT64 ); + testcase( v==(-1) ); + longvalue = ~v; + longvalue++; prefix = '-'; }else{ longvalue = v; - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + prefix = flag_prefix; } }else{ if( bArgList ){ longvalue = (u64)getIntArg(pArgList); - }else if( flag_longlong ){ - longvalue = va_arg(ap,u64); }else if( flag_long ){ - longvalue = va_arg(ap,unsigned long int); + if( flag_long==2 ){ + longvalue = va_arg(ap,u64); + }else{ + longvalue = va_arg(ap,unsigned long int); + } }else{ longvalue = va_arg(ap,unsigned int); } @@ -24843,16 +31789,16 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( flag_zeropad && precision0 ); } length = (int)(&zOut[nOut-1]-bufpt); - for(idx=precision-length; idx>0; idx--){ + while( precision>length ){ *(--bufpt) = '0'; /* Zero pad */ + length++; + } + if( cThousand ){ + int nn = (length - 1)/3; /* Number of "," to insert */ + int ix = (length - 1)%3 + 1; + bufpt -= nn; + for(idx=0; nn>0; idx++){ + bufpt[idx] = bufpt[idx+nn]; + ix--; + if( ix==0 ){ + bufpt[++idx] = cThousand; + nn--; + ix = 3; + } + } } if( prefix ) *(--bufpt) = prefix; /* Add sign */ if( flag_alternateform && infop->prefix ){ /* Add "0" or "0x" */ @@ -24887,61 +31848,69 @@ SQLITE_PRIVATE void sqlite3VXPrintf( break; case etFLOAT: case etEXP: - case etGENERIC: + case etGENERIC: { + FpDecode s; + int iRound; + int j; + if( bArgList ){ realvalue = getDoubleArg(pArgList); }else{ realvalue = va_arg(ap,double); } -#ifdef SQLITE_OMIT_FLOATING_POINT - length = 0; -#else if( precision<0 ) precision = 6; /* Set default precision */ - if( realvalue<0.0 ){ - realvalue = -realvalue; - prefix = '-'; +#ifdef SQLITE_FP_PRECISION_LIMIT + if( precision>SQLITE_FP_PRECISION_LIMIT ){ + precision = SQLITE_FP_PRECISION_LIMIT; + } +#endif + if( xtype==etFLOAT ){ + iRound = -precision; + }else if( xtype==etGENERIC ){ + if( precision==0 ) precision = 1; + iRound = precision; }else{ - if( flag_plussign ) prefix = '+'; - else if( flag_blanksign ) prefix = ' '; - else prefix = 0; + iRound = precision+1; } - if( xtype==etGENERIC && precision>0 ) precision--; - testcase( precision>0xfff ); - for(idx=precision&0xfff, rounder=0.5; idx>0; idx--, rounder*=0.1){} - if( xtype==etFLOAT ) realvalue += rounder; - /* Normalize realvalue to within 10.0 > realvalue >= 1.0 */ - exp = 0; - if( sqlite3IsNaN((double)realvalue) ){ - bufpt = "NaN"; - length = 3; - break; - } - if( realvalue>0.0 ){ - LONGDOUBLE_TYPE scale = 1.0; - while( realvalue>=1e100*scale && exp<=350 ){ scale *= 1e100;exp+=100;} - while( realvalue>=1e10*scale && exp<=350 ){ scale *= 1e10; exp+=10; } - while( realvalue>=10.0*scale && exp<=350 ){ scale *= 10.0; exp++; } - realvalue /= scale; - while( realvalue<1e-8 ){ realvalue *= 1e8; exp-=8; } - while( realvalue<1.0 ){ realvalue *= 10.0; exp--; } - if( exp>350 ){ + sqlite3FpDecode(&s, realvalue, iRound, flag_altform2 ? 26 : 16); + if( s.isSpecial ){ + if( s.isSpecial==2 ){ + bufpt = flag_zeropad ? "null" : "NaN"; + length = sqlite3Strlen30(bufpt); + break; + }else if( flag_zeropad ){ + s.z[0] = '9'; + s.iDP = 1000; + s.n = 1; + }else{ + memcpy(buf, "-Inf", 5); bufpt = buf; - buf[0] = prefix; - memcpy(buf+(prefix!=0),"Inf",4); - length = 3+(prefix!=0); + if( s.sign=='-' ){ + /* no-op */ + }else if( flag_prefix ){ + buf[0] = flag_prefix; + }else{ + bufpt++; + } + length = sqlite3Strlen30(bufpt); break; } } - bufpt = buf; + if( s.sign=='-' ){ + prefix = '-'; + }else{ + prefix = flag_prefix; + } + + exp = s.iDP-1; + /* ** If the field type is etGENERIC, then convert to either etEXP ** or etFLOAT, as appropriate. */ - if( xtype!=etFLOAT ){ - realvalue += rounder; - if( realvalue>=10.0 ){ realvalue *= 0.1; exp++; } - } if( xtype==etGENERIC ){ + assert( precision>0 ); + precision--; flag_rtz = !flag_alternateform; if( exp<-4 || exp>precision ){ xtype = etEXP; @@ -24955,29 +31924,32 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( xtype==etEXP ){ e2 = 0; }else{ - e2 = exp; + e2 = s.iDP - 1; } - if( MAX(e2,0)+(i64)precision+(i64)width > etBUFSIZE - 15 ){ - bufpt = zExtra - = sqlite3Malloc( MAX(e2,0)+(i64)precision+(i64)width+15 ); - if( bufpt==0 ){ - setStrAccumError(pAccum, STRACCUM_NOMEM); - return; + bufpt = buf; + { + i64 szBufNeeded; /* Size of a temporary buffer needed */ + szBufNeeded = MAX(e2,0)+(i64)precision+(i64)width+15; + if( cThousand && e2>0 ) szBufNeeded += (e2+2)/3; + if( szBufNeeded > etBUFSIZE ){ + bufpt = zExtra = printfTempBuf(pAccum, szBufNeeded); + if( bufpt==0 ) return; } } zOut = bufpt; - nsd = 16 + flag_altform2*10; flag_dp = (precision>0 ?1:0) | flag_alternateform | flag_altform2; /* The sign in front of the number */ if( prefix ){ *(bufpt++) = prefix; } /* Digits prior to the decimal point */ + j = 0; if( e2<0 ){ *(bufpt++) = '0'; }else{ for(; e2>=0; e2--){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + *(bufpt++) = j1 ) *(bufpt++) = ','; } } /* The decimal point */ @@ -24986,13 +31958,12 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } /* "0" digits after the decimal point but before the first ** significant digit of the number */ - for(e2++; e2<0; precision--, e2++){ - assert( precision>0 ); + for(e2++; e2<0 && precision>0; precision--, e2++){ *(bufpt++) = '0'; } /* Significant digits after the decimal point */ while( (precision--)>0 ){ - *(bufpt++) = et_getdigit(&realvalue,&nsd); + *(bufpt++) = jcharset]; if( exp<0 ){ *(bufpt++) = '-'; exp = -exp; @@ -25041,8 +32013,8 @@ SQLITE_PRIVATE void sqlite3VXPrintf( while( nPad-- ) bufpt[i++] = '0'; length = width; } -#endif /* !defined(SQLITE_OMIT_FLOATING_POINT) */ break; + } case etSIZE: if( !bArgList ){ *(va_arg(ap,int*)) = pAccum->nChar; @@ -25057,22 +32029,65 @@ SQLITE_PRIVATE void sqlite3VXPrintf( case etCHARX: if( bArgList ){ bufpt = getTextArg(pArgList); - c = bufpt ? bufpt[0] : 0; + length = 1; + if( bufpt ){ + buf[0] = c = *(bufpt++); + if( (c&0xc0)==0xc0 ){ + while( length<4 && (bufpt[0]&0xc0)==0x80 ){ + buf[length++] = *(bufpt++); + } + } + }else{ + buf[0] = 0; + } }else{ - c = va_arg(ap,int); + unsigned int ch = va_arg(ap,unsigned int); + if( ch<0x00080 ){ + buf[0] = ch & 0xff; + length = 1; + }else if( ch<0x00800 ){ + buf[0] = 0xc0 + (u8)((ch>>6)&0x1f); + buf[1] = 0x80 + (u8)(ch & 0x3f); + length = 2; + }else if( ch<0x10000 ){ + buf[0] = 0xe0 + (u8)((ch>>12)&0x0f); + buf[1] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[2] = 0x80 + (u8)(ch & 0x3f); + length = 3; + }else{ + buf[0] = 0xf0 + (u8)((ch>>18) & 0x07); + buf[1] = 0x80 + (u8)((ch>>12) & 0x3f); + buf[2] = 0x80 + (u8)((ch>>6) & 0x3f); + buf[3] = 0x80 + (u8)(ch & 0x3f); + length = 4; + } } if( precision>1 ){ + i64 nPrior = 1; width -= precision-1; if( width>1 && !flag_leftjustify ){ - sqlite3AppendChar(pAccum, width-1, ' '); + sqlite3_str_appendchar(pAccum, width-1, ' '); width = 0; } - sqlite3AppendChar(pAccum, precision-1, c); + sqlite3_str_append(pAccum, buf, length); + precision--; + while( precision > 1 ){ + i64 nCopyBytes; + if( nPrior > precision-1 ) nPrior = precision - 1; + nCopyBytes = length*nPrior; + if( nCopyBytes + pAccum->nChar >= pAccum->nAlloc ){ + sqlite3StrAccumEnlarge(pAccum, nCopyBytes); + } + if( pAccum->accError ) break; + sqlite3_str_append(pAccum, + &pAccum->zText[pAccum->nChar-nCopyBytes], nCopyBytes); + precision -= nPrior; + nPrior *= 2; + } } - length = 1; - buf[0] = c; bufpt = buf; - break; + flag_altform2 = 1; + goto adjust_width_for_utf8; case etSTRING: case etDYNSTRING: if( bArgList ){ @@ -25084,19 +32099,52 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( bufpt==0 ){ bufpt = ""; }else if( xtype==etDYNSTRING ){ + if( pAccum->nChar==0 + && pAccum->mxAlloc + && width==0 + && precision<0 + && pAccum->accError==0 + ){ + /* Special optimization for sqlite3_mprintf("%z..."): + ** Extend an existing memory allocation rather than creating + ** a new one. */ + assert( (pAccum->printfFlags&SQLITE_PRINTF_MALLOCED)==0 ); + pAccum->zText = bufpt; + pAccum->nAlloc = sqlite3DbMallocSize(pAccum->db, bufpt); + pAccum->nChar = 0x7fffffff & (int)strlen(bufpt); + pAccum->printfFlags |= SQLITE_PRINTF_MALLOCED; + length = 0; + break; + } zExtra = bufpt; } if( precision>=0 ){ - for(length=0; length 0 && z[0] ){ + SQLITE_SKIP_UTF8(z); + } + length = (int)(z - (unsigned char*)bufpt); + }else{ + for(length=0; length0 ){ + /* Adjust width to account for extra bytes in UTF-8 characters */ + int ii = length - 1; + while( ii>=0 ) if( (bufpt[ii--] & 0xc0)==0x80 ) width++; } break; - case etSQLESCAPE: /* Escape ' characters */ - case etSQLESCAPE2: /* Escape ' and enclose in '...' */ - case etSQLESCAPE3: { /* Escape " characters */ - int i, j, k, n, isnull; - int needQuote; + case etSQLESCAPE: /* %q: Escape ' characters */ + case etSQLESCAPE2: /* %Q: Escape ' and enclose in '...' */ + case etSQLESCAPE3: { /* %w: Escape " characters */ + i64 i, j, k, n; + int needQuote, isnull; char ch; char q = ((xtype==etSQLESCAPE3)?'"':'\''); /* Quote character */ char *escarg; @@ -25108,18 +32156,23 @@ SQLITE_PRIVATE void sqlite3VXPrintf( } isnull = escarg==0; if( isnull ) escarg = (xtype==etSQLESCAPE2 ? "NULL" : "(NULL)"); + /* For %q, %Q, and %w, the precision is the number of bytes (or + ** characters if the ! flags is present) to use from the input. + ** Because of the extra quoting characters inserted, the number + ** of output characters may be larger than the precision. + */ k = precision; for(i=n=0; k!=0 && (ch=escarg[i])!=0; i++, k--){ if( ch==q ) n++; + if( flag_altform2 && (ch&0xc0)==0xc0 ){ + while( (escarg[i+1]&0xc0)==0x80 ){ i++; } + } } needQuote = !isnull && xtype==etSQLESCAPE2; n += i + 3; if( n>etBUFSIZE ){ - bufpt = zExtra = sqlite3Malloc( n ); - if( bufpt==0 ){ - setStrAccumError(pAccum, STRACCUM_NOMEM); - return; - } + bufpt = zExtra = printfTempBuf(pAccum, n); + if( bufpt==0 ) return; }else{ bufpt = buf; } @@ -25133,31 +32186,60 @@ SQLITE_PRIVATE void sqlite3VXPrintf( if( needQuote ) bufpt[j++] = q; bufpt[j] = 0; length = j; - /* The precision in %q and %Q means how many input characters to - ** consume, not the length of the output... - ** if( precision>=0 && precisionn ){ - sqlite3StrAccumAppend(pAccum, (const char*)pToken->z, pToken->n); + if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + if( flag_alternateform ){ + /* %#T means an Expr pointer that uses Expr.u.zToken */ + Expr *pExpr = va_arg(ap,Expr*); + if( ALWAYS(pExpr) && ALWAYS(!ExprHasProperty(pExpr,EP_IntValue)) ){ + sqlite3_str_appendall(pAccum, (const char*)pExpr->u.zToken); + sqlite3RecordErrorOffsetOfExpr(pAccum->db, pExpr); + } + }else{ + /* %T means a Token pointer */ + Token *pToken = va_arg(ap, Token*); + assert( bArgList==0 ); + if( pToken && pToken->n ){ + sqlite3_str_append(pAccum, (const char*)pToken->z, pToken->n); + sqlite3RecordErrorByteOffset(pAccum->db, pToken->z); + } } length = width = 0; break; } - case etSRCLIST: { - SrcList *pSrc = va_arg(ap, SrcList*); - int k = va_arg(ap, int); - struct SrcList_item *pItem = &pSrc->a[k]; + case etSRCITEM: { + SrcItem *pItem; + if( (pAccum->printfFlags & SQLITE_PRINTF_INTERNAL)==0 ) return; + pItem = va_arg(ap, SrcItem*); assert( bArgList==0 ); - assert( k>=0 && knSrc ); - if( pItem->zDatabase ){ - sqlite3StrAccumAppendAll(pAccum, pItem->zDatabase); - sqlite3StrAccumAppend(pAccum, ".", 1); + if( pItem->zAlias && !flag_altform2 ){ + sqlite3_str_appendall(pAccum, pItem->zAlias); + }else if( pItem->zName ){ + if( pItem->fg.fixedSchema==0 + && pItem->fg.isSubquery==0 + && pItem->u4.zDatabase!=0 + ){ + sqlite3_str_appendall(pAccum, pItem->u4.zDatabase); + sqlite3_str_append(pAccum, ".", 1); + } + sqlite3_str_appendall(pAccum, pItem->zName); + }else if( pItem->zAlias ){ + sqlite3_str_appendall(pAccum, pItem->zAlias); + }else if( ALWAYS(pItem->fg.isSubquery) ){/* Because of tag-20240424-1 */ + Select *pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); + if( pSel->selFlags & SF_NestedFrom ){ + sqlite3_str_appendf(pAccum, "(join-%u)", pSel->selId); + }else if( pSel->selFlags & SF_MultiValue ){ + assert( !pItem->fg.isTabFunc && !pItem->fg.isIndexedBy ); + sqlite3_str_appendf(pAccum, "%u-ROW VALUES CLAUSE", + pItem->u1.nRow); + }else{ + sqlite3_str_appendf(pAccum, "(subquery-%u)", pSel->selId); + } } - sqlite3StrAccumAppendAll(pAccum, pItem->zName); length = width = 0; break; } @@ -25169,12 +32251,19 @@ SQLITE_PRIVATE void sqlite3VXPrintf( /* ** The text of the conversion is pointed to by "bufpt" and is ** "length" characters long. The field width is "width". Do - ** the output. + ** the output. Both length and width are in bytes, not characters, + ** at this point. If the "!" flag was present on string conversions + ** indicating that width and precision should be expressed in characters, + ** then the values have been translated prior to reaching this point. */ width -= length; - if( width>0 && !flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); - sqlite3StrAccumAppend(pAccum, bufpt, length); - if( width>0 && flag_leftjustify ) sqlite3AppendChar(pAccum, width, ' '); + if( width>0 ){ + if( !flag_leftjustify ) sqlite3_str_appendchar(pAccum, width, ' '); + sqlite3_str_append(pAccum, bufpt, length); + if( flag_leftjustify ) sqlite3_str_appendchar(pAccum, width, ' '); + }else{ + sqlite3_str_append(pAccum, bufpt, length); + } if( zExtra ){ sqlite3DbFree(pAccum->db, zExtra); @@ -25183,6 +32272,45 @@ SQLITE_PRIVATE void sqlite3VXPrintf( }/* End for loop over the format string */ } /* End of function */ + +/* +** The z string points to the first character of a token that is +** associated with an error. If db does not already have an error +** byte offset recorded, try to compute the error byte offset for +** z and set the error byte offset in db. +*/ +SQLITE_PRIVATE void sqlite3RecordErrorByteOffset(sqlite3 *db, const char *z){ + const Parse *pParse; + const char *zText; + const char *zEnd; + assert( z!=0 ); + if( NEVER(db==0) ) return; + if( db->errByteOffset!=(-2) ) return; + pParse = db->pParse; + if( NEVER(pParse==0) ) return; + zText =pParse->zTail; + if( NEVER(zText==0) ) return; + zEnd = &zText[strlen(zText)]; + if( SQLITE_WITHIN(z,zText,zEnd) ){ + db->errByteOffset = (int)(z-zText); + } +} + +/* +** If pExpr has a byte offset for the start of a token, record that as +** as the error offset. +*/ +SQLITE_PRIVATE void sqlite3RecordErrorOffsetOfExpr(sqlite3 *db, const Expr *pExpr){ + while( pExpr + && (ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) || pExpr->w.iOfst<=0) + ){ + pExpr = pExpr->pLeft; + } + if( pExpr==0 ) return; + if( ExprHasProperty(pExpr, EP_FromDDL) ) return; + db->errByteOffset = pExpr->w.iOfst; +} + /* ** Enlarge the memory allocation on a StrAccum object so that it is ** able to accept at least N more bytes of text. @@ -25190,31 +32318,28 @@ SQLITE_PRIVATE void sqlite3VXPrintf( ** Return the number of bytes of text that StrAccum is able to accept ** after the attempted enlargement. The value returned might be zero. */ -static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ +SQLITE_PRIVATE int sqlite3StrAccumEnlarge(StrAccum *p, i64 N){ char *zNew; - assert( p->nChar+(i64)N >= p->nAlloc ); /* Only called if really needed */ + assert( p->nChar+N >= p->nAlloc ); /* Only called if really needed */ if( p->accError ){ - testcase(p->accError==STRACCUM_TOOBIG); - testcase(p->accError==STRACCUM_NOMEM); + testcase(p->accError==SQLITE_TOOBIG); + testcase(p->accError==SQLITE_NOMEM); return 0; } if( p->mxAlloc==0 ){ - N = p->nAlloc - p->nChar - 1; - setStrAccumError(p, STRACCUM_TOOBIG); - return N; + sqlite3StrAccumSetError(p, SQLITE_TOOBIG); + return p->nAlloc - p->nChar - 1; }else{ char *zOld = isMalloced(p) ? p->zText : 0; - i64 szNew = p->nChar; - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); - szNew += N + 1; + i64 szNew = p->nChar + N + 1; if( szNew+p->nChar<=p->mxAlloc ){ /* Force exponential buffer size growth as long as it does not overflow, ** to avoid having to call this routine too often */ szNew += p->nChar; } if( szNew > p->mxAlloc ){ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_TOOBIG); + sqlite3_str_reset(p); + sqlite3StrAccumSetError(p, SQLITE_TOOBIG); return 0; }else{ p->nAlloc = (int)szNew; @@ -25222,7 +32347,7 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ if( p->db ){ zNew = sqlite3DbRealloc(p->db, zOld, p->nAlloc); }else{ - zNew = sqlite3_realloc64(zOld, p->nAlloc); + zNew = sqlite3Realloc(zOld, p->nAlloc); } if( zNew ){ assert( p->zText!=0 || p->nChar==0 ); @@ -25231,23 +32356,23 @@ static int sqlite3StrAccumEnlarge(StrAccum *p, int N){ p->nAlloc = sqlite3DbMallocSize(p->db, zNew); p->printfFlags |= SQLITE_PRINTF_MALLOCED; }else{ - sqlite3StrAccumReset(p); - setStrAccumError(p, STRACCUM_NOMEM); + sqlite3_str_reset(p); + sqlite3StrAccumSetError(p, SQLITE_NOMEM); return 0; } } - return N; + assert( N>=0 && N<=0x7fffffff ); + return (int)N; } /* ** Append N copies of character c to the given string buffer. */ -SQLITE_PRIVATE void sqlite3AppendChar(StrAccum *p, int N, char c){ +SQLITE_API void sqlite3_str_appendchar(sqlite3_str *p, int N, char c){ testcase( p->nChar + (i64)N > 0x7fffffff ); if( p->nChar+(i64)N >= p->nAlloc && (N = sqlite3StrAccumEnlarge(p, N))<=0 ){ return; } - assert( (p->zText==p->zBase)==!isMalloced(p) ); while( (N--)>0 ) p->zText[p->nChar++] = c; } @@ -25255,9 +32380,9 @@ SQLITE_PRIVATE void sqlite3AppendChar(StrAccum *p, int N, char c){ ** The StrAccum "p" is not large enough to accept N new bytes of z[]. ** So enlarge if first, then do the append. ** -** This is a helper routine to sqlite3StrAccumAppend() that does special-case +** This is a helper routine to sqlite3_str_append() that does special-case ** work (enlarging the buffer) using tail recursion, so that the -** sqlite3StrAccumAppend() routine can use fast calling semantics. +** sqlite3_str_append() routine can use fast calling semantics. */ static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ N = sqlite3StrAccumEnlarge(p, N); @@ -25265,21 +32390,20 @@ static void SQLITE_NOINLINE enlargeAndAppend(StrAccum *p, const char *z, int N){ memcpy(&p->zText[p->nChar], z, N); p->nChar += N; } - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); } /* ** Append N bytes of text from z to the StrAccum object. Increase the ** size of the memory allocation for StrAccum if necessary. */ -SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ +SQLITE_API void sqlite3_str_append(sqlite3_str *p, const char *z, int N){ assert( z!=0 || N==0 ); assert( p->zText!=0 || p->nChar==0 || p->accError ); assert( N>=0 ); - assert( p->accError==0 || p->nAlloc==0 ); + assert( p->accError==0 || p->nAlloc==0 || p->mxAlloc==0 ); if( p->nChar+N >= p->nAlloc ){ enlargeAndAppend(p,z,N); - }else{ + }else if( N ){ assert( p->zText ); p->nChar += N; memcpy(&p->zText[p->nChar-N], z, N); @@ -25289,8 +32413,8 @@ SQLITE_PRIVATE void sqlite3StrAccumAppend(StrAccum *p, const char *z, int N){ /* ** Append the complete text of zero-terminated string z[] to the p string. */ -SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ - sqlite3StrAccumAppend(p, z, sqlite3Strlen30(z)); +SQLITE_API void sqlite3_str_appendall(sqlite3_str *p, const char *z){ + sqlite3_str_append(p, z, sqlite3Strlen30(z)); } @@ -25299,32 +32423,95 @@ SQLITE_PRIVATE void sqlite3StrAccumAppendAll(StrAccum *p, const char *z){ ** Return a pointer to the resulting string. Return a NULL ** pointer if any kind of error was encountered. */ +static SQLITE_NOINLINE char *strAccumFinishRealloc(StrAccum *p){ + char *zText; + assert( p->mxAlloc>0 && !isMalloced(p) ); + zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); + if( zText ){ + memcpy(zText, p->zText, p->nChar+1); + p->printfFlags |= SQLITE_PRINTF_MALLOCED; + }else{ + sqlite3StrAccumSetError(p, SQLITE_NOMEM); + } + p->zText = zText; + return zText; +} SQLITE_PRIVATE char *sqlite3StrAccumFinish(StrAccum *p){ if( p->zText ){ - assert( (p->zText==p->zBase)==!isMalloced(p) ); p->zText[p->nChar] = 0; if( p->mxAlloc>0 && !isMalloced(p) ){ - p->zText = sqlite3DbMallocRaw(p->db, p->nChar+1 ); - if( p->zText ){ - memcpy(p->zText, p->zBase, p->nChar+1); - p->printfFlags |= SQLITE_PRINTF_MALLOCED; - }else{ - setStrAccumError(p, STRACCUM_NOMEM); - } + return strAccumFinishRealloc(p); } } return p->zText; } +/* +** Use the content of the StrAccum passed as the second argument +** as the result of an SQL function. +*/ +SQLITE_PRIVATE void sqlite3ResultStrAccum(sqlite3_context *pCtx, StrAccum *p){ + if( p->accError ){ + sqlite3_result_error_code(pCtx, p->accError); + sqlite3_str_reset(p); + }else if( isMalloced(p) ){ + sqlite3_result_text(pCtx, p->zText, p->nChar, SQLITE_DYNAMIC); + }else{ + sqlite3_result_text(pCtx, "", 0, SQLITE_STATIC); + sqlite3_str_reset(p); + } +} + +/* +** This singleton is an sqlite3_str object that is returned if +** sqlite3_malloc() fails to provide space for a real one. This +** sqlite3_str object accepts no new text and always returns +** an SQLITE_NOMEM error. +*/ +static sqlite3_str sqlite3OomStr = { + 0, 0, 0, 0, 0, SQLITE_NOMEM, 0 +}; + +/* Finalize a string created using sqlite3_str_new(). +*/ +SQLITE_API char *sqlite3_str_finish(sqlite3_str *p){ + char *z; + if( p!=0 && p!=&sqlite3OomStr ){ + z = sqlite3StrAccumFinish(p); + sqlite3_free(p); + }else{ + z = 0; + } + return z; +} + +/* Return any error code associated with p */ +SQLITE_API int sqlite3_str_errcode(sqlite3_str *p){ + return p ? p->accError : SQLITE_NOMEM; +} + +/* Return the current length of p in bytes */ +SQLITE_API int sqlite3_str_length(sqlite3_str *p){ + return p ? p->nChar : 0; +} + +/* Return the current value for p */ +SQLITE_API char *sqlite3_str_value(sqlite3_str *p){ + if( p==0 || p->nChar==0 ) return 0; + p->zText[p->nChar] = 0; + return p->zText; +} + /* ** Reset an StrAccum string. Reclaim all malloced memory. */ -SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ - assert( (p->zText==0 || p->zText==p->zBase)==!isMalloced(p) ); +SQLITE_API void sqlite3_str_reset(StrAccum *p){ if( isMalloced(p) ){ sqlite3DbFree(p->db, p->zText); p->printfFlags &= ~SQLITE_PRINTF_MALLOCED; } + p->nAlloc = 0; + p->nChar = 0; p->zText = 0; } @@ -25343,15 +32530,27 @@ SQLITE_PRIVATE void sqlite3StrAccumReset(StrAccum *p){ ** allocations will ever occur. */ SQLITE_PRIVATE void sqlite3StrAccumInit(StrAccum *p, sqlite3 *db, char *zBase, int n, int mx){ - p->zText = p->zBase = zBase; + p->zText = zBase; p->db = db; - p->nChar = 0; p->nAlloc = n; p->mxAlloc = mx; + p->nChar = 0; p->accError = 0; p->printfFlags = 0; } +/* Allocate and initialize a new dynamic string object */ +SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3 *db){ + sqlite3_str *p = sqlite3_malloc64(sizeof(*p)); + if( p ){ + sqlite3StrAccumInit(p, 0, 0, 0, + db ? db->aLimit[SQLITE_LIMIT_LENGTH] : SQLITE_MAX_LENGTH); + }else{ + p = &sqlite3OomStr; + } + return p; +} + /* ** Print into memory obtained from sqliteMalloc(). Use the internal ** %-conversion extensions. @@ -25364,9 +32563,9 @@ SQLITE_PRIVATE char *sqlite3VMPrintf(sqlite3 *db, const char *zFormat, va_list a sqlite3StrAccumInit(&acc, db, zBase, sizeof(zBase), db->aLimit[SQLITE_LIMIT_LENGTH]); acc.printfFlags = SQLITE_PRINTF_INTERNAL; - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); - if( acc.accError==STRACCUM_NOMEM ){ + if( acc.accError==SQLITE_NOMEM ){ sqlite3OomFault(db); } return z; @@ -25389,12 +32588,12 @@ SQLITE_PRIVATE char *sqlite3MPrintf(sqlite3 *db, const char *zFormat, ...){ ** Print into memory obtained from sqlite3_malloc(). Omit the internal ** %-conversion extensions. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char *zFormat, va_list ap){ +SQLITE_API char *sqlite3_vmprintf(const char *zFormat, va_list ap){ char *z; char zBase[SQLITE_PRINT_BUF_SIZE]; StrAccum acc; -#ifdef SQLITE_ENABLE_API_ARMOR +#ifdef SQLITE_ENABLE_API_ARMOR if( zFormat==0 ){ (void)SQLITE_MISUSE_BKPT; return 0; @@ -25404,7 +32603,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char *zFormat, va_list ap if( sqlite3_initialize() ) return 0; #endif sqlite3StrAccumInit(&acc, 0, zBase, sizeof(zBase), SQLITE_MAX_LENGTH); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); z = sqlite3StrAccumFinish(&acc); return z; } @@ -25413,7 +32612,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vmprintf(const char *zFormat, va_list ap ** Print into memory obtained from sqlite3_malloc()(). Omit the internal ** %-conversion extensions. */ -SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char *zFormat, ...){ +SQLITE_API char *sqlite3_mprintf(const char *zFormat, ...){ va_list ap; char *z; #ifndef SQLITE_OMIT_AUTOINIT @@ -25438,7 +32637,7 @@ SQLITE_API char *SQLITE_CDECL sqlite3_mprintf(const char *zFormat, ...){ ** ** sqlite3_vsnprintf() is the varargs version. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ +SQLITE_API char *sqlite3_vsnprintf(int n, char *zBuf, const char *zFormat, va_list ap){ StrAccum acc; if( n<=0 ) return zBuf; #ifdef SQLITE_ENABLE_API_ARMOR @@ -25449,16 +32648,27 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_vsnprintf(int n, char *zBuf, const char } #endif sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); - sqlite3VXPrintf(&acc, zFormat, ap); - return sqlite3StrAccumFinish(&acc); + sqlite3_str_vappendf(&acc, zFormat, ap); + zBuf[acc.nChar] = 0; + return zBuf; } -SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ - char *z; +SQLITE_API char *sqlite3_snprintf(int n, char *zBuf, const char *zFormat, ...){ + StrAccum acc; va_list ap; + if( n<=0 ) return zBuf; +#ifdef SQLITE_ENABLE_API_ARMOR + if( zBuf==0 || zFormat==0 ) { + (void)SQLITE_MISUSE_BKPT; + if( zBuf ) zBuf[0] = 0; + return zBuf; + } +#endif + sqlite3StrAccumInit(&acc, 0, zBuf, n, 0); va_start(ap,zFormat); - z = sqlite3_vsnprintf(n, zBuf, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); - return z; + zBuf[acc.nChar] = 0; + return zBuf; } /* @@ -25470,7 +32680,7 @@ SQLITE_API char *SQLITE_CDECL sqlite3_snprintf(int n, char *zBuf, const char *zF ** allocate memory because it might be called while the memory allocator ** mutex is held. ** -** sqlite3VXPrintf() might ask for *temporary* memory allocations for +** sqlite3_str_vappendf() might ask for *temporary* memory allocations for ** certain format characters (%q) or for very large precisions or widths. ** Care must be taken that any sqlite3_log() calls that occur while the ** memory mutex is held do not use these mechanisms. @@ -25480,7 +32690,7 @@ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ char zMsg[SQLITE_PRINT_BUF_SIZE*3]; /* Complete log message */ sqlite3StrAccumInit(&acc, 0, zMsg, sizeof(zMsg), 0); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); sqlite3GlobalConfig.xLog(sqlite3GlobalConfig.pLogArg, iErrCode, sqlite3StrAccumFinish(&acc)); } @@ -25488,7 +32698,7 @@ static void renderLogMsg(int iErrCode, const char *zFormat, va_list ap){ /* ** Format and write a message to the log if logging is enabled. */ -SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...){ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...){ va_list ap; /* Vararg list */ if( sqlite3GlobalConfig.xLog ){ va_start(ap, zFormat); @@ -25506,29 +32716,105 @@ SQLITE_API void SQLITE_CDECL sqlite3_log(int iErrCode, const char *zFormat, ...) SQLITE_PRIVATE void sqlite3DebugPrintf(const char *zFormat, ...){ va_list ap; StrAccum acc; - char zBuf[500]; + char zBuf[SQLITE_PRINT_BUF_SIZE*10]; sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); va_start(ap,zFormat); - sqlite3VXPrintf(&acc, zFormat, ap); + sqlite3_str_vappendf(&acc, zFormat, ap); va_end(ap); sqlite3StrAccumFinish(&acc); +#ifdef SQLITE_OS_TRACE_PROC + { + extern void SQLITE_OS_TRACE_PROC(const char *zBuf, int nBuf); + SQLITE_OS_TRACE_PROC(zBuf, sizeof(zBuf)); + } +#else fprintf(stdout,"%s", zBuf); fflush(stdout); +#endif } #endif /* -** variable-argument wrapper around sqlite3VXPrintf(). The bFlags argument +** variable-argument wrapper around sqlite3_str_vappendf(). The bFlags argument ** can contain the bit SQLITE_PRINTF_INTERNAL enable internal formats. */ -SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){ +SQLITE_API void sqlite3_str_appendf(StrAccum *p, const char *zFormat, ...){ va_list ap; va_start(ap,zFormat); - sqlite3VXPrintf(p, zFormat, ap); + sqlite3_str_vappendf(p, zFormat, ap); va_end(ap); } + +/***************************************************************************** +** Reference counted string/blob storage +*****************************************************************************/ + +/* +** Increase the reference count of the string by one. +** +** The input parameter is returned. +*/ +SQLITE_PRIVATE char *sqlite3RCStrRef(char *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + p->nRCRef++; + return z; +} + +/* +** Decrease the reference count by one. Free the string when the +** reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3RCStrUnref(void *z){ + RCStr *p = (RCStr*)z; + assert( p!=0 ); + p--; + assert( p->nRCRef>0 ); + if( p->nRCRef>=2 ){ + p->nRCRef--; + }else{ + sqlite3_free(p); + } +} + +/* +** Create a new string that is capable of holding N bytes of text, not counting +** the zero byte at the end. The string is uninitialized. +** +** The reference count is initially 1. Call sqlite3RCStrUnref() to free the +** newly allocated string. +** +** This routine returns 0 on an OOM. +*/ +SQLITE_PRIVATE char *sqlite3RCStrNew(u64 N){ + RCStr *p = sqlite3_malloc64( N + sizeof(*p) + 1 ); + if( p==0 ) return 0; + p->nRCRef = 1; + return (char*)&p[1]; +} + +/* +** Change the size of the string so that it is able to hold N bytes. +** The string might be reallocated, so return the new allocation. +*/ +SQLITE_PRIVATE char *sqlite3RCStrResize(char *z, u64 N){ + RCStr *p = (RCStr*)z; + RCStr *pNew; + assert( p!=0 ); + p--; + assert( p->nRCRef==1 ); + pNew = sqlite3_realloc64(p, N+sizeof(RCStr)+1); + if( pNew==0 ){ + sqlite3_free(p); + return 0; + }else{ + return (char*)&pNew[1]; + } +} + /************** End of printf.c **********************************************/ /************** Begin file treeview.c ****************************************/ /* @@ -25545,7 +32831,7 @@ SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){ ** ** This file contains C code to implement the TreeView debugging routines. ** These routines print a parse tree to standard output for debugging and -** analysis. +** analysis. ** ** The interfaces in this file is only available when compiling ** with SQLITE_DEBUG. @@ -25557,48 +32843,55 @@ SQLITE_PRIVATE void sqlite3XPrintf(StrAccum *p, const char *zFormat, ...){ ** Add a new subitem to the tree. The moreToFollow flag indicates that this ** is not the last item in the tree. */ -static TreeView *sqlite3TreeViewPush(TreeView *p, u8 moreToFollow){ +static void sqlite3TreeViewPush(TreeView **pp, u8 moreToFollow){ + TreeView *p = *pp; if( p==0 ){ - p = sqlite3_malloc64( sizeof(*p) ); - if( p==0 ) return 0; + *pp = p = sqlite3_malloc64( sizeof(*p) ); + if( p==0 ) return; memset(p, 0, sizeof(*p)); }else{ p->iLevel++; } assert( moreToFollow==0 || moreToFollow==1 ); - if( p->iLevelbLine) ) p->bLine[p->iLevel] = moreToFollow; - return p; + if( p->iLevel<(int)sizeof(p->bLine) ) p->bLine[p->iLevel] = moreToFollow; } /* ** Finished with one layer of the tree */ -static void sqlite3TreeViewPop(TreeView *p){ +static void sqlite3TreeViewPop(TreeView **pp){ + TreeView *p = *pp; if( p==0 ) return; p->iLevel--; - if( p->iLevel<0 ) sqlite3_free(p); + if( p->iLevel<0 ){ + sqlite3_free(p); + *pp = 0; + } } /* ** Generate a single line of output for the tree, with a prefix that contains ** all the appropriate tree lines */ -static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ +SQLITE_PRIVATE void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ va_list ap; int i; StrAccum acc; - char zBuf[500]; + char zBuf[1000]; sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); if( p ){ - for(i=0; iiLevel && ibLine)-1; i++){ - sqlite3StrAccumAppend(&acc, p->bLine[i] ? "| " : " ", 4); + for(i=0; iiLevel && i<(int)sizeof(p->bLine)-1; i++){ + sqlite3_str_append(&acc, p->bLine[i] ? "| " : " ", 4); } - sqlite3StrAccumAppend(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); + sqlite3_str_append(&acc, p->bLine[i] ? "|-- " : "'-- ", 4); + } + if( zFormat!=0 ){ + va_start(ap, zFormat); + sqlite3_str_vappendf(&acc, zFormat, ap); + va_end(ap); + assert( acc.nChar>0 || acc.accError ); + sqlite3_str_append(&acc, "\n", 1); } - va_start(ap, zFormat); - sqlite3VXPrintf(&acc, zFormat, ap); - va_end(ap); - if( zBuf[acc.nChar-1]!='\n' ) sqlite3StrAccumAppend(&acc, "\n", 1); sqlite3StrAccumFinish(&acc); fprintf(stdout,"%s", zBuf); fflush(stdout); @@ -25608,10 +32901,57 @@ static void sqlite3TreeViewLine(TreeView *p, const char *zFormat, ...){ ** Shorthand for starting a new tree item that consists of a single label */ static void sqlite3TreeViewItem(TreeView *p, const char *zLabel,u8 moreFollows){ - p = sqlite3TreeViewPush(p, moreFollows); + sqlite3TreeViewPush(&p, moreFollows); sqlite3TreeViewLine(p, "%s", zLabel); } +/* +** Show a list of Column objects in tree format. +*/ +SQLITE_PRIVATE void sqlite3TreeViewColumnList( + TreeView *pView, + const Column *aCol, + int nCol, + u8 moreToFollow +){ + int i; + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewLine(pView, "COLUMNS"); + for(i=0; inCte>0 ){ - pView = sqlite3TreeViewPush(pView, 1); + sqlite3TreeViewPush(&pView, moreToFollow); for(i=0; inCte; i++){ StrAccum x; char zLine[1000]; const struct Cte *pCte = &pWith->a[i]; sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, "%s", pCte->zName); + sqlite3_str_appendf(&x, "%s", pCte->zName); if( pCte->pCols && pCte->pCols->nExpr>0 ){ char cSep = '('; int j; for(j=0; jpCols->nExpr; j++){ - sqlite3XPrintf(&x, "%c%s", cSep, pCte->pCols->a[j].zName); + sqlite3_str_appendf(&x, "%c%s", cSep, pCte->pCols->a[j].zEName); cSep = ','; } - sqlite3XPrintf(&x, ")"); + sqlite3_str_appendf(&x, ")"); + } + if( pCte->eM10d!=M10d_Any ){ + sqlite3_str_appendf(&x, " %sMATERIALIZED", + pCte->eM10d==M10d_No ? "NOT " : ""); + } + if( pCte->pUse ){ + sqlite3_str_appendf(&x, " (pUse=0x%p, nUse=%d)", pCte->pUse, + pCte->pUse->nUse); } - sqlite3XPrintf(&x, " AS"); sqlite3StrAccumFinish(&x); sqlite3TreeViewItem(pView, zLine, inCte-1); sqlite3TreeViewSelect(pView, pCte->pSelect, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } } +/* +** Generate a human-readable description of a SrcList object. +*/ +SQLITE_PRIVATE void sqlite3TreeViewSrcList(TreeView *pView, const SrcList *pSrc){ + int i; + if( pSrc==0 ) return; + for(i=0; inSrc; i++){ + const SrcItem *pItem = &pSrc->a[i]; + StrAccum x; + int n = 0; + char zLine[1000]; + sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); + x.printfFlags |= SQLITE_PRINTF_INTERNAL; + sqlite3_str_appendf(&x, "{%d:*} %!S", pItem->iCursor, pItem); + if( pItem->pSTab ){ + sqlite3_str_appendf(&x, " tab=%Q nCol=%d ptr=%p used=%llx%s", + pItem->pSTab->zName, pItem->pSTab->nCol, pItem->pSTab, + pItem->colUsed, + pItem->fg.rowidUsed ? "+rowid" : ""); + } + if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))==(JT_LEFT|JT_RIGHT) ){ + sqlite3_str_appendf(&x, " FULL-OUTER-JOIN"); + }else if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&x, " LEFT-JOIN"); + }else if( pItem->fg.jointype & JT_RIGHT ){ + sqlite3_str_appendf(&x, " RIGHT-JOIN"); + }else if( pItem->fg.jointype & JT_CROSS ){ + sqlite3_str_appendf(&x, " CROSS-JOIN"); + } + if( pItem->fg.jointype & JT_LTORJ ){ + sqlite3_str_appendf(&x, " LTORJ"); + } + if( pItem->fg.fromDDL ){ + sqlite3_str_appendf(&x, " DDL"); + } + if( pItem->fg.isCte ){ + sqlite3_str_appendf(&x, " CteUse=0x%p", pItem->u2.pCteUse); + } + if( pItem->fg.isOn || (pItem->fg.isUsing==0 && pItem->u3.pOn!=0) ){ + sqlite3_str_appendf(&x, " ON"); + } + if( pItem->fg.isTabFunc ) sqlite3_str_appendf(&x, " isTabFunc"); + if( pItem->fg.isCorrelated ) sqlite3_str_appendf(&x, " isCorrelated"); + if( pItem->fg.isMaterialized ) sqlite3_str_appendf(&x, " isMaterialized"); + if( pItem->fg.viaCoroutine ) sqlite3_str_appendf(&x, " viaCoroutine"); + if( pItem->fg.notCte ) sqlite3_str_appendf(&x, " notCte"); + if( pItem->fg.isNestedFrom ) sqlite3_str_appendf(&x, " isNestedFrom"); + if( pItem->fg.fixedSchema ) sqlite3_str_appendf(&x, " fixedSchema"); + if( pItem->fg.hadSchema ) sqlite3_str_appendf(&x, " hadSchema"); + if( pItem->fg.isSubquery ) sqlite3_str_appendf(&x, " isSubquery"); + + sqlite3StrAccumFinish(&x); + sqlite3TreeViewItem(pView, zLine, inSrc-1); + n = 0; + if( pItem->fg.isSubquery ) n++; + if( pItem->fg.isTabFunc ) n++; + if( pItem->fg.isUsing ) n++; + if( pItem->fg.isUsing ){ + sqlite3TreeViewIdList(pView, pItem->u3.pUsing, (--n)>0, "USING"); + } + if( pItem->fg.isSubquery ){ + assert( n==1 ); + if( pItem->pSTab ){ + Table *pTab = pItem->pSTab; + sqlite3TreeViewColumnList(pView, pTab->aCol, pTab->nCol, 1); + } + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "SUBQUERY"); + sqlite3TreeViewPop(&pView); + sqlite3TreeViewSelect(pView, pItem->u4.pSubq->pSelect, 0); + } + if( pItem->fg.isTabFunc ){ + sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); + } + sqlite3TreeViewPop(&pView); + } +} /* -** Generate a human-readable description of a the Select object. +** Generate a human-readable description of a Select object. */ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 moreToFollow){ int n = 0; int cnt = 0; - pView = sqlite3TreeViewPush(pView, moreToFollow); + if( p==0 ){ + sqlite3TreeViewLine(pView, "nil-SELECT"); + return; + } + sqlite3TreeViewPush(&pView, moreToFollow); if( p->pWith ){ sqlite3TreeViewWith(pView, p->pWith, 1); cnt = 1; - sqlite3TreeViewPush(pView, 1); + sqlite3TreeViewPush(&pView, 1); } do{ - sqlite3TreeViewLine(pView, "SELECT%s%s (0x%p) selFlags=0x%x nSelectRow=%d", - ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), - ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), p, p->selFlags, - (int)p->nSelectRow - ); - if( cnt++ ) sqlite3TreeViewPop(pView); + if( p->selFlags & SF_WhereBegin ){ + sqlite3TreeViewLine(pView, "sqlite3WhereBegin()"); + }else{ + sqlite3TreeViewLine(pView, + "SELECT%s%s (%u/%p) selFlags=0x%x nSelectRow=%d", + ((p->selFlags & SF_Distinct) ? " DISTINCT" : ""), + ((p->selFlags & SF_Aggregate) ? " agg_flag" : ""), + p->selId, p, p->selFlags, + (int)p->nSelectRow + ); + } + if( cnt++ ) sqlite3TreeViewPop(&pView); if( p->pPrior ){ n = 1000; }else{ n = 0; - if( p->pSrc && p->pSrc->nSrc ) n++; + if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ) n++; if( p->pWhere ) n++; if( p->pGroupBy ) n++; if( p->pHaving ) n++; if( p->pOrderBy ) n++; if( p->pLimit ) n++; - if( p->pOffset ) n++; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) n++; + if( p->pWinDefn ) n++; +#endif } - sqlite3TreeViewExprList(pView, p->pEList, (n--)>0, "result-set"); - if( p->pSrc && p->pSrc->nSrc ){ - int i; - pView = sqlite3TreeViewPush(pView, (n--)>0); - sqlite3TreeViewLine(pView, "FROM"); - for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - StrAccum x; - char zLine[100]; - sqlite3StrAccumInit(&x, 0, zLine, sizeof(zLine), 0); - sqlite3XPrintf(&x, "{%d,*}", pItem->iCursor); - if( pItem->zDatabase ){ - sqlite3XPrintf(&x, " %s.%s", pItem->zDatabase, pItem->zName); - }else if( pItem->zName ){ - sqlite3XPrintf(&x, " %s", pItem->zName); - } - if( pItem->pTab ){ - sqlite3XPrintf(&x, " tabname=%Q", pItem->pTab->zName); - } - if( pItem->zAlias ){ - sqlite3XPrintf(&x, " (AS %s)", pItem->zAlias); - } - if( pItem->fg.jointype & JT_LEFT ){ - sqlite3XPrintf(&x, " LEFT-JOIN"); - } - sqlite3StrAccumFinish(&x); - sqlite3TreeViewItem(pView, zLine, ipSrc->nSrc-1); - if( pItem->pSelect ){ - sqlite3TreeViewSelect(pView, pItem->pSelect, 0); - } - if( pItem->fg.isTabFunc ){ - sqlite3TreeViewExprList(pView, pItem->u1.pFuncArg, 0, "func-args:"); - } - sqlite3TreeViewPop(pView); + if( p->pEList ){ + sqlite3TreeViewExprList(pView, p->pEList, n>0, "result-set"); + } + n--; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + Window *pX; + sqlite3TreeViewPush(&pView, (n--)>0); + sqlite3TreeViewLine(pView, "window-functions"); + for(pX=p->pWin; pX; pX=pX->pNextWin){ + sqlite3TreeViewWinFunc(pView, pX, pX->pNextWin!=0); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); + } +#endif + if( p->pSrc && p->pSrc->nSrc && p->pSrc->nAlloc ){ + sqlite3TreeViewPush(&pView, (n--)>0); + sqlite3TreeViewLine(pView, "FROM"); + sqlite3TreeViewSrcList(pView, p->pSrc); + sqlite3TreeViewPop(&pView); } if( p->pWhere ){ sqlite3TreeViewItem(pView, "WHERE", (n--)>0); sqlite3TreeViewExpr(pView, p->pWhere, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } if( p->pGroupBy ){ sqlite3TreeViewExprList(pView, p->pGroupBy, (n--)>0, "GROUPBY"); @@ -25731,20 +33153,30 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m if( p->pHaving ){ sqlite3TreeViewItem(pView, "HAVING", (n--)>0); sqlite3TreeViewExpr(pView, p->pHaving, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWinDefn ){ + Window *pX; + sqlite3TreeViewItem(pView, "WINDOW", (n--)>0); + for(pX=p->pWinDefn; pX; pX=pX->pNextWin){ + sqlite3TreeViewWindow(pView, pX, pX->pNextWin!=0); + } + sqlite3TreeViewPop(&pView); + } +#endif if( p->pOrderBy ){ sqlite3TreeViewExprList(pView, p->pOrderBy, (n--)>0, "ORDERBY"); } if( p->pLimit ){ sqlite3TreeViewItem(pView, "LIMIT", (n--)>0); - sqlite3TreeViewExpr(pView, p->pLimit, 0); - sqlite3TreeViewPop(pView); - } - if( p->pOffset ){ - sqlite3TreeViewItem(pView, "OFFSET", (n--)>0); - sqlite3TreeViewExpr(pView, p->pOffset, 0); - sqlite3TreeViewPop(pView); + sqlite3TreeViewExpr(pView, p->pLimit->pLeft, p->pLimit->pRight!=0); + if( p->pLimit->pRight ){ + sqlite3TreeViewItem(pView, "OFFSET", 0); + sqlite3TreeViewExpr(pView, p->pLimit->pRight, 0); + sqlite3TreeViewPop(&pView); + } + sqlite3TreeViewPop(&pView); } if( p->pPrior ){ const char *zOp = "UNION"; @@ -25757,24 +33189,161 @@ SQLITE_PRIVATE void sqlite3TreeViewSelect(TreeView *pView, const Select *p, u8 m } p = p->pPrior; }while( p!=0 ); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a description of starting or stopping bounds +*/ +SQLITE_PRIVATE void sqlite3TreeViewBound( + TreeView *pView, /* View context */ + u8 eBound, /* UNBOUNDED, CURRENT, PRECEDING, FOLLOWING */ + Expr *pExpr, /* Value for PRECEDING or FOLLOWING */ + u8 moreToFollow /* True if more to follow */ +){ + switch( eBound ){ + case TK_UNBOUNDED: { + sqlite3TreeViewItem(pView, "UNBOUNDED", moreToFollow); + sqlite3TreeViewPop(&pView); + break; + } + case TK_CURRENT: { + sqlite3TreeViewItem(pView, "CURRENT", moreToFollow); + sqlite3TreeViewPop(&pView); + break; + } + case TK_PRECEDING: { + sqlite3TreeViewItem(pView, "PRECEDING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(&pView); + break; + } + case TK_FOLLOWING: { + sqlite3TreeViewItem(pView, "FOLLOWING", moreToFollow); + sqlite3TreeViewExpr(pView, pExpr, 0); + sqlite3TreeViewPop(&pView); + break; + } + } +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWindow(TreeView *pView, const Window *pWin, u8 more){ + int nElement = 0; + if( pWin==0 ) return; + if( pWin->pFilter ){ + sqlite3TreeViewItem(pView, "FILTER", 1); + sqlite3TreeViewExpr(pView, pWin->pFilter, 0); + sqlite3TreeViewPop(&pView); + if( pWin->eFrmType==TK_FILTER ) return; + } + sqlite3TreeViewPush(&pView, more); + if( pWin->zName ){ + sqlite3TreeViewLine(pView, "OVER %s (%p)", pWin->zName, pWin); + }else{ + sqlite3TreeViewLine(pView, "OVER (%p)", pWin); + } + if( pWin->zBase ) nElement++; + if( pWin->pOrderBy ) nElement++; + if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ) nElement++; + if( pWin->eExclude ) nElement++; + if( pWin->zBase ){ + sqlite3TreeViewPush(&pView, (--nElement)>0); + sqlite3TreeViewLine(pView, "window: %s", pWin->zBase); + sqlite3TreeViewPop(&pView); + } + if( pWin->pPartition ){ + sqlite3TreeViewExprList(pView, pWin->pPartition, nElement>0,"PARTITION-BY"); + } + if( pWin->pOrderBy ){ + sqlite3TreeViewExprList(pView, pWin->pOrderBy, (--nElement)>0, "ORDER-BY"); + } + if( pWin->eFrmType!=0 && pWin->eFrmType!=TK_FILTER ){ + char zBuf[30]; + const char *zFrmType = "ROWS"; + if( pWin->eFrmType==TK_RANGE ) zFrmType = "RANGE"; + if( pWin->eFrmType==TK_GROUPS ) zFrmType = "GROUPS"; + sqlite3_snprintf(sizeof(zBuf),zBuf,"%s%s",zFrmType, + pWin->bImplicitFrame ? " (implied)" : ""); + sqlite3TreeViewItem(pView, zBuf, (--nElement)>0); + sqlite3TreeViewBound(pView, pWin->eStart, pWin->pStart, 1); + sqlite3TreeViewBound(pView, pWin->eEnd, pWin->pEnd, 0); + sqlite3TreeViewPop(&pView); + } + if( pWin->eExclude ){ + char zBuf[30]; + const char *zExclude; + switch( pWin->eExclude ){ + case TK_NO: zExclude = "NO OTHERS"; break; + case TK_CURRENT: zExclude = "CURRENT ROW"; break; + case TK_GROUP: zExclude = "GROUP"; break; + case TK_TIES: zExclude = "TIES"; break; + default: + sqlite3_snprintf(sizeof(zBuf),zBuf,"invalid(%d)", pWin->eExclude); + zExclude = zBuf; + break; + } + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "EXCLUDE %s", zExclude); + sqlite3TreeViewPop(&pView); + } + sqlite3TreeViewPop(&pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Generate a human-readable explanation for a Window Function object +*/ +SQLITE_PRIVATE void sqlite3TreeViewWinFunc(TreeView *pView, const Window *pWin, u8 more){ + if( pWin==0 ) return; + sqlite3TreeViewPush(&pView, more); + sqlite3TreeViewLine(pView, "WINFUNC %s(%d)", + pWin->pWFunc->zName, pWin->pWFunc->nArg); + sqlite3TreeViewWindow(pView, pWin, 0); + sqlite3TreeViewPop(&pView); +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** Generate a human-readable explanation of an expression tree. */ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 moreToFollow){ const char *zBinOp = 0; /* Binary operator */ const char *zUniOp = 0; /* Unary operator */ - char zFlgs[30]; - pView = sqlite3TreeViewPush(pView, moreToFollow); + char zFlgs[200]; + sqlite3TreeViewPush(&pView, moreToFollow); if( pExpr==0 ){ sqlite3TreeViewLine(pView, "nil"); - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); return; } - if( pExpr->flags ){ - sqlite3_snprintf(sizeof(zFlgs),zFlgs," flags=0x%x",pExpr->flags); + if( pExpr->flags || pExpr->affExpr || pExpr->vvaFlags || pExpr->pAggInfo ){ + StrAccum x; + sqlite3StrAccumInit(&x, 0, zFlgs, sizeof(zFlgs), 0); + sqlite3_str_appendf(&x, " fg.af=%x.%c", + pExpr->flags, pExpr->affExpr ? pExpr->affExpr : 'n'); + if( ExprHasProperty(pExpr, EP_OuterON) ){ + sqlite3_str_appendf(&x, " outer.iJoin=%d", pExpr->w.iJoin); + } + if( ExprHasProperty(pExpr, EP_InnerON) ){ + sqlite3_str_appendf(&x, " inner.iJoin=%d", pExpr->w.iJoin); + } + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + sqlite3_str_appendf(&x, " DDL"); + } + if( ExprHasVVAProperty(pExpr, EP_Immutable) ){ + sqlite3_str_appendf(&x, " IMMUTABLE"); + } + if( pExpr->pAggInfo!=0 ){ + sqlite3_str_appendf(&x, " agg-column[%d]", pExpr->iAgg); + } + sqlite3StrAccumFinish(&x); }else{ zFlgs[0] = 0; } @@ -25787,10 +33356,22 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_COLUMN: { if( pExpr->iTable<0 ){ /* This only happens when coding check constraints */ - sqlite3TreeViewLine(pView, "COLUMN(%d)%s", pExpr->iColumn, zFlgs); + char zOp2[16]; + if( pExpr->op2 ){ + sqlite3_snprintf(sizeof(zOp2),zOp2," op2=0x%02x",pExpr->op2); + }else{ + zOp2[0] = 0; + } + sqlite3TreeViewLine(pView, "COLUMN(%d)%s%s", + pExpr->iColumn, zFlgs, zOp2); }else{ - sqlite3TreeViewLine(pView, "{%d:%d}%s", - pExpr->iTable, pExpr->iColumn, zFlgs); + assert( ExprUseYTab(pExpr) ); + sqlite3TreeViewLine(pView, "{%d:%d} pTab=%p%s", + pExpr->iTable, pExpr->iColumn, + pExpr->y.pTab, zFlgs); + } + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } break; } @@ -25804,11 +33385,13 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); break; } #endif case TK_STRING: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView,"%Q", pExpr->u.zToken); break; } @@ -25816,13 +33399,20 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewLine(pView,"NULL"); break; } + case TK_TRUEFALSE: { + sqlite3TreeViewLine(pView,"%s%s", + sqlite3ExprTruthValue(pExpr) ? "TRUE" : "FALSE", zFlgs); + break; + } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView,"%s", pExpr->u.zToken); break; } #endif case TK_VARIABLE: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView,"VARIABLE(%s,%d)", pExpr->u.zToken, pExpr->iColumn); break; @@ -25832,12 +33422,14 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m break; } case TK_ID: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView,"ID \"%w\"", pExpr->u.zToken); break; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView,"CAST %Q", pExpr->u.zToken); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; @@ -25864,6 +33456,7 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_RSHIFT: zBinOp = "RSHIFT"; break; case TK_CONCAT: zBinOp = "CONCAT"; break; case TK_DOT: zBinOp = "DOT"; break; + case TK_LIMIT: zBinOp = "LIMIT"; break; case TK_UMINUS: zUniOp = "UMINUS"; break; case TK_UPLUS: zUniOp = "UPLUS"; break; @@ -25872,14 +33465,37 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_ISNULL: zUniOp = "ISNULL"; break; case TK_NOTNULL: zUniOp = "NOTNULL"; break; + case TK_TRUTH: { + int x; + const char *azOp[] = { + "IS-FALSE", "IS-TRUE", "IS-NOT-FALSE", "IS-NOT-TRUE" + }; + assert( pExpr->op2==TK_IS || pExpr->op2==TK_ISNOT ); + assert( pExpr->pRight ); + assert( sqlite3ExprSkipCollateAndLikely(pExpr->pRight)->op + == TK_TRUEFALSE ); + x = (pExpr->op2==TK_ISNOT)*2 + sqlite3ExprTruthValue(pExpr->pRight); + zUniOp = azOp[x]; + break; + } + case TK_SPAN: { + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3TreeViewLine(pView, "SPAN %Q", pExpr->u.zToken); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } case TK_COLLATE: { - sqlite3TreeViewLine(pView, "COLLATE %Q", pExpr->u.zToken); + /* COLLATE operators without the EP_Collate flag are intended to + ** emulate collation associated with a table column. These show + ** up in the treeview output as "SOFT-COLLATE". Explicit COLLATE + ** operators that appear in the original SQL always have the + ** EP_Collate bit set and appear in treeview output as just "COLLATE" */ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3TreeViewLine(pView, "%sCOLLATE %Q%s", + !ExprHasProperty(pExpr, EP_Collate) ? "SOFT-" : "", + pExpr->u.zToken, zFlgs); sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } @@ -25887,37 +33503,86 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m case TK_AGG_FUNCTION: case TK_FUNCTION: { ExprList *pFarg; /* List of function arguments */ + Window *pWin; if( ExprHasProperty(pExpr, EP_TokenOnly) ){ pFarg = 0; + pWin = 0; }else{ + assert( ExprUseXList(pExpr) ); pFarg = pExpr->x.pList; +#ifndef SQLITE_OMIT_WINDOWFUNC + pWin = IsWindowFunc(pExpr) ? pExpr->y.pWin : 0; +#else + pWin = 0; +#endif } + assert( !ExprHasProperty(pExpr, EP_IntValue) ); if( pExpr->op==TK_AGG_FUNCTION ){ - sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q", - pExpr->op2, pExpr->u.zToken); + sqlite3TreeViewLine(pView, "AGG_FUNCTION%d %Q%s agg=%d[%d]/%p", + pExpr->op2, pExpr->u.zToken, zFlgs, + pExpr->pAggInfo ? pExpr->pAggInfo->selId : 0, + pExpr->iAgg, pExpr->pAggInfo); + }else if( pExpr->op2!=0 ){ + const char *zOp2; + char zBuf[8]; + sqlite3_snprintf(sizeof(zBuf),zBuf,"0x%02x",pExpr->op2); + zOp2 = zBuf; + if( pExpr->op2==NC_IsCheck ) zOp2 = "NC_IsCheck"; + if( pExpr->op2==NC_IdxExpr ) zOp2 = "NC_IdxExpr"; + if( pExpr->op2==NC_PartIdx ) zOp2 = "NC_PartIdx"; + if( pExpr->op2==NC_GenCol ) zOp2 = "NC_GenCol"; + sqlite3TreeViewLine(pView, "FUNCTION %Q%s op2=%s", + pExpr->u.zToken, zFlgs, zOp2); }else{ - sqlite3TreeViewLine(pView, "FUNCTION %Q", pExpr->u.zToken); + sqlite3TreeViewLine(pView, "FUNCTION %Q%s", pExpr->u.zToken, zFlgs); } if( pFarg ){ - sqlite3TreeViewExprList(pView, pFarg, 0, 0); + sqlite3TreeViewExprList(pView, pFarg, pWin!=0 || pExpr->pLeft, 0); + if( pExpr->pLeft ){ + Expr *pOB = pExpr->pLeft; + assert( pOB->op==TK_ORDER ); + assert( ExprUseXList(pOB) ); + sqlite3TreeViewExprList(pView, pOB->x.pList, pWin!=0, "ORDERBY"); + } } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + sqlite3TreeViewWindow(pView, pWin, 0); + } +#endif + break; + } + case TK_ORDER: { + sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, "ORDERBY"); break; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: { - sqlite3TreeViewLine(pView, "EXISTS-expr"); + assert( ExprUseXSelect(pExpr) ); + sqlite3TreeViewLine(pView, "EXISTS-expr flags=0x%x", pExpr->flags); sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); break; } case TK_SELECT: { - sqlite3TreeViewLine(pView, "SELECT-expr"); + assert( ExprUseXSelect(pExpr) ); + sqlite3TreeViewLine(pView, "subquery-expr flags=0x%x", pExpr->flags); sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); break; } case TK_IN: { - sqlite3TreeViewLine(pView, "IN"); + sqlite3_str *pStr = sqlite3_str_new(0); + char *z; + sqlite3_str_appendf(pStr, "IN flags=0x%x", pExpr->flags); + if( pExpr->iTable ) sqlite3_str_appendf(pStr, " iTable=%d",pExpr->iTable); + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + sqlite3_str_appendf(pStr, " subrtn(%d,%d)", + pExpr->y.sub.regReturn, pExpr->y.sub.iAddr); + } + z = sqlite3_str_finish(pStr); + sqlite3TreeViewLine(pView, z); + sqlite3_free(z); sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( ExprUseXSelect(pExpr) ){ sqlite3TreeViewSelect(pView, pExpr->x.pSelect, 0); }else{ sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); @@ -25938,10 +33603,13 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m ** Z is stored in pExpr->pList->a[1].pExpr. */ case TK_BETWEEN: { - Expr *pX = pExpr->pLeft; - Expr *pY = pExpr->x.pList->a[0].pExpr; - Expr *pZ = pExpr->x.pList->a[1].pExpr; - sqlite3TreeViewLine(pView, "BETWEEN"); + const Expr *pX, *pY, *pZ; + pX = pExpr->pLeft; + assert( ExprUseXList(pExpr) ); + assert( pExpr->x.pList->nExpr==2 ); + pY = pExpr->x.pList->a[0].pExpr; + pZ = pExpr->x.pList->a[1].pExpr; + sqlite3TreeViewLine(pView, "BETWEEN%s", zFlgs); sqlite3TreeViewExpr(pView, pX, 1); sqlite3TreeViewExpr(pView, pY, 1); sqlite3TreeViewExpr(pView, pZ, 0); @@ -25955,29 +33623,75 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m ** is set to the column of the pseudo-table to read, or to -1 to ** read the rowid field. */ - sqlite3TreeViewLine(pView, "%s(%d)", + sqlite3TreeViewLine(pView, "%s(%d)", pExpr->iTable ? "NEW" : "OLD", pExpr->iColumn); break; } case TK_CASE: { sqlite3TreeViewLine(pView, "CASE"); sqlite3TreeViewExpr(pView, pExpr->pLeft, 1); + assert( ExprUseXList(pExpr) ); sqlite3TreeViewExprList(pView, pExpr->x.pList, 0, 0); break; } #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { const char *zType = "unk"; - switch( pExpr->affinity ){ + switch( pExpr->affExpr ){ case OE_Rollback: zType = "rollback"; break; case OE_Abort: zType = "abort"; break; case OE_Fail: zType = "fail"; break; case OE_Ignore: zType = "ignore"; break; } - sqlite3TreeViewLine(pView, "RAISE %s(%Q)", zType, pExpr->u.zToken); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3TreeViewLine(pView, "RAISE %s", zType); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); break; } #endif + case TK_MATCH: { + sqlite3TreeViewLine(pView, "MATCH {%d:%d}%s", + pExpr->iTable, pExpr->iColumn, zFlgs); + sqlite3TreeViewExpr(pView, pExpr->pRight, 0); + break; + } + case TK_VECTOR: { + char *z = sqlite3_mprintf("VECTOR%s",zFlgs); + assert( ExprUseXList(pExpr) ); + sqlite3TreeViewBareExprList(pView, pExpr->x.pList, z); + sqlite3_free(z); + break; + } + case TK_SELECT_COLUMN: { + sqlite3TreeViewLine(pView, "SELECT-COLUMN %d of [0..%d]%s", + pExpr->iColumn, pExpr->iTable-1, + pExpr->pRight==pExpr->pLeft ? " (SELECT-owner)" : ""); + assert( ExprUseXSelect(pExpr->pLeft) ); + sqlite3TreeViewSelect(pView, pExpr->pLeft->x.pSelect, 0); + break; + } + case TK_IF_NULL_ROW: { + sqlite3TreeViewLine(pView, "IF-NULL-ROW %d", pExpr->iTable); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + break; + } + case TK_ERROR: { + Expr tmp; + sqlite3TreeViewLine(pView, "ERROR"); + tmp = *pExpr; + tmp.op = pExpr->op2; + sqlite3TreeViewExpr(pView, &tmp, 0); + break; + } + case TK_ROW: { + if( pExpr->iColumn<=0 ){ + sqlite3TreeViewLine(pView, "First FROM table rowid"); + }else{ + sqlite3TreeViewLine(pView, "First FROM table column %d", + pExpr->iColumn-1); + } + break; + } default: { sqlite3TreeViewLine(pView, "op=%d", pExpr->op); break; @@ -25989,39 +33703,446 @@ SQLITE_PRIVATE void sqlite3TreeViewExpr(TreeView *pView, const Expr *pExpr, u8 m sqlite3TreeViewExpr(pView, pExpr->pRight, 0); }else if( zUniOp ){ sqlite3TreeViewLine(pView, "%s%s", zUniOp, zFlgs); - sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); + sqlite3TreeViewExpr(pView, pExpr->pLeft, 0); } - sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } + /* ** Generate a human-readable explanation of an expression list. */ +SQLITE_PRIVATE void sqlite3TreeViewBareExprList( + TreeView *pView, + const ExprList *pList, + const char *zLabel +){ + if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; + if( pList==0 ){ + sqlite3TreeViewLine(pView, "%s (empty)", zLabel); + }else{ + int i; + sqlite3TreeViewLine(pView, "%s", zLabel); + for(i=0; inExpr; i++){ + int j = pList->a[i].u.x.iOrderByCol; + u8 sortFlags = pList->a[i].fg.sortFlags; + char *zName = pList->a[i].zEName; + int moreToFollow = inExpr - 1; + if( j || zName || sortFlags ){ + sqlite3TreeViewPush(&pView, moreToFollow); + moreToFollow = 0; + sqlite3TreeViewLine(pView, 0); + if( zName ){ + switch( pList->a[i].fg.eEName ){ + default: + fprintf(stdout, "AS %s ", zName); + break; + case ENAME_TAB: + fprintf(stdout, "TABLE-ALIAS-NAME(\"%s\") ", zName); + if( pList->a[i].fg.bUsed ) fprintf(stdout, "(used) "); + if( pList->a[i].fg.bUsingTerm ) fprintf(stdout, "(USING-term) "); + if( pList->a[i].fg.bNoExpand ) fprintf(stdout, "(NoExpand) "); + break; + case ENAME_SPAN: + fprintf(stdout, "SPAN(\"%s\") ", zName); + break; + } + } + if( j ){ + fprintf(stdout, "iOrderByCol=%d ", j); + } + if( sortFlags & KEYINFO_ORDER_DESC ){ + fprintf(stdout, "DESC "); + }else if( sortFlags & KEYINFO_ORDER_BIGNULL ){ + fprintf(stdout, "NULLS-LAST"); + } + fprintf(stdout, "\n"); + fflush(stdout); + } + sqlite3TreeViewExpr(pView, pList->a[i].pExpr, moreToFollow); + if( j || zName || sortFlags ){ + sqlite3TreeViewPop(&pView); + } + } + } +} SQLITE_PRIVATE void sqlite3TreeViewExprList( TreeView *pView, const ExprList *pList, u8 moreToFollow, const char *zLabel ){ - int i; - pView = sqlite3TreeViewPush(pView, moreToFollow); + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewBareExprList(pView, pList, zLabel); + sqlite3TreeViewPop(&pView); +} + +/* +** Generate a human-readable explanation of an id-list. +*/ +SQLITE_PRIVATE void sqlite3TreeViewBareIdList( + TreeView *pView, + const IdList *pList, + const char *zLabel +){ if( zLabel==0 || zLabel[0]==0 ) zLabel = "LIST"; if( pList==0 ){ sqlite3TreeViewLine(pView, "%s (empty)", zLabel); }else{ + int i; sqlite3TreeViewLine(pView, "%s", zLabel); - for(i=0; inExpr; i++){ - int j = pList->a[i].u.x.iOrderByCol; - if( j ){ - sqlite3TreeViewPush(pView, 0); - sqlite3TreeViewLine(pView, "iOrderByCol=%d", j); + for(i=0; inId; i++){ + char *zName = pList->a[i].zName; + int moreToFollow = inId - 1; + if( zName==0 ) zName = "(null)"; + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewLine(pView, 0); + if( pList->eU4==EU4_NONE ){ + fprintf(stdout, "%s\n", zName); + }else if( pList->eU4==EU4_IDX ){ + fprintf(stdout, "%s (%d)\n", zName, pList->a[i].u4.idx); + }else{ + assert( pList->eU4==EU4_EXPR ); + if( pList->a[i].u4.pExpr==0 ){ + fprintf(stdout, "%s (pExpr=NULL)\n", zName); + }else{ + fprintf(stdout, "%s\n", zName); + sqlite3TreeViewPush(&pView, inId-1); + sqlite3TreeViewExpr(pView, pList->a[i].u4.pExpr, 0); + sqlite3TreeViewPop(&pView); + } } - sqlite3TreeViewExpr(pView, pList->a[i].pExpr, inExpr-1); - if( j ) sqlite3TreeViewPop(pView); + sqlite3TreeViewPop(&pView); } } - sqlite3TreeViewPop(pView); } +SQLITE_PRIVATE void sqlite3TreeViewIdList( + TreeView *pView, + const IdList *pList, + u8 moreToFollow, + const char *zLabel +){ + sqlite3TreeViewPush(&pView, moreToFollow); + sqlite3TreeViewBareIdList(pView, pList, zLabel); + sqlite3TreeViewPop(&pView); +} + +/* +** Generate a human-readable explanation of a list of Upsert objects +*/ +SQLITE_PRIVATE void sqlite3TreeViewUpsert( + TreeView *pView, + const Upsert *pUpsert, + u8 moreToFollow +){ + if( pUpsert==0 ) return; + sqlite3TreeViewPush(&pView, moreToFollow); + while( pUpsert ){ + int n; + sqlite3TreeViewPush(&pView, pUpsert->pNextUpsert!=0 || moreToFollow); + sqlite3TreeViewLine(pView, "ON CONFLICT DO %s", + pUpsert->isDoUpdate ? "UPDATE" : "NOTHING"); + n = (pUpsert->pUpsertSet!=0) + (pUpsert->pUpsertWhere!=0); + sqlite3TreeViewExprList(pView, pUpsert->pUpsertTarget, (n--)>0, "TARGET"); + sqlite3TreeViewExprList(pView, pUpsert->pUpsertSet, (n--)>0, "SET"); + if( pUpsert->pUpsertWhere ){ + sqlite3TreeViewItem(pView, "WHERE", (n--)>0); + sqlite3TreeViewExpr(pView, pUpsert->pUpsertWhere, 0); + sqlite3TreeViewPop(&pView); + } + sqlite3TreeViewPop(&pView); + pUpsert = pUpsert->pNextUpsert; + } + sqlite3TreeViewPop(&pView); +} + +#if TREETRACE_ENABLED +/* +** Generate a human-readable diagram of the data structure that go +** into generating an DELETE statement. +*/ +SQLITE_PRIVATE void sqlite3TreeViewDelete( + const With *pWith, + const SrcList *pTabList, + const Expr *pWhere, + const ExprList *pOrderBy, + const Expr *pLimit, + const Trigger *pTrigger +){ + int n = 0; + TreeView *pView = 0; + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, "DELETE"); + if( pWith ) n++; + if( pTabList ) n++; + if( pWhere ) n++; + if( pOrderBy ) n++; + if( pLimit ) n++; + if( pTrigger ) n++; + if( pWith ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewWith(pView, pWith, 0); + sqlite3TreeViewPop(&pView); + } + if( pTabList ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "FROM"); + sqlite3TreeViewSrcList(pView, pTabList); + sqlite3TreeViewPop(&pView); + } + if( pWhere ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "WHERE"); + sqlite3TreeViewExpr(pView, pWhere, 0); + sqlite3TreeViewPop(&pView); + } + if( pOrderBy ){ + sqlite3TreeViewExprList(pView, pOrderBy, (--n)>0, "ORDER-BY"); + } + if( pLimit ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "LIMIT"); + sqlite3TreeViewExpr(pView, pLimit, 0); + sqlite3TreeViewPop(&pView); + } + if( pTrigger ){ + sqlite3TreeViewTrigger(pView, pTrigger, (--n)>0, 1); + } + sqlite3TreeViewPop(&pView); +} +#endif /* TREETRACE_ENABLED */ + +#if TREETRACE_ENABLED +/* +** Generate a human-readable diagram of the data structure that go +** into generating an INSERT statement. +*/ +SQLITE_PRIVATE void sqlite3TreeViewInsert( + const With *pWith, + const SrcList *pTabList, + const IdList *pColumnList, + const Select *pSelect, + const ExprList *pExprList, + int onError, + const Upsert *pUpsert, + const Trigger *pTrigger +){ + TreeView *pView = 0; + int n = 0; + const char *zLabel = "INSERT"; + switch( onError ){ + case OE_Replace: zLabel = "REPLACE"; break; + case OE_Ignore: zLabel = "INSERT OR IGNORE"; break; + case OE_Rollback: zLabel = "INSERT OR ROLLBACK"; break; + case OE_Abort: zLabel = "INSERT OR ABORT"; break; + case OE_Fail: zLabel = "INSERT OR FAIL"; break; + } + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, zLabel); + if( pWith ) n++; + if( pTabList ) n++; + if( pColumnList ) n++; + if( pSelect ) n++; + if( pExprList ) n++; + if( pUpsert ) n++; + if( pTrigger ) n++; + if( pWith ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewWith(pView, pWith, 0); + sqlite3TreeViewPop(&pView); + } + if( pTabList ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "INTO"); + sqlite3TreeViewSrcList(pView, pTabList); + sqlite3TreeViewPop(&pView); + } + if( pColumnList ){ + sqlite3TreeViewIdList(pView, pColumnList, (--n)>0, "COLUMNS"); + } + if( pSelect ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "DATA-SOURCE"); + sqlite3TreeViewSelect(pView, pSelect, 0); + sqlite3TreeViewPop(&pView); + } + if( pExprList ){ + sqlite3TreeViewExprList(pView, pExprList, (--n)>0, "VALUES"); + } + if( pUpsert ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "UPSERT"); + sqlite3TreeViewUpsert(pView, pUpsert, 0); + sqlite3TreeViewPop(&pView); + } + if( pTrigger ){ + sqlite3TreeViewTrigger(pView, pTrigger, (--n)>0, 1); + } + sqlite3TreeViewPop(&pView); +} +#endif /* TREETRACE_ENABLED */ + +#if TREETRACE_ENABLED +/* +** Generate a human-readable diagram of the data structure that go +** into generating an UPDATE statement. +*/ +SQLITE_PRIVATE void sqlite3TreeViewUpdate( + const With *pWith, + const SrcList *pTabList, + const ExprList *pChanges, + const Expr *pWhere, + int onError, + const ExprList *pOrderBy, + const Expr *pLimit, + const Upsert *pUpsert, + const Trigger *pTrigger +){ + int n = 0; + TreeView *pView = 0; + const char *zLabel = "UPDATE"; + switch( onError ){ + case OE_Replace: zLabel = "UPDATE OR REPLACE"; break; + case OE_Ignore: zLabel = "UPDATE OR IGNORE"; break; + case OE_Rollback: zLabel = "UPDATE OR ROLLBACK"; break; + case OE_Abort: zLabel = "UPDATE OR ABORT"; break; + case OE_Fail: zLabel = "UPDATE OR FAIL"; break; + } + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewLine(pView, zLabel); + if( pWith ) n++; + if( pTabList ) n++; + if( pChanges ) n++; + if( pWhere ) n++; + if( pOrderBy ) n++; + if( pLimit ) n++; + if( pUpsert ) n++; + if( pTrigger ) n++; + if( pWith ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewWith(pView, pWith, 0); + sqlite3TreeViewPop(&pView); + } + if( pTabList ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "FROM"); + sqlite3TreeViewSrcList(pView, pTabList); + sqlite3TreeViewPop(&pView); + } + if( pChanges ){ + sqlite3TreeViewExprList(pView, pChanges, (--n)>0, "SET"); + } + if( pWhere ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "WHERE"); + sqlite3TreeViewExpr(pView, pWhere, 0); + sqlite3TreeViewPop(&pView); + } + if( pOrderBy ){ + sqlite3TreeViewExprList(pView, pOrderBy, (--n)>0, "ORDER-BY"); + } + if( pLimit ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "LIMIT"); + sqlite3TreeViewExpr(pView, pLimit, 0); + sqlite3TreeViewPop(&pView); + } + if( pUpsert ){ + sqlite3TreeViewPush(&pView, (--n)>0); + sqlite3TreeViewLine(pView, "UPSERT"); + sqlite3TreeViewUpsert(pView, pUpsert, 0); + sqlite3TreeViewPop(&pView); + } + if( pTrigger ){ + sqlite3TreeViewTrigger(pView, pTrigger, (--n)>0, 1); + } + sqlite3TreeViewPop(&pView); +} +#endif /* TREETRACE_ENABLED */ + +#ifndef SQLITE_OMIT_TRIGGER +/* +** Show a human-readable graph of a TriggerStep +*/ +SQLITE_PRIVATE void sqlite3TreeViewTriggerStep( + TreeView *pView, + const TriggerStep *pStep, + u8 moreToFollow, + u8 showFullList +){ + int cnt = 0; + if( pStep==0 ) return; + sqlite3TreeViewPush(&pView, + moreToFollow || (showFullList && pStep->pNext!=0)); + do{ + if( cnt++ && pStep->pNext==0 ){ + sqlite3TreeViewPop(&pView); + sqlite3TreeViewPush(&pView, 0); + } + sqlite3TreeViewLine(pView, "%s", pStep->zSpan ? pStep->zSpan : "RETURNING"); + }while( showFullList && (pStep = pStep->pNext)!=0 ); + sqlite3TreeViewPop(&pView); +} + +/* +** Show a human-readable graph of a Trigger +*/ +SQLITE_PRIVATE void sqlite3TreeViewTrigger( + TreeView *pView, + const Trigger *pTrigger, + u8 moreToFollow, + u8 showFullList +){ + int cnt = 0; + if( pTrigger==0 ) return; + sqlite3TreeViewPush(&pView, + moreToFollow || (showFullList && pTrigger->pNext!=0)); + do{ + if( cnt++ && pTrigger->pNext==0 ){ + sqlite3TreeViewPop(&pView); + sqlite3TreeViewPush(&pView, 0); + } + sqlite3TreeViewLine(pView, "TRIGGER %s", pTrigger->zName); + sqlite3TreeViewPush(&pView, 0); + sqlite3TreeViewTriggerStep(pView, pTrigger->step_list, 0, 1); + sqlite3TreeViewPop(&pView); + }while( showFullList && (pTrigger = pTrigger->pNext)!=0 ); + sqlite3TreeViewPop(&pView); +} +#endif /* SQLITE_OMIT_TRIGGER */ + + +/* +** These simplified versions of the tree-view routines omit unnecessary +** parameters. These variants are intended to be used from a symbolic +** debugger, such as "gdb", during interactive debugging sessions. +** +** This routines are given external linkage so that they will always be +** accessible to the debugging, and to avoid warnings about unused +** functions. But these routines only exist in debugging builds, so they +** do not contaminate the interface. +*/ +SQLITE_PRIVATE void sqlite3ShowExpr(const Expr *p){ sqlite3TreeViewExpr(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowExprList(const ExprList *p){ sqlite3TreeViewExprList(0,p,0,0);} +SQLITE_PRIVATE void sqlite3ShowIdList(const IdList *p){ sqlite3TreeViewIdList(0,p,0,0); } +SQLITE_PRIVATE void sqlite3ShowSrcList(const SrcList *p){ sqlite3TreeViewSrcList(0,p); } +SQLITE_PRIVATE void sqlite3ShowSelect(const Select *p){ sqlite3TreeViewSelect(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowWith(const With *p){ sqlite3TreeViewWith(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowUpsert(const Upsert *p){ sqlite3TreeViewUpsert(0,p,0); } +#ifndef SQLITE_OMIT_TRIGGER +SQLITE_PRIVATE void sqlite3ShowTriggerStep(const TriggerStep *p){ + sqlite3TreeViewTriggerStep(0,p,0,0); +} +SQLITE_PRIVATE void sqlite3ShowTriggerStepList(const TriggerStep *p){ + sqlite3TreeViewTriggerStep(0,p,0,1); +} +SQLITE_PRIVATE void sqlite3ShowTrigger(const Trigger *p){ sqlite3TreeViewTrigger(0,p,0,0); } +SQLITE_PRIVATE void sqlite3ShowTriggerList(const Trigger *p){ sqlite3TreeViewTrigger(0,p,0,1);} +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE void sqlite3ShowWindow(const Window *p){ sqlite3TreeViewWindow(0,p,0); } +SQLITE_PRIVATE void sqlite3ShowWinFunc(const Window *p){ sqlite3TreeViewWinFunc(0,p,0); } +#endif #endif /* SQLITE_DEBUG */ @@ -26051,16 +34172,41 @@ SQLITE_PRIVATE void sqlite3TreeViewExprList( ** This structure is the current state of the generator. */ static SQLITE_WSD struct sqlite3PrngType { - unsigned char isInit; /* True if initialized */ - unsigned char i, j; /* State variables */ - unsigned char s[256]; /* State variables */ + u32 s[16]; /* 64 bytes of chacha20 state */ + u8 out[64]; /* Output bytes */ + u8 n; /* Output bytes remaining */ } sqlite3Prng; + +/* The RFC-7539 ChaCha20 block function +*/ +#define ROTL(a,b) (((a) << (b)) | ((a) >> (32 - (b)))) +#define QR(a, b, c, d) ( \ + a += b, d ^= a, d = ROTL(d,16), \ + c += d, b ^= c, b = ROTL(b,12), \ + a += b, d ^= a, d = ROTL(d, 8), \ + c += d, b ^= c, b = ROTL(b, 7)) +static void chacha_block(u32 *out, const u32 *in){ + int i; + u32 x[16]; + memcpy(x, in, 64); + for(i=0; i<10; i++){ + QR(x[0], x[4], x[ 8], x[12]); + QR(x[1], x[5], x[ 9], x[13]); + QR(x[2], x[6], x[10], x[14]); + QR(x[3], x[7], x[11], x[15]); + QR(x[0], x[5], x[10], x[15]); + QR(x[1], x[6], x[11], x[12]); + QR(x[2], x[7], x[ 8], x[13]); + QR(x[3], x[4], x[ 9], x[14]); + } + for(i=0; i<16; i++) out[i] = x[i]+in[i]; +} + /* ** Return N random bytes. */ -SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *pBuf){ - unsigned char t; +SQLITE_API void sqlite3_randomness(int N, void *pBuf){ unsigned char *zBuf = pBuf; /* The "wsdPrng" macro will resolve to the pseudo-random number generator @@ -26090,52 +34236,50 @@ SQLITE_API void SQLITE_STDCALL sqlite3_randomness(int N, void *pBuf){ sqlite3_mutex_enter(mutex); if( N<=0 || pBuf==0 ){ - wsdPrng.isInit = 0; + wsdPrng.s[0] = 0; sqlite3_mutex_leave(mutex); return; } /* Initialize the state of the random number generator once, - ** the first time this routine is called. The seed value does - ** not need to contain a lot of randomness since we are not - ** trying to do secure encryption or anything like that... - ** - ** Nothing in this file or anywhere else in SQLite does any kind of - ** encryption. The RC4 algorithm is being used as a PRNG (pseudo-random - ** number generator) not as an encryption device. + ** the first time this routine is called. */ - if( !wsdPrng.isInit ){ - int i; - char k[256]; - wsdPrng.j = 0; - wsdPrng.i = 0; - sqlite3OsRandomness(sqlite3_vfs_find(0), 256, k); - for(i=0; i<256; i++){ - wsdPrng.s[i] = (u8)i; + if( wsdPrng.s[0]==0 ){ + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + static const u32 chacha20_init[] = { + 0x61707865, 0x3320646e, 0x79622d32, 0x6b206574 + }; + memcpy(&wsdPrng.s[0], chacha20_init, 16); + if( NEVER(pVfs==0) ){ + memset(&wsdPrng.s[4], 0, 44); + }else{ + sqlite3OsRandomness(pVfs, 44, (char*)&wsdPrng.s[4]); } - for(i=0; i<256; i++){ - wsdPrng.j += wsdPrng.s[i] + k[i]; - t = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = wsdPrng.s[i]; - wsdPrng.s[i] = t; - } - wsdPrng.isInit = 1; + wsdPrng.s[15] = wsdPrng.s[12]; + wsdPrng.s[12] = 0; + wsdPrng.n = 0; } assert( N>0 ); - do{ - wsdPrng.i++; - t = wsdPrng.s[wsdPrng.i]; - wsdPrng.j += t; - wsdPrng.s[wsdPrng.i] = wsdPrng.s[wsdPrng.j]; - wsdPrng.s[wsdPrng.j] = t; - t += wsdPrng.s[wsdPrng.i]; - *(zBuf++) = wsdPrng.s[t]; - }while( --N ); + while( 1 /* exit by break */ ){ + if( N<=wsdPrng.n ){ + memcpy(zBuf, &wsdPrng.out[wsdPrng.n-N], N); + wsdPrng.n -= N; + break; + } + if( wsdPrng.n>0 ){ + memcpy(zBuf, wsdPrng.out, wsdPrng.n); + N -= wsdPrng.n; + zBuf += wsdPrng.n; + } + wsdPrng.s[12]++; + chacha_block((u32*)wsdPrng.out, wsdPrng.s); + wsdPrng.n = 64; + } sqlite3_mutex_leave(mutex); } -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE /* ** For testing purposes, we sometimes want to preserve the state of ** PRNG and restore the PRNG to its saved state at a later time, or @@ -26160,7 +34304,7 @@ SQLITE_PRIVATE void sqlite3PrngRestoreState(void){ sizeof(sqlite3Prng) ); } -#endif /* SQLITE_OMIT_BUILTIN_TEST */ +#endif /* SQLITE_UNTESTABLE */ /************** End of random.c **********************************************/ /************** Begin file threads.c *****************************************/ @@ -26233,13 +34377,13 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( memset(p, 0, sizeof(*p)); p->xTask = xTask; p->pIn = pIn; - /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a ** function that returns SQLITE_ERROR when passed the argument 200, that - ** forces worker threads to run sequentially and deterministically + ** forces worker threads to run sequentially and deterministically ** for testing purposes. */ if( sqlite3FaultSim(200) ){ rc = 1; - }else{ + }else{ rc = pthread_create(&p->tid, 0, xTask, pIn); } if( rc ){ @@ -26321,9 +34465,9 @@ SQLITE_PRIVATE int sqlite3ThreadCreate( *ppThread = 0; p = sqlite3Malloc(sizeof(*p)); if( p==0 ) return SQLITE_NOMEM_BKPT; - /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a + /* If the SQLITE_TESTCTRL_FAULT_INSTALL callback is registered to a ** function that returns SQLITE_ERROR when passed the argument 200, that - ** forces worker threads to run sequentially and deterministically + ** forces worker threads to run sequentially and deterministically ** (via the sqlite3FaultSim() term of the conditional) for testing ** purposes. */ if( sqlite3GlobalConfig.bCoreMutex==0 || sqlite3FaultSim(200) ){ @@ -26452,7 +34596,7 @@ SQLITE_PRIVATE int sqlite3ThreadJoin(SQLiteThread *p, void **ppOut){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains routines used to translate between UTF-8, +** This file contains routines used to translate between UTF-8, ** UTF-16, UTF-16BE, and UTF-16LE. ** ** Notes on UTF-8: @@ -26548,26 +34692,6 @@ static const unsigned char sqlite3Utf8Trans1[] = { } \ } -#define READ_UTF16LE(zIn, TERM, c){ \ - c = (*zIn++); \ - c += ((*zIn++)<<8); \ - if( c>=0xD800 && c<0xE000 && TERM ){ \ - int c2 = (*zIn++); \ - c2 += ((*zIn++)<<8); \ - c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ - } \ -} - -#define READ_UTF16BE(zIn, TERM, c){ \ - c = ((*zIn++)<<8); \ - c += (*zIn++); \ - if( c>=0xD800 && c<0xE000 && TERM ){ \ - int c2 = ((*zIn++)<<8); \ - c2 += (*zIn++); \ - c = (c2&0x03FF) + ((c&0x003F)<<10) + (((c&0x03C0)+0x0040)<<10); \ - } \ -} - /* ** Translate a single UTF-8 character. Return the unicode value. ** @@ -26599,7 +34723,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { c = *(zIn++); \ if( c>=0xc0 ){ \ c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + while( zIn0 ); + c = z[0]; + if( c>=0xc0 ){ + c = sqlite3Utf8Trans1[c-0xc0]; + if( n>4 ) n = 4; + while( idb==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -26658,13 +34813,15 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) { - char zBuf[100]; - sqlite3VdbeMemPrettyPrint(pMem, zBuf); - fprintf(stderr, "INPUT: %s\n", zBuf); + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(pMem, &acc); + fprintf(stderr, "INPUT: %s\n", sqlite3StrAccumFinish(&acc)); } #endif - /* If the translation is between UTF-16 little and big endian, then + /* If the translation is between UTF-16 little and big endian, then ** all that is required is to swap the byte order. This case is handled ** differently from the others. */ @@ -26696,14 +34853,14 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired ** nul-terminator. */ pMem->n &= ~1; - len = pMem->n * 2 + 1; + len = 2 * (sqlite3_int64)pMem->n + 1; }else{ /* When converting from UTF-8 to UTF-16 the maximum growth is caused ** when a 1-byte UTF-8 character is translated into a 2-byte UTF-16 ** character. Two bytes are required in the output buffer for the ** nul-terminator. */ - len = pMem->n * 2 + 2; + len = 2 * (sqlite3_int64)pMem->n + 2; } /* Set zIn to point at the start of the input buffer and zTerm to point 1 @@ -26742,13 +34899,59 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired if( pMem->enc==SQLITE_UTF16LE ){ /* UTF-16 Little-endian -> UTF-8 */ while( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF + if( c>=0xdc00 || zIn>=zTerm ){ + c = 0xfffd; + }else{ + int c2 = *(zIn++); + c2 += (*(zIn++))<<8; + if( c2<0xdc00 || c2>=0xe000 ){ + zIn -= 2; + c = 0xfffd; + }else{ + c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; + } + } +#else + if( zIn UTF-8 */ while( zIn=0xd800 && c<0xe000 ){ +#ifdef SQLITE_REPLACE_INVALID_UTF + if( c>=0xdc00 || zIn>=zTerm ){ + c = 0xfffd; + }else{ + int c2 = (*(zIn++))<<8; + c2 += *(zIn++); + if( c2<0xdc00 || c2>=0xe000 ){ + zIn -= 2; + c = 0xfffd; + }else{ + c = ((c&0x3ff)<<10) + (c2&0x3ff) + 0x10000; + } + } +#else + if( zInn+(desiredEnc==SQLITE_UTF8?1:2))<=len ); - c = pMem->flags; + c = MEM_Str|MEM_Term|(pMem->flags&(MEM_AffMask|MEM_Subtype)); sqlite3VdbeMemRelease(pMem); - pMem->flags = MEM_Str|MEM_Term|(c&(MEM_AffMask|MEM_Subtype)); + pMem->flags = c; pMem->enc = desiredEnc; pMem->z = (char*)zOut; pMem->zMalloc = pMem->z; @@ -26768,16 +34971,20 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemTranslate(Mem *pMem, u8 desired translate_out: #if defined(TRANSLATE_TRACE) && defined(SQLITE_DEBUG) { - char zBuf[100]; - sqlite3VdbeMemPrettyPrint(pMem, zBuf); - fprintf(stderr, "OUTPUT: %s\n", zBuf); + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(pMem, &acc); + fprintf(stderr, "OUTPUT: %s\n", sqlite3StrAccumFinish(&acc)); } #endif return SQLITE_OK; } +#endif /* SQLITE_OMIT_UTF16 */ +#ifndef SQLITE_OMIT_UTF16 /* -** This routine checks for a byte-order mark at the beginning of the +** This routine checks for a byte-order mark at the beginning of the ** UTF-16 string stored in *pMem. If one is present, it is removed and ** the encoding of the Mem adjusted. This routine does not do any ** byte-swapping, it just sets Mem.enc appropriately. @@ -26800,7 +35007,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem){ bom = SQLITE_UTF16LE; } } - + if( bom ){ rc = sqlite3VdbeMemMakeWriteable(pMem); if( rc==SQLITE_OK ){ @@ -26820,7 +35027,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemHandleBom(Mem *pMem){ ** pZ is a UTF-8 encoded unicode string. If nByte is less than zero, ** return the number of unicode characters in pZ up to (but not including) ** the first 0x00 byte. If nByte is not less than zero, return the -** number of unicode characters in the first nByte of pZ (or up to +** number of unicode characters in the first nByte of pZ (or up to ** the first 0x00, whichever comes first). */ SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){ @@ -26840,7 +35047,7 @@ SQLITE_PRIVATE int sqlite3Utf8CharLen(const char *zIn, int nByte){ return r; } -/* This test function is not currently used by the automated test-suite. +/* This test function is not currently used by the automated test-suite. ** Hence it is only available in debug builds. */ #if defined(SQLITE_TEST) && defined(SQLITE_DEBUG) @@ -26894,27 +35101,26 @@ SQLITE_PRIVATE char *sqlite3Utf16to8(sqlite3 *db, const void *z, int nByte, u8 e } /* -** zIn is a UTF-16 encoded unicode string at least nChar characters long. +** zIn is a UTF-16 encoded unicode string at least nByte bytes long. ** Return the number of bytes in the first nChar unicode characters -** in pZ. nChar must be non-negative. +** in pZ. nChar must be non-negative. Surrogate pairs count as a single +** character. */ -SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *zIn, int nChar){ +SQLITE_PRIVATE int sqlite3Utf16ByteLen(const void *zIn, int nByte, int nChar){ int c; unsigned char const *z = zIn; + unsigned char const *zEnd = &z[nByte-1]; int n = 0; - - if( SQLITE_UTF16NATIVE==SQLITE_UTF16BE ){ - while( n=0xd8 && c<0xdc && z<=zEnd && z[0]>=0xdc && z[0]<0xe0 ) z += 2; + n++; } - return (int)(z-(unsigned char const *)zIn); + return (int)(z-(unsigned char const *)zIn) + - (SQLITE_UTF16NATIVE==SQLITE_UTF16LE); } #if defined(SQLITE_TEST) @@ -26944,30 +35150,6 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ assert( c==t ); assert( (z-zBuf)==n ); } - for(i=0; i<0x00110000; i++){ - if( i>=0xD800 && i<0xE000 ) continue; - z = zBuf; - WRITE_UTF16LE(z, i); - n = (int)(z-zBuf); - assert( n>0 && n<=4 ); - z[0] = 0; - z = zBuf; - READ_UTF16LE(z, 1, c); - assert( c==i ); - assert( (z-zBuf)==n ); - } - for(i=0; i<0x00110000; i++){ - if( i>=0xD800 && i<0xE000 ) continue; - z = zBuf; - WRITE_UTF16BE(z, i); - n = (int)(z-zBuf); - assert( n>0 && n<=4 ); - z[0] = 0; - z = zBuf; - READ_UTF16BE(z, 1, c); - assert( c==i ); - assert( (z-zBuf)==n ); - } } #endif /* SQLITE_TEST */ #endif /* SQLITE_OMIT_UTF16 */ @@ -26993,32 +35175,30 @@ SQLITE_PRIVATE void sqlite3UtfSelfTest(void){ */ /* #include "sqliteInt.h" */ /* #include */ -#if HAVE_ISNAN || SQLITE_HAVE_ISNAN -# include +#ifndef SQLITE_OMIT_FLOATING_POINT +#include #endif /* -** Routine needed to support the testcase() macro. -*/ -#ifdef SQLITE_COVERAGE_TEST -SQLITE_PRIVATE void sqlite3Coverage(int x){ - static unsigned dummy = 0; - dummy += (unsigned)x; -} -#endif - -/* -** Give a callback to the test harness that can be used to simulate faults -** in places where it is difficult or expensive to do so purely by means -** of inputs. +** Calls to sqlite3FaultSim() are used to simulate a failure during testing, +** or to bypass normal error detection during testing in order to let +** execute proceed further downstream. ** -** The intent of the integer argument is to let the fault simulator know -** which of multiple sqlite3FaultSim() calls has been hit. +** In deployment, sqlite3FaultSim() *always* return SQLITE_OK (0). The +** sqlite3FaultSim() function only returns non-zero during testing. ** -** Return whatever integer value the test callback returns, or return -** SQLITE_OK if no test callback is installed. +** During testing, if the test harness has set a fault-sim callback using +** a call to sqlite3_test_control(SQLITE_TESTCTRL_FAULT_INSTALL), then +** each call to sqlite3FaultSim() is relayed to that application-supplied +** callback and the integer return value form the application-supplied +** callback is returned by sqlite3FaultSim(). +** +** The integer argument to sqlite3FaultSim() is a code to identify which +** sqlite3FaultSim() instance is being invoked. Each call to sqlite3FaultSim() +** should have a unique code. To prevent legacy testing applications from +** breaking, the codes should not be changed or reused. */ -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ int (*xCallback)(int) = sqlite3GlobalConfig.xTestCallback; return xCallback ? xCallback(iTest) : SQLITE_OK; @@ -27035,36 +35215,10 @@ SQLITE_PRIVATE int sqlite3FaultSim(int iTest){ SQLITE_PRIVATE int sqlite3IsNaN(double x){ int rc; /* The value return */ #if !SQLITE_HAVE_ISNAN && !HAVE_ISNAN - /* - ** Systems that support the isnan() library function should probably - ** make use of it by compiling with -DSQLITE_HAVE_ISNAN. But we have - ** found that many systems do not have a working isnan() function so - ** this implementation is provided as an alternative. - ** - ** This NaN test sometimes fails if compiled on GCC with -ffast-math. - ** On the other hand, the use of -ffast-math comes with the following - ** warning: - ** - ** This option [-ffast-math] should never be turned on by any - ** -O option since it can result in incorrect output for programs - ** which depend on an exact implementation of IEEE or ISO - ** rules/specifications for math functions. - ** - ** Under MSVC, this NaN test may fail if compiled with a floating- - ** point precision mode other than /fp:precise. From the MSDN - ** documentation: - ** - ** The compiler [with /fp:precise] will properly handle comparisons - ** involving NaN. For example, x != x evaluates to true if x is NaN - ** ... - */ -#ifdef __FAST_MATH__ -# error SQLite will not work correctly with the -ffast-math option of GCC. -#endif - volatile double y = x; - volatile double z = y; - rc = (y!=z); -#else /* if HAVE_ISNAN */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsNaN(y); +#else rc = isnan(x); #endif /* HAVE_ISNAN */ testcase( rc ); @@ -27072,6 +35226,19 @@ SQLITE_PRIVATE int sqlite3IsNaN(double x){ } #endif /* SQLITE_OMIT_FLOATING_POINT */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* +** Return true if the floating point value is NaN or +Inf or -Inf. +*/ +SQLITE_PRIVATE int sqlite3IsOverflow(double x){ + int rc; /* The value return */ + u64 y; + memcpy(&y,&x,sizeof(y)); + rc = IsOvfl(y); + return rc; +} +#endif /* SQLITE_OMIT_FLOATING_POINT */ + /* ** Compute a string length that is limited to what can be stored in ** lower 30 bits of a 32-bit signed integer. @@ -27086,15 +35253,21 @@ SQLITE_PRIVATE int sqlite3Strlen30(const char *z){ } /* -** Return the declared type of a column. Or return zDflt if the column +** Return the declared type of a column. Or return zDflt if the column ** has no declared type. ** ** The column type is an extra string stored after the zero-terminator on ** the column name if and only if the COLFLAG_HASTYPE flag is set. */ SQLITE_PRIVATE char *sqlite3ColumnType(Column *pCol, char *zDflt){ - if( (pCol->colFlags & COLFLAG_HASTYPE)==0 ) return zDflt; - return pCol->zName + strlen(pCol->zName) + 1; + if( pCol->colFlags & COLFLAG_HASTYPE ){ + return pCol->zCnName + strlen(pCol->zCnName) + 1; + }else if( pCol->eCType ){ + assert( pCol->eCType<=SQLITE_N_STDTYPE ); + return (char*)sqlite3StdType[pCol->eCType-1]; + }else{ + return zDflt; + } } /* @@ -27115,7 +35288,22 @@ static SQLITE_NOINLINE void sqlite3ErrorFinish(sqlite3 *db, int err_code){ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ assert( db!=0 ); db->errCode = err_code; - if( err_code || db->pErr ) sqlite3ErrorFinish(db, err_code); + if( err_code || db->pErr ){ + sqlite3ErrorFinish(db, err_code); + }else{ + db->errByteOffset = -1; + } +} + +/* +** The equivalent of sqlite3Error(db, SQLITE_OK). Clear the error state +** and error message. +*/ +SQLITE_PRIVATE void sqlite3ErrorClear(sqlite3 *db){ + assert( db!=0 ); + db->errCode = SQLITE_OK; + db->errByteOffset = -1; + if( db->pErr ) sqlite3ValueSetNull(db->pErr); } /* @@ -27124,6 +35312,23 @@ SQLITE_PRIVATE void sqlite3Error(sqlite3 *db, int err_code){ */ SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ if( rc==SQLITE_IOERR_NOMEM ) return; +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) + if( rc==SQLITE_IOERR_IN_PAGE ){ + int ii; + int iErr; + sqlite3BtreeEnterAll(db); + for(ii=0; iinDb; ii++){ + if( db->aDb[ii].pBt ){ + iErr = sqlite3PagerWalSystemErrno(sqlite3BtreePager(db->aDb[ii].pBt)); + if( iErr ){ + db->iSysErrno = iErr; + } + } + } + sqlite3BtreeLeaveAll(db); + return; + } +#endif rc &= 0xff; if( rc==SQLITE_CANTOPEN || rc==SQLITE_IOERR ){ db->iSysErrno = sqlite3OsGetLastError(db->pVfs); @@ -27135,17 +35340,8 @@ SQLITE_PRIVATE void sqlite3SystemError(sqlite3 *db, int rc){ ** handle "db". The error code is set to "err_code". ** ** If it is not NULL, string zFormat specifies the format of the -** error string in the style of the printf functions: The following -** format characters are allowed: -** -** %s Insert a string -** %z A string that should be freed after use -** %d Insert an integer -** %T Insert a token -** %S Insert the first element of a SrcList -** -** zFormat and any string tokens that follow it are assumed to be -** encoded in UTF-8. +** error string. zFormat and any string tokens that follow it are +** assumed to be encoded in UTF-8. ** ** To clear the most recent error for sqlite handle "db", sqlite3Error ** should be called with err_code set to SQLITE_OK and zFormat set @@ -27167,15 +35363,32 @@ SQLITE_PRIVATE void sqlite3ErrorWithMsg(sqlite3 *db, int err_code, const char *z } } +/* +** Check for interrupts and invoke progress callback. +*/ +SQLITE_PRIVATE void sqlite3ProgressCheck(Parse *p){ + sqlite3 *db = p->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + if( p->rc==SQLITE_INTERRUPT ){ + p->nProgressSteps = 0; + }else if( (++p->nProgressSteps)>=db->nProgressOps ){ + if( db->xProgress(db->pProgressArg) ){ + p->nErr++; + p->rc = SQLITE_INTERRUPT; + } + p->nProgressSteps = 0; + } + } +#endif +} + /* ** Add an error message to pParse->zErrMsg and increment pParse->nErr. -** The following formatting characters are allowed: -** -** %s Insert a string -** %z A string that should be freed after use -** %d Insert an integer -** %T Insert a token -** %S Insert the first element of a SrcList ** ** This function should be used to report any error that occurs while ** compiling an SQL statement (i.e. within sqlite3_prepare()). The @@ -27188,19 +35401,41 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ char *zMsg; va_list ap; sqlite3 *db = pParse->db; + assert( db!=0 ); + assert( db->pParse==pParse || db->pParse->pToplevel==pParse ); + db->errByteOffset = -2; va_start(ap, zFormat); zMsg = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); + if( db->errByteOffset<-1 ) db->errByteOffset = -1; if( db->suppressErr ){ sqlite3DbFree(db, zMsg); + if( db->mallocFailed ){ + pParse->nErr++; + pParse->rc = SQLITE_NOMEM; + } }else{ pParse->nErr++; sqlite3DbFree(db, pParse->zErrMsg); pParse->zErrMsg = zMsg; pParse->rc = SQLITE_ERROR; + pParse->pWith = 0; } } +/* +** If database connection db is currently parsing SQL, then transfer +** error code errCode to that parser if the parser has not already +** encountered some other kind of error. +*/ +SQLITE_PRIVATE int sqlite3ErrorToParser(sqlite3 *db, int errCode){ + Parse *pParse; + if( db==0 || (pParse = db->pParse)==0 ) return errCode; + pParse->rc = errCode; + pParse->nErr++; + return errCode; +} + /* ** Convert an SQL-style quoted string into a normal string by removing ** the quote characters. The conversion is done in-place. If the @@ -27214,7 +35449,7 @@ SQLITE_PRIVATE void sqlite3ErrorMsg(Parse *pParse, const char *zFormat, ...){ ** dequoted string, exclusive of the zero terminator, if dequoting does ** occur. ** -** 2002-Feb-14: This routine is extended to remove MS-Access style +** 2002-02-14: This routine is extended to remove MS-Access style ** brackets from around identifiers. For example: "[a-b-c]" becomes ** "a-b-c". */ @@ -27240,6 +35475,72 @@ SQLITE_PRIVATE void sqlite3Dequote(char *z){ } z[j] = 0; } +SQLITE_PRIVATE void sqlite3DequoteExpr(Expr *p){ + assert( !ExprHasProperty(p, EP_IntValue) ); + assert( sqlite3Isquote(p->u.zToken[0]) ); + p->flags |= p->u.zToken[0]=='"' ? EP_Quoted|EP_DblQuoted : EP_Quoted; + sqlite3Dequote(p->u.zToken); +} + +/* +** Expression p is a QNUMBER (quoted number). Dequote the value in p->u.zToken +** and set the type to INTEGER or FLOAT. "Quoted" integers or floats are those +** that contain '_' characters that must be removed before further processing. +*/ +SQLITE_PRIVATE void sqlite3DequoteNumber(Parse *pParse, Expr *p){ + assert( p!=0 || pParse->db->mallocFailed ); + if( p ){ + const char *pIn = p->u.zToken; + char *pOut = p->u.zToken; + int bHex = (pIn[0]=='0' && (pIn[1]=='x' || pIn[1]=='X')); + int iValue; + assert( p->op==TK_QNUMBER ); + p->op = TK_INTEGER; + do { + if( *pIn!=SQLITE_DIGIT_SEPARATOR ){ + *pOut++ = *pIn; + if( *pIn=='e' || *pIn=='E' || *pIn=='.' ) p->op = TK_FLOAT; + }else{ + if( (bHex==0 && (!sqlite3Isdigit(pIn[-1]) || !sqlite3Isdigit(pIn[1]))) + || (bHex==1 && (!sqlite3Isxdigit(pIn[-1]) || !sqlite3Isxdigit(pIn[1]))) + ){ + sqlite3ErrorMsg(pParse, "unrecognized token: \"%s\"", p->u.zToken); + } + } + }while( *pIn++ ); + if( bHex ) p->op = TK_INTEGER; + + /* tag-20240227-a: If after dequoting, the number is an integer that + ** fits in 32 bits, then it must be converted into EP_IntValue. Other + ** parts of the code expect this. See also tag-20240227-b. */ + if( p->op==TK_INTEGER && sqlite3GetInt32(p->u.zToken, &iValue) ){ + p->u.iValue = iValue; + p->flags |= EP_IntValue; + } + } +} + +/* +** If the input token p is quoted, try to adjust the token to remove +** the quotes. This is not always possible: +** +** "abc" -> abc +** "ab""cd" -> (not possible because of the interior "") +** +** Remove the quotes if possible. This is a optimization. The overall +** system should still return the correct answer even if this routine +** is always a no-op. +*/ +SQLITE_PRIVATE void sqlite3DequoteToken(Token *p){ + unsigned int i; + if( p->n<2 ) return; + if( !sqlite3Isquote(p->z[0]) ) return; + for(i=1; in-1; i++){ + if( sqlite3Isquote(p->z[i]) ) return; + } + p->n -= 2; + p->z++; +} /* ** Generate a Token object from a string @@ -27262,7 +35563,7 @@ SQLITE_PRIVATE void sqlite3TokenInit(Token *p, char *z){ ** case-independent fashion, using the same definition of "case ** independence" that SQLite uses internally when comparing identifiers. */ -SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *zLeft, const char *zRight){ +SQLITE_API int sqlite3_stricmp(const char *zLeft, const char *zRight){ if( zLeft==0 ){ return zRight ? -1 : 0; }else if( zRight==0 ){ @@ -27272,18 +35573,24 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stricmp(const char *zLeft, const char *zRi } SQLITE_PRIVATE int sqlite3StrICmp(const char *zLeft, const char *zRight){ unsigned char *a, *b; - int c; + int c, x; a = (unsigned char *)zLeft; b = (unsigned char *)zRight; for(;;){ - c = (int)UpperToLower[*a] - (int)UpperToLower[*b]; - if( c || *a==0 ) break; + c = *a; + x = *b; + if( c==x ){ + if( c==0 ) break; + }else{ + c = (int)UpperToLower[c] - (int)UpperToLower[x]; + if( c ) break; + } a++; b++; } return c; } -SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ +SQLITE_API int sqlite3_strnicmp(const char *zLeft, const char *zRight, int N){ register unsigned char *a, *b; if( zLeft==0 ){ return zRight ? -1 : 0; @@ -27296,6 +35603,55 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zR return N<0 ? 0 : UpperToLower[*a] - UpperToLower[*b]; } +/* +** Compute an 8-bit hash on a string that is insensitive to case differences +*/ +SQLITE_PRIVATE u8 sqlite3StrIHash(const char *z){ + u8 h = 0; + if( z==0 ) return 0; + while( z[0] ){ + h += UpperToLower[(unsigned char)z[0]]; + z++; + } + return h; +} + +/* Double-Double multiplication. (x[0],x[1]) *= (y,yy) +** +** Reference: +** T. J. Dekker, "A Floating-Point Technique for Extending the +** Available Precision". 1971-07-26. +*/ +static void dekkerMul2(volatile double *x, double y, double yy){ + /* + ** The "volatile" keywords on parameter x[] and on local variables + ** below are needed force intermediate results to be truncated to + ** binary64 rather than be carried around in an extended-precision + ** format. The truncation is necessary for the Dekker algorithm to + ** work. Intel x86 floating point might omit the truncation without + ** the use of volatile. + */ + volatile double tx, ty, p, q, c, cc; + double hx, hy; + u64 m; + memcpy(&m, (void*)&x[0], 8); + m &= 0xfffffffffc000000LL; + memcpy(&hx, &m, 8); + tx = x[0] - hx; + memcpy(&m, &y, 8); + m &= 0xfffffffffc000000LL; + memcpy(&hy, &m, 8); + ty = y - hy; + p = hx*hy; + q = hx*ty + tx*hy; + c = p+q; + cc = p - c + q + tx*ty; + cc = x[0]*yy + x[1]*y + cc; + x[0] = c + cc; + x[1] = c - x[0]; + x[1] += cc; +} + /* ** The string z[] is an text representation of a real number. ** Convert this string to a double and write it into *pResult. @@ -27304,8 +35660,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zR ** uses the encoding enc. The string is not necessarily zero-terminated. ** ** Return TRUE if the result is a valid real number (or integer) and FALSE -** if the string is empty or contains extraneous text. Valid numbers -** are in one of these formats: +** if the string is empty or contains extraneous text. More specifically +** return +** 1 => The input string is a pure integer +** 2 or more => The input has a decimal point or eNNN clause +** 0 or less => The input string is not a valid number +** -1 => Not a valid number, but has a valid prefix which +** includes a decimal point and/or an eNNN clause +** +** Valid numbers are in one of these formats: ** ** [+-]digits[E[+-]digits] ** [+-]digits.[digits][E[+-]digits] @@ -27318,32 +35681,41 @@ SQLITE_API int SQLITE_STDCALL sqlite3_strnicmp(const char *zLeft, const char *zR ** returns FALSE but it still converts the prefix and writes the result ** into *pResult. */ +#if defined(_MSC_VER) +#pragma warning(disable : 4756) +#endif SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 enc){ #ifndef SQLITE_OMIT_FLOATING_POINT int incr; - const char *zEnd = z + length; + const char *zEnd; /* sign * significand * (10 ^ (esign * exponent)) */ int sign = 1; /* sign of significand */ - i64 s = 0; /* significand */ + u64 s = 0; /* significand */ int d = 0; /* adjust exponent for shifting decimal point */ int esign = 1; /* sign of exponent */ int e = 0; /* exponent */ int eValid = 1; /* True exponent is either not used or is well-formed */ - double result; - int nDigits = 0; - int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int nDigit = 0; /* Number of digits processed */ + int eType = 1; /* 1: pure integer, 2+: fractional -1 or less: bad UTF16 */ + double rr[2]; + u64 s2; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); *pResult = 0.0; /* Default return value, in case of an error */ + if( length==0 ) return 0; if( enc==SQLITE_UTF8 ){ incr = 1; + zEnd = z + length; }else{ int i; incr = 2; + length &= ~1; assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + testcase( enc==SQLITE_UTF16LE ); + testcase( enc==SQLITE_UTF16BE ); for(i=3-enc; i=((LARGEST_UINT64-9)/10) ){ + /* skip non-significant significand digits + ** (increase exponent by d to shift decimal left) */ + while( z=zEnd ) goto do_atof_calc; /* if decimal point is present */ if( *z=='.' ){ z+=incr; + eType++; /* copy digits from after decimal to significand ** (decrease exponent by d to shift decimal right) */ while( z=zEnd ) goto do_atof_calc; @@ -27390,8 +35765,9 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en if( *z=='e' || *z=='E' ){ z+=incr; eValid = 0; + eType++; - /* This branch is needed to avoid a (harmless) buffer overread. The + /* This branch is needed to avoid a (harmless) buffer overread. The ** special comment alerts the mutation tester that the correct answer ** is obtained even if the branch is omitted */ if( z>=zEnd ) goto do_atof_calc; /*PREVENTS-HARMLESS-OVERREAD*/ @@ -27415,84 +35791,109 @@ SQLITE_PRIVATE int sqlite3AtoF(const char *z, double *pResult, int length, u8 en while( z0 && s<(LARGEST_UINT64/10) ){ + s *= 10; + e--; + } + while( e<0 && (s%10)==0 ){ + s /= 10; + e++; } - if( s==0 ) { - /* In the IEEE 754 standard, zero is signed. */ - result = sign<0 ? -(double)0 : (double)0; - } else { - /* Attempt to reduce exponent. - ** - ** Branches that are not required for the correct answer but which only - ** help to obtain the correct answer faster are marked with special - ** comments, as a hint to the mutation tester. - */ - while( e>0 ){ /*OPTIMIZATION-IF-TRUE*/ - if( esign>0 ){ - if( s>=(LARGEST_INT64/10) ) break; /*OPTIMIZATION-IF-FALSE*/ - s *= 10; - }else{ - if( s%10!=0 ) break; /*OPTIMIZATION-IF-FALSE*/ - s /= 10; - } - e--; + rr[0] = (double)s; + s2 = (u64)rr[0]; +#if defined(_MSC_VER) && _MSC_VER<1700 + if( s2==0x8000000000000000LL ){ s2 = 2*(u64)(0.5*rr[0]); } +#endif + rr[1] = s>=s2 ? (double)(s - s2) : -(double)(s2 - s); + if( e>0 ){ + while( e>=100 ){ + e -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); } - - /* adjust the sign of significand */ - s = sign<0 ? -s : s; - - if( e==0 ){ /*OPTIMIZATION-IF-TRUE*/ - result = (double)s; - }else{ - LONGDOUBLE_TYPE scale = 1.0; - /* attempt to handle extremely small/large numbers better */ - if( e>307 ){ /*OPTIMIZATION-IF-TRUE*/ - if( e<342 ){ /*OPTIMIZATION-IF-TRUE*/ - while( e%308 ) { scale *= 1.0e+1; e -= 1; } - if( esign<0 ){ - result = s / scale; - result /= 1.0e+308; - }else{ - result = s * scale; - result *= 1.0e+308; - } - }else{ assert( e>=342 ); - if( esign<0 ){ - result = 0.0*s; - }else{ - result = 1e308*1e308*s; /* Infinity */ - } - } - }else{ - /* 1.0e+22 is the largest power of 10 than can be - ** represented exactly. */ - while( e%22 ) { scale *= 1.0e+1; e -= 1; } - while( e>0 ) { scale *= 1.0e+22; e -= 22; } - if( esign<0 ){ - result = s / scale; - }else{ - result = s * scale; - } - } + while( e>=10 ){ + e -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( e>=1 ){ + e -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + }else{ + while( e<=-100 ){ + e += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( e<=-10 ){ + e += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( e<=-1 ){ + e += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); } } + *pResult = rr[0]+rr[1]; + if( sqlite3IsNaN(*pResult) ) *pResult = 1e300*1e300; + if( sign<0 ) *pResult = -*pResult; + assert( !sqlite3IsNaN(*pResult) ); - /* store the result */ - *pResult = result; - - /* return true if number and no extra non-whitespace chracters after */ - return z==zEnd && nDigits>0 && eValid && nonNum==0; +atof_return: + /* return true if number and no extra non-whitespace characters after */ + if( z==zEnd && nDigit>0 && eValid && eType>0 ){ + return eType; + }else if( eType>=2 && (eType==3 || eValid) && nDigit>0 ){ + return -1; + }else{ + return 0; + } #else return !sqlite3Atoi64(z, pResult, length, enc); #endif /* SQLITE_OMIT_FLOATING_POINT */ } +#if defined(_MSC_VER) +#pragma warning(default : 4756) +#endif + +/* +** Render an signed 64-bit integer as text. Store the result in zOut[] and +** return the length of the string that was stored, in bytes. The value +** returned does not include the zero terminator at the end of the output +** string. +** +** The caller must ensure that zOut[] is at least 21 bytes in size. +*/ +SQLITE_PRIVATE int sqlite3Int64ToText(i64 v, char *zOut){ + int i; + u64 x; + char zTemp[22]; + if( v<0 ){ + x = (v==SMALLEST_INT64) ? ((u64)1)<<63 : (u64)-v; + }else{ + x = v; + } + i = sizeof(zTemp)-2; + zTemp[sizeof(zTemp)-1] = 0; + while( 1 /*exit-by-break*/ ){ + zTemp[i] = (x%10) + '0'; + x = x/10; + if( x==0 ) break; + i--; + }; + if( v<0 ) zTemp[--i] = '-'; + memcpy(zOut, &zTemp[i], sizeof(zTemp)-i); + return sizeof(zTemp)-1-i; +} /* ** Compare the 19-character string zNum against the text representation @@ -27529,16 +35930,13 @@ static int compare2pow63(const char *zNum, int incr){ ** Convert zNum to a 64-bit signed integer. zNum must be decimal. This ** routine does *not* accept hexadecimal notation. ** -** If the zNum value is representable as a 64-bit twos-complement -** integer, then write that value into *pNum and return 0. +** Returns: ** -** If zNum is exactly 9223372036854775808, return 2. This special -** case is broken out because while 9223372036854775808 cannot be a -** signed 64-bit integer, its negative -9223372036854775808 can be. -** -** If zNum is too big for a 64-bit integer and is not -** 9223372036854775808 or if zNum contains any non-numeric text, -** then return 1. +** -1 Not even a prefix of the input text looks like an integer +** 0 Successful transformation. Fits in a 64-bit signed integer. +** 1 Excess non-space text after the integer value +** 2 Integer too large for a 64-bit signed integer or is malformed +** 3 Special case of 9223372036854775808 ** ** length is the number of bytes in the string (bytes, not characters). ** The string is not necessarily zero-terminated. The encoding is @@ -27551,6 +35949,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc int i; int c = 0; int nonNum = 0; /* True if input contains UTF16 with high byte non-zero */ + int rc; /* Baseline return code */ const char *zStart; const char *zEnd = zNum + length; assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); @@ -27558,6 +35957,7 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc incr = 1; }else{ incr = 2; + length &= ~1; assert( SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); for(i=3-enc; i='0' && c<='9'; i+=incr){ u = u*10 + c - '0'; } + testcase( i==18*incr ); + testcase( i==19*incr ); + testcase( i==20*incr ); if( u>LARGEST_INT64 ){ + /* This test and assignment is needed only to suppress UB warnings + ** from clang and -fsanitize=undefined. This test and assignment make + ** the code a little larger and slower, and no harm comes from omitting + ** them, but we must appease the undefined-behavior pharisees. */ *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; }else if( neg ){ *pNum = -(i64)u; }else{ *pNum = (i64)u; } - testcase( i==18 ); - testcase( i==19 ); - testcase( i==20 ); - if( &zNum[i]19*incr /* Too many digits */ - || nonNum /* UTF16 with high-order bytes non-zero */ - ){ - /* zNum is empty or contains non-numeric text or is longer - ** than 19 digits (thus guaranteeing that it is too large) */ - return 1; - }else if( i<19*incr ){ + rc = 0; + if( i==0 && zStart==zNum ){ /* No digits */ + rc = -1; + }else if( nonNum ){ /* UTF16 with high-order bytes non-zero */ + rc = 1; + }else if( &zNum[i]19*incr ? 1 : compare2pow63(zNum, incr); if( c<0 ){ /* zNum is less than 9223372036854775808 so it fits */ assert( u<=LARGEST_INT64 ); - return 0; - }else if( c>0 ){ - /* zNum is greater than 9223372036854775808 so it overflows */ - return 1; + return rc; }else{ - /* zNum is exactly 9223372036854775808. Fits if negative. The - ** special case 2 overflow if positive */ - assert( u-1==LARGEST_INT64 ); - return neg ? 0 : 2; + *pNum = neg ? SMALLEST_INT64 : LARGEST_INT64; + if( c>0 ){ + /* zNum is greater than 9223372036854775808 so it overflows */ + return 2; + }else{ + /* zNum is exactly 9223372036854775808. Fits if negative. The + ** special case 2 overflow if positive */ + assert( u-1==LARGEST_INT64 ); + return neg ? rc : 3; + } } } } @@ -27627,8 +36041,9 @@ SQLITE_PRIVATE int sqlite3Atoi64(const char *zNum, i64 *pNum, int length, u8 enc ** Returns: ** ** 0 Successful transformation. Fits in a 64-bit signed integer. -** 1 Integer too large for a 64-bit signed integer or is malformed -** 2 Special case of 9223372036854775808 +** 1 Excess text after the integer value +** 2 Integer too large for a 64-bit signed integer or is malformed +** 3 Special case of 9223372036854775808 */ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ #ifndef SQLITE_OMIT_HEX_INTEGER @@ -27642,11 +36057,15 @@ SQLITE_PRIVATE int sqlite3DecOrHexToI64(const char *z, i64 *pOut){ u = u*16 + sqlite3HexToInt(z[k]); } memcpy(pOut, &u, 8); - return (z[k]==0 && k-i<=16) ? 0 : 1; + if( k-i>16 ) return 2; + if( z[k]!=0 ) return 1; + return 0; }else #endif /* SQLITE_OMIT_HEX_INTEGER */ { - return sqlite3Atoi64(z, pOut, sqlite3Strlen30(z), SQLITE_UTF8); + int n = (int)(0x3fffffff&strspn(z,"+- \n\t0123456789")); + if( z[n] ) n++; + return sqlite3Atoi64(z, pOut, n, SQLITE_UTF8); } } @@ -27678,7 +36097,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ u32 u = 0; zNum += 2; while( zNum[0]=='0' ) zNum++; - for(i=0; sqlite3Isxdigit(zNum[i]) && i<8; i++){ + for(i=0; i<8 && sqlite3Isxdigit(zNum[i]); i++){ u = u*16 + sqlite3HexToInt(zNum[i]); } if( (u&0x80000000)==0 && sqlite3Isxdigit(zNum[i])==0 ){ @@ -27689,6 +36108,7 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ } } #endif + if( !sqlite3Isdigit(zNum[0]) ) return 0; while( zNum[0]=='0' ) zNum++; for(i=0; i<11 && (c = zNum[i] - '0')>=0 && c<=9; i++){ v = v*10 + c; @@ -27720,10 +36140,164 @@ SQLITE_PRIVATE int sqlite3GetInt32(const char *zNum, int *pValue){ */ SQLITE_PRIVATE int sqlite3Atoi(const char *z){ int x = 0; - if( z ) sqlite3GetInt32(z, &x); + sqlite3GetInt32(z, &x); return x; } +/* +** Decode a floating-point value into an approximate decimal +** representation. +** +** If iRound<=0 then round to -iRound significant digits to the +** the left of the decimal point, or to a maximum of mxRound total +** significant digits. +** +** If iRound>0 round to min(iRound,mxRound) significant digits total. +** +** mxRound must be positive. +** +** The significant digits of the decimal representation are +** stored in p->z[] which is a often (but not always) a pointer +** into the middle of p->zBuf[]. There are p->n significant digits. +** The p->z[] array is *not* zero-terminated. +*/ +SQLITE_PRIVATE void sqlite3FpDecode(FpDecode *p, double r, int iRound, int mxRound){ + int i; + u64 v; + int e, exp = 0; + double rr[2]; + + p->isSpecial = 0; + p->z = p->zBuf; + assert( mxRound>0 ); + + /* Convert negative numbers to positive. Deal with Infinity, 0.0, and + ** NaN. */ + if( r<0.0 ){ + p->sign = '-'; + r = -r; + }else if( r==0.0 ){ + p->sign = '+'; + p->n = 1; + p->iDP = 1; + p->z = "0"; + return; + }else{ + p->sign = '+'; + } + memcpy(&v,&r,8); + e = v>>52; + if( (e&0x7ff)==0x7ff ){ + p->isSpecial = 1 + (v!=0x7ff0000000000000LL); + p->n = 0; + p->iDP = 0; + return; + } + + /* Multiply r by powers of ten until it lands somewhere in between + ** 1.0e+19 and 1.0e+17. + ** + ** Use Dekker-style double-double computation to increase the + ** precision. + ** + ** The error terms on constants like 1.0e+100 computed using the + ** decimal extension, for example as follows: + ** + ** SELECT decimal_exp(decimal_sub('1.0e+100',decimal(1.0e+100))); + */ + rr[0] = r; + rr[1] = 0.0; + if( rr[0]>9.223372036854774784e+18 ){ + while( rr[0]>9.223372036854774784e+118 ){ + exp += 100; + dekkerMul2(rr, 1.0e-100, -1.99918998026028836196e-117); + } + while( rr[0]>9.223372036854774784e+28 ){ + exp += 10; + dekkerMul2(rr, 1.0e-10, -3.6432197315497741579e-27); + } + while( rr[0]>9.223372036854774784e+18 ){ + exp += 1; + dekkerMul2(rr, 1.0e-01, -5.5511151231257827021e-18); + } + }else{ + while( rr[0]<9.223372036854774784e-83 ){ + exp -= 100; + dekkerMul2(rr, 1.0e+100, -1.5902891109759918046e+83); + } + while( rr[0]<9.223372036854774784e+07 ){ + exp -= 10; + dekkerMul2(rr, 1.0e+10, 0.0); + } + while( rr[0]<9.22337203685477478e+17 ){ + exp -= 1; + dekkerMul2(rr, 1.0e+01, 0.0); + } + } + v = rr[1]<0.0 ? (u64)rr[0]-(u64)(-rr[1]) : (u64)rr[0]+(u64)rr[1]; + + /* Extract significant digits. */ + i = sizeof(p->zBuf)-1; + assert( v>0 ); + while( v ){ p->zBuf[i--] = (v%10) + '0'; v /= 10; } + assert( i>=0 && izBuf)-1 ); + p->n = sizeof(p->zBuf) - 1 - i; + assert( p->n>0 ); + assert( p->nzBuf) ); + p->iDP = p->n + exp; + if( iRound<=0 ){ + iRound = p->iDP - iRound; + if( iRound==0 && p->zBuf[i+1]>='5' ){ + iRound = 1; + p->zBuf[i--] = '0'; + p->n++; + p->iDP++; + } + } + if( iRound>0 && (iRoundn || p->n>mxRound) ){ + char *z = &p->zBuf[i+1]; + if( iRound>mxRound ) iRound = mxRound; + p->n = iRound; + if( z[iRound]>='5' ){ + int j = iRound-1; + while( 1 /*exit-by-break*/ ){ + z[j]++; + if( z[j]<='9' ) break; + z[j] = '0'; + if( j==0 ){ + p->z[i--] = '1'; + p->n++; + p->iDP++; + break; + }else{ + j--; + } + } + } + } + p->z = &p->zBuf[i+1]; + assert( i+p->n < sizeof(p->zBuf) ); + while( ALWAYS(p->n>0) && p->z[p->n-1]=='0' ){ p->n--; } +} + +/* +** Try to convert z into an unsigned 32-bit integer. Return true on +** success and false if there is an error. +** +** Only decimal notation is accepted. +*/ +SQLITE_PRIVATE int sqlite3GetUInt32(const char *z, u32 *pI){ + u64 v = 0; + int i; + for(i=0; sqlite3Isdigit(z[i]); i++){ + v = v*10 + z[i] - '0'; + if( v>4294967296LL ){ *pI = 0; return 0; } + } + if( i==0 || z[i]!=0 ){ *pI = 0; return 0; } + *pI = (u32)v; + return 1; +} + /* ** The variable-length integer encoding is as follows: ** @@ -27764,7 +36338,7 @@ static int SQLITE_NOINLINE putVarint64(unsigned char *p, u64 v){ v >>= 7; } return 9; - } + } n = 0; do{ buf[n++] = (u8)((v & 0x7f) | 0x80); @@ -27810,23 +36384,12 @@ SQLITE_PRIVATE int sqlite3PutVarint(unsigned char *p, u64 v){ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ u32 a,b,s; - a = *p; - /* a: p0 (unmasked) */ - if (!(a&0x80)) - { - *v = a; + if( ((signed char*)p)[0]>=0 ){ + *v = *p; return 1; } - - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - a &= 0x7f; - a = a<<7; - a |= b; - *v = a; + if( ((signed char*)p)[1]>=0 ){ + *v = ((u32)(p[0]&0x7f)<<7) | p[1]; return 2; } @@ -27834,8 +36397,9 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ assert( SLOT_2_0 == ((0x7f<<14) | (0x7f)) ); assert( SLOT_4_2_0 == ((0xfU<<28) | (0x7f<<14) | (0x7f)) ); - p++; - a = a<<14; + a = ((u32)p[0])<<14; + b = p[1]; + p += 2; a |= *p; /* a: p0<<14 | p2 (unmasked) */ if (!(a&0x80)) @@ -27974,127 +36538,37 @@ SQLITE_PRIVATE u8 sqlite3GetVarint(const unsigned char *p, u64 *v){ ** If the varint stored in p[0] is larger than can fit in a 32-bit unsigned ** integer, then set *v to 0xffffffff. ** -** A MACRO version, getVarint32, is provided which inlines the -** single-byte case. All code should use the MACRO version as +** A MACRO version, getVarint32, is provided which inlines the +** single-byte case. All code should use the MACRO version as ** this function assumes the single-byte case has already been handled. */ SQLITE_PRIVATE u8 sqlite3GetVarint32(const unsigned char *p, u32 *v){ - u32 a,b; + u64 v64; + u8 n; - /* The 1-byte case. Overwhelmingly the most common. Handled inline - ** by the getVarin32() macro */ - a = *p; - /* a: p0 (unmasked) */ -#ifndef getVarint32 - if (!(a&0x80)) - { - /* Values between 0 and 127 */ - *v = a; - return 1; - } -#endif + /* Assume that the single-byte case has already been handled by + ** the getVarint32() macro */ + assert( (p[0] & 0x80)!=0 ); - /* The 2-byte case */ - p++; - b = *p; - /* b: p1 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 128 and 16383 */ - a &= 0x7f; - a = a<<7; - *v = a | b; + if( (p[1] & 0x80)==0 ){ + /* This is the two-byte case */ + *v = ((p[0]&0x7f)<<7) | p[1]; return 2; } - - /* The 3-byte case */ - p++; - a = a<<14; - a |= *p; - /* a: p0<<14 | p2 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 16384 and 2097151 */ - a &= (0x7f<<14)|(0x7f); - b &= 0x7f; - b = b<<7; - *v = a | b; + if( (p[2] & 0x80)==0 ){ + /* This is the three-byte case */ + *v = ((p[0]&0x7f)<<14) | ((p[1]&0x7f)<<7) | p[2]; return 3; } - - /* A 32-bit varint is used to store size information in btrees. - ** Objects are rarely larger than 2MiB limit of a 3-byte varint. - ** A 3-byte varint is sufficient, for example, to record the size - ** of a 1048569-byte BLOB or string. - ** - ** We only unroll the first 1-, 2-, and 3- byte cases. The very - ** rare larger cases can be handled by the slower 64-bit varint - ** routine. - */ -#if 1 - { - u64 v64; - u8 n; - - p -= 2; - n = sqlite3GetVarint(p, &v64); - assert( n>3 && n<=9 ); - if( (v64 & SQLITE_MAX_U32)!=v64 ){ - *v = 0xffffffff; - }else{ - *v = (u32)v64; - } - return n; - } - -#else - /* For following code (kept for historical record only) shows an - ** unrolling for the 3- and 4-byte varint cases. This code is - ** slightly faster, but it is also larger and much harder to test. - */ - p++; - b = b<<14; - b |= *p; - /* b: p1<<14 | p3 (unmasked) */ - if (!(b&0x80)) - { - /* Values between 2097152 and 268435455 */ - b &= (0x7f<<14)|(0x7f); - a &= (0x7f<<14)|(0x7f); - a = a<<7; - *v = a | b; - return 4; - } - - p++; - a = a<<14; - a |= *p; - /* a: p0<<28 | p2<<14 | p4 (unmasked) */ - if (!(a&0x80)) - { - /* Values between 268435456 and 34359738367 */ - a &= SLOT_4_2_0; - b &= SLOT_4_2_0; - b = b<<7; - *v = a | b; - return 5; - } - - /* We can only reach this point when reading a corrupt database - ** file. In that case we are not in any hurry. Use the (relatively - ** slow) general-purpose sqlite3GetVarint() routine to extract the - ** value. */ - { - u64 v64; - u8 n; - - p -= 4; - n = sqlite3GetVarint(p, &v64); - assert( n>5 && n<=9 ); + /* four or more bytes */ + n = sqlite3GetVarint(p, &v64); + assert( n>3 && n<=9 ); + if( (v64 & SQLITE_MAX_U32)!=v64 ){ + *v = 0xffffffff; + }else{ *v = (u32)v64; - return n; } -#endif + return n; } /* @@ -28116,13 +36590,11 @@ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ u32 x; memcpy(&x,p,4); return x; -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(__GNUC__) && GCC_VERSION>=4003000 +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u32 x; memcpy(&x,p,4); return __builtin_bswap32(x); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u32 x; memcpy(&x,p,4); return _byteswap_ulong(x); @@ -28134,12 +36606,10 @@ SQLITE_PRIVATE u32 sqlite3Get4byte(const u8 *p){ SQLITE_PRIVATE void sqlite3Put4byte(unsigned char *p, u32 v){ #if SQLITE_BYTEORDER==4321 memcpy(p,&v,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(__GNUC__) && GCC_VERSION>=4003000 +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 u32 x = __builtin_bswap32(v); memcpy(p,&x,4); -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 u32 x = _byteswap_ulong(v); memcpy(p,&x,4); #else @@ -28168,7 +36638,7 @@ SQLITE_PRIVATE u8 sqlite3HexToInt(int h){ return (u8)(h & 0xf); } -#if !defined(SQLITE_OMIT_BLOB_LITERAL) || defined(SQLITE_HAS_CODEC) +#if !defined(SQLITE_OMIT_BLOB_LITERAL) /* ** Convert a BLOB literal of the form "x'hhhhhh'" into its binary ** value. Return a pointer to its binary value. Space to hold the @@ -28189,7 +36659,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ } return zBlob; } -#endif /* !SQLITE_OMIT_BLOB_LITERAL || SQLITE_HAS_CODEC */ +#endif /* !SQLITE_OMIT_BLOB_LITERAL */ /* ** Log an error that is an API call on a connection pointer that should @@ -28197,7 +36667,7 @@ SQLITE_PRIVATE void *sqlite3HexToBlob(sqlite3 *db, const char *z, int n){ ** argument. The zType is a word like "NULL" or "closed" or "invalid". */ static void logBadConnection(const char *zType){ - sqlite3_log(SQLITE_MISUSE, + sqlite3_log(SQLITE_MISUSE, "API call with %s database connection pointer", zType ); @@ -28218,13 +36688,13 @@ static void logBadConnection(const char *zType){ ** used as an argument to sqlite3_errmsg() or sqlite3_close(). */ SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3 *db){ - u32 magic; + u8 eOpenState; if( db==0 ){ logBadConnection("NULL"); return 0; } - magic = db->magic; - if( magic!=SQLITE_MAGIC_OPEN ){ + eOpenState = db->eOpenState; + if( eOpenState!=SQLITE_STATE_OPEN ){ if( sqlite3SafetyCheckSickOrOk(db) ){ testcase( sqlite3GlobalConfig.xLog!=0 ); logBadConnection("unopened"); @@ -28235,11 +36705,11 @@ SQLITE_PRIVATE int sqlite3SafetyCheckOk(sqlite3 *db){ } } SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ - u32 magic; - magic = db->magic; - if( magic!=SQLITE_MAGIC_SICK && - magic!=SQLITE_MAGIC_OPEN && - magic!=SQLITE_MAGIC_BUSY ){ + u8 eOpenState; + eOpenState = db->eOpenState; + if( eOpenState!=SQLITE_STATE_SICK && + eOpenState!=SQLITE_STATE_OPEN && + eOpenState!=SQLITE_STATE_BUSY ){ testcase( sqlite3GlobalConfig.xLog!=0 ); logBadConnection("invalid"); return 0; @@ -28249,12 +36719,15 @@ SQLITE_PRIVATE int sqlite3SafetyCheckSickOrOk(sqlite3 *db){ } /* -** Attempt to add, substract, or multiply the 64-bit signed value iB against +** Attempt to add, subtract, or multiply the 64-bit signed value iB against ** the other 64-bit signed integer at *pA and store the result in *pA. ** Return 0 on success. Or if the operation would have resulted in an ** overflow, leave *pA unchanged and return 1. */ SQLITE_PRIVATE int sqlite3AddInt64(i64 *pA, i64 iB){ +#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER) + return __builtin_add_overflow(*pA, iB, pA); +#else i64 iA = *pA; testcase( iA==0 ); testcase( iA==1 ); testcase( iB==-1 ); testcase( iB==0 ); @@ -28268,9 +36741,13 @@ SQLITE_PRIVATE int sqlite3AddInt64(i64 *pA, i64 iB){ if( iA<0 && -(iA + LARGEST_INT64) > iB + 1 ) return 1; } *pA += iB; - return 0; + return 0; +#endif } SQLITE_PRIVATE int sqlite3SubInt64(i64 *pA, i64 iB){ +#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER) + return __builtin_sub_overflow(*pA, iB, pA); +#else testcase( iB==SMALLEST_INT64+1 ); if( iB==SMALLEST_INT64 ){ testcase( (*pA)==(-1) ); testcase( (*pA)==0 ); @@ -28280,42 +36757,32 @@ SQLITE_PRIVATE int sqlite3SubInt64(i64 *pA, i64 iB){ }else{ return sqlite3AddInt64(pA, -iB); } +#endif } -#define TWOPOWER32 (((i64)1)<<32) -#define TWOPOWER31 (((i64)1)<<31) SQLITE_PRIVATE int sqlite3MulInt64(i64 *pA, i64 iB){ +#if GCC_VERSION>=5004000 && !defined(__INTEL_COMPILER) + return __builtin_mul_overflow(*pA, iB, pA); +#else i64 iA = *pA; - i64 iA1, iA0, iB1, iB0, r; - - iA1 = iA/TWOPOWER32; - iA0 = iA % TWOPOWER32; - iB1 = iB/TWOPOWER32; - iB0 = iB % TWOPOWER32; - if( iA1==0 ){ - if( iB1==0 ){ - *pA *= iB; - return 0; + if( iB>0 ){ + if( iA>LARGEST_INT64/iB ) return 1; + if( iA0 ){ + if( iBLARGEST_INT64/-iB ) return 1; } - r = iA0*iB1; - }else if( iB1==0 ){ - r = iA1*iB0; - }else{ - /* If both iA1 and iB1 are non-zero, overflow will result */ - return 1; } - testcase( r==(-TWOPOWER31)-1 ); - testcase( r==(-TWOPOWER31) ); - testcase( r==TWOPOWER31 ); - testcase( r==TWOPOWER31-1 ); - if( r<(-TWOPOWER31) || r>=TWOPOWER31 ) return 1; - r *= TWOPOWER32; - if( sqlite3AddInt64(&r, iA0*iB0) ) return 1; - *pA = r; + *pA = iA*iB; return 0; +#endif } /* -** Compute the absolute value of a 32-bit signed integer, of possible. Or +** Compute the absolute value of a 32-bit signed integer, of possible. Or ** if the integer has a value of -2147483648, return +2147483647 */ SQLITE_PRIVATE int sqlite3AbsInt32(int x){ @@ -28355,11 +36822,11 @@ SQLITE_PRIVATE void sqlite3FileSuffix3(const char *zBaseFilename, char *z){ } #endif -/* +/* ** Find (an approximate) sum of two LogEst values. This computation is ** not a simple "+" operator because LogEst is stored as a logarithmic ** value. -** +** */ SQLITE_PRIVATE LogEst sqlite3LogEstAdd(LogEst a, LogEst b){ static const unsigned char x[] = { @@ -28395,13 +36862,18 @@ SQLITE_PRIVATE LogEst sqlite3LogEst(u64 x){ if( x<2 ) return 0; while( x<8 ){ y -= 10; x <<= 1; } }else{ +#if GCC_VERSION>=5004000 + int i = 60 - __builtin_clzll(x); + y += i*10; + x >>= i; +#else while( x>255 ){ y += 40; x >>= 4; } /*OPTIMIZATION-IF-TRUE*/ while( x>15 ){ y += 10; x >>= 1; } +#endif } return a[x&7] + y - 10; } -#ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Convert a double into a LogEst ** In other words, compute an approximation for 10*log2(x). @@ -28416,16 +36888,9 @@ SQLITE_PRIVATE LogEst sqlite3LogEstFromDouble(double x){ e = (a>>52) - 1022; return e*10; } -#endif /* SQLITE_OMIT_VIRTUALTABLE */ -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_ENABLE_STAT3_OR_STAT4) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) /* ** Convert a LogEst into an integer. -** -** Note that this routine is only used when one or more of various -** non-standard compile-time options is enabled. */ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ u64 n; @@ -28433,17 +36898,112 @@ SQLITE_PRIVATE u64 sqlite3LogEstToInt(LogEst x){ x /= 10; if( n>=5 ) n -= 2; else if( n>=1 ) n -= 1; -#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || \ - defined(SQLITE_EXPLAIN_ESTIMATED_ROWS) if( x>60 ) return (u64)LARGEST_INT64; -#else - /* If only SQLITE_ENABLE_STAT3_OR_STAT4 is on, then the largest input - ** possible to this routine is 310, resulting in a maximum x of 31 */ - assert( x<=60 ); -#endif return x>=3 ? (n+8)<<(x-3) : (n+8)>>(3-x); } -#endif /* defined SCANSTAT or STAT4 or ESTIMATED_ROWS */ + +/* +** Add a new name/number pair to a VList. This might require that the +** VList object be reallocated, so return the new VList. If an OOM +** error occurs, the original VList returned and the +** db->mallocFailed flag is set. +** +** A VList is really just an array of integers. To destroy a VList, +** simply pass it to sqlite3DbFree(). +** +** The first integer is the number of integers allocated for the whole +** VList. The second integer is the number of integers actually used. +** Each name/number pair is encoded by subsequent groups of 3 or more +** integers. +** +** Each name/number pair starts with two integers which are the numeric +** value for the pair and the size of the name/number pair, respectively. +** The text name overlays one or more following integers. The text name +** is always zero-terminated. +** +** Conceptually: +** +** struct VList { +** int nAlloc; // Number of allocated slots +** int nUsed; // Number of used slots +** struct VListEntry { +** int iValue; // Value for this entry +** int nSlot; // Slots used by this entry +** // ... variable name goes here +** } a[0]; +** } +** +** During code generation, pointers to the variable names within the +** VList are taken. When that happens, nAlloc is set to zero as an +** indication that the VList may never again be enlarged, since the +** accompanying realloc() would invalidate the pointers. +*/ +SQLITE_PRIVATE VList *sqlite3VListAdd( + sqlite3 *db, /* The database connection used for malloc() */ + VList *pIn, /* The input VList. Might be NULL */ + const char *zName, /* Name of symbol to add */ + int nName, /* Bytes of text in zName */ + int iVal /* Value to associate with zName */ +){ + int nInt; /* number of sizeof(int) objects needed for zName */ + char *z; /* Pointer to where zName will be stored */ + int i; /* Index in pIn[] where zName is stored */ + + nInt = nName/4 + 3; + assert( pIn==0 || pIn[0]>=3 ); /* Verify ok to add new elements */ + if( pIn==0 || pIn[1]+nInt > pIn[0] ){ + /* Enlarge the allocation */ + sqlite3_int64 nAlloc = (pIn ? 2*(sqlite3_int64)pIn[0] : 10) + nInt; + VList *pOut = sqlite3DbRealloc(db, pIn, nAlloc*sizeof(int)); + if( pOut==0 ) return pIn; + if( pIn==0 ) pOut[1] = 2; + pIn = pOut; + pIn[0] = nAlloc; + } + i = pIn[1]; + pIn[i] = iVal; + pIn[i+1] = nInt; + z = (char*)&pIn[i+2]; + pIn[1] = i+nInt; + assert( pIn[1]<=pIn[0] ); + memcpy(z, zName, nName); + z[nName] = 0; + return pIn; +} + +/* +** Return a pointer to the name of a variable in the given VList that +** has the value iVal. Or return a NULL if there is no such variable in +** the list +*/ +SQLITE_PRIVATE const char *sqlite3VListNumToName(VList *pIn, int iVal){ + int i, mx; + if( pIn==0 ) return 0; + mx = pIn[1]; + i = 2; + do{ + if( pIn[i]==iVal ) return (char*)&pIn[i+2]; + i += pIn[i+1]; + }while( iht ){ /*OPTIMIZATION-IF-TRUE*/ struct _ht *pEntry; @@ -28608,15 +37174,16 @@ static HashElem *findElementWithHash( elem = pH->first; count = pH->count; } - *pHash = h; - while( count-- ){ + if( pHash ) *pHash = h; + while( count ){ assert( elem!=0 ); - if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ + if( sqlite3StrICmp(elem->pKey,pKey)==0 ){ return elem; } elem = elem->next; + count--; } - return 0; + return &nullElement; } /* Remove a single entry from the hash table given a pointer to that @@ -28629,7 +37196,7 @@ static void removeElementGivenHash( ){ struct _ht *pEntry; if( elem->prev ){ - elem->prev->next = elem->next; + elem->prev->next = elem->next; }else{ pH->first = elem->next; } @@ -28641,8 +37208,8 @@ static void removeElementGivenHash( if( pEntry->chain==elem ){ pEntry->chain = elem->next; } + assert( pEntry->count>0 ); pEntry->count--; - assert( pEntry->count>=0 ); } sqlite3_free( elem ); pH->count--; @@ -28658,13 +37225,9 @@ static void removeElementGivenHash( ** found, or NULL if there is no match. */ SQLITE_PRIVATE void *sqlite3HashFind(const Hash *pH, const char *pKey){ - HashElem *elem; /* The element that matches key */ - unsigned int h; /* A hash on key */ - assert( pH!=0 ); assert( pKey!=0 ); - elem = findElementWithHash(pH, pKey, &h); - return elem ? elem->data : 0; + return findElementWithHash(pH, pKey, 0)->data; } /* Insert an element into the hash table pH. The key is pKey @@ -28689,7 +37252,7 @@ SQLITE_PRIVATE void *sqlite3HashInsert(Hash *pH, const char *pKey, void *data){ assert( pH!=0 ); assert( pKey!=0 ); elem = findElementWithHash(pH,pKey,&h); - if( elem ){ + if( elem->data ){ void *old_data = elem->data; if( data==0 ){ removeElementGivenHash(pH,elem,h); @@ -28732,170 +37295,1181 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* 0 */ "Savepoint" OpHelp(""), /* 1 */ "AutoCommit" OpHelp(""), /* 2 */ "Transaction" OpHelp(""), - /* 3 */ "SorterNext" OpHelp(""), - /* 4 */ "PrevIfOpen" OpHelp(""), - /* 5 */ "NextIfOpen" OpHelp(""), - /* 6 */ "Prev" OpHelp(""), - /* 7 */ "Next" OpHelp(""), - /* 8 */ "Checkpoint" OpHelp(""), - /* 9 */ "JournalMode" OpHelp(""), - /* 10 */ "Vacuum" OpHelp(""), - /* 11 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), - /* 12 */ "VUpdate" OpHelp("data=r[P3@P2]"), - /* 13 */ "Goto" OpHelp(""), - /* 14 */ "Gosub" OpHelp(""), - /* 15 */ "InitCoroutine" OpHelp(""), - /* 16 */ "Yield" OpHelp(""), - /* 17 */ "MustBeInt" OpHelp(""), - /* 18 */ "Jump" OpHelp(""), + /* 3 */ "Checkpoint" OpHelp(""), + /* 4 */ "JournalMode" OpHelp(""), + /* 5 */ "Vacuum" OpHelp(""), + /* 6 */ "VFilter" OpHelp("iplan=r[P3] zplan='P4'"), + /* 7 */ "VUpdate" OpHelp("data=r[P3@P2]"), + /* 8 */ "Init" OpHelp("Start at P2"), + /* 9 */ "Goto" OpHelp(""), + /* 10 */ "Gosub" OpHelp(""), + /* 11 */ "InitCoroutine" OpHelp(""), + /* 12 */ "Yield" OpHelp(""), + /* 13 */ "MustBeInt" OpHelp(""), + /* 14 */ "Jump" OpHelp(""), + /* 15 */ "Once" OpHelp(""), + /* 16 */ "If" OpHelp(""), + /* 17 */ "IfNot" OpHelp(""), + /* 18 */ "IsType" OpHelp("if typeof(P1.P3) in P5 goto P2"), /* 19 */ "Not" OpHelp("r[P2]= !r[P1]"), - /* 20 */ "Once" OpHelp(""), - /* 21 */ "If" OpHelp(""), - /* 22 */ "IfNot" OpHelp(""), - /* 23 */ "SeekLT" OpHelp("key=r[P3@P4]"), - /* 24 */ "SeekLE" OpHelp("key=r[P3@P4]"), - /* 25 */ "SeekGE" OpHelp("key=r[P3@P4]"), - /* 26 */ "SeekGT" OpHelp("key=r[P3@P4]"), - /* 27 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), - /* 28 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), - /* 29 */ "NoConflict" OpHelp("key=r[P3@P4]"), - /* 30 */ "NotFound" OpHelp("key=r[P3@P4]"), - /* 31 */ "Found" OpHelp("key=r[P3@P4]"), - /* 32 */ "NotExists" OpHelp("intkey=r[P3]"), - /* 33 */ "Last" OpHelp(""), - /* 34 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), - /* 35 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), - /* 36 */ "Ne" OpHelp("if r[P1]!=r[P3] goto P2"), - /* 37 */ "Eq" OpHelp("if r[P1]==r[P3] goto P2"), - /* 38 */ "Gt" OpHelp("if r[P1]>r[P3] goto P2"), - /* 39 */ "Le" OpHelp("if r[P1]<=r[P3] goto P2"), - /* 40 */ "Lt" OpHelp("if r[P1]=r[P3] goto P2"), - /* 42 */ "SorterSort" OpHelp(""), - /* 43 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), - /* 44 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), - /* 45 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), - /* 47 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), - /* 48 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), - /* 49 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), - /* 50 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), - /* 51 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), - /* 52 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), - /* 53 */ "Sort" OpHelp(""), - /* 54 */ "BitNot" OpHelp("r[P1]= ~r[P1]"), - /* 55 */ "Rewind" OpHelp(""), - /* 56 */ "IdxLE" OpHelp("key=r[P3@P4]"), - /* 57 */ "IdxGT" OpHelp("key=r[P3@P4]"), - /* 58 */ "IdxLT" OpHelp("key=r[P3@P4]"), - /* 59 */ "IdxGE" OpHelp("key=r[P3@P4]"), - /* 60 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), - /* 61 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), - /* 62 */ "Program" OpHelp(""), - /* 63 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), - /* 64 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), - /* 65 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]-=P3, goto P2"), - /* 66 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), - /* 67 */ "IncrVacuum" OpHelp(""), - /* 68 */ "VNext" OpHelp(""), - /* 69 */ "Init" OpHelp("Start at P2"), - /* 70 */ "Return" OpHelp(""), - /* 71 */ "EndCoroutine" OpHelp(""), - /* 72 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), - /* 73 */ "Halt" OpHelp(""), - /* 74 */ "Integer" OpHelp("r[P2]=P1"), - /* 75 */ "Int64" OpHelp("r[P2]=P4"), - /* 76 */ "String" OpHelp("r[P2]='P4' (len=P1)"), - /* 77 */ "Null" OpHelp("r[P2..P3]=NULL"), - /* 78 */ "SoftNull" OpHelp("r[P1]=NULL"), - /* 79 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), - /* 80 */ "Variable" OpHelp("r[P2]=parameter(P1,P4)"), - /* 81 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), - /* 82 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), - /* 83 */ "SCopy" OpHelp("r[P2]=r[P1]"), - /* 84 */ "IntCopy" OpHelp("r[P2]=r[P1]"), - /* 85 */ "ResultRow" OpHelp("output=r[P1@P2]"), - /* 86 */ "CollSeq" OpHelp(""), - /* 87 */ "Function0" OpHelp("r[P3]=func(r[P2@P5])"), - /* 88 */ "Function" OpHelp("r[P3]=func(r[P2@P5])"), - /* 89 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), - /* 90 */ "RealAffinity" OpHelp(""), - /* 91 */ "Cast" OpHelp("affinity(r[P1])"), - /* 92 */ "Permutation" OpHelp(""), - /* 93 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), - /* 94 */ "Column" OpHelp("r[P3]=PX"), - /* 95 */ "Affinity" OpHelp("affinity(r[P1@P2])"), - /* 96 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), - /* 97 */ "String8" OpHelp("r[P2]='P4'"), + /* 20 */ "IfNullRow" OpHelp("if P1.nullRow then r[P3]=NULL, goto P2"), + /* 21 */ "SeekLT" OpHelp("key=r[P3@P4]"), + /* 22 */ "SeekLE" OpHelp("key=r[P3@P4]"), + /* 23 */ "SeekGE" OpHelp("key=r[P3@P4]"), + /* 24 */ "SeekGT" OpHelp("key=r[P3@P4]"), + /* 25 */ "IfNotOpen" OpHelp("if( !csr[P1] ) goto P2"), + /* 26 */ "IfNoHope" OpHelp("key=r[P3@P4]"), + /* 27 */ "NoConflict" OpHelp("key=r[P3@P4]"), + /* 28 */ "NotFound" OpHelp("key=r[P3@P4]"), + /* 29 */ "Found" OpHelp("key=r[P3@P4]"), + /* 30 */ "SeekRowid" OpHelp("intkey=r[P3]"), + /* 31 */ "NotExists" OpHelp("intkey=r[P3]"), + /* 32 */ "Last" OpHelp(""), + /* 33 */ "IfSizeBetween" OpHelp(""), + /* 34 */ "SorterSort" OpHelp(""), + /* 35 */ "Sort" OpHelp(""), + /* 36 */ "Rewind" OpHelp(""), + /* 37 */ "SorterNext" OpHelp(""), + /* 38 */ "Prev" OpHelp(""), + /* 39 */ "Next" OpHelp(""), + /* 40 */ "IdxLE" OpHelp("key=r[P3@P4]"), + /* 41 */ "IdxGT" OpHelp("key=r[P3@P4]"), + /* 42 */ "IdxLT" OpHelp("key=r[P3@P4]"), + /* 43 */ "Or" OpHelp("r[P3]=(r[P1] || r[P2])"), + /* 44 */ "And" OpHelp("r[P3]=(r[P1] && r[P2])"), + /* 45 */ "IdxGE" OpHelp("key=r[P3@P4]"), + /* 46 */ "RowSetRead" OpHelp("r[P3]=rowset(P1)"), + /* 47 */ "RowSetTest" OpHelp("if r[P3] in rowset(P1) goto P2"), + /* 48 */ "Program" OpHelp(""), + /* 49 */ "FkIfZero" OpHelp("if fkctr[P1]==0 goto P2"), + /* 50 */ "IfPos" OpHelp("if r[P1]>0 then r[P1]-=P3, goto P2"), + /* 51 */ "IsNull" OpHelp("if r[P1]==NULL goto P2"), + /* 52 */ "NotNull" OpHelp("if r[P1]!=NULL goto P2"), + /* 53 */ "Ne" OpHelp("IF r[P3]!=r[P1]"), + /* 54 */ "Eq" OpHelp("IF r[P3]==r[P1]"), + /* 55 */ "Gt" OpHelp("IF r[P3]>r[P1]"), + /* 56 */ "Le" OpHelp("IF r[P3]<=r[P1]"), + /* 57 */ "Lt" OpHelp("IF r[P3]=r[P1]"), + /* 59 */ "ElseEq" OpHelp(""), + /* 60 */ "IfNotZero" OpHelp("if r[P1]!=0 then r[P1]--, goto P2"), + /* 61 */ "DecrJumpZero" OpHelp("if (--r[P1])==0 goto P2"), + /* 62 */ "IncrVacuum" OpHelp(""), + /* 63 */ "VNext" OpHelp(""), + /* 64 */ "Filter" OpHelp("if key(P3@P4) not in filter(P1) goto P2"), + /* 65 */ "PureFunc" OpHelp("r[P3]=func(r[P2@NP])"), + /* 66 */ "Function" OpHelp("r[P3]=func(r[P2@NP])"), + /* 67 */ "Return" OpHelp(""), + /* 68 */ "EndCoroutine" OpHelp(""), + /* 69 */ "HaltIfNull" OpHelp("if r[P3]=null halt"), + /* 70 */ "Halt" OpHelp(""), + /* 71 */ "Integer" OpHelp("r[P2]=P1"), + /* 72 */ "Int64" OpHelp("r[P2]=P4"), + /* 73 */ "String" OpHelp("r[P2]='P4' (len=P1)"), + /* 74 */ "BeginSubrtn" OpHelp("r[P2]=NULL"), + /* 75 */ "Null" OpHelp("r[P2..P3]=NULL"), + /* 76 */ "SoftNull" OpHelp("r[P1]=NULL"), + /* 77 */ "Blob" OpHelp("r[P2]=P4 (len=P1)"), + /* 78 */ "Variable" OpHelp("r[P2]=parameter(P1)"), + /* 79 */ "Move" OpHelp("r[P2@P3]=r[P1@P3]"), + /* 80 */ "Copy" OpHelp("r[P2@P3+1]=r[P1@P3+1]"), + /* 81 */ "SCopy" OpHelp("r[P2]=r[P1]"), + /* 82 */ "IntCopy" OpHelp("r[P2]=r[P1]"), + /* 83 */ "FkCheck" OpHelp(""), + /* 84 */ "ResultRow" OpHelp("output=r[P1@P2]"), + /* 85 */ "CollSeq" OpHelp(""), + /* 86 */ "AddImm" OpHelp("r[P1]=r[P1]+P2"), + /* 87 */ "RealAffinity" OpHelp(""), + /* 88 */ "Cast" OpHelp("affinity(r[P1])"), + /* 89 */ "Permutation" OpHelp(""), + /* 90 */ "Compare" OpHelp("r[P1@P3] <-> r[P2@P3]"), + /* 91 */ "IsTrue" OpHelp("r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4"), + /* 92 */ "ZeroOrNull" OpHelp("r[P2] = 0 OR NULL"), + /* 93 */ "Offset" OpHelp("r[P3] = sqlite_offset(P1)"), + /* 94 */ "Column" OpHelp("r[P3]=PX cursor P1 column P2"), + /* 95 */ "TypeCheck" OpHelp("typecheck(r[P1@P2])"), + /* 96 */ "Affinity" OpHelp("affinity(r[P1@P2])"), + /* 97 */ "MakeRecord" OpHelp("r[P3]=mkrec(r[P1@P2])"), /* 98 */ "Count" OpHelp("r[P2]=count()"), /* 99 */ "ReadCookie" OpHelp(""), /* 100 */ "SetCookie" OpHelp(""), /* 101 */ "ReopenIdx" OpHelp("root=P2 iDb=P3"), /* 102 */ "OpenRead" OpHelp("root=P2 iDb=P3"), - /* 103 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), - /* 104 */ "OpenAutoindex" OpHelp("nColumn=P2"), - /* 105 */ "OpenEphemeral" OpHelp("nColumn=P2"), - /* 106 */ "SorterOpen" OpHelp(""), - /* 107 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), - /* 108 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), - /* 109 */ "Close" OpHelp(""), - /* 110 */ "ColumnsUsed" OpHelp(""), - /* 111 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), - /* 112 */ "NewRowid" OpHelp("r[P2]=rowid"), - /* 113 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), - /* 114 */ "InsertInt" OpHelp("intkey=P3 data=r[P2]"), - /* 115 */ "Delete" OpHelp(""), - /* 116 */ "ResetCount" OpHelp(""), - /* 117 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), - /* 118 */ "SorterData" OpHelp("r[P2]=data"), - /* 119 */ "RowKey" OpHelp("r[P2]=key"), - /* 120 */ "RowData" OpHelp("r[P2]=data"), - /* 121 */ "Rowid" OpHelp("r[P2]=rowid"), - /* 122 */ "NullRow" OpHelp(""), - /* 123 */ "SorterInsert" OpHelp(""), - /* 124 */ "IdxInsert" OpHelp("key=r[P2]"), - /* 125 */ "IdxDelete" OpHelp("key=r[P2@P3]"), - /* 126 */ "Seek" OpHelp("Move P3 to P1.rowid"), - /* 127 */ "IdxRowid" OpHelp("r[P2]=rowid"), - /* 128 */ "Destroy" OpHelp(""), - /* 129 */ "Clear" OpHelp(""), - /* 130 */ "ResetSorter" OpHelp(""), - /* 131 */ "CreateIndex" OpHelp("r[P2]=root iDb=P1"), - /* 132 */ "CreateTable" OpHelp("r[P2]=root iDb=P1"), - /* 133 */ "Real" OpHelp("r[P2]=P4"), - /* 134 */ "ParseSchema" OpHelp(""), - /* 135 */ "LoadAnalysis" OpHelp(""), - /* 136 */ "DropTable" OpHelp(""), - /* 137 */ "DropIndex" OpHelp(""), - /* 138 */ "DropTrigger" OpHelp(""), - /* 139 */ "IntegrityCk" OpHelp(""), - /* 140 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), - /* 141 */ "Param" OpHelp(""), - /* 142 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), - /* 143 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), - /* 144 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), - /* 145 */ "AggStep0" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 146 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), - /* 147 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), - /* 148 */ "Expire" OpHelp(""), - /* 149 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), - /* 150 */ "VBegin" OpHelp(""), - /* 151 */ "VCreate" OpHelp(""), - /* 152 */ "VDestroy" OpHelp(""), - /* 153 */ "VOpen" OpHelp(""), - /* 154 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), - /* 155 */ "VRename" OpHelp(""), - /* 156 */ "Pagecount" OpHelp(""), - /* 157 */ "MaxPgcnt" OpHelp(""), - /* 158 */ "CursorHint" OpHelp(""), - /* 159 */ "Noop" OpHelp(""), - /* 160 */ "Explain" OpHelp(""), + /* 103 */ "BitAnd" OpHelp("r[P3]=r[P1]&r[P2]"), + /* 104 */ "BitOr" OpHelp("r[P3]=r[P1]|r[P2]"), + /* 105 */ "ShiftLeft" OpHelp("r[P3]=r[P2]<>r[P1]"), + /* 107 */ "Add" OpHelp("r[P3]=r[P1]+r[P2]"), + /* 108 */ "Subtract" OpHelp("r[P3]=r[P2]-r[P1]"), + /* 109 */ "Multiply" OpHelp("r[P3]=r[P1]*r[P2]"), + /* 110 */ "Divide" OpHelp("r[P3]=r[P2]/r[P1]"), + /* 111 */ "Remainder" OpHelp("r[P3]=r[P2]%r[P1]"), + /* 112 */ "Concat" OpHelp("r[P3]=r[P2]+r[P1]"), + /* 113 */ "OpenWrite" OpHelp("root=P2 iDb=P3"), + /* 114 */ "OpenDup" OpHelp(""), + /* 115 */ "BitNot" OpHelp("r[P2]= ~r[P1]"), + /* 116 */ "OpenAutoindex" OpHelp("nColumn=P2"), + /* 117 */ "OpenEphemeral" OpHelp("nColumn=P2"), + /* 118 */ "String8" OpHelp("r[P2]='P4'"), + /* 119 */ "SorterOpen" OpHelp(""), + /* 120 */ "SequenceTest" OpHelp("if( cursor[P1].ctr++ ) pc = P2"), + /* 121 */ "OpenPseudo" OpHelp("P3 columns in r[P2]"), + /* 122 */ "Close" OpHelp(""), + /* 123 */ "ColumnsUsed" OpHelp(""), + /* 124 */ "SeekScan" OpHelp("Scan-ahead up to P1 rows"), + /* 125 */ "SeekHit" OpHelp("set P2<=seekHit<=P3"), + /* 126 */ "Sequence" OpHelp("r[P2]=cursor[P1].ctr++"), + /* 127 */ "NewRowid" OpHelp("r[P2]=rowid"), + /* 128 */ "Insert" OpHelp("intkey=r[P3] data=r[P2]"), + /* 129 */ "RowCell" OpHelp(""), + /* 130 */ "Delete" OpHelp(""), + /* 131 */ "ResetCount" OpHelp(""), + /* 132 */ "SorterCompare" OpHelp("if key(P1)!=trim(r[P3],P4) goto P2"), + /* 133 */ "SorterData" OpHelp("r[P2]=data"), + /* 134 */ "RowData" OpHelp("r[P2]=data"), + /* 135 */ "Rowid" OpHelp("r[P2]=PX rowid of P1"), + /* 136 */ "NullRow" OpHelp(""), + /* 137 */ "SeekEnd" OpHelp(""), + /* 138 */ "IdxInsert" OpHelp("key=r[P2]"), + /* 139 */ "SorterInsert" OpHelp("key=r[P2]"), + /* 140 */ "IdxDelete" OpHelp("key=r[P2@P3]"), + /* 141 */ "DeferredSeek" OpHelp("Move P3 to P1.rowid if needed"), + /* 142 */ "IdxRowid" OpHelp("r[P2]=rowid"), + /* 143 */ "FinishSeek" OpHelp(""), + /* 144 */ "Destroy" OpHelp(""), + /* 145 */ "Clear" OpHelp(""), + /* 146 */ "ResetSorter" OpHelp(""), + /* 147 */ "CreateBtree" OpHelp("r[P2]=root iDb=P1 flags=P3"), + /* 148 */ "SqlExec" OpHelp(""), + /* 149 */ "ParseSchema" OpHelp(""), + /* 150 */ "LoadAnalysis" OpHelp(""), + /* 151 */ "DropTable" OpHelp(""), + /* 152 */ "DropIndex" OpHelp(""), + /* 153 */ "DropTrigger" OpHelp(""), + /* 154 */ "Real" OpHelp("r[P2]=P4"), + /* 155 */ "IntegrityCk" OpHelp(""), + /* 156 */ "RowSetAdd" OpHelp("rowset(P1)=r[P2]"), + /* 157 */ "Param" OpHelp(""), + /* 158 */ "FkCounter" OpHelp("fkctr[P1]+=P2"), + /* 159 */ "MemMax" OpHelp("r[P1]=max(r[P1],r[P2])"), + /* 160 */ "OffsetLimit" OpHelp("if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1)"), + /* 161 */ "AggInverse" OpHelp("accum=r[P3] inverse(r[P2@P5])"), + /* 162 */ "AggStep" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 163 */ "AggStep1" OpHelp("accum=r[P3] step(r[P2@P5])"), + /* 164 */ "AggValue" OpHelp("r[P3]=value N=P2"), + /* 165 */ "AggFinal" OpHelp("accum=r[P1] N=P2"), + /* 166 */ "Expire" OpHelp(""), + /* 167 */ "CursorLock" OpHelp(""), + /* 168 */ "CursorUnlock" OpHelp(""), + /* 169 */ "TableLock" OpHelp("iDb=P1 root=P2 write=P3"), + /* 170 */ "VBegin" OpHelp(""), + /* 171 */ "VCreate" OpHelp(""), + /* 172 */ "VDestroy" OpHelp(""), + /* 173 */ "VOpen" OpHelp(""), + /* 174 */ "VCheck" OpHelp(""), + /* 175 */ "VInitIn" OpHelp("r[P2]=ValueList(P1,P3)"), + /* 176 */ "VColumn" OpHelp("r[P3]=vcolumn(P2)"), + /* 177 */ "VRename" OpHelp(""), + /* 178 */ "Pagecount" OpHelp(""), + /* 179 */ "MaxPgcnt" OpHelp(""), + /* 180 */ "ClrSubtype" OpHelp("r[P1].subtype = 0"), + /* 181 */ "GetSubtype" OpHelp("r[P2] = r[P1].subtype"), + /* 182 */ "SetSubtype" OpHelp("r[P2].subtype = r[P1]"), + /* 183 */ "FilterAdd" OpHelp("filter(P1) += key(P3@P4)"), + /* 184 */ "Trace" OpHelp(""), + /* 185 */ "CursorHint" OpHelp(""), + /* 186 */ "ReleaseReg" OpHelp("release r[P1@P2] mask P3"), + /* 187 */ "Noop" OpHelp(""), + /* 188 */ "Explain" OpHelp(""), + /* 189 */ "Abortable" OpHelp(""), }; return azName[i]; } #endif /************** End of opcodes.c *********************************************/ +/************** Begin file os_kv.c *******************************************/ +/* +** 2022-09-06 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an experimental VFS layer that operates on a +** Key/Value storage engine where both keys and values must be pure +** text. +*/ +/* #include */ +#if SQLITE_OS_KV || (SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL)) + +/***************************************************************************** +** Debugging logic +*/ + +/* SQLITE_KV_TRACE() is used for tracing calls to kvstorage routines. */ +#if 0 +#define SQLITE_KV_TRACE(X) printf X +#else +#define SQLITE_KV_TRACE(X) +#endif + +/* SQLITE_KV_LOG() is used for tracing calls to the VFS interface */ +#if 0 +#define SQLITE_KV_LOG(X) printf X +#else +#define SQLITE_KV_LOG(X) +#endif + + +/* +** Forward declaration of objects used by this VFS implementation +*/ +typedef struct KVVfsFile KVVfsFile; + +/* A single open file. There are only two files represented by this +** VFS - the database and the rollback journal. +*/ +struct KVVfsFile { + sqlite3_file base; /* IO methods */ + const char *zClass; /* Storage class */ + int isJournal; /* True if this is a journal file */ + unsigned int nJrnl; /* Space allocated for aJrnl[] */ + char *aJrnl; /* Journal content */ + int szPage; /* Last known page size */ + sqlite3_int64 szDb; /* Database file size. -1 means unknown */ + char *aData; /* Buffer to hold page data */ +}; +#define SQLITE_KVOS_SZ 133073 + +/* +** Methods for KVVfsFile +*/ +static int kvvfsClose(sqlite3_file*); +static int kvvfsReadDb(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsReadJrnl(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int kvvfsWriteDb(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsWriteJrnl(sqlite3_file*,const void*,int iAmt, sqlite3_int64); +static int kvvfsTruncateDb(sqlite3_file*, sqlite3_int64 size); +static int kvvfsTruncateJrnl(sqlite3_file*, sqlite3_int64 size); +static int kvvfsSyncDb(sqlite3_file*, int flags); +static int kvvfsSyncJrnl(sqlite3_file*, int flags); +static int kvvfsFileSizeDb(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsFileSizeJrnl(sqlite3_file*, sqlite3_int64 *pSize); +static int kvvfsLock(sqlite3_file*, int); +static int kvvfsUnlock(sqlite3_file*, int); +static int kvvfsCheckReservedLock(sqlite3_file*, int *pResOut); +static int kvvfsFileControlDb(sqlite3_file*, int op, void *pArg); +static int kvvfsFileControlJrnl(sqlite3_file*, int op, void *pArg); +static int kvvfsSectorSize(sqlite3_file*); +static int kvvfsDeviceCharacteristics(sqlite3_file*); + +/* +** Methods for sqlite3_vfs +*/ +static int kvvfsOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +static int kvvfsDelete(sqlite3_vfs*, const char *zName, int syncDir); +static int kvvfsAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int kvvfsFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *kvvfsDlOpen(sqlite3_vfs*, const char *zFilename); +static int kvvfsRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int kvvfsSleep(sqlite3_vfs*, int microseconds); +static int kvvfsCurrentTime(sqlite3_vfs*, double*); +static int kvvfsCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs sqlite3OsKvvfsObject = { + 1, /* iVersion */ + sizeof(KVVfsFile), /* szOsFile */ + 1024, /* mxPathname */ + 0, /* pNext */ + "kvvfs", /* zName */ + 0, /* pAppData */ + kvvfsOpen, /* xOpen */ + kvvfsDelete, /* xDelete */ + kvvfsAccess, /* xAccess */ + kvvfsFullPathname, /* xFullPathname */ + kvvfsDlOpen, /* xDlOpen */ + 0, /* xDlError */ + 0, /* xDlSym */ + 0, /* xDlClose */ + kvvfsRandomness, /* xRandomness */ + kvvfsSleep, /* xSleep */ + kvvfsCurrentTime, /* xCurrentTime */ + 0, /* xGetLastError */ + kvvfsCurrentTimeInt64 /* xCurrentTimeInt64 */ +}; + +/* Methods for sqlite3_file objects referencing a database file +*/ +static sqlite3_io_methods kvvfs_db_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadDb, /* xRead */ + kvvfsWriteDb, /* xWrite */ + kvvfsTruncateDb, /* xTruncate */ + kvvfsSyncDb, /* xSync */ + kvvfsFileSizeDb, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlDb, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/* Methods for sqlite3_file objects referencing a rollback journal +*/ +static sqlite3_io_methods kvvfs_jrnl_io_methods = { + 1, /* iVersion */ + kvvfsClose, /* xClose */ + kvvfsReadJrnl, /* xRead */ + kvvfsWriteJrnl, /* xWrite */ + kvvfsTruncateJrnl, /* xTruncate */ + kvvfsSyncJrnl, /* xSync */ + kvvfsFileSizeJrnl, /* xFileSize */ + kvvfsLock, /* xLock */ + kvvfsUnlock, /* xUnlock */ + kvvfsCheckReservedLock, /* xCheckReservedLock */ + kvvfsFileControlJrnl, /* xFileControl */ + kvvfsSectorSize, /* xSectorSize */ + kvvfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + 0, /* xFetch */ + 0 /* xUnfetch */ +}; + +/****** Storage subsystem **************************************************/ +#include +#include +#include + +/* Forward declarations for the low-level storage engine +*/ +static int kvstorageWrite(const char*, const char *zKey, const char *zData); +static int kvstorageDelete(const char*, const char *zKey); +static int kvstorageRead(const char*, const char *zKey, char *zBuf, int nBuf); +#define KVSTORAGE_KEY_SZ 32 + +/* Expand the key name with an appropriate prefix and put the result +** zKeyOut[]. The zKeyOut[] buffer is assumed to hold at least +** KVSTORAGE_KEY_SZ bytes. +*/ +static void kvstorageMakeKey( + const char *zClass, + const char *zKeyIn, + char *zKeyOut +){ + sqlite3_snprintf(KVSTORAGE_KEY_SZ, zKeyOut, "kvvfs-%s-%s", zClass, zKeyIn); +} + +/* Write content into a key. zClass is the particular namespace of the +** underlying key/value store to use - either "local" or "session". +** +** Both zKey and zData are zero-terminated pure text strings. +** +** Return the number of errors. +*/ +static int kvstorageWrite( + const char *zClass, + const char *zKey, + const char *zData +){ + FILE *fd; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + fd = fopen(zXKey, "wb"); + if( fd ){ + SQLITE_KV_TRACE(("KVVFS-WRITE %-15s (%d) %.50s%s\n", zXKey, + (int)strlen(zData), zData, + strlen(zData)>50 ? "..." : "")); + fputs(zData, fd); + fclose(fd); + return 0; + }else{ + return 1; + } +} + +/* Delete a key (with its corresponding data) from the key/value +** namespace given by zClass. If the key does not previously exist, +** this routine is a no-op. +*/ +static int kvstorageDelete(const char *zClass, const char *zKey){ + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + unlink(zXKey); + SQLITE_KV_TRACE(("KVVFS-DELETE %-15s\n", zXKey)); + return 0; +} + +/* Read the value associated with a zKey from the key/value namespace given +** by zClass and put the text data associated with that key in the first +** nBuf bytes of zBuf[]. The value might be truncated if zBuf is not large +** enough to hold it all. The value put into zBuf must always be zero +** terminated, even if it gets truncated because nBuf is not large enough. +** +** Return the total number of bytes in the data, without truncation, and +** not counting the final zero terminator. Return -1 if the key does +** not exist. +** +** If nBuf<=0 then this routine simply returns the size of the data without +** actually reading it. +*/ +static int kvstorageRead( + const char *zClass, + const char *zKey, + char *zBuf, + int nBuf +){ + FILE *fd; + struct stat buf; + char zXKey[KVSTORAGE_KEY_SZ]; + kvstorageMakeKey(zClass, zKey, zXKey); + if( access(zXKey, R_OK)!=0 + || stat(zXKey, &buf)!=0 + || !S_ISREG(buf.st_mode) + ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + } + if( nBuf<=0 ){ + return (int)buf.st_size; + }else if( nBuf==1 ){ + zBuf[0] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%d)\n", zXKey, + (int)buf.st_size)); + return (int)buf.st_size; + } + if( nBuf > buf.st_size + 1 ){ + nBuf = buf.st_size + 1; + } + fd = fopen(zXKey, "rb"); + if( fd==0 ){ + SQLITE_KV_TRACE(("KVVFS-READ %-15s (-1)\n", zXKey)); + return -1; + }else{ + sqlite3_int64 n = fread(zBuf, 1, nBuf-1, fd); + fclose(fd); + zBuf[n] = 0; + SQLITE_KV_TRACE(("KVVFS-READ %-15s (%lld) %.50s%s\n", zXKey, + n, zBuf, n>50 ? "..." : "")); + return (int)n; + } +} + +/* +** An internal level of indirection which enables us to replace the +** kvvfs i/o methods with JavaScript implementations in WASM builds. +** Maintenance reminder: if this struct changes in any way, the JSON +** rendering of its structure must be updated in +** sqlite3_wasm_enum_json(). There are no binary compatibility +** concerns, so it does not need an iVersion member. This file is +** necessarily always compiled together with sqlite3_wasm_enum_json(), +** and JS code dynamically creates the mapping of members based on +** that JSON description. +*/ +typedef struct sqlite3_kvvfs_methods sqlite3_kvvfs_methods; +struct sqlite3_kvvfs_methods { + int (*xRead)(const char *zClass, const char *zKey, char *zBuf, int nBuf); + int (*xWrite)(const char *zClass, const char *zKey, const char *zData); + int (*xDelete)(const char *zClass, const char *zKey); + const int nKeySize; +}; + +/* +** This object holds the kvvfs I/O methods which may be swapped out +** for JavaScript-side implementations in WASM builds. In such builds +** it cannot be const, but in native builds it should be so that +** the compiler can hopefully optimize this level of indirection out. +** That said, kvvfs is intended primarily for use in WASM builds. +** +** Note that this is not explicitly flagged as static because the +** amalgamation build will tag it with SQLITE_PRIVATE. +*/ +#ifndef SQLITE_WASM +const +#endif +SQLITE_PRIVATE sqlite3_kvvfs_methods sqlite3KvvfsMethods = { +kvstorageRead, +kvstorageWrite, +kvstorageDelete, +KVSTORAGE_KEY_SZ +}; + +/****** Utility subroutines ************************************************/ + +/* +** Encode binary into the text encoded used to persist on disk. +** The output text is stored in aOut[], which must be at least +** nData+1 bytes in length. +** +** Return the actual length of the encoded text, not counting the +** zero terminator at the end. +** +** Encoding format +** --------------- +** +** * Non-zero bytes are encoded as upper-case hexadecimal +** +** * A sequence of one or more zero-bytes that are not at the +** beginning of the buffer are encoded as a little-endian +** base-26 number using a..z. "a" means 0. "b" means 1, +** "z" means 25. "ab" means 26. "ac" means 52. And so forth. +** +** * Because there is no overlap between the encoding characters +** of hexadecimal and base-26 numbers, it is always clear where +** one stops and the next begins. +*/ +static int kvvfsEncode(const char *aData, int nData, char *aOut){ + int i, j; + const unsigned char *a = (const unsigned char*)aData; + for(i=j=0; i>4]; + aOut[j++] = "0123456789ABCDEF"[c&0xf]; + }else{ + /* A sequence of 1 or more zeros is stored as a little-endian + ** base-26 number using a..z as the digits. So one zero is "b". + ** Two zeros is "c". 25 zeros is "z", 26 zeros is "ab", 27 is "bb", + ** and so forth. + */ + int k; + for(k=1; i+k0 ){ + aOut[j++] = 'a'+(k%26); + k /= 26; + } + } + } + aOut[j] = 0; + return j; +} + +static const signed char kvvfsHexValue[256] = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, + -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 +}; + +/* +** Decode the text encoding back to binary. The binary content is +** written into pOut, which must be at least nOut bytes in length. +** +** The return value is the number of bytes actually written into aOut[]. +*/ +static int kvvfsDecode(const char *a, char *aOut, int nOut){ + int i, j; + int c; + const unsigned char *aIn = (const unsigned char*)a; + i = 0; + j = 0; + while( 1 ){ + c = kvvfsHexValue[aIn[i]]; + if( c<0 ){ + int n = 0; + int mult = 1; + c = aIn[i]; + if( c==0 ) break; + while( c>='a' && c<='z' ){ + n += (c - 'a')*mult; + mult *= 26; + c = aIn[++i]; + } + if( j+n>nOut ) return -1; + memset(&aOut[j], 0, n); + j += n; + if( c==0 || mult==1 ) break; /* progress stalled if mult==1 */ + }else{ + aOut[j] = c<<4; + c = kvvfsHexValue[aIn[++i]]; + if( c<0 ) break; + aOut[j++] += c; + i++; + } + } + return j; +} + +/* +** Decode a complete journal file. Allocate space in pFile->aJrnl +** and store the decoding there. Or leave pFile->aJrnl set to NULL +** if an error is encountered. +** +** The first few characters of the text encoding will be a little-endian +** base-26 number (digits a..z) that is the total number of bytes +** in the decoded journal file image. This base-26 number is followed +** by a single space, then the encoding of the journal. The space +** separator is required to act as a terminator for the base-26 number. +*/ +static void kvvfsDecodeJournal( + KVVfsFile *pFile, /* Store decoding in pFile->aJrnl */ + const char *zTxt, /* Text encoding. Zero-terminated */ + int nTxt /* Bytes in zTxt, excluding zero terminator */ +){ + unsigned int n = 0; + int c, i, mult; + i = 0; + mult = 1; + while( (c = zTxt[i++])>='a' && c<='z' ){ + n += (zTxt[i] - 'a')*mult; + mult *= 26; + } + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = sqlite3_malloc64( n ); + if( pFile->aJrnl==0 ){ + pFile->nJrnl = 0; + return; + } + pFile->nJrnl = n; + n = kvvfsDecode(zTxt+i, pFile->aJrnl, pFile->nJrnl); + if( nnJrnl ){ + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + } +} + +/* +** Read or write the "sz" element, containing the database file size. +*/ +static sqlite3_int64 kvvfsReadFileSize(KVVfsFile *pFile){ + char zData[50]; + zData[0] = 0; + sqlite3KvvfsMethods.xRead(pFile->zClass, "sz", zData, sizeof(zData)-1); + return strtoll(zData, 0, 0); +} +static int kvvfsWriteFileSize(KVVfsFile *pFile, sqlite3_int64 sz){ + char zData[50]; + sqlite3_snprintf(sizeof(zData), zData, "%lld", sz); + return sqlite3KvvfsMethods.xWrite(pFile->zClass, "sz", zData); +} + +/****** sqlite3_io_methods methods ******************************************/ + +/* +** Close an kvvfs-file. +*/ +static int kvvfsClose(sqlite3_file *pProtoFile){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + + SQLITE_KV_LOG(("xClose %s %s\n", pFile->zClass, + pFile->isJournal ? "journal" : "db")); + sqlite3_free(pFile->aJrnl); + sqlite3_free(pFile->aData); + return SQLITE_OK; +} + +/* +** Read from the -journal file. +*/ +static int kvvfsReadJrnl( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + assert( pFile->isJournal ); + SQLITE_KV_LOG(("xRead('%s-journal',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( pFile->aJrnl==0 ){ + int szTxt = kvstorageRead(pFile->zClass, "jrnl", 0, 0); + char *aTxt; + if( szTxt<=4 ){ + return SQLITE_IOERR; + } + aTxt = sqlite3_malloc64( szTxt+1 ); + if( aTxt==0 ) return SQLITE_NOMEM; + kvstorageRead(pFile->zClass, "jrnl", aTxt, szTxt+1); + kvvfsDecodeJournal(pFile, aTxt, szTxt); + sqlite3_free(aTxt); + if( pFile->aJrnl==0 ) return SQLITE_IOERR; + } + if( iOfst+iAmt>pFile->nJrnl ){ + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, pFile->aJrnl+iOfst, iAmt); + return SQLITE_OK; +} + +/* +** Read from the database file. +*/ +static int kvvfsReadDb( + sqlite3_file *pProtoFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + int got, n; + char zKey[30]; + char *aData = pFile->aData; + assert( iOfst>=0 ); + assert( iAmt>=0 ); + SQLITE_KV_LOG(("xRead('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + if( iOfst+iAmt>=512 ){ + if( (iOfst % iAmt)!=0 ){ + return SQLITE_IOERR_READ; + } + if( (iAmt & (iAmt-1))!=0 || iAmt<512 || iAmt>65536 ){ + return SQLITE_IOERR_READ; + } + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + }else{ + pgno = 1; + } + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + got = sqlite3KvvfsMethods.xRead(pFile->zClass, zKey, + aData, SQLITE_KVOS_SZ-1); + if( got<0 ){ + n = 0; + }else{ + aData[got] = 0; + if( iOfst+iAmt<512 ){ + int k = iOfst+iAmt; + aData[k*2] = 0; + n = kvvfsDecode(aData, &aData[2000], SQLITE_KVOS_SZ-2000); + if( n>=iOfst+iAmt ){ + memcpy(zBuf, &aData[2000+iOfst], iAmt); + n = iAmt; + }else{ + n = 0; + } + }else{ + n = kvvfsDecode(aData, zBuf, iAmt); + } + } + if( nzClass, iAmt, iOfst)); + if( iEnd>=0x10000000 ) return SQLITE_FULL; + if( pFile->aJrnl==0 || pFile->nJrnlaJrnl, iEnd); + if( aNew==0 ){ + return SQLITE_IOERR_NOMEM; + } + pFile->aJrnl = aNew; + if( pFile->nJrnlaJrnl+pFile->nJrnl, 0, iOfst-pFile->nJrnl); + } + pFile->nJrnl = iEnd; + } + memcpy(pFile->aJrnl+iOfst, zBuf, iAmt); + return SQLITE_OK; +} + +/* +** Write into the database file. +*/ +static int kvvfsWriteDb( + sqlite3_file *pProtoFile, + const void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + unsigned int pgno; + char zKey[30]; + char *aData = pFile->aData; + SQLITE_KV_LOG(("xWrite('%s-db',%d,%lld)\n", pFile->zClass, iAmt, iOfst)); + assert( iAmt>=512 && iAmt<=65536 ); + assert( (iAmt & (iAmt-1))==0 ); + assert( pFile->szPage<0 || pFile->szPage==iAmt ); + pFile->szPage = iAmt; + pgno = 1 + iOfst/iAmt; + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + kvvfsEncode(zBuf, iAmt, aData); + if( sqlite3KvvfsMethods.xWrite(pFile->zClass, zKey, aData) ){ + return SQLITE_IOERR; + } + if( iOfst+iAmt > pFile->szDb ){ + pFile->szDb = iOfst + iAmt; + } + return SQLITE_OK; +} + +/* +** Truncate an kvvfs-file. +*/ +static int kvvfsTruncateJrnl(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xTruncate('%s-journal',%lld)\n", pFile->zClass, size)); + assert( size==0 ); + sqlite3KvvfsMethods.xDelete(pFile->zClass, "jrnl"); + sqlite3_free(pFile->aJrnl); + pFile->aJrnl = 0; + pFile->nJrnl = 0; + return SQLITE_OK; +} +static int kvvfsTruncateDb(sqlite3_file *pProtoFile, sqlite_int64 size){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + if( pFile->szDb>size + && pFile->szPage>0 + && (size % pFile->szPage)==0 + ){ + char zKey[50]; + unsigned int pgno, pgnoMax; + SQLITE_KV_LOG(("xTruncate('%s-db',%lld)\n", pFile->zClass, size)); + pgno = 1 + size/pFile->szPage; + pgnoMax = 2 + pFile->szDb/pFile->szPage; + while( pgno<=pgnoMax ){ + sqlite3_snprintf(sizeof(zKey), zKey, "%u", pgno); + sqlite3KvvfsMethods.xDelete(pFile->zClass, zKey); + pgno++; + } + pFile->szDb = size; + return kvvfsWriteFileSize(pFile, size) ? SQLITE_IOERR : SQLITE_OK; + } + return SQLITE_IOERR; +} + +/* +** Sync an kvvfs-file. +*/ +static int kvvfsSyncJrnl(sqlite3_file *pProtoFile, int flags){ + int i, n; + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + char *zOut; + SQLITE_KV_LOG(("xSync('%s-journal')\n", pFile->zClass)); + if( pFile->nJrnl<=0 ){ + return kvvfsTruncateJrnl(pProtoFile, 0); + } + zOut = sqlite3_malloc64( pFile->nJrnl*2 + 50 ); + if( zOut==0 ){ + return SQLITE_IOERR_NOMEM; + } + n = pFile->nJrnl; + i = 0; + do{ + zOut[i++] = 'a' + (n%26); + n /= 26; + }while( n>0 ); + zOut[i++] = ' '; + kvvfsEncode(pFile->aJrnl, pFile->nJrnl, &zOut[i]); + i = sqlite3KvvfsMethods.xWrite(pFile->zClass, "jrnl", zOut); + sqlite3_free(zOut); + return i ? SQLITE_IOERR : SQLITE_OK; +} +static int kvvfsSyncDb(sqlite3_file *pProtoFile, int flags){ + return SQLITE_OK; +} + +/* +** Return the current file-size of an kvvfs-file. +*/ +static int kvvfsFileSizeJrnl(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-journal')\n", pFile->zClass)); + *pSize = pFile->nJrnl; + return SQLITE_OK; +} +static int kvvfsFileSizeDb(sqlite3_file *pProtoFile, sqlite_int64 *pSize){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + SQLITE_KV_LOG(("xFileSize('%s-db')\n", pFile->zClass)); + if( pFile->szDb>=0 ){ + *pSize = pFile->szDb; + }else{ + *pSize = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Lock an kvvfs-file. +*/ +static int kvvfsLock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xLock(%s,%d)\n", pFile->zClass, eLock)); + + if( eLock!=SQLITE_LOCK_NONE ){ + pFile->szDb = kvvfsReadFileSize(pFile); + } + return SQLITE_OK; +} + +/* +** Unlock an kvvfs-file. +*/ +static int kvvfsUnlock(sqlite3_file *pProtoFile, int eLock){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + assert( !pFile->isJournal ); + SQLITE_KV_LOG(("xUnlock(%s,%d)\n", pFile->zClass, eLock)); + if( eLock==SQLITE_LOCK_NONE ){ + pFile->szDb = -1; + } + return SQLITE_OK; +} + +/* +** Check if another file-handle holds a RESERVED lock on an kvvfs-file. +*/ +static int kvvfsCheckReservedLock(sqlite3_file *pProtoFile, int *pResOut){ + SQLITE_KV_LOG(("xCheckReservedLock\n")); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** File control method. For custom operations on an kvvfs-file. +*/ +static int kvvfsFileControlJrnl(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on journal\n", op)); + return SQLITE_NOTFOUND; +} +static int kvvfsFileControlDb(sqlite3_file *pProtoFile, int op, void *pArg){ + SQLITE_KV_LOG(("xFileControl(%d) on database\n", op)); + if( op==SQLITE_FCNTL_SYNC ){ + KVVfsFile *pFile = (KVVfsFile *)pProtoFile; + int rc = SQLITE_OK; + SQLITE_KV_LOG(("xSync('%s-db')\n", pFile->zClass)); + if( pFile->szDb>0 && 0!=kvvfsWriteFileSize(pFile, pFile->szDb) ){ + rc = SQLITE_IOERR; + } + return rc; + } + return SQLITE_NOTFOUND; +} + +/* +** Return the sector-size in bytes for an kvvfs-file. +*/ +static int kvvfsSectorSize(sqlite3_file *pFile){ + return 512; +} + +/* +** Return the device characteristic flags supported by an kvvfs-file. +*/ +static int kvvfsDeviceCharacteristics(sqlite3_file *pProtoFile){ + return 0; +} + +/****** sqlite3_vfs methods *************************************************/ + +/* +** Open an kvvfs file handle. +*/ +static int kvvfsOpen( + sqlite3_vfs *pProtoVfs, + const char *zName, + sqlite3_file *pProtoFile, + int flags, + int *pOutFlags +){ + KVVfsFile *pFile = (KVVfsFile*)pProtoFile; + if( zName==0 ) zName = ""; + SQLITE_KV_LOG(("xOpen(\"%s\")\n", zName)); + if( strcmp(zName, "local")==0 + || strcmp(zName, "session")==0 + ){ + pFile->isJournal = 0; + pFile->base.pMethods = &kvvfs_db_io_methods; + }else + if( strcmp(zName, "local-journal")==0 + || strcmp(zName, "session-journal")==0 + ){ + pFile->isJournal = 1; + pFile->base.pMethods = &kvvfs_jrnl_io_methods; + }else{ + return SQLITE_CANTOPEN; + } + if( zName[0]=='s' ){ + pFile->zClass = "session"; + }else{ + pFile->zClass = "local"; + } + pFile->aData = sqlite3_malloc64(SQLITE_KVOS_SZ); + if( pFile->aData==0 ){ + return SQLITE_NOMEM; + } + pFile->aJrnl = 0; + pFile->nJrnl = 0; + pFile->szPage = -1; + pFile->szDb = -1; + return SQLITE_OK; +} + +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int kvvfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + if( strcmp(zPath, "local-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("local", "jrnl"); + }else + if( strcmp(zPath, "session-journal")==0 ){ + sqlite3KvvfsMethods.xDelete("session", "jrnl"); + } + return SQLITE_OK; +} + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +*/ +static int kvvfsAccess( + sqlite3_vfs *pProtoVfs, + const char *zPath, + int flags, + int *pResOut +){ + SQLITE_KV_LOG(("xAccess(\"%s\")\n", zPath)); + if( strcmp(zPath, "local-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "session-journal")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "jrnl", 0, 0)>0; + }else + if( strcmp(zPath, "local")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("local", "sz", 0, 0)>0; + }else + if( strcmp(zPath, "session")==0 ){ + *pResOut = sqlite3KvvfsMethods.xRead("session", "sz", 0, 0)>0; + }else + { + *pResOut = 0; + } + SQLITE_KV_LOG(("xAccess returns %d\n",*pResOut)); + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int kvvfsFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + size_t nPath; +#ifdef SQLITE_OS_KV_ALWAYS_LOCAL + zPath = "local"; +#endif + nPath = strlen(zPath); + SQLITE_KV_LOG(("xFullPathname(\"%s\")\n", zPath)); + if( nOut +static int kvvfsCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *pTimeOut){ + static const sqlite3_int64 unixEpoch = 24405875*(sqlite3_int64)8640000; + struct timeval sNow; + (void)gettimeofday(&sNow, 0); /* Cannot fail given valid arguments */ + *pTimeOut = unixEpoch + 1000*(sqlite3_int64)sNow.tv_sec + sNow.tv_usec/1000; + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV || SQLITE_OS_UNIX */ + +#if SQLITE_OS_KV +/* +** This routine is called initialize the KV-vfs as the default VFS. +*/ +SQLITE_API int sqlite3_os_init(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 1); +} +SQLITE_API int sqlite3_os_end(void){ + return SQLITE_OK; +} +#endif /* SQLITE_OS_KV */ + +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) +SQLITE_PRIVATE int sqlite3KvvfsInit(void){ + return sqlite3_vfs_register(&sqlite3OsKvvfsObject, 0); +} +#endif + +/************** End of os_kv.c ***********************************************/ /************** Begin file os_unix.c *****************************************/ /* ** 2004 May 22 @@ -28921,7 +38495,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** This source file is organized into divisions where the logic for various ** subfunctions is contained within the appropriate division. PLEASE ** KEEP THE STRUCTURE OF THIS FILE INTACT. New code should be placed -** in the correct division and should be clearly labeled. +** in the correct division and should be clearly labelled. ** ** The layout of divisions is as follows: ** @@ -28960,7 +38534,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ ** Styles 4, 5, and 7 are only available of SQLITE_ENABLE_LOCKING_STYLE ** is defined to 1. The SQLITE_ENABLE_LOCKING_STYLE also enables automatic ** selection of the appropriate locking style based on the filesystem -** where the database is located. +** where the database is located. */ #if !defined(SQLITE_ENABLE_LOCKING_STYLE) # if defined(__APPLE__) @@ -28971,7 +38545,7 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #endif /* Use pread() and pwrite() if they are available */ -#if defined(__APPLE__) +#if defined(__APPLE__) || defined(__linux__) # define HAVE_PREAD 1 # define HAVE_PWRITE 1 #endif @@ -28986,30 +38560,49 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ /* ** standard include files. */ -#include -#include +#include /* amalgamator: keep */ +#include /* amalgamator: keep */ #include -#include +#include +#include /* amalgamator: keep */ /* #include */ -#include +#include /* amalgamator: keep */ #include -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) # include #endif #if SQLITE_ENABLE_LOCKING_STYLE -# include +/* # include */ # include # include #endif /* SQLITE_ENABLE_LOCKING_STYLE */ -#if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ - (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) -# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ - && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0)) -# define HAVE_GETHOSTUUID 1 -# else -# warning "gethostuuid() is disabled." +/* +** Try to determine if gethostuuid() is available based on standard +** macros. This might sometimes compute the wrong value for some +** obscure platforms. For those cases, simply compile with one of +** the following: +** +** -DHAVE_GETHOSTUUID=0 +** -DHAVE_GETHOSTUUID=1 +** +** None if this matters except when building on Apple products with +** -DSQLITE_ENABLE_LOCKING_STYLE. +*/ +#ifndef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 0 +# if defined(__APPLE__) && ((__MAC_OS_X_VERSION_MIN_REQUIRED > 1050) || \ + (__IPHONE_OS_VERSION_MIN_REQUIRED > 2000)) +# if (!defined(TARGET_OS_EMBEDDED) || (TARGET_OS_EMBEDDED==0)) \ + && (!defined(TARGET_IPHONE_SIMULATOR) || (TARGET_IPHONE_SIMULATOR==0))\ + && (!defined(TARGET_OS_MACCATALYST) || (TARGET_OS_MACCATALYST==0)) +# undef HAVE_GETHOSTUUID +# define HAVE_GETHOSTUUID 1 +# else +# warning "gethostuuid() is disabled." +# endif # endif #endif @@ -29034,12 +38627,10 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ #define SQLITE_FSFLAGS_IS_MSDOS 0x1 /* -** If we are to be thread-safe, include the pthreads header and define -** the SQLITE_UNIX_THREADS macro. +** If we are to be thread-safe, include the pthreads header. */ #if SQLITE_THREADSAFE /* # include */ -# define SQLITE_UNIX_THREADS 1 #endif /* @@ -29066,12 +38657,49 @@ SQLITE_PRIVATE const char *sqlite3OpcodeName(int i){ */ #define SQLITE_MAX_SYMLINKS 100 +/* +** Remove and stub certain info for WASI (WebAssembly System +** Interface) builds. +*/ +#ifdef SQLITE_WASI +# undef HAVE_FCHMOD +# undef HAVE_FCHOWN +# undef HAVE_MREMAP +# define HAVE_MREMAP 0 +# ifndef SQLITE_DEFAULT_UNIX_VFS +# define SQLITE_DEFAULT_UNIX_VFS "unix-dotfile" + /* ^^^ should SQLITE_DEFAULT_UNIX_VFS be "unix-none"? */ +# endif +# ifndef F_RDLCK +# define F_RDLCK 0 +# define F_WRLCK 1 +# define F_UNLCK 2 +# if __LONG_MAX == 0x7fffffffL +# define F_GETLK 12 +# define F_SETLK 13 +# define F_SETLKW 14 +# else +# define F_GETLK 5 +# define F_SETLK 6 +# define F_SETLKW 7 +# endif +# endif +#else /* !SQLITE_WASI */ +# ifndef HAVE_FCHMOD +# define HAVE_FCHMOD +# endif +#endif /* SQLITE_WASI */ + +#ifdef SQLITE_WASI +# define osGetpid(X) (pid_t)1 +#else /* Always cast the getpid() return type for compatibility with ** kernel modules in VxWorks. */ -#define osGetpid(X) (pid_t)getpid() +# define osGetpid(X) (pid_t)getpid() +#endif /* -** Only set the lastErrno if the error code is a real error and not +** Only set the lastErrno if the error code is a real error and not ** a normal expected return code of SQLITE_BUSY or SQLITE_OK */ #define IS_LOCK_ERROR(x) ((x != SQLITE_OK) && (x != SQLITE_BUSY)) @@ -29108,7 +38736,7 @@ struct unixFile { unsigned short int ctrlFlags; /* Behavioral bits. UNIXFILE_* flags */ int lastErrno; /* The unix errno from last I/O error */ void *lockingContext; /* Locking style specific state */ - UnixUnusedFd *pUnused; /* Pre-allocated UnixUnusedFd */ + UnixUnusedFd *pPreallocatedUnused; /* Pre-allocated UnixUnusedFd */ const char *zPath; /* Name of the file */ unixShm *pShm; /* Shared memory segment information */ int szChunk; /* Configured by FCNTL_CHUNK_SIZE */ @@ -29119,16 +38747,17 @@ struct unixFile { sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ void *pMapRegion; /* Memory mapped region */ #endif -#ifdef __QNXNTO__ int sectorSize; /* Device sector size */ int deviceCharacteristics; /* Precomputed device characteristics */ -#endif #if SQLITE_ENABLE_LOCKING_STYLE int openFlags; /* The flags specified at open() */ #endif #if SQLITE_ENABLE_LOCKING_STYLE || defined(__APPLE__) unsigned fsFlags; /* cached details from statfs() */ #endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + unsigned iBusyTimeout; /* Wait this many millisec on locks */ +#endif #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID */ #endif @@ -29138,7 +38767,7 @@ struct unixFile { ** whenever any part of the database changes. An assertion fault will ** occur if a file is updated without also updating the transaction ** counter. This test is made to avoid new problems similar to the - ** one described by ticket #3584. + ** one described by ticket #3584. */ unsigned char transCntrChng; /* True if the transaction counter changed */ unsigned char dbUpdate; /* True if any part of database file changed */ @@ -29147,7 +38776,7 @@ struct unixFile { #endif #ifdef SQLITE_TEST - /* In test mode, increase the size of this structure a bit so that + /* In test mode, increase the size of this structure a bit so that ** it is larger than the struct CrashFile defined in test6.c. */ char aPadding[32]; @@ -29166,7 +38795,7 @@ static pid_t randomnessPid = 0; #define UNIXFILE_EXCL 0x01 /* Connections from one process only */ #define UNIXFILE_RDONLY 0x02 /* Connection is read only */ #define UNIXFILE_PERSIST_WAL 0x04 /* Persistent WAL mode */ -#ifndef SQLITE_DISABLE_DIRSYNC +#if !defined(SQLITE_DISABLE_DIRSYNC) && !defined(_AIX) # define UNIXFILE_DIRSYNC 0x08 /* Directory sync needed */ #else # define UNIXFILE_DIRSYNC 0x00 @@ -29179,205 +38808,7 @@ static pid_t randomnessPid = 0; /* ** Include code that is common to all os_*.c files */ -/************** Include os_common.h in the middle of os_unix.c ***************/ -/************** Begin file os_common.h ***************************************/ -/* -** 2004 May 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains macros and a little bit of code that is common to -** all of the platform-specific files (os_*.c) and is #included into those -** files. -** -** This file should be #included by the os_*.c files only. It is not a -** general purpose header file. -*/ -#ifndef _OS_COMMON_H_ -#define _OS_COMMON_H_ - -/* -** At least two bugs have slipped in because we changed the MEMORY_DEBUG -** macro to SQLITE_DEBUG and some older makefiles have not yet made the -** switch. The following code should catch this problem at compile-time. -*/ -#ifdef MEMORY_DEBUG -# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." -#endif - -/* -** Macros for performance tracing. Normally turned off. Only works -** on i486 hardware. -*/ -#ifdef SQLITE_PERFORMANCE_TRACE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - #error Need implementation of sqlite3Hwtime() for your platform. - - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(_HWTIME_H_) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ - -static sqlite_uint64 g_start; -static sqlite_uint64 g_elapsed; -#define TIMER_START g_start=sqlite3Hwtime() -#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start -#define TIMER_ELAPSED g_elapsed -#else -#define TIMER_START -#define TIMER_END -#define TIMER_ELAPSED ((sqlite_uint64)0) -#endif - -/* -** If we compile with the SQLITE_TEST macro set, then the following block -** of code will give us the ability to simulate a disk I/O error. This -** is used for testing the I/O recovery logic. -*/ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_io_error_hit; -SQLITE_API extern int sqlite3_io_error_hardhit; -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_persist; -SQLITE_API extern int sqlite3_io_error_benign; -SQLITE_API extern int sqlite3_diskfull_pending; -SQLITE_API extern int sqlite3_diskfull; -#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) -#define SimulateIOError(CODE) \ - if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ - || sqlite3_io_error_pending-- == 1 ) \ - { local_ioerr(); CODE; } -static void local_ioerr(){ - IOTRACE(("IOERR\n")); - sqlite3_io_error_hit++; - if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; -} -#define SimulateDiskfullError(CODE) \ - if( sqlite3_diskfull_pending ){ \ - if( sqlite3_diskfull_pending == 1 ){ \ - local_ioerr(); \ - sqlite3_diskfull = 1; \ - sqlite3_io_error_hit = 1; \ - CODE; \ - }else{ \ - sqlite3_diskfull_pending--; \ - } \ - } -#else -#define SimulateIOErrorBenign(X) -#define SimulateIOError(A) -#define SimulateDiskfullError(A) -#endif /* defined(SQLITE_TEST) */ - -/* -** When testing, keep a count of the number of open files. -*/ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_open_file_count; -#define OpenCounter(X) sqlite3_open_file_count+=(X) -#else -#define OpenCounter(X) -#endif /* defined(SQLITE_TEST) */ - -#endif /* !defined(_OS_COMMON_H_) */ - -/************** End of os_common.h *******************************************/ -/************** Continuing where we left off in os_unix.c ********************/ +/* #include "os_common.h" */ /* ** Define various macros that are missing from some systems. @@ -29425,6 +38856,20 @@ SQLITE_API extern int sqlite3_open_file_count; # define lseek lseek64 #endif +#ifdef __linux__ +/* +** Linux-specific IOCTL magic numbers used for controlling F2FS +*/ +#define F2FS_IOCTL_MAGIC 0xf5 +#define F2FS_IOC_START_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 1) +#define F2FS_IOC_COMMIT_ATOMIC_WRITE _IO(F2FS_IOCTL_MAGIC, 2) +#define F2FS_IOC_START_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 3) +#define F2FS_IOC_ABORT_VOLATILE_WRITE _IO(F2FS_IOCTL_MAGIC, 5) +#define F2FS_IOC_GET_FEATURES _IOR(F2FS_IOCTL_MAGIC, 12, u32) +#define F2FS_FEATURE_ATOMIC_WRITE 0x0004 +#endif /* __linux__ */ + + /* ** Different Unix systems declare open() in different ways. Same use ** open(const char*,int,mode_t). Others use open(const char*,int,...). @@ -29476,7 +38921,7 @@ static struct unix_syscall { #ifdef __DJGPP__ { "fstat", 0, 0 }, #define osFstat(a,b,c) 0 -#else +#else { "fstat", (sqlite3_syscall_ptr)fstat, 0 }, #define osFstat ((int(*)(int,struct stat*))aSyscall[5].pCurrent) #endif @@ -29523,7 +38968,11 @@ static struct unix_syscall { #define osPwrite64 ((ssize_t(*)(int,const void*,size_t,off64_t))\ aSyscall[13].pCurrent) +#if defined(HAVE_FCHMOD) { "fchmod", (sqlite3_syscall_ptr)fchmod, 0 }, +#else + { "fchmod", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osFchmod ((int(*)(int,mode_t))aSyscall[14].pCurrent) #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE @@ -29552,22 +39001,28 @@ static struct unix_syscall { #endif #define osFchown ((int(*)(int,uid_t,gid_t))aSyscall[20].pCurrent) +#if defined(HAVE_FCHOWN) { "geteuid", (sqlite3_syscall_ptr)geteuid, 0 }, +#else + { "geteuid", (sqlite3_syscall_ptr)0, 0 }, +#endif #define osGeteuid ((uid_t(*)(void))aSyscall[21].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) { "mmap", (sqlite3_syscall_ptr)mmap, 0 }, #else { "mmap", (sqlite3_syscall_ptr)0, 0 }, #endif #define osMmap ((void*(*)(void*,size_t,int,int,int,off_t))aSyscall[22].pCurrent) -#if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 +#if (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) \ + && !defined(SQLITE_WASI) { "munmap", (sqlite3_syscall_ptr)munmap, 0 }, #else { "munmap", (sqlite3_syscall_ptr)0, 0 }, #endif -#define osMunmap ((void*(*)(void*,size_t))aSyscall[23].pCurrent) +#define osMunmap ((int(*)(void*,size_t))aSyscall[23].pCurrent) #if HAVE_MREMAP && (!defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0) { "mremap", (sqlite3_syscall_ptr)mremap, 0 }, @@ -29597,6 +39052,18 @@ static struct unix_syscall { #endif #define osLstat ((int(*)(const char*,struct stat*))aSyscall[27].pCurrent) +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) +# ifdef __ANDROID__ + { "ioctl", (sqlite3_syscall_ptr)(int(*)(int, int, ...))ioctl, 0 }, +#define osIoctl ((int(*)(int,int,...))aSyscall[28].pCurrent) +# else + { "ioctl", (sqlite3_syscall_ptr)ioctl, 0 }, +#define osIoctl ((int(*)(int,unsigned long,...))aSyscall[28].pCurrent) +# endif +#else + { "ioctl", (sqlite3_syscall_ptr)0, 0 }, +#endif + }; /* End of the overrideable system calls */ @@ -29615,7 +39082,7 @@ static int robustFchown(int fd, uid_t uid, gid_t gid){ /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "unix" VFSes. Return SQLITE_OK opon successfully updating the +** "unix" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -29698,7 +39165,7 @@ static const char *unixNextSystemCall(sqlite3_vfs *p, const char *zName){ /* ** Do not accept any file descriptor less than this value, in order to avoid -** opening database file using file descriptors that are commonly used for +** opening database file using file descriptors that are commonly used for ** standard input, output, and error. */ #ifndef SQLITE_MINIMUM_FILE_DESCRIPTOR @@ -29736,18 +39203,21 @@ static int robust_open(const char *z, int f, mode_t m){ break; } if( fd>=SQLITE_MINIMUM_FILE_DESCRIPTOR ) break; + if( (f & (O_EXCL|O_CREAT))==(O_EXCL|O_CREAT) ){ + (void)osUnlink(z); + } osClose(fd); - sqlite3_log(SQLITE_WARNING, + sqlite3_log(SQLITE_WARNING, "attempt to open \"%s\" as file descriptor %d", z, fd); fd = -1; - if( osOpen("/dev/null", f, m)<0 ) break; + if( osOpen("/dev/null", O_RDONLY, m)<0 ) break; } if( fd>=0 ){ if( m!=0 ){ struct stat statbuf; - if( osFstat(fd, &statbuf)==0 + if( osFstat(fd, &statbuf)==0 && statbuf.st_size==0 - && (statbuf.st_mode&0777)!=m + && (statbuf.st_mode&0777)!=m ){ osFchmod(fd, m); } @@ -29762,26 +39232,40 @@ static int robust_open(const char *z, int f, mode_t m){ /* ** Helper functions to obtain and relinquish the global mutex. The ** global mutex is used to protect the unixInodeInfo and -** vxworksFileId objects used by this file, all of which may be +** vxworksFileId objects used by this file, all of which may be ** shared by multiple threads. ** -** Function unixMutexHeld() is used to assert() that the global mutex -** is held when required. This function is only used as part of assert() +** Function unixMutexHeld() is used to assert() that the global mutex +** is held when required. This function is only used as part of assert() ** statements. e.g. ** ** unixEnterMutex() ** assert( unixMutexHeld() ); ** unixEnterLeave() +** +** To prevent deadlock, the global unixBigLock must must be acquired +** before the unixInodeInfo.pLockMutex mutex, if both are held. It is +** OK to get the pLockMutex without holding unixBigLock first, but if +** that happens, the unixBigLock mutex must not be acquired until after +** pLockMutex is released. +** +** OK: enter(unixBigLock), enter(pLockInfo) +** OK: enter(unixBigLock) +** OK: enter(pLockInfo) +** ERROR: enter(pLockInfo), enter(unixBigLock) */ +static sqlite3_mutex *unixBigLock = 0; static void unixEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + assert( sqlite3_mutex_notheld(unixBigLock) ); /* Not a recursive mutex */ + sqlite3_mutex_enter(unixBigLock); } static void unixLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + assert( sqlite3_mutex_held(unixBigLock) ); + sqlite3_mutex_leave(unixBigLock); } #ifdef SQLITE_DEBUG static int unixMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(unixBigLock); } #endif @@ -29874,7 +39358,7 @@ static int lockTrace(int fd, int op, struct flock *p){ static int robust_ftruncate(int h, sqlite3_int64 sz){ int rc; #ifdef __ANDROID__ - /* On Android, ftruncate() always uses 32-bit offsets, even if + /* On Android, ftruncate() always uses 32-bit offsets, even if ** _FILE_OFFSET_BITS=64 is defined. This means it is unsafe to attempt to ** truncate a file to any size larger than 2GiB. Silently ignore any ** such attempts. */ @@ -29890,32 +39374,32 @@ static int robust_ftruncate(int h, sqlite3_int64 sz){ ** This routine translates a standard POSIX errno code into something ** useful to the clients of the sqlite3 functions. Specifically, it is ** intended to translate a variety of "try again" errors into SQLITE_BUSY -** and a variety of "please close the file descriptor NOW" errors into +** and a variety of "please close the file descriptor NOW" errors into ** SQLITE_IOERR -** +** ** Errors during initialization of locks, or file system support for locks, ** should handle ENOLCK, ENOTSUP, EOPNOTSUPP separately. */ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { - assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || - (sqliteIOErr == SQLITE_IOERR_UNLOCK) || + assert( (sqliteIOErr == SQLITE_IOERR_LOCK) || + (sqliteIOErr == SQLITE_IOERR_UNLOCK) || (sqliteIOErr == SQLITE_IOERR_RDLOCK) || (sqliteIOErr == SQLITE_IOERR_CHECKRESERVEDLOCK) ); switch (posixError) { - case EACCES: + case EACCES: case EAGAIN: case ETIMEDOUT: case EBUSY: case EINTR: - case ENOLCK: - /* random NFS retry error, unless during file system support + case ENOLCK: + /* random NFS retry error, unless during file system support * introspection, in which it actually means what it says */ return SQLITE_BUSY; - - case EPERM: + + case EPERM: return SQLITE_PERM; - - default: + + default: return sqliteIOErr; } } @@ -29930,7 +39414,7 @@ static int sqliteErrorFromPosixError(int posixError, int sqliteIOErr) { ** ** A pointer to an instance of the following structure can be used as a ** unique file ID in VxWorks. Each instance of this structure contains -** a copy of the canonical filename. There is also a reference count. +** a copy of the canonical filename. There is also a reference count. ** The structure is reclaimed when the number of pointers to it drops to ** zero. ** @@ -29946,7 +39430,7 @@ struct vxworksFileId { }; #if OS_VXWORKS -/* +/* ** All unique filenames are held on a linked list headed by this ** variable: */ @@ -30018,7 +39502,7 @@ static struct vxworksFileId *vxworksFindFileId(const char *zAbsoluteName){ */ unixEnterMutex(); for(pCandidate=vxworksFileList; pCandidate; pCandidate=pCandidate->pNext){ - if( pCandidate->nName==n + if( pCandidate->nName==n && memcmp(pCandidate->zCanonicalName, pNew->zCanonicalName, n)==0 ){ sqlite3_free(pNew); @@ -30111,7 +39595,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** cnt>0 means there are cnt shared locks on the file. ** ** Any attempt to lock or unlock a file first checks the locking -** structure. The fcntl() system call is only invoked to set a +** structure. The fcntl() system call is only invoked to set a ** POSIX lock if the internal lock structure transitions between ** a locked and an unlocked state. ** @@ -30120,7 +39604,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** If you close a file descriptor that points to a file that has locks, ** all locks on that file that are owned by the current process are ** released. To work around this problem, each unixInodeInfo object -** maintains a count of the number of pending locks on tha inode. +** maintains a count of the number of pending locks on the inode. ** When an attempt is made to close an unixFile, if there are ** other unixFile open on the same inode that are holding locks, the call ** to close() the file descriptor is deferred until all of the locks clear. @@ -30134,7 +39618,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** not posix compliant. Under LinuxThreads, a lock created by thread ** A cannot be modified or overridden by a different thread B. ** Only thread A can modify the lock. Locking behavior is correct -** if the appliation uses the newer Native Posix Thread Library (NPTL) +** if the application uses the newer Native Posix Thread Library (NPTL) ** on linux - with NPTL a lock created by thread A can override locks ** in thread B. But there is no way to know at compile-time which ** threading library is being used. So there is no way to know at @@ -30144,7 +39628,7 @@ static void vxworksReleaseFileId(struct vxworksFileId *pId){ ** ** SQLite used to support LinuxThreads. But support for LinuxThreads ** was dropped beginning with version 3.7.0. SQLite will still work with -** LinuxThreads provided that (1) there is no more than one connection +** LinuxThreads provided that (1) there is no more than one connection ** per database file in the same process and (2) database connections ** do not move across threads. */ @@ -30158,28 +39642,52 @@ struct unixFileId { #if OS_VXWORKS struct vxworksFileId *pId; /* Unique file ID for vxworks. */ #else - ino_t ino; /* Inode number */ + /* We are told that some versions of Android contain a bug that + ** sizes ino_t at only 32-bits instead of 64-bits. (See + ** https://android-review.googlesource.com/#/c/115351/3/dist/sqlite3.c) + ** To work around this, always allocate 64-bits for the inode number. + ** On small machines that only have 32-bit inodes, this wastes 4 bytes, + ** but that should not be a big deal. */ + /* WAS: ino_t ino; */ + u64 ino; /* Inode number */ #endif }; /* ** An instance of the following structure is allocated for each open -** inode. Or, on LinuxThreads, there is one of these structures for -** each inode opened by each thread. +** inode. ** ** A single inode can have multiple file descriptors, so each unixFile ** structure contains a pointer to an instance of this object and this ** object keeps a count of the number of unixFile pointing to it. +** +** Mutex rules: +** +** (1) Only the pLockMutex mutex must be held in order to read or write +** any of the locking fields: +** nShared, nLock, eFileLock, bProcessLock, pUnused +** +** (2) When nRef>0, then the following fields are unchanging and can +** be read (but not written) without holding any mutex: +** fileId, pLockMutex +** +** (3) With the exceptions above, all the fields may only be read +** or written while holding the global unixBigLock mutex. +** +** Deadlock prevention: The global unixBigLock mutex may not +** be acquired while holding the pLockMutex mutex. If both unixBigLock +** and pLockMutex are needed, then unixBigLock must be acquired first. */ struct unixInodeInfo { struct unixFileId fileId; /* The lookup key */ - int nShared; /* Number of SHARED locks held */ - unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ - unsigned char bProcessLock; /* An exclusive process lock is held */ + sqlite3_mutex *pLockMutex; /* Hold this mutex for... */ + int nShared; /* Number of SHARED locks held */ + int nLock; /* Number of outstanding file locks */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ int nRef; /* Number of pointers to this structure */ unixShmNode *pShmNode; /* Shared memory associated with this inode */ - int nLock; /* Number of outstanding file locks */ - UnixUnusedFd *pUnused; /* Unused file descriptors to close */ unixInodeInfo *pNext; /* List of all unixInodeInfo objects */ unixInodeInfo *pPrev; /* .... doubly linked */ #if SQLITE_ENABLE_LOCKING_STYLE @@ -30193,8 +39701,26 @@ struct unixInodeInfo { /* ** A lists of all unixInodeInfo objects. +** +** Must hold unixBigLock in order to read or write this variable. */ -static unixInodeInfo *inodeList = 0; +static unixInodeInfo *inodeList = 0; /* All unixInodeInfo objects */ + +#ifdef SQLITE_DEBUG +/* +** True if the inode mutex (on the unixFile.pFileMutex field) is held, or not. +** This routine is used only within assert() to help verify correct mutex +** usage. +*/ +int unixFileMutexHeld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_held(pFile->pInode->pLockMutex); +} +int unixFileMutexNotheld(unixFile *pFile){ + assert( pFile->pInode ); + return sqlite3_mutex_notheld(pFile->pInode->pLockMutex); +} +#endif /* ** @@ -30207,7 +39733,7 @@ static unixInodeInfo *inodeList = 0; ** strerror_r(). ** ** The first argument passed to the macro should be the error code that -** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). +** will be returned to SQLite (e.g. SQLITE_IOERR_DELETE, SQLITE_CANTOPEN). ** The two subsequent arguments should be the name of the OS function that ** failed (e.g. "unlink", "open") and the associated file-system path, ** if any. @@ -30225,7 +39751,7 @@ static int unixLogErrorAtLine( /* If this is not a threadsafe build (SQLITE_THREADSAFE==0), then use ** the strerror() function to obtain the human-readable error message ** equivalent to errno. Otherwise, use strerror_r(). - */ + */ #if SQLITE_THREADSAFE && defined(HAVE_STRERROR_R) char aErr[80]; memset(aErr, 0, sizeof(aErr)); @@ -30233,18 +39759,22 @@ static int unixLogErrorAtLine( /* If STRERROR_R_CHAR_P (set by autoconf scripts) or __USE_GNU is defined, ** assume that the system provides the GNU version of strerror_r() that - ** returns a pointer to a buffer containing the error message. That pointer - ** may point to aErr[], or it may point to some static storage somewhere. - ** Otherwise, assume that the system provides the POSIX version of + ** returns a pointer to a buffer containing the error message. That pointer + ** may point to aErr[], or it may point to some static storage somewhere. + ** Otherwise, assume that the system provides the POSIX version of ** strerror_r(), which always writes an error message into aErr[]. ** ** If the code incorrectly assumes that it is the POSIX version that is ** available, the error message will often be an empty string. Not a - ** huge problem. Incorrectly concluding that the GNU version is available + ** huge problem. Incorrectly concluding that the GNU version is available ** could lead to a segfault though. + ** + ** Forum post 3f13857fa4062301 reports that the Android SDK may use + ** int-type return, depending on its version. */ -#if defined(STRERROR_R_CHAR_P) || defined(__USE_GNU) - zErr = +#if (defined(STRERROR_R_CHAR_P) || defined(__USE_GNU)) \ + && !defined(ANDROID) && !defined(__ANDROID__) + zErr = # endif strerror_r(iErrno, aErr, sizeof(aErr)-1); @@ -30294,12 +39824,13 @@ static void storeLastErrno(unixFile *pFile, int error){ } /* -** Close all file descriptors accumuated in the unixInodeInfo->pUnused list. -*/ +** Close all file descriptors accumulated in the unixInodeInfo->pUnused list. +*/ static void closePendingFds(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; UnixUnusedFd *p; UnixUnusedFd *pNext; + assert( unixFileMutexHeld(pFile) ); for(p=pInode->pUnused; p; p=pNext){ pNext = p->pNext; robust_close(pFile, p->fd, __LINE__); @@ -30311,17 +39842,20 @@ static void closePendingFds(unixFile *pFile){ /* ** Release a unixInodeInfo structure previously allocated by findInodeInfo(). ** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** The global mutex must be held when this routine is called, but the mutex +** on the inode being deleted must NOT be held. */ static void releaseInodeInfo(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; assert( unixMutexHeld() ); + assert( unixFileMutexNotheld(pFile) ); if( ALWAYS(pInode) ){ pInode->nRef--; if( pInode->nRef==0 ){ assert( pInode->pShmNode==0 ); + sqlite3_mutex_enter(pInode->pLockMutex); closePendingFds(pFile); + sqlite3_mutex_leave(pInode->pLockMutex); if( pInode->pPrev ){ assert( pInode->pPrev->pNext==pInode ); pInode->pPrev->pNext = pInode->pNext; @@ -30333,6 +39867,7 @@ static void releaseInodeInfo(unixFile *pFile){ assert( pInode->pNext->pPrev==pInode ); pInode->pNext->pPrev = pInode->pPrev; } + sqlite3_mutex_free(pInode->pLockMutex); sqlite3_free(pInode); } } @@ -30343,8 +39878,7 @@ static void releaseInodeInfo(unixFile *pFile){ ** describes that file descriptor. Create a new one if necessary. The ** return value might be uninitialized if an error occurs. ** -** The mutex entered using the unixEnterMutex() function must be held -** when this function is called. +** The global mutex must held when calling this routine. ** ** Return an appropriate error code. */ @@ -30403,8 +39937,9 @@ static int findInodeInfo( #if OS_VXWORKS fileId.pId = pFile->pId; #else - fileId.ino = statbuf.st_ino; + fileId.ino = (u64)statbuf.st_ino; #endif + assert( unixMutexHeld() ); pInode = inodeList; while( pInode && memcmp(&fileId, &pInode->fileId, sizeof(fileId)) ){ pInode = pInode->pNext; @@ -30416,7 +39951,15 @@ static int findInodeInfo( } memset(pInode, 0, sizeof(*pInode)); memcpy(&pInode->fileId, &fileId, sizeof(fileId)); + if( sqlite3GlobalConfig.bCoreMutex ){ + pInode->pLockMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pInode->pLockMutex==0 ){ + sqlite3_free(pInode); + return SQLITE_NOMEM_BKPT; + } + } pInode->nRef = 1; + assert( unixMutexHeld() ); pInode->pNext = inodeList; pInode->pPrev = 0; if( inodeList ) inodeList->pPrev = pInode; @@ -30437,7 +39980,8 @@ static int fileHasMoved(unixFile *pFile){ #else struct stat buf; return pFile->pInode!=0 && - (osStat(pFile->zPath, &buf)!=0 || buf.st_ino!=pFile->pInode->fileId.ino); + (osStat(pFile->zPath, &buf)!=0 + || (u64)buf.st_ino!=pFile->pInode->fileId.ino); #endif } @@ -30493,7 +40037,7 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ assert( pFile ); assert( pFile->eFileLock<=SHARED_LOCK ); - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ + sqlite3_mutex_enter(pFile->pInode->pLockMutex); /* Check if a thread in this process holds such a lock */ if( pFile->pInode->eFileLock>SHARED_LOCK ){ @@ -30517,16 +40061,57 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ } } #endif - - unixLeaveMutex(); + + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (unix)\n", pFile->h, rc, reserved)); *pResOut = reserved; return rc; } +/* Forward declaration*/ +static int unixSleep(sqlite3_vfs*,int); + /* -** Attempt to set a system-lock on the file pFile. The lock is +** Set a posix-advisory-lock. +** +** There are two versions of this routine. If compiled with +** SQLITE_ENABLE_SETLK_TIMEOUT then the routine has an extra parameter +** which is a pointer to a unixFile. If the unixFile->iBusyTimeout +** value is set, then it is the number of milliseconds to wait before +** failing the lock. The iBusyTimeout value is always reset back to +** zero on each call. +** +** If SQLITE_ENABLE_SETLK_TIMEOUT is not defined, then do a non-blocking +** attempt to set the lock. +*/ +#ifndef SQLITE_ENABLE_SETLK_TIMEOUT +# define osSetPosixAdvisoryLock(h,x,t) osFcntl(h,F_SETLK,x) +#else +static int osSetPosixAdvisoryLock( + int h, /* The file descriptor on which to take the lock */ + struct flock *pLock, /* The description of the lock */ + unixFile *pFile /* Structure holding timeout value */ +){ + int tm = pFile->iBusyTimeout; + int rc = osFcntl(h,F_SETLK,pLock); + while( rc<0 && tm>0 ){ + /* On systems that support some kind of blocking file lock with a timeout, + ** make appropriate changes here to invoke that blocking file lock. On + ** generic posix, however, there is no such API. So we simply try the + ** lock once every millisecond until either the timeout expires, or until + ** the lock is obtained. */ + unixSleep(0,1000); + rc = osFcntl(h,F_SETLK,pLock); + tm--; + } + return rc; +} +#endif /* SQLITE_ENABLE_SETLK_TIMEOUT */ + + +/* +** Attempt to set a system-lock on the file pFile. The lock is ** described by pLock. ** ** If the pFile was opened read/write from unix-excl, then the only lock @@ -30547,8 +40132,8 @@ static int unixCheckReservedLock(sqlite3_file *id, int *pResOut){ static int unixFileLock(unixFile *pFile, struct flock *pLock){ int rc; unixInodeInfo *pInode = pFile->pInode; - assert( unixMutexHeld() ); assert( pInode!=0 ); + assert( sqlite3_mutex_held(pInode->pLockMutex) ); if( (pFile->ctrlFlags & (UNIXFILE_EXCL|UNIXFILE_RDONLY))==UNIXFILE_EXCL ){ if( pInode->bProcessLock==0 ){ struct flock lock; @@ -30557,7 +40142,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ lock.l_start = SHARED_FIRST; lock.l_len = SHARED_SIZE; lock.l_type = F_WRLCK; - rc = osFcntl(pFile->h, F_SETLK, &lock); + rc = osSetPosixAdvisoryLock(pFile->h, &lock, pFile); if( rc<0 ) return rc; pInode->bProcessLock = 1; pInode->nLock++; @@ -30565,7 +40150,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ rc = 0; } }else{ - rc = osFcntl(pFile->h, F_SETLK, pLock); + rc = osSetPosixAdvisoryLock(pFile->h, pLock, pFile); } return rc; } @@ -30587,7 +40172,7 @@ static int unixFileLock(unixFile *pFile, struct flock *pLock){ ** ** UNLOCKED -> SHARED ** SHARED -> RESERVED -** SHARED -> (PENDING) -> EXCLUSIVE +** SHARED -> EXCLUSIVE ** RESERVED -> (PENDING) -> EXCLUSIVE ** PENDING -> EXCLUSIVE ** @@ -30602,7 +40187,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** slightly in order to be compatible with Windows95 systems simultaneously ** accessing the same database file, in case that is ever required. ** - ** Symbols defined in os.h indentify the 'pending byte' and the 'reserved + ** Symbols defined in os.h identify the 'pending byte' and the 'reserved ** byte', each single bytes at well known offsets, and the 'shared byte ** range', a range of 510 bytes at a well known offset. ** @@ -30610,7 +40195,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** byte'. If this is successful, 'shared byte range' is read-locked ** and the lock on the 'pending byte' released. (Legacy note: When ** SQLite was first developed, Windows95 systems were still very common, - ** and Widnows95 lacks a shared-lock capability. So on Windows95, a + ** and Windows95 lacks a shared-lock capability. So on Windows95, a ** single randomly selected by from the 'shared byte range' is locked. ** Windows95 is now pretty much extinct, but this work-around for the ** lack of shared-locks on Windows95 lives on, for backwards @@ -30618,21 +40203,22 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** ** A process may only obtain a RESERVED lock after it has a SHARED lock. ** A RESERVED lock is implemented by grabbing a write-lock on the - ** 'reserved byte'. + ** 'reserved byte'. ** - ** A process may only obtain a PENDING lock after it has obtained a - ** SHARED lock. A PENDING lock is implemented by obtaining a write-lock - ** on the 'pending byte'. This ensures that no new SHARED locks can be - ** obtained, but existing SHARED locks are allowed to persist. A process - ** does not have to obtain a RESERVED lock on the way to a PENDING lock. - ** This property is used by the algorithm for rolling back a journal file - ** after a crash. + ** An EXCLUSIVE lock may only be requested after either a SHARED or + ** RESERVED lock is held. An EXCLUSIVE lock is implemented by obtaining + ** a write-lock on the entire 'shared byte range'. Since all other locks + ** require a read-lock on one of the bytes within this range, this ensures + ** that no other locks are held on the database. ** - ** An EXCLUSIVE lock, obtained after a PENDING lock is held, is - ** implemented by obtaining a write-lock on the entire 'shared byte - ** range'. Since all other locks require a read-lock on one of the bytes - ** within this range, this ensures that no other locks are held on the - ** database. + ** If a process that holds a RESERVED lock requests an EXCLUSIVE, then + ** a PENDING lock is obtained first. A PENDING lock is implemented by + ** obtaining a write-lock on the 'pending byte'. This ensures that no new + ** SHARED locks can be obtained, but existing SHARED locks are allowed to + ** persist. If the call to this function fails to obtain the EXCLUSIVE + ** lock in this case, it holds the PENDING lock instead. The client may + ** then re-attempt the EXCLUSIVE lock later on, after existing SHARED + ** locks have cleared. */ int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; @@ -30658,7 +40244,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct. ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); @@ -30667,13 +40253,13 @@ static int unixLock(sqlite3_file *id, int eFileLock){ /* This mutex is needed because pFile->pInode is shared across threads */ - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. */ - if( (pFile->eFileLock!=pInode->eFileLock && + if( (pFile->eFileLock!=pInode->eFileLock && (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; @@ -30684,7 +40270,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ ** has a SHARED or RESERVED lock, then increment reference counts and ** return SQLITE_OK. */ - if( eFileLock==SHARED_LOCK && + if( eFileLock==SHARED_LOCK && (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ assert( eFileLock==SHARED_LOCK ); assert( pFile->eFileLock==0 ); @@ -30702,8 +40288,8 @@ static int unixLock(sqlite3_file *id, int eFileLock){ */ lock.l_len = 1L; lock.l_whence = SEEK_SET; - if( eFileLock==SHARED_LOCK - || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLockeFileLock==RESERVED_LOCK) ){ lock.l_type = (eFileLock==SHARED_LOCK?F_RDLCK:F_WRLCK); lock.l_start = PENDING_BYTE; @@ -30714,6 +40300,9 @@ static int unixLock(sqlite3_file *id, int eFileLock){ storeLastErrno(pFile, tErrno); } goto end_lock; + }else if( eFileLock==EXCLUSIVE_LOCK ){ + pFile->eFileLock = PENDING_LOCK; + pInode->eFileLock = PENDING_LOCK; } } @@ -30741,7 +40330,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ if( unixFileLock(pFile, &lock) && rc==SQLITE_OK ){ /* This could happen with a network mount */ tErrno = errno; - rc = SQLITE_IOERR_UNLOCK; + rc = SQLITE_IOERR_UNLOCK; } if( rc ){ @@ -30783,7 +40372,7 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } } } - + #ifdef SQLITE_DEBUG /* Set up the transaction-counter change checking flags when @@ -30801,18 +40390,14 @@ static int unixLock(sqlite3_file *id, int eFileLock){ } #endif - if( rc==SQLITE_OK ){ pFile->eFileLock = eFileLock; pInode->eFileLock = eFileLock; - }else if( eFileLock==EXCLUSIVE_LOCK ){ - pFile->eFileLock = PENDING_LOCK; - pInode->eFileLock = PENDING_LOCK; } end_lock: - unixLeaveMutex(); - OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), + sqlite3_mutex_leave(pInode->pLockMutex); + OSTRACE(("LOCK %d %s %s (unix)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; } @@ -30823,11 +40408,12 @@ end_lock: */ static void setPendingFd(unixFile *pFile){ unixInodeInfo *pInode = pFile->pInode; - UnixUnusedFd *p = pFile->pUnused; + UnixUnusedFd *p = pFile->pPreallocatedUnused; + assert( unixFileMutexHeld(pFile) ); p->pNext = pInode->pUnused; pInode->pUnused = p; pFile->h = -1; - pFile->pUnused = 0; + pFile->pPreallocatedUnused = 0; } /* @@ -30836,11 +40422,11 @@ static void setPendingFd(unixFile *pFile){ ** ** If the locking level of the file descriptor is already at or below ** the requested locking level, this routine is a no-op. -** +** ** If handleNFSUnlock is true, then on downgrading an EXCLUSIVE_LOCK to SHARED ** the byte range is divided into 2 parts and the first part is unlocked then -** set to a read lock, then the other part is simply unlocked. This works -** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to +** set to a read lock, then the other part is simply unlocked. This works +** around a bug in BSD NFS lockd (also seen on MacOSX 10.3+) that fails to ** remove the write lock on a region when a read lock is set. */ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ @@ -30858,8 +40444,8 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); @@ -30878,7 +40464,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ /* downgrading to a shared lock on NFS involves clearing the write lock ** before establishing the readlock - to avoid a race condition we downgrade - ** the lock in 2 blocks, so that part of the range will be covered by a + ** the lock in 2 blocks, so that part of the range will be covered by a ** write lock until the rest is covered by a read lock: ** 1: [WWWWW] ** 2: [....W] @@ -30894,7 +40480,7 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ if( handleNFSUnlock ){ int tErrno; /* Error code from system call errors */ off_t divSize = SHARED_SIZE - 1; - + lock.l_type = F_UNLCK; lock.l_whence = SEEK_SET; lock.l_start = SHARED_FIRST; @@ -30936,11 +40522,11 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ lock.l_len = SHARED_SIZE; if( unixFileLock(pFile, &lock) ){ /* In theory, the call to unixFileLock() cannot fail because another - ** process is holding an incompatible lock. If it does, this + ** process is holding an incompatible lock. If it does, this ** indicates that the other process is not following the locking ** protocol. If this happens, return SQLITE_IOERR_RDLOCK. Returning - ** SQLITE_BUSY would confuse the upper layer (in practice it causes - ** an assert to fail). */ + ** SQLITE_BUSY would confuse the upper layer (in practice it causes + ** an assert to fail). */ rc = SQLITE_IOERR_RDLOCK; storeLastErrno(pFile, errno); goto end_unlock; @@ -30985,14 +40571,14 @@ static int posixUnlock(sqlite3_file *id, int eFileLock, int handleNFSUnlock){ */ pInode->nLock--; assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - closePendingFds(pFile); - } + if( pInode->nLock==0 ) closePendingFds(pFile); } end_unlock: - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } return rc; } @@ -31016,7 +40602,7 @@ static void unixUnmapfile(unixFile *pFd); #endif /* -** This function performs the parts of the "close file" operation +** This function performs the parts of the "close file" operation ** common to all locking schemes. It closes the directory and file ** handles, if they are valid, and sets all fields of the unixFile ** structure to 0. @@ -31052,7 +40638,7 @@ static int closeUnixFile(sqlite3_file *id){ #endif OSTRACE(("CLOSE %-3d\n", pFile->h)); OpenCounter(-1); - sqlite3_free(pFile->pUnused); + sqlite3_free(pFile->pPreallocatedUnused); memset(pFile, 0, sizeof(unixFile)); return SQLITE_OK; } @@ -31063,23 +40649,30 @@ static int closeUnixFile(sqlite3_file *id){ static int unixClose(sqlite3_file *id){ int rc = SQLITE_OK; unixFile *pFile = (unixFile *)id; + unixInodeInfo *pInode = pFile->pInode; + + assert( pInode!=0 ); verifyDbFile(pFile); unixUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); /* unixFile.pInode is always valid here. Otherwise, a different close ** routine (e.g. nolockClose()) would be called instead. */ assert( pFile->pInode->nLock>0 || pFile->pInode->bProcessLock==0 ); - if( ALWAYS(pFile->pInode) && pFile->pInode->nLock ){ + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ /* If there are outstanding locks, do not actually close the file just ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->pUnused list. It will be automatically closed + ** descriptor to pInode->pUnused list. It will be automatically closed ** when the last lock is cleared. */ setPendingFd(pFile); } + sqlite3_mutex_leave(pInode->pLockMutex); releaseInodeInfo(pFile); + assert( pFile->pShm==0 ); rc = closeUnixFile(id); unixLeaveMutex(); return rc; @@ -31159,26 +40752,22 @@ static int nolockClose(sqlite3_file *id) { /* ** This routine checks if there is a RESERVED lock held on the specified -** file by this or any other process. If such a lock is held, set *pResOut -** to a non-zero value otherwise *pResOut is set to zero. The return value -** is set to SQLITE_OK unless an I/O error occurs during lock checking. -** -** In dotfile locking, either a lock exists or it does not. So in this -** variation of CheckReservedLock(), *pResOut is set to true if any lock -** is held on the file and false if the file is unlocked. +** file by this or any other process. If the caller holds a SHARED +** or greater lock when it is called, then it is assumed that no other +** client may hold RESERVED. Or, if the caller holds no lock, then it +** is assumed another client holds RESERVED if the lock-file exists. */ static int dotlockCheckReservedLock(sqlite3_file *id, int *pResOut) { - int rc = SQLITE_OK; - int reserved = 0; unixFile *pFile = (unixFile*)id; - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - - assert( pFile ); - reserved = osAccess((const char*)pFile->lockingContext, 0)==0; - OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, rc, reserved)); - *pResOut = reserved; - return rc; + + if( pFile->eFileLock>=SHARED_LOCK ){ + *pResOut = 0; + }else{ + *pResOut = osAccess((const char*)pFile->lockingContext, 0)==0; + } + OSTRACE(("TEST WR-LOCK %d %d %d (dotlock)\n", pFile->h, 0, *pResOut)); + return SQLITE_OK; } /* @@ -31227,7 +40816,7 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { #endif return SQLITE_OK; } - + /* grab an exclusive lock */ rc = osMkdir(zLockFile, 0777); if( rc<0 ){ @@ -31242,8 +40831,8 @@ static int dotlockLock(sqlite3_file *id, int eFileLock) { } } return rc; - } - + } + /* got it, set the type and return ok */ pFile->eFileLock = eFileLock; return rc; @@ -31267,7 +40856,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { OSTRACE(("UNLOCK %d %d was %d pid=%d (dotlock)\n", pFile->h, eFileLock, pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); - + /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; @@ -31280,7 +40869,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { pFile->eFileLock = SHARED_LOCK; return SQLITE_OK; } - + /* To fully unlock the database, delete the lock file */ assert( eFileLock==NO_LOCK ); rc = osRmdir(zLockFile); @@ -31292,7 +40881,7 @@ static int dotlockUnlock(sqlite3_file *id, int eFileLock) { rc = SQLITE_IOERR_UNLOCK; storeLastErrno(pFile, tErrno); } - return rc; + return rc; } pFile->eFileLock = NO_LOCK; return SQLITE_OK; @@ -31339,7 +40928,7 @@ static int robust_flock(int fd, int op){ #else # define robust_flock(a,b) flock(a,b) #endif - + /* ** This routine checks if there is a RESERVED lock held on the specified @@ -31348,54 +40937,33 @@ static int robust_flock(int fd, int op){ ** is set to SQLITE_OK unless an I/O error occurs during lock checking. */ static int flockCheckReservedLock(sqlite3_file *id, int *pResOut){ - int rc = SQLITE_OK; - int reserved = 0; +#ifdef SQLITE_DEBUG unixFile *pFile = (unixFile*)id; - - SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - - assert( pFile ); - - /* Check if a thread in this process holds such a lock */ - if( pFile->eFileLock>SHARED_LOCK ){ - reserved = 1; - } - - /* Otherwise see if some other process holds it. */ - if( !reserved ){ - /* attempt to get the lock */ - int lrc = robust_flock(pFile->h, LOCK_EX | LOCK_NB); - if( !lrc ){ - /* got the lock, unlock it */ - lrc = robust_flock(pFile->h, LOCK_UN); - if ( lrc ) { - int tErrno = errno; - /* unlock failed with an error */ - lrc = SQLITE_IOERR_UNLOCK; - storeLastErrno(pFile, tErrno); - rc = lrc; - } - } else { - int tErrno = errno; - reserved = 1; - /* someone else might have it reserved */ - lrc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_LOCK); - if( IS_LOCK_ERROR(lrc) ){ - storeLastErrno(pFile, tErrno); - rc = lrc; - } - } - } - OSTRACE(("TEST WR-LOCK %d %d %d (flock)\n", pFile->h, rc, reserved)); +#else + UNUSED_PARAMETER(id); +#endif -#ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ - rc = SQLITE_OK; - reserved=1; - } -#endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ - *pResOut = reserved; - return rc; + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); + + assert( pFile ); + assert( pFile->eFileLock<=SHARED_LOCK ); + + /* The flock VFS only ever takes exclusive locks (see function flockLock). + ** Therefore, if this connection is holding any lock at all, no other + ** connection may be holding a RESERVED lock. So set *pResOut to 0 + ** in this case. + ** + ** Or, this connection may be holding no lock. In that case, set *pResOut to + ** 0 as well. The caller will then attempt to take an EXCLUSIVE lock on the + ** db in order to roll the hot journal back. If there is another connection + ** holding a lock, that attempt will fail and an SQLITE_BUSY returned to + ** the user. With other VFS, we try to avoid this, in order to allow a reader + ** to proceed while a writer is preparing its transaction. But that won't + ** work with the flock VFS - as it always takes EXCLUSIVE locks - so it is + ** not a problem in this case. */ + *pResOut = 0; + + return SQLITE_OK; } /* @@ -31433,15 +41001,15 @@ static int flockLock(sqlite3_file *id, int eFileLock) { assert( pFile ); - /* if we already have a lock, it is exclusive. + /* if we already have a lock, it is exclusive. ** Just adjust level and punt on outta here. */ if (pFile->eFileLock > NO_LOCK) { pFile->eFileLock = eFileLock; return SQLITE_OK; } - + /* grab an exclusive lock */ - + if (robust_flock(pFile->h, LOCK_EX | LOCK_NB)) { int tErrno = errno; /* didn't get, must be busy */ @@ -31453,10 +41021,10 @@ static int flockLock(sqlite3_file *id, int eFileLock) { /* got it, set the type and return ok */ pFile->eFileLock = eFileLock; } - OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), + OSTRACE(("LOCK %d %s %s (flock)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS - if( (rc & SQLITE_IOERR) == SQLITE_IOERR ){ + if( (rc & 0xff) == SQLITE_IOERR ){ rc = SQLITE_BUSY; } #endif /* SQLITE_IGNORE_FLOCK_LOCK_ERRORS */ @@ -31473,23 +41041,23 @@ static int flockLock(sqlite3_file *id, int eFileLock) { */ static int flockUnlock(sqlite3_file *id, int eFileLock) { unixFile *pFile = (unixFile*)id; - + assert( pFile ); OSTRACE(("UNLOCK %d %d was %d pid=%d (flock)\n", pFile->h, eFileLock, pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); - + /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; } - + /* shared can just be set because we always have an exclusive */ if (eFileLock==SHARED_LOCK) { pFile->eFileLock = eFileLock; return SQLITE_OK; } - + /* no, really, unlock. */ if( robust_flock(pFile->h, LOCK_UN) ){ #ifdef SQLITE_IGNORE_FLOCK_LOCK_ERRORS @@ -31540,14 +41108,14 @@ static int semXCheckReservedLock(sqlite3_file *id, int *pResOut) { unixFile *pFile = (unixFile*)id; SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - + assert( pFile ); /* Check if a thread in this process holds such a lock */ if( pFile->eFileLock>SHARED_LOCK ){ reserved = 1; } - + /* Otherwise see if some other process holds it. */ if( !reserved ){ sem_t *pSem = pFile->pInode->pSem; @@ -31606,14 +41174,14 @@ static int semXLock(sqlite3_file *id, int eFileLock) { sem_t *pSem = pFile->pInode->pSem; int rc = SQLITE_OK; - /* if we already have a lock, it is exclusive. + /* if we already have a lock, it is exclusive. ** Just adjust level and punt on outta here. */ if (pFile->eFileLock > NO_LOCK) { pFile->eFileLock = eFileLock; rc = SQLITE_OK; goto sem_end_lock; } - + /* lock semaphore now but bail out when already locked. */ if( sem_trywait(pSem)==-1 ){ rc = SQLITE_BUSY; @@ -31643,18 +41211,18 @@ static int semXUnlock(sqlite3_file *id, int eFileLock) { OSTRACE(("UNLOCK %d %d was %d pid=%d (sem)\n", pFile->h, eFileLock, pFile->eFileLock, osGetpid(0))); assert( eFileLock<=SHARED_LOCK ); - + /* no-op if possible */ if( pFile->eFileLock==eFileLock ){ return SQLITE_OK; } - + /* shared can just be set because we always have an exclusive */ if (eFileLock==SHARED_LOCK) { pFile->eFileLock = eFileLock; return SQLITE_OK; } - + /* no, really unlock. */ if ( sem_post(pSem)==-1 ) { int rc, tErrno = errno; @@ -31662,7 +41230,7 @@ static int semXUnlock(sqlite3_file *id, int eFileLock) { if( IS_LOCK_ERROR(rc) ){ storeLastErrno(pFile, tErrno); } - return rc; + return rc; } pFile->eFileLock = NO_LOCK; return SQLITE_OK; @@ -31676,6 +41244,7 @@ static int semXClose(sqlite3_file *id) { unixFile *pFile = (unixFile*)id; semXUnlock(id, NO_LOCK); assert( pFile ); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); releaseInodeInfo(pFile); unixLeaveMutex(); @@ -31727,7 +41296,7 @@ struct ByteRangeLockPB2 /* ** This is a utility for setting or clearing a bit-range lock on an ** AFP filesystem. -** +** ** Return SQLITE_OK on success, SQLITE_BUSY on failure. */ static int afpSetLock( @@ -31739,14 +41308,14 @@ static int afpSetLock( ){ struct ByteRangeLockPB2 pb; int err; - + pb.unLockFlag = setLockFlag ? 0 : 1; pb.startEndFlag = 0; pb.offset = offset; - pb.length = length; + pb.length = length; pb.fd = pFile->h; - - OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", + + OSTRACE(("AFPSETLOCK [%s] for %d%s in range %llx:%llx\n", (setLockFlag?"ON":"OFF"), pFile->h, (pb.fd==-1?"[testval-1]":""), offset, length)); err = fsctl(path, afpfsByteRangeLock2FSCTL, &pb, 0); @@ -31781,27 +41350,26 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ int reserved = 0; unixFile *pFile = (unixFile*)id; afpLockingContext *context; - + SimulateIOError( return SQLITE_IOERR_CHECKRESERVEDLOCK; ); - + assert( pFile ); context = (afpLockingContext *) pFile->lockingContext; if( context->reserved ){ *pResOut = 1; return SQLITE_OK; } - unixEnterMutex(); /* Because pFile->pInode is shared across threads */ - + sqlite3_mutex_enter(pFile->pInode->pLockMutex); /* Check if a thread in this process holds such a lock */ if( pFile->pInode->eFileLock>SHARED_LOCK ){ reserved = 1; } - + /* Otherwise see if some other process holds it. */ if( !reserved ){ /* lock the RESERVED byte */ - int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); + int lrc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1,1); if( SQLITE_OK==lrc ){ /* if we succeeded in taking the reserved lock, unlock it to restore ** the original state */ @@ -31814,10 +41382,10 @@ static int afpCheckReservedLock(sqlite3_file *id, int *pResOut){ rc=lrc; } } - - unixLeaveMutex(); + + sqlite3_mutex_leave(pFile->pInode->pLockMutex); OSTRACE(("TEST WR-LOCK %d %d %d (afp)\n", pFile->h, rc, reserved)); - + *pResOut = reserved; return rc; } @@ -31851,7 +41419,7 @@ static int afpLock(sqlite3_file *id, int eFileLock){ unixFile *pFile = (unixFile*)id; unixInodeInfo *pInode = pFile->pInode; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; - + assert( pFile ); OSTRACE(("LOCK %d %s was %s(%s,%d) pid=%d (afp)\n", pFile->h, azFileLock(eFileLock), azFileLock(pFile->eFileLock), @@ -31869,33 +41437,33 @@ static int afpLock(sqlite3_file *id, int eFileLock){ /* Make sure the locking sequence is correct ** (1) We never move from unlocked to anything higher than shared lock. - ** (2) SQLite never explicitly requests a pendig lock. + ** (2) SQLite never explicitly requests a pending lock. ** (3) A shared lock is always held when a reserve lock is requested. */ assert( pFile->eFileLock!=NO_LOCK || eFileLock==SHARED_LOCK ); assert( eFileLock!=PENDING_LOCK ); assert( eFileLock!=RESERVED_LOCK || pFile->eFileLock==SHARED_LOCK ); - + /* This mutex is needed because pFile->pInode is shared across threads */ - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); /* If some thread using this PID has a lock via a different unixFile* ** handle that precludes the requested lock, return BUSY. */ - if( (pFile->eFileLock!=pInode->eFileLock && + if( (pFile->eFileLock!=pInode->eFileLock && (pInode->eFileLock>=PENDING_LOCK || eFileLock>SHARED_LOCK)) ){ rc = SQLITE_BUSY; goto afp_end_lock; } - + /* If a SHARED lock is requested, and some thread using this PID already ** has a SHARED or RESERVED lock, then increment reference counts and ** return SQLITE_OK. */ - if( eFileLock==SHARED_LOCK && + if( eFileLock==SHARED_LOCK && (pInode->eFileLock==SHARED_LOCK || pInode->eFileLock==RESERVED_LOCK) ){ assert( eFileLock==SHARED_LOCK ); assert( pFile->eFileLock==0 ); @@ -31905,12 +41473,12 @@ static int afpLock(sqlite3_file *id, int eFileLock){ pInode->nLock++; goto afp_end_lock; } - + /* A PENDING lock is needed before acquiring a SHARED lock and before ** acquiring an EXCLUSIVE lock. For the SHARED lock, the PENDING will ** be released. */ - if( eFileLock==SHARED_LOCK + if( eFileLock==SHARED_LOCK || (eFileLock==EXCLUSIVE_LOCK && pFile->eFileLocknShared==0 ); assert( pInode->eFileLock==0 ); - + mask = (sizeof(long)==8) ? LARGEST_INT64 : 0x7fffffff; /* Now get the read-lock SHARED_LOCK */ /* note that the quality of the randomness doesn't matter that much */ - lk = random(); + lk = random(); pInode->sharedByte = (lk & mask)%(SHARED_SIZE - 1); - lrc1 = afpSetLock(context->dbPath, pFile, + lrc1 = afpSetLock(context->dbPath, pFile, SHARED_FIRST+pInode->sharedByte, 1, 1); if( IS_LOCK_ERROR(lrc1) ){ lrc1Errno = pFile->lastErrno; } /* Drop the temporary PENDING lock */ lrc2 = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); - + if( IS_LOCK_ERROR(lrc1) ) { storeLastErrno(pFile, lrc1Errno); rc = lrc1; @@ -31978,34 +41546,34 @@ static int afpLock(sqlite3_file *id, int eFileLock){ } if (!failed && eFileLock == EXCLUSIVE_LOCK) { /* Acquire an EXCLUSIVE lock */ - - /* Remove the shared lock before trying the range. we'll need to + + /* Remove the shared lock before trying the range. we'll need to ** reestablish the shared lock if we can't get the afpUnlock */ if( !(failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 0)) ){ int failed2 = SQLITE_OK; - /* now attemmpt to get the exclusive lock range */ - failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, + /* now attempt to get the exclusive lock range */ + failed = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 1); - if( failed && (failed2 = afpSetLock(context->dbPath, pFile, + if( failed && (failed2 = afpSetLock(context->dbPath, pFile, SHARED_FIRST + pInode->sharedByte, 1, 1)) ){ /* Can't reestablish the shared lock. Sqlite can't deal, this is ** a critical I/O error */ - rc = ((failed & SQLITE_IOERR) == SQLITE_IOERR) ? failed2 : + rc = ((failed & 0xff) == SQLITE_IOERR) ? failed2 : SQLITE_IOERR_LOCK; goto afp_end_lock; - } + } }else{ - rc = failed; + rc = failed; } } if( failed ){ rc = failed; } } - + if( rc==SQLITE_OK ){ pFile->eFileLock = eFileLock; pInode->eFileLock = eFileLock; @@ -32013,10 +41581,10 @@ static int afpLock(sqlite3_file *id, int eFileLock){ pFile->eFileLock = PENDING_LOCK; pInode->eFileLock = PENDING_LOCK; } - + afp_end_lock: - unixLeaveMutex(); - OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), + sqlite3_mutex_leave(pInode->pLockMutex); + OSTRACE(("LOCK %d %s %s (afp)\n", pFile->h, azFileLock(eFileLock), rc==SQLITE_OK ? "ok" : "failed")); return rc; } @@ -32034,9 +41602,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unixInodeInfo *pInode; afpLockingContext *context = (afpLockingContext *) pFile->lockingContext; int skipShared = 0; -#ifdef SQLITE_TEST - int h = pFile->h; -#endif assert( pFile ); OSTRACE(("UNLOCK %d %d was %d(%d,%d) pid=%d (afp)\n", pFile->h, eFileLock, @@ -32047,15 +41612,12 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( pFile->eFileLock<=eFileLock ){ return SQLITE_OK; } - unixEnterMutex(); pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); assert( pInode->nShared!=0 ); if( pFile->eFileLock>SHARED_LOCK ){ assert( pInode->eFileLock==pFile->eFileLock ); - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); - + #ifdef SQLITE_DEBUG /* When reducing a lock such that other processes can start ** reading the database file again, make sure that the @@ -32070,7 +41632,7 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { || pFile->transCntrChng==1 ); pFile->inNormalWrite = 0; #endif - + if( pFile->eFileLock==EXCLUSIVE_LOCK ){ rc = afpSetLock(context->dbPath, pFile, SHARED_FIRST, SHARED_SIZE, 0); if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1) ){ @@ -32083,11 +41645,11 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { } if( rc==SQLITE_OK && pFile->eFileLock>=PENDING_LOCK ){ rc = afpSetLock(context->dbPath, pFile, PENDING_BYTE, 1, 0); - } + } if( rc==SQLITE_OK && pFile->eFileLock>=RESERVED_LOCK && context->reserved ){ rc = afpSetLock(context->dbPath, pFile, RESERVED_BYTE, 1, 0); - if( !rc ){ - context->reserved = 0; + if( !rc ){ + context->reserved = 0; } } if( rc==SQLITE_OK && (eFileLock==SHARED_LOCK || pInode->nShared>1)){ @@ -32103,9 +41665,6 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { unsigned long long sharedLockByte = SHARED_FIRST+pInode->sharedByte; pInode->nShared--; if( pInode->nShared==0 ){ - SimulateIOErrorBenign(1); - SimulateIOError( h=(-1) ) - SimulateIOErrorBenign(0); if( !skipShared ){ rc = afpSetLock(context->dbPath, pFile, sharedLockByte, 1, 0); } @@ -32117,33 +41676,39 @@ static int afpUnlock(sqlite3_file *id, int eFileLock) { if( rc==SQLITE_OK ){ pInode->nLock--; assert( pInode->nLock>=0 ); - if( pInode->nLock==0 ){ - closePendingFds(pFile); - } + if( pInode->nLock==0 ) closePendingFds(pFile); } } - - unixLeaveMutex(); - if( rc==SQLITE_OK ) pFile->eFileLock = eFileLock; + + sqlite3_mutex_leave(pInode->pLockMutex); + if( rc==SQLITE_OK ){ + pFile->eFileLock = eFileLock; + } return rc; } /* -** Close a file & cleanup AFP specific locking context +** Close a file & cleanup AFP specific locking context */ static int afpClose(sqlite3_file *id) { int rc = SQLITE_OK; unixFile *pFile = (unixFile*)id; assert( id!=0 ); afpUnlock(id, NO_LOCK); + assert( unixFileMutexNotheld(pFile) ); unixEnterMutex(); - if( pFile->pInode && pFile->pInode->nLock ){ - /* If there are outstanding locks, do not actually close the file just - ** yet because that would clear those locks. Instead, add the file - ** descriptor to pInode->aPending. It will be automatically closed when - ** the last lock is cleared. - */ - setPendingFd(pFile); + if( pFile->pInode ){ + unixInodeInfo *pInode = pFile->pInode; + sqlite3_mutex_enter(pInode->pLockMutex); + if( pInode->nLock ){ + /* If there are outstanding locks, do not actually close the file just + ** yet because that would clear those locks. Instead, add the file + ** descriptor to pInode->aPending. It will be automatically closed when + ** the last lock is cleared. + */ + setPendingFd(pFile); + } + sqlite3_mutex_leave(pInode->pLockMutex); } releaseInodeInfo(pFile); sqlite3_free(pFile->lockingContext); @@ -32181,7 +41746,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ /* ** The code above is the NFS lock implementation. The code is specific ** to MacOSX and does not work on other unix platforms. No alternative -** is available. +** is available. ** ********************* End of the NFS lock implementation ********************** ******************************************************************************/ @@ -32189,7 +41754,7 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ /****************************************************************************** **************** Non-locking sqlite3_file methods ***************************** ** -** The next division contains implementations for all methods of the +** The next division contains implementations for all methods of the ** sqlite3_file object other than the locking methods. The locking ** methods were defined in divisions above (one locking method per ** division). Those methods that are common to all locking modes @@ -32197,15 +41762,9 @@ static int nfsUnlock(sqlite3_file *id, int eFileLock){ */ /* -** Seek to the offset passed as the second argument, then read cnt +** Seek to the offset passed as the second argument, then read cnt ** bytes into pBuf. Return the number of bytes actually read. ** -** NB: If you define USE_PREAD or USE_PREAD64, then it might also -** be necessary to define _XOPEN_SOURCE to be 500. This varies from -** one system to another. Since SQLite does not define USE_PREAD -** in any form by default, we will not attempt to define _XOPEN_SOURCE. -** See tickets #2741 and #2681. -** ** To avoid stomping the errno value on a failed read the lastErrno value ** is set before returning. */ @@ -32259,8 +41818,8 @@ static int seekAndRead(unixFile *id, sqlite3_int64 offset, void *pBuf, int cnt){ ** wrong. */ static int unixRead( - sqlite3_file *id, - void *pBuf, + sqlite3_file *id, + void *pBuf, int amt, sqlite3_int64 offset ){ @@ -32270,17 +41829,17 @@ static int unixRead( assert( offset>=0 ); assert( amt>0 ); - /* If this is a database file (not a journal, master-journal or temp + /* If this is a database file (not a journal, super-journal or temp ** file), the bytes in the locking range should never be read or written. */ #if 0 - assert( pFile->pUnused==0 + assert( pFile->pPreallocatedUnused==0 || offset>=PENDING_BYTE+512 - || offset+amt<=PENDING_BYTE + || offset+amt<=PENDING_BYTE ); #endif #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -32300,7 +41859,24 @@ static int unixRead( if( got==amt ){ return SQLITE_OK; }else if( got<0 ){ - /* lastErrno set by seekAndRead */ + /* pFile->lastErrno has been set by seekAndRead(). + ** Usually we return SQLITE_IOERR_READ here, though for some + ** kinds of errors we return SQLITE_IOERR_CORRUPTFS. The + ** SQLITE_IOERR_CORRUPTFS will be converted into SQLITE_CORRUPT + ** prior to returning to the application by the sqlite3ApiExit() + ** routine. + */ + switch( pFile->lastErrno ){ + case ERANGE: + case EIO: +#ifdef ENXIO + case ENXIO: +#endif +#ifdef EDEVERR + case EDEVERR: +#endif + return SQLITE_IOERR_CORRUPTFS; + } return SQLITE_IOERR_READ; }else{ storeLastErrno(pFile, 0); /* not a system error */ @@ -32313,7 +41889,7 @@ static int unixRead( /* ** Attempt to seek the file-descriptor passed as the first argument to ** absolute offset iOff, then attempt to write nBuf bytes of data from -** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, +** pBuf to it. If an error occurs, return -1 and set *piErrno. Otherwise, ** return the actual number of bytes written (which may be less than ** nBuf). */ @@ -32373,22 +41949,22 @@ static int seekAndWrite(unixFile *id, i64 offset, const void *pBuf, int cnt){ ** or some other error code on failure. */ static int unixWrite( - sqlite3_file *id, - const void *pBuf, + sqlite3_file *id, + const void *pBuf, int amt, - sqlite3_int64 offset + sqlite3_int64 offset ){ unixFile *pFile = (unixFile*)id; int wrote = 0; assert( id ); assert( amt>0 ); - /* If this is a database file (not a journal, master-journal or temp + /* If this is a database file (not a journal, super-journal or temp ** file), the bytes in the locking range should never be read or written. */ #if 0 - assert( pFile->pUnused==0 + assert( pFile->pPreallocatedUnused==0 || offset>=PENDING_BYTE+512 - || offset+amt<=PENDING_BYTE + || offset+amt<=PENDING_BYTE ); #endif @@ -32415,7 +41991,7 @@ static int unixWrite( #endif #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -32430,7 +42006,7 @@ static int unixWrite( } } #endif - + while( (wrote = seekAndWrite(pFile, offset, pBuf, amt))0 ){ amt -= wrote; offset += wrote; @@ -32496,8 +42072,8 @@ SQLITE_API int sqlite3_fullsync_count = 0; ** ** SQLite sets the dataOnly flag if the size of the file is unchanged. ** The idea behind dataOnly is that it should only write the file content -** to disk, not the inode. We only set dataOnly if the file size is -** unchanged since the file size is part of the inode. However, +** to disk, not the inode. We only set dataOnly if the file size is +** unchanged since the file size is part of the inode. However, ** Ted Ts'o tells us that fdatasync() will also write the inode if the ** file size has changed. The only real difference between fdatasync() ** and fsync(), Ted tells us, is that fdatasync() will not flush the @@ -32511,7 +42087,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ int rc; /* The following "ifdef/elif/else/" block has the same structure as - ** the one below. It is replicated here solely to avoid cluttering + ** the one below. It is replicated here solely to avoid cluttering ** up the real code with the UNUSED_PARAMETER() macros. */ #ifdef SQLITE_NO_SYNC @@ -32525,7 +42101,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ UNUSED_PARAMETER(dataOnly); #endif - /* Record the number of times that we do a normal fsync() and + /* Record the number of times that we do a normal fsync() and ** FULLSYNC. This is used during testing to verify that this procedure ** gets called with the correct arguments. */ @@ -32537,7 +42113,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ /* If we compiled with the SQLITE_NO_SYNC flag, then syncing is a ** no-op. But go ahead and call fstat() to validate the file ** descriptor as we need a method to provoke a failure during - ** coverate testing. + ** coverage testing. */ #ifdef SQLITE_NO_SYNC { @@ -32551,11 +42127,11 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ rc = 1; } /* If the FULLFSYNC failed, fall back to attempting an fsync(). - ** It shouldn't be possible for fullfsync to fail on the local + ** It shouldn't be possible for fullfsync to fail on the local ** file system (on OSX), so failure indicates that FULLFSYNC - ** isn't supported for this file system. So, attempt an fsync - ** and (for now) ignore the overhead of a superfluous fcntl call. - ** It'd be better to detect fullfsync support once and avoid + ** isn't supported for this file system. So, attempt an fsync + ** and (for now) ignore the overhead of a superfluous fcntl call. + ** It'd be better to detect fullfsync support once and avoid ** the fcntl call every time sync is called. */ if( rc ) rc = fsync(fd); @@ -32565,7 +42141,7 @@ static int full_fsync(int fd, int fullSync, int dataOnly){ ** so currently we default to the macro that redefines fdatasync to fsync */ rc = fsync(fd); -#else +#else rc = fdatasync(fd); #if OS_VXWORKS if( rc==-1 && errno==ENOTSUP ){ @@ -32726,7 +42302,7 @@ static int unixTruncate(sqlite3_file *id, i64 nByte){ #if SQLITE_MAX_MMAP_SIZE>0 /* If the file was just truncated to a size smaller than the currently ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. + ** use read() and write() to access data beyond this point from now on. */ if( nBytemmapSize ){ pFile->mmapSize = nByte; @@ -32772,8 +42348,8 @@ static int unixFileSize(sqlite3_file *id, i64 *pSize){ static int proxyFileControl(sqlite3_file*,int,void*); #endif -/* -** This function is called to handle the SQLITE_FCNTL_SIZE_HINT +/* +** This function is called to handle the SQLITE_FCNTL_SIZE_HINT ** file-control operation. Enlarge the database to nBytes in size ** (rounded up to the next chunk-size). If the database is already ** nBytes or larger, this routine is a no-op. @@ -32782,7 +42358,7 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ if( pFile->szChunk>0 ){ i64 nSize; /* Required file size */ struct stat buf; /* Used to hold return values of fstat() */ - + if( osFstat(pFile->h, &buf) ){ return SQLITE_IOERR_FSTAT; } @@ -32791,16 +42367,16 @@ static int fcntlSizeHint(unixFile *pFile, i64 nByte){ if( nSize>(i64)buf.st_size ){ #if defined(HAVE_POSIX_FALLOCATE) && HAVE_POSIX_FALLOCATE - /* The code below is handling the return value of osFallocate() - ** correctly. posix_fallocate() is defined to "returns zero on success, + /* The code below is handling the return value of osFallocate() + ** correctly. posix_fallocate() is defined to "returns zero on success, ** or an error number on failure". See the manpage for details. */ int err; do{ err = osFallocate(pFile->h, buf.st_size, nSize-buf.st_size); }while( err==EINTR ); - if( err ) return SQLITE_IOERR_WRITE; + if( err && err!=EINVAL ) return SQLITE_IOERR_WRITE; #else - /* If the OS does not have posix_fallocate(), fake it. Write a + /* If the OS does not have posix_fallocate(), fake it. Write a ** single byte to the last byte in each block that falls entirely ** within the extended region. Then, if required, a single byte ** at offset (nSize-1), to set the size of the file correctly. @@ -32859,6 +42435,9 @@ static void unixModeBit(unixFile *pFile, unsigned char mask, int *pArg){ /* Forward declaration */ static int unixGetTempname(int nBuf, char *zBuf); +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) + static int unixFcntlExternalReader(unixFile*, int*); +#endif /* ** Information and control of an open file handle. @@ -32866,6 +42445,21 @@ static int unixGetTempname(int nBuf, char *zBuf); static int unixFileControl(sqlite3_file *id, int op, void *pArg){ unixFile *pFile = (unixFile*)id; switch( op ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + case SQLITE_FCNTL_BEGIN_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_START_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_BEGIN_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_COMMIT_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_COMMIT_ATOMIC_WRITE); + return rc ? SQLITE_IOERR_COMMIT_ATOMIC : SQLITE_OK; + } + case SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE: { + int rc = osIoctl(pFile->h, F2FS_IOC_ABORT_VOLATILE_WRITE); + return rc ? SQLITE_IOERR_ROLLBACK_ATOMIC : SQLITE_OK; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + case SQLITE_FCNTL_LOCKSTATE: { *(int*)pArg = pFile->eFileLock; return SQLITE_OK; @@ -32909,6 +42503,20 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ *(int*)pArg = fileHasMoved(pFile); return SQLITE_OK; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + case SQLITE_FCNTL_LOCK_TIMEOUT: { + int iOld = pFile->iBusyTimeout; +#if SQLITE_ENABLE_SETLK_TIMEOUT==1 + pFile->iBusyTimeout = *(int*)pArg; +#elif SQLITE_ENABLE_SETLK_TIMEOUT==2 + pFile->iBusyTimeout = !!(*(int*)pArg); +#else +# error "SQLITE_ENABLE_SETLK_TIMEOUT must be set to 1 or 2" +#endif + *(int*)pArg = iOld; + return SQLITE_OK; + } +#endif #if SQLITE_MAX_MMAP_SIZE>0 case SQLITE_FCNTL_MMAP_SIZE: { i64 newLimit = *(i64*)pArg; @@ -32916,6 +42524,14 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ if( newLimit>sqlite3GlobalConfig.mxMmap ){ newLimit = sqlite3GlobalConfig.mxMmap; } + + /* The value of newLimit may be eventually cast to (size_t) and passed + ** to mmap(). Restrict its value to 2GB if (size_t) is not at least a + ** 64-bit type. */ + if( newLimit>0 && sizeof(size_t)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; @@ -32944,43 +42560,64 @@ static int unixFileControl(sqlite3_file *id, int op, void *pArg){ return proxyFileControl(id,op,pArg); } #endif /* SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) */ + + case SQLITE_FCNTL_EXTERNAL_READER: { +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) + return unixFcntlExternalReader((unixFile*)id, (int*)pArg); +#else + *(int*)pArg = 0; + return SQLITE_OK; +#endif + } } return SQLITE_NOTFOUND; } /* -** Return the sector size in bytes of the underlying block device for -** the specified file. This is almost always 512 bytes, but may be -** larger for some devices. +** If pFd->sectorSize is non-zero when this function is called, it is a +** no-op. Otherwise, the values of pFd->sectorSize and +** pFd->deviceCharacteristics are set according to the file-system +** characteristics. ** -** SQLite code assumes this function cannot fail. It also assumes that -** if two files are created in the same file-system directory (i.e. -** a database and its journal file) that the sector size will be the -** same for both. +** There are two versions of this function. One for QNX and one for all +** other systems. */ -#ifndef __QNXNTO__ -static int unixSectorSize(sqlite3_file *NotUsed){ - UNUSED_PARAMETER(NotUsed); - return SQLITE_DEFAULT_SECTOR_SIZE; -} -#endif +#ifndef __QNXNTO__ +static void setDeviceCharacteristics(unixFile *pFd){ + assert( pFd->deviceCharacteristics==0 || pFd->sectorSize!=0 ); + if( pFd->sectorSize==0 ){ +#if defined(__linux__) && defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int res; + u32 f = 0; -/* -** The following version of unixSectorSize() is optimized for QNX. -*/ -#ifdef __QNXNTO__ + /* Check for support for F2FS atomic batch writes. */ + res = osIoctl(pFd->h, F2FS_IOC_GET_FEATURES, &f); + if( res==0 && (f & F2FS_FEATURE_ATOMIC_WRITE) ){ + pFd->deviceCharacteristics = SQLITE_IOCAP_BATCH_ATOMIC; + } +#endif /* __linux__ && SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + /* Set the POWERSAFE_OVERWRITE flag if requested. */ + if( pFd->ctrlFlags & UNIXFILE_PSOW ){ + pFd->deviceCharacteristics |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; + } + pFd->deviceCharacteristics |= SQLITE_IOCAP_SUBPAGE_READ; + + pFd->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; + } +} +#else #include #include -static int unixSectorSize(sqlite3_file *id){ - unixFile *pFile = (unixFile*)id; +static void setDeviceCharacteristics(unixFile *pFile){ if( pFile->sectorSize == 0 ){ struct statvfs fsInfo; - + /* Set defaults for non-supported filesystems */ pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; pFile->deviceCharacteristics = 0; if( fstatvfs(pFile->h, &fsInfo) == -1 ) { - return pFile->sectorSize; + return; } if( !strcmp(fsInfo.f_basetype, "tmp") ) { @@ -33015,7 +42652,7 @@ static int unixSectorSize(sqlite3_file *id){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -33023,7 +42660,7 @@ static int unixSectorSize(sqlite3_file *id){ pFile->sectorSize = fsInfo.f_bsize; pFile->deviceCharacteristics = /* full bitset of atomics from max sector size and smaller */ - ((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2 | + (((pFile->sectorSize / 512 * SQLITE_IOCAP_ATOMIC512) << 1) - 2) | SQLITE_IOCAP_SEQUENTIAL | /* The ram filesystem has no write behind ** so it is ordered */ 0; @@ -33041,9 +42678,24 @@ static int unixSectorSize(sqlite3_file *id){ pFile->deviceCharacteristics = 0; pFile->sectorSize = SQLITE_DEFAULT_SECTOR_SIZE; } - return pFile->sectorSize; } -#endif /* __QNXNTO__ */ +#endif + +/* +** Return the sector size in bytes of the underlying block device for +** the specified file. This is almost always 512 bytes, but may be +** larger for some devices. +** +** SQLite code assumes this function cannot fail. It also assumes that +** if two files are created in the same file-system directory (i.e. +** a database and its journal file) that the sector size will be the +** same for both. +*/ +static int unixSectorSize(sqlite3_file *id){ + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->sectorSize; +} /* ** Return the device characteristics for the file. @@ -33059,16 +42711,9 @@ static int unixSectorSize(sqlite3_file *id){ ** available to turn it off and URI query parameter available to turn it off. */ static int unixDeviceCharacteristics(sqlite3_file *id){ - unixFile *p = (unixFile*)id; - int rc = 0; -#ifdef __QNXNTO__ - if( p->sectorSize==0 ) unixSectorSize(id); - rc = p->deviceCharacteristics; -#endif - if( p->ctrlFlags & UNIXFILE_PSOW ){ - rc |= SQLITE_IOCAP_POWERSAFE_OVERWRITE; - } - return rc; + unixFile *pFd = (unixFile*)id; + setDeviceCharacteristics(pFd); + return pFd->deviceCharacteristics; } #if !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 @@ -33076,7 +42721,7 @@ static int unixDeviceCharacteristics(sqlite3_file *id){ /* ** Return the system page size. ** -** This function should not be called directly by other code in this file. +** This function should not be called directly by other code in this file. ** Instead, it should be called via macro osGetpagesize(). */ static int unixGetpagesize(void){ @@ -33091,10 +42736,10 @@ static int unixGetpagesize(void){ #endif /* !defined(SQLITE_OMIT_WAL) || SQLITE_MAX_MMAP_SIZE>0 */ -#ifndef SQLITE_OMIT_WAL +#if !defined(SQLITE_WASI) && !defined(SQLITE_OMIT_WAL) /* -** Object used to represent an shared memory buffer. +** Object used to represent an shared memory buffer. ** ** When multiple threads all reference the same wal-index, each thread ** has its own unixShm object, but they all point to a single instance @@ -33114,28 +42759,50 @@ static int unixGetpagesize(void){ ** nRef ** ** The following fields are read-only after the object is created: -** -** fid +** +** hShm ** zFilename ** -** Either unixShmNode.mutex must be held or unixShmNode.nRef==0 and +** Either unixShmNode.pShmMutex must be held or unixShmNode.nRef==0 and ** unixMutexHeld() is true when reading or writing any other field ** in this structure. +** +** aLock[SQLITE_SHM_NLOCK]: +** This array records the various locks held by clients on each of the +** SQLITE_SHM_NLOCK slots. If the aLock[] entry is set to 0, then no +** locks are held by the process on this slot. If it is set to -1, then +** some client holds an EXCLUSIVE lock on the locking slot. If the aLock[] +** value is set to a positive value, then it is the number of shared +** locks currently held on the slot. +** +** aMutex[SQLITE_SHM_NLOCK]: +** Normally, when SQLITE_ENABLE_SETLK_TIMEOUT is not defined, mutex +** pShmMutex is used to protect the aLock[] array and the right to +** call fcntl() on unixShmNode.hShm to obtain or release locks. +** +** If SQLITE_ENABLE_SETLK_TIMEOUT is defined though, we use an array +** of mutexes - one for each locking slot. To read or write locking +** slot aLock[iSlot], the caller must hold the corresponding mutex +** aMutex[iSlot]. Similarly, to call fcntl() to obtain or release a +** lock corresponding to slot iSlot, mutex aMutex[iSlot] must be held. */ struct unixShmNode { unixInodeInfo *pInode; /* unixInodeInfo that owns this SHM node */ - sqlite3_mutex *mutex; /* Mutex to access this object */ + sqlite3_mutex *pShmMutex; /* Mutex to access this object */ char *zFilename; /* Name of the mmapped file */ - int h; /* Open file descriptor */ + int hShm; /* Open file descriptor */ int szRegion; /* Size of shared-memory regions */ u16 nRegion; /* Size of array apRegion */ u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ char **apRegion; /* Array of mapped shared-memory regions */ int nRef; /* Number of unixShm objects pointing to this */ unixShm *pFirst; /* All unixShm objects pointing to this */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3_mutex *aMutex[SQLITE_SHM_NLOCK]; +#endif + int aLock[SQLITE_SHM_NLOCK]; /* # shared locks on slot, -1==excl lock */ #ifdef SQLITE_DEBUG - u8 exclMask; /* Mask of exclusive locks held */ - u8 sharedMask; /* Mask of shared locks held */ u8 nextShmId; /* Next available unixShm.id value */ #endif }; @@ -33147,16 +42814,16 @@ struct unixShmNode { ** The following fields are initialized when this object is created and ** are read-only thereafter: ** -** unixShm.pFile +** unixShm.pShmNode ** unixShm.id ** -** All other fields are read/write. The unixShm.pFile->mutex must be held -** while accessing any read/write fields. +** All other fields are read/write. The unixShm.pShmNode->pShmMutex must +** be held while accessing any read/write fields. */ struct unixShm { unixShmNode *pShmNode; /* The underlying unixShmNode object */ unixShm *pNext; /* Next unixShm with the same unixShmNode */ - u8 hasMutex; /* True if holding the unixShmNode mutex */ + u8 hasMutex; /* True if holding the unixShmNode->pShmMutex */ u8 id; /* Id of this connection within its unixShmNode */ u16 sharedMask; /* Mask of shared locks held */ u16 exclMask; /* Mask of exclusive locks held */ @@ -33168,6 +42835,40 @@ struct unixShm { #define UNIX_SHM_BASE ((22+SQLITE_SHM_NLOCK)*4) /* first lock byte */ #define UNIX_SHM_DMS (UNIX_SHM_BASE+SQLITE_SHM_NLOCK) /* deadman switch */ +/* +** Use F_GETLK to check whether or not there are any readers with open +** wal-mode transactions in other processes on database file pFile. If +** no error occurs, return SQLITE_OK and set (*piOut) to 1 if there are +** such transactions, or 0 otherwise. If an error occurs, return an +** SQLite error code. The final value of *piOut is undefined in this +** case. +*/ +static int unixFcntlExternalReader(unixFile *pFile, int *piOut){ + int rc = SQLITE_OK; + *piOut = 0; + if( pFile->pShm){ + unixShmNode *pShmNode = pFile->pShm->pShmNode; + struct flock f; + + memset(&f, 0, sizeof(f)); + f.l_type = F_WRLCK; + f.l_whence = SEEK_SET; + f.l_start = UNIX_SHM_BASE + 3; + f.l_len = SQLITE_SHM_NLOCK - 3; + + sqlite3_mutex_enter(pShmNode->pShmMutex); + if( osFcntl(pShmNode->hShm, F_GETLK, &f)<0 ){ + rc = SQLITE_IOERR_LOCK; + }else{ + *piOut = (f.l_type!=F_UNLCK); + } + sqlite3_mutex_leave(pShmNode->pShmMutex); + } + + return rc; +} + + /* ** Apply posix advisory locks for all bytes from ofst through ofst+n-1. ** @@ -33184,64 +42885,78 @@ static int unixShmSystemLock( struct flock f; /* The posix advisory locking structure */ int rc = SQLITE_OK; /* Result code form fcntl() */ - /* Access to the unixShmNode object is serialized by the caller */ pShmNode = pFile->pInode->pShmNode; - assert( sqlite3_mutex_held(pShmNode->mutex) || pShmNode->nRef==0 ); + + /* Assert that the parameters are within expected range and that the + ** correct mutex or mutexes are held. */ + assert( pShmNode->nRef>=0 ); + assert( (ofst==UNIX_SHM_DMS && n==1) + || (ofst>=UNIX_SHM_BASE && ofst+n<=(UNIX_SHM_BASE+SQLITE_SHM_NLOCK)) + ); + if( ofst==UNIX_SHM_DMS ){ + assert( pShmNode->nRef>0 || unixMutexHeld() ); + assert( pShmNode->nRef==0 || sqlite3_mutex_held(pShmNode->pShmMutex) ); + }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int ii; + for(ii=ofst-UNIX_SHM_BASE; iiaMutex[ii]) ); + } +#else + assert( sqlite3_mutex_held(pShmNode->pShmMutex) ); + assert( pShmNode->nRef>0 ); +#endif + } /* Shared locks never span more than one byte */ assert( n==1 || lockType!=F_RDLCK ); /* Locks are within range */ assert( n>=1 && n<=SQLITE_SHM_NLOCK ); + assert( ofst>=UNIX_SHM_BASE && ofst<=(UNIX_SHM_DMS+SQLITE_SHM_NLOCK) ); - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ + int res; /* Initialize the locking parameters */ - memset(&f, 0, sizeof(f)); f.l_type = lockType; f.l_whence = SEEK_SET; f.l_start = ofst; f.l_len = n; - - rc = osFcntl(pShmNode->h, F_SETLK, &f); - rc = (rc!=(-1)) ? SQLITE_OK : SQLITE_BUSY; + res = osSetPosixAdvisoryLock(pShmNode->hShm, &f, pFile); + if( res==-1 ){ +#if defined(SQLITE_ENABLE_SETLK_TIMEOUT) && SQLITE_ENABLE_SETLK_TIMEOUT==1 + rc = (pFile->iBusyTimeout ? SQLITE_BUSY_TIMEOUT : SQLITE_BUSY); +#else + rc = SQLITE_BUSY; +#endif + } } - /* Update the global lock state and do debug tracing */ + /* Do debug tracing */ #ifdef SQLITE_DEBUG - { u16 mask; OSTRACE(("SHM-LOCK ")); - mask = ofst>31 ? 0xffff : (1<<(ofst+n)) - (1<exclMask &= ~mask; - pShmNode->sharedMask &= ~mask; + OSTRACE(("unlock %d..%d ok\n", ofst, ofst+n-1)); }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock %d ok", ofst)); - pShmNode->exclMask &= ~mask; - pShmNode->sharedMask |= mask; + OSTRACE(("read-lock %d..%d ok\n", ofst, ofst+n-1)); }else{ assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d ok", ofst)); - pShmNode->exclMask |= mask; - pShmNode->sharedMask &= ~mask; + OSTRACE(("write-lock %d..%d ok\n", ofst, ofst+n-1)); } }else{ if( lockType==F_UNLCK ){ - OSTRACE(("unlock %d failed", ofst)); + OSTRACE(("unlock %d..%d failed\n", ofst, ofst+n-1)); }else if( lockType==F_RDLCK ){ - OSTRACE(("read-lock failed")); + OSTRACE(("read-lock %d..%d failed\n", ofst, ofst+n-1)); }else{ assert( lockType==F_WRLCK ); - OSTRACE(("write-lock %d failed", ofst)); + OSTRACE(("write-lock %d..%d failed\n", ofst, ofst+n-1)); } } - OSTRACE((" - afterwards %03x,%03x\n", - pShmNode->sharedMask, pShmNode->exclMask)); - } #endif - return rc; + return rc; } /* @@ -33274,18 +42989,23 @@ static void unixShmPurge(unixFile *pFd){ int nShmPerMap = unixShmRegionPerMap(); int i; assert( p->pInode==pFd->pInode ); - sqlite3_mutex_free(p->mutex); + sqlite3_mutex_free(p->pShmMutex); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + for(i=0; iaMutex[i]); + } +#endif for(i=0; inRegion; i+=nShmPerMap){ - if( p->h>=0 ){ + if( p->hShm>=0 ){ osMunmap(p->apRegion[i], p->szRegion); }else{ sqlite3_free(p->apRegion[i]); } } sqlite3_free(p->apRegion); - if( p->h>=0 ){ - robust_close(pFd, p->h, __LINE__); - p->h = -1; + if( p->hShm>=0 ){ + robust_close(pFd, p->hShm, __LINE__); + p->hShm = -1; } p->pInode->pShmNode = 0; sqlite3_free(p); @@ -33293,20 +43013,96 @@ static void unixShmPurge(unixFile *pFd){ } /* -** Open a shared-memory area associated with open database file pDbFd. +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int unixLockSharedMemory(unixFile *pDbFd, unixShmNode *pShmNode){ + struct flock lock; + int rc = SQLITE_OK; + + /* Use F_GETLK to determine the locks other processes are holding + ** on the DMS byte. If it indicates that another process is holding + ** a SHARED lock, then this process may also take a SHARED lock + ** and proceed with opening the *-shm file. + ** + ** Or, if no other process is holding any lock, then this process + ** is the first to open it. In this case take an EXCLUSIVE lock on the + ** DMS byte and truncate the *-shm file to zero bytes in size. Then + ** downgrade to a SHARED lock on the DMS byte. + ** + ** If another process is holding an EXCLUSIVE lock on the DMS byte, + ** return SQLITE_BUSY to the caller (it will try again). An earlier + ** version of this code attempted the SHARED lock at this point. But + ** this introduced a subtle race condition: if the process holding + ** EXCLUSIVE failed just before truncating the *-shm file, then this + ** process might open and use the *-shm file without truncating it. + ** And if the *-shm file has been corrupted by a power failure or + ** system crash, the database itself may also become corrupt. */ + lock.l_whence = SEEK_SET; + lock.l_start = UNIX_SHM_DMS; + lock.l_len = 1; + lock.l_type = F_WRLCK; + if( osFcntl(pShmNode->hShm, F_GETLK, &lock)!=0 ) { + rc = SQLITE_IOERR_LOCK; + }else if( lock.l_type==F_UNLCK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + rc = SQLITE_READONLY_CANTINIT; + }else{ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* Do not use a blocking lock here. If the lock cannot be obtained + ** immediately, it means some other connection is truncating the + ** *-shm file. And after it has done so, it will not release its + ** lock, but only downgrade it to a shared lock. So no point in + ** blocking here. The call below to obtain the shared DMS lock may + ** use a blocking lock. */ + int iSaveTimeout = pDbFd->iBusyTimeout; + pDbFd->iBusyTimeout = 0; +#endif + rc = unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + pDbFd->iBusyTimeout = iSaveTimeout; +#endif + /* The first connection to attach must truncate the -shm file. We + ** truncate to 3 bytes (an arbitrary small number, less than the + ** -shm header size) rather than 0 as a system debugging aid, to + ** help detect if a -shm file truncation is legitimate or is the work + ** or a rogue process. */ + if( rc==SQLITE_OK && robust_ftruncate(pShmNode->hShm, 3) ){ + rc = unixLogError(SQLITE_IOERR_SHMOPEN,"ftruncate",pShmNode->zFilename); + } + } + }else if( lock.l_type==F_WRLCK ){ + rc = SQLITE_BUSY; + } + + if( rc==SQLITE_OK ){ + assert( lock.l_type==F_UNLCK || lock.l_type==F_RDLCK ); + rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); + } + return rc; +} + +/* +** Open a shared-memory area associated with open database file pDbFd. ** This particular implementation uses mmapped files. ** ** The file used to implement shared-memory is in the same directory ** as the open database file and has the same name as the open database ** file with the "-shm" suffix added. For example, if the database file ** is "/home/user1/config.db" then the file that is created and mmapped -** for shared memory will be called "/home/user1/config.db-shm". +** for shared memory will be called "/home/user1/config.db-shm". ** ** Another approach to is to use files in /dev/shm or /dev/tmp or an ** some other tmpfs mount. But if a file in a different directory ** from the database file is used, then differing access permissions ** or a chroot() might cause two different processes on the same -** database to end up using different files for shared memory - +** database to end up using different files for shared memory - ** meaning that their memory would not really be shared - resulting ** in database corruption. Nevertheless, this tmpfs file usage ** can be enabled at compile-time using -DSQLITE_SHM_DIRECTORY="/dev/shm" @@ -33330,9 +43126,9 @@ static void unixShmPurge(unixFile *pFd){ static int unixOpenSharedMemory(unixFile *pDbFd){ struct unixShm *p = 0; /* The connection to be opened */ struct unixShmNode *pShmNode; /* The underlying mmapped file */ - int rc; /* Result code */ + int rc = SQLITE_OK; /* Result code */ unixInodeInfo *pInode; /* The inode of fd */ - char *zShmFilename; /* Name of the file used for SHM */ + char *zShm; /* Name of the file used for SHM */ int nShmFilename; /* Size of the SHM filename in bytes */ /* Allocate space for the new unixShm object. */ @@ -33344,6 +43140,7 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ /* Check to see if a unixShmNode object already exists. Reuse an existing ** one if present. Create a new one if necessary. */ + assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); pInode = pDbFd->pInode; pShmNode = pInode->pShmNode; @@ -33373,57 +43170,61 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ goto shm_open_err; } memset(pShmNode, 0, sizeof(*pShmNode)+nShmFilename); - zShmFilename = pShmNode->zFilename = (char*)&pShmNode[1]; + zShm = pShmNode->zFilename = (char*)&pShmNode[1]; #ifdef SQLITE_SHM_DIRECTORY - sqlite3_snprintf(nShmFilename, zShmFilename, + sqlite3_snprintf(nShmFilename, zShm, SQLITE_SHM_DIRECTORY "/sqlite-shm-%x-%x", (u32)sStat.st_ino, (u32)sStat.st_dev); #else - sqlite3_snprintf(nShmFilename, zShmFilename, "%s-shm", zBasePath); - sqlite3FileSuffix3(pDbFd->zPath, zShmFilename); + sqlite3_snprintf(nShmFilename, zShm, "%s-shm", zBasePath); + sqlite3FileSuffix3(pDbFd->zPath, zShm); #endif - pShmNode->h = -1; + pShmNode->hShm = -1; pDbFd->pInode->pShmNode = pShmNode; pShmNode->pInode = pDbFd->pInode; if( sqlite3GlobalConfig.bCoreMutex ){ - pShmNode->mutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); - if( pShmNode->mutex==0 ){ + pShmNode->pShmMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->pShmMutex==0 ){ rc = SQLITE_NOMEM_BKPT; goto shm_open_err; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + { + int ii; + for(ii=0; iiaMutex[ii] = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( pShmNode->aMutex[ii]==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto shm_open_err; + } + } + } +#endif } if( pInode->bProcessLock==0 ){ - int openFlags = O_RDWR | O_CREAT; - if( sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ - openFlags = O_RDONLY; - pShmNode->isReadonly = 1; + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + pShmNode->hShm = robust_open(zShm, O_RDWR|O_CREAT|O_NOFOLLOW, + (sStat.st_mode&0777)); } - pShmNode->h = robust_open(zShmFilename, openFlags, (sStat.st_mode&0777)); - if( pShmNode->h<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShmFilename); - goto shm_open_err; + if( pShmNode->hShm<0 ){ + pShmNode->hShm = robust_open(zShm, O_RDONLY|O_NOFOLLOW, + (sStat.st_mode&0777)); + if( pShmNode->hShm<0 ){ + rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zShm); + goto shm_open_err; + } + pShmNode->isReadonly = 1; } /* If this process is running as root, make sure that the SHM file ** is owned by the same user that owns the original database. Otherwise, ** the original owner will not be able to connect. */ - robustFchown(pShmNode->h, sStat.st_uid, sStat.st_gid); - - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - rc = SQLITE_OK; - if( unixShmSystemLock(pDbFd, F_WRLCK, UNIX_SHM_DMS, 1)==SQLITE_OK ){ - if( robust_ftruncate(pShmNode->h, 0) ){ - rc = unixLogError(SQLITE_IOERR_SHMOPEN, "ftruncate", zShmFilename); - } - } - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pDbFd, F_RDLCK, UNIX_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + robustFchown(pShmNode->hShm, sStat.st_uid, sStat.st_gid); + + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } } @@ -33440,14 +43241,14 @@ static int unixOpenSharedMemory(unixFile *pDbFd){ ** the cover of the unixEnterMutex() mutex and the pointer from the ** new (struct unixShm) object to the pShmNode has been set. All that is ** left to do is to link the new object into the linked list starting - ** at pShmNode->pFirst. This must be done while holding the pShmNode->mutex - ** mutex. + ** at pShmNode->pFirst. This must be done while holding the + ** pShmNode->pShmMutex. */ - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; - sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + sqlite3_mutex_leave(pShmNode->pShmMutex); + return rc; /* Jump here on any error */ shm_open_err: @@ -33458,22 +43259,22 @@ shm_open_err: } /* -** This function is called to obtain a pointer to region iRegion of the -** shared-memory associated with the database file fd. Shared-memory regions -** are numbered starting from zero. Each shared-memory region is szRegion +** This function is called to obtain a pointer to region iRegion of the +** shared-memory associated with the database file fd. Shared-memory regions +** are numbered starting from zero. Each shared-memory region is szRegion ** bytes in size. ** ** If an error occurs, an error code is returned and *pp is set to NULL. ** ** Otherwise, if the bExtend parameter is 0 and the requested shared-memory ** region has not been allocated (by any client, including one running in a -** separate process), then *pp is set to NULL and SQLITE_OK returned. If -** bExtend is non-zero and the requested shared-memory region has not yet +** separate process), then *pp is set to NULL and SQLITE_OK returned. If +** bExtend is non-zero and the requested shared-memory region has not yet ** been allocated, it is allocated by this function. ** ** If the shared-memory region has already been allocated or is allocated by -** this call as described above, then it is mapped into this processes -** address space (if it is not already), *pp is set to point to the mapped +** this call as described above, then it is mapped into this processes +** address space (if it is not already), *pp is set to point to the mapped ** memory and SQLITE_OK returned. */ static int unixShmMap( @@ -33498,11 +43299,16 @@ static int unixShmMap( p = pDbFd->pShm; pShmNode = p->pShmNode; - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); + if( pShmNode->isUnlocked ){ + rc = unixLockSharedMemory(pDbFd, pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); assert( pShmNode->pInode==pDbFd->pInode ); - assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); - assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); /* Minimum number of regions required to be mapped. */ nReqRegion = ((iRegion+nShmPerMap) / nShmPerMap) * nShmPerMap; @@ -33514,16 +43320,16 @@ static int unixShmMap( pShmNode->szRegion = szRegion; - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ /* The requested region is not mapped into this processes address space. ** Check to see if it has been allocated (i.e. if the wal-index file is ** large enough to contain the requested region). */ - if( osFstat(pShmNode->h, &sStat) ){ + if( osFstat(pShmNode->hShm, &sStat) ){ rc = SQLITE_IOERR_SHMSIZE; goto shmpage_out; } - + if( sStat.st_sizeh, iPg*pgsz + pgsz-1, "", 1, &x)!=1 ){ + if( seekAndWriteFd(pShmNode->hShm, iPg*pgsz + pgsz-1,"",1,&x)!=1 ){ const char *zFile = pShmNode->zFilename; rc = unixLogError(SQLITE_IOERR_SHMSIZE, "write", zFile); goto shmpage_out; @@ -33570,22 +43376,22 @@ static int unixShmMap( int nMap = szRegion*nShmPerMap; int i; void *pMem; - if( pShmNode->h>=0 ){ + if( pShmNode->hShm>=0 ){ pMem = osMmap(0, nMap, - pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, - MAP_SHARED, pShmNode->h, szRegion*(i64)pShmNode->nRegion + pShmNode->isReadonly ? PROT_READ : PROT_READ|PROT_WRITE, + MAP_SHARED, pShmNode->hShm, szRegion*(i64)pShmNode->nRegion ); if( pMem==MAP_FAILED ){ rc = unixLogError(SQLITE_IOERR_SHMMAP, "mmap", pShmNode->zFilename); goto shmpage_out; } }else{ - pMem = sqlite3_malloc64(szRegion); + pMem = sqlite3_malloc64(nMap); if( pMem==0 ){ rc = SQLITE_NOMEM_BKPT; goto shmpage_out; } - memset(pMem, 0, szRegion); + memset(pMem, 0, nMap); } for(i=0; iisReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); return rc; } +/* +** Check that the pShmNode->aLock[] array comports with the locking bitmasks +** held by each client. Return true if it does, or false otherwise. This +** is to be used in an assert(). e.g. +** +** assert( assertLockingArrayOk(pShmNode) ); +*/ +#ifdef SQLITE_DEBUG +static int assertLockingArrayOk(unixShmNode *pShmNode){ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + return 1; +#else + unixShm *pX; + int aLock[SQLITE_SHM_NLOCK]; + + memset(aLock, 0, sizeof(aLock)); + for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ + int i; + for(i=0; iexclMask & (1<sharedMask & (1<=0 ); + aLock[i]++; + } + } + } + + assert( 0==memcmp(pShmNode->aLock, aLock, sizeof(aLock)) ); + return (memcmp(pShmNode->aLock, aLock, sizeof(aLock))==0); +#endif +} +#endif + /* ** Change the lock state for a shared-memory segment. ** -** Note that the relationship between SHAREd and EXCLUSIVE locks is a little +** Note that the relationship between SHARED and EXCLUSIVE locks is a little ** different here than in posix. In xShmLock(), one can go from unlocked ** to shared and back or from unlocked to exclusive and back. But one may ** not go from shared to exclusive or from exclusive to shared. @@ -33621,11 +43462,17 @@ static int unixShmLock( int flags /* What to do with the lock */ ){ unixFile *pDbFd = (unixFile*)fd; /* Connection holding shared memory */ - unixShm *p = pDbFd->pShm; /* The shared memory being locked */ - unixShm *pX; /* For looping over all siblings */ - unixShmNode *pShmNode = p->pShmNode; /* The underlying file iNode */ + unixShm *p; /* The shared memory being locked */ + unixShmNode *pShmNode; /* The underlying file iNode */ int rc = SQLITE_OK; /* Result code */ - u16 mask; /* Mask of locks to take or release */ + u16 mask = (1<<(ofst+n)) - (1<pShm; + if( p==0 ) return SQLITE_IOERR_SHMLOCK; + pShmNode = p->pShmNode; + if( NEVER(pShmNode==0) ) return SQLITE_IOERR_SHMLOCK; + aLock = pShmNode->aLock; assert( pShmNode==pDbFd->pInode->pShmNode ); assert( pShmNode->pInode==pDbFd->pInode ); @@ -33636,92 +43483,175 @@ static int unixShmLock( || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED) || flags==(SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE) ); assert( n==1 || (flags & SQLITE_SHM_EXCLUSIVE)!=0 ); - assert( pShmNode->h>=0 || pDbFd->pInode->bProcessLock==1 ); - assert( pShmNode->h<0 || pDbFd->pInode->bProcessLock==0 ); + assert( pShmNode->hShm>=0 || pDbFd->pInode->bProcessLock==1 ); + assert( pShmNode->hShm<0 || pDbFd->pInode->bProcessLock==0 ); - mask = (1<<(ofst+n)) - (1<1 || mask==(1<mutex); - if( flags & SQLITE_SHM_UNLOCK ){ - u16 allMask = 0; /* Mask of locks held by siblings */ - - /* See if any siblings hold this same lock */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( pX==p ) continue; - assert( (pX->exclMask & (p->exclMask|p->sharedMask))==0 ); - allMask |= pX->sharedMask; - } - - /* Unlock the system-level locks */ - if( (mask & allMask)==0 ){ - rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } - - /* Undo the local locks */ - if( rc==SQLITE_OK ){ - p->exclMask &= ~mask; - p->sharedMask &= ~mask; - } - }else if( flags & SQLITE_SHM_SHARED ){ - u16 allShared = 0; /* Union of locks held by connections other than "p" */ - - /* Find out which shared locks are already held by sibling connections. - ** If any sibling already holds an exclusive lock, go ahead and return - ** SQLITE_BUSY. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - allShared |= pX->sharedMask; - } - - /* Get shared locks at the system level, if necessary */ - if( rc==SQLITE_OK ){ - if( (allShared & mask)==0 ){ - rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); - }else{ - rc = SQLITE_OK; - } - } - - /* Get the local shared locks */ - if( rc==SQLITE_OK ){ - p->sharedMask |= mask; - } - }else{ - /* Make sure no sibling connections hold locks that will block this - ** lock. If any do, return SQLITE_BUSY right away. - */ - for(pX=pShmNode->pFirst; pX; pX=pX->pNext){ - if( (pX->exclMask & mask)!=0 || (pX->sharedMask & mask)!=0 ){ - rc = SQLITE_BUSY; - break; - } - } - - /* Get the exclusive locks at the system level. Then if successful - ** also mark the local connection as being locked. - */ - if( rc==SQLITE_OK ){ - rc = unixShmSystemLock(pDbFd, F_WRLCK, ofst+UNIX_SHM_BASE, n); - if( rc==SQLITE_OK ){ - assert( (p->sharedMask & mask)==0 ); - p->exclMask |= mask; - } - } + /* Check that, if this to be a blocking lock, no locks that occur later + ** in the following list than the lock being obtained are already held: + ** + ** 1. Checkpointer lock (ofst==1). + ** 2. Write lock (ofst==0). + ** 3. Read locks (ofst>=3 && ofstexclMask|p->sharedMask); + assert( (flags & SQLITE_SHM_UNLOCK) || pDbFd->iBusyTimeout==0 || ( + (ofst!=2) /* not RECOVER */ + && (ofst!=1 || lockMask==0 || lockMask==2) + && (ofst!=0 || lockMask<3) + && (ofst<3 || lockMask<(1<mutex); +#endif + + /* Check if there is any work to do. There are three cases: + ** + ** a) An unlock operation where there are locks to unlock, + ** b) An shared lock where the requested lock is not already held + ** c) An exclusive lock where the requested lock is not already held + ** + ** The SQLite core never requests an exclusive lock that it already holds. + ** This is assert()ed below. + */ + assert( flags!=(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK) + || 0==(p->exclMask & mask) + ); + if( ((flags & SQLITE_SHM_UNLOCK) && ((p->exclMask|p->sharedMask) & mask)) + || (flags==(SQLITE_SHM_SHARED|SQLITE_SHM_LOCK) && 0==(p->sharedMask & mask)) + || (flags==(SQLITE_SHM_EXCLUSIVE|SQLITE_SHM_LOCK)) + ){ + + /* Take the required mutexes. In SETLK_TIMEOUT mode (blocking locks), if + ** this is an attempt on an exclusive lock use sqlite3_mutex_try(). If any + ** other thread is holding this mutex, then it is either holding or about + ** to hold a lock exclusive to the one being requested, and we may + ** therefore return SQLITE_BUSY to the caller. + ** + ** Doing this prevents some deadlock scenarios. For example, thread 1 may + ** be a checkpointer blocked waiting on the WRITER lock. And thread 2 + ** may be a normal SQL client upgrading to a write transaction. In this + ** case thread 2 does a non-blocking request for the WRITER lock. But - + ** if it were to use sqlite3_mutex_enter() then it would effectively + ** become a (doomed) blocking request, as thread 2 would block until thread + ** 1 obtained WRITER and released the mutex. Since thread 2 already holds + ** a lock on a read-locking slot at this point, this breaks the + ** anti-deadlock rules (see above). */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int iMutex; + for(iMutex=ofst; iMutexaMutex[iMutex]); + if( rc!=SQLITE_OK ) goto leave_shmnode_mutexes; + }else{ + sqlite3_mutex_enter(pShmNode->aMutex[iMutex]); + } + } +#else + sqlite3_mutex_enter(pShmNode->pShmMutex); +#endif + + if( ALWAYS(rc==SQLITE_OK) ){ + if( flags & SQLITE_SHM_UNLOCK ){ + /* Case (a) - unlock. */ + int bUnlock = 1; + assert( (p->exclMask & p->sharedMask)==0 ); + assert( !(flags & SQLITE_SHM_EXCLUSIVE) || (p->exclMask & mask)==mask ); + assert( !(flags & SQLITE_SHM_SHARED) || (p->sharedMask & mask)==mask ); + + /* If this is a SHARED lock being unlocked, it is possible that other + ** clients within this process are holding the same SHARED lock. In + ** this case, set bUnlock to 0 so that the posix lock is not removed + ** from the file-descriptor below. */ + if( flags & SQLITE_SHM_SHARED ){ + assert( n==1 ); + assert( aLock[ofst]>=1 ); + if( aLock[ofst]>1 ){ + bUnlock = 0; + aLock[ofst]--; + p->sharedMask &= ~mask; + } + } + + if( bUnlock ){ + rc = unixShmSystemLock(pDbFd, F_UNLCK, ofst+UNIX_SHM_BASE, n); + if( rc==SQLITE_OK ){ + memset(&aLock[ofst], 0, sizeof(int)*n); + p->sharedMask &= ~mask; + p->exclMask &= ~mask; + } + } + }else if( flags & SQLITE_SHM_SHARED ){ + /* Case (b) - a shared lock. */ + + if( aLock[ofst]<0 ){ + /* An exclusive lock is held by some other connection. BUSY. */ + rc = SQLITE_BUSY; + }else if( aLock[ofst]==0 ){ + rc = unixShmSystemLock(pDbFd, F_RDLCK, ofst+UNIX_SHM_BASE, n); + } + + /* Get the local shared locks */ + if( rc==SQLITE_OK ){ + p->sharedMask |= mask; + aLock[ofst]++; + } + }else{ + /* Case (c) - an exclusive lock. */ + int ii; + + assert( flags==(SQLITE_SHM_LOCK|SQLITE_SHM_EXCLUSIVE) ); + assert( (p->sharedMask & mask)==0 ); + assert( (p->exclMask & mask)==0 ); + + /* Make sure no sibling connections hold locks that will block this + ** lock. If any do, return SQLITE_BUSY right away. */ + for(ii=ofst; iiexclMask |= mask; + for(ii=ofst; ii=ofst; iMutex--){ + sqlite3_mutex_leave(pShmNode->aMutex[iMutex]); + } +#else + sqlite3_mutex_leave(pShmNode->pShmMutex); +#endif + } + OSTRACE(("SHM-LOCK shmid-%d, pid-%d got %03x,%03x\n", p->id, osGetpid(0), p->sharedMask, p->exclMask)); return rc; } /* -** Implement a memory barrier or memory fence on shared memory. +** Implement a memory barrier or memory fence on shared memory. ** ** All loads and stores begun before the barrier must complete before ** any load or store begun after the barrier. @@ -33731,12 +43661,15 @@ static void unixShmBarrier( ){ UNUSED_PARAMETER(fd); sqlite3MemoryBarrier(); /* compiler-defined memory barrier */ + assert( fd->pMethods->xLock==nolockLock + || unixFileMutexNotheld((unixFile*)fd) + ); unixEnterMutex(); /* Also mutex, for redundancy */ unixLeaveMutex(); } /* -** Close a connection to shared-memory. Delete the underlying +** Close a connection to shared-memory. Delete the underlying ** storage if deleteFlag is true. ** ** If there is no shared memory associated with the connection then this @@ -33761,22 +43694,23 @@ static int unixShmUnmap( /* Remove connection p from the set of connections associated ** with pShmNode */ - sqlite3_mutex_enter(pShmNode->mutex); + sqlite3_mutex_enter(pShmNode->pShmMutex); for(pp=&pShmNode->pFirst; (*pp)!=p; pp = &(*pp)->pNext){} *pp = p->pNext; /* Free the connection p */ sqlite3_free(p); pDbFd->pShm = 0; - sqlite3_mutex_leave(pShmNode->mutex); + sqlite3_mutex_leave(pShmNode->pShmMutex); /* If pShmNode->nRef has reached 0, then close the underlying ** shared-memory file, too */ + assert( unixFileMutexNotheld(pDbFd) ); unixEnterMutex(); assert( pShmNode->nRef>0 ); pShmNode->nRef--; if( pShmNode->nRef==0 ){ - if( deleteFlag && pShmNode->h>=0 ){ + if( deleteFlag && pShmNode->hShm>=0 ){ osUnlink(pShmNode->zFilename); } unixShmPurge(pDbFd); @@ -33809,7 +43743,7 @@ static void unixUnmapfile(unixFile *pFd){ } /* -** Attempt to set the size of the memory mapping maintained by file +** Attempt to set the size of the memory mapping maintained by file ** descriptor pFd to nNew bytes. Any existing mapping is discarded. ** ** If successful, this function sets the following variables: @@ -33901,14 +43835,14 @@ static void unixRemapfile( /* ** Memory map or remap the file opened by file-descriptor pFd (if the file -** is already mapped, the existing mapping is replaced by the new). Or, if -** there already exists a mapping for this file, and there are still +** is already mapped, the existing mapping is replaced by the new). Or, if +** there already exists a mapping for this file, and there are still ** outstanding xFetch() references to it, this function is a no-op. ** -** If parameter nByte is non-negative, then it is the requested size of -** the mapping to create. Otherwise, if nByte is less than zero, then the +** If parameter nByte is non-negative, then it is the requested size of +** the mapping to create. Otherwise, if nByte is less than zero, then the ** requested size is the size of the file on disk. The actual size of the -** created mapping is either the requested size or the value configured +** created mapping is either the requested size or the value configured ** using SQLITE_FCNTL_MMAP_LIMIT, whichever is smaller. ** ** SQLITE_OK is returned if no error occurs (even if the mapping is not @@ -33949,7 +43883,7 @@ static int unixMapfile(unixFile *pFd, i64 nMap){ ** Finally, if an error does occur, return an SQLite error code. The final ** value of *pp is undefined in this case. ** -** If this function does return a pointer, the caller must eventually +** If this function does return a pointer, the caller must eventually ** release the reference by calling unixUnfetch(). */ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ @@ -33960,11 +43894,16 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = unixMapfile(pFd, -1); if( rc!=SQLITE_OK ) return rc; } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -33974,13 +43913,13 @@ static int unixFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ } /* -** If the third argument is non-NULL, then this function releases a +** If the third argument is non-NULL, then this function releases a ** reference obtained by an earlier call to unixFetch(). The second ** argument passed to this function must be the same as the corresponding -** argument that was passed to the unixFetch() invocation. +** argument that was passed to the unixFetch() invocation. ** -** Or, if the third argument is NULL, then this function is being called -** to inform the VFS layer that, according to POSIX, any existing mapping +** Or, if the third argument is NULL, then this function is being called +** to inform the VFS layer that, according to POSIX, any existing mapping ** may now be invalid and should be unmapped. */ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ @@ -33988,7 +43927,7 @@ static int unixUnfetch(sqlite3_file *fd, i64 iOff, void *p){ unixFile *pFd = (unixFile *)fd; /* The underlying database file */ UNUSED_PARAMETER(iOff); - /* If p==0 (unmap the entire file) then there must be no outstanding + /* If p==0 (unmap the entire file) then there must be no outstanding ** xFetch references. Or, if p!=0 (meaning it is an xFetch reference), ** then there must be at least one outstanding. */ assert( (p==0)==(pFd->nFetchOut==0) ); @@ -34098,7 +44037,7 @@ IOMETHODS( IOMETHODS( nolockIoFinder, /* Finder function name */ nolockIoMethods, /* sqlite3_io_methods object name */ - 3, /* shared memory is disabled */ + 3, /* shared memory and mmap are enabled */ nolockClose, /* xClose method */ nolockLock, /* xLock method */ nolockUnlock, /* xUnlock method */ @@ -34196,8 +44135,8 @@ IOMETHODS( #endif #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE -/* -** This "finder" function attempts to determine the best locking strategy +/* +** This "finder" function attempts to determine the best locking strategy ** for the database file "filePath". It then returns the sqlite3_io_methods ** object that implements that strategy. ** @@ -34239,8 +44178,8 @@ static const sqlite3_io_methods *autolockIoFinderImpl( } /* Default case. Handles, amongst others, "nfs". - ** Test byte-range lock using fcntl(). If the call succeeds, - ** assume that the file-system supports POSIX style locks. + ** Test byte-range lock using fcntl(). If the call succeeds, + ** assume that the file-system supports POSIX style locks. */ lockInfo.l_len = 1; lockInfo.l_start = 0; @@ -34256,7 +44195,7 @@ static const sqlite3_io_methods *autolockIoFinderImpl( return &dotlockIoMethods; } } -static const sqlite3_io_methods +static const sqlite3_io_methods *(*const autolockIoFinder)(const char*,unixFile*) = autolockIoFinderImpl; #endif /* defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE */ @@ -34292,7 +44231,7 @@ static const sqlite3_io_methods *vxworksIoFinderImpl( return &semIoMethods; } } -static const sqlite3_io_methods +static const sqlite3_io_methods *(*const vxworksIoFinder)(const char*,unixFile*) = vxworksIoFinderImpl; #endif /* OS_VXWORKS */ @@ -34326,17 +44265,6 @@ static int fillInUnixFile( assert( pNew->pInode==NULL ); - /* Usually the path zFilename should not be a relative pathname. The - ** exception is when opening the proxy "conch" file in builds that - ** include the special Apple locking styles. - */ -#if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE - assert( zFilename==0 || zFilename[0]=='/' - || pVfs->pAppData==(void*)&autolockIoFinder ); -#else - assert( zFilename==0 || zFilename[0]=='/' ); -#endif - /* No locking occurs in temporary files */ assert( zFilename!=0 || (ctrlFlags & UNIXFILE_NOLOCK)!=0 ); @@ -34431,14 +44359,14 @@ static int fillInUnixFile( robust_close(pNew, h, __LINE__); h = -1; } - unixLeaveMutex(); + unixLeaveMutex(); } } #endif else if( pLockingStyle == &dotlockIoMethods ){ /* Dotfile locking uses the file path so it needs to be included in - ** the dotlockLockingContext + ** the dotlockLockingContext */ char *zLockFile; int nFilename; @@ -34476,7 +44404,7 @@ static int fillInUnixFile( unixLeaveMutex(); } #endif - + storeLastErrno(pNew, 0); #if OS_VXWORKS if( rc!=SQLITE_OK ){ @@ -34489,32 +44417,42 @@ static int fillInUnixFile( if( rc!=SQLITE_OK ){ if( h>=0 ) robust_close(pNew, h, __LINE__); }else{ - pNew->pMethod = pLockingStyle; + pId->pMethods = pLockingStyle; OpenCounter(+1); verifyDbFile(pNew); } return rc; } +/* +** Directories to consider for temp files. +*/ +static const char *azTempDirs[] = { + 0, + 0, + "/var/tmp", + "/usr/tmp", + "/tmp", + "." +}; + +/* +** Initialize first two members of azTempDirs[] array. +*/ +static void unixTempFileInit(void){ + azTempDirs[0] = getenv("SQLITE_TMPDIR"); + azTempDirs[1] = getenv("TMPDIR"); +} + /* ** Return the name of a directory in which to put temporary files. ** If no suitable temporary file directory can be found, return NULL. */ static const char *unixTempFileDir(void){ - static const char *azDirs[] = { - 0, - 0, - "/var/tmp", - "/usr/tmp", - "/tmp", - "." - }; unsigned int i = 0; struct stat buf; const char *zDir = sqlite3_temp_directory; - if( !azDirs[0] ) azDirs[0] = getenv("SQLITE_TMPDIR"); - if( !azDirs[1] ) azDirs[1] = getenv("TMPDIR"); while(1){ if( zDir!=0 && osStat(zDir, &buf)==0 @@ -34523,8 +44461,8 @@ static const char *unixTempFileDir(void){ ){ return zDir; } - if( i>=sizeof(azDirs)/sizeof(azDirs[0]) ) break; - zDir = azDirs[i++]; + if( i>=sizeof(azTempDirs)/sizeof(azTempDirs[0]) ) break; + zDir = azTempDirs[i++]; } return 0; } @@ -34537,26 +44475,35 @@ static const char *unixTempFileDir(void){ static int unixGetTempname(int nBuf, char *zBuf){ const char *zDir; int iLimit = 0; + int rc = SQLITE_OK; /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this - ** function failing. + ** function failing. */ zBuf[0] = 0; SimulateIOError( return SQLITE_IOERR ); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); zDir = unixTempFileDir(); - if( zDir==0 ) return SQLITE_IOERR_GETTEMPPATH; - do{ - u64 r; - sqlite3_randomness(sizeof(r), &r); - assert( nBuf>2 ); - zBuf[nBuf-2] = 0; - sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", - zDir, r, 0); - if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ) return SQLITE_ERROR; - }while( osAccess(zBuf,0)==0 ); - return SQLITE_OK; + if( zDir==0 ){ + rc = SQLITE_IOERR_GETTEMPPATH; + }else{ + do{ + u64 r; + sqlite3_randomness(sizeof(r), &r); + assert( nBuf>2 ); + zBuf[nBuf-2] = 0; + sqlite3_snprintf(nBuf, zBuf, "%s/"SQLITE_TEMP_FILE_PREFIX"%llx%c", + zDir, r, 0); + if( zBuf[nBuf-2]!=0 || (iLimit++)>10 ){ + rc = SQLITE_ERROR; + break; + } + }while( osAccess(zBuf,0)==0 ); + } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; } #if SQLITE_ENABLE_LOCKING_STYLE && defined(__APPLE__) @@ -34569,8 +44516,8 @@ static int proxyTransformUnixFile(unixFile*, const char*); #endif /* -** Search for an unused file descriptor that was opened on the database -** file (not a journal or master-journal file) identified by pathname +** Search for an unused file descriptor that was opened on the database +** file (not a journal or super-journal file) identified by pathname ** zPath with SQLITE_OPEN_XXX flags matching those passed as the second ** argument to this function. ** @@ -34578,7 +44525,7 @@ static int proxyTransformUnixFile(unixFile*, const char*); ** but the associated file descriptor could not be closed because some ** other file descriptor open on the same file is holding a file-lock. ** Refer to comments in the unixClose() function and the lengthy comment -** describing "Posix Advisory Locking" at the start of this file for +** describing "Posix Advisory Locking" at the start of this file for ** further details. Also, ticket #4018. ** ** If a suitable file descriptor is found, then it is returned. If no @@ -34589,12 +44536,14 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ /* Do not search for an unused file descriptor on vxworks. Not because ** vxworks would not benefit from the change (it might, we're not sure), - ** but because no way to test it is currently available. It is better - ** not to risk breaking vxworks support for the sake of such an obscure + ** but because no way to test it is currently available. It is better + ** not to risk breaking vxworks support for the sake of such an obscure ** feature. */ #if !OS_VXWORKS struct stat sStat; /* Results of stat() call */ + unixEnterMutex(); + /* A stat() call may fail for various reasons. If this happens, it is ** almost certain that an open() call on the same path will also fail. ** For this reason, if an error occurs in the stat() call here, it is @@ -34603,49 +44552,73 @@ static UnixUnusedFd *findReusableFd(const char *zPath, int flags){ ** ** Even if a subsequent open() call does succeed, the consequences of ** not searching for a reusable file descriptor are not dire. */ - if( 0==osStat(zPath, &sStat) ){ + if( inodeList!=0 && 0==osStat(zPath, &sStat) ){ unixInodeInfo *pInode; - unixEnterMutex(); pInode = inodeList; while( pInode && (pInode->fileId.dev!=sStat.st_dev - || pInode->fileId.ino!=sStat.st_ino) ){ + || pInode->fileId.ino!=(u64)sStat.st_ino) ){ pInode = pInode->pNext; } if( pInode ){ UnixUnusedFd **pp; + assert( sqlite3_mutex_notheld(pInode->pLockMutex) ); + sqlite3_mutex_enter(pInode->pLockMutex); + flags &= (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); for(pp=&pInode->pUnused; *pp && (*pp)->flags!=flags; pp=&((*pp)->pNext)); pUnused = *pp; if( pUnused ){ *pp = pUnused->pNext; } + sqlite3_mutex_leave(pInode->pLockMutex); } - unixLeaveMutex(); } + unixLeaveMutex(); #endif /* if !OS_VXWORKS */ return pUnused; } +/* +** Find the mode, uid and gid of file zFile. +*/ +static int getFileMode( + const char *zFile, /* File name */ + mode_t *pMode, /* OUT: Permissions of zFile */ + uid_t *pUid, /* OUT: uid of zFile. */ + gid_t *pGid /* OUT: gid of zFile. */ +){ + struct stat sStat; /* Output of stat() on database file */ + int rc = SQLITE_OK; + if( 0==osStat(zFile, &sStat) ){ + *pMode = sStat.st_mode & 0777; + *pUid = sStat.st_uid; + *pGid = sStat.st_gid; + }else{ + rc = SQLITE_IOERR_FSTAT; + } + return rc; +} + /* ** This function is called by unixOpen() to determine the unix permissions ** to create new files with. If no error occurs, then SQLITE_OK is returned ** and a value suitable for passing as the third argument to open(2) is -** written to *pMode. If an IO error occurs, an SQLite error code is +** written to *pMode. If an IO error occurs, an SQLite error code is ** returned and the value of *pMode is not modified. ** ** In most cases, this routine sets *pMode to 0, which will become ** an indication to robust_open() to create the file using ** SQLITE_DEFAULT_FILE_PERMISSIONS adjusted by the umask. -** But if the file being opened is a WAL or regular journal file, then -** this function queries the file-system for the permissions on the -** corresponding database file and sets *pMode to this value. Whenever -** possible, WAL and journal files are created using the same permissions +** But if the file being opened is a WAL or regular journal file, then +** this function queries the file-system for the permissions on the +** corresponding database file and sets *pMode to this value. Whenever +** possible, WAL and journal files are created using the same permissions ** as the associated database file. ** ** If the SQLITE_ENABLE_8_3_NAMES option is enabled, then the ** original filename is unavailable. But 8_3_NAMES is only used for ** FAT filesystems and permissions do not matter there, so just use -** the default permissions. +** the default permissions. In 8_3_NAMES mode, leave *pMode set to zero. */ static int findCreateFileMode( const char *zPath, /* Path of file (possibly) being created */ @@ -34661,7 +44634,6 @@ static int findCreateFileMode( if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ char zDb[MAX_PATHNAME+1]; /* Database file path */ int nDb; /* Number of valid bytes in zDb */ - struct stat sStat; /* Output of stat() on database file */ /* zPath is a path to a WAL or journal file. The following block derives ** the path to the associated database file from zPath. This block handles @@ -34672,42 +44644,43 @@ static int findCreateFileMode( ** "-journalNN" ** "-walNN" ** - ** where NN is a decimal number. The NN naming schemes are + ** where NN is a decimal number. The NN naming schemes are ** used by the test_multiplex.c module. + ** + ** In normal operation, the journal file name will always contain + ** a '-' character. However in 8+3 filename mode, or if a corrupt + ** rollback journal specifies a super-journal with a goofy name, then + ** the '-' might be missing or the '-' might be the first character in + ** the filename. In that case, just return SQLITE_OK with *pMode==0. */ - nDb = sqlite3Strlen30(zPath) - 1; - while( zPath[nDb]!='-' ){ -#ifndef SQLITE_ENABLE_8_3_NAMES - /* In the normal case (8+3 filenames disabled) the journal filename - ** is guaranteed to contain a '-' character. */ - assert( nDb>0 ); - assert( sqlite3Isalnum(zPath[nDb]) ); -#else - /* If 8+3 names are possible, then the journal file might not contain - ** a '-' character. So check for that case and return early. */ - if( nDb==0 || zPath[nDb]=='.' ) return SQLITE_OK; -#endif + nDb = sqlite3Strlen30(zPath) - 1; + while( nDb>0 && zPath[nDb]!='.' ){ + if( zPath[nDb]=='-' ){ + memcpy(zDb, zPath, nDb); + zDb[nDb] = '\0'; + rc = getFileMode(zDb, pMode, pUid, pGid); + break; + } nDb--; } - memcpy(zDb, zPath, nDb); - zDb[nDb] = '\0'; - - if( 0==osStat(zDb, &sStat) ){ - *pMode = sStat.st_mode & 0777; - *pUid = sStat.st_uid; - *pGid = sStat.st_gid; - }else{ - rc = SQLITE_IOERR_FSTAT; - } }else if( flags & SQLITE_OPEN_DELETEONCLOSE ){ *pMode = 0600; + }else if( flags & SQLITE_OPEN_URI ){ + /* If this is a main database file and the file was opened using a URI + ** filename, check for the "modeof" parameter. If present, interpret + ** its value as a filename and try to copy the mode, uid and gid from + ** that file. */ + const char *z = sqlite3_uri_parameter(zPath, "modeof"); + if( z ){ + rc = getFileMode(z, pMode, pUid, pGid); + } } return rc; } /* ** Open the file zPath. -** +** ** Previously, the SQLite OS layer used three functions in place of this ** one: ** @@ -34718,13 +44691,13 @@ static int findCreateFileMode( ** These calls correspond to the following combinations of flags: ** ** ReadWrite() -> (READWRITE | CREATE) -** ReadOnly() -> (READONLY) +** ReadOnly() -> (READONLY) ** OpenExclusive() -> (READWRITE | CREATE | EXCLUSIVE) ** ** The old OpenExclusive() accepted a boolean argument - "delFlag". If ** true, the file was configured to be automatically deleted when the -** file handle closed. To achieve the same effect using this new -** interface, add the DELETEONCLOSE flag to those specified above for +** file handle closed. To achieve the same effect using this new +** interface, add the DELETEONCLOSE flag to those specified above for ** OpenExclusive(). */ static int unixOpen( @@ -34737,7 +44710,7 @@ static int unixOpen( unixFile *p = (unixFile *)pFile; int fd = -1; /* File descriptor returned by open() */ int openFlags = 0; /* Flags to pass to open() */ - int eType = flags&0xFFFFFF00; /* Type of file to open */ + int eType = flags&0x0FFF00; /* Type of file to open */ int noLock; /* True to omit locking primitives */ int rc = SQLITE_OK; /* Function Return Code */ int ctrlFlags = 0; /* UNIXFILE_* flags */ @@ -34754,13 +44727,13 @@ static int unixOpen( struct statfs fsInfo; #endif - /* If creating a master or main-file journal, this function will open + /* If creating a super- or main-file journal, this function will open ** a file-descriptor on the directory too. The first time unixSync() ** is called the directory file descriptor will be fsync()ed and close()d. */ - int syncDir = (isCreate && ( - eType==SQLITE_OPEN_MASTER_JOURNAL - || eType==SQLITE_OPEN_MAIN_JOURNAL + int isNewJrnl = (isCreate && ( + eType==SQLITE_OPEN_SUPER_JOURNAL + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_WAL )); @@ -34770,9 +44743,9 @@ static int unixOpen( char zTmpname[MAX_PATHNAME+2]; const char *zName = zPath; - /* Check the following statements are true: + /* Check the following statements are true: ** - ** (a) Exactly one of the READWRITE and READONLY flags must be set, and + ** (a) Exactly one of the READWRITE and READONLY flags must be set, and ** (b) if CREATE is set, then READWRITE must also be set, and ** (c) if EXCLUSIVE is set, then CREATE must also be set. ** (d) if DELETEONCLOSE is set, then CREATE must also be set. @@ -34782,17 +44755,17 @@ static int unixOpen( assert(isExclusive==0 || isCreate); assert(isDelete==0 || isCreate); - /* The main DB, main journal, WAL file and master journal are never + /* The main DB, main journal, WAL file and super-journal are never ** automatically deleted. Nor are they ever temporary files. */ assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_SUPER_JOURNAL ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); /* Assert that the upper layer has set one of the "file-type" flags. */ - assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB - || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB + || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); @@ -34805,9 +44778,13 @@ static int unixOpen( randomnessPid = osGetpid(0); sqlite3_randomness(0,0); } - memset(p, 0, sizeof(unixFile)); +#ifdef SQLITE_ASSERT_NO_FILES + /* Applications that never read or write a persistent disk files */ + assert( zName==0 ); +#endif + if( eType==SQLITE_OPEN_MAIN_DB ){ UnixUnusedFd *pUnused; pUnused = findReusableFd(zName, flags); @@ -34819,7 +44796,7 @@ static int unixOpen( return SQLITE_NOMEM_BKPT; } } - p->pUnused = pUnused; + p->pPreallocatedUnused = pUnused; /* Database filenames are double-zero terminated if they are not ** URIs with parameters. Hence, they can always be passed into @@ -34828,7 +44805,7 @@ static int unixOpen( }else if( !zName ){ /* If zName is NULL, the upper layer is requesting a temp file. */ - assert(isDelete && !syncDir); + assert(isDelete && !isNewJrnl); rc = unixGetTempname(pVfs->mxPathname, zTmpname); if( rc!=SQLITE_OK ){ return rc; @@ -34842,13 +44819,13 @@ static int unixOpen( /* Determine the value of the flags parameter passed to POSIX function ** open(). These must be calculated even if open() is not called, as - ** they may be stored as part of the file handle and used by the + ** they may be stored as part of the file handle and used by the ** 'conch file' locking functions later on. */ if( isReadonly ) openFlags |= O_RDONLY; if( isReadWrite ) openFlags |= O_RDWR; if( isCreate ) openFlags |= O_CREAT; if( isExclusive ) openFlags |= (O_EXCL|O_NOFOLLOW); - openFlags |= (O_LARGEFILE|O_BINARY); + openFlags |= (O_LARGEFILE|O_BINARY|O_NOFOLLOW); if( fd<0 ){ mode_t openMode; /* Permissions to create file with */ @@ -34856,32 +44833,54 @@ static int unixOpen( gid_t gid; /* Groupid for the file */ rc = findCreateFileMode(zName, flags, &openMode, &uid, &gid); if( rc!=SQLITE_OK ){ - assert( !p->pUnused ); + assert( !p->pPreallocatedUnused ); assert( eType==SQLITE_OPEN_WAL || eType==SQLITE_OPEN_MAIN_JOURNAL ); return rc; } fd = robust_open(zName, openFlags, openMode); OSTRACE(("OPENX %-3d %s 0%o\n", fd, zName, openFlags)); assert( !isExclusive || (openFlags & O_CREAT)!=0 ); - if( fd<0 && errno!=EISDIR && isReadWrite ){ - /* Failed to open the file for read/write access. Try read-only. */ - flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); - openFlags &= ~(O_RDWR|O_CREAT); - flags |= SQLITE_OPEN_READONLY; - openFlags |= O_RDONLY; - isReadonly = 1; - fd = robust_open(zName, openFlags, openMode); + if( fd<0 ){ + if( isNewJrnl && errno==EACCES && osAccess(zName, F_OK) ){ + /* If unable to create a journal because the directory is not + ** writable, change the error code to indicate that. */ + rc = SQLITE_READONLY_DIRECTORY; + }else if( errno!=EISDIR && isReadWrite ){ + /* Failed to open the file for read/write access. Try read-only. */ + UnixUnusedFd *pReadonly = 0; + flags &= ~(SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE); + openFlags &= ~(O_RDWR|O_CREAT); + flags |= SQLITE_OPEN_READONLY; + openFlags |= O_RDONLY; + isReadonly = 1; + pReadonly = findReusableFd(zName, flags); + if( pReadonly ){ + fd = pReadonly->fd; + sqlite3_free(pReadonly); + }else{ + fd = robust_open(zName, openFlags, openMode); + } + } } if( fd<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + int rc2 = unixLogError(SQLITE_CANTOPEN_BKPT, "open", zName); + if( rc==SQLITE_OK ) rc = rc2; goto open_finished; } - /* If this process is running as root and if creating a new rollback - ** journal or WAL file, set the ownership of the journal or WAL to be - ** the same as the original database. + /* The owner of the rollback journal or WAL file should always be the + ** same as the owner of the database file. Try to ensure that this is + ** the case. The chown() system call will be a no-op if the current + ** process lacks root privileges, be we should at least try. Without + ** this step, if a root process opens a database file, it can leave + ** behinds a journal/WAL that is owned by root and hence make the + ** database inaccessible to unprivileged processes. + ** + ** If openMode==0, then that means uid and gid are not set correctly + ** (probably because SQLite is configured to use 8+3 filename mode) and + ** in that case we do not want to attempt the chown(). */ - if( flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL) ){ + if( openMode && (flags & (SQLITE_OPEN_WAL|SQLITE_OPEN_MAIN_JOURNAL))!=0 ){ robustFchown(fd, uid, gid); } } @@ -34890,9 +44889,10 @@ static int unixOpen( *pOutFlags = flags; } - if( p->pUnused ){ - p->pUnused->fd = fd; - p->pUnused->flags = flags; + if( p->pPreallocatedUnused ){ + p->pPreallocatedUnused->fd = fd; + p->pPreallocatedUnused->flags = + flags & (SQLITE_OPEN_READONLY|SQLITE_OPEN_READWRITE); } if( isDelete ){ @@ -34913,7 +44913,7 @@ static int unixOpen( p->openFlags = openFlags; } #endif - + #if defined(__APPLE__) || SQLITE_ENABLE_LOCKING_STYLE if( fstatfs(fd, &fsInfo) == -1 ){ storeLastErrno(p, errno); @@ -34933,7 +44933,7 @@ static int unixOpen( if( isReadonly ) ctrlFlags |= UNIXFILE_RDONLY; noLock = eType!=SQLITE_OPEN_MAIN_DB; if( noLock ) ctrlFlags |= UNIXFILE_NOLOCK; - if( syncDir ) ctrlFlags |= UNIXFILE_DIRSYNC; + if( isNewJrnl ) ctrlFlags |= UNIXFILE_DIRSYNC; if( flags & SQLITE_OPEN_URI ) ctrlFlags |= UNIXFILE_URI; #if SQLITE_ENABLE_LOCKING_STYLE @@ -34944,7 +44944,7 @@ static int unixOpen( char *envforce = getenv("SQLITE_FORCE_PROXY_LOCKING"); int useProxy = 0; - /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means + /* SQLITE_FORCE_PROXY_LOCKING==1 means force always use proxy, 0 means ** never use proxy, NULL means use proxy for non-local files only. */ if( envforce!=NULL ){ useProxy = atoi(envforce)>0; @@ -34956,9 +44956,9 @@ static int unixOpen( if( rc==SQLITE_OK ){ rc = proxyTransformUnixFile((unixFile*)pFile, ":auto:"); if( rc!=SQLITE_OK ){ - /* Use unixClose to clean up the resources added in fillInUnixFile - ** and clear all the structure's references. Specifically, - ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op + /* Use unixClose to clean up the resources added in fillInUnixFile + ** and clear all the structure's references. Specifically, + ** pFile->pMethods will be NULL so sqlite3OsClose will be a no-op */ unixClose(pFile); return rc; @@ -34968,12 +44968,15 @@ static int unixOpen( } } #endif - + + assert( zPath==0 || zPath[0]=='/' + || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL + ); rc = fillInUnixFile(pVfs, fd, pFile, zPath, ctrlFlags); open_finished: if( rc!=SQLITE_OK ){ - sqlite3_free(p->pUnused); + sqlite3_free(p->pPreallocatedUnused); } return rc; } @@ -35047,7 +45050,8 @@ static int unixAccess( if( flags==SQLITE_ACCESS_EXISTS ){ struct stat buf; - *pResOut = (0==osStat(zPath, &buf) && buf.st_size>0); + *pResOut = 0==osStat(zPath, &buf) && + (!S_ISREG(buf.st_mode) || buf.st_size>0); }else{ *pResOut = osAccess(zPath, W_OK|R_OK)==0; } @@ -35055,38 +45059,105 @@ static int unixAccess( } /* -** +** A pathname under construction */ -static int mkFullPathname( - const char *zPath, /* Input path */ - char *zOut, /* Output buffer */ - int nOut /* Allocated size of buffer zOut */ +typedef struct DbPath DbPath; +struct DbPath { + int rc; /* Non-zero following any error */ + int nSymlink; /* Number of symlinks resolved */ + char *zOut; /* Write the pathname here */ + int nOut; /* Bytes of space available to zOut[] */ + int nUsed; /* Bytes of zOut[] currently being used */ +}; + +/* Forward reference */ +static void appendAllPathElements(DbPath*,const char*); + +/* +** Append a single path element to the DbPath under construction +*/ +static void appendOnePathElement( + DbPath *pPath, /* Path under construction, to which to append zName */ + const char *zName, /* Name to append to pPath. Not zero-terminated */ + int nName /* Number of significant bytes in zName */ ){ - int nPath = sqlite3Strlen30(zPath); - int iOff = 0; - if( zPath[0]!='/' ){ - if( osGetcwd(zOut, nOut-2)==0 ){ - return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); + assert( nName>0 ); + assert( zName!=0 ); + if( zName[0]=='.' ){ + if( nName==1 ) return; + if( zName[1]=='.' && nName==2 ){ + if( pPath->nUsed>1 ){ + assert( pPath->zOut[0]=='/' ); + while( pPath->zOut[--pPath->nUsed]!='/' ){} + } + return; } - iOff = sqlite3Strlen30(zOut); - zOut[iOff++] = '/'; } - if( (iOff+nPath+1)>nOut ){ - /* SQLite assumes that xFullPathname() nul-terminates the output buffer - ** even if it returns an error. */ - zOut[iOff] = '\0'; - return SQLITE_CANTOPEN_BKPT; + if( pPath->nUsed + nName + 2 >= pPath->nOut ){ + pPath->rc = SQLITE_ERROR; + return; } - sqlite3_snprintf(nOut-iOff, &zOut[iOff], "%s", zPath); - return SQLITE_OK; + pPath->zOut[pPath->nUsed++] = '/'; + memcpy(&pPath->zOut[pPath->nUsed], zName, nName); + pPath->nUsed += nName; +#if defined(HAVE_READLINK) && defined(HAVE_LSTAT) + if( pPath->rc==SQLITE_OK ){ + const char *zIn; + struct stat buf; + pPath->zOut[pPath->nUsed] = 0; + zIn = pPath->zOut; + if( osLstat(zIn, &buf)!=0 ){ + if( errno!=ENOENT ){ + pPath->rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); + } + }else if( S_ISLNK(buf.st_mode) ){ + ssize_t got; + char zLnk[SQLITE_MAX_PATHLEN+2]; + if( pPath->nSymlink++ > SQLITE_MAX_SYMLINK ){ + pPath->rc = SQLITE_CANTOPEN_BKPT; + return; + } + got = osReadlink(zIn, zLnk, sizeof(zLnk)-2); + if( got<=0 || got>=(ssize_t)sizeof(zLnk)-2 ){ + pPath->rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); + return; + } + zLnk[got] = 0; + if( zLnk[0]=='/' ){ + pPath->nUsed = 0; + }else{ + pPath->nUsed -= nName + 1; + } + appendAllPathElements(pPath, zLnk); + } + } +#endif +} + +/* +** Append all path elements in zPath to the DbPath under construction. +*/ +static void appendAllPathElements( + DbPath *pPath, /* Path under construction, to which to append zName */ + const char *zPath /* Path to append to pPath. Is zero-terminated */ +){ + int i = 0; + int j = 0; + do{ + while( zPath[i] && zPath[i]!='/' ){ i++; } + if( i>j ){ + appendOnePathElement(pPath, &zPath[j], i-j); + } + j = i+1; + }while( zPath[i++] ); } /* ** Turn a relative pathname into a full pathname. The relative path ** is stored as a nul-terminated string in the buffer pointed to by -** zPath. +** zPath. ** -** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes +** zOut points to a buffer of at least sqlite3_vfs.mxPathname bytes ** (in this case, MAX_PATHNAME bytes). The full-path is written to ** this buffer before returning. */ @@ -35096,84 +45167,27 @@ static int unixFullPathname( int nOut, /* Size of output buffer in bytes */ char *zOut /* Output buffer */ ){ -#if !defined(HAVE_READLINK) || !defined(HAVE_LSTAT) - return mkFullPathname(zPath, zOut, nOut); -#else - int rc = SQLITE_OK; - int nByte; - int nLink = 1; /* Number of symbolic links followed so far */ - const char *zIn = zPath; /* Input path for each iteration of loop */ - char *zDel = 0; - - assert( pVfs->mxPathname==MAX_PATHNAME ); + DbPath path; UNUSED_PARAMETER(pVfs); - - /* It's odd to simulate an io-error here, but really this is just - ** using the io-error infrastructure to test that SQLite handles this - ** function failing. This function could fail if, for example, the - ** current working directory has been unlinked. - */ - SimulateIOError( return SQLITE_ERROR ); - - do { - - /* Call stat() on path zIn. Set bLink to true if the path is a symbolic - ** link, or false otherwise. */ - int bLink = 0; - struct stat buf; - if( osLstat(zIn, &buf)!=0 ){ - if( errno!=ENOENT ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "lstat", zIn); - } - }else{ - bLink = S_ISLNK(buf.st_mode); + path.rc = 0; + path.nUsed = 0; + path.nSymlink = 0; + path.nOut = nOut; + path.zOut = zOut; + if( zPath[0]!='/' ){ + char zPwd[SQLITE_MAX_PATHLEN+2]; + if( osGetcwd(zPwd, sizeof(zPwd)-2)==0 ){ + return unixLogError(SQLITE_CANTOPEN_BKPT, "getcwd", zPath); } - - if( bLink ){ - if( zDel==0 ){ - zDel = sqlite3_malloc(nOut); - if( zDel==0 ) rc = SQLITE_NOMEM_BKPT; - }else if( ++nLink>SQLITE_MAX_SYMLINKS ){ - rc = SQLITE_CANTOPEN_BKPT; - } - - if( rc==SQLITE_OK ){ - nByte = osReadlink(zIn, zDel, nOut-1); - if( nByte<0 ){ - rc = unixLogError(SQLITE_CANTOPEN_BKPT, "readlink", zIn); - }else{ - if( zDel[0]!='/' ){ - int n; - for(n = sqlite3Strlen30(zIn); n>0 && zIn[n-1]!='/'; n--); - if( nByte+n+1>nOut ){ - rc = SQLITE_CANTOPEN_BKPT; - }else{ - memmove(&zDel[n], zDel, nByte+1); - memcpy(zDel, zIn, n); - nByte += n; - } - } - zDel[nByte] = '\0'; - } - } - - zIn = zDel; - } - - assert( rc!=SQLITE_OK || zIn!=zOut || zIn[0]=='/' ); - if( rc==SQLITE_OK && zIn!=zOut ){ - rc = mkFullPathname(zIn, zOut, nOut); - } - if( bLink==0 ) break; - zIn = zOut; - }while( rc==SQLITE_OK ); - - sqlite3_free(zDel); - return rc; -#endif /* HAVE_READLINK && HAVE_LSTAT */ + appendAllPathElements(&path, zPwd); + } + appendAllPathElements(&path, zPath); + zOut[path.nUsed] = 0; + if( path.rc || path.nUsed<2 ) return SQLITE_CANTOPEN_BKPT; + if( path.nSymlink ) return SQLITE_OK_SYMLINK; + return SQLITE_OK; } - #ifndef SQLITE_OMIT_LOAD_EXTENSION /* ** Interfaces for opening a shared library, finding entry points @@ -35203,7 +45217,7 @@ static void unixDlError(sqlite3_vfs *NotUsed, int nBuf, char *zBufOut){ unixLeaveMutex(); } static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ - /* + /* ** GCC with -pedantic-errors says that C90 does not allow a void* to be ** cast into a pointer to a function. And yet the library dlsym() routine ** returns a void* which is really a pointer to a function. So how do we @@ -35213,7 +45227,7 @@ static void (*unixDlSym(sqlite3_vfs *NotUsed, void *p, const char*zSym))(void){ ** parameters void* and const char* and returning a pointer to a function. ** We initialize x by assigning it a pointer to the dlsym() function. ** (That assignment requires a cast.) Then we call the function that - ** x points to. + ** x points to. ** ** This work-around is unlikely to work correctly on any system where ** you really cannot cast a function pointer into void*. But then, on the @@ -35256,7 +45270,7 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** tests repeatable. */ memset(zBuf, 0, nBuf); - randomnessPid = osGetpid(0); + randomnessPid = osGetpid(0); #if !defined(SQLITE_TEST) && !defined(SQLITE_OMIT_RANDOMNESS) { int fd, got; @@ -35287,16 +45301,22 @@ static int unixRandomness(sqlite3_vfs *NotUsed, int nBuf, char *zBuf){ ** than the argument. */ static int unixSleep(sqlite3_vfs *NotUsed, int microseconds){ -#if OS_VXWORKS +#if !defined(HAVE_NANOSLEEP) || HAVE_NANOSLEEP+0 struct timespec sp; - sp.tv_sec = microseconds / 1000000; sp.tv_nsec = (microseconds % 1000000) * 1000; + + /* Almost all modern unix systems support nanosleep(). But if you are + ** compiling for one of the rare exceptions, you can use + ** -DHAVE_NANOSLEEP=0 (perhaps in conjuction with -DHAVE_USLEEP if + ** usleep() is available) in order to bypass the use of nanosleep() */ nanosleep(&sp, NULL); + UNUSED_PARAMETER(NotUsed); return microseconds; #elif defined(HAVE_USLEEP) && HAVE_USLEEP - usleep(microseconds); + if( microseconds>=1000000 ) sleep(microseconds/1000000); + if( microseconds%1000000 ) usleep(microseconds%1000000); UNUSED_PARAMETER(NotUsed); return microseconds; #else @@ -35323,7 +45343,7 @@ SQLITE_API int sqlite3_current_time = 0; /* Fake system time in seconds since 1 ** epoch of noon in Greenwich on November 24, 4714 B.C according to the ** proleptic Gregorian calendar. ** -** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date +** On success, return SQLITE_OK. Return SQLITE_ERROR if the time and date ** cannot be found. */ static int unixCurrentTimeInt64(sqlite3_vfs *NotUsed, sqlite3_int64 *piNow){ @@ -35430,7 +45450,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** To address the performance and cache coherency issues, proxy file locking ** changes the way database access is controlled by limiting access to a ** single host at a time and moving file locks off of the database file -** and onto a proxy file on the local file system. +** and onto a proxy file on the local file system. ** ** ** Using proxy locks @@ -35456,19 +45476,19 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** actual proxy file name is generated from the name and path of the ** database file. For example: ** -** For database path "/Users/me/foo.db" +** For database path "/Users/me/foo.db" ** The lock path will be "/sqliteplocks/_Users_me_foo.db:auto:") ** ** Once a lock proxy is configured for a database connection, it can not ** be removed, however it may be switched to a different proxy path via ** the above APIs (assuming the conch file is not being held by another -** connection or process). +** connection or process). ** ** ** How proxy locking works ** ----------------------- ** -** Proxy file locking relies primarily on two new supporting files: +** Proxy file locking relies primarily on two new supporting files: ** ** * conch file to limit access to the database file to a single host ** at a time @@ -35495,11 +45515,11 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** host (the conch ensures that they all use the same local lock file). ** ** Requesting the lock proxy does not immediately take the conch, it is -** only taken when the first request to lock database file is made. +** only taken when the first request to lock database file is made. ** This matches the semantics of the traditional locking behavior, where ** opening a connection to a database file does not take a lock on it. -** The shared lock and an open file descriptor are maintained until -** the connection to the database is closed. +** The shared lock and an open file descriptor are maintained until +** the connection to the database is closed. ** ** The proxy file and the lock file are never deleted so they only need ** to be created the first time they are used. @@ -35513,7 +45533,7 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** automatically configured for proxy locking, lock files are ** named automatically using the same logic as ** PRAGMA lock_proxy_file=":auto:" -** +** ** SQLITE_PROXY_DEBUG ** ** Enables the logging of error messages during host id file @@ -35528,8 +45548,8 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ ** ** Permissions to use when creating a directory for storing the ** lock proxy files, only used when LOCKPROXYDIR is not set. -** -** +** +** ** As mentioned above, when compiled with SQLITE_PREFER_PROXY_LOCKING, ** setting the environment variable SQLITE_FORCE_PROXY_LOCKING to 1 will ** force proxy locking to be used for every database file opened, and 0 @@ -35539,12 +45559,12 @@ static int unixGetLastError(sqlite3_vfs *NotUsed, int NotUsed2, char *NotUsed3){ */ /* -** Proxy locking is only available on MacOSX +** Proxy locking is only available on MacOSX */ #if defined(__APPLE__) && SQLITE_ENABLE_LOCKING_STYLE /* -** The proxyLockingContext has the path and file structures for the remote +** The proxyLockingContext has the path and file structures for the remote ** and local proxy files in it */ typedef struct proxyLockingContext proxyLockingContext; @@ -35560,10 +45580,10 @@ struct proxyLockingContext { sqlite3_io_methods const *pOldMethod; /* Original I/O methods for close */ }; -/* -** The proxy lock file path for the database at dbPath is written into lPath, +/* +** The proxy lock file path for the database at dbPath is written into lPath, ** which must point to valid, writable memory large enough for a maxLen length -** file path. +** file path. */ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ int len; @@ -35580,7 +45600,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ lPath, errno, osGetpid(0))); return SQLITE_IOERR_LOCK; } - len = strlcat(lPath, "sqliteplocks", maxLen); + len = strlcat(lPath, "sqliteplocks", maxLen); } # else len = strlcpy(lPath, "/tmp/", maxLen); @@ -35590,7 +45610,7 @@ static int proxyGetLockPath(const char *dbPath, char *lPath, size_t maxLen){ if( lPath[len-1]!='/' ){ len = strlcat(lPath, "/", maxLen); } - + /* transform the db path to a unique cache name */ dbLen = (int)strlen(dbPath); for( i=0; i 0) ){ /* only mkdir if leaf dir != "." or "/" or ".." */ - if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') + if( i-start>2 || (i-start==1 && buf[start] != '.' && buf[start] != '/') || (i-start==2 && buf[start] != '.' && buf[start+1] != '.') ){ buf[i]='\0'; if( osMkdir(buf, SQLITE_DEFAULT_PROXYDIR_PERMISSIONS) ){ @@ -35654,7 +45674,7 @@ static int proxyCreateUnixFile( int fd = -1; unixFile *pNew; int rc = SQLITE_OK; - int openFlags = O_RDWR | O_CREAT; + int openFlags = O_RDWR | O_CREAT | O_NOFOLLOW; sqlite3_vfs dummyVfs; int terrno = 0; UnixUnusedFd *pUnused = NULL; @@ -35684,7 +45704,7 @@ static int proxyCreateUnixFile( } } if( fd<0 ){ - openFlags = O_RDONLY; + openFlags = O_RDONLY | O_NOFOLLOW; fd = robust_open(path, openFlags, 0); terrno = errno; } @@ -35695,13 +45715,13 @@ static int proxyCreateUnixFile( switch (terrno) { case EACCES: return SQLITE_PERM; - case EIO: + case EIO: return SQLITE_IOERR_LOCK; /* even though it is the conch */ default: return SQLITE_CANTOPEN_BKPT; } } - + pNew = (unixFile *)sqlite3_malloc64(sizeof(*pNew)); if( pNew==NULL ){ rc = SQLITE_NOMEM_BKPT; @@ -35714,14 +45734,14 @@ static int proxyCreateUnixFile( dummyVfs.zName = "dummy"; pUnused->fd = fd; pUnused->flags = openFlags; - pNew->pUnused = pUnused; - + pNew->pPreallocatedUnused = pUnused; + rc = fillInUnixFile(&dummyVfs, fd, (sqlite3_file*)pNew, path, 0); if( rc==SQLITE_OK ){ *ppFile = pNew; return SQLITE_OK; } -end_create_proxy: +end_create_proxy: robust_close(pNew, fd, __LINE__); sqlite3_free(pNew); sqlite3_free(pUnused); @@ -35735,18 +45755,18 @@ SQLITE_API int sqlite3_hostid_num = 0; #define PROXY_HOSTIDLEN 16 /* conch file host id length */ -#ifdef HAVE_GETHOSTUUID +#if HAVE_GETHOSTUUID /* Not always defined in the headers as it ought to be */ extern int gethostuuid(uuid_t id, const struct timespec *wait); #endif -/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN +/* get the host ID via gethostuuid(), pHostID must point to PROXY_HOSTIDLEN ** bytes of writable memory. */ static int proxyGetHostID(unsigned char *pHostID, int *pError){ assert(PROXY_HOSTIDLEN == sizeof(uuid_t)); memset(pHostID, 0, PROXY_HOSTIDLEN); -#ifdef HAVE_GETHOSTUUID +#if HAVE_GETHOSTUUID { struct timespec timeout = {1, 0}; /* 1 sec timeout */ if( gethostuuid(pHostID, &timeout) ){ @@ -35766,7 +45786,7 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ pHostID[0] = (char)(pHostID[0] + (char)(sqlite3_hostid_num & 0xFF)); } #endif - + return SQLITE_OK; } @@ -35777,14 +45797,14 @@ static int proxyGetHostID(unsigned char *pHostID, int *pError){ #define PROXY_PATHINDEX (PROXY_HEADERLEN+PROXY_HOSTIDLEN) #define PROXY_MAXCONCHLEN (PROXY_HEADERLEN+PROXY_HOSTIDLEN+MAXPATHLEN) -/* -** Takes an open conch file, copies the contents to a new path and then moves +/* +** Takes an open conch file, copies the contents to a new path and then moves ** it back. The newly created file's file descriptor is assigned to the -** conch file structure and finally the original conch file descriptor is +** conch file structure and finally the original conch file descriptor is ** closed. Returns zero if successful. */ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *conchFile = pCtx->conchFile; char tPath[MAXPATHLEN]; char buf[PROXY_MAXCONCHLEN]; @@ -35798,7 +45818,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ /* create a new path by replace the trailing '-conch' with '-break' */ pathLen = strlcpy(tPath, cPath, MAXPATHLEN); - if( pathLen>MAXPATHLEN || pathLen<6 || + if( pathLen>MAXPATHLEN || pathLen<6 || (strlcpy(&tPath[pathLen-5], "break", 6) != 5) ){ sqlite3_snprintf(sizeof(errmsg),errmsg,"path error (len %d)",(int)pathLen); goto end_breaklock; @@ -35810,7 +45830,7 @@ static int proxyBreakConchLock(unixFile *pFile, uuid_t myHostID){ goto end_breaklock; } /* write it out to the temporary break file */ - fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL), 0); + fd = robust_open(tPath, (O_RDWR|O_CREAT|O_EXCL|O_NOFOLLOW), 0); if( fd<0 ){ sqlite3_snprintf(sizeof(errmsg), errmsg, "create failed (%d)", errno); goto end_breaklock; @@ -35840,24 +45860,24 @@ end_breaklock: return rc; } -/* Take the requested lock on the conch file and break a stale lock if the +/* Take the requested lock on the conch file and break a stale lock if the ** host id matches. */ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; unixFile *conchFile = pCtx->conchFile; int rc = SQLITE_OK; int nTries = 0; struct timespec conchModTime; - + memset(&conchModTime, 0, sizeof(conchModTime)); do { rc = conchFile->pMethod->xLock((sqlite3_file*)conchFile, lockType); nTries ++; if( rc==SQLITE_BUSY ){ /* If the lock failed (busy): - * 1st try: get the mod time of the conch, wait 0.5s and try again. - * 2nd try: fail if the mod time changed or host id is different, wait + * 1st try: get the mod time of the conch, wait 0.5s and try again. + * 2nd try: fail if the mod time changed or host id is different, wait * 10 sec and try again * 3rd try: break the lock unless the mod time has changed. */ @@ -35866,20 +45886,20 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ storeLastErrno(pFile, errno); return SQLITE_IOERR_LOCK; } - + if( nTries==1 ){ conchModTime = buf.st_mtimespec; - usleep(500000); /* wait 0.5 sec and try the lock again*/ - continue; + unixSleep(0,500000); /* wait 0.5 sec and try the lock again*/ + continue; } assert( nTries>1 ); - if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || + if( conchModTime.tv_sec != buf.st_mtimespec.tv_sec || conchModTime.tv_nsec != buf.st_mtimespec.tv_nsec ){ return SQLITE_BUSY; } - - if( nTries==2 ){ + + if( nTries==2 ){ char tBuf[PROXY_MAXCONCHLEN]; int len = osPread(conchFile->h, tBuf, PROXY_MAXCONCHLEN, 0); if( len<0 ){ @@ -35895,10 +45915,10 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ /* don't break the lock on short read or a version mismatch */ return SQLITE_BUSY; } - usleep(10000000); /* wait 10 sec and try the lock again */ - continue; + unixSleep(0,10000000); /* wait 10 sec and try the lock again */ + continue; } - + assert( nTries==3 ); if( 0==proxyBreakConchLock(pFile, myHostID) ){ rc = SQLITE_OK; @@ -35911,19 +45931,19 @@ static int proxyConchLock(unixFile *pFile, uuid_t myHostID, int lockType){ } } } while( rc==SQLITE_BUSY && nTries<3 ); - + return rc; } -/* Takes the conch by taking a shared lock and read the contents conch, if -** lockPath is non-NULL, the host ID and lock file path must match. A NULL -** lockPath means that the lockPath in the conch file will be used if the -** host IDs match, or a new lock path will be generated automatically +/* Takes the conch by taking a shared lock and read the contents conch, if +** lockPath is non-NULL, the host ID and lock file path must match. A NULL +** lockPath means that the lockPath in the conch file will be used if the +** host IDs match, or a new lock path will be generated automatically ** and written to the conch file. */ static int proxyTakeConch(unixFile *pFile){ - proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; - + proxyLockingContext *pCtx = (proxyLockingContext *)pFile->lockingContext; + if( pCtx->conchHeld!=0 ){ return SQLITE_OK; }else{ @@ -35939,7 +45959,7 @@ static int proxyTakeConch(unixFile *pFile){ int readLen = 0; int tryOldLockPath = 0; int forceNewLockPath = 0; - + OSTRACE(("TAKECONCH %d for %s pid=%d\n", conchFile->h, (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), osGetpid(0))); @@ -35960,21 +45980,21 @@ static int proxyTakeConch(unixFile *pFile){ storeLastErrno(pFile, conchFile->lastErrno); rc = SQLITE_IOERR_READ; goto end_takeconch; - }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || + }else if( readLen<=(PROXY_HEADERLEN+PROXY_HOSTIDLEN) || readBuf[0]!=(char)PROXY_CONCHVERSION ){ - /* a short read or version format mismatch means we need to create a new - ** conch file. + /* a short read or version format mismatch means we need to create a new + ** conch file. */ createConch = 1; } /* if the host id matches and the lock path already exists in the conch - ** we'll try to use the path there, if we can't open that path, we'll - ** retry with a new auto-generated path + ** we'll try to use the path there, if we can't open that path, we'll + ** retry with a new auto-generated path */ do { /* in case we need to try again for an :auto: named lock file */ if( !createConch && !forceNewLockPath ){ - hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, + hostIdMatch = !memcmp(&readBuf[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); /* if the conch has data compare the contents */ if( !pCtx->lockProxyPath ){ @@ -35983,7 +46003,7 @@ static int proxyTakeConch(unixFile *pFile){ */ if( hostIdMatch ){ size_t pathLen = (readLen - PROXY_PATHINDEX); - + if( pathLen>=MAXPATHLEN ){ pathLen=MAXPATHLEN-1; } @@ -35999,23 +46019,23 @@ static int proxyTakeConch(unixFile *pFile){ readLen-PROXY_PATHINDEX) ){ /* conch host and lock path match */ - goto end_takeconch; + goto end_takeconch; } } - + /* if the conch isn't writable and doesn't match, we can't take it */ if( (conchFile->openFlags&O_RDWR) == 0 ){ rc = SQLITE_BUSY; goto end_takeconch; } - + /* either the conch didn't match or we need to create a new one */ if( !pCtx->lockProxyPath ){ proxyGetLockPath(pCtx->dbPath, lockPath, MAXPATHLEN); tempLockPath = lockPath; /* create a copy of the lock path _only_ if the conch is taken */ } - + /* update conch with host and path (this will fail if other process ** has a shared lock already), if the host id matches, use the big ** stick. @@ -36026,7 +46046,7 @@ static int proxyTakeConch(unixFile *pFile){ /* We are trying for an exclusive lock but another thread in this ** same process is still holding a shared lock. */ rc = SQLITE_BUSY; - } else { + } else { rc = proxyConchLock(pFile, myHostID, EXCLUSIVE_LOCK); } }else{ @@ -36035,7 +46055,7 @@ static int proxyTakeConch(unixFile *pFile){ if( rc==SQLITE_OK ){ char writeBuffer[PROXY_MAXCONCHLEN]; int writeSize = 0; - + writeBuffer[0] = (char)PROXY_CONCHVERSION; memcpy(&writeBuffer[PROXY_HEADERLEN], myHostID, PROXY_HOSTIDLEN); if( pCtx->lockProxyPath!=NULL ){ @@ -36048,8 +46068,8 @@ static int proxyTakeConch(unixFile *pFile){ robust_ftruncate(conchFile->h, writeSize); rc = unixWrite((sqlite3_file *)conchFile, writeBuffer, writeSize, 0); full_fsync(conchFile->h,0,0); - /* If we created a new conch file (not just updated the contents of a - ** valid conch file), try to match the permissions of the database + /* If we created a new conch file (not just updated the contents of a + ** valid conch file), try to match the permissions of the database */ if( rc==SQLITE_OK && createConch ){ struct stat buf; @@ -36073,14 +46093,14 @@ static int proxyTakeConch(unixFile *pFile){ } }else{ int code = errno; - fprintf(stderr, "STAT FAILED[%d] with %d %s\n", + fprintf(stderr, "STAT FAILED[%d] with %d %s\n", err, code, strerror(code)); #endif } } } conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, SHARED_LOCK); - + end_takeconch: OSTRACE(("TRANSPROXY: CLOSE %d\n", pFile->h)); if( rc==SQLITE_OK && pFile->openFlags ){ @@ -36103,7 +46123,7 @@ static int proxyTakeConch(unixFile *pFile){ rc = proxyCreateUnixFile(path, &pCtx->lockProxy, 1); if( rc!=SQLITE_OK && rc!=SQLITE_NOMEM && tryOldLockPath ){ /* we couldn't create the proxy lock file with the old lock file path - ** so try again via auto-naming + ** so try again via auto-naming */ forceNewLockPath = 1; tryOldLockPath = 0; @@ -36123,7 +46143,7 @@ static int proxyTakeConch(unixFile *pFile){ } if( rc==SQLITE_OK ){ pCtx->conchHeld = 1; - + if( pCtx->lockProxy->pMethod == &afpIoMethods ){ afpLockingContext *afpCtx; afpCtx = (afpLockingContext *)pCtx->lockProxy->lockingContext; @@ -36135,7 +46155,7 @@ static int proxyTakeConch(unixFile *pFile){ OSTRACE(("TAKECONCH %d %s\n", conchFile->h, rc==SQLITE_OK?"ok":"failed")); return rc; - } while (1); /* in case we need to retry the :auto: lock file - + } while (1); /* in case we need to retry the :auto: lock file - ** we should never get here except via the 'continue' call. */ } } @@ -36151,7 +46171,7 @@ static int proxyReleaseConch(unixFile *pFile){ pCtx = (proxyLockingContext *)pFile->lockingContext; conchFile = pCtx->conchFile; OSTRACE(("RELEASECONCH %d for %s pid=%d\n", conchFile->h, - (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), + (pCtx->lockProxyPath ? pCtx->lockProxyPath : ":auto:"), osGetpid(0))); if( pCtx->conchHeld>0 ){ rc = conchFile->pMethod->xUnlock((sqlite3_file*)conchFile, NO_LOCK); @@ -36179,13 +46199,13 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ char *conchPath; /* buffer in which to construct conch name */ /* Allocate space for the conch filename and initialize the name to - ** the name of the original database file. */ + ** the name of the original database file. */ *pConchPath = conchPath = (char *)sqlite3_malloc64(len + 8); if( conchPath==0 ){ return SQLITE_NOMEM_BKPT; } memcpy(conchPath, dbPath, len+1); - + /* now insert a "." before the last / character */ for( i=(len-1); i>=0; i-- ){ if( conchPath[i]=='/' ){ @@ -36208,7 +46228,7 @@ static int proxyCreateConchPathname(char *dbPath, char **pConchPath){ /* Takes a fully configured proxy locking-style unix file and switches -** the local lock file path +** the local lock file path */ static int switchLockProxyPath(unixFile *pFile, const char *path) { proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; @@ -36217,7 +46237,7 @@ static int switchLockProxyPath(unixFile *pFile, const char *path) { if( pFile->eFileLock!=NO_LOCK ){ return SQLITE_BUSY; - } + } /* nothing to do if the path is NULL, :auto: or matches the existing path */ if( !path || path[0]=='\0' || !strcmp(path, ":auto:") || @@ -36235,7 +46255,7 @@ static int switchLockProxyPath(unixFile *pFile, const char *path) { sqlite3_free(oldPath); pCtx->lockProxyPath = sqlite3DbStrDup(0, path); } - + return rc; } @@ -36249,7 +46269,7 @@ static int switchLockProxyPath(unixFile *pFile, const char *path) { static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ #if defined(__APPLE__) if( pFile->pMethod == &afpIoMethods ){ - /* afp style keeps a reference to the db path in the filePath field + /* afp style keeps a reference to the db path in the filePath field ** of the struct */ assert( (int)strlen((char*)pFile->lockingContext)<=MAXPATHLEN ); strlcpy(dbPath, ((afpLockingContext *)pFile->lockingContext)->dbPath, @@ -36270,9 +46290,9 @@ static int proxyGetDbPathForUnixFile(unixFile *pFile, char *dbPath){ } /* -** Takes an already filled in unix file and alters it so all file locking +** Takes an already filled in unix file and alters it so all file locking ** will be performed on the local proxy lock file. The following fields -** are preserved in the locking context so that they can be restored and +** are preserved in the locking context so that they can be restored and ** the unix structure properly cleaned up at close time: ** ->lockingContext ** ->pMethod @@ -36282,7 +46302,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { char dbPath[MAXPATHLEN+1]; /* Name of the database file */ char *lockPath=NULL; int rc = SQLITE_OK; - + if( pFile->eFileLock!=NO_LOCK ){ return SQLITE_BUSY; } @@ -36292,7 +46312,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { }else{ lockPath=(char *)path; } - + OSTRACE(("TRANSPROXY %d for %s pid=%d\n", pFile->h, (lockPath ? lockPath : ":auto:"), osGetpid(0))); @@ -36326,7 +46346,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { rc = SQLITE_OK; } } - } + } if( rc==SQLITE_OK && lockPath ){ pCtx->lockProxyPath = sqlite3DbStrDup(0, lockPath); } @@ -36338,7 +46358,7 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { } } if( rc==SQLITE_OK ){ - /* all memory is allocated, proxys are created and assigned, + /* all memory is allocated, proxys are created and assigned, ** switch the locking context and pMethod then return. */ pCtx->oldLockingContext = pFile->lockingContext; @@ -36346,12 +46366,12 @@ static int proxyTransformUnixFile(unixFile *pFile, const char *path) { pCtx->pOldMethod = pFile->pMethod; pFile->pMethod = &proxyIoMethods; }else{ - if( pCtx->conchFile ){ + if( pCtx->conchFile ){ pCtx->conchFile->pMethod->xClose((sqlite3_file *)pCtx->conchFile); sqlite3_free(pCtx->conchFile); } sqlite3DbFree(0, pCtx->lockProxyPath); - sqlite3_free(pCtx->conchFilePath); + sqlite3_free(pCtx->conchFilePath); sqlite3_free(pCtx); } OSTRACE(("TRANSPROXY %d %s\n", pFile->h, @@ -36389,7 +46409,7 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ if( isProxyStyle ){ /* turn off proxy locking - not supported. If support is added for ** switching proxy locking mode off then it will need to fail if - ** the journal mode is WAL mode. + ** the journal mode is WAL mode. */ rc = SQLITE_ERROR /*SQLITE_PROTOCOL? SQLITE_MISUSE?*/; }else{ @@ -36399,9 +46419,9 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ }else{ const char *proxyPath = (const char *)pArg; if( isProxyStyle ){ - proxyLockingContext *pCtx = + proxyLockingContext *pCtx = (proxyLockingContext*)pFile->lockingContext; - if( !strcmp(pArg, ":auto:") + if( !strcmp(pArg, ":auto:") || (pCtx->lockProxyPath && !strncmp(pCtx->lockProxyPath, proxyPath, MAXPATHLEN)) ){ @@ -36420,7 +46440,7 @@ static int proxyFileControl(sqlite3_file *id, int op, void *pArg){ assert( 0 ); /* The call assures that only valid opcodes are sent */ } } - /*NOTREACHED*/ + /*NOTREACHED*/ assert(0); return SQLITE_ERROR; } @@ -36526,7 +46546,7 @@ static int proxyClose(sqlite3_file *id) { unixFile *lockProxy = pCtx->lockProxy; unixFile *conchFile = pCtx->conchFile; int rc = SQLITE_OK; - + if( lockProxy ){ rc = lockProxy->pMethod->xUnlock((sqlite3_file*)lockProxy, NO_LOCK); if( rc ) return rc; @@ -36563,7 +46583,7 @@ static int proxyClose(sqlite3_file *id) { ** The proxy locking style is intended for use with AFP filesystems. ** And since AFP is only supported on MacOSX, the proxy locking is also ** restricted to MacOSX. -** +** ** ******************* End of the proxy lock implementation ********************** ******************************************************************************/ @@ -36581,8 +46601,8 @@ static int proxyClose(sqlite3_file *id) { ** necessarily been initialized when this routine is called, and so they ** should not be used. */ -SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ - /* +SQLITE_API int sqlite3_os_init(void){ + /* ** The following macro defines an initializer for an sqlite3_vfs object. ** The name of the VFS is NAME. The pAppData is a pointer to a pointer ** to the "finder" function. (pAppData is a pointer to a pointer because @@ -36598,7 +46618,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ ** ** Most finders simply return a pointer to a fixed sqlite3_io_methods ** object. But the "autolockIoFinder" available on MacOSX does a little - ** more than that; it looks at the filesystem type that hosts the + ** more than that; it looks at the filesystem type that hosts the ** database file and tries to choose an locking method appropriate for ** that filesystem time. */ @@ -36664,13 +46684,44 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ /* Double-check that the aSyscall[] array has been constructed ** correctly. See ticket [bb3a86e890c8e96ab] */ - assert( ArraySize(aSyscall)==28 ); + assert( ArraySize(aSyscall)==29 ); /* Register all VFSes defined in the aVfs[] array */ for(i=0; i<(sizeof(aVfs)/sizeof(sqlite3_vfs)); i++){ +#ifdef SQLITE_DEFAULT_UNIX_VFS + sqlite3_vfs_register(&aVfs[i], + 0==strcmp(aVfs[i].zName,SQLITE_DEFAULT_UNIX_VFS)); +#else sqlite3_vfs_register(&aVfs[i], i==0); +#endif } - return SQLITE_OK; +#ifdef SQLITE_OS_KV_OPTIONAL + sqlite3KvvfsInit(); +#endif + unixBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); + +#ifndef SQLITE_OMIT_WAL + /* Validate lock assumptions */ + assert( SQLITE_SHM_NLOCK==8 ); /* Number of available locks */ + assert( UNIX_SHM_BASE==120 ); /* Start of locking area */ + /* Locks: + ** WRITE UNIX_SHM_BASE 120 + ** CKPT UNIX_SHM_BASE+1 121 + ** RECOVER UNIX_SHM_BASE+2 122 + ** READ-0 UNIX_SHM_BASE+3 123 + ** READ-1 UNIX_SHM_BASE+4 124 + ** READ-2 UNIX_SHM_BASE+5 125 + ** READ-3 UNIX_SHM_BASE+6 126 + ** READ-4 UNIX_SHM_BASE+7 127 + ** DMS UNIX_SHM_BASE+8 128 + */ + assert( UNIX_SHM_DMS==128 ); /* Byte offset of the deadman-switch */ +#endif + + /* Initialize temp file dir array. */ + unixTempFileInit(); + + return SQLITE_OK; } /* @@ -36680,10 +46731,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ ** to release dynamically allocated objects. But not on unix. ** This routine is a no-op for unix. */ -SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ - return SQLITE_OK; +SQLITE_API int sqlite3_os_end(void){ + unixBigLock = 0; + return SQLITE_OK; } - + #endif /* SQLITE_OS_UNIX */ /************** End of os_unix.c *********************************************/ @@ -36708,205 +46760,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ /* ** Include code that is common to all os_*.c files */ -/************** Include os_common.h in the middle of os_win.c ****************/ -/************** Begin file os_common.h ***************************************/ -/* -** 2004 May 22 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains macros and a little bit of code that is common to -** all of the platform-specific files (os_*.c) and is #included into those -** files. -** -** This file should be #included by the os_*.c files only. It is not a -** general purpose header file. -*/ -#ifndef _OS_COMMON_H_ -#define _OS_COMMON_H_ - -/* -** At least two bugs have slipped in because we changed the MEMORY_DEBUG -** macro to SQLITE_DEBUG and some older makefiles have not yet made the -** switch. The following code should catch this problem at compile-time. -*/ -#ifdef MEMORY_DEBUG -# error "The MEMORY_DEBUG macro is obsolete. Use SQLITE_DEBUG instead." -#endif - -/* -** Macros for performance tracing. Normally turned off. Only works -** on i486 hardware. -*/ -#ifdef SQLITE_PERFORMANCE_TRACE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of os_common.h ****************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - #error Need implementation of sqlite3Hwtime() for your platform. - - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(_HWTIME_H_) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in os_common.h ******************/ - -static sqlite_uint64 g_start; -static sqlite_uint64 g_elapsed; -#define TIMER_START g_start=sqlite3Hwtime() -#define TIMER_END g_elapsed=sqlite3Hwtime()-g_start -#define TIMER_ELAPSED g_elapsed -#else -#define TIMER_START -#define TIMER_END -#define TIMER_ELAPSED ((sqlite_uint64)0) -#endif - -/* -** If we compile with the SQLITE_TEST macro set, then the following block -** of code will give us the ability to simulate a disk I/O error. This -** is used for testing the I/O recovery logic. -*/ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_io_error_hit; -SQLITE_API extern int sqlite3_io_error_hardhit; -SQLITE_API extern int sqlite3_io_error_pending; -SQLITE_API extern int sqlite3_io_error_persist; -SQLITE_API extern int sqlite3_io_error_benign; -SQLITE_API extern int sqlite3_diskfull_pending; -SQLITE_API extern int sqlite3_diskfull; -#define SimulateIOErrorBenign(X) sqlite3_io_error_benign=(X) -#define SimulateIOError(CODE) \ - if( (sqlite3_io_error_persist && sqlite3_io_error_hit) \ - || sqlite3_io_error_pending-- == 1 ) \ - { local_ioerr(); CODE; } -static void local_ioerr(){ - IOTRACE(("IOERR\n")); - sqlite3_io_error_hit++; - if( !sqlite3_io_error_benign ) sqlite3_io_error_hardhit++; -} -#define SimulateDiskfullError(CODE) \ - if( sqlite3_diskfull_pending ){ \ - if( sqlite3_diskfull_pending == 1 ){ \ - local_ioerr(); \ - sqlite3_diskfull = 1; \ - sqlite3_io_error_hit = 1; \ - CODE; \ - }else{ \ - sqlite3_diskfull_pending--; \ - } \ - } -#else -#define SimulateIOErrorBenign(X) -#define SimulateIOError(A) -#define SimulateDiskfullError(A) -#endif /* defined(SQLITE_TEST) */ - -/* -** When testing, keep a count of the number of open files. -*/ -#if defined(SQLITE_TEST) -SQLITE_API extern int sqlite3_open_file_count; -#define OpenCounter(X) sqlite3_open_file_count+=(X) -#else -#define OpenCounter(X) -#endif /* defined(SQLITE_TEST) */ - -#endif /* !defined(_OS_COMMON_H_) */ - -/************** End of os_common.h *******************************************/ -/************** Continuing where we left off in os_win.c *********************/ +/* #include "os_common.h" */ /* ** Include the header file for the Windows VFS. @@ -37172,12 +47026,22 @@ struct winFile { int nFetchOut; /* Number of outstanding xFetch references */ HANDLE hMap; /* Handle for accessing memory mapping */ void *pMapRegion; /* Area memory mapped */ - sqlite3_int64 mmapSize; /* Usable size of mapped region */ - sqlite3_int64 mmapSizeActual; /* Actual size of mapped region */ + sqlite3_int64 mmapSize; /* Size of mapped region */ sqlite3_int64 mmapSizeMax; /* Configured FCNTL_MMAP_SIZE value */ #endif }; +/* +** The winVfsAppData structure is used for the pAppData member for all of the +** Win32 VFS variants. +*/ +typedef struct winVfsAppData winVfsAppData; +struct winVfsAppData { + const sqlite3_io_methods *pMethod; /* The file I/O methods to use. */ + void *pAppData; /* The extra pAppData, if any. */ + BOOL bNoLock; /* Non-zero if locking is disabled. */ +}; + /* ** Allowed values for winFile.ctrlFlags */ @@ -37192,22 +47056,6 @@ struct winFile { # define SQLITE_WIN32_DBG_BUF_SIZE ((int)(4096-sizeof(DWORD))) #endif -/* - * The value used with sqlite3_win32_set_directory() to specify that - * the data directory should be changed. - */ -#ifndef SQLITE_WIN32_DATA_DIRECTORY_TYPE -# define SQLITE_WIN32_DATA_DIRECTORY_TYPE (1) -#endif - -/* - * The value used with sqlite3_win32_set_directory() to specify that - * the temporary directory should be changed. - */ -#ifndef SQLITE_WIN32_TEMP_DIRECTORY_TYPE -# define SQLITE_WIN32_TEMP_DIRECTORY_TYPE (2) -#endif - /* * If compiled with SQLITE_WIN32_MALLOC on Windows, we will use the * various Win32 API heap functions instead of our own. @@ -37229,7 +47077,34 @@ struct winFile { ****************************************************************************** */ #ifndef SQLITE_WIN32_HEAP_CREATE -# define SQLITE_WIN32_HEAP_CREATE (TRUE) +# define SQLITE_WIN32_HEAP_CREATE (TRUE) +#endif + +/* + * This is the maximum possible initial size of the Win32-specific heap, in + * bytes. + */ +#ifndef SQLITE_WIN32_HEAP_MAX_INIT_SIZE +# define SQLITE_WIN32_HEAP_MAX_INIT_SIZE (4294967295U) +#endif + +/* + * This is the extra space for the initial size of the Win32-specific heap, + * in bytes. This value may be zero. + */ +#ifndef SQLITE_WIN32_HEAP_INIT_EXTRA +# define SQLITE_WIN32_HEAP_INIT_EXTRA (4194304) +#endif + +/* + * Calculate the maximum legal cache size, in pages, based on the maximum + * possible initial heap size and the default page size, setting aside the + * needed extra space. + */ +#ifndef SQLITE_WIN32_MAX_CACHE_SIZE +# define SQLITE_WIN32_MAX_CACHE_SIZE (((SQLITE_WIN32_HEAP_MAX_INIT_SIZE) - \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) / \ + (SQLITE_DEFAULT_PAGE_SIZE)) #endif /* @@ -37238,25 +47113,36 @@ struct winFile { */ #ifndef SQLITE_WIN32_CACHE_SIZE # if SQLITE_DEFAULT_CACHE_SIZE>=0 -# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) +# define SQLITE_WIN32_CACHE_SIZE (SQLITE_DEFAULT_CACHE_SIZE) # else -# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) +# define SQLITE_WIN32_CACHE_SIZE (-(SQLITE_DEFAULT_CACHE_SIZE)) # endif #endif +/* + * Make sure that the calculated cache size, in pages, cannot cause the + * initial size of the Win32-specific heap to exceed the maximum amount + * of memory that can be specified in the call to HeapCreate. + */ +#if SQLITE_WIN32_CACHE_SIZE>SQLITE_WIN32_MAX_CACHE_SIZE +# undef SQLITE_WIN32_CACHE_SIZE +# define SQLITE_WIN32_CACHE_SIZE (2000) +#endif + /* * The initial size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_INIT_SIZE -# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ - (SQLITE_DEFAULT_PAGE_SIZE) + 4194304) +# define SQLITE_WIN32_HEAP_INIT_SIZE ((SQLITE_WIN32_CACHE_SIZE) * \ + (SQLITE_DEFAULT_PAGE_SIZE) + \ + (SQLITE_WIN32_HEAP_INIT_EXTRA)) #endif /* * The maximum size of the Win32-specific heap. This value may be zero. */ #ifndef SQLITE_WIN32_HEAP_MAX_SIZE -# define SQLITE_WIN32_HEAP_MAX_SIZE (0) +# define SQLITE_WIN32_HEAP_MAX_SIZE (0) #endif /* @@ -37264,7 +47150,7 @@ struct winFile { * zero for the default behavior. */ #ifndef SQLITE_WIN32_HEAP_FLAGS -# define SQLITE_WIN32_HEAP_FLAGS (0) +# define SQLITE_WIN32_HEAP_FLAGS (0) #endif @@ -38015,7 +47901,7 @@ static struct win_syscall { /* ** This is the xSetSystemCall() method of sqlite3_vfs for all of the -** "win32" VFSes. Return SQLITE_OK opon successfully updating the +** "win32" VFSes. Return SQLITE_OK upon successfully updating the ** system call pointer, or SQLITE_NOTFOUND if there is no configurable ** system call named zName. */ @@ -38104,7 +47990,7 @@ static const char *winNextSystemCall(sqlite3_vfs *p, const char *zName){ ** "pnLargest" argument, if non-zero, will be used to return the size of the ** largest committed free block in the heap, in bytes. */ -SQLITE_API int SQLITE_STDCALL sqlite3_win32_compact_heap(LPUINT pnLargest){ +SQLITE_API int sqlite3_win32_compact_heap(LPUINT pnLargest){ int rc = SQLITE_OK; UINT nLargest = 0; HANDLE hHeap; @@ -38144,19 +48030,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_compact_heap(LPUINT pnLargest){ ** the sqlite3_memory_used() function does not return zero, SQLITE_BUSY will ** be returned and no changes will be made to the Win32 native heap. */ -SQLITE_API int SQLITE_STDCALL sqlite3_win32_reset_heap(){ +SQLITE_API int sqlite3_win32_reset_heap(){ int rc; - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) /* The main static mutex */ MUTEX_LOGIC( sqlite3_mutex *pMem; ) /* The memsys static mutex */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) MUTEX_LOGIC( pMem = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MEM); ) - sqlite3_mutex_enter(pMaster); + sqlite3_mutex_enter(pMainMtx); sqlite3_mutex_enter(pMem); winMemAssertMagic(); if( winMemGetHeap()!=NULL && winMemGetOwned() && sqlite3_memory_used()==0 ){ /* ** At this point, there should be no outstanding memory allocations on - ** the heap. Also, since both the master and memsys locks are currently + ** the heap. Also, since both the main and memsys locks are currently ** being held by us, no other function (i.e. from another thread) should ** be able to even access the heap. Attempt to destroy and recreate our ** isolated Win32 native heap now. @@ -38179,7 +48065,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_reset_heap(){ rc = SQLITE_BUSY; } sqlite3_mutex_leave(pMem); - sqlite3_mutex_leave(pMaster); + sqlite3_mutex_leave(pMainMtx); return rc; } #endif /* SQLITE_WIN32_MALLOC */ @@ -38189,7 +48075,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_reset_heap(){ ** (if available). */ -SQLITE_API void SQLITE_STDCALL sqlite3_win32_write_debug(const char *zBuf, int nBuf){ +SQLITE_API void sqlite3_win32_write_debug(const char *zBuf, int nBuf){ char zDbgBuf[SQLITE_WIN32_DBG_BUF_SIZE]; int nMin = MIN(nBuf, (SQLITE_WIN32_DBG_BUF_SIZE - 1)); /* may be negative. */ if( nMin<-1 ) nMin = -1; /* all negative values become -1. */ @@ -38235,7 +48121,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_win32_write_debug(const char *zBuf, int n static HANDLE sleepObj = NULL; #endif -SQLITE_API void SQLITE_STDCALL sqlite3_win32_sleep(DWORD milliseconds){ +SQLITE_API void sqlite3_win32_sleep(DWORD milliseconds){ #if SQLITE_OS_WINRT if ( sleepObj==NULL ){ sleepObj = osCreateEventExW(NULL, NULL, CREATE_EVENT_MANUAL_RESET, @@ -38284,7 +48170,7 @@ SQLITE_PRIVATE DWORD sqlite3Win32Wait(HANDLE hObject){ ** This function determines if the machine is running a version of Windows ** based on the NT kernel. */ -SQLITE_API int SQLITE_STDCALL sqlite3_win32_is_nt(void){ +SQLITE_API int sqlite3_win32_is_nt(void){ #if SQLITE_OS_WINRT /* ** NOTE: The WinRT sub-platform is always assumed to be based on the NT @@ -38672,7 +48558,7 @@ static char *winUtf8ToMbcs(const char *zText, int useAnsi){ /* ** This is a public wrapper for the winUtf8ToUnicode() function. */ -SQLITE_API LPWSTR SQLITE_STDCALL sqlite3_win32_utf8_to_unicode(const char *zText){ +SQLITE_API LPWSTR sqlite3_win32_utf8_to_unicode(const char *zText){ #ifdef SQLITE_ENABLE_API_ARMOR if( !zText ){ (void)SQLITE_MISUSE_BKPT; @@ -38688,7 +48574,7 @@ SQLITE_API LPWSTR SQLITE_STDCALL sqlite3_win32_utf8_to_unicode(const char *zText /* ** This is a public wrapper for the winUnicodeToUtf8() function. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ +SQLITE_API char *sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText){ #ifdef SQLITE_ENABLE_API_ARMOR if( !zWideText ){ (void)SQLITE_MISUSE_BKPT; @@ -38704,7 +48590,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_win32_unicode_to_utf8(LPCWSTR zWideText) /* ** This is a public wrapper for the winMbcsToUtf8() function. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_win32_mbcs_to_utf8(const char *zText){ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8(const char *zText){ #ifdef SQLITE_ENABLE_API_ARMOR if( !zText ){ (void)SQLITE_MISUSE_BKPT; @@ -38720,7 +48606,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_win32_mbcs_to_utf8(const char *zText){ /* ** This is a public wrapper for the winMbcsToUtf8() function. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ +SQLITE_API char *sqlite3_win32_mbcs_to_utf8_v2(const char *zText, int useAnsi){ #ifdef SQLITE_ENABLE_API_ARMOR if( !zText ){ (void)SQLITE_MISUSE_BKPT; @@ -38736,7 +48622,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_win32_mbcs_to_utf8_v2(const char *zText, /* ** This is a public wrapper for the winUtf8ToMbcs() function. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_win32_utf8_to_mbcs(const char *zText){ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs(const char *zText){ #ifdef SQLITE_ENABLE_API_ARMOR if( !zText ){ (void)SQLITE_MISUSE_BKPT; @@ -38752,7 +48638,7 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_win32_utf8_to_mbcs(const char *zText){ /* ** This is a public wrapper for the winUtf8ToMbcs() function. */ -SQLITE_API char *SQLITE_STDCALL sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ +SQLITE_API char *sqlite3_win32_utf8_to_mbcs_v2(const char *zText, int useAnsi){ #ifdef SQLITE_ENABLE_API_ARMOR if( !zText ){ (void)SQLITE_MISUSE_BKPT; @@ -38766,18 +48652,20 @@ SQLITE_API char *SQLITE_STDCALL sqlite3_win32_utf8_to_mbcs_v2(const char *zText, } /* -** This function sets the data directory or the temporary directory based on -** the provided arguments. The type argument must be 1 in order to set the -** data directory or 2 in order to set the temporary directory. The zValue -** argument is the name of the directory to use. The return value will be -** SQLITE_OK if successful. +** This function is the same as sqlite3_win32_set_directory (below); however, +** it accepts a UTF-8 string. */ -SQLITE_API int SQLITE_STDCALL sqlite3_win32_set_directory(DWORD type, LPCWSTR zValue){ +SQLITE_API int sqlite3_win32_set_directory8( + unsigned long type, /* Identifier for directory being set or reset */ + const char *zValue /* New value for directory being set or reset */ +){ char **ppDirectory = 0; + int rc; #ifndef SQLITE_OMIT_AUTOINIT - int rc = sqlite3_initialize(); + rc = sqlite3_initialize(); if( rc ) return rc; #endif + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( type==SQLITE_WIN32_DATA_DIRECTORY_TYPE ){ ppDirectory = &sqlite3_data_directory; }else if( type==SQLITE_WIN32_TEMP_DIRECTORY_TYPE ){ @@ -38788,18 +48676,56 @@ SQLITE_API int SQLITE_STDCALL sqlite3_win32_set_directory(DWORD type, LPCWSTR zV ); assert( !ppDirectory || sqlite3MemdebugHasType(*ppDirectory, MEMTYPE_HEAP) ); if( ppDirectory ){ - char *zValueUtf8 = 0; + char *zCopy = 0; if( zValue && zValue[0] ){ - zValueUtf8 = winUnicodeToUtf8(zValue); - if ( zValueUtf8==0 ){ - return SQLITE_NOMEM_BKPT; + zCopy = sqlite3_mprintf("%s", zValue); + if ( zCopy==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto set_directory8_done; } } sqlite3_free(*ppDirectory); - *ppDirectory = zValueUtf8; - return SQLITE_OK; + *ppDirectory = zCopy; + rc = SQLITE_OK; + }else{ + rc = SQLITE_ERROR; } - return SQLITE_ERROR; +set_directory8_done: + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return rc; +} + +/* +** This function is the same as sqlite3_win32_set_directory (below); however, +** it accepts a UTF-16 string. +*/ +SQLITE_API int sqlite3_win32_set_directory16( + unsigned long type, /* Identifier for directory being set or reset */ + const void *zValue /* New value for directory being set or reset */ +){ + int rc; + char *zUtf8 = 0; + if( zValue ){ + zUtf8 = sqlite3_win32_unicode_to_utf8(zValue); + if( zUtf8==0 ) return SQLITE_NOMEM_BKPT; + } + rc = sqlite3_win32_set_directory8(type, zUtf8); + if( zUtf8 ) sqlite3_free(zUtf8); + return rc; +} + +/* +** This function sets the data directory or the temporary directory based on +** the provided arguments. The type argument must be 1 in order to set the +** data directory or 2 in order to set the temporary directory. The zValue +** argument is the name of the directory to use. The return value will be +** SQLITE_OK if successful. +*/ +SQLITE_API int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +){ + return sqlite3_win32_set_directory16(type, zValue); } /* @@ -39499,7 +49425,12 @@ static int winClose(sqlite3_file *id){ }while( rc==0 && ++cnt < MX_CLOSE_ATTEMPT && (sqlite3_win32_sleep(100), 1) ); #if SQLITE_OS_WINCE #define WINCE_DELETION_ATTEMPTS 3 - winceDestroyLock(pFile); + { + winVfsAppData *pAppData = (winVfsAppData*)pFile->pVfs->pAppData; + if( pAppData==NULL || !pAppData->bNoLock ){ + winceDestroyLock(pFile); + } + } if( pFile->zDeleteOnClose ){ int cnt = 0; while( @@ -39550,7 +49481,7 @@ static int winRead( pFile->h, pBuf, amt, offset, pFile->locktype)); #if SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this read request as possible by transfering + /* Deal with as much of this read request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -39628,7 +49559,7 @@ static int winWrite( pFile->h, pBuf, amt, offset, pFile->locktype)); #if defined(SQLITE_MMAP_READWRITE) && SQLITE_MAX_MMAP_SIZE>0 - /* Deal with as much of this write request as possible by transfering + /* Deal with as much of this write request as possible by transferring ** data from the memory mapping using memcpy(). */ if( offsetmmapSize ){ if( offset+amt <= pFile->mmapSize ){ @@ -39721,6 +49652,29 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ winFile *pFile = (winFile*)id; /* File handle object */ int rc = SQLITE_OK; /* Return code for this function */ DWORD lastErrno; +#if SQLITE_MAX_MMAP_SIZE>0 + sqlite3_int64 oldMmapSize; + if( pFile->nFetchOut>0 ){ + /* File truncation is a no-op if there are outstanding memory mapped + ** pages. This is because truncating the file means temporarily unmapping + ** the file, and that might delete memory out from under existing cursors. + ** + ** This can result in incremental vacuum not truncating the file, + ** if there is an active read cursor when the incremental vacuum occurs. + ** No real harm comes of this - the database file is not corrupted, + ** though some folks might complain that the file is bigger than it + ** needs to be. + ** + ** The only feasible work-around is to defer the truncation until after + ** all references to memory-mapped content are closed. That is doable, + ** but involves adding a few branches in the common write code path which + ** could slow down normal operations slightly. Hence, we have decided for + ** now to simply make transactions a no-op if there are pending reads. We + ** can maybe revisit this decision in the future. + */ + return SQLITE_OK; + } +#endif assert( pFile ); SimulateIOError(return SQLITE_IOERR_TRUNCATE); @@ -39736,6 +49690,15 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ nByte = ((nByte + pFile->szChunk - 1)/pFile->szChunk) * pFile->szChunk; } +#if SQLITE_MAX_MMAP_SIZE>0 + if( pFile->pMapRegion ){ + oldMmapSize = pFile->mmapSize; + }else{ + oldMmapSize = 0; + } + winUnmapfile(pFile); +#endif + /* SetEndOfFile() returns non-zero when successful, or zero when it fails. */ if( winSeekFile(pFile, nByte) ){ rc = winLogError(SQLITE_IOERR_TRUNCATE, pFile->lastErrno, @@ -39748,12 +49711,12 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ } #if SQLITE_MAX_MMAP_SIZE>0 - /* If the file was truncated to a size smaller than the currently - ** mapped region, reduce the effective mapping size as well. SQLite will - ** use read() and write() to access data beyond this point from now on. - */ - if( pFile->pMapRegion && nBytemmapSize ){ - pFile->mmapSize = nByte; + if( rc==SQLITE_OK && oldMmapSize>0 ){ + if( oldMmapSize>nByte ){ + winMapfile(pFile, -1); + }else{ + winMapfile(pFile, oldMmapSize); + } } #endif @@ -39765,7 +49728,7 @@ static int winTruncate(sqlite3_file *id, sqlite3_int64 nByte){ #ifdef SQLITE_TEST /* ** Count the number of fullsyncs and normal syncs. This is used to test -** that syncs and fullsyncs are occuring at the right times. +** that syncs and fullsyncs are occurring at the right times. */ SQLITE_API int sqlite3_sync_count = 0; SQLITE_API int sqlite3_fullsync_count = 0; @@ -40122,7 +50085,7 @@ static int winLock(sqlite3_file *id, int locktype){ */ if( locktype==EXCLUSIVE_LOCK && res ){ assert( pFile->locktype>=SHARED_LOCK ); - res = winUnlockReadLock(pFile); + (void)winUnlockReadLock(pFile); res = winLockFile(&pFile->h, SQLITE_LOCKFILE_FLAGS, SHARED_FIRST, 0, SHARED_SIZE, 0); if( res ){ @@ -40231,6 +50194,44 @@ static int winUnlock(sqlite3_file *id, int locktype){ return rc; } +/****************************************************************************** +****************************** No-op Locking ********************************** +** +** Of the various locking implementations available, this is by far the +** simplest: locking is ignored. No attempt is made to lock the database +** file for reading or writing. +** +** This locking mode is appropriate for use on read-only databases +** (ex: databases that are burned into CD-ROM, for example.) It can +** also be used if the application employs some external mechanism to +** prevent simultaneous access of the same database by two or more +** database connections. But there is a serious risk of database +** corruption if this locking mode is used in situations where multiple +** database connections are accessing the same database file at the same +** time and one or more of those connections are writing. +*/ + +static int winNolockLock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +static int winNolockCheckReservedLock(sqlite3_file *id, int *pResOut){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(pResOut); + return SQLITE_OK; +} + +static int winNolockUnlock(sqlite3_file *id, int locktype){ + UNUSED_PARAMETER(id); + UNUSED_PARAMETER(locktype); + return SQLITE_OK; +} + +/******************* End of the no-op lock implementation ********************* +******************************************************************************/ + /* ** If *pArg is initially negative then this is a query. Set *pArg to ** 1 or 0 depending on whether or not bit mask of pFile->ctrlFlags is set. @@ -40250,6 +50251,7 @@ static void winModeBit(winFile *pFile, unsigned char mask, int *pArg){ /* Forward references to VFS helper methods used for temporary files */ static int winGetTempname(sqlite3_vfs *, char **); static int winIsDir(const void *); +static BOOL winIsLongPathPrefix(const char *); static BOOL winIsDriveLetterAndColon(const char *); /* @@ -40322,6 +50324,12 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); return SQLITE_OK; } + case SQLITE_FCNTL_WIN32_GET_HANDLE: { + LPHANDLE phFile = (LPHANDLE)pArg; + *phFile = pFile->h; + OSTRACE(("FCNTL file=%p, rc=SQLITE_OK\n", pFile->h)); + return SQLITE_OK; + } #ifdef SQLITE_TEST case SQLITE_FCNTL_WIN32_SET_HANDLE: { LPHANDLE phFile = (LPHANDLE)pArg; @@ -40349,6 +50357,14 @@ static int winFileControl(sqlite3_file *id, int op, void *pArg){ if( newLimit>sqlite3GlobalConfig.mxMmap ){ newLimit = sqlite3GlobalConfig.mxMmap; } + + /* The value of newLimit may be eventually cast to (SIZE_T) and passed + ** to MapViewOfFile(). Restrict its value to 2GB if (SIZE_T) is not at + ** least a 64-bit type. */ + if( newLimit>0 && sizeof(SIZE_T)<8 ){ + newLimit = (newLimit & 0x7FFFFFFF); + } + *(i64*)pArg = pFile->mmapSizeMax; if( newLimit>=0 && newLimit!=pFile->mmapSizeMax && pFile->nFetchOut==0 ){ pFile->mmapSizeMax = newLimit; @@ -40386,7 +50402,7 @@ static int winSectorSize(sqlite3_file *id){ */ static int winDeviceCharacteristics(sqlite3_file *id){ winFile *p = (winFile*)id; - return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | + return SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN | SQLITE_IOCAP_SUBPAGE_READ | ((p->ctrlFlags & WINFILE_PSOW)?SQLITE_IOCAP_POWERSAFE_OVERWRITE:0); } @@ -40413,15 +50429,16 @@ static SYSTEM_INFO winSysInfo; ** assert( winShmMutexHeld() ); ** winShmLeaveMutex() */ +static sqlite3_mutex *winBigLock = 0; static void winShmEnterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_enter(winBigLock); } static void winShmLeaveMutex(void){ - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + sqlite3_mutex_leave(winBigLock); } #ifndef NDEBUG static int winShmMutexHeld(void) { - return sqlite3_mutex_held(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1)); + return sqlite3_mutex_held(winBigLock); } #endif @@ -40455,6 +50472,9 @@ struct winShmNode { int szRegion; /* Size of shared-memory regions */ int nRegion; /* Size of array apRegion */ + u8 isReadonly; /* True if read-only */ + u8 isUnlocked; /* True if no DMS lock held */ + struct ShmRegion { HANDLE hMap; /* File handle from CreateFileMapping */ void *pMap; @@ -40509,30 +50529,30 @@ struct winShm { /* ** Apply advisory locks for all n bytes beginning at ofst. */ -#define _SHM_UNLCK 1 -#define _SHM_RDLCK 2 -#define _SHM_WRLCK 3 +#define WINSHM_UNLCK 1 +#define WINSHM_RDLCK 2 +#define WINSHM_WRLCK 3 static int winShmSystemLock( winShmNode *pFile, /* Apply locks to this open shared-memory segment */ - int lockType, /* _SHM_UNLCK, _SHM_RDLCK, or _SHM_WRLCK */ + int lockType, /* WINSHM_UNLCK, WINSHM_RDLCK, or WINSHM_WRLCK */ int ofst, /* Offset to first byte to be locked/unlocked */ int nByte /* Number of bytes to lock or unlock */ ){ int rc = 0; /* Result code form Lock/UnlockFileEx() */ /* Access to the winShmNode object is serialized by the caller */ - assert( sqlite3_mutex_held(pFile->mutex) || pFile->nRef==0 ); + assert( pFile->nRef==0 || sqlite3_mutex_held(pFile->mutex) ); OSTRACE(("SHM-LOCK file=%p, lock=%d, offset=%d, size=%d\n", pFile->hFile.h, lockType, ofst, nByte)); /* Release/Acquire the system-level lock */ - if( lockType==_SHM_UNLCK ){ + if( lockType==WINSHM_UNLCK ){ rc = winUnlockFile(&pFile->hFile.h, ofst, 0, nByte, 0); }else{ /* Initialize the locking parameters */ DWORD dwFlags = LOCKFILE_FAIL_IMMEDIATELY; - if( lockType == _SHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; + if( lockType == WINSHM_WRLCK ) dwFlags |= LOCKFILE_EXCLUSIVE_LOCK; rc = winLockFile(&pFile->hFile.h, dwFlags, ofst, 0, nByte, 0); } @@ -40544,7 +50564,7 @@ static int winShmSystemLock( } OSTRACE(("SHM-LOCK file=%p, func=%s, errno=%lu, rc=%s\n", - pFile->hFile.h, (lockType == _SHM_UNLCK) ? "winUnlockFile" : + pFile->hFile.h, (lockType == WINSHM_UNLCK) ? "winUnlockFile" : "winLockFile", pFile->lastErrno, sqlite3ErrName(rc))); return rc; @@ -40602,6 +50622,37 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ } } +/* +** The DMS lock has not yet been taken on shm file pShmNode. Attempt to +** take it now. Return SQLITE_OK if successful, or an SQLite error +** code otherwise. +** +** If the DMS cannot be locked because this is a readonly_shm=1 +** connection and no other process already holds a lock, return +** SQLITE_READONLY_CANTINIT and set pShmNode->isUnlocked=1. +*/ +static int winLockSharedMemory(winShmNode *pShmNode){ + int rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, WIN_SHM_DMS, 1); + + if( rc==SQLITE_OK ){ + if( pShmNode->isReadonly ){ + pShmNode->isUnlocked = 1; + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return SQLITE_READONLY_CANTINIT; + }else if( winTruncate((sqlite3_file*)&pShmNode->hFile, 0) ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + return winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), + "winLockSharedMemory", pShmNode->zFilename); + } + } + + if( rc==SQLITE_OK ){ + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); + } + + return winShmSystemLock(pShmNode, WINSHM_RDLCK, WIN_SHM_DMS, 1); +} + /* ** Open the shared-memory area associated with database file pDbFd. ** @@ -40611,9 +50662,9 @@ static void winShmPurge(sqlite3_vfs *pVfs, int deleteFlag){ */ static int winOpenSharedMemory(winFile *pDbFd){ struct winShm *p; /* The connection to be opened */ - struct winShmNode *pShmNode = 0; /* The underlying mmapped file */ - int rc; /* Result code */ - struct winShmNode *pNew; /* Newly allocated winShmNode */ + winShmNode *pShmNode = 0; /* The underlying mmapped file */ + int rc = SQLITE_OK; /* Result code */ + winShmNode *pNew; /* Newly allocated winShmNode */ int nName; /* Size of zName in bytes */ assert( pDbFd->pShm==0 ); /* Not previously opened */ @@ -40646,6 +50697,9 @@ static int winOpenSharedMemory(winFile *pDbFd){ if( pShmNode ){ sqlite3_free(pNew); }else{ + int inFlags = SQLITE_OPEN_WAL; + int outFlags = 0; + pShmNode = pNew; pNew = 0; ((winFile*)(&pShmNode->hFile))->h = INVALID_HANDLE_VALUE; @@ -40660,30 +50714,23 @@ static int winOpenSharedMemory(winFile *pDbFd){ } } - rc = winOpen(pDbFd->pVfs, - pShmNode->zFilename, /* Name of the file (UTF-8) */ - (sqlite3_file*)&pShmNode->hFile, /* File handle here */ - SQLITE_OPEN_WAL | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - 0); - if( SQLITE_OK!=rc ){ + if( 0==sqlite3_uri_boolean(pDbFd->zPath, "readonly_shm", 0) ){ + inFlags |= SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE; + }else{ + inFlags |= SQLITE_OPEN_READONLY; + } + rc = winOpen(pDbFd->pVfs, pShmNode->zFilename, + (sqlite3_file*)&pShmNode->hFile, + inFlags, &outFlags); + if( rc!=SQLITE_OK ){ + rc = winLogError(rc, osGetLastError(), "winOpenShm", + pShmNode->zFilename); goto shm_open_err; } + if( outFlags==SQLITE_OPEN_READONLY ) pShmNode->isReadonly = 1; - /* Check to see if another process is holding the dead-man switch. - ** If not, truncate the file to zero length. - */ - if( winShmSystemLock(pShmNode, _SHM_WRLCK, WIN_SHM_DMS, 1)==SQLITE_OK ){ - rc = winTruncate((sqlite3_file *)&pShmNode->hFile, 0); - if( rc!=SQLITE_OK ){ - rc = winLogError(SQLITE_IOERR_SHMOPEN, osGetLastError(), - "winOpenShm", pDbFd->zPath); - } - } - if( rc==SQLITE_OK ){ - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, WIN_SHM_DMS, 1); - } - if( rc ) goto shm_open_err; + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK && rc!=SQLITE_READONLY_CANTINIT ) goto shm_open_err; } /* Make the new connection a child of the winShmNode */ @@ -40706,11 +50753,11 @@ static int winOpenSharedMemory(winFile *pDbFd){ p->pNext = pShmNode->pFirst; pShmNode->pFirst = p; sqlite3_mutex_leave(pShmNode->mutex); - return SQLITE_OK; + return rc; /* Jump here on any error */ shm_open_err: - winShmSystemLock(pShmNode, _SHM_UNLCK, WIN_SHM_DMS, 1); + winShmSystemLock(pShmNode, WINSHM_UNLCK, WIN_SHM_DMS, 1); winShmPurge(pDbFd->pVfs, 0); /* This call frees pShmNode if required */ sqlite3_free(p); sqlite3_free(pNew); @@ -40772,10 +50819,14 @@ static int winShmLock( winFile *pDbFd = (winFile*)fd; /* Connection holding shared memory */ winShm *p = pDbFd->pShm; /* The shared memory being locked */ winShm *pX; /* For looping over all siblings */ - winShmNode *pShmNode = p->pShmNode; + winShmNode *pShmNode; int rc = SQLITE_OK; /* Result code */ u16 mask; /* Mask of locks to take or release */ + if( p==0 ) return SQLITE_IOERR_SHMLOCK; + pShmNode = p->pShmNode; + if( NEVER(pShmNode==0) ) return SQLITE_IOERR_SHMLOCK; + assert( ofst>=0 && ofst+n<=SQLITE_SHM_NLOCK ); assert( n>=1 ); assert( flags==(SQLITE_SHM_LOCK | SQLITE_SHM_SHARED) @@ -40799,7 +50850,7 @@ static int winShmLock( /* Unlock the system-level locks */ if( (mask & allMask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_UNLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_UNLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -40827,7 +50878,7 @@ static int winShmLock( /* Get shared locks at the system level, if necessary */ if( rc==SQLITE_OK ){ if( (allShared & mask)==0 ){ - rc = winShmSystemLock(pShmNode, _SHM_RDLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_RDLCK, ofst+WIN_SHM_BASE, n); }else{ rc = SQLITE_OK; } @@ -40852,7 +50903,7 @@ static int winShmLock( ** also mark the local connection as being locked. */ if( rc==SQLITE_OK ){ - rc = winShmSystemLock(pShmNode, _SHM_WRLCK, ofst+WIN_SHM_BASE, n); + rc = winShmSystemLock(pShmNode, WINSHM_WRLCK, ofst+WIN_SHM_BASE, n); if( rc==SQLITE_OK ){ assert( (p->sharedMask & mask)==0 ); p->exclMask |= mask; @@ -40910,16 +50961,24 @@ static int winShmMap( winFile *pDbFd = (winFile*)fd; winShm *pShm = pDbFd->pShm; winShmNode *pShmNode; + DWORD protect = PAGE_READWRITE; + DWORD flags = FILE_MAP_WRITE | FILE_MAP_READ; int rc = SQLITE_OK; if( !pShm ){ rc = winOpenSharedMemory(pDbFd); if( rc!=SQLITE_OK ) return rc; pShm = pDbFd->pShm; + assert( pShm!=0 ); } pShmNode = pShm->pShmNode; sqlite3_mutex_enter(pShmNode->mutex); + if( pShmNode->isUnlocked ){ + rc = winLockSharedMemory(pShmNode); + if( rc!=SQLITE_OK ) goto shmpage_out; + pShmNode->isUnlocked = 0; + } assert( szRegion==pShmNode->szRegion || pShmNode->nRegion==0 ); if( pShmNode->nRegion<=iRegion ){ @@ -40966,21 +51025,26 @@ static int winShmMap( } pShmNode->aRegion = apNew; + if( pShmNode->isReadonly ){ + protect = PAGE_READONLY; + flags = FILE_MAP_READ; + } + while( pShmNode->nRegion<=iRegion ){ HANDLE hMap = NULL; /* file-mapping handle */ void *pMap = 0; /* Mapped memory region */ #if SQLITE_OS_WINRT hMap = osCreateFileMappingFromApp(pShmNode->hFile.h, - NULL, PAGE_READWRITE, nByte, NULL + NULL, protect, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_WIDE) hMap = osCreateFileMappingW(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #elif defined(SQLITE_WIN32_HAS_ANSI) && SQLITE_WIN32_CREATEFILEMAPPINGA hMap = osCreateFileMappingA(pShmNode->hFile.h, - NULL, PAGE_READWRITE, 0, nByte, NULL + NULL, protect, 0, nByte, NULL ); #endif OSTRACE(("SHM-MAP-CREATE pid=%lu, region=%d, size=%d, rc=%s\n", @@ -40990,11 +51054,11 @@ static int winShmMap( int iOffset = pShmNode->nRegion*szRegion; int iOffsetShift = iOffset % winSysInfo.dwAllocationGranularity; #if SQLITE_OS_WINRT - pMap = osMapViewOfFileFromApp(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFileFromApp(hMap, flags, iOffset - iOffsetShift, szRegion + iOffsetShift ); #else - pMap = osMapViewOfFile(hMap, FILE_MAP_WRITE | FILE_MAP_READ, + pMap = osMapViewOfFile(hMap, flags, 0, iOffset - iOffsetShift, szRegion + iOffsetShift ); #endif @@ -41025,6 +51089,7 @@ shmpage_out: }else{ *pp = 0; } + if( pShmNode->isReadonly && rc==SQLITE_OK ) rc = SQLITE_READONLY; sqlite3_mutex_leave(pShmNode->mutex); return rc; } @@ -41043,9 +51108,9 @@ shmpage_out: static int winUnmapfile(winFile *pFile){ assert( pFile!=0 ); OSTRACE(("UNMAP-FILE pid=%lu, pFile=%p, hMap=%p, pMapRegion=%p, " - "mmapSize=%lld, mmapSizeActual=%lld, mmapSizeMax=%lld\n", + "mmapSize=%lld, mmapSizeMax=%lld\n", osGetCurrentProcessId(), pFile, pFile->hMap, pFile->pMapRegion, - pFile->mmapSize, pFile->mmapSizeActual, pFile->mmapSizeMax)); + pFile->mmapSize, pFile->mmapSizeMax)); if( pFile->pMapRegion ){ if( !osUnmapViewOfFile(pFile->pMapRegion) ){ pFile->lastErrno = osGetLastError(); @@ -41057,7 +51122,6 @@ static int winUnmapfile(winFile *pFile){ } pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapSizeActual = 0; } if( pFile->hMap!=NULL ){ if( !osCloseHandle(pFile->hMap) ){ @@ -41168,7 +51232,6 @@ static int winMapfile(winFile *pFd, sqlite3_int64 nByte){ } pFd->pMapRegion = pNew; pFd->mmapSize = nMap; - pFd->mmapSizeActual = nMap; } OSTRACE(("MAP-FILE pid=%lu, pFile=%p, rc=SQLITE_OK\n", @@ -41200,6 +51263,11 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ #if SQLITE_MAX_MMAP_SIZE>0 if( pFd->mmapSizeMax>0 ){ + /* Ensure that there is always at least a 256 byte buffer of addressable + ** memory following the returned page. If the database is corrupt, + ** SQLite may overread the page slightly (in practice only a few bytes, + ** but 256 is safe, round, number). */ + const int nEofBuffer = 256; if( pFd->pMapRegion==0 ){ int rc = winMapfile(pFd, -1); if( rc!=SQLITE_OK ){ @@ -41208,7 +51276,8 @@ static int winFetch(sqlite3_file *fd, i64 iOff, int nAmt, void **pp){ return rc; } } - if( pFd->mmapSize >= iOff+nAmt ){ + if( pFd->mmapSize >= (iOff+nAmt+nEofBuffer) ){ + assert( pFd->pMapRegion!=0 ); *pp = &((u8 *)pFd->pMapRegion)[iOff]; pFd->nFetchOut++; } @@ -41295,6 +51364,44 @@ static const sqlite3_io_methods winIoMethod = { winUnfetch /* xUnfetch */ }; +/* +** This vector defines all the methods that can operate on an +** sqlite3_file for win32 without performing any locking. +*/ +static const sqlite3_io_methods winIoNolockMethod = { + 3, /* iVersion */ + winClose, /* xClose */ + winRead, /* xRead */ + winWrite, /* xWrite */ + winTruncate, /* xTruncate */ + winSync, /* xSync */ + winFileSize, /* xFileSize */ + winNolockLock, /* xLock */ + winNolockUnlock, /* xUnlock */ + winNolockCheckReservedLock, /* xCheckReservedLock */ + winFileControl, /* xFileControl */ + winSectorSize, /* xSectorSize */ + winDeviceCharacteristics, /* xDeviceCharacteristics */ + winShmMap, /* xShmMap */ + winShmLock, /* xShmLock */ + winShmBarrier, /* xShmBarrier */ + winShmUnmap, /* xShmUnmap */ + winFetch, /* xFetch */ + winUnfetch /* xUnfetch */ +}; + +static winVfsAppData winAppData = { + &winIoMethod, /* pMethod */ + 0, /* pAppData */ + 0 /* bNoLock */ +}; + +static winVfsAppData winNolockAppData = { + &winIoNolockMethod, /* pMethod */ + 0, /* pAppData */ + 1 /* bNoLock */ +}; + /**************************************************************************** **************************** sqlite3_vfs methods **************************** ** @@ -41364,6 +51471,19 @@ static int winMakeEndInDirSep(int nBuf, char *zBuf){ return 0; } +/* +** If sqlite3_temp_directory is defined, take the mutex and return true. +** +** If sqlite3_temp_directory is NULL (undefined), omit the mutex and +** return false. +*/ +static int winTempDirDefined(void){ + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + if( sqlite3_temp_directory!=0 ) return 1; + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); + return 0; +} + /* ** Create a temporary file name and store the resulting pointer into pzBuf. ** The pointer returned in pzBuf must be freed via sqlite3_free(). @@ -41374,6 +51494,7 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789"; size_t i, j; + DWORD pid; int nPre = sqlite3Strlen30(SQLITE_TEMP_FILE_PREFIX); int nMax, nBuf, nDir, nLen; char *zBuf; @@ -41400,20 +51521,23 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ */ nDir = nMax - (nPre + 15); assert( nDir>0 ); - if( sqlite3_temp_directory ){ + if( winTempDirDefined() ){ int nDirLen = sqlite3Strlen30(sqlite3_temp_directory); if( nDirLen>0 ){ if( !winIsDirSep(sqlite3_temp_directory[nDirLen-1]) ){ nDirLen++; } if( nDirLen>nDir ){ + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); sqlite3_free(zBuf); OSTRACE(("TEMP-FILENAME rc=SQLITE_ERROR\n")); return winLogError(SQLITE_ERROR, 0, "winGetTempname1", 0); } sqlite3_snprintf(nMax, zBuf, "%s", sqlite3_temp_directory); } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); } + #if defined(__CYGWIN__) else{ static const char *azDirs[] = { @@ -41583,7 +51707,10 @@ static int winGetTempname(sqlite3_vfs *pVfs, char **pzBuf){ j = sqlite3Strlen30(zBuf); sqlite3_randomness(15, &zBuf[j]); + pid = osGetCurrentProcessId(); for(i=0; i<15; i++, j++){ + zBuf[j] += pid & 0xff; + pid >>= 8; zBuf[j] = (char)zChars[ ((unsigned char)zBuf[j])%(sizeof(zChars)-1) ]; } zBuf[j] = 0; @@ -41623,11 +51750,19 @@ static int winIsDir(const void *zConverted){ return (attr!=INVALID_FILE_ATTRIBUTES) && (attr&FILE_ATTRIBUTE_DIRECTORY); } +/* forward reference */ +static int winAccess( + sqlite3_vfs *pVfs, /* Not used on win32 */ + const char *zFilename, /* Name of file to check */ + int flags, /* Type of test to make on this file */ + int *pResOut /* OUT: Result */ +); + /* ** Open a file. */ static int winOpen( - sqlite3_vfs *pVfs, /* Used to get maximum path name length */ + sqlite3_vfs *pVfs, /* Used to get maximum path length and AppData */ const char *zName, /* Name of the file (UTF-8) */ sqlite3_file *id, /* Write the SQLite file handle here */ int flags, /* Open mode flags */ @@ -41642,6 +51777,7 @@ static int winOpen( #if SQLITE_OS_WINCE int isTemp = 0; #endif + winVfsAppData *pAppData; winFile *pFile = (winFile*)id; void *zConverted; /* Filename in OS encoding */ const char *zUtf8Name = zName; /* Filename in UTF-8 encoding */ @@ -41654,7 +51790,7 @@ static int winOpen( int rc = SQLITE_OK; /* Function Return Code */ #if !defined(NDEBUG) || SQLITE_OS_WINCE - int eType = flags&0xFFFFFF00; /* Type of file to open */ + int eType = flags&0x0FFF00; /* Type of file to open */ #endif int isExclusive = (flags & SQLITE_OPEN_EXCLUSIVE); @@ -41665,7 +51801,7 @@ static int winOpen( #ifndef NDEBUG int isOpenJournal = (isCreate && ( - eType==SQLITE_OPEN_MASTER_JOURNAL + eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_WAL )); @@ -41686,17 +51822,17 @@ static int winOpen( assert(isExclusive==0 || isCreate); assert(isDelete==0 || isCreate); - /* The main DB, main journal, WAL file and master journal are never + /* The main DB, main journal, WAL file and super-journal are never ** automatically deleted. Nor are they ever temporary files. */ assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_DB ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MAIN_JOURNAL ); - assert( (!isDelete && zName) || eType!=SQLITE_OPEN_MASTER_JOURNAL ); + assert( (!isDelete && zName) || eType!=SQLITE_OPEN_SUPER_JOURNAL ); assert( (!isDelete && zName) || eType!=SQLITE_OPEN_WAL ); /* Assert that the upper layer has set one of the "file-type" flags. */ assert( eType==SQLITE_OPEN_MAIN_DB || eType==SQLITE_OPEN_TEMP_DB || eType==SQLITE_OPEN_MAIN_JOURNAL || eType==SQLITE_OPEN_TEMP_JOURNAL - || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_MASTER_JOURNAL + || eType==SQLITE_OPEN_SUBJOURNAL || eType==SQLITE_OPEN_SUPER_JOURNAL || eType==SQLITE_OPEN_TRANSIENT_DB || eType==SQLITE_OPEN_WAL ); @@ -41768,7 +51904,11 @@ static int winOpen( dwCreationDisposition = OPEN_EXISTING; } - dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + if( 0==sqlite3_uri_boolean(zName, "exclusive", 0) ){ + dwShareMode = FILE_SHARE_READ | FILE_SHARE_WRITE; + }else{ + dwShareMode = 0; + } if( isDelete ){ #if SQLITE_OS_WINCE @@ -41798,37 +51938,58 @@ static int winOpen( extendedParameters.dwSecurityQosFlags = SECURITY_ANONYMOUS; extendedParameters.lpSecurityAttributes = NULL; extendedParameters.hTemplateFile = NULL; - while( (h = osCreateFile2((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, - dwCreationDisposition, - &extendedParameters))==INVALID_HANDLE_VALUE && - winRetryIoerr(&cnt, &lastErrno) ){ - /* Noop */ - } + do{ + h = osCreateFile2((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, + dwCreationDisposition, + &extendedParameters); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); #else - while( (h = osCreateFileW((LPCWSTR)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - winRetryIoerr(&cnt, &lastErrno) ){ - /* Noop */ - } + do{ + h = osCreateFileW((LPCWSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); #endif } #ifdef SQLITE_WIN32_HAS_ANSI else{ - while( (h = osCreateFileA((LPCSTR)zConverted, - dwDesiredAccess, - dwShareMode, NULL, - dwCreationDisposition, - dwFlagsAndAttributes, - NULL))==INVALID_HANDLE_VALUE && - winRetryIoerr(&cnt, &lastErrno) ){ - /* Noop */ - } + do{ + h = osCreateFileA((LPCSTR)zConverted, + dwDesiredAccess, + dwShareMode, NULL, + dwCreationDisposition, + dwFlagsAndAttributes, + NULL); + if( h!=INVALID_HANDLE_VALUE ) break; + if( isReadWrite ){ + int rc2, isRO = 0; + sqlite3BeginBenignMalloc(); + rc2 = winAccess(pVfs, zUtf8Name, SQLITE_ACCESS_READ, &isRO); + sqlite3EndBenignMalloc(); + if( rc2==SQLITE_OK && isRO ) break; + } + }while( winRetryIoerr(&cnt, &lastErrno) ); } #endif winLogIoerr(cnt, __LINE__); @@ -41837,8 +51998,6 @@ static int winOpen( dwDesiredAccess, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); if( h==INVALID_HANDLE_VALUE ){ - pFile->lastErrno = lastErrno; - winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); sqlite3_free(zConverted); sqlite3_free(zTmpname); if( isReadWrite && !isExclusive ){ @@ -41847,6 +52006,8 @@ static int winOpen( ~(SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE)), pOutFlags); }else{ + pFile->lastErrno = lastErrno; + winLogError(SQLITE_CANTOPEN, pFile->lastErrno, "winOpen", zUtf8Name); return SQLITE_CANTOPEN_BKPT; } } @@ -41863,15 +52024,20 @@ static int winOpen( "rc=%s\n", h, zUtf8Name, dwDesiredAccess, pOutFlags, pOutFlags ? *pOutFlags : 0, (h==INVALID_HANDLE_VALUE) ? "failed" : "ok")); + pAppData = (winVfsAppData*)pVfs->pAppData; + #if SQLITE_OS_WINCE - if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB - && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK - ){ - osCloseHandle(h); - sqlite3_free(zConverted); - sqlite3_free(zTmpname); - OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); - return rc; + { + if( isReadWrite && eType==SQLITE_OPEN_MAIN_DB + && ((pAppData==NULL) || !pAppData->bNoLock) + && (rc = winceCreateLock(zName, pFile))!=SQLITE_OK + ){ + osCloseHandle(h); + sqlite3_free(zConverted); + sqlite3_free(zTmpname); + OSTRACE(("OPEN-CE-LOCK name=%s, rc=%s\n", zName, sqlite3ErrName(rc))); + return rc; + } } if( isTemp ){ pFile->zDeleteOnClose = zConverted; @@ -41882,13 +52048,15 @@ static int winOpen( } sqlite3_free(zTmpname); - pFile->pMethod = &winIoMethod; + id->pMethods = pAppData ? pAppData->pMethod : &winIoMethod; pFile->pVfs = pVfs; pFile->h = h; if( isReadonly ){ pFile->ctrlFlags |= WINFILE_RDONLY; } - if( sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) ){ + if( (flags & SQLITE_OPEN_MAIN_DB) + && sqlite3_uri_boolean(zName, "psow", SQLITE_POWERSAFE_OVERWRITE) + ){ pFile->ctrlFlags |= WINFILE_PSOW; } pFile->lastErrno = NO_ERROR; @@ -41897,7 +52065,6 @@ static int winOpen( pFile->hMap = NULL; pFile->pMapRegion = 0; pFile->mmapSize = 0; - pFile->mmapSizeActual = 0; pFile->mmapSizeMax = sqlite3GlobalConfig.szMmap; #endif @@ -42041,6 +52208,13 @@ static int winAccess( OSTRACE(("ACCESS name=%s, flags=%x, pResOut=%p\n", zFilename, flags, pResOut)); + if( zFilename==0 ){ + *pResOut = 0; + OSTRACE(("ACCESS name=%s, pResOut=%p, *pResOut=%d, rc=SQLITE_OK\n", + zFilename, pResOut, *pResOut)); + return SQLITE_OK; + } + zConverted = winConvertFromUtf8Filename(zFilename); if( zConverted==0 ){ OSTRACE(("ACCESS name=%s, rc=SQLITE_IOERR_NOMEM\n", zFilename)); @@ -42099,6 +52273,17 @@ static int winAccess( return SQLITE_OK; } +/* +** Returns non-zero if the specified path name starts with the "long path" +** prefix. +*/ +static BOOL winIsLongPathPrefix( + const char *zPathname +){ + return ( zPathname[0]=='\\' && zPathname[1]=='\\' + && zPathname[2]=='?' && zPathname[3]=='\\' ); +} + /* ** Returns non-zero if the specified path name starts with a drive letter ** followed by a colon character. @@ -42151,12 +52336,25 @@ static BOOL winIsVerbatimPathname( ** pathname into zOut[]. zOut[] will be at least pVfs->mxPathname ** bytes in size. */ -static int winFullPathname( +static int winFullPathnameNoMutex( sqlite3_vfs *pVfs, /* Pointer to vfs object */ const char *zRelative, /* Possibly relative input path */ int nFull, /* Size of output buffer in bytes */ char *zFull /* Output buffer */ ){ +#if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) + DWORD nByte; + void *zConverted; + char *zOut; +#endif + + /* If this path name begins with "/X:" or "\\?\", where "X" is any + ** alphabetic character, discard the initial "/" from the pathname. + */ + if( zRelative[0]=='/' && (winIsDriveLetterAndColon(zRelative+1) + || winIsLongPathPrefix(zRelative+1)) ){ + zRelative++; + } #if defined(__CYGWIN__) SimulateIOError( return SQLITE_ERROR ); @@ -42235,17 +52433,6 @@ static int winFullPathname( #endif #if !SQLITE_OS_WINCE && !SQLITE_OS_WINRT && !defined(__CYGWIN__) - DWORD nByte; - void *zConverted; - char *zOut; - - /* If this path name begins with "/X:", where "X" is any alphabetic - ** character, discard the initial "/" from the pathname. - */ - if( zRelative[0]=='/' && winIsDriveLetterAndColon(zRelative+1) ){ - zRelative++; - } - /* It's odd to simulate an io-error here, but really this is just ** using the io-error infrastructure to test that SQLite handles this ** function failing. This function could fail if, for example, the @@ -42328,6 +52515,20 @@ static int winFullPathname( } #endif } +static int winFullPathname( + sqlite3_vfs *pVfs, /* Pointer to vfs object */ + const char *zRelative, /* Possibly relative input path */ + int nFull, /* Size of output buffer in bytes */ + char *zFull /* Output buffer */ +){ + int rc; + MUTEX_LOGIC( sqlite3_mutex *pMutex; ) + MUTEX_LOGIC( pMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR); ) + sqlite3_mutex_enter(pMutex); + rc = winFullPathnameNoMutex(pVfs, zRelative, nFull, zFull); + sqlite3_mutex_leave(pMutex); + return rc; +} #ifndef SQLITE_OMIT_LOAD_EXTENSION /* @@ -42433,9 +52634,6 @@ static int winRandomness(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ EntropyGatherer e; UNUSED_PARAMETER(pVfs); memset(zBuf, 0, nBuf); -#if defined(_MSC_VER) && _MSC_VER>=1400 && !SQLITE_OS_WINCE - rand_s((unsigned int*)zBuf); /* rand_s() is not available with MinGW */ -#endif /* defined(_MSC_VER) && _MSC_VER>=1400 */ e.a = (unsigned char*)zBuf; e.na = nBuf; e.nXor = 0; @@ -42602,55 +52800,105 @@ static int winGetLastError(sqlite3_vfs *pVfs, int nBuf, char *zBuf){ /* ** Initialize and deinitialize the operating system interface. */ -SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ +SQLITE_API int sqlite3_os_init(void){ static sqlite3_vfs winVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #if defined(SQLITE_WIN32_HAS_WIDE) static sqlite3_vfs winLongPathVfs = { - 3, /* iVersion */ - sizeof(winFile), /* szOsFile */ + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ - 0, /* pNext */ - "win32-longpath", /* zName */ - 0, /* pAppData */ - winOpen, /* xOpen */ - winDelete, /* xDelete */ - winAccess, /* xAccess */ - winFullPathname, /* xFullPathname */ - winDlOpen, /* xDlOpen */ - winDlError, /* xDlError */ - winDlSym, /* xDlSym */ - winDlClose, /* xDlClose */ - winRandomness, /* xRandomness */ - winSleep, /* xSleep */ - winCurrentTime, /* xCurrentTime */ - winGetLastError, /* xGetLastError */ - winCurrentTimeInt64, /* xCurrentTimeInt64 */ - winSetSystemCall, /* xSetSystemCall */ - winGetSystemCall, /* xGetSystemCall */ - winNextSystemCall, /* xNextSystemCall */ + 0, /* pNext */ + "win32-longpath", /* zName */ + &winAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#endif + static sqlite3_vfs winNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WIN32_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ + }; +#if defined(SQLITE_WIN32_HAS_WIDE) + static sqlite3_vfs winLongPathNolockVfs = { + 3, /* iVersion */ + sizeof(winFile), /* szOsFile */ + SQLITE_WINNT_MAX_PATH_BYTES, /* mxPathname */ + 0, /* pNext */ + "win32-longpath-none", /* zName */ + &winNolockAppData, /* pAppData */ + winOpen, /* xOpen */ + winDelete, /* xDelete */ + winAccess, /* xAccess */ + winFullPathname, /* xFullPathname */ + winDlOpen, /* xDlOpen */ + winDlError, /* xDlError */ + winDlSym, /* xDlSym */ + winDlClose, /* xDlClose */ + winRandomness, /* xRandomness */ + winSleep, /* xSleep */ + winCurrentTime, /* xCurrentTime */ + winGetLastError, /* xGetLastError */ + winCurrentTimeInt64, /* xCurrentTimeInt64 */ + winSetSystemCall, /* xSetSystemCall */ + winGetSystemCall, /* xGetSystemCall */ + winNextSystemCall, /* xNextSystemCall */ }; #endif @@ -42674,22 +52922,976 @@ SQLITE_API int SQLITE_STDCALL sqlite3_os_init(void){ sqlite3_vfs_register(&winLongPathVfs, 0); #endif + sqlite3_vfs_register(&winNolockVfs, 0); + +#if defined(SQLITE_WIN32_HAS_WIDE) + sqlite3_vfs_register(&winLongPathNolockVfs, 0); +#endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + return SQLITE_OK; } -SQLITE_API int SQLITE_STDCALL sqlite3_os_end(void){ +SQLITE_API int sqlite3_os_end(void){ #if SQLITE_OS_WINRT if( sleepObj!=NULL ){ osCloseHandle(sleepObj); sleepObj = NULL; } #endif + +#ifndef SQLITE_OMIT_WAL + winBigLock = 0; +#endif + return SQLITE_OK; } #endif /* SQLITE_OS_WIN */ /************** End of os_win.c **********************************************/ +/************** Begin file memdb.c *******************************************/ +/* +** 2016-09-07 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an in-memory VFS. A database is held as a contiguous +** block of memory. +** +** This file also implements interface sqlite3_serialize() and +** sqlite3_deserialize(). +*/ +/* #include "sqliteInt.h" */ +#ifndef SQLITE_OMIT_DESERIALIZE + +/* +** Forward declaration of objects used by this utility +*/ +typedef struct sqlite3_vfs MemVfs; +typedef struct MemFile MemFile; +typedef struct MemStore MemStore; + +/* Access to a lower-level VFS that (might) implement dynamic loading, +** access to randomness, etc. +*/ +#define ORIGVFS(p) ((sqlite3_vfs*)((p)->pAppData)) + +/* Storage for a memdb file. +** +** An memdb object can be shared or separate. Shared memdb objects can be +** used by more than one database connection. Mutexes are used by shared +** memdb objects to coordinate access. Separate memdb objects are only +** connected to a single database connection and do not require additional +** mutexes. +** +** Shared memdb objects have .zFName!=0 and .pMutex!=0. They are created +** using "file:/name?vfs=memdb". The first character of the name must be +** "/" or else the object will be a separate memdb object. All shared +** memdb objects are stored in memdb_g.apMemStore[] in an arbitrary order. +** +** Separate memdb objects are created using a name that does not begin +** with "/" or using sqlite3_deserialize(). +** +** Access rules for shared MemStore objects: +** +** * .zFName is initialized when the object is created and afterwards +** is unchanged until the object is destroyed. So it can be accessed +** at any time as long as we know the object is not being destroyed, +** which means while either the SQLITE_MUTEX_STATIC_VFS1 or +** .pMutex is held or the object is not part of memdb_g.apMemStore[]. +** +** * Can .pMutex can only be changed while holding the +** SQLITE_MUTEX_STATIC_VFS1 mutex or while the object is not part +** of memdb_g.apMemStore[]. +** +** * Other fields can only be changed while holding the .pMutex mutex +** or when the .nRef is less than zero and the object is not part of +** memdb_g.apMemStore[]. +** +** * The .aData pointer has the added requirement that it can can only +** be changed (for resizing) when nMmap is zero. +** +*/ +struct MemStore { + sqlite3_int64 sz; /* Size of the file */ + sqlite3_int64 szAlloc; /* Space allocated to aData */ + sqlite3_int64 szMax; /* Maximum allowed size of the file */ + unsigned char *aData; /* content of the file */ + sqlite3_mutex *pMutex; /* Used by shared stores only */ + int nMmap; /* Number of memory mapped pages */ + unsigned mFlags; /* Flags */ + int nRdLock; /* Number of readers */ + int nWrLock; /* Number of writers. (Always 0 or 1) */ + int nRef; /* Number of users of this MemStore */ + char *zFName; /* The filename for shared stores */ +}; + +/* An open file */ +struct MemFile { + sqlite3_file base; /* IO methods */ + MemStore *pStore; /* The storage */ + int eLock; /* Most recent lock against this file */ +}; + +/* +** File-scope variables for holding the memdb files that are accessible +** to multiple database connections in separate threads. +** +** Must hold SQLITE_MUTEX_STATIC_VFS1 to access any part of this object. +*/ +static struct MemFS { + int nMemStore; /* Number of shared MemStore objects */ + MemStore **apMemStore; /* Array of all shared MemStore objects */ +} memdb_g; + +/* +** Methods for MemFile +*/ +static int memdbClose(sqlite3_file*); +static int memdbRead(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); +static int memdbWrite(sqlite3_file*,const void*,int iAmt, sqlite3_int64 iOfst); +static int memdbTruncate(sqlite3_file*, sqlite3_int64 size); +static int memdbSync(sqlite3_file*, int flags); +static int memdbFileSize(sqlite3_file*, sqlite3_int64 *pSize); +static int memdbLock(sqlite3_file*, int); +static int memdbUnlock(sqlite3_file*, int); +/* static int memdbCheckReservedLock(sqlite3_file*, int *pResOut);// not used */ +static int memdbFileControl(sqlite3_file*, int op, void *pArg); +/* static int memdbSectorSize(sqlite3_file*); // not used */ +static int memdbDeviceCharacteristics(sqlite3_file*); +static int memdbFetch(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); +static int memdbUnfetch(sqlite3_file*, sqlite3_int64 iOfst, void *p); + +/* +** Methods for MemVfs +*/ +static int memdbOpen(sqlite3_vfs*, const char *, sqlite3_file*, int , int *); +/* static int memdbDelete(sqlite3_vfs*, const char *zName, int syncDir); */ +static int memdbAccess(sqlite3_vfs*, const char *zName, int flags, int *); +static int memdbFullPathname(sqlite3_vfs*, const char *zName, int, char *zOut); +static void *memdbDlOpen(sqlite3_vfs*, const char *zFilename); +static void memdbDlError(sqlite3_vfs*, int nByte, char *zErrMsg); +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char*zSym))(void); +static void memdbDlClose(sqlite3_vfs*, void*); +static int memdbRandomness(sqlite3_vfs*, int nByte, char *zOut); +static int memdbSleep(sqlite3_vfs*, int microseconds); +/* static int memdbCurrentTime(sqlite3_vfs*, double*); */ +static int memdbGetLastError(sqlite3_vfs*, int, char *); +static int memdbCurrentTimeInt64(sqlite3_vfs*, sqlite3_int64*); + +static sqlite3_vfs memdb_vfs = { + 2, /* iVersion */ + 0, /* szOsFile (set when registered) */ + 1024, /* mxPathname */ + 0, /* pNext */ + "memdb", /* zName */ + 0, /* pAppData (set when registered) */ + memdbOpen, /* xOpen */ + 0, /* memdbDelete, */ /* xDelete */ + memdbAccess, /* xAccess */ + memdbFullPathname, /* xFullPathname */ + memdbDlOpen, /* xDlOpen */ + memdbDlError, /* xDlError */ + memdbDlSym, /* xDlSym */ + memdbDlClose, /* xDlClose */ + memdbRandomness, /* xRandomness */ + memdbSleep, /* xSleep */ + 0, /* memdbCurrentTime, */ /* xCurrentTime */ + memdbGetLastError, /* xGetLastError */ + memdbCurrentTimeInt64, /* xCurrentTimeInt64 */ + 0, /* xSetSystemCall */ + 0, /* xGetSystemCall */ + 0, /* xNextSystemCall */ +}; + +static const sqlite3_io_methods memdb_io_methods = { + 3, /* iVersion */ + memdbClose, /* xClose */ + memdbRead, /* xRead */ + memdbWrite, /* xWrite */ + memdbTruncate, /* xTruncate */ + memdbSync, /* xSync */ + memdbFileSize, /* xFileSize */ + memdbLock, /* xLock */ + memdbUnlock, /* xUnlock */ + 0, /* memdbCheckReservedLock, */ /* xCheckReservedLock */ + memdbFileControl, /* xFileControl */ + 0, /* memdbSectorSize,*/ /* xSectorSize */ + memdbDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, /* xShmMap */ + 0, /* xShmLock */ + 0, /* xShmBarrier */ + 0, /* xShmUnmap */ + memdbFetch, /* xFetch */ + memdbUnfetch /* xUnfetch */ +}; + +/* +** Enter/leave the mutex on a MemStore +*/ +#if defined(SQLITE_THREADSAFE) && SQLITE_THREADSAFE==0 +static void memdbEnter(MemStore *p){ + UNUSED_PARAMETER(p); +} +static void memdbLeave(MemStore *p){ + UNUSED_PARAMETER(p); +} +#else +static void memdbEnter(MemStore *p){ + sqlite3_mutex_enter(p->pMutex); +} +static void memdbLeave(MemStore *p){ + sqlite3_mutex_leave(p->pMutex); +} +#endif + + + +/* +** Close an memdb-file. +** Free the underlying MemStore object when its refcount drops to zero +** or less. +*/ +static int memdbClose(sqlite3_file *pFile){ + MemStore *p = ((MemFile*)pFile)->pStore; + if( p->zFName ){ + int i; +#ifndef SQLITE_MUTEX_OMIT + sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + sqlite3_mutex_enter(pVfsMutex); + for(i=0; ALWAYS(inRef==1 ){ + memdb_g.apMemStore[i] = memdb_g.apMemStore[--memdb_g.nMemStore]; + if( memdb_g.nMemStore==0 ){ + sqlite3_free(memdb_g.apMemStore); + memdb_g.apMemStore = 0; + } + } + break; + } + } + sqlite3_mutex_leave(pVfsMutex); + }else{ + memdbEnter(p); + } + p->nRef--; + if( p->nRef<=0 ){ + if( p->mFlags & SQLITE_DESERIALIZE_FREEONCLOSE ){ + sqlite3_free(p->aData); + } + memdbLeave(p); + sqlite3_mutex_free(p->pMutex); + sqlite3_free(p); + }else{ + memdbLeave(p); + } + return SQLITE_OK; +} + +/* +** Read data from an memdb-file. +*/ +static int memdbRead( + sqlite3_file *pFile, + void *zBuf, + int iAmt, + sqlite_int64 iOfst +){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + if( iOfst+iAmt>p->sz ){ + memset(zBuf, 0, iAmt); + if( iOfstsz ) memcpy(zBuf, p->aData+iOfst, p->sz - iOfst); + memdbLeave(p); + return SQLITE_IOERR_SHORT_READ; + } + memcpy(zBuf, p->aData+iOfst, iAmt); + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Try to enlarge the memory allocation to hold at least sz bytes +*/ +static int memdbEnlarge(MemStore *p, sqlite3_int64 newSz){ + unsigned char *pNew; + if( (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)==0 || NEVER(p->nMmap>0) ){ + return SQLITE_FULL; + } + if( newSz>p->szMax ){ + return SQLITE_FULL; + } + newSz *= 2; + if( newSz>p->szMax ) newSz = p->szMax; + pNew = sqlite3Realloc(p->aData, newSz); + if( pNew==0 ) return SQLITE_IOERR_NOMEM; + p->aData = pNew; + p->szAlloc = newSz; + return SQLITE_OK; +} + +/* +** Write data to an memdb-file. +*/ +static int memdbWrite( + sqlite3_file *pFile, + const void *z, + int iAmt, + sqlite_int64 iOfst +){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + if( NEVER(p->mFlags & SQLITE_DESERIALIZE_READONLY) ){ + /* Can't happen: memdbLock() will return SQLITE_READONLY before + ** reaching this point */ + memdbLeave(p); + return SQLITE_IOERR_WRITE; + } + if( iOfst+iAmt>p->sz ){ + int rc; + if( iOfst+iAmt>p->szAlloc + && (rc = memdbEnlarge(p, iOfst+iAmt))!=SQLITE_OK + ){ + memdbLeave(p); + return rc; + } + if( iOfst>p->sz ) memset(p->aData+p->sz, 0, iOfst-p->sz); + p->sz = iOfst+iAmt; + } + memcpy(p->aData+iOfst, z, iAmt); + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Truncate an memdb-file. +** +** In rollback mode (which is always the case for memdb, as it does not +** support WAL mode) the truncate() method is only used to reduce +** the size of a file, never to increase the size. +*/ +static int memdbTruncate(sqlite3_file *pFile, sqlite_int64 size){ + MemStore *p = ((MemFile*)pFile)->pStore; + int rc = SQLITE_OK; + memdbEnter(p); + if( size>p->sz ){ + /* This can only happen with a corrupt wal mode db */ + rc = SQLITE_CORRUPT; + }else{ + p->sz = size; + } + memdbLeave(p); + return rc; +} + +/* +** Sync an memdb-file. +*/ +static int memdbSync(sqlite3_file *pFile, int flags){ + UNUSED_PARAMETER(pFile); + UNUSED_PARAMETER(flags); + return SQLITE_OK; +} + +/* +** Return the current file-size of an memdb-file. +*/ +static int memdbFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + *pSize = p->sz; + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Lock an memdb-file. +*/ +static int memdbLock(sqlite3_file *pFile, int eLock){ + MemFile *pThis = (MemFile*)pFile; + MemStore *p = pThis->pStore; + int rc = SQLITE_OK; + if( eLock<=pThis->eLock ) return SQLITE_OK; + memdbEnter(p); + + assert( p->nWrLock==0 || p->nWrLock==1 ); + assert( pThis->eLock<=SQLITE_LOCK_SHARED || p->nWrLock==1 ); + assert( pThis->eLock==SQLITE_LOCK_NONE || p->nRdLock>=1 ); + + if( eLock>SQLITE_LOCK_SHARED && (p->mFlags & SQLITE_DESERIALIZE_READONLY) ){ + rc = SQLITE_READONLY; + }else{ + switch( eLock ){ + case SQLITE_LOCK_SHARED: { + assert( pThis->eLock==SQLITE_LOCK_NONE ); + if( p->nWrLock>0 ){ + rc = SQLITE_BUSY; + }else{ + p->nRdLock++; + } + break; + }; + + case SQLITE_LOCK_RESERVED: + case SQLITE_LOCK_PENDING: { + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + if( ALWAYS(pThis->eLock==SQLITE_LOCK_SHARED) ){ + if( p->nWrLock>0 ){ + rc = SQLITE_BUSY; + }else{ + p->nWrLock = 1; + } + } + break; + } + + default: { + assert( eLock==SQLITE_LOCK_EXCLUSIVE ); + assert( pThis->eLock>=SQLITE_LOCK_SHARED ); + if( p->nRdLock>1 ){ + rc = SQLITE_BUSY; + }else if( pThis->eLock==SQLITE_LOCK_SHARED ){ + p->nWrLock = 1; + } + break; + } + } + } + if( rc==SQLITE_OK ) pThis->eLock = eLock; + memdbLeave(p); + return rc; +} + +/* +** Unlock an memdb-file. +*/ +static int memdbUnlock(sqlite3_file *pFile, int eLock){ + MemFile *pThis = (MemFile*)pFile; + MemStore *p = pThis->pStore; + if( eLock>=pThis->eLock ) return SQLITE_OK; + memdbEnter(p); + + assert( eLock==SQLITE_LOCK_SHARED || eLock==SQLITE_LOCK_NONE ); + if( eLock==SQLITE_LOCK_SHARED ){ + if( ALWAYS(pThis->eLock>SQLITE_LOCK_SHARED) ){ + p->nWrLock--; + } + }else{ + if( pThis->eLock>SQLITE_LOCK_SHARED ){ + p->nWrLock--; + } + p->nRdLock--; + } + + pThis->eLock = eLock; + memdbLeave(p); + return SQLITE_OK; +} + +#if 0 +/* +** This interface is only used for crash recovery, which does not +** occur on an in-memory database. +*/ +static int memdbCheckReservedLock(sqlite3_file *pFile, int *pResOut){ + *pResOut = 0; + return SQLITE_OK; +} +#endif + + +/* +** File control method. For custom operations on an memdb-file. +*/ +static int memdbFileControl(sqlite3_file *pFile, int op, void *pArg){ + MemStore *p = ((MemFile*)pFile)->pStore; + int rc = SQLITE_NOTFOUND; + memdbEnter(p); + if( op==SQLITE_FCNTL_VFSNAME ){ + *(char**)pArg = sqlite3_mprintf("memdb(%p,%lld)", p->aData, p->sz); + rc = SQLITE_OK; + } + if( op==SQLITE_FCNTL_SIZE_LIMIT ){ + sqlite3_int64 iLimit = *(sqlite3_int64*)pArg; + if( iLimitsz ){ + if( iLimit<0 ){ + iLimit = p->szMax; + }else{ + iLimit = p->sz; + } + } + p->szMax = iLimit; + *(sqlite3_int64*)pArg = iLimit; + rc = SQLITE_OK; + } + memdbLeave(p); + return rc; +} + +#if 0 /* Not used because of SQLITE_IOCAP_POWERSAFE_OVERWRITE */ +/* +** Return the sector-size in bytes for an memdb-file. +*/ +static int memdbSectorSize(sqlite3_file *pFile){ + return 1024; +} +#endif + +/* +** Return the device characteristic flags supported by an memdb-file. +*/ +static int memdbDeviceCharacteristics(sqlite3_file *pFile){ + UNUSED_PARAMETER(pFile); + return SQLITE_IOCAP_ATOMIC | + SQLITE_IOCAP_POWERSAFE_OVERWRITE | + SQLITE_IOCAP_SAFE_APPEND | + SQLITE_IOCAP_SEQUENTIAL; +} + +/* Fetch a page of a memory-mapped file */ +static int memdbFetch( + sqlite3_file *pFile, + sqlite3_int64 iOfst, + int iAmt, + void **pp +){ + MemStore *p = ((MemFile*)pFile)->pStore; + memdbEnter(p); + if( iOfst+iAmt>p->sz || (p->mFlags & SQLITE_DESERIALIZE_RESIZEABLE)!=0 ){ + *pp = 0; + }else{ + p->nMmap++; + *pp = (void*)(p->aData + iOfst); + } + memdbLeave(p); + return SQLITE_OK; +} + +/* Release a memory-mapped page */ +static int memdbUnfetch(sqlite3_file *pFile, sqlite3_int64 iOfst, void *pPage){ + MemStore *p = ((MemFile*)pFile)->pStore; + UNUSED_PARAMETER(iOfst); + UNUSED_PARAMETER(pPage); + memdbEnter(p); + p->nMmap--; + memdbLeave(p); + return SQLITE_OK; +} + +/* +** Open an mem file handle. +*/ +static int memdbOpen( + sqlite3_vfs *pVfs, + const char *zName, + sqlite3_file *pFd, + int flags, + int *pOutFlags +){ + MemFile *pFile = (MemFile*)pFd; + MemStore *p = 0; + int szName; + UNUSED_PARAMETER(pVfs); + + memset(pFile, 0, sizeof(*pFile)); + szName = sqlite3Strlen30(zName); + if( szName>1 && (zName[0]=='/' || zName[0]=='\\') ){ + int i; +#ifndef SQLITE_MUTEX_OMIT + sqlite3_mutex *pVfsMutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_VFS1); +#endif + sqlite3_mutex_enter(pVfsMutex); + for(i=0; izFName,zName)==0 ){ + p = memdb_g.apMemStore[i]; + break; + } + } + if( p==0 ){ + MemStore **apNew; + p = sqlite3Malloc( sizeof(*p) + szName + 3 ); + if( p==0 ){ + sqlite3_mutex_leave(pVfsMutex); + return SQLITE_NOMEM; + } + apNew = sqlite3Realloc(memdb_g.apMemStore, + sizeof(apNew[0])*(memdb_g.nMemStore+1) ); + if( apNew==0 ){ + sqlite3_free(p); + sqlite3_mutex_leave(pVfsMutex); + return SQLITE_NOMEM; + } + apNew[memdb_g.nMemStore++] = p; + memdb_g.apMemStore = apNew; + memset(p, 0, sizeof(*p)); + p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE|SQLITE_DESERIALIZE_FREEONCLOSE; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; + p->zFName = (char*)&p[1]; + memcpy(p->zFName, zName, szName+1); + p->pMutex = sqlite3_mutex_alloc(SQLITE_MUTEX_FAST); + if( p->pMutex==0 ){ + memdb_g.nMemStore--; + sqlite3_free(p); + sqlite3_mutex_leave(pVfsMutex); + return SQLITE_NOMEM; + } + p->nRef = 1; + memdbEnter(p); + }else{ + memdbEnter(p); + p->nRef++; + } + sqlite3_mutex_leave(pVfsMutex); + }else{ + p = sqlite3Malloc( sizeof(*p) ); + if( p==0 ){ + return SQLITE_NOMEM; + } + memset(p, 0, sizeof(*p)); + p->mFlags = SQLITE_DESERIALIZE_RESIZEABLE | SQLITE_DESERIALIZE_FREEONCLOSE; + p->szMax = sqlite3GlobalConfig.mxMemdbSize; + } + pFile->pStore = p; + if( pOutFlags!=0 ){ + *pOutFlags = flags | SQLITE_OPEN_MEMORY; + } + pFd->pMethods = &memdb_io_methods; + memdbLeave(p); + return SQLITE_OK; +} + +#if 0 /* Only used to delete rollback journals, super-journals, and WAL + ** files, none of which exist in memdb. So this routine is never used */ +/* +** Delete the file located at zPath. If the dirSync argument is true, +** ensure the file-system modifications are synced to disk before +** returning. +*/ +static int memdbDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ + return SQLITE_IOERR_DELETE; +} +#endif + +/* +** Test for access permissions. Return true if the requested permission +** is available, or false otherwise. +** +** With memdb, no files ever exist on disk. So always return false. +*/ +static int memdbAccess( + sqlite3_vfs *pVfs, + const char *zPath, + int flags, + int *pResOut +){ + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(zPath); + UNUSED_PARAMETER(flags); + *pResOut = 0; + return SQLITE_OK; +} + +/* +** Populate buffer zOut with the full canonical pathname corresponding +** to the pathname in zPath. zOut is guaranteed to point to a buffer +** of at least (INST_MAX_PATHNAME+1) bytes. +*/ +static int memdbFullPathname( + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, + char *zOut +){ + UNUSED_PARAMETER(pVfs); + sqlite3_snprintf(nOut, zOut, "%s", zPath); + return SQLITE_OK; +} + +/* +** Open the dynamic library located at zPath and return a handle. +*/ +static void *memdbDlOpen(sqlite3_vfs *pVfs, const char *zPath){ + return ORIGVFS(pVfs)->xDlOpen(ORIGVFS(pVfs), zPath); +} + +/* +** Populate the buffer zErrMsg (size nByte bytes) with a human readable +** utf-8 string describing the most recent error encountered associated +** with dynamic libraries. +*/ +static void memdbDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ + ORIGVFS(pVfs)->xDlError(ORIGVFS(pVfs), nByte, zErrMsg); +} + +/* +** Return a pointer to the symbol zSymbol in the dynamic library pHandle. +*/ +static void (*memdbDlSym(sqlite3_vfs *pVfs, void *p, const char *zSym))(void){ + return ORIGVFS(pVfs)->xDlSym(ORIGVFS(pVfs), p, zSym); +} + +/* +** Close the dynamic library handle pHandle. +*/ +static void memdbDlClose(sqlite3_vfs *pVfs, void *pHandle){ + ORIGVFS(pVfs)->xDlClose(ORIGVFS(pVfs), pHandle); +} + +/* +** Populate the buffer pointed to by zBufOut with nByte bytes of +** random data. +*/ +static int memdbRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ + return ORIGVFS(pVfs)->xRandomness(ORIGVFS(pVfs), nByte, zBufOut); +} + +/* +** Sleep for nMicro microseconds. Return the number of microseconds +** actually slept. +*/ +static int memdbSleep(sqlite3_vfs *pVfs, int nMicro){ + return ORIGVFS(pVfs)->xSleep(ORIGVFS(pVfs), nMicro); +} + +#if 0 /* Never used. Modern cores only call xCurrentTimeInt64() */ +/* +** Return the current time as a Julian Day number in *pTimeOut. +*/ +static int memdbCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ + return ORIGVFS(pVfs)->xCurrentTime(ORIGVFS(pVfs), pTimeOut); +} +#endif + +static int memdbGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + return ORIGVFS(pVfs)->xGetLastError(ORIGVFS(pVfs), a, b); +} +static int memdbCurrentTimeInt64(sqlite3_vfs *pVfs, sqlite3_int64 *p){ + return ORIGVFS(pVfs)->xCurrentTimeInt64(ORIGVFS(pVfs), p); +} + +/* +** Translate a database connection pointer and schema name into a +** MemFile pointer. +*/ +static MemFile *memdbFromDbSchema(sqlite3 *db, const char *zSchema){ + MemFile *p = 0; + MemStore *pStore; + int rc = sqlite3_file_control(db, zSchema, SQLITE_FCNTL_FILE_POINTER, &p); + if( rc ) return 0; + if( p->base.pMethods!=&memdb_io_methods ) return 0; + pStore = p->pStore; + memdbEnter(pStore); + if( pStore->zFName!=0 ) p = 0; + memdbLeave(pStore); + return p; +} + +/* +** Return the serialization of a database +*/ +SQLITE_API unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which database within the connection */ + sqlite3_int64 *piSize, /* Write size here, if not NULL */ + unsigned int mFlags /* Maybe SQLITE_SERIALIZE_NOCOPY */ +){ + MemFile *p; + int iDb; + Btree *pBt; + sqlite3_int64 sz; + int szPage = 0; + sqlite3_stmt *pStmt = 0; + unsigned char *pOut; + char *zSql; + int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + if( zSchema==0 ) zSchema = db->aDb[0].zDbSName; + p = memdbFromDbSchema(db, zSchema); + iDb = sqlite3FindDbName(db, zSchema); + if( piSize ) *piSize = -1; + if( iDb<0 ) return 0; + if( p ){ + MemStore *pStore = p->pStore; + assert( pStore->pMutex==0 ); + if( piSize ) *piSize = pStore->sz; + if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ + pOut = pStore->aData; + }else{ + pOut = sqlite3_malloc64( pStore->sz ); + if( pOut ) memcpy(pOut, pStore->aData, pStore->sz); + } + return pOut; + } + pBt = db->aDb[iDb].pBt; + if( pBt==0 ) return 0; + szPage = sqlite3BtreeGetPageSize(pBt); + zSql = sqlite3_mprintf("PRAGMA \"%w\".page_count", zSchema); + rc = zSql ? sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0) : SQLITE_NOMEM; + sqlite3_free(zSql); + if( rc ) return 0; + rc = sqlite3_step(pStmt); + if( rc!=SQLITE_ROW ){ + pOut = 0; + }else{ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + if( sz==0 ){ + sqlite3_reset(pStmt); + sqlite3_exec(db, "BEGIN IMMEDIATE; COMMIT;", 0, 0, 0); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_ROW ){ + sz = sqlite3_column_int64(pStmt, 0)*szPage; + } + } + if( piSize ) *piSize = sz; + if( mFlags & SQLITE_SERIALIZE_NOCOPY ){ + pOut = 0; + }else{ + pOut = sqlite3_malloc64( sz ); + if( pOut ){ + int nPage = sqlite3_column_int(pStmt, 0); + Pager *pPager = sqlite3BtreePager(pBt); + int pgno; + for(pgno=1; pgno<=nPage; pgno++){ + DbPage *pPage = 0; + unsigned char *pTo = pOut + szPage*(sqlite3_int64)(pgno-1); + rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pPage, 0); + if( rc==SQLITE_OK ){ + memcpy(pTo, sqlite3PagerGetData(pPage), szPage); + }else{ + memset(pTo, 0, szPage); + } + sqlite3PagerUnref(pPage); + } + } + } + } + sqlite3_finalize(pStmt); + return pOut; +} + +/* Convert zSchema to a MemDB and initialize its content. +*/ +SQLITE_API int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +){ + MemFile *p; + char *zSql; + sqlite3_stmt *pStmt = 0; + int rc; + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } + if( szDb<0 ) return SQLITE_MISUSE_BKPT; + if( szBuf<0 ) return SQLITE_MISUSE_BKPT; +#endif + + sqlite3_mutex_enter(db->mutex); + if( zSchema==0 ) zSchema = db->aDb[0].zDbSName; + iDb = sqlite3FindDbName(db, zSchema); + testcase( iDb==1 ); + if( iDb<2 && iDb!=0 ){ + rc = SQLITE_ERROR; + goto end_deserialize; + } + zSql = sqlite3_mprintf("ATTACH x AS %Q", zSchema); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); + sqlite3_free(zSql); + } + if( rc ) goto end_deserialize; + db->init.iDb = (u8)iDb; + db->init.reopenMemdb = 1; + rc = sqlite3_step(pStmt); + db->init.reopenMemdb = 0; + if( rc!=SQLITE_DONE ){ + rc = SQLITE_ERROR; + goto end_deserialize; + } + p = memdbFromDbSchema(db, zSchema); + if( p==0 ){ + rc = SQLITE_ERROR; + }else{ + MemStore *pStore = p->pStore; + pStore->aData = pData; + pData = 0; + pStore->sz = szDb; + pStore->szAlloc = szBuf; + pStore->szMax = szBuf; + if( pStore->szMaxszMax = sqlite3GlobalConfig.mxMemdbSize; + } + pStore->mFlags = mFlags; + rc = SQLITE_OK; + } + +end_deserialize: + sqlite3_finalize(pStmt); + if( pData && (mFlags & SQLITE_DESERIALIZE_FREEONCLOSE)!=0 ){ + sqlite3_free(pData); + } + sqlite3_mutex_leave(db->mutex); + return rc; +} + +/* +** Return true if the VFS is the memvfs. +*/ +SQLITE_PRIVATE int sqlite3IsMemdb(const sqlite3_vfs *pVfs){ + return pVfs==&memdb_vfs; +} + +/* +** This routine is called when the extension is loaded. +** Register the new VFS. +*/ +SQLITE_PRIVATE int sqlite3MemdbInit(void){ + sqlite3_vfs *pLower = sqlite3_vfs_find(0); + unsigned int sz; + if( NEVER(pLower==0) ) return SQLITE_ERROR; + sz = pLower->szOsFile; + memdb_vfs.pAppData = pLower; + /* The following conditional can only be true when compiled for + ** Windows x86 and SQLITE_MAX_MMAP_SIZE=0. We always leave + ** it in, to be safe, but it is marked as NO_TEST since there + ** is no way to reach it under most builds. */ + if( szu.aHash[h] ){ if (p->nSet<(BITVEC_NINT-1)) { goto bitvec_set_end; @@ -42986,7 +54188,7 @@ SQLITE_PRIVATE u32 sqlite3BitvecSize(Bitvec *p){ return p->iSize; } -#ifndef SQLITE_OMIT_BUILTIN_TEST +#ifndef SQLITE_UNTESTABLE /* ** Let V[] be an array of unsigned characters sufficient to hold ** up to N bits. Let I be an integer between 0 and N. 0<=IpExtra; + printf("%3d: nRef %2lld flgs %02x data ", i, pPg->nRef, pPg->flags); + a = (unsigned char *)pLower->pBuf; + for(j=0; j<12; j++) printf("%02x", a[j]); + printf(" ptr %p\n", pPg); + } + } + static void pcacheDump(PCache *pCache){ + int N; + int i; + sqlite3_pcache_page *pLower; + if( sqlite3PcacheTrace<2 ) return; if( pCache->pCache==0 ) return; N = sqlite3PcachePagecount(pCache); if( N>sqlite3PcacheMxDump ) N = sqlite3PcacheMxDump; for(i=1; i<=N; i++){ pLower = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, i, 0); - if( pLower==0 ) continue; - pPg = (PgHdr*)pLower->pExtra; - printf("%3d: nRef %2d flgs %02x data ", i, pPg->nRef, pPg->flags); - a = (unsigned char *)pLower->pBuf; - for(j=0; j<12; j++) printf("%02x", a[j]); - printf("\n"); - if( pPg->pPage==0 ){ + pcachePageTrace(i, pLower); + if( pLower && ((PgHdr*)pLower)->pPage==0 ){ sqlite3GlobalConfig.pcache2.xUnpin(pCache->pCache, pLower, 0); } } } - #else +#else # define pcacheTrace(X) +# define pcachePageTrace(PGNO, X) # define pcacheDump(X) #endif +/* +** Return 1 if pPg is on the dirty list for pCache. Return 0 if not. +** This routine runs inside of assert() statements only. +*/ +#if defined(SQLITE_ENABLE_EXPENSIVE_ASSERT) +static int pageOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 1; + } + return 0; +} +static int pageNotOnDirtyList(PCache *pCache, PgHdr *pPg){ + PgHdr *p; + for(p=pCache->pDirty; p; p=p->pDirtyNext){ + if( p==pPg ) return 0; + } + return 1; +} +#else +# define pageOnDirtyList(A,B) 1 +# define pageNotOnDirtyList(A,B) 1 +#endif + /* ** Check invariants on a PgHdr entry. Return true if everything is OK. ** Return false if any invariant is violated. @@ -43211,17 +54445,22 @@ struct PCache { ** ** assert( sqlite3PcachePageSanity(pPg) ); */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3PcachePageSanity(PgHdr *pPg){ PCache *pCache; assert( pPg!=0 ); - assert( pPg->pgno>0 ); /* Page number is 1 or more */ + assert( pPg->pgno>0 || pPg->pPager==0 ); /* Page number is 1 or more */ pCache = pPg->pCache; assert( pCache!=0 ); /* Every page has an associated PCache */ if( pPg->flags & PGHDR_CLEAN ){ assert( (pPg->flags & PGHDR_DIRTY)==0 );/* Cannot be both CLEAN and DIRTY */ - assert( pCache->pDirty!=pPg ); /* CLEAN pages not on dirty list */ - assert( pCache->pDirtyTail!=pPg ); + assert( pageNotOnDirtyList(pCache, pPg) );/* CLEAN pages not on dirtylist */ + }else{ + assert( (pPg->flags & PGHDR_DIRTY)!=0 );/* If not CLEAN must be DIRTY */ + assert( pPg->pDirtyNext==0 || pPg->pDirtyNext->pDirtyPrev==pPg ); + assert( pPg->pDirtyPrev==0 || pPg->pDirtyPrev->pDirtyNext==pPg ); + assert( pPg->pDirtyPrev!=0 || pCache->pDirty==pPg ); + assert( pageOnDirtyList(pCache, pPg) ); } /* WRITEABLE pages must also be DIRTY */ if( pPg->flags & PGHDR_WRITEABLE ){ @@ -43271,12 +54510,12 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( addRemove & PCACHE_DIRTYLIST_REMOVE ){ assert( pPage->pDirtyNext || pPage==p->pDirtyTail ); assert( pPage->pDirtyPrev || pPage==p->pDirty ); - + /* Update the PCache1.pSynced variable if necessary. */ if( p->pSynced==pPage ){ p->pSynced = pPage->pDirtyPrev; } - + if( pPage->pDirtyNext ){ pPage->pDirtyNext->pDirtyPrev = pPage->pDirtyPrev; }else{ @@ -43286,7 +54525,7 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ if( pPage->pDirtyPrev ){ pPage->pDirtyPrev->pDirtyNext = pPage->pDirtyNext; }else{ - /* If there are now no dirty pages in the cache, set eCreate to 2. + /* If there are now no dirty pages in the cache, set eCreate to 2. ** This is an optimization that allows sqlite3PcacheFetch() to skip ** searching for a dirty page to eject from the cache when it might ** otherwise have to. */ @@ -43298,12 +54537,9 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ p->eCreate = 2; } } - pPage->pDirtyNext = 0; - pPage->pDirtyPrev = 0; } if( addRemove & PCACHE_DIRTYLIST_ADD ){ - assert( pPage->pDirtyNext==0 && pPage->pDirtyPrev==0 && p->pDirty!=pPage ); - + pPage->pDirtyPrev = 0; pPage->pDirtyNext = p->pDirty; if( pPage->pDirtyNext ){ assert( pPage->pDirtyNext->pDirtyPrev==0 ); @@ -43318,11 +54554,11 @@ static void pcacheManageDirtyList(PgHdr *pPage, u8 addRemove){ p->pDirty = pPage; /* If pSynced is NULL and this page has a clear NEED_SYNC flag, set - ** pSynced to point to it. Checking the NEED_SYNC flag is an + ** pSynced to point to it. Checking the NEED_SYNC flag is an ** optimization, as if pSynced points to a page with the NEED_SYNC - ** flag set sqlite3PcacheFetchStress() searches through all newer + ** flag set sqlite3PcacheFetchStress() searches through all newer ** entries of the dirty-list for a page with NEED_SYNC clear anyway. */ - if( !p->pSynced + if( !p->pSynced && 0==(pPage->flags&PGHDR_NEED_SYNC) /*OPTIMIZATION-IF-FALSE*/ ){ p->pSynced = pPage; @@ -43353,16 +54589,20 @@ static int numberOfCachePages(PCache *p){ ** suggested cache size is set to N. */ return p->szCache; }else{ - /* IMPLEMENTATION-OF: R-61436-13639 If the argument N is negative, then - ** the number of cache pages is adjusted to use approximately abs(N*1024) - ** bytes of memory. */ - return (int)((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + i64 n; + /* IMPLEMENTATION-OF: R-59858-46238 If the argument N is negative, then the + ** number of cache pages is adjusted to be a number of pages that would + ** use approximately abs(N*1024) bytes of memory based on the current + ** page size. */ + n = ((-1024*(i64)p->szCache)/(p->szPage+p->szExtra)); + if( n>1000000000 ) n = 1000000000; + return (int)n; } } /*************************************************** General Interfaces ****** ** -** Initialize and shutdown the page cache subsystem. Neither of these +** Initialize and shutdown the page cache subsystem. Neither of these ** functions are threadsafe. */ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ @@ -43371,6 +54611,7 @@ SQLITE_PRIVATE int sqlite3PcacheInitialize(void){ ** built-in default page cache is used instead of the application defined ** page cache. */ sqlite3PCacheSetDefault(); + assert( sqlite3GlobalConfig.pcache2.xInit!=0 ); } return sqlite3GlobalConfig.pcache2.xInit(sqlite3GlobalConfig.pcache2.pArg); } @@ -43388,9 +54629,15 @@ SQLITE_PRIVATE int sqlite3PcacheSize(void){ return sizeof(PCache); } /* ** Create a new PCache object. Storage space to hold the object -** has already been allocated and is passed in as the p pointer. -** The caller discovers how much space needs to be allocated by +** has already been allocated and is passed in as the p pointer. +** The caller discovers how much space needs to be allocated by ** calling sqlite3PcacheSize(). +** +** szExtra is some extra space allocated for each page. The first +** 8 bytes of the extra space will be zeroed as the page is allocated, +** but remaining content will be uninitialized. Though it is opaque +** to this module, the extra space really ends up being the MemPage +** structure in the pager. */ SQLITE_PRIVATE int sqlite3PcacheOpen( int szPage, /* Size of every page */ @@ -43403,6 +54650,7 @@ SQLITE_PRIVATE int sqlite3PcacheOpen( memset(p, 0, sizeof(PCache)); p->szPage = 1; p->szExtra = szExtra; + assert( szExtra>=8 ); /* First 8 bytes will be zeroed */ p->bPurgeable = bPurgeable; p->eCreate = 2; p->xStress = xStress; @@ -43472,7 +54720,6 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( assert( pCache!=0 ); assert( pCache->pCache!=0 ); assert( createFlag==3 || createFlag==0 ); - assert( pgno>0 ); assert( pCache->eCreate==((pCache->bPurgeable && pCache->pDirty) ? 1 : 2) ); /* eCreate defines what to do if the page does not exist. @@ -43487,15 +54734,16 @@ SQLITE_PRIVATE sqlite3_pcache_page *sqlite3PcacheFetch( assert( createFlag==0 || pCache->eCreate==eCreate ); assert( createFlag==0 || eCreate==1+(!pCache->bPurgeable||!pCache->pDirty) ); pRes = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, pgno, eCreate); - pcacheTrace(("%p.FETCH %d%s (result: %p)\n",pCache,pgno, + pcacheTrace(("%p.FETCH %d%s (result: %p) ",pCache,pgno, createFlag?" create":"",pRes)); + pcachePageTrace(pgno, pRes); return pRes; } /* ** If the sqlite3PcacheFetch() routine is unable to allocate a new ** page because no clean pages are available for reuse and the cache -** size limit has been reached, then this routine can be invoked to +** size limit has been reached, then this routine can be invoked to ** try harder to allocate a page. This routine might invoke the stress ** callback to spill dirty pages to the journal. It will then try to ** allocate the new page and will only fail to allocate a new page on @@ -43512,17 +54760,17 @@ SQLITE_PRIVATE int sqlite3PcacheFetchStress( if( pCache->eCreate==2 ) return 0; if( sqlite3PcachePagecount(pCache)>pCache->szSpill ){ - /* Find a dirty page to write-out and recycle. First try to find a + /* Find a dirty page to write-out and recycle. First try to find a ** page that does not require a journal-sync (one with PGHDR_NEED_SYNC - ** cleared), but if that is not possible settle for any other + ** cleared), but if that is not possible settle for any other ** unreferenced dirty page. ** ** If the LRU page in the dirty list that has a clear PGHDR_NEED_SYNC ** flag is currently referenced, then the following may leave pSynced ** set incorrectly (pointing to other than the LRU page with NEED_SYNC ** cleared). This is Ok, as pSynced is just an optimization. */ - for(pPg=pCache->pSynced; - pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); + for(pPg=pCache->pSynced; + pPg && (pPg->nRef || (pPg->flags&PGHDR_NEED_SYNC)); pPg=pPg->pDirtyPrev ); pCache->pSynced = pPg; @@ -43532,10 +54780,10 @@ SQLITE_PRIVATE int sqlite3PcacheFetchStress( if( pPg ){ int rc; #ifdef SQLITE_LOG_CACHE_SPILL - sqlite3_log(SQLITE_FULL, + sqlite3_log(SQLITE_FULL, "spill page %d making room for %d - cache used: %d/%d", pPg->pgno, pgno, - sqlite3GlobalConfig.pcache.xPagecount(pCache->pCache), + sqlite3GlobalConfig.pcache2.xPagecount(pCache->pCache), numberOfCachePages(pCache)); #endif pcacheTrace(("%p.SPILL %d\n",pCache,pPg->pgno)); @@ -43568,11 +54816,12 @@ static SQLITE_NOINLINE PgHdr *pcacheFetchFinishWithInit( assert( pPage!=0 ); pPgHdr = (PgHdr*)pPage->pExtra; assert( pPgHdr->pPage==0 ); - memset(pPgHdr, 0, sizeof(PgHdr)); + memset(&pPgHdr->pDirty, 0, sizeof(PgHdr) - offsetof(PgHdr,pDirty)); pPgHdr->pPage = pPage; pPgHdr->pData = pPage->pBuf; pPgHdr->pExtra = (void *)&pPgHdr[1]; - memset(pPgHdr->pExtra, 0, pCache->szExtra); + memset(pPgHdr->pExtra, 0, 8); + assert( EIGHT_BYTE_ALIGNMENT( pPgHdr->pExtra ) ); pPgHdr->pCache = pCache; pPgHdr->pgno = pgno; pPgHdr->flags = PGHDR_CLEAN; @@ -43614,12 +54863,9 @@ SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3PcacheRelease(PgHdr *p){ if( (--p->nRef)==0 ){ if( p->flags&PGHDR_CLEAN ){ pcacheUnpin(p); - }else if( p->pDirtyPrev!=0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* Move the page to the head of the dirty list. If p->pDirtyPrev==0, - ** then page p is already at the head of the dirty list and the - ** following call would be a no-op. Hence the OPTIMIZATION-IF-FALSE - ** tag above. */ + }else{ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } } @@ -43663,6 +54909,7 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ pcacheTrace(("%p.DIRTY %d\n",p->pCache,p->pgno)); assert( (p->flags & (PGHDR_DIRTY|PGHDR_CLEAN))==PGHDR_DIRTY ); pcacheManageDirtyList(p, PCACHE_DIRTYLIST_ADD); + assert( sqlite3PcachePageSanity(p) ); } assert( sqlite3PcachePageSanity(p) ); } @@ -43674,16 +54921,15 @@ SQLITE_PRIVATE void sqlite3PcacheMakeDirty(PgHdr *p){ */ SQLITE_PRIVATE void sqlite3PcacheMakeClean(PgHdr *p){ assert( sqlite3PcachePageSanity(p) ); - if( ALWAYS((p->flags & PGHDR_DIRTY)!=0) ){ - assert( (p->flags & PGHDR_CLEAN)==0 ); - pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); - p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); - p->flags |= PGHDR_CLEAN; - pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); - assert( sqlite3PcachePageSanity(p) ); - if( p->nRef==0 ){ - pcacheUnpin(p); - } + assert( (p->flags & PGHDR_DIRTY)!=0 ); + assert( (p->flags & PGHDR_CLEAN)==0 ); + pcacheManageDirtyList(p, PCACHE_DIRTYLIST_REMOVE); + p->flags &= ~(PGHDR_DIRTY|PGHDR_NEED_SYNC|PGHDR_WRITEABLE); + p->flags |= PGHDR_CLEAN; + pcacheTrace(("%p.CLEAN %d\n",p->pCache,p->pgno)); + assert( sqlite3PcachePageSanity(p) ); + if( p->nRef==0 ){ + pcacheUnpin(p); } } @@ -43722,18 +54968,28 @@ SQLITE_PRIVATE void sqlite3PcacheClearSyncFlags(PCache *pCache){ } /* -** Change the page number of page p to newPgno. +** Change the page number of page p to newPgno. */ SQLITE_PRIVATE void sqlite3PcacheMove(PgHdr *p, Pgno newPgno){ PCache *pCache = p->pCache; + sqlite3_pcache_page *pOther; assert( p->nRef>0 ); assert( newPgno>0 ); assert( sqlite3PcachePageSanity(p) ); pcacheTrace(("%p.MOVE %d -> %d\n",pCache,p->pgno,newPgno)); + pOther = sqlite3GlobalConfig.pcache2.xFetch(pCache->pCache, newPgno, 0); + if( pOther ){ + PgHdr *pXPage = (PgHdr*)pOther->pExtra; + assert( pXPage->nRef==0 ); + pXPage->nRef++; + pCache->nRefSum++; + sqlite3PcacheDrop(pXPage); + } sqlite3GlobalConfig.pcache2.xRekey(pCache->pCache, p->pPage, p->pgno,newPgno); p->pgno = newPgno; if( (p->flags&PGHDR_DIRTY) && (p->flags&PGHDR_NEED_SYNC) ){ pcacheManageDirtyList(p, PCACHE_DIRTYLIST_FRONT); + assert( sqlite3PcachePageSanity(p) ); } } @@ -43785,7 +55041,7 @@ SQLITE_PRIVATE void sqlite3PcacheClose(PCache *pCache){ sqlite3GlobalConfig.pcache2.xDestroy(pCache->pCache); } -/* +/* ** Discard the contents of the cache. */ SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ @@ -43794,34 +55050,36 @@ SQLITE_PRIVATE void sqlite3PcacheClear(PCache *pCache){ /* ** Merge two lists of pages connected by pDirty and in pgno order. -** Do not both fixing the pDirtyPrev pointers. +** Do not bother fixing the pDirtyPrev pointers. */ static PgHdr *pcacheMergeDirtyList(PgHdr *pA, PgHdr *pB){ PgHdr result, *pTail; pTail = &result; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ if( pA->pgnopgno ){ pTail->pDirty = pA; pTail = pA; pA = pA->pDirty; + if( pA==0 ){ + pTail->pDirty = pB; + break; + } }else{ pTail->pDirty = pB; pTail = pB; pB = pB->pDirty; + if( pB==0 ){ + pTail->pDirty = pA; + break; + } } } - if( pA ){ - pTail->pDirty = pA; - }else if( pB ){ - pTail->pDirty = pB; - }else{ - pTail->pDirty = 0; - } return result.pDirty; } /* -** Sort the list of pages in accending order by pgno. Pages are +** Sort the list of pages in ascending order by pgno. Pages are ** connected by pDirty pointers. The pDirtyPrev pointers are ** corrupted by this sort. ** @@ -43857,7 +55115,8 @@ static PgHdr *pcacheSortDirtyList(PgHdr *pIn){ } p = a[0]; for(i=1; ipDirty); } -/* +/* ** Return the total number of references to all pages held by the cache. ** ** This is not the total number of pages referenced, but the sum of the ** reference count for all pages. */ -SQLITE_PRIVATE int sqlite3PcacheRefCount(PCache *pCache){ +SQLITE_PRIVATE i64 sqlite3PcacheRefCount(PCache *pCache){ return pCache->nRefSum; } /* ** Return the number of references to the page supplied as an argument. */ -SQLITE_PRIVATE int sqlite3PcachePageRefcount(PgHdr *p){ +SQLITE_PRIVATE i64 sqlite3PcachePageRefcount(PgHdr *p){ return p->nRef; } -/* +/* ** Return the total number of pages in the cache. */ SQLITE_PRIVATE int sqlite3PcachePagecount(PCache *pCache){ @@ -43932,7 +55191,7 @@ SQLITE_PRIVATE int sqlite3PcacheSetSpillsize(PCache *p, int mxPage){ p->szSpill = mxPage; } res = numberOfCachePages(p); - if( resszSpill ) res = p->szSpill; + if( resszSpill ) res = p->szSpill; return res; } @@ -43962,6 +55221,15 @@ SQLITE_PRIVATE int sqlite3PCachePercentDirty(PCache *pCache){ return nCache ? (int)(((i64)nDirty * 100) / nCache) : 0; } +#ifdef SQLITE_DIRECT_OVERFLOW_READ +/* +** Return true if there are one or more dirty pages in the cache. Else false. +*/ +SQLITE_PRIVATE int sqlite3PCacheIsDirty(PCache *pCache){ + return (pCache->pDirty!=0); +} +#endif + #if defined(SQLITE_CHECK_PAGES) || defined(SQLITE_DEBUG) /* ** For all dirty pages currently in the cache, invoke the specified @@ -44019,12 +55287,13 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** size can vary according to architecture, compile-time options, and ** SQLite library version number. ** -** If SQLITE_PCACHE_SEPARATE_HEADER is defined, then the extension is obtained -** using a separate memory allocation from the database page content. This -** seeks to overcome the "clownshoe" problem (also called "internal -** fragmentation" in academic literature) of allocating a few bytes more -** than a power of two with the memory allocator rounding up to the next -** power of two, and leaving the rounded-up space unused. +** Historical note: It used to be that if the SQLITE_PCACHE_SEPARATE_HEADER +** was defined, then the page content would be held in a separate memory +** allocation from the PgHdr1. This was intended to avoid clownshoe memory +** allocations. However, the btree layer needs a small (16-byte) overrun +** area after the page content buffer. The header serves as that overrun +** area. Therefore SQLITE_PCACHE_SEPARATE_HEADER was discontinued to avoid +** any possibility of a memory error. ** ** This module tracks pointers to PgHdr1 objects. Only pcache.c communicates ** with this module. Information is passed back and forth as PgHdr1 pointers. @@ -44043,14 +55312,14 @@ SQLITE_PRIVATE void sqlite3PcacheIterateDirty(PCache *pCache, void (*xIter)(PgHd ** ** The third case is a chunk of heap memory (defaulting to 100 pages worth) ** that is allocated when the page cache is created. The size of the local -** bulk allocation can be adjusted using +** bulk allocation can be adjusted using ** ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, (void*)0, 0, N). ** ** If N is positive, then N pages worth of memory are allocated using a single ** sqlite3Malloc() call and that memory is used for the first N pages allocated. ** Or if N is negative, then -1024*N bytes of memory are allocated and used -** for as many pages as can be accomodated. +** for as many pages as can be accommodated. ** ** Only one of (2) or (3) can be used. Once the memory available to (2) or ** (3) is exhausted, subsequent allocations fail over to the general-purpose @@ -44068,24 +55337,51 @@ typedef struct PgFreeslot PgFreeslot; typedef struct PGroup PGroup; /* -** Each cache entry is represented by an instance of the following -** structure. Unless SQLITE_PCACHE_SEPARATE_HEADER is defined, a buffer of -** PgHdr1.pCache->szPage bytes is allocated directly before this structure -** in memory. +** Each cache entry is represented by an instance of the following +** structure. A buffer of PgHdr1.pCache->szPage bytes is allocated +** directly before this structure and is used to cache the page content. +** +** When reading a corrupt database file, it is possible that SQLite might +** read a few bytes (no more than 16 bytes) past the end of the page buffer. +** It will only read past the end of the page buffer, never write. This +** object is positioned immediately after the page buffer to serve as an +** overrun area, so that overreads are harmless. +** +** Variables isBulkLocal and isAnchor were once type "u8". That works, +** but causes a 2-byte gap in the structure for most architectures (since +** pointers must be either 4 or 8-byte aligned). As this structure is located +** in memory directly after the associated page data, if the database is +** corrupt, code at the b-tree layer may overread the page buffer and +** read part of this structure before the corruption is detected. This +** can cause a valgrind error if the uninitialized gap is accessed. Using u16 +** ensures there is no such gap, and therefore no bytes of uninitialized +** memory in the structure. +** +** The pLruNext and pLruPrev pointers form a double-linked circular list +** of all pages that are unpinned. The PGroup.lru element (which should be +** the only element on the list with PgHdr1.isAnchor set to 1) forms the +** beginning and the end of the list. */ struct PgHdr1 { - sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ - unsigned int iKey; /* Key value (page number) */ - u8 isPinned; /* Page in use, not on the LRU list */ - u8 isBulkLocal; /* This page from bulk local storage */ - u8 isAnchor; /* This is the PGroup.lru element */ - PgHdr1 *pNext; /* Next in hash table chain */ - PCache1 *pCache; /* Cache that currently owns this page */ - PgHdr1 *pLruNext; /* Next in LRU list of unpinned pages */ - PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + sqlite3_pcache_page page; /* Base class. Must be first. pBuf & pExtra */ + unsigned int iKey; /* Key value (page number) */ + u16 isBulkLocal; /* This page from bulk local storage */ + u16 isAnchor; /* This is the PGroup.lru element */ + PgHdr1 *pNext; /* Next in hash table chain */ + PCache1 *pCache; /* Cache that currently owns this page */ + PgHdr1 *pLruNext; /* Next in circular LRU list of unpinned pages */ + PgHdr1 *pLruPrev; /* Previous in LRU list of unpinned pages */ + /* NB: pLruPrev is only valid if pLruNext!=0 */ }; -/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set +/* +** A page is pinned if it is not on the LRU list. To be "pinned" means +** that the page is in active use and must not be deallocated. +*/ +#define PAGE_IS_PINNED(p) ((p)->pLruNext==0) +#define PAGE_IS_UNPINNED(p) ((p)->pLruNext!=0) + +/* Each page cache (or PCache) belongs to a PGroup. A PGroup is a set ** of one or more PCaches that are able to recycle each other's unpinned ** pages when they are under memory pressure. A PGroup is an instance of ** the following object. @@ -44112,7 +55408,7 @@ struct PGroup { unsigned int nMaxPage; /* Sum of nMax for purgeable caches */ unsigned int nMinPage; /* Sum of nMin for purgeable caches */ unsigned int mxPinned; /* nMaxpage + 10 - nMinPage */ - unsigned int nCurrentPage; /* Number of purgeable pages allocated */ + unsigned int nPurgeable; /* Number of purgeable pages allocated */ PgHdr1 lru; /* The beginning and end of the LRU list */ }; @@ -44121,16 +55417,18 @@ struct PGroup { ** temporary or transient database) has a single page cache which ** is an instance of this object. ** -** Pointers to structures of this type are cast and returned as +** Pointers to structures of this type are cast and returned as ** opaque sqlite3_pcache* handles. */ struct PCache1 { /* Cache configuration parameters. Page size (szPage) and the purgeable - ** flag (bPurgeable) are set when the cache is created. nMax may be + ** flag (bPurgeable) and the pnPurgeable pointer are all set when the + ** cache is created and are never changed thereafter. nMax may be ** modified at any time by a call to the pcache1Cachesize() method. ** The PGroup mutex must be held when accessing nMax. */ PGroup *pGroup; /* PGroup this cache belongs to */ + unsigned int *pnPurgeable; /* Pointer to pGroup->nPurgeable */ int szPage; /* Size of database content section */ int szExtra; /* sizeof(MemPage)+sizeof(PgHdr) */ int szAlloc; /* Total size of one pcache line */ @@ -44139,6 +55437,7 @@ struct PCache1 { unsigned int nMax; /* Configured "cache_size" value */ unsigned int n90pct; /* nMax*9/10 */ unsigned int iMaxKey; /* Largest key seen since xTruncate() */ + unsigned int nPurgeableDummy; /* pnPurgeable points here when not used*/ /* Hash table of all pages. The following variables may only be accessed ** when the accessor is holding the PGroup mutex. @@ -44172,7 +55471,7 @@ static SQLITE_WSD struct PCacheGlobal { */ int isInit; /* True if initialized */ int separateCache; /* Use a new PGroup for each PCache */ - int nInitPage; /* Initial bulk allocation size */ + int nInitPage; /* Initial bulk allocation size */ int szSlot; /* Size of each free slot */ int nSlot; /* The number of pcache slots */ int nReserve; /* Try to keep nFreeSlot above this */ @@ -44213,7 +55512,7 @@ static SQLITE_WSD struct PCacheGlobal { /* -** This function is called during initialization if a static buffer is +** This function is called during initialization if a static buffer is ** supplied to use for the page-cache by passing the SQLITE_CONFIG_PAGECACHE ** verb to sqlite3_config(). Parameter pBuf points to an allocation large ** enough to contain 'n' buffers of 'sz' bytes each. @@ -44225,6 +55524,7 @@ SQLITE_PRIVATE void sqlite3PCacheBufferSetup(void *pBuf, int sz, int n){ if( pcache1.isInit ){ PgFreeslot *p; if( pBuf==0 ) sz = n = 0; + if( n==0 ) sz = 0; sz = ROUNDDOWN8(sz); pcache1.szSlot = sz; pcache1.nSlot = pcache1.nFreeSlot = n; @@ -44259,31 +55559,32 @@ static int pcache1InitBulk(PCache1 *pCache){ szBulk = -1024 * (i64)pcache1.nInitPage; } if( szBulk > pCache->szAlloc*(i64)pCache->nMax ){ - szBulk = pCache->szAlloc*pCache->nMax; + szBulk = pCache->szAlloc*(i64)pCache->nMax; } zBulk = pCache->pBulk = sqlite3Malloc( szBulk ); sqlite3EndBenignMalloc(); if( zBulk ){ int nBulk = sqlite3MallocSize(zBulk)/pCache->szAlloc; - int i; - for(i=0; iszPage]; pX->page.pBuf = zBulk; - pX->page.pExtra = &pX[1]; + pX->page.pExtra = (u8*)pX + ROUND8(sizeof(*pX)); + assert( EIGHT_BYTE_ALIGNMENT( pX->page.pExtra ) ); pX->isBulkLocal = 1; pX->isAnchor = 0; pX->pNext = pCache->pFree; + pX->pLruPrev = 0; /* Initializing this saves a valgrind error */ pCache->pFree = pX; zBulk += pCache->szAlloc; - } + }while( --nBulk ); } return pCache->pFree!=0; } /* ** Malloc function used within this file to allocate space from the buffer -** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no -** such buffer exists or there is no space left in it, this function falls +** configured using sqlite3_config(SQLITE_CONFIG_PAGECACHE) option. If no +** such buffer exists or there is no space left in it, this function falls ** back to sqlite3Malloc(). ** ** Multiple threads can run this routine at the same time. Global variables @@ -44383,44 +55684,35 @@ static PgHdr1 *pcache1AllocPage(PCache1 *pCache, int benignMalloc){ assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); if( pCache->pFree || (pCache->nPage==0 && pcache1InitBulk(pCache)) ){ + assert( pCache->pFree!=0 ); p = pCache->pFree; pCache->pFree = p->pNext; p->pNext = 0; }else{ #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT /* The group mutex must be released before pcache1Alloc() is called. This - ** is because it might call sqlite3_release_memory(), which assumes that + ** is because it might call sqlite3_release_memory(), which assumes that ** this mutex is not held. */ assert( pcache1.separateCache==0 ); assert( pCache->pGroup==&pcache1.grp ); pcache1LeaveMutex(pCache->pGroup); #endif if( benignMalloc ){ sqlite3BeginBenignMalloc(); } -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - pPg = pcache1Alloc(pCache->szPage); - p = sqlite3Malloc(sizeof(PgHdr1) + pCache->szExtra); - if( !pPg || !p ){ - pcache1Free(pPg); - sqlite3_free(p); - pPg = 0; - } -#else pPg = pcache1Alloc(pCache->szAlloc); - p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; -#endif if( benignMalloc ){ sqlite3EndBenignMalloc(); } #ifdef SQLITE_ENABLE_MEMORY_MANAGEMENT pcache1EnterMutex(pCache->pGroup); #endif if( pPg==0 ) return 0; + p = (PgHdr1 *)&((u8 *)pPg)[pCache->szPage]; p->page.pBuf = pPg; - p->page.pExtra = &p[1]; + p->page.pExtra = (u8*)p + ROUND8(sizeof(*p)); + assert( EIGHT_BYTE_ALIGNMENT( p->page.pExtra ) ); p->isBulkLocal = 0; p->isAnchor = 0; + p->pLruPrev = 0; /* Initializing this saves a valgrind error */ } - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage++; - } + (*pCache->pnPurgeable)++; return p; } @@ -44437,13 +55729,8 @@ static void pcache1FreePage(PgHdr1 *p){ pCache->pFree = p; }else{ pcache1Free(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - sqlite3_free(p); -#endif - } - if( pCache->bPurgeable ){ - pCache->pGroup->nCurrentPage--; } + (*pCache->pnPurgeable)--; } /* @@ -44452,6 +55739,7 @@ static void pcache1FreePage(PgHdr1 *p){ ** exists, this function falls back to sqlite3Malloc(). */ SQLITE_PRIVATE void *sqlite3PageMalloc(int sz){ + assert( sz<=65536+8 ); /* These allocations are never very large */ return pcache1Alloc(sz); } @@ -44531,35 +55819,32 @@ static void pcache1ResizeHash(PCache1 *p){ } /* -** This function is used internally to remove the page pPage from the +** This function is used internally to remove the page pPage from the ** PGroup LRU list, if is part of it. If pPage is not part of the PGroup ** LRU list, then this function is a no-op. ** ** The PGroup mutex must be held when this function is called. */ static PgHdr1 *pcache1PinPage(PgHdr1 *pPage){ - PCache1 *pCache; - assert( pPage!=0 ); - assert( pPage->isPinned==0 ); - pCache = pPage->pCache; + assert( PAGE_IS_UNPINNED(pPage) ); assert( pPage->pLruNext ); assert( pPage->pLruPrev ); - assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); + assert( sqlite3_mutex_held(pPage->pCache->pGroup->mutex) ); pPage->pLruPrev->pLruNext = pPage->pLruNext; pPage->pLruNext->pLruPrev = pPage->pLruPrev; pPage->pLruNext = 0; - pPage->pLruPrev = 0; - pPage->isPinned = 1; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev as it is never accessed if pLruNext is 0 */ assert( pPage->isAnchor==0 ); - assert( pCache->pGroup->lru.isAnchor==1 ); - pCache->nRecyclable--; + assert( pPage->pCache->pGroup->lru.isAnchor==1 ); + pPage->pCache->nRecyclable--; return pPage; } /* -** Remove the page supplied as an argument from the hash table +** Remove the page supplied as an argument from the hash table ** (PCache1.apHash structure) that it is currently stored in. ** Also free the page if freePage is true. ** @@ -44587,11 +55872,11 @@ static void pcache1EnforceMaxPage(PCache1 *pCache){ PGroup *pGroup = pCache->pGroup; PgHdr1 *p; assert( sqlite3_mutex_held(pGroup->mutex) ); - while( pGroup->nCurrentPage>pGroup->nMaxPage + while( pGroup->nPurgeable>pGroup->nMaxPage && (p=pGroup->lru.pLruPrev)->isAnchor==0 ){ assert( p->pCache->pGroup==pGroup ); - assert( p->isPinned==0 ); + assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); } @@ -44602,8 +55887,8 @@ static void pcache1EnforceMaxPage(PCache1 *pCache){ } /* -** Discard all pages from cache pCache with a page number (key value) -** greater than or equal to iLimit. Any pinned pages that meet this +** Discard all pages from cache pCache with a page number (key value) +** greater than or equal to iLimit. Any pinned pages that meet this ** criteria are unpinned before they are discarded. ** ** The PCache mutex must be held when this function is called. @@ -44612,25 +55897,45 @@ static void pcache1TruncateUnsafe( PCache1 *pCache, /* The cache to truncate */ unsigned int iLimit /* Drop pages with this pgno or larger */ ){ - TESTONLY( unsigned int nPage = 0; ) /* To assert pCache->nPage is correct */ - unsigned int h; + TESTONLY( int nPage = 0; ) /* To assert pCache->nPage is correct */ + unsigned int h, iStop; assert( sqlite3_mutex_held(pCache->pGroup->mutex) ); - for(h=0; hnHash; h++){ - PgHdr1 **pp = &pCache->apHash[h]; + assert( pCache->iMaxKey >= iLimit ); + assert( pCache->nHash > 0 ); + if( pCache->iMaxKey - iLimit < pCache->nHash ){ + /* If we are just shaving the last few pages off the end of the + ** cache, then there is no point in scanning the entire hash table. + ** Only scan those hash slots that might contain pages that need to + ** be removed. */ + h = iLimit % pCache->nHash; + iStop = pCache->iMaxKey % pCache->nHash; + TESTONLY( nPage = -10; ) /* Disable the pCache->nPage validity check */ + }else{ + /* This is the general case where many pages are being removed. + ** It is necessary to scan the entire hash table */ + h = pCache->nHash/2; + iStop = h - 1; + } + for(;;){ + PgHdr1 **pp; PgHdr1 *pPage; + assert( hnHash ); + pp = &pCache->apHash[h]; while( (pPage = *pp)!=0 ){ if( pPage->iKey>=iLimit ){ pCache->nPage--; *pp = pPage->pNext; - if( !pPage->isPinned ) pcache1PinPage(pPage); + if( PAGE_IS_UNPINNED(pPage) ) pcache1PinPage(pPage); pcache1FreePage(pPage); }else{ pp = &pPage->pNext; - TESTONLY( nPage++; ) + TESTONLY( if( nPage>=0 ) nPage++; ) } } + if( h==iStop ) break; + h = (h+1) % pCache->nHash; } - assert( pCache->nPage==nPage ); + assert( nPage<0 || pCache->nPage==(unsigned)nPage ); } /******************************************************************************/ @@ -44654,7 +55959,7 @@ static int pcache1Init(void *NotUsed){ ** ** * Use a unified cache in single-threaded applications that have ** configured a start-time buffer for use as page-cache memory using - ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL + ** sqlite3_config(SQLITE_CONFIG_PAGECACHE, pBuf, sz, N) with non-NULL ** pBuf argument. ** ** * Otherwise use separate caches (mode-1) @@ -44689,7 +55994,7 @@ static int pcache1Init(void *NotUsed){ /* ** Implementation of the sqlite3_pcache.xShutdown method. -** Note that the static mutex allocated in xInit does +** Note that the static mutex allocated in xInit does ** not need to be freed. */ static void pcache1Shutdown(void *NotUsed){ @@ -44723,6 +56028,7 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ }else{ pGroup = &pcache1.grp; } + pcache1EnterMutex(pGroup); if( pGroup->lru.isAnchor==0 ){ pGroup->lru.isAnchor = 1; pGroup->lru.pLruPrev = pGroup->lru.pLruNext = &pGroup->lru; @@ -44732,12 +56038,14 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ pCache->szExtra = szExtra; pCache->szAlloc = szPage + szExtra + ROUND8(sizeof(PgHdr1)); pCache->bPurgeable = (bPurgeable ? 1 : 0); - pcache1EnterMutex(pGroup); pcache1ResizeHash(pCache); if( bPurgeable ){ pCache->nMin = 10; pGroup->nMinPage += pCache->nMin; pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; + pCache->pnPurgeable = &pGroup->nPurgeable; + }else{ + pCache->pnPurgeable = &pCache->nPurgeableDummy; } pcache1LeaveMutex(pGroup); if( pCache->nHash==0 ){ @@ -44749,18 +56057,24 @@ static sqlite3_pcache *pcache1Create(int szPage, int szExtra, int bPurgeable){ } /* -** Implementation of the sqlite3_pcache.xCachesize method. +** Implementation of the sqlite3_pcache.xCachesize method. ** ** Configure the cache_size limit for a cache. */ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ PCache1 *pCache = (PCache1 *)p; + u32 n; + assert( nMax>=0 ); if( pCache->bPurgeable ){ PGroup *pGroup = pCache->pGroup; pcache1EnterMutex(pGroup); - pGroup->nMaxPage += (nMax - pCache->nMax); + n = (u32)nMax; + if( n > 0x7fff0000 - pGroup->nMaxPage + pCache->nMax ){ + n = 0x7fff0000 - pGroup->nMaxPage + pCache->nMax; + } + pGroup->nMaxPage += (n - pCache->nMax); pGroup->mxPinned = pGroup->nMaxPage + 10 - pGroup->nMinPage; - pCache->nMax = nMax; + pCache->nMax = n; pCache->n90pct = pCache->nMax*9/10; pcache1EnforceMaxPage(pCache); pcache1LeaveMutex(pGroup); @@ -44768,7 +56082,7 @@ static void pcache1Cachesize(sqlite3_pcache *p, int nMax){ } /* -** Implementation of the sqlite3_pcache.xShrink method. +** Implementation of the sqlite3_pcache.xShrink method. ** ** Free up as much memory as possible. */ @@ -44776,7 +56090,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ PCache1 *pCache = (PCache1*)p; if( pCache->bPurgeable ){ PGroup *pGroup = pCache->pGroup; - int savedMaxPage; + unsigned int savedMaxPage; pcache1EnterMutex(pGroup); savedMaxPage = pGroup->nMaxPage; pGroup->nMaxPage = 0; @@ -44787,7 +56101,7 @@ static void pcache1Shrink(sqlite3_pcache *p){ } /* -** Implementation of the sqlite3_pcache.xPagecount method. +** Implementation of the sqlite3_pcache.xPagecount method. */ static int pcache1Pagecount(sqlite3_pcache *p){ int n; @@ -44808,8 +56122,8 @@ static int pcache1Pagecount(sqlite3_pcache *p){ ** for these steps, the main pcache1Fetch() procedure can run faster. */ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( - PCache1 *pCache, - unsigned int iKey, + PCache1 *pCache, + unsigned int iKey, int createFlag ){ unsigned int nPinned; @@ -44839,7 +56153,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ){ PCache1 *pOther; pPage = pGroup->lru.pLruPrev; - assert( pPage->isPinned==0 ); + assert( PAGE_IS_UNPINNED(pPage) ); pcache1RemoveFromHash(pPage, 0); pcache1PinPage(pPage); pOther = pPage->pCache; @@ -44847,12 +56161,12 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( pcache1FreePage(pPage); pPage = 0; }else{ - pGroup->nCurrentPage -= (pOther->bPurgeable - pCache->bPurgeable); + pGroup->nPurgeable -= (pOther->bPurgeable - pCache->bPurgeable); } } - /* Step 5. If a usable page buffer has still not been found, - ** attempt to allocate a new one. + /* Step 5. If a usable page buffer has still not been found, + ** attempt to allocate a new one. */ if( !pPage ){ pPage = pcache1AllocPage(pCache, createFlag==1); @@ -44864,9 +56178,9 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( pPage->iKey = iKey; pPage->pNext = pCache->apHash[h]; pPage->pCache = pCache; - pPage->pLruPrev = 0; pPage->pLruNext = 0; - pPage->isPinned = 1; + /* pPage->pLruPrev = 0; + ** No need to clear pLruPrev since it is not accessed when pLruNext==0 */ *(void **)pPage->page.pExtra = 0; pCache->apHash[h] = pPage; if( iKey>pCache->iMaxKey ){ @@ -44877,13 +56191,13 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( } /* -** Implementation of the sqlite3_pcache.xFetch method. +** Implementation of the sqlite3_pcache.xFetch method. ** ** Fetch a page by key value. ** ** Whether or not a new page may be allocated by this function depends on ** the value of the createFlag argument. 0 means do not allocate a new -** page. 1 means allocate a new page if space is easily available. 2 +** page. 1 means allocate a new page if space is easily available. 2 ** means to try really hard to allocate a new page. ** ** For a non-purgeable cache (a cache used as the storage for an in-memory @@ -44894,7 +56208,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** There are three different approaches to obtaining space for a page, ** depending on the value of parameter createFlag (which may be 0, 1 or 2). ** -** 1. Regardless of the value of createFlag, the cache is searched for a +** 1. Regardless of the value of createFlag, the cache is searched for a ** copy of the requested page. If one is found, it is returned. ** ** 2. If createFlag==0 and the page is not already in the cache, NULL is @@ -44908,13 +56222,13 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** PCache1.nMax, or ** ** (b) the number of pages pinned by the cache is greater than -** the sum of nMax for all purgeable caches, less the sum of +** the sum of nMax for all purgeable caches, less the sum of ** nMin for all other purgeable caches, or ** ** 4. If none of the first three conditions apply and the cache is marked ** as purgeable, and if one of the following is true: ** -** (a) The number of pages allocated for the cache is already +** (a) The number of pages allocated for the cache is already ** PCache1.nMax, or ** ** (b) The number of pages allocated for all purgeable caches is @@ -44926,7 +56240,7 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** ** then attempt to recycle a page from the LRU list. If it is the right ** size, return the recycled buffer. Otherwise, free the buffer and -** proceed to step 5. +** proceed to step 5. ** ** 5. Otherwise, allocate and return a new page buffer. ** @@ -44936,8 +56250,8 @@ static SQLITE_NOINLINE PgHdr1 *pcache1FetchStage2( ** invokes the appropriate routine. */ static PgHdr1 *pcache1FetchNoMutex( - sqlite3_pcache *p, - unsigned int iKey, + sqlite3_pcache *p, + unsigned int iKey, int createFlag ){ PCache1 *pCache = (PCache1 *)p; @@ -44952,7 +56266,7 @@ static PgHdr1 *pcache1FetchNoMutex( ** Otherwise (page not in hash and createFlag!=0) continue with ** subsequent steps to try to create the page. */ if( pPage ){ - if( !pPage->isPinned ){ + if( PAGE_IS_UNPINNED(pPage) ){ return pcache1PinPage(pPage); }else{ return pPage; @@ -44966,8 +56280,8 @@ static PgHdr1 *pcache1FetchNoMutex( } #if PCACHE1_MIGHT_USE_GROUP_MUTEX static PgHdr1 *pcache1FetchWithMutex( - sqlite3_pcache *p, - unsigned int iKey, + sqlite3_pcache *p, + unsigned int iKey, int createFlag ){ PCache1 *pCache = (PCache1 *)p; @@ -44981,8 +56295,8 @@ static PgHdr1 *pcache1FetchWithMutex( } #endif static sqlite3_pcache_page *pcache1Fetch( - sqlite3_pcache *p, - unsigned int iKey, + sqlite3_pcache *p, + unsigned int iKey, int createFlag ){ #if PCACHE1_MIGHT_USE_GROUP_MUTEX || defined(SQLITE_DEBUG) @@ -45012,24 +56326,24 @@ static sqlite3_pcache_page *pcache1Fetch( ** Mark a page as unpinned (eligible for asynchronous recycling). */ static void pcache1Unpin( - sqlite3_pcache *p, - sqlite3_pcache_page *pPg, + sqlite3_pcache *p, + sqlite3_pcache_page *pPg, int reuseUnlikely ){ PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PGroup *pGroup = pCache->pGroup; - + assert( pPage->pCache==pCache ); pcache1EnterMutex(pGroup); - /* It is an error to call this function if the page is already + /* It is an error to call this function if the page is already ** part of the PGroup LRU list. */ - assert( pPage->pLruPrev==0 && pPage->pLruNext==0 ); - assert( pPage->isPinned==1 ); + assert( pPage->pLruNext==0 ); + assert( PAGE_IS_PINNED(pPage) ); - if( reuseUnlikely || pGroup->nCurrentPage>pGroup->nMaxPage ){ + if( reuseUnlikely || pGroup->nPurgeable>pGroup->nMaxPage ){ pcache1RemoveFromHash(pPage, 1); }else{ /* Add the page to the PGroup LRU list. */ @@ -45038,14 +56352,13 @@ static void pcache1Unpin( (pPage->pLruNext = *ppFirst)->pLruPrev = pPage; *ppFirst = pPage; pCache->nRecyclable++; - pPage->isPinned = 0; } pcache1LeaveMutex(pCache->pGroup); } /* -** Implementation of the sqlite3_pcache.xRekey method. +** Implementation of the sqlite3_pcache.xRekey method. */ static void pcache1Rekey( sqlite3_pcache *p, @@ -45056,23 +56369,26 @@ static void pcache1Rekey( PCache1 *pCache = (PCache1 *)p; PgHdr1 *pPage = (PgHdr1 *)pPg; PgHdr1 **pp; - unsigned int h; + unsigned int hOld, hNew; assert( pPage->iKey==iOld ); assert( pPage->pCache==pCache ); + assert( iOld!=iNew ); /* The page number really is changing */ pcache1EnterMutex(pCache->pGroup); - h = iOld%pCache->nHash; - pp = &pCache->apHash[h]; + assert( pcache1FetchNoMutex(p, iOld, 0)==pPage ); /* pPg really is iOld */ + hOld = iOld%pCache->nHash; + pp = &pCache->apHash[hOld]; while( (*pp)!=pPage ){ pp = &(*pp)->pNext; } *pp = pPage->pNext; - h = iNew%pCache->nHash; + assert( pcache1FetchNoMutex(p, iNew, 0)==0 ); /* iNew not in cache */ + hNew = iNew%pCache->nHash; pPage->iKey = iNew; - pPage->pNext = pCache->apHash[h]; - pCache->apHash[h] = pPage; + pPage->pNext = pCache->apHash[hNew]; + pCache->apHash[hNew] = pPage; if( iNew>pCache->iMaxKey ){ pCache->iMaxKey = iNew; } @@ -45081,7 +56397,7 @@ static void pcache1Rekey( } /* -** Implementation of the sqlite3_pcache.xTruncate method. +** Implementation of the sqlite3_pcache.xTruncate method. ** ** Discard all unpinned pages in the cache with a page number equal to ** or greater than parameter iLimit. Any pinned pages with a page number @@ -45098,7 +56414,7 @@ static void pcache1Truncate(sqlite3_pcache *p, unsigned int iLimit){ } /* -** Implementation of the sqlite3_pcache.xDestroy method. +** Implementation of the sqlite3_pcache.xDestroy method. ** ** Destroy a cache allocated using pcache1Create(). */ @@ -45107,7 +56423,7 @@ static void pcache1Destroy(sqlite3_pcache *p){ PGroup *pGroup = pCache->pGroup; assert( pCache->bPurgeable || (pCache->nMax==0 && pCache->nMin==0) ); pcache1EnterMutex(pGroup); - pcache1TruncateUnsafe(pCache, 0); + if( pCache->nPage ) pcache1TruncateUnsafe(pCache, 0); assert( pGroup->nMaxPage >= pCache->nMax ); pGroup->nMaxPage -= pCache->nMax; assert( pGroup->nMinPage >= pCache->nMin ); @@ -45164,14 +56480,14 @@ SQLITE_PRIVATE sqlite3_mutex *sqlite3Pcache1Mutex(void){ ** by the current thread may be sqlite3_free()ed. ** ** nReq is the number of bytes of memory required. Once this much has -** been released, the function returns. The return value is the total number +** been released, the function returns. The return value is the total number ** of bytes of memory released. */ SQLITE_PRIVATE int sqlite3PcacheReleaseMemory(int nReq){ int nFree = 0; assert( sqlite3_mutex_notheld(pcache1.grp.mutex) ); assert( sqlite3_mutex_notheld(pcache1.mutex) ); - if( sqlite3GlobalConfig.nPage==0 ){ + if( sqlite3GlobalConfig.pPage==0 ){ PgHdr1 *p; pcache1EnterMutex(&pcache1.grp); while( (nReq<0 || nFreeisAnchor==0 ){ nFree += pcache1MemSize(p->page.pBuf); -#ifdef SQLITE_PCACHE_SEPARATE_HEADER - nFree += sqlite3MemSize(p); -#endif - assert( p->isPinned==0 ); + assert( PAGE_IS_UNPINNED(p) ); pcache1PinPage(p); pcache1RemoveFromHash(p, 1); } @@ -45206,10 +56519,10 @@ SQLITE_PRIVATE void sqlite3PcacheStats( PgHdr1 *p; int nRecyclable = 0; for(p=pcache1.grp.lru.pLruNext; p && !p->isAnchor; p=p->pLruNext){ - assert( p->isPinned==0 ); + assert( PAGE_IS_UNPINNED(p) ); nRecyclable++; } - *pnCurrent = pcache1.grp.nCurrentPage; + *pnCurrent = pcache1.grp.nPurgeable; *pnMax = (int)pcache1.grp.nMaxPage; *pnMin = (int)pcache1.grp.nMinPage; *pnRecyclable = nRecyclable; @@ -45255,14 +56568,14 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** extracts the least value from the RowSet. ** ** The INSERT primitive might allocate additional memory. Memory is -** allocated in chunks so most INSERTs do no allocation. There is an +** allocated in chunks so most INSERTs do no allocation. There is an ** upper bound on the size of allocated memory. No memory is freed ** until DESTROY. ** ** The TEST primitive includes a "batch" number. The TEST primitive ** will only see elements that were inserted before the last change ** in the batch number. In other words, if an INSERT occurs between -** two TESTs where the TESTs have the same batch nubmer, then the +** two TESTs where the TESTs have the same batch number, then the ** value added by the INSERT will not be visible to the second TEST. ** The initial batch number is zero, so if the very first TEST contains ** a non-zero batch number, it will see all prior INSERTs. @@ -45303,7 +56616,7 @@ SQLITE_PRIVATE void sqlite3PcacheStats( ** in the list, pLeft points to the tree, and v is unused. The ** RowSet.pForest value points to the head of this forest list. */ -struct RowSetEntry { +struct RowSetEntry { i64 v; /* ROWID value for this entry */ struct RowSetEntry *pRight; /* Right subtree (larger entries) or list */ struct RowSetEntry *pLeft; /* Left subtree (smaller entries) */ @@ -45344,30 +56657,23 @@ struct RowSet { #define ROWSET_NEXT 0x02 /* True if sqlite3RowSetNext() has been called */ /* -** Turn bulk memory into a RowSet object. N bytes of memory -** are available at pSpace. The db pointer is used as a memory context -** for any subsequent allocations that need to occur. -** Return a pointer to the new RowSet object. -** -** It must be the case that N is sufficient to make a Rowset. If not -** an assertion fault occurs. -** -** If N is larger than the minimum, use the surplus as an initial -** allocation of entries available to be filled. +** Allocate a RowSet object. Return NULL if a memory allocation +** error occurs. */ -SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int N){ - RowSet *p; - assert( N >= ROUND8(sizeof(*p)) ); - p = pSpace; - p->pChunk = 0; - p->db = db; - p->pEntry = 0; - p->pLast = 0; - p->pForest = 0; - p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); - p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); - p->rsFlags = ROWSET_SORTED; - p->iBatch = 0; +SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db){ + RowSet *p = sqlite3DbMallocRawNN(db, sizeof(*p)); + if( p ){ + int N = sqlite3DbMallocSize(db, p); + p->pChunk = 0; + p->db = db; + p->pEntry = 0; + p->pLast = 0; + p->pForest = 0; + p->pFresh = (struct RowSetEntry*)(ROUND8(sizeof(*p)) + (char*)p); + p->nFresh = (u16)((N - ROUND8(sizeof(*p)))/sizeof(struct RowSetEntry)); + p->rsFlags = ROWSET_SORTED; + p->iBatch = 0; + } return p; } @@ -45376,7 +56682,8 @@ SQLITE_PRIVATE RowSet *sqlite3RowSetInit(sqlite3 *db, void *pSpace, unsigned int ** the RowSet has allocated over its lifetime. This routine is ** the destructor for the RowSet. */ -SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ +SQLITE_PRIVATE void sqlite3RowSetClear(void *pArg){ + RowSet *p = (RowSet*)pArg; struct RowSetChunk *pChunk, *pNextChunk; for(pChunk=p->pChunk; pChunk; pChunk = pNextChunk){ pNextChunk = pChunk->pNextChunk; @@ -45390,10 +56697,20 @@ SQLITE_PRIVATE void sqlite3RowSetClear(RowSet *p){ p->rsFlags = ROWSET_SORTED; } +/* +** Deallocate all chunks from a RowSet. This frees all memory that +** the RowSet has allocated over its lifetime. This routine is +** the destructor for the RowSet. +*/ +SQLITE_PRIVATE void sqlite3RowSetDelete(void *pArg){ + sqlite3RowSetClear(pArg); + sqlite3DbFree(((RowSet*)pArg)->db, pArg); +} + /* ** Allocate a new RowSetEntry object that is associated with the ** given RowSet. Return a pointer to the new and completely uninitialized -** objected. +** object. ** ** In an OOM situation, the RowSet.db->mallocFailed flag is set and this ** routine returns NULL. @@ -45451,7 +56768,7 @@ SQLITE_PRIVATE void sqlite3RowSetInsert(RowSet *p, i64 rowid){ /* ** Merge two lists of RowSetEntry objects. Remove duplicates. ** -** The input lists are connected via pRight pointers and are +** The input lists are connected via pRight pointers and are ** assumed to each already be in sorted order. */ static struct RowSetEntry *rowSetEntryMerge( @@ -45462,35 +56779,33 @@ static struct RowSetEntry *rowSetEntryMerge( struct RowSetEntry *pTail; pTail = &head; - while( pA && pB ){ + assert( pA!=0 && pB!=0 ); + for(;;){ assert( pA->pRight==0 || pA->v<=pA->pRight->v ); assert( pB->pRight==0 || pB->v<=pB->pRight->v ); - if( pA->vv ){ - pTail->pRight = pA; + if( pA->v<=pB->v ){ + if( pA->vv ) pTail = pTail->pRight = pA; pA = pA->pRight; - pTail = pTail->pRight; - }else if( pB->vv ){ - pTail->pRight = pB; - pB = pB->pRight; - pTail = pTail->pRight; + if( pA==0 ){ + pTail->pRight = pB; + break; + } }else{ - pA = pA->pRight; + pTail = pTail->pRight = pB; + pB = pB->pRight; + if( pB==0 ){ + pTail->pRight = pA; + break; + } } } - if( pA ){ - assert( pA->pRight==0 || pA->v<=pA->pRight->v ); - pTail->pRight = pA; - }else{ - assert( pB==0 || pB->pRight==0 || pB->v<=pB->pRight->v ); - pTail->pRight = pB; - } return head.pRight; } /* ** Sort all elements on the list of RowSetEntry objects into order of ** increasing v. -*/ +*/ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ unsigned int i; struct RowSetEntry *pNext, *aBucket[40]; @@ -45506,9 +56821,10 @@ static struct RowSetEntry *rowSetEntrySort(struct RowSetEntry *pIn){ aBucket[i] = pIn; pIn = pNext; } - pIn = 0; - for(i=0; i1 ){ /*OPTIMIZATION-IF-TRUE*/ /* This branch causes a *balanced* tree to be generated. A valid tree @@ -45670,7 +56986,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 if( p ){ struct RowSetEntry **ppPrevTree = &pRowSet->pForest; if( (pRowSet->rsFlags & ROWSET_SORTED)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - /* Only sort the current set of entiries if they need it */ + /* Only sort the current set of entries if they need it */ p = rowSetEntrySort(p); } for(pTree = pRowSet->pForest; pTree; pTree=pTree->pRight){ @@ -45732,7 +57048,7 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** ************************************************************************* ** This is the implementation of the page cache subsystem or "pager". -** +** ** The pager is used to access a database disk file. It implements ** atomic commit and rollback through the use of a journal file that ** is separate from the database file. The pager also implements file @@ -45755,26 +57071,26 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 ** May you share freely, never taking more than you give. ** ************************************************************************* -** This header file defines the interface to the write-ahead logging -** system. Refer to the comments below and the header comment attached to +** This header file defines the interface to the write-ahead logging +** system. Refer to the comments below and the header comment attached to ** the implementation of each function in log.c for further details. */ -#ifndef _WAL_H_ -#define _WAL_H_ +#ifndef SQLITE_WAL_H +#define SQLITE_WAL_H /* #include "sqliteInt.h" */ -/* Additional values that can be added to the sync_flags argument of -** sqlite3WalFrames(): +/* Macros for extracting appropriate sync flags for either transaction +** commits (WAL_SYNC_FLAGS(X)) or for checkpoint ops (CKPT_SYNC_FLAGS(X)): */ -#define WAL_SYNC_TRANSACTIONS 0x20 /* Sync at the end of each transaction */ -#define SQLITE_SYNC_MASK 0x13 /* Mask off the SQLITE_SYNC_* values */ +#define WAL_SYNC_FLAGS(X) ((X)&0x03) +#define CKPT_SYNC_FLAGS(X) (((X)>>2)&0x03) #ifdef SQLITE_OMIT_WAL # define sqlite3WalOpen(x,y,z) 0 # define sqlite3WalLimit(x,y) -# define sqlite3WalClose(w,x,y,z) 0 +# define sqlite3WalClose(v,w,x,y,z) 0 # define sqlite3WalBeginReadTransaction(y,z) 0 # define sqlite3WalEndReadTransaction(z) # define sqlite3WalDbsize(y) 0 @@ -45784,30 +57100,31 @@ SQLITE_PRIVATE int sqlite3RowSetTest(RowSet *pRowSet, int iBatch, sqlite3_int64 # define sqlite3WalSavepoint(y,z) # define sqlite3WalSavepointUndo(y,z) 0 # define sqlite3WalFrames(u,v,w,x,y,z) 0 -# define sqlite3WalCheckpoint(r,s,t,u,v,w,x,y,z) 0 +# define sqlite3WalCheckpoint(q,r,s,t,u,v,w,x,y,z) 0 # define sqlite3WalCallback(z) 0 # define sqlite3WalExclusiveMode(y,z) 0 # define sqlite3WalHeapMemory(z) 0 # define sqlite3WalFramesize(z) 0 # define sqlite3WalFindFrame(x,y,z) 0 # define sqlite3WalFile(x) 0 +# undef SQLITE_USE_SEH #else #define WAL_SAVEPOINT_NDATA 4 -/* Connection to a write-ahead log (WAL) file. -** There is one object of this type for each pager. +/* Connection to a write-ahead log (WAL) file. +** There is one object of this type for each pager. */ typedef struct Wal Wal; /* Open and close a connection to a write-ahead log. */ SQLITE_PRIVATE int sqlite3WalOpen(sqlite3_vfs*, sqlite3_file*, const char *, int, i64, Wal**); -SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, int sync_flags, int, u8 *); +SQLITE_PRIVATE int sqlite3WalClose(Wal *pWal, sqlite3*, int sync_flags, int, u8 *); /* Set the limiting size of a WAL file. */ SQLITE_PRIVATE void sqlite3WalLimit(Wal*, i64); -/* Used by readers to open (lock) and close (unlock) a snapshot. A +/* Used by readers to open (lock) and close (unlock) a snapshot. A ** snapshot is like a read-transaction. It is the state of the database ** at an instant in time. sqlite3WalOpenSnapshot gets a read lock and ** preserves the current state even if the other threads or processes @@ -45842,9 +57159,10 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData); /* Write a frame or frames to the log. */ SQLITE_PRIVATE int sqlite3WalFrames(Wal *pWal, int, PgHdr *, Pgno, int, int); -/* Copy pages from the log to the database file */ +/* Copy pages from the log to the database file */ SQLITE_PRIVATE int sqlite3WalCheckpoint( Wal *pWal, /* Write-ahead log connection */ + sqlite3 *db, /* Check this handle's interrupt flag */ int eMode, /* One of PASSIVE, FULL and RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ @@ -45869,13 +57187,16 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op); /* Return true if the argument is non-NULL and the WAL module is using ** heap-memory for the wal-index. Otherwise, if the argument is NULL or the -** WAL module is using shared-memory, return false. +** WAL module is using shared-memory, return false. */ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal); #ifdef SQLITE_ENABLE_SNAPSHOT SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot); SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal); +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot); +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal); #endif #ifdef SQLITE_ENABLE_ZIPVFS @@ -45888,8 +57209,17 @@ SQLITE_PRIVATE int sqlite3WalFramesize(Wal *pWal); /* Return the sqlite3_file object for the WAL file */ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock); +SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db); +#endif + +#ifdef SQLITE_USE_SEH +SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal*); +#endif + #endif /* ifndef SQLITE_OMIT_WAL */ -#endif /* _WAL_H_ */ +#endif /* SQLITE_WAL_H */ /************** End of wal.h *************************************************/ /************** Continuing where we left off in pager.c **********************/ @@ -45908,60 +57238,60 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal); ** ** Definition: A page of the database file is said to be "overwriteable" if ** one or more of the following are true about the page: -** +** ** (a) The original content of the page as it was at the beginning of ** the transaction has been written into the rollback journal and ** synced. -** +** ** (b) The page was a freelist leaf page at the start of the transaction. -** +** ** (c) The page number is greater than the largest page that existed in ** the database file at the start of the transaction. -** +** ** (1) A page of the database file is never overwritten unless one of the ** following are true: -** +** ** (a) The page and all other pages on the same sector are overwriteable. -** +** ** (b) The atomic page write optimization is enabled, and the entire ** transaction other than the update of the transaction sequence ** number consists of a single page change. -** +** ** (2) The content of a page written into the rollback journal exactly matches ** both the content in the database when the rollback journal was written ** and the content in the database at the beginning of the current ** transaction. -** +** ** (3) Writes to the database file are an integer multiple of the page size ** in length and are aligned on a page boundary. -** +** ** (4) Reads from the database file are either aligned on a page boundary and ** an integer multiple of the page size in length or are taken from the ** first 100 bytes of the database file. -** +** ** (5) All writes to the database file are synced prior to the rollback journal ** being deleted, truncated, or zeroed. -** -** (6) If a master journal file is used, then all writes to the database file -** are synced prior to the master journal being deleted. -** +** +** (6) If a super-journal file is used, then all writes to the database file +** are synced prior to the super-journal being deleted. +** ** Definition: Two databases (or the same database at two points it time) ** are said to be "logically equivalent" if they give the same answer to ** all queries. Note in particular the content of freelist leaf ** pages can be changed arbitrarily without affecting the logical equivalence ** of the database. -** +** ** (7) At any time, if any subset, including the empty set and the total set, -** of the unsynced changes to a rollback journal are removed and the +** of the unsynced changes to a rollback journal are removed and the ** journal is rolled back, the resulting database file will be logically ** equivalent to the database file at the beginning of the transaction. -** +** ** (8) When a transaction is rolled back, the xTruncate method of the VFS ** is called to restore the database file to the same size it was at ** the beginning of the transaction. (In some VFSes, the xTruncate ** method is a no-op, but that does not change the fact the SQLite will ** invoke it.) -** +** ** (9) Whenever the database file is modified, at least one bit in the range ** of bytes from 24 through 39 inclusive will be changed prior to releasing ** the EXCLUSIVE lock, thus signaling other connections on the same @@ -45994,14 +57324,14 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ /* ** The following two macros are used within the PAGERTRACE() macros above -** to print out file-descriptors. +** to print out file-descriptors. ** ** PAGERID() takes a pointer to a Pager struct as its argument. The ** associated file-descriptor is returned. FILEHANDLEID() takes an sqlite3_file ** struct as its argument. */ -#define PAGERID(p) ((int)(p->fd)) -#define FILEHANDLEID(fd) ((int)fd) +#define PAGERID(p) (SQLITE_PTR_TO_INT(p->fd)) +#define FILEHANDLEID(fd) (SQLITE_PTR_TO_INT(fd)) /* ** The Pager.eState variable stores the current 'state' of a pager. A @@ -46015,7 +57345,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** | | | ** | V | ** |<-------WRITER_LOCKED------> ERROR -** | | ^ +** | | ^ ** | V | ** |<------WRITER_CACHEMOD-------->| ** | | | @@ -46027,7 +57357,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** ** List of state transitions and the C [function] that performs each: -** +** ** OPEN -> READER [sqlite3PagerSharedLock] ** READER -> OPEN [pager_unlock] ** @@ -46039,7 +57369,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** WRITER_*** -> ERROR [pager_error] ** ERROR -> OPEN [pager_unlock] -** +** ** ** OPEN: ** @@ -46053,9 +57383,9 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** READER: ** -** In this state all the requirements for reading the database in +** In this state all the requirements for reading the database in ** rollback (non-WAL) mode are met. Unless the pager is (or recently -** was) in exclusive-locking mode, a user-level read transaction is +** was) in exclusive-locking mode, a user-level read transaction is ** open. The database size is known in this state. ** ** A connection running with locking_mode=normal enters this state when @@ -46065,28 +57395,28 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** this state even after the read-transaction is closed. The only way ** a locking_mode=exclusive connection can transition from READER to OPEN ** is via the ERROR state (see below). -** +** ** * A read transaction may be active (but a write-transaction cannot). ** * A SHARED or greater lock is held on the database file. -** * The dbSize variable may be trusted (even if a user-level read +** * The dbSize variable may be trusted (even if a user-level read ** transaction is not active). The dbOrigSize and dbFileSize variables ** may not be trusted at this point. ** * If the database is a WAL database, then the WAL connection is open. -** * Even if a read-transaction is not open, it is guaranteed that +** * Even if a read-transaction is not open, it is guaranteed that ** there is no hot-journal in the file-system. ** ** WRITER_LOCKED: ** ** The pager moves to this state from READER when a write-transaction -** is first opened on the database. In WRITER_LOCKED state, all locks -** required to start a write-transaction are held, but no actual +** is first opened on the database. In WRITER_LOCKED state, all locks +** required to start a write-transaction are held, but no actual ** modifications to the cache or database have taken place. ** -** In rollback mode, a RESERVED or (if the transaction was opened with +** In rollback mode, a RESERVED or (if the transaction was opened with ** BEGIN EXCLUSIVE) EXCLUSIVE lock is obtained on the database file when -** moving to this state, but the journal file is not written to or opened -** to in this state. If the transaction is committed or rolled back while -** in WRITER_LOCKED state, all that is required is to unlock the database +** moving to this state, but the journal file is not written to or opened +** to in this state. If the transaction is committed or rolled back while +** in WRITER_LOCKED state, all that is required is to unlock the database ** file. ** ** IN WAL mode, WalBeginWriteTransaction() is called to lock the log file. @@ -46094,7 +57424,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** is made to obtain an EXCLUSIVE lock on the database file. ** ** * A write transaction is active. -** * If the connection is open in rollback-mode, a RESERVED or greater +** * If the connection is open in rollback-mode, a RESERVED or greater ** lock is held on the database file. ** * If the connection is open in WAL-mode, a WAL write transaction ** is open (i.e. sqlite3WalBeginWriteTransaction() has been successfully @@ -46113,7 +57443,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** * A write transaction is active. ** * A RESERVED or greater lock is held on the database file. -** * The journal file is open and the first header has been written +** * The journal file is open and the first header has been written ** to it, but the header has not been synced to disk. ** * The contents of the page cache have been modified. ** @@ -46126,7 +57456,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** * A write transaction is active. ** * An EXCLUSIVE or greater lock is held on the database file. -** * The journal file is open and the first header has been written +** * The journal file is open and the first header has been written ** and synced to disk. ** * The contents of the page cache have been modified (and possibly ** written to disk). @@ -46138,8 +57468,8 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** A rollback-mode pager changes to WRITER_FINISHED state from WRITER_DBMOD ** state after the entire transaction has been successfully written into the ** database file. In this state the transaction may be committed simply -** by finalizing the journal file. Once in WRITER_FINISHED state, it is -** not possible to modify the database further. At this point, the upper +** by finalizing the journal file. Once in WRITER_FINISHED state, it is +** not possible to modify the database further. At this point, the upper ** layer must either commit or rollback the transaction. ** ** * A write transaction is active. @@ -46147,19 +57477,19 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** * All writing and syncing of journal and database data has finished. ** If no error occurred, all that remains is to finalize the journal to ** commit the transaction. If an error did occur, the caller will need -** to rollback the transaction. +** to rollback the transaction. ** ** ERROR: ** ** The ERROR state is entered when an IO or disk-full error (including -** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it -** difficult to be sure that the in-memory pager state (cache contents, +** SQLITE_IOERR_NOMEM) occurs at a point in the code that makes it +** difficult to be sure that the in-memory pager state (cache contents, ** db size etc.) are consistent with the contents of the file-system. ** ** Temporary pager files may enter the ERROR state, but in-memory pagers ** cannot. ** -** For example, if an IO error occurs while performing a rollback, +** For example, if an IO error occurs while performing a rollback, ** the contents of the page-cache may be left in an inconsistent state. ** At this point it would be dangerous to change back to READER state ** (as usually happens after a rollback). Any subsequent readers might @@ -46169,13 +57499,13 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** instead of READER following such an error. ** ** Once it has entered the ERROR state, any attempt to use the pager -** to read or write data returns an error. Eventually, once all +** to read or write data returns an error. Eventually, once all ** outstanding transactions have been abandoned, the pager is able to -** transition back to OPEN state, discarding the contents of the +** transition back to OPEN state, discarding the contents of the ** page-cache and any other in-memory state at the same time. Everything -** is reloaded from disk (and, if necessary, hot-journal rollback peformed) +** is reloaded from disk (and, if necessary, hot-journal rollback performed) ** when a read-transaction is next opened on the pager (transitioning -** the pager into READER state). At that point the system has recovered +** the pager into READER state). At that point the system has recovered ** from the error. ** ** Specifically, the pager jumps into the ERROR state if: @@ -46191,21 +57521,21 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** memory. ** ** In other cases, the error is returned to the b-tree layer. The b-tree -** layer then attempts a rollback operation. If the error condition +** layer then attempts a rollback operation. If the error condition ** persists, the pager enters the ERROR state via condition (1) above. ** ** Condition (3) is necessary because it can be triggered by a read-only ** statement executed within a transaction. In this case, if the error ** code were simply returned to the user, the b-tree layer would not ** automatically attempt a rollback, as it assumes that an error in a -** read-only statement cannot leave the pager in an internally inconsistent +** read-only statement cannot leave the pager in an internally inconsistent ** state. ** ** * The Pager.errCode variable is set to something other than SQLITE_OK. ** * There are one or more outstanding references to pages (after the ** last reference is dropped the pager should move back to OPEN state). ** * The pager is not an in-memory pager. -** +** ** ** Notes: ** @@ -46215,7 +57545,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** * Normally, a connection open in exclusive mode is never in PAGER_OPEN ** state. There are two exceptions: immediately after exclusive-mode has -** been turned on (and before any read or write transactions are +** been turned on (and before any read or write transactions are ** executed), and when the pager is leaving the "error state". ** ** * See also: assert_pager_state(). @@ -46229,7 +57559,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ #define PAGER_ERROR 6 /* -** The Pager.eLock variable is almost always set to one of the +** The Pager.eLock variable is almost always set to one of the ** following locking-states, according to the lock currently held on ** the database file: NO_LOCK, SHARED_LOCK, RESERVED_LOCK or EXCLUSIVE_LOCK. ** This variable is kept up to date as locks are taken and released by @@ -46244,20 +57574,20 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** to a less exclusive (lower) value than the lock that is actually held ** at the system level, but it is never set to a more exclusive value. ** -** This is usually safe. If an xUnlock fails or appears to fail, there may +** This is usually safe. If an xUnlock fails or appears to fail, there may ** be a few redundant xLock() calls or a lock may be held for longer than ** required, but nothing really goes wrong. ** ** The exception is when the database file is unlocked as the pager moves -** from ERROR to OPEN state. At this point there may be a hot-journal file +** from ERROR to OPEN state. At this point there may be a hot-journal file ** in the file-system that needs to be rolled back (as part of an OPEN->SHARED ** transition, by the same pager or any other). If the call to xUnlock() ** fails at this point and the pager is left holding an EXCLUSIVE lock, this ** can confuse the call to xCheckReservedLock() call made later as part ** of hot-journal detection. ** -** xCheckReservedLock() is defined as returning true "if there is a RESERVED -** lock held by this process or any others". So xCheckReservedLock may +** xCheckReservedLock() is defined as returning true "if there is a RESERVED +** lock held by this process or any others". So xCheckReservedLock may ** return true because the caller itself is holding an EXCLUSIVE lock (but ** doesn't know it because of a previous error in xUnlock). If this happens ** a hot-journal may be mistaken for a journal being created by an active @@ -46268,32 +57598,18 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** database in the ERROR state, Pager.eLock is set to UNKNOWN_LOCK. It ** is only changed back to a real locking state after a successful call ** to xLock(EXCLUSIVE). Also, the code to do the OPEN->SHARED state transition -** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK +** omits the check for a hot-journal if Pager.eLock is set to UNKNOWN_LOCK ** lock. Instead, it assumes a hot-journal exists and obtains an EXCLUSIVE ** lock on the database file before attempting to roll it back. See function ** PagerSharedLock() for more detail. ** -** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in +** Pager.eLock may only be set to UNKNOWN_LOCK when the pager is in ** PAGER_OPEN state. */ #define UNKNOWN_LOCK (EXCLUSIVE_LOCK+1) /* -** A macro used for invoking the codec if there is one -*/ -#ifdef SQLITE_HAS_CODEC -# define CODEC1(P,D,N,X,E) \ - if( P->xCodec && P->xCodec(P->pCodec,D,N,X)==0 ){ E; } -# define CODEC2(P,D,N,X,E,O) \ - if( P->xCodec==0 ){ O=(char*)D; }else \ - if( (O=(char*)(P->xCodec(P->pCodec,D,N,X)))==0 ){ E; } -#else -# define CODEC1(P,D,N,X,E) /* NO-OP */ -# define CODEC2(P,D,N,X,E,O) O=(char*)D -#endif - -/* -** The maximum allowed sector size. 64KiB. If the xSectorsize() method +** The maximum allowed sector size. 64KiB. If the xSectorsize() method ** returns a value larger than this, then MAX_SECTOR_SIZE is used instead. ** This could conceivably cause corruption following a power failure on ** such a system. This is currently an undocumented limit. @@ -46309,7 +57625,7 @@ int sqlite3PagerTrace=1; /* True to enable tracing */ ** ** When a savepoint is created, the PagerSavepoint.iHdrOffset field is ** set to 0. If a journal-header is written into the main journal while -** the savepoint is active, then iHdrOffset is set to the byte offset +** the savepoint is active, then iHdrOffset is set to the byte offset ** immediately following the last journal record written into the main ** journal before the journal-header. This is required during savepoint ** rollback (see pagerPlaybackSavepoint()). @@ -46321,6 +57637,7 @@ struct PagerSavepoint { Bitvec *pInSavepoint; /* Set of pages in this savepoint */ Pgno nOrig; /* Original number of pages in file */ Pgno iSubRec; /* Index of first record in sub-journal */ + int bTruncateOnRelease; /* If stmt journal may be truncated on RELEASE */ #ifndef SQLITE_OMIT_WAL u32 aWalData[WAL_SAVEPOINT_NDATA]; /* WAL savepoint context */ #endif @@ -46359,44 +57676,44 @@ struct PagerSavepoint { ** ** changeCountDone ** -** This boolean variable is used to make sure that the change-counter -** (the 4-byte header field at byte offset 24 of the database file) is -** not updated more often than necessary. +** This boolean variable is used to make sure that the change-counter +** (the 4-byte header field at byte offset 24 of the database file) is +** not updated more often than necessary. ** -** It is set to true when the change-counter field is updated, which +** It is set to true when the change-counter field is updated, which ** can only happen if an exclusive lock is held on the database file. -** It is cleared (set to false) whenever an exclusive lock is +** It is cleared (set to false) whenever an exclusive lock is ** relinquished on the database file. Each time a transaction is committed, ** The changeCountDone flag is inspected. If it is true, the work of ** updating the change-counter is omitted for the current transaction. ** -** This mechanism means that when running in exclusive mode, a connection +** This mechanism means that when running in exclusive mode, a connection ** need only update the change-counter once, for the first transaction ** committed. ** -** setMaster +** setSuper ** ** When PagerCommitPhaseOne() is called to commit a transaction, it may -** (or may not) specify a master-journal name to be written into the +** (or may not) specify a super-journal name to be written into the ** journal file before it is synced to disk. ** -** Whether or not a journal file contains a master-journal pointer affects -** the way in which the journal file is finalized after the transaction is +** Whether or not a journal file contains a super-journal pointer affects +** the way in which the journal file is finalized after the transaction is ** committed or rolled back when running in "journal_mode=PERSIST" mode. -** If a journal file does not contain a master-journal pointer, it is +** If a journal file does not contain a super-journal pointer, it is ** finalized by overwriting the first journal header with zeroes. If -** it does contain a master-journal pointer the journal file is finalized -** by truncating it to zero bytes, just as if the connection were +** it does contain a super-journal pointer the journal file is finalized +** by truncating it to zero bytes, just as if the connection were ** running in "journal_mode=truncate" mode. ** -** Journal files that contain master journal pointers cannot be finalized +** Journal files that contain super-journal pointers cannot be finalized ** simply by overwriting the first journal-header with zeroes, as the -** master journal pointer could interfere with hot-journal rollback of any +** super-journal pointer could interfere with hot-journal rollback of any ** subsequently interrupted transaction that reuses the journal file. ** ** The flag is cleared as soon as the journal file is finalized (either ** by PagerCommitPhaseTwo or PagerRollback). If an IO error prevents the -** journal file from being successfully finalized, the setMaster flag +** journal file from being successfully finalized, the setSuper flag ** is cleared anyway (and the pager will move to ERROR state). ** ** doNotSpill @@ -46412,12 +57729,12 @@ struct PagerSavepoint { ** to allocate a new page to prevent the journal file from being written ** while it is being traversed by code in pager_playback(). The SPILLFLAG_OFF ** case is a user preference. -** +** ** If the SPILLFLAG_NOSYNC bit is set, writing to the database from ** pagerStress() is permitted, but syncing the journal file is not. ** This flag is set by sqlite3PagerWrite() when the file-system sector-size ** is larger than the database page-size in order to prevent a journal sync -** from happening in between the journalling of two pages on the same sector. +** from happening in between the journalling of two pages on the same sector. ** ** subjInMemory ** @@ -46425,16 +57742,16 @@ struct PagerSavepoint { ** is opened as an in-memory journal file. If false, then in-memory ** sub-journals are only used for in-memory pager files. ** -** This variable is updated by the upper layer each time a new +** This variable is updated by the upper layer each time a new ** write-transaction is opened. ** ** dbSize, dbOrigSize, dbFileSize ** ** Variable dbSize is set to the number of pages in the database file. ** It is valid in PAGER_READER and higher states (all states except for -** OPEN and ERROR). +** OPEN and ERROR). ** -** dbSize is set based on the size of the database file, which may be +** dbSize is set based on the size of the database file, which may be ** larger than the size of the database (the value stored at offset ** 28 of the database header by the btree). If the size of the file ** is not an integer multiple of the page-size, the value stored in @@ -46445,10 +57762,10 @@ struct PagerSavepoint { ** ** During a write-transaction, if pages with page-numbers greater than ** dbSize are modified in the cache, dbSize is updated accordingly. -** Similarly, if the database is truncated using PagerTruncateImage(), +** Similarly, if the database is truncated using PagerTruncateImage(), ** dbSize is updated. ** -** Variables dbOrigSize and dbFileSize are valid in states +** Variables dbOrigSize and dbFileSize are valid in states ** PAGER_WRITER_LOCKED and higher. dbOrigSize is a copy of the dbSize ** variable at the start of the transaction. It is used during rollback, ** and to determine whether or not pages need to be journalled before @@ -46457,12 +57774,12 @@ struct PagerSavepoint { ** Throughout a write-transaction, dbFileSize contains the size of ** the file on disk in pages. It is set to a copy of dbSize when the ** write-transaction is first opened, and updated when VFS calls are made -** to write or truncate the database file on disk. +** to write or truncate the database file on disk. ** -** The only reason the dbFileSize variable is required is to suppress -** unnecessary calls to xTruncate() after committing a transaction. If, -** when a transaction is committed, the dbFileSize variable indicates -** that the database file is larger than the database image (Pager.dbSize), +** The only reason the dbFileSize variable is required is to suppress +** unnecessary calls to xTruncate() after committing a transaction. If, +** when a transaction is committed, the dbFileSize variable indicates +** that the database file is larger than the database image (Pager.dbSize), ** pager_truncate() is called. The pager_truncate() call uses xFilesize() ** to measure the database file on disk, and then truncates it if required. ** dbFileSize is not used when rolling back a transaction. In this case @@ -46473,21 +57790,33 @@ struct PagerSavepoint { ** dbHintSize ** ** The dbHintSize variable is used to limit the number of calls made to -** the VFS xFileControl(FCNTL_SIZE_HINT) method. +** the VFS xFileControl(FCNTL_SIZE_HINT) method. ** ** dbHintSize is set to a copy of the dbSize variable when a ** write-transaction is opened (at the same time as dbFileSize and ** dbOrigSize). If the xFileControl(FCNTL_SIZE_HINT) method is called, ** dbHintSize is increased to the number of pages that correspond to the -** size-hint passed to the method call. See pager_write_pagelist() for +** size-hint passed to the method call. See pager_write_pagelist() for ** details. ** ** errCode ** ** The Pager.errCode variable is only ever used in PAGER_ERROR state. It -** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode -** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX +** is set to zero in all other states. In PAGER_ERROR state, Pager.errCode +** is always set to SQLITE_FULL, SQLITE_IOERR or one of the SQLITE_IOERR_XXX ** sub-codes. +** +** syncFlags, walSyncFlags +** +** syncFlags is either SQLITE_SYNC_NORMAL (0x02) or SQLITE_SYNC_FULL (0x03). +** syncFlags is used for rollback mode. walSyncFlags is used for WAL mode +** and contains the flags used to sync the checkpoint operations in the +** lower two bits, and sync flags used for transaction commits in the WAL +** file in bits 0x04 and 0x08. In other words, to get the correct sync flags +** for checkpoint operations, use (walSyncFlags&0x03) and to get the correct +** sync flags for transaction commit, use ((walSyncFlags>>2)&0x03). Note +** that with synchronous=NORMAL in WAL mode, transaction commit is not synced +** meaning that the 0x04 and 0x08 bits are both zero. */ struct Pager { sqlite3_vfs *pVfs; /* OS functions to use for IO */ @@ -46497,13 +57826,13 @@ struct Pager { u8 noSync; /* Do not sync the journal if true */ u8 fullSync; /* Do extra syncs of the journal for robustness */ u8 extraSync; /* sync directory after journal delete */ - u8 ckptSyncFlags; /* SYNC_NORMAL or SYNC_FULL for checkpoint */ - u8 walSyncFlags; /* SYNC_NORMAL or SYNC_FULL for wal writes */ u8 syncFlags; /* SYNC_NORMAL or SYNC_FULL otherwise */ + u8 walSyncFlags; /* See description above */ u8 tempFile; /* zFilename is a temporary or immutable file */ u8 noLock; /* Do not lock (except in WAL mode) */ u8 readOnly; /* True for a read-only database */ u8 memDb; /* True to inhibit all file I/O */ + u8 memVfs; /* VFS-implemented memory database */ /************************************************************************** ** The following block contains those class members that change during @@ -46517,7 +57846,7 @@ struct Pager { u8 eState; /* Pager state (OPEN, READER, WRITER_LOCKED..) */ u8 eLock; /* Current lock held on database file */ u8 changeCountDone; /* Set after incrementing the change-counter */ - u8 setMaster; /* True if a m-j name has been written to jrnl */ + u8 setSuper; /* Super-jrnl name is written into jrnl */ u8 doNotSpill; /* Do not spill the cache when non-zero */ u8 subjInMemory; /* True to use in-memory sub-journals */ u8 bUseFetch; /* True to use xFetch() */ @@ -46553,24 +57882,20 @@ struct Pager { i16 nReserve; /* Number of unused bytes at end of each page */ u32 vfsFlags; /* Flags for sqlite3_vfs.xOpen() */ u32 sectorSize; /* Assumed sector size during rollback */ - int pageSize; /* Number of bytes in a page */ Pgno mxPgno; /* Maximum allowed size of the database */ + Pgno lckPgno; /* Page number for the locking page */ + i64 pageSize; /* Number of bytes in a page */ i64 journalSizeLimit; /* Size limit for persistent journal files */ char *zFilename; /* Name of the database file */ char *zJournal; /* Name of the journal file */ int (*xBusyHandler)(void*); /* Function to call when busy */ void *pBusyHandlerArg; /* Context argument for xBusyHandler */ - int aStat[3]; /* Total cache hits, misses and writes */ + u32 aStat[4]; /* Total cache hits, misses, writes, spills */ #ifdef SQLITE_TEST int nRead; /* Database pages read */ #endif void (*xReiniter)(DbPage*); /* Call this routine when reloading pages */ -#ifdef SQLITE_HAS_CODEC - void *(*xCodec)(void*,void*,Pgno,int); /* Routine for en/decoding data */ - void (*xCodecSizeChng)(void*,int,int); /* Notify of page size changes */ - void (*xCodecFree)(void*); /* Destructor for the codec */ - void *pCodec; /* First argument to xCodec... methods */ -#endif + int (*xGet)(Pager*,Pgno,DbPage**,int); /* Routine to fetch a patch */ char *pTmpSpace; /* Pager.pageSize bytes of space for tmp use */ PCache *pPCache; /* Pointer to page cache object */ #ifndef SQLITE_OMIT_WAL @@ -46581,12 +57906,13 @@ struct Pager { /* ** Indexes for use with Pager.aStat[]. The Pager.aStat[] array contains -** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS +** the values accessed by passing SQLITE_DBSTATUS_CACHE_HIT, CACHE_MISS ** or CACHE_WRITE to sqlite3_db_status(). */ #define PAGER_STAT_HIT 0 #define PAGER_STAT_MISS 1 #define PAGER_STAT_WRITE 2 +#define PAGER_STAT_SPILL 3 /* ** The following global variables hold counters used for @@ -46638,7 +57964,7 @@ static const unsigned char aJournalMagic[] = { #define JOURNAL_PG_SZ(pPager) ((pPager->pageSize) + 8) /* -** The journal header size for this pager. This is usually the same +** The journal header size for this pager. This is usually the same ** size as a single disk sector. See also setSectorSize(). */ #define JOURNAL_HDR_SZ(pPager) (pPager->sectorSize) @@ -46665,11 +57991,6 @@ static const unsigned char aJournalMagic[] = { # define USEFETCH(x) 0 #endif -/* -** The maximum legal page number is (2^31 - 1). -*/ -#define PAGER_MAX_PGNO 2147483647 - /* ** The argument to this macro is a file descriptor (type sqlite3_file*). ** Return 0 if it is not open, or non-zero (but not 1) if it is. @@ -46684,14 +58005,39 @@ static const unsigned char aJournalMagic[] = { */ #define isOpen(pFd) ((pFd)->pMethods!=0) +#ifdef SQLITE_DIRECT_OVERFLOW_READ /* -** Return true if this pager uses a write-ahead log instead of the usual -** rollback journal. Otherwise false. +** Return true if page pgno can be read directly from the database file +** by the b-tree layer. This is the case if: +** +** (1) the database file is open +** (2) the VFS for the database is able to do unaligned sub-page reads +** (3) there are no dirty pages in the cache, and +** (4) the desired page is not currently in the wal file. */ +SQLITE_PRIVATE int sqlite3PagerDirectReadOk(Pager *pPager, Pgno pgno){ + assert( pPager!=0 ); + assert( pPager->fd!=0 ); + if( pPager->fd->pMethods==0 ) return 0; /* Case (1) */ + assert( pPager->fd->pMethods->xDeviceCharacteristics!=0 ); + if( (pPager->fd->pMethods->xDeviceCharacteristics(pPager->fd) + & SQLITE_IOCAP_SUBPAGE_READ)==0 ){ + return 0; /* Case (2) */ + } + if( sqlite3PCacheIsDirty(pPager->pPCache) ) return 0; /* Failed (3) */ #ifndef SQLITE_OMIT_WAL -static int pagerUseWal(Pager *pPager){ - return (pPager->pWal!=0); + if( pPager->pWal ){ + u32 iRead = 0; + (void)sqlite3WalFindFrame(pPager->pWal, pgno, &iRead); + return iRead==0; /* Condition (4) */ + } +#endif + return 1; } +#endif + +#ifndef SQLITE_OMIT_WAL +# define pagerUseWal(x) ((x)->pWal!=0) #else # define pagerUseWal(x) 0 # define pagerRollbackWal(x) 0 @@ -46700,7 +58046,7 @@ static int pagerUseWal(Pager *pPager){ # define pagerBeginReadTransaction(z) SQLITE_OK #endif -#ifndef NDEBUG +#ifndef NDEBUG /* ** Usage: ** @@ -46729,25 +58075,25 @@ static int assert_pager_state(Pager *p){ assert( p->tempFile==0 || p->eLock==EXCLUSIVE_LOCK ); assert( p->tempFile==0 || pPager->changeCountDone ); - /* If the useJournal flag is clear, the journal-mode must be "OFF". + /* If the useJournal flag is clear, the journal-mode must be "OFF". ** And if the journal-mode is "OFF", the journal file must not be open. */ assert( p->journalMode==PAGER_JOURNALMODE_OFF || p->useJournal ); assert( p->journalMode!=PAGER_JOURNALMODE_OFF || !isOpen(p->jfd) ); - /* Check that MEMDB implies noSync. And an in-memory journal. Since - ** this means an in-memory pager performs no IO at all, it cannot encounter - ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing - ** a journal file. (although the in-memory journal implementation may - ** return SQLITE_IOERR_NOMEM while the journal file is being written). It - ** is therefore not possible for an in-memory pager to enter the ERROR + /* Check that MEMDB implies noSync. And an in-memory journal. Since + ** this means an in-memory pager performs no IO at all, it cannot encounter + ** either SQLITE_IOERR or SQLITE_FULL during rollback or while finalizing + ** a journal file. (although the in-memory journal implementation may + ** return SQLITE_IOERR_NOMEM while the journal file is being written). It + ** is therefore not possible for an in-memory pager to enter the ERROR ** state. */ if( MEMDB ){ assert( !isOpen(p->fd) ); assert( p->noSync ); - assert( p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_MEMORY + assert( p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_MEMORY ); assert( p->eState!=PAGER_ERROR && p->eState!=PAGER_OPEN ); assert( pagerUseWal(p)==0 ); @@ -46781,7 +58127,7 @@ static int assert_pager_state(Pager *p){ assert( pPager->dbSize==pPager->dbOrigSize ); assert( pPager->dbOrigSize==pPager->dbFileSize ); assert( pPager->dbOrigSize==pPager->dbHintSize ); - assert( pPager->setMaster==0 ); + assert( pPager->setSuper==0 ); break; case PAGER_WRITER_CACHEMOD: @@ -46794,9 +58140,9 @@ static int assert_pager_state(Pager *p){ ** to journal_mode=wal. */ assert( p->eLock>=RESERVED_LOCK ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL ); } assert( pPager->dbOrigSize==pPager->dbFileSize ); @@ -46808,9 +58154,10 @@ static int assert_pager_state(Pager *p){ assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); assert( p->eLock>=EXCLUSIVE_LOCK ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); assert( pPager->dbOrigSize<=pPager->dbHintSize ); break; @@ -46819,9 +58166,10 @@ static int assert_pager_state(Pager *p){ assert( p->eLock==EXCLUSIVE_LOCK ); assert( pPager->errCode==SQLITE_OK ); assert( !pagerUseWal(pPager) ); - assert( isOpen(p->jfd) - || p->journalMode==PAGER_JOURNALMODE_OFF - || p->journalMode==PAGER_JOURNALMODE_WAL + assert( isOpen(p->jfd) + || p->journalMode==PAGER_JOURNALMODE_OFF + || p->journalMode==PAGER_JOURNALMODE_WAL + || (sqlite3OsDeviceCharacteristics(p->fd)&SQLITE_IOCAP_BATCH_ATOMIC) ); break; @@ -46839,7 +58187,7 @@ static int assert_pager_state(Pager *p){ } #endif /* ifndef NDEBUG */ -#ifdef SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Return a pointer to a human readable string in a static buffer ** containing the state of the Pager object passed as an argument. This @@ -46847,8 +58195,12 @@ static int assert_pager_state(Pager *p){ ** to "print *pPager" in gdb: ** ** (gdb) printf "%s", print_pager_state(pPager) +** +** This routine has external linkage in order to suppress compiler warnings +** about an unused function. It is enclosed within SQLITE_DEBUG and so does +** not appear in normal builds. */ -static char *print_pager_state(Pager *p){ +char *print_pager_state(Pager *p){ static char zRet[1024]; sqlite3_snprintf(1024, zRet, @@ -46890,6 +58242,29 @@ static char *print_pager_state(Pager *p){ } #endif +/* Forward references to the various page getters */ +static int getPageNormal(Pager*,Pgno,DbPage**,int); +static int getPageError(Pager*,Pgno,DbPage**,int); +#if SQLITE_MAX_MMAP_SIZE>0 +static int getPageMMap(Pager*,Pgno,DbPage**,int); +#endif + +/* +** Set the Pager.xGet method for the appropriate routine used to fetch +** content from the pager. +*/ +static void setGetterMethod(Pager *pPager){ + if( pPager->errCode ){ + pPager->xGet = getPageError; +#if SQLITE_MAX_MMAP_SIZE>0 + }else if( USEFETCH(pPager) ){ + pPager->xGet = getPageMMap; +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + }else{ + pPager->xGet = getPageNormal; + } +} + /* ** Return true if it is necessary to write page *pPg into the sub-journal. ** A page needs to be written into the sub-journal if there exists one @@ -46907,6 +58282,9 @@ static int subjRequiresPage(PgHdr *pPg){ for(i=0; inSavepoint; i++){ p = &pPager->aSavepoint[i]; if( p->nOrig>=pgno && 0==sqlite3BitvecTestNotNull(p->pInSavepoint, pgno) ){ + for(i=i+1; inSavepoint; i++){ + pPager->aSavepoint[i].bTruncateOnRelease = 0; + } return 1; } } @@ -46960,7 +58338,7 @@ static int write32bits(sqlite3_file *fd, i64 offset, u32 val){ ** succeeds, set the Pager.eLock variable to match the (attempted) new lock. ** ** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is -** called, do not modify it. See the comment above the #define of +** called, do not modify it. See the comment above the #define of ** UNKNOWN_LOCK for an explanation of this. */ static int pagerUnlockDb(Pager *pPager, int eLock){ @@ -46977,17 +58355,18 @@ static int pagerUnlockDb(Pager *pPager, int eLock){ } IOTRACE(("UNLOCK %p %d\n", pPager, eLock)) } + pPager->changeCountDone = pPager->tempFile; /* ticket fb3b3024ea238d5c */ return rc; } /* ** Lock the database file to level eLock, which must be either SHARED_LOCK, ** RESERVED_LOCK or EXCLUSIVE_LOCK. If the caller is successful, set the -** Pager.eLock variable to the new locking state. +** Pager.eLock variable to the new locking state. ** -** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is -** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. -** See the comment above the #define of UNKNOWN_LOCK for an explanation +** Except, if Pager.eLock is set to UNKNOWN_LOCK when this function is +** called, do not modify it unless the new locking state is EXCLUSIVE_LOCK. +** See the comment above the #define of UNKNOWN_LOCK for an explanation ** of this. */ static int pagerLockDb(Pager *pPager, int eLock){ @@ -47005,34 +58384,47 @@ static int pagerLockDb(Pager *pPager, int eLock){ } /* -** This function determines whether or not the atomic-write optimization -** can be used with this pager. The optimization can be used if: +** This function determines whether or not the atomic-write or +** atomic-batch-write optimizations can be used with this pager. The +** atomic-write optimization can be used if: ** ** (a) the value returned by OsDeviceCharacteristics() indicates that ** a database page may be written atomically, and ** (b) the value returned by OsSectorSize() is less than or equal ** to the page size. ** -** The optimization is also always enabled for temporary files. It is -** an error to call this function if pPager is opened on an in-memory -** database. +** If it can be used, then the value returned is the size of the journal +** file when it contains rollback data for exactly one page. ** -** If the optimization cannot be used, 0 is returned. If it can be used, -** then the value returned is the size of the journal file when it -** contains rollback data for exactly one page. +** The atomic-batch-write optimization can be used if OsDeviceCharacteristics() +** returns a value with the SQLITE_IOCAP_BATCH_ATOMIC bit set. -1 is +** returned in this case. +** +** If neither optimization can be used, 0 is returned. */ -#ifdef SQLITE_ENABLE_ATOMIC_WRITE static int jrnlBufferSize(Pager *pPager){ assert( !MEMDB ); - if( !pPager->tempFile ){ - int dc; /* Device characteristics */ - int nSector; /* Sector size */ - int szPage; /* Page size */ - assert( isOpen(pPager->fd) ); - dc = sqlite3OsDeviceCharacteristics(pPager->fd); - nSector = pPager->sectorSize; - szPage = pPager->pageSize; +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) + int dc; /* Device characteristics */ + + assert( isOpen(pPager->fd) ); + dc = sqlite3OsDeviceCharacteristics(pPager->fd); +#else + UNUSED_PARAMETER(pPager); +#endif + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->dbSize>0 && (dc&SQLITE_IOCAP_BATCH_ATOMIC) ){ + return -1; + } +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + { + int nSector = pPager->sectorSize; + int szPage = pPager->pageSize; assert(SQLITE_IOCAP_ATOMIC512==(512>>8)); assert(SQLITE_IOCAP_ATOMIC64K==(65536>>8)); @@ -47042,11 +58434,11 @@ static int jrnlBufferSize(Pager *pPager){ } return JOURNAL_HDR_SZ(pPager) + JOURNAL_PG_SZ(pPager); -} -#else -# define jrnlBufferSize(x) 0 #endif + return 0; +} + /* ** If SQLITE_CHECK_PAGES is defined then we do some sanity checking ** on the cache using a hash function. This is used for testing @@ -47092,71 +58484,73 @@ static void checkPage(PgHdr *pPg){ /* ** When this is called the journal file for pager pPager must be open. -** This function attempts to read a master journal file name from the -** end of the file and, if successful, copies it into memory supplied -** by the caller. See comments above writeMasterJournal() for the format -** used to store a master journal file name at the end of a journal file. +** This function attempts to read a super-journal file name from the +** end of the file and, if successful, copies it into memory supplied +** by the caller. See comments above writeSuperJournal() for the format +** used to store a super-journal file name at the end of a journal file. ** -** zMaster must point to a buffer of at least nMaster bytes allocated by +** zSuper must point to a buffer of at least nSuper bytes allocated by ** the caller. This should be sqlite3_vfs.mxPathname+1 (to ensure there is -** enough space to write the master journal name). If the master journal -** name in the journal is longer than nMaster bytes (including a -** nul-terminator), then this is handled as if no master journal name +** enough space to write the super-journal name). If the super-journal +** name in the journal is longer than nSuper bytes (including a +** nul-terminator), then this is handled as if no super-journal name ** were present in the journal. ** -** If a master journal file name is present at the end of the journal -** file, then it is copied into the buffer pointed to by zMaster. A -** nul-terminator byte is appended to the buffer following the master -** journal file name. +** If a super-journal file name is present at the end of the journal +** file, then it is copied into the buffer pointed to by zSuper. A +** nul-terminator byte is appended to the buffer following the +** super-journal file name. ** -** If it is determined that no master journal file name is present -** zMaster[0] is set to 0 and SQLITE_OK returned. +** If it is determined that no super-journal file name is present +** zSuper[0] is set to 0 and SQLITE_OK returned. ** ** If an error occurs while reading from the journal file, an SQLite ** error code is returned. */ -static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ +static int readSuperJournal(sqlite3_file *pJrnl, char *zSuper, u32 nSuper){ int rc; /* Return code */ - u32 len; /* Length in bytes of master journal name */ + u32 len; /* Length in bytes of super-journal name */ i64 szJ; /* Total size in bytes of journal file pJrnl */ u32 cksum; /* MJ checksum value read from journal */ u32 u; /* Unsigned loop counter */ unsigned char aMagic[8]; /* A buffer to hold the magic header */ - zMaster[0] = '\0'; + zSuper[0] = '\0'; if( SQLITE_OK!=(rc = sqlite3OsFileSize(pJrnl, &szJ)) || szJ<16 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-16, &len)) - || len>=nMaster - || len==0 + || len>=nSuper + || len>szJ-16 + || len==0 || SQLITE_OK!=(rc = read32bits(pJrnl, szJ-12, &cksum)) || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, aMagic, 8, szJ-8)) || memcmp(aMagic, aJournalMagic, 8) - || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zMaster, len, szJ-16-len)) + || SQLITE_OK!=(rc = sqlite3OsRead(pJrnl, zSuper, len, szJ-16-len)) ){ return rc; } - /* See if the checksum matches the master journal name */ + /* See if the checksum matches the super-journal name */ for(u=0; ujournalOff, assuming a sector +** Return the offset of the sector boundary at or immediately +** following the value in pPager->journalOff, assuming a sector ** size of pPager->sectorSize bytes. ** ** i.e for a sector size of 512: @@ -47167,7 +58561,7 @@ static int readMasterJournal(sqlite3_file *pJrnl, char *zMaster, u32 nMaster){ ** 512 512 ** 100 512 ** 2000 2048 -** +** */ static i64 journalHdrOffset(Pager *pPager){ i64 offset = 0; @@ -47189,12 +58583,12 @@ static i64 journalHdrOffset(Pager *pPager){ ** ** If doTruncate is non-zero or the Pager.journalSizeLimit variable is ** set to 0, then truncate the journal file to zero bytes in size. Otherwise, -** zero the 28-byte header at the start of the journal file. In either case, -** if the pager is not in no-sync mode, sync the journal file immediately +** zero the 28-byte header at the start of the journal file. In either case, +** if the pager is not in no-sync mode, sync the journal file immediately ** after writing or truncating it. ** ** If Pager.journalSizeLimit is set to a positive, non-zero value, and -** following the truncation or zeroing described above the size of the +** following the truncation or zeroing described above the size of the ** journal file in bytes is larger than this value, then truncate the ** journal file to Pager.journalSizeLimit bytes. The journal file does ** not need to be synced following this operation. @@ -47220,8 +58614,8 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ rc = sqlite3OsSync(pPager->jfd, SQLITE_SYNC_DATAONLY|pPager->syncFlags); } - /* At this point the transaction is committed but the write lock - ** is still held on the file. If there is a size limit configured for + /* At this point the transaction is committed but the write lock + ** is still held on the file. If there is a size limit configured for ** the persistent journal and the journal file currently consumes more ** space than that limit allows for, truncate it now. There is no need ** to sync the file following this operation. @@ -47249,7 +58643,7 @@ static int zeroJournalHdr(Pager *pPager, int doTruncate){ ** - 4 bytes: Initial database page count. ** - 4 bytes: Sector size used by the process that wrote this journal. ** - 4 bytes: Database page size. -** +** ** Followed by (JOURNAL_HDR_SZ - 28) bytes of unused space. */ static int writeJournalHdr(Pager *pPager){ @@ -47265,8 +58659,8 @@ static int writeJournalHdr(Pager *pPager){ nHeader = JOURNAL_HDR_SZ(pPager); } - /* If there are active savepoints and any of them were created - ** since the most recent journal header was written, update the + /* If there are active savepoints and any of them were created + ** since the most recent journal header was written, update the ** PagerSavepoint.iHdrOffset fields now. */ for(ii=0; iinSavepoint; ii++){ @@ -47277,10 +58671,10 @@ static int writeJournalHdr(Pager *pPager){ pPager->journalHdr = pPager->journalOff = journalHdrOffset(pPager); - /* + /* ** Write the nRec Field - the number of page records that follow this ** journal header. Normally, zero is written to this value at this time. - ** After the records are added to the journal (and the journal synced, + ** After the records are added to the journal (and the journal synced, ** if in full-sync mode), the zero is overwritten with the true number ** of records (see syncJournal()). ** @@ -47299,7 +58693,7 @@ static int writeJournalHdr(Pager *pPager){ */ assert( isOpen(pPager->fd) || pPager->noSync ); if( pPager->noSync || (pPager->journalMode==PAGER_JOURNALMODE_MEMORY) - || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_SAFE_APPEND) ){ memcpy(zHeader, aJournalMagic, sizeof(aJournalMagic)); put32bits(&zHeader[sizeof(aJournalMagic)], 0xffffffff); @@ -47307,9 +58701,32 @@ static int writeJournalHdr(Pager *pPager){ memset(zHeader, 0, sizeof(aJournalMagic)+4); } - /* The random check-hash initializer */ - sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + + + /* The random check-hash initializer */ + if( pPager->journalMode!=PAGER_JOURNALMODE_MEMORY ){ + sqlite3_randomness(sizeof(pPager->cksumInit), &pPager->cksumInit); + } +#ifdef SQLITE_DEBUG + else{ + /* The Pager.cksumInit variable is usually randomized above to protect + ** against there being existing records in the journal file. This is + ** dangerous, as following a crash they may be mistaken for records + ** written by the current transaction and rolled back into the database + ** file, causing corruption. The following assert statements verify + ** that this is not required in "journal_mode=memory" mode, as in that + ** case the journal file is always 0 bytes in size at this point. + ** It is advantageous to avoid the sqlite3_randomness() call if possible + ** as it takes the global PRNG mutex. */ + i64 sz = 0; + sqlite3OsFileSize(pPager->jfd, &sz); + assert( sz==0 ); + assert( pPager->journalOff==journalHdrOffset(pPager) ); + assert( sqlite3JournalIsInMemory(pPager->jfd) ); + } +#endif put32bits(&zHeader[sizeof(aJournalMagic)+4], pPager->cksumInit); + /* The initial database size */ put32bits(&zHeader[sizeof(aJournalMagic)+8], pPager->dbOrigSize); /* The assumed sector size for this process */ @@ -47326,23 +58743,23 @@ static int writeJournalHdr(Pager *pPager){ memset(&zHeader[sizeof(aJournalMagic)+20], 0, nHeader-(sizeof(aJournalMagic)+20)); - /* In theory, it is only necessary to write the 28 bytes that the - ** journal header consumes to the journal file here. Then increment the - ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next + /* In theory, it is only necessary to write the 28 bytes that the + ** journal header consumes to the journal file here. Then increment the + ** Pager.journalOff variable by JOURNAL_HDR_SZ so that the next ** record is written to the following sector (leaving a gap in the file ** that will be implicitly filled in by the OS). ** - ** However it has been discovered that on some systems this pattern can + ** However it has been discovered that on some systems this pattern can ** be significantly slower than contiguously writing data to the file, - ** even if that means explicitly writing data to the block of + ** even if that means explicitly writing data to the block of ** (JOURNAL_HDR_SZ - 28) bytes that will not be used. So that is what - ** is done. + ** is done. ** - ** The loop is required here in case the sector-size is larger than the + ** The loop is required here in case the sector-size is larger than the ** database page size. Since the zHeader buffer is only Pager.pageSize ** bytes in size, more than one call to sqlite3OsWrite() may be required ** to populate the entire journal header sector. - */ + */ for(nWrite=0; rc==SQLITE_OK&&nWritejournalHdr, nHeader)) rc = sqlite3OsWrite(pPager->jfd, zHeader, nHeader, pPager->journalOff); @@ -47440,29 +58857,29 @@ static int readJournalHdr( /* Check that the values read from the page-size and sector-size fields ** are within range. To be 'in range', both values need to be a power - ** of two greater than or equal to 512 or 32, and not greater than their + ** of two greater than or equal to 512 or 32, and not greater than their ** respective compile time maximum limits. */ if( iPageSize<512 || iSectorSize<32 || iPageSize>SQLITE_MAX_PAGE_SIZE || iSectorSize>MAX_SECTOR_SIZE - || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 + || ((iPageSize-1)&iPageSize)!=0 || ((iSectorSize-1)&iSectorSize)!=0 ){ - /* If the either the page-size or sector-size in the journal-header is - ** invalid, then the process that wrote the journal-header must have - ** crashed before the header was synced. In this case stop reading + /* If the either the page-size or sector-size in the journal-header is + ** invalid, then the process that wrote the journal-header must have + ** crashed before the header was synced. In this case stop reading ** the journal file here. */ return SQLITE_DONE; } - /* Update the page-size to match the value read from the journal. - ** Use a testcase() macro to make sure that malloc failure within + /* Update the page-size to match the value read from the journal. + ** Use a testcase() macro to make sure that malloc failure within ** PagerSetPagesize() is tested. */ rc = sqlite3PagerSetPagesize(pPager, &iPageSize, -1); testcase( rc!=SQLITE_OK ); - /* Update the assumed sector-size to match the value used by + /* Update the assumed sector-size to match the value used by ** the process that created this journal. If this journal was ** created by a process other than this one, then this routine ** is being called from within pager_playback(). The local value @@ -47477,50 +58894,50 @@ static int readJournalHdr( /* -** Write the supplied master journal name into the journal file for pager -** pPager at the current location. The master journal name must be the last +** Write the supplied super-journal name into the journal file for pager +** pPager at the current location. The super-journal name must be the last ** thing written to a journal file. If the pager is in full-sync mode, the ** journal file descriptor is advanced to the next sector boundary before ** anything is written. The format is: ** -** + 4 bytes: PAGER_MJ_PGNO. -** + N bytes: Master journal filename in utf-8. -** + 4 bytes: N (length of master journal name in bytes, no nul-terminator). -** + 4 bytes: Master journal name checksum. +** + 4 bytes: PAGER_SJ_PGNO. +** + N bytes: super-journal filename in utf-8. +** + 4 bytes: N (length of super-journal name in bytes, no nul-terminator). +** + 4 bytes: super-journal name checksum. ** + 8 bytes: aJournalMagic[]. ** -** The master journal page checksum is the sum of the bytes in the master -** journal name, where each byte is interpreted as a signed 8-bit integer. +** The super-journal page checksum is the sum of the bytes in the super-journal +** name, where each byte is interpreted as a signed 8-bit integer. ** -** If zMaster is a NULL pointer (occurs for a single database transaction), +** If zSuper is a NULL pointer (occurs for a single database transaction), ** this call is a no-op. */ -static int writeMasterJournal(Pager *pPager, const char *zMaster){ +static int writeSuperJournal(Pager *pPager, const char *zSuper){ int rc; /* Return code */ - int nMaster; /* Length of string zMaster */ + int nSuper; /* Length of string zSuper */ i64 iHdrOff; /* Offset of header in journal file */ i64 jrnlSize; /* Size of journal file on disk */ - u32 cksum = 0; /* Checksum of string zMaster */ + u32 cksum = 0; /* Checksum of string zSuper */ - assert( pPager->setMaster==0 ); + assert( pPager->setSuper==0 ); assert( !pagerUseWal(pPager) ); - if( !zMaster - || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + if( !zSuper + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY || !isOpen(pPager->jfd) ){ return SQLITE_OK; } - pPager->setMaster = 1; + pPager->setSuper = 1; assert( pPager->journalHdr <= pPager->journalOff ); - /* Calculate the length in bytes and the checksum of zMaster */ - for(nMaster=0; zMaster[nMaster]; nMaster++){ - cksum += zMaster[nMaster]; + /* Calculate the length in bytes and the checksum of zSuper */ + for(nSuper=0; zSuper[nSuper]; nSuper++){ + cksum += zSuper[nSuper]; } /* If in full-sync mode, advance to the next disk sector before writing - ** the master journal name. This is in case the previous page written to + ** the super-journal name. This is in case the previous page written to ** the journal has already been synced. */ if( pPager->fullSync ){ @@ -47528,30 +58945,30 @@ static int writeMasterJournal(Pager *pPager, const char *zMaster){ } iHdrOff = pPager->journalOff; - /* Write the master journal data to the end of the journal file. If + /* Write the super-journal data to the end of the journal file. If ** an error occurs, return the error code to the caller. */ - if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_MJ_PGNO(pPager)))) - || (0 != (rc = sqlite3OsWrite(pPager->jfd, zMaster, nMaster, iHdrOff+4))) - || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster, nMaster))) - || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nMaster+4, cksum))) + if( (0 != (rc = write32bits(pPager->jfd, iHdrOff, PAGER_SJ_PGNO(pPager)))) + || (0 != (rc = sqlite3OsWrite(pPager->jfd, zSuper, nSuper, iHdrOff+4))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper, nSuper))) + || (0 != (rc = write32bits(pPager->jfd, iHdrOff+4+nSuper+4, cksum))) || (0 != (rc = sqlite3OsWrite(pPager->jfd, aJournalMagic, 8, - iHdrOff+4+nMaster+8))) + iHdrOff+4+nSuper+8))) ){ return rc; } - pPager->journalOff += (nMaster+20); + pPager->journalOff += (nSuper+20); - /* If the pager is in peristent-journal mode, then the physical - ** journal-file may extend past the end of the master-journal name - ** and 8 bytes of magic data just written to the file. This is + /* If the pager is in persistent-journal mode, then the physical + ** journal-file may extend past the end of the super-journal name + ** and 8 bytes of magic data just written to the file. This is ** dangerous because the code to rollback a hot-journal file - ** will not be able to find the master-journal name to determine - ** whether or not the journal is hot. + ** will not be able to find the super-journal name to determine + ** whether or not the journal is hot. ** - ** Easiest thing to do in this scenario is to truncate the journal + ** Easiest thing to do in this scenario is to truncate the journal ** file to the required size. - */ + */ if( SQLITE_OK==(rc = sqlite3OsFileSize(pPager->jfd, &jrnlSize)) && jrnlSize>pPager->journalOff ){ @@ -47573,7 +58990,6 @@ static void pager_reset(Pager *pPager){ ** Return the pPager->iDataVersion value */ SQLITE_PRIVATE u32 sqlite3PagerDataVersion(Pager *pPager){ - assert( pPager->eState>PAGER_OPEN ); return pPager->iDataVersion; } @@ -47597,7 +59013,7 @@ static void releaseAllSavepoints(Pager *pPager){ } /* -** Set the bit number pgno in the PagerSavepoint.pInSavepoint +** Set the bit number pgno in the PagerSavepoint.pInSavepoint ** bitvecs of all open savepoints. Return SQLITE_OK if successful ** or SQLITE_NOMEM if a malloc failure occurs. */ @@ -47626,8 +59042,8 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ ** not exhibit the UNDELETABLE_WHEN_OPEN property, the journal file is ** closed (if it is open). ** -** If the pager is in ERROR state when this function is called, the -** contents of the pager cache are discarded before switching back to +** If the pager is in ERROR state when this function is called, the +** contents of the pager cache are discarded before switching back to ** the OPEN state. Regardless of whether the pager is in exclusive-mode ** or not, any journal file left in the file-system will be treated ** as a hot-journal and rolled back the next time a read-transaction @@ -47635,9 +59051,9 @@ static int addToSavepointBitvecs(Pager *pPager, Pgno pgno){ */ static void pager_unlock(Pager *pPager){ - assert( pPager->eState==PAGER_READER - || pPager->eState==PAGER_OPEN - || pPager->eState==PAGER_ERROR + assert( pPager->eState==PAGER_READER + || pPager->eState==PAGER_OPEN + || pPager->eState==PAGER_ERROR ); sqlite3BitvecDestroy(pPager->pInJournal); @@ -47684,7 +59100,6 @@ static void pager_unlock(Pager *pPager){ ** code is cleared and the cache reset in the block below. */ assert( pPager->errCode || pPager->eState!=PAGER_ERROR ); - pPager->changeCountDone = 0; pPager->eState = PAGER_OPEN; } @@ -47704,27 +59119,28 @@ static void pager_unlock(Pager *pPager){ } if( USEFETCH(pPager) ) sqlite3OsUnfetch(pPager->fd, 0, 0); pPager->errCode = SQLITE_OK; + setGetterMethod(pPager); } pPager->journalOff = 0; pPager->journalHdr = 0; - pPager->setMaster = 0; + pPager->setSuper = 0; } /* ** This function is called whenever an IOERR or FULL error that requires -** the pager to transition into the ERROR state may ahve occurred. -** The first argument is a pointer to the pager structure, the second -** the error-code about to be returned by a pager API function. The -** value returned is a copy of the second argument to this function. +** the pager to transition into the ERROR state may have occurred. +** The first argument is a pointer to the pager structure, the second +** the error-code about to be returned by a pager API function. The +** value returned is a copy of the second argument to this function. ** ** If the second argument is SQLITE_FULL, SQLITE_IOERR or one of the ** IOERR sub-codes, the pager enters the ERROR state and the error code ** is stored in Pager.errCode. While the pager remains in the ERROR state, ** all major API calls on the Pager will immediately return Pager.errCode. ** -** The ERROR state indicates that the contents of the pager-cache -** cannot be trusted. This state can be cleared by completely discarding +** The ERROR state indicates that the contents of the pager-cache +** cannot be trusted. This state can be cleared by completely discarding ** the contents of the pager-cache. If a transaction was active when ** the persistent error occurred, then the rollback journal may need ** to be replayed to restore the contents of the database file (as if @@ -47741,6 +59157,7 @@ static int pager_error(Pager *pPager, int rc){ if( rc2==SQLITE_FULL || rc2==SQLITE_IOERR ){ pPager->errCode = rc; pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); } return rc; } @@ -47771,27 +59188,27 @@ static int pagerFlushOnCommit(Pager *pPager, int bCommit){ } /* -** This routine ends a transaction. A transaction is usually ended by -** either a COMMIT or a ROLLBACK operation. This routine may be called +** This routine ends a transaction. A transaction is usually ended by +** either a COMMIT or a ROLLBACK operation. This routine may be called ** after rollback of a hot-journal, or if an error occurs while opening ** the journal file or writing the very first journal-header of a ** database transaction. -** +** ** This routine is never called in PAGER_ERROR state. If it is called ** in PAGER_NONE or PAGER_SHARED state and the lock held is less ** exclusive than a RESERVED lock, it is a no-op. ** ** Otherwise, any active savepoints are released. ** -** If the journal file is open, then it is "finalized". Once a journal -** file has been finalized it is not possible to use it to roll back a +** If the journal file is open, then it is "finalized". Once a journal +** file has been finalized it is not possible to use it to roll back a ** transaction. Nor will it be considered to be a hot-journal by this ** or any other database connection. Exactly how a journal is finalized ** depends on whether or not the pager is running in exclusive mode and ** the current journal-mode (Pager.journalMode value), as follows: ** ** journalMode==MEMORY -** Journal file descriptor is simply closed. This destroys an +** Journal file descriptor is simply closed. This destroys an ** in-memory journal. ** ** journalMode==TRUNCATE @@ -47811,19 +59228,19 @@ static int pagerFlushOnCommit(Pager *pPager, int bCommit){ ** journalMode==PERSIST is used instead. ** ** After the journal is finalized, the pager moves to PAGER_READER state. -** If running in non-exclusive rollback mode, the lock on the file is +** If running in non-exclusive rollback mode, the lock on the file is ** downgraded to a SHARED_LOCK. ** ** SQLITE_OK is returned if no error occurs. If an error occurs during ** any of the IO operations to finalize the journal file or unlock the -** database then the IO error code is returned to the user. If the +** database then the IO error code is returned to the user. If the ** operation to finalize the journal file fails, then the code still ** tries to unlock the database file if not in exclusive mode. If the ** unlock operation fails as well, then the first error code related ** to the first error encountered (the journal finalization one) is ** returned. */ -static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ +static int pager_end_transaction(Pager *pPager, int hasSuper, int bCommit){ int rc = SQLITE_OK; /* Error code from journal finalization operation */ int rc2 = SQLITE_OK; /* Error code from db file unlock operation */ @@ -47835,9 +59252,9 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ ** 1. After a successful hot-journal rollback, it is called with ** eState==PAGER_NONE and eLock==EXCLUSIVE_LOCK. ** - ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE + ** 2. If a connection with locking_mode=exclusive holding an EXCLUSIVE ** lock switches back to locking_mode=normal and then executes a - ** read-transaction, this function is called with eState==PAGER_READER + ** read-transaction, this function is called with eState==PAGER_READER ** and eLock==EXCLUSIVE_LOCK when the read-transaction is closed. */ assert( assert_pager_state(pPager) ); @@ -47847,7 +59264,9 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ } releaseAllSavepoints(pPager); - assert( isOpen(pPager->jfd) || pPager->pInJournal==0 ); + assert( isOpen(pPager->jfd) || pPager->pInJournal==0 + || (sqlite3OsDeviceCharacteristics(pPager->fd)&SQLITE_IOCAP_BATCH_ATOMIC) + ); if( isOpen(pPager->jfd) ){ assert( !pagerUseWal(pPager) ); @@ -47873,7 +59292,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ }else if( pPager->journalMode==PAGER_JOURNALMODE_PERSIST || (pPager->exclusiveMode && pPager->journalMode!=PAGER_JOURNALMODE_WAL) ){ - rc = zeroJournalHdr(pPager, hasMaster||pPager->tempFile); + rc = zeroJournalHdr(pPager, hasSuper||pPager->tempFile); pPager->journalOff = 0; }else{ /* This branch may be executed with Pager.journalMode==MEMORY if @@ -47883,9 +59302,9 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ */ int bDelete = !pPager->tempFile; assert( sqlite3JournalIsInMemory(pPager->jfd)==0 ); - assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE - || pPager->journalMode==PAGER_JOURNALMODE_MEMORY - || pPager->journalMode==PAGER_JOURNALMODE_WAL + assert( pPager->journalMode==PAGER_JOURNALMODE_DELETE + || pPager->journalMode==PAGER_JOURNALMODE_MEMORY + || pPager->journalMode==PAGER_JOURNALMODE_WAL ); sqlite3OsClose(pPager->jfd); if( bDelete ){ @@ -47909,7 +59328,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ pPager->pInJournal = 0; pPager->nRec = 0; if( rc==SQLITE_OK ){ - if( pagerFlushOnCommit(pPager, bCommit) ){ + if( MEMDB || pagerFlushOnCommit(pPager, bCommit) ){ sqlite3PcacheCleanAll(pPager->pPCache); }else{ sqlite3PcacheClearWritable(pPager->pPCache); @@ -47918,8 +59337,8 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ } if( pagerUseWal(pPager) ){ - /* Drop the WAL write-lock, if any. Also, if the connection was in - ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE + /* Drop the WAL write-lock, if any. Also, if the connection was in + ** locking_mode=exclusive mode but is no longer, drop the EXCLUSIVE ** lock held on the database file. */ rc2 = sqlite3WalEndWriteTransaction(pPager->pWal); @@ -47927,7 +59346,7 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ }else if( rc==SQLITE_OK && bCommit && pPager->dbFileSize>pPager->dbSize ){ /* This branch is taken when committing a transaction in rollback-journal ** mode if the database file on disk is larger than the database image. - ** At this point the journal has been finalized and the transaction + ** At this point the journal has been finalized and the transaction ** successfully committed, but the EXCLUSIVE lock is still held on the ** file. So it is safe to truncate the database file to its minimum ** required size. */ @@ -47935,37 +59354,39 @@ static int pager_end_transaction(Pager *pPager, int hasMaster, int bCommit){ rc = pager_truncate(pPager, pPager->dbSize); } - if( rc==SQLITE_OK && bCommit && isOpen(pPager->fd) ){ + if( rc==SQLITE_OK && bCommit ){ rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_COMMIT_PHASETWO, 0); if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; } - if( !pPager->exclusiveMode + if( !pPager->exclusiveMode && (!pagerUseWal(pPager) || sqlite3WalExclusiveMode(pPager->pWal, 0)) ){ rc2 = pagerUnlockDb(pPager, SHARED_LOCK); - pPager->changeCountDone = 0; } pPager->eState = PAGER_READER; - pPager->setMaster = 0; + pPager->setSuper = 0; return (rc==SQLITE_OK?rc2:rc); } +/* Forward reference */ +static int pager_playback(Pager *pPager, int isHot); + /* -** Execute a rollback if a transaction is active and unlock the -** database file. +** Execute a rollback if a transaction is active and unlock the +** database file. ** -** If the pager has already entered the ERROR state, do not attempt +** If the pager has already entered the ERROR state, do not attempt ** the rollback at this time. Instead, pager_unlock() is called. The ** call to pager_unlock() will discard all in-memory pages, unlock -** the database file and move the pager back to OPEN state. If this -** means that there is a hot-journal left in the file-system, the next -** connection to obtain a shared lock on the pager (which may be this one) +** the database file and move the pager back to OPEN state. If this +** means that there is a hot-journal left in the file-system, the next +** connection to obtain a shared lock on the pager (which may be this one) ** will roll it back. ** ** If the pager has not already entered the ERROR state, but an IO or -** malloc error occurs during a rollback, then this will itself cause +** malloc error occurs during a rollback, then this will itself cause ** the pager to enter the ERROR state. Which will be cleared by the ** call to pager_unlock(), as described above. */ @@ -47980,16 +59401,31 @@ static void pagerUnlockAndRollback(Pager *pPager){ assert( pPager->eState==PAGER_READER ); pager_end_transaction(pPager, 0, 0); } + }else if( pPager->eState==PAGER_ERROR + && pPager->journalMode==PAGER_JOURNALMODE_MEMORY + && isOpen(pPager->jfd) + ){ + /* Special case for a ROLLBACK due to I/O error with an in-memory + ** journal: We have to rollback immediately, before the journal is + ** closed, because once it is closed, all content is forgotten. */ + int errCode = pPager->errCode; + u8 eLock = pPager->eLock; + pPager->eState = PAGER_OPEN; + pPager->errCode = SQLITE_OK; + pPager->eLock = EXCLUSIVE_LOCK; + pager_playback(pPager, 1); + pPager->errCode = errCode; + pPager->eLock = eLock; } pager_unlock(pPager); } /* ** Parameter aData must point to a buffer of pPager->pageSize bytes -** of data. Compute and return a checksum based ont the contents of the +** of data. Compute and return a checksum based on the contents of the ** page of data and the current value of pPager->cksumInit. ** -** This is not a real checksum. It is really just the sum of the +** This is not a real checksum. It is really just the sum of the ** random initial value (pPager->cksumInit) and every 200th byte ** of the page data, starting with byte offset (pPager->pageSize%200). ** Each byte is interpreted as an 8-bit unsigned integer. @@ -47997,8 +59433,8 @@ static void pagerUnlockAndRollback(Pager *pPager){ ** Changing the formula used to compute this checksum results in an ** incompatible journal file format. ** -** If journal corruption occurs due to a power failure, the most likely -** scenario is that one end or the other of the record will be changed. +** If journal corruption occurs due to a power failure, the most likely +** scenario is that one end or the other of the record will be changed. ** It is much less likely that the two ends of the journal record will be ** correct and the middle be corrupt. Thus, this "checksum" scheme, ** though fast and simple, catches the mostly likely kind of corruption. @@ -48013,42 +59449,13 @@ static u32 pager_cksum(Pager *pPager, const u8 *aData){ return cksum; } -/* -** Report the current page size and number of reserved bytes back -** to the codec. -*/ -#ifdef SQLITE_HAS_CODEC -static void pagerReportSize(Pager *pPager){ - if( pPager->xCodecSizeChng ){ - pPager->xCodecSizeChng(pPager->pCodec, pPager->pageSize, - (int)pPager->nReserve); - } -} -#else -# define pagerReportSize(X) /* No-op if we do not support a codec */ -#endif - -#ifdef SQLITE_HAS_CODEC -/* -** Make sure the number of reserved bits is the same in the destination -** pager as it is in the source. This comes up when a VACUUM changes the -** number of reserved bits to the "optimal" amount. -*/ -SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ - if( pDest->nReserve!=pSrc->nReserve ){ - pDest->nReserve = pSrc->nReserve; - pagerReportSize(pDest); - } -} -#endif - /* ** Read a single page from either the journal file (if isMainJrnl==1) or ** from the sub-journal (if isMainJrnl==0) and playback that page. ** The page begins at offset *pOffset into the file. The *pOffset ** value is increased to the start of the next page in the journal. ** -** The main rollback journal uses checksums - the statement journal does +** The main rollback journal uses checksums - the statement journal does ** not. ** ** If the page number of the page record read from the (sub-)journal file @@ -48068,8 +59475,8 @@ SQLITE_PRIVATE void sqlite3PagerAlignReserve(Pager *pDest, Pager *pSrc){ ** is successfully read from the (sub-)journal file but appears to be ** corrupted, SQLITE_DONE is returned. Data is considered corrupted in ** two circumstances: -** -** * If the record page-number is illegal (0 or PAGER_MJ_PGNO), or +** +** * If the record page-number is illegal (0 or PAGER_SJ_PGNO), or ** * If the record is being rolled back from the main journal file ** and the checksum field does not match the record content. ** @@ -48103,7 +59510,7 @@ static int pager_playback_one_page( assert( aData ); /* Temp storage must have already been allocated */ assert( pagerUseWal(pPager)==0 || (!isMainJrnl && isSavepnt) ); - /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction + /* Either the state is greater than PAGER_WRITER_CACHEMOD (a transaction ** or savepoint rollback done at the request of the caller) or this is ** a hot-journal rollback. If it is a hot-journal rollback, the pager ** is in state OPEN and holds an EXCLUSIVE lock. Hot-journal rollback @@ -48129,7 +59536,7 @@ static int pager_playback_one_page( ** it could cause invalid data to be written into the journal. We need to ** detect this invalid data (with high probability) and ignore it. */ - if( pgno==0 || pgno==PAGER_MJ_PGNO(pPager) ){ + if( pgno==0 || pgno==PAGER_SJ_PGNO(pPager) ){ assert( !isSavepnt ); return SQLITE_DONE; } @@ -48155,7 +59562,6 @@ static int pager_playback_one_page( */ if( pgno==1 && pPager->nReserve!=((u8*)aData)[20] ){ pPager->nReserve = ((u8*)aData)[20]; - pagerReportSize(pPager); } /* If the pager is in CACHEMOD state, then there must be a copy of this @@ -48170,7 +59576,7 @@ static int pager_playback_one_page( ** assert()able. ** ** If in WRITER_DBMOD, WRITER_FINISHED or OPEN state, then we update the - ** pager cache if it exists and the main file. The page is then marked + ** pager cache if it exists and the main file. The page is then marked ** not dirty. Since this code is only executed in PAGER_OPEN state for ** a hot-journal rollback, it is guaranteed that the page-cache is empty ** if the pager is in OPEN state. @@ -48216,30 +59622,36 @@ static int pager_playback_one_page( i64 ofst = (pgno-1)*(i64)pPager->pageSize; testcase( !isSavepnt && pPg!=0 && (pPg->flags&PGHDR_NEED_SYNC)!=0 ); assert( !pagerUseWal(pPager) ); + + /* Write the data read from the journal back into the database file. + ** This is usually safe even for an encrypted database - as the data + ** was encrypted before it was written to the journal file. The exception + ** is if the data was just read from an in-memory sub-journal. In that + ** case it must be encrypted here before it is copied into the database + ** file. */ rc = sqlite3OsWrite(pPager->fd, (u8 *)aData, pPager->pageSize, ofst); + if( pgno>pPager->dbFileSize ){ pPager->dbFileSize = pgno; } if( pPager->pBackup ){ - CODEC1(pPager, aData, pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3BackupUpdate(pPager->pBackup, pgno, (u8*)aData); - CODEC2(pPager, aData, pgno, 7, rc=SQLITE_NOMEM_BKPT, aData); } }else if( !isMainJrnl && pPg==0 ){ /* If this is a rollback of a savepoint and data was not written to ** the database and the page is not in-memory, there is a potential - ** problem. When the page is next fetched by the b-tree layer, it - ** will be read from the database file, which may or may not be - ** current. + ** problem. When the page is next fetched by the b-tree layer, it + ** will be read from the database file, which may or may not be + ** current. ** ** There are a couple of different ways this can happen. All are quite - ** obscure. When running in synchronous mode, this can only happen + ** obscure. When running in synchronous mode, this can only happen ** if the page is on the free-list at the start of the transaction, then ** populated, then moved using sqlite3PagerMovepage(). ** ** The solution is to add an in-memory page to the cache containing - ** the data just read from the sub-journal. Mark the page as dirty - ** and if the pager requires a journal-sync, then mark the page as + ** the data just read from the sub-journal. Mark the page as dirty + ** and if the pager requires a journal-sync, then mark the page as ** requiring a journal-sync before it is written. */ assert( isSavepnt ); @@ -48273,162 +59685,167 @@ static int pager_playback_one_page( if( pgno==1 ){ memcpy(&pPager->dbFileVers, &((u8*)pData)[24],sizeof(pPager->dbFileVers)); } - - /* Decode the page just read from disk */ - CODEC1(pPager, pData, pPg->pgno, 3, rc=SQLITE_NOMEM_BKPT); sqlite3PcacheRelease(pPg); } return rc; } /* -** Parameter zMaster is the name of a master journal file. A single journal -** file that referred to the master journal file has just been rolled back. -** This routine checks if it is possible to delete the master journal file, +** Parameter zSuper is the name of a super-journal file. A single journal +** file that referred to the super-journal file has just been rolled back. +** This routine checks if it is possible to delete the super-journal file, ** and does so if it is. ** -** Argument zMaster may point to Pager.pTmpSpace. So that buffer is not +** Argument zSuper may point to Pager.pTmpSpace. So that buffer is not ** available for use within this function. ** -** When a master journal file is created, it is populated with the names -** of all of its child journals, one after another, formatted as utf-8 -** encoded text. The end of each child journal file is marked with a -** nul-terminator byte (0x00). i.e. the entire contents of a master journal +** When a super-journal file is created, it is populated with the names +** of all of its child journals, one after another, formatted as utf-8 +** encoded text. The end of each child journal file is marked with a +** nul-terminator byte (0x00). i.e. the entire contents of a super-journal ** file for a transaction involving two databases might be: ** ** "/home/bill/a.db-journal\x00/home/bill/b.db-journal\x00" ** -** A master journal file may only be deleted once all of its child +** A super-journal file may only be deleted once all of its child ** journals have been rolled back. ** -** This function reads the contents of the master-journal file into +** This function reads the contents of the super-journal file into ** memory and loops through each of the child journal names. For ** each child journal, it checks if: ** ** * if the child journal exists, and if so -** * if the child journal contains a reference to master journal -** file zMaster +** * if the child journal contains a reference to super-journal +** file zSuper ** ** If a child journal can be found that matches both of the criteria ** above, this function returns without doing anything. Otherwise, if -** no such child journal can be found, file zMaster is deleted from +** no such child journal can be found, file zSuper is deleted from ** the file-system using sqlite3OsDelete(). ** ** If an IO error within this function, an error code is returned. This ** function allocates memory by calling sqlite3Malloc(). If an allocation -** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors +** fails, SQLITE_NOMEM is returned. Otherwise, if no IO or malloc errors ** occur, SQLITE_OK is returned. ** ** TODO: This function allocates a single block of memory to load -** the entire contents of the master journal file. This could be -** a couple of kilobytes or so - potentially larger than the page +** the entire contents of the super-journal file. This could be +** a couple of kilobytes or so - potentially larger than the page ** size. */ -static int pager_delmaster(Pager *pPager, const char *zMaster){ +static int pager_delsuper(Pager *pPager, const char *zSuper){ sqlite3_vfs *pVfs = pPager->pVfs; int rc; /* Return code */ - sqlite3_file *pMaster; /* Malloc'd master-journal file descriptor */ + sqlite3_file *pSuper; /* Malloc'd super-journal file descriptor */ sqlite3_file *pJournal; /* Malloc'd child-journal file descriptor */ - char *zMasterJournal = 0; /* Contents of master journal file */ - i64 nMasterJournal; /* Size of master journal file */ + char *zSuperJournal = 0; /* Contents of super-journal file */ + i64 nSuperJournal; /* Size of super-journal file */ char *zJournal; /* Pointer to one journal within MJ file */ - char *zMasterPtr; /* Space to hold MJ filename from a journal file */ - int nMasterPtr; /* Amount of space allocated to zMasterPtr[] */ + char *zSuperPtr; /* Space to hold super-journal filename */ + char *zFree = 0; /* Free this buffer */ + int nSuperPtr; /* Amount of space allocated to zSuperPtr[] */ - /* Allocate space for both the pJournal and pMaster file descriptors. - ** If successful, open the master journal file for reading. + /* Allocate space for both the pJournal and pSuper file descriptors. + ** If successful, open the super-journal file for reading. */ - pMaster = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); - pJournal = (sqlite3_file *)(((u8 *)pMaster) + pVfs->szOsFile); - if( !pMaster ){ + pSuper = (sqlite3_file *)sqlite3MallocZero(pVfs->szOsFile * 2); + if( !pSuper ){ rc = SQLITE_NOMEM_BKPT; + pJournal = 0; }else{ - const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_MASTER_JOURNAL); - rc = sqlite3OsOpen(pVfs, zMaster, pMaster, flags, 0); + const int flags = (SQLITE_OPEN_READONLY|SQLITE_OPEN_SUPER_JOURNAL); + rc = sqlite3OsOpen(pVfs, zSuper, pSuper, flags, 0); + pJournal = (sqlite3_file *)(((u8 *)pSuper) + pVfs->szOsFile); } - if( rc!=SQLITE_OK ) goto delmaster_out; + if( rc!=SQLITE_OK ) goto delsuper_out; - /* Load the entire master journal file into space obtained from - ** sqlite3_malloc() and pointed to by zMasterJournal. Also obtain - ** sufficient space (in zMasterPtr) to hold the names of master - ** journal files extracted from regular rollback-journals. + /* Load the entire super-journal file into space obtained from + ** sqlite3_malloc() and pointed to by zSuperJournal. Also obtain + ** sufficient space (in zSuperPtr) to hold the names of super-journal + ** files extracted from regular rollback-journals. */ - rc = sqlite3OsFileSize(pMaster, &nMasterJournal); - if( rc!=SQLITE_OK ) goto delmaster_out; - nMasterPtr = pVfs->mxPathname+1; - zMasterJournal = sqlite3Malloc(nMasterJournal + nMasterPtr + 1); - if( !zMasterJournal ){ + rc = sqlite3OsFileSize(pSuper, &nSuperJournal); + if( rc!=SQLITE_OK ) goto delsuper_out; + nSuperPtr = pVfs->mxPathname+1; + zFree = sqlite3Malloc(4 + nSuperJournal + nSuperPtr + 2); + if( !zFree ){ rc = SQLITE_NOMEM_BKPT; - goto delmaster_out; + goto delsuper_out; } - zMasterPtr = &zMasterJournal[nMasterJournal+1]; - rc = sqlite3OsRead(pMaster, zMasterJournal, (int)nMasterJournal, 0); - if( rc!=SQLITE_OK ) goto delmaster_out; - zMasterJournal[nMasterJournal] = 0; + zFree[0] = zFree[1] = zFree[2] = zFree[3] = 0; + zSuperJournal = &zFree[4]; + zSuperPtr = &zSuperJournal[nSuperJournal+2]; + rc = sqlite3OsRead(pSuper, zSuperJournal, (int)nSuperJournal, 0); + if( rc!=SQLITE_OK ) goto delsuper_out; + zSuperJournal[nSuperJournal] = 0; + zSuperJournal[nSuperJournal+1] = 0; - zJournal = zMasterJournal; - while( (zJournal-zMasterJournal)pageSize bytes). +** DBMOD or OPEN state, this function is a no-op. Otherwise, the size +** of the file is changed to nPage pages (nPage*pPager->pageSize bytes). ** If the file on disk is currently larger than nPage pages, then use the VFS ** xTruncate() method to truncate it. ** -** Or, it might be the case that the file on disk is smaller than -** nPage pages. Some operating system implementations can get confused if -** you try to truncate a file to some size that is larger than it -** currently is, so detect this case and write a single zero byte to +** Or, it might be the case that the file on disk is smaller than +** nPage pages. Some operating system implementations can get confused if +** you try to truncate a file to some size that is larger than it +** currently is, so detect this case and write a single zero byte to ** the end of the new file instead. ** ** If successful, return SQLITE_OK. If an IO error occurs while modifying @@ -48438,9 +59855,11 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ int rc = SQLITE_OK; assert( pPager->eState!=PAGER_ERROR ); assert( pPager->eState!=PAGER_READER ); - - if( isOpen(pPager->fd) - && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) + PAGERTRACE(("Truncate %d npage %u\n", PAGERID(pPager), nPage)); + + + if( isOpen(pPager->fd) + && (pPager->eState>=PAGER_WRITER_DBMOD || pPager->eState==PAGER_OPEN) ){ i64 currentSize, newSize; int szPage = pPager->pageSize; @@ -48456,6 +59875,7 @@ static int pager_truncate(Pager *pPager, Pgno nPage){ memset(pTmp, 0, szPage); testcase( (newSize-szPage) == currentSize ); testcase( (newSize-szPage) > currentSize ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_SIZE_HINT, &newSize); rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, newSize-szPage); } if( rc==SQLITE_OK ){ @@ -48484,9 +59904,9 @@ SQLITE_PRIVATE int sqlite3SectorSize(sqlite3_file *pFile){ /* ** Set the value of the Pager.sectorSize variable for the given ** pager based on the value returned by the xSectorSize method -** of the open database file. The sector size will be used -** to determine the size and alignment of journal header and -** master journal pointers within created journal files. +** of the open database file. The sector size will be used +** to determine the size and alignment of journal header and +** super-journal pointers within created journal files. ** ** For temporary files the effective sector size is always 512 bytes. ** @@ -48508,7 +59928,7 @@ static void setSectorSize(Pager *pPager){ assert( isOpen(pPager->fd) || pPager->tempFile ); if( pPager->tempFile - || (sqlite3OsDeviceCharacteristics(pPager->fd) & + || (sqlite3OsDeviceCharacteristics(pPager->fd) & SQLITE_IOCAP_POWERSAFE_OVERWRITE)!=0 ){ /* Sector size doesn't matter for temporary files. Also, the file @@ -48522,15 +59942,15 @@ static void setSectorSize(Pager *pPager){ /* ** Playback the journal and thus restore the database file to -** the state it was in before we started making changes. +** the state it was in before we started making changes. ** -** The journal file format is as follows: +** The journal file format is as follows: ** ** (1) 8 byte prefix. A copy of aJournalMagic[]. ** (2) 4 byte big-endian integer which is the number of valid page records ** in the journal. If this value is 0xffffffff, then compute the ** number of page records from the journal size. -** (3) 4 byte big-endian integer which is the initial value for the +** (3) 4 byte big-endian integer which is the initial value for the ** sanity checksum. ** (4) 4 byte integer which is the number of pages to truncate the ** database to during a rollback. @@ -48559,7 +59979,7 @@ static void setSectorSize(Pager *pPager){ ** from the file size. This value is used when the user selects the ** no-sync option for the journal. A power failure could lead to corruption ** in this case. But for things like temporary table (which will be -** deleted when the power is restored) we don't care. +** deleted when the power is restored) we don't care. ** ** If the file opened as the journal file is not a well-formed ** journal file then all pages up to the first corrupted page are rolled @@ -48571,7 +59991,7 @@ static void setSectorSize(Pager *pPager){ ** and an error code is returned. ** ** The isHot parameter indicates that we are trying to rollback a journal -** that might be a hot journal. Or, it could be that the journal is +** that might be a hot journal. Or, it could be that the journal is ** preserved because of JOURNALMODE_PERSIST or JOURNALMODE_TRUNCATE. ** If the journal really is hot, reset the pager cache prior rolling ** back any content. If the journal is merely persistent, no reset is @@ -48585,9 +60005,10 @@ static int pager_playback(Pager *pPager, int isHot){ Pgno mxPg = 0; /* Size of the original file in pages */ int rc; /* Result code of a subroutine */ int res = 1; /* Value returned by sqlite3OsAccess() */ - char *zMaster = 0; /* Name of master journal file if any */ + char *zSuper = 0; /* Name of super-journal file if any */ int needPagerReset; /* True to reset page prior to first page rollback */ int nPlayback = 0; /* Total number of pages restored from journal */ + u32 savedPageSize = pPager->pageSize; /* Figure out how many records are in the journal. Abort early if ** the journal is empty. @@ -48598,8 +60019,8 @@ static int pager_playback(Pager *pPager, int isHot){ goto end_playback; } - /* Read the master journal name from the journal, if it is present. - ** If a master journal file name is specified, but the file is not + /* Read the super-journal name from the journal, if it is present. + ** If a super-journal file name is specified, but the file is not ** present on disk, then the journal is not hot and does not need to be ** played back. ** @@ -48609,21 +60030,21 @@ static int pager_playback(Pager *pPager, int isHot){ ** mxPathname is 512, which is the same as the minimum allowable value ** for pageSize. */ - zMaster = pPager->pTmpSpace; - rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); - if( rc==SQLITE_OK && zMaster[0] ){ - rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); + zSuper = pPager->pTmpSpace; + rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); + if( rc==SQLITE_OK && zSuper[0] ){ + rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); } - zMaster = 0; + zSuper = 0; if( rc!=SQLITE_OK || !res ){ goto end_playback; } pPager->journalOff = 0; needPagerReset = isHot; - /* This loop terminates either when a readJournalHdr() or - ** pager_playback_one_page() call returns SQLITE_DONE or an IO error - ** occurs. + /* This loop terminates either when a readJournalHdr() or + ** pager_playback_one_page() call returns SQLITE_DONE or an IO error + ** occurs. */ while( 1 ){ /* Read the next journal header from the journal file. If there are @@ -48632,7 +60053,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** This indicates nothing more needs to be rolled back. */ rc = readJournalHdr(pPager, isHot, szJ, &nRec, &mxPg); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK ){ if( rc==SQLITE_DONE ){ rc = SQLITE_OK; } @@ -48660,7 +60081,7 @@ static int pager_playback(Pager *pPager, int isHot){ ** chunk of the journal contains zero pages to be rolled back. But ** when doing a ROLLBACK and the nRec==0 chunk is the last chunk in ** the journal, it means that the journal might contain additional - ** pages that need to be rolled back and that the number of pages + ** pages that need to be rolled back and that the number of pages ** should be computed based on the journal file size. */ if( nRec==0 && !isHot && @@ -48677,9 +60098,12 @@ static int pager_playback(Pager *pPager, int isHot){ goto end_playback; } pPager->dbSize = mxPg; + if( pPager->mxPgnomxPgno = mxPg; + } } - /* Copy original pages out of the journal and back into the + /* Copy original pages out of the journal and back into the ** database file and/or page cache. */ for(u=0; ufd->pMethods ){ - sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0); - } + sqlite3OsFileControlHint(pPager->fd,SQLITE_FCNTL_DB_UNCHANGED,0); #endif - /* If this playback is happening automatically as a result of an IO or - ** malloc error that occurred after the change-counter was updated but - ** before the transaction was committed, then the change-counter - ** modification may just have been reverted. If this happens in exclusive + /* If this playback is happening automatically as a result of an IO or + ** malloc error that occurred after the change-counter was updated but + ** before the transaction was committed, then the change-counter + ** modification may just have been reverted. If this happens in exclusive ** mode, then subsequent transactions performed by the connection will not ** update the change-counter at all. This may lead to cache inconsistency ** problems for other processes at some point in the future. So, just @@ -48740,8 +60165,12 @@ end_playback: pPager->changeCountDone = pPager->tempFile; if( rc==SQLITE_OK ){ - zMaster = pPager->pTmpSpace; - rc = readMasterJournal(pPager->jfd, zMaster, pPager->pVfs->mxPathname+1); + /* Leave 4 bytes of space before the super-journal filename in memory. + ** This is because it may end up being passed to sqlite3OsOpen(), in + ** which case it requires 4 0x00 bytes in memory immediately before + ** the filename. */ + zSuper = &pPager->pTmpSpace[4]; + rc = readSuperJournal(pPager->jfd, zSuper, pPager->pVfs->mxPathname+1); testcase( rc!=SQLITE_OK ); } if( rc==SQLITE_OK @@ -48750,14 +60179,16 @@ end_playback: rc = sqlite3PagerSync(pPager, 0); } if( rc==SQLITE_OK ){ - rc = pager_end_transaction(pPager, zMaster[0]!='\0', 0); + rc = pager_end_transaction(pPager, zSuper[0]!='\0', 0); testcase( rc!=SQLITE_OK ); } - if( rc==SQLITE_OK && zMaster[0] && res ){ - /* If there was a master journal and this routine will return success, - ** see if it is possible to delete the master journal. + if( rc==SQLITE_OK && zSuper[0] && res ){ + /* If there was a super-journal and this routine will return success, + ** see if it is possible to delete the super-journal. */ - rc = pager_delmaster(pPager, zMaster); + assert( zSuper==&pPager->pTmpSpace[4] ); + memset(pPager->pTmpSpace, 0, 4); + rc = pager_delsuper(pPager, zSuper); testcase( rc!=SQLITE_OK ); } if( isHot && nPlayback ){ @@ -48775,7 +60206,8 @@ end_playback: /* -** Read the content for page pPg out of the database file and into +** Read the content for page pPg out of the database file (or out of +** the WAL if that is where the most recent copy if found) into ** pPg->pData. A shared lock or greater must be held on the database ** file before this function is called. ** @@ -48785,30 +60217,33 @@ end_playback: ** If an IO error occurs, then the IO error is returned to the caller. ** Otherwise, SQLITE_OK is returned. */ -static int readDbPage(PgHdr *pPg, u32 iFrame){ +static int readDbPage(PgHdr *pPg){ Pager *pPager = pPg->pPager; /* Pager object associated with page pPg */ - Pgno pgno = pPg->pgno; /* Page number to read */ int rc = SQLITE_OK; /* Return code */ - int pgsz = pPager->pageSize; /* Number of bytes to read */ + +#ifndef SQLITE_OMIT_WAL + u32 iFrame = 0; /* Frame of WAL containing pgno */ assert( pPager->eState>=PAGER_READER && !MEMDB ); assert( isOpen(pPager->fd) ); -#ifndef SQLITE_OMIT_WAL + if( pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); + if( rc ) return rc; + } if( iFrame ){ - /* Try to pull the page from the write-ahead log. */ - rc = sqlite3WalReadFrame(pPager->pWal, iFrame, pgsz, pPg->pData); + rc = sqlite3WalReadFrame(pPager->pWal, iFrame,pPager->pageSize,pPg->pData); }else #endif { - i64 iOffset = (pgno-1)*(i64)pPager->pageSize; - rc = sqlite3OsRead(pPager->fd, pPg->pData, pgsz, iOffset); + i64 iOffset = (pPg->pgno-1)*(i64)pPager->pageSize; + rc = sqlite3OsRead(pPager->fd, pPg->pData, pPager->pageSize, iOffset); if( rc==SQLITE_IOERR_SHORT_READ ){ rc = SQLITE_OK; } } - if( pgno==1 ){ + if( pPg->pgno==1 ){ if( rc ){ /* If the read is unsuccessful, set the dbFileVers[] to something ** that will never be a valid file version. dbFileVers[] is a copy @@ -48828,13 +60263,11 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ memcpy(&pPager->dbFileVers, dbFileVers, sizeof(pPager->dbFileVers)); } } - CODEC1(pPager, pPg->pData, pgno, 3, rc = SQLITE_NOMEM_BKPT); - PAGER_INCR(sqlite3_pager_readdb_count); PAGER_INCR(pPager->nRead); - IOTRACE(("PGIN %p %d\n", pPager, pgno)); + IOTRACE(("PGIN %p %d\n", pPager, pPg->pgno)); PAGERTRACE(("FETCH %d page %d hash(%08x)\n", - PAGERID(pPager), pgno, pager_pagehash(pPg))); + PAGERID(pPager), pPg->pgno, pager_pagehash(pPg))); return rc; } @@ -48849,6 +60282,7 @@ static int readDbPage(PgHdr *pPg, u32 iFrame){ */ static void pager_write_changecounter(PgHdr *pPg){ u32 change_counter; + if( NEVER(pPg==0) ) return; /* Increment the value just read and write it back to byte 24. */ change_counter = sqlite3Get4byte((u8*)pPg->pPager->dbFileVers)+1; @@ -48863,15 +60297,15 @@ static void pager_write_changecounter(PgHdr *pPg){ #ifndef SQLITE_OMIT_WAL /* -** This function is invoked once for each page that has already been +** This function is invoked once for each page that has already been ** written into the log file when a WAL transaction is rolled back. -** Parameter iPg is the page number of said page. The pCtx argument +** Parameter iPg is the page number of said page. The pCtx argument ** is actually a pointer to the Pager structure. ** ** If page iPg is present in the cache, and has no outstanding references, ** it is discarded. Otherwise, if there are one or more outstanding ** references, the page content is reloaded from the database. If the -** attempt to reload content from the database is required and fails, +** attempt to reload content from the database is required and fails, ** return an SQLite error code. Otherwise, SQLITE_OK. */ static int pagerUndoCallback(void *pCtx, Pgno iPg){ @@ -48885,11 +60319,7 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ if( sqlite3PcachePageRefcount(pPg)==1 ){ sqlite3PcacheDrop(pPg); }else{ - u32 iFrame = 0; - rc = sqlite3WalFindFrame(pPager->pWal, pPg->pgno, &iFrame); - if( rc==SQLITE_OK ){ - rc = readDbPage(pPg, iFrame); - } + rc = readDbPage(pPg); if( rc==SQLITE_OK ){ pPager->xReiniter(pPg); } @@ -48901,7 +60331,7 @@ static int pagerUndoCallback(void *pCtx, Pgno iPg){ ** updated as data is copied out of the rollback journal and into the ** database. This is not generally possible with a WAL database, as ** rollback involves simply truncating the log file. Therefore, if one - ** or more frames have already been written to the log (and therefore + ** or more frames have already been written to the log (and therefore ** also copied into the backup databases) as part of this transaction, ** the backups must be restarted. */ @@ -48918,7 +60348,7 @@ static int pagerRollbackWal(Pager *pPager){ PgHdr *pList; /* List of dirty pages to revert */ /* For all pages in the cache that are currently dirty or have already - ** been written (but not committed) to the log file, do one of the + ** been written (but not committed) to the log file, do one of the ** following: ** ** + Discard the cached page (if refcount==0), or @@ -48940,11 +60370,11 @@ static int pagerRollbackWal(Pager *pPager){ ** This function is a wrapper around sqlite3WalFrames(). As well as logging ** the contents of the list of pages headed by pList (connected by pDirty), ** this function notifies any active backup processes that the pages have -** changed. +** changed. ** ** The list of pages passed into this routine is always sorted by page number. ** Hence, if page 1 appears anywhere on the list, it will be the first page. -*/ +*/ static int pagerWalFrames( Pager *pPager, /* Pager object */ PgHdr *pList, /* List of frames to log */ @@ -48958,7 +60388,7 @@ static int pagerWalFrames( assert( pPager->pWal ); assert( pList ); #ifdef SQLITE_DEBUG - /* Verify that the page list is in accending order */ + /* Verify that the page list is in ascending order */ for(p=pList; p && p->pDirty; p=p->pDirty){ assert( p->pgno < p->pDirty->pgno ); } @@ -48985,7 +60415,7 @@ static int pagerWalFrames( pPager->aStat[PAGER_STAT_WRITE] += nList; if( pList->pgno==1 ) pager_write_changecounter(pList); - rc = sqlite3WalFrames(pPager->pWal, + rc = sqlite3WalFrames(pPager->pWal, pPager->pageSize, pList, nTruncate, isCommit, pPager->walSyncFlags ); if( rc==SQLITE_OK && pPager->pBackup ){ @@ -49061,7 +60491,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ nPage = sqlite3WalDbsize(pPager->pWal); /* If the number of pages in the database is not available from the - ** WAL sub-system, determine the page counte based on the size of + ** WAL sub-system, determine the page count based on the size of ** the database file. If the size of the database file is not an ** integer multiple of the page-size, round up the result. */ @@ -49089,7 +60519,7 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ #ifndef SQLITE_OMIT_WAL /* ** Check if the *-wal file that corresponds to the database opened by pPager -** exists if the database is not empy, or verify that the *-wal file does +** exists if the database is not empty, or verify that the *-wal file does ** not exist (by deleting it) if the database file is empty. ** ** If the database is not empty and the *-wal file exists, open the pager @@ -49100,9 +60530,9 @@ static int pagerPagecount(Pager *pPager, Pgno *pnPage){ ** Return SQLITE_OK or an error code. ** ** The caller must hold a SHARED lock on the database file to call this -** function. Because an EXCLUSIVE lock on the db file is required to delete -** a WAL on a none-empty database, this ensures there is no race condition -** between the xAccess() below and an xDelete() being executed by some +** function. Because an EXCLUSIVE lock on the db file is required to delete +** a WAL on a none-empty database, this ensures there is no race condition +** between the xAccess() below and an xDelete() being executed by some ** other connection. */ static int pagerOpenWalIfPresent(Pager *pPager){ @@ -49112,23 +60542,21 @@ static int pagerOpenWalIfPresent(Pager *pPager){ if( !pPager->tempFile ){ int isWal; /* True if WAL file exists */ - Pgno nPage; /* Size of the database file */ - - rc = pagerPagecount(pPager, &nPage); - if( rc ) return rc; - if( nPage==0 ){ - rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); - if( rc==SQLITE_IOERR_DELETE_NOENT ) rc = SQLITE_OK; - isWal = 0; - }else{ - rc = sqlite3OsAccess( - pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal - ); - } + rc = sqlite3OsAccess( + pPager->pVfs, pPager->zWal, SQLITE_ACCESS_EXISTS, &isWal + ); if( rc==SQLITE_OK ){ if( isWal ){ - testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); - rc = sqlite3PagerOpenWal(pPager, 0); + Pgno nPage; /* Size of the database file */ + + rc = pagerPagecount(pPager, &nPage); + if( rc ) return rc; + if( nPage==0 ){ + rc = sqlite3OsDelete(pPager->pVfs, pPager->zWal, 0); + }else{ + testcase( sqlite3PcachePagecount(pPager->pPCache)==0 ); + rc = sqlite3PagerOpenWal(pPager, 0); + } }else if( pPager->journalMode==PAGER_JOURNALMODE_WAL ){ pPager->journalMode = PAGER_JOURNALMODE_DELETE; } @@ -49140,21 +60568,21 @@ static int pagerOpenWalIfPresent(Pager *pPager){ /* ** Playback savepoint pSavepoint. Or, if pSavepoint==NULL, then playback -** the entire master journal file. The case pSavepoint==NULL occurs when -** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction +** the entire super-journal file. The case pSavepoint==NULL occurs when +** a ROLLBACK TO command is invoked on a SAVEPOINT that is a transaction ** savepoint. ** -** When pSavepoint is not NULL (meaning a non-transaction savepoint is +** When pSavepoint is not NULL (meaning a non-transaction savepoint is ** being rolled back), then the rollback consists of up to three stages, ** performed in the order specified: ** ** * Pages are played back from the main journal starting at byte -** offset PagerSavepoint.iOffset and continuing to +** offset PagerSavepoint.iOffset and continuing to ** PagerSavepoint.iHdrOffset, or to the end of the main journal ** file if PagerSavepoint.iHdrOffset is zero. ** ** * If PagerSavepoint.iHdrOffset is not zero, then pages are played -** back starting from the journal header immediately following +** back starting from the journal header immediately following ** PagerSavepoint.iHdrOffset to the end of the main journal file. ** ** * Pages are then played back from the sub-journal file, starting @@ -49170,7 +60598,7 @@ static int pagerOpenWalIfPresent(Pager *pPager){ ** journal file. There is no need for a bitvec in this case. ** ** In either case, before playback commences the Pager.dbSize variable -** is reset to the value that it held at the start of the savepoint +** is reset to the value that it held at the start of the savepoint ** (or transaction). No page with a page-number greater than this value ** is played back. If one is encountered it is simply skipped. */ @@ -49191,7 +60619,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ } } - /* Set the database size back to the value it was before the savepoint + /* Set the database size back to the value it was before the savepoint ** being reverted was opened. */ pPager->dbSize = pSavepoint ? pSavepoint->nOrig : pPager->dbOrigSize; @@ -49244,7 +60672,7 @@ static int pagerPlaybackSavepoint(Pager *pPager, PagerSavepoint *pSavepoint){ ** test is related to ticket #2565. See the discussion in the ** pager_playback() function for additional information. */ - if( nJRec==0 + if( nJRec==0 && pPager->journalHdr+JOURNAL_HDR_SZ(pPager)==pPager->journalOff ){ nJRec = (u32)((szJ - pPager->journalOff)/JOURNAL_PG_SZ(pPager)); @@ -49308,6 +60736,7 @@ static void pagerFixMaplimit(Pager *pPager){ sqlite3_int64 sz; sz = pPager->szMmap; pPager->bUseFetch = (sz>0); + setGetterMethod(pPager); sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_MMAP_SIZE, &sz); } #endif @@ -49379,7 +60808,6 @@ SQLITE_PRIVATE void sqlite3PagerShrink(Pager *pPager){ ** Numeric values associated with these states are OFF==1, NORMAL=2, ** and FULL=3. */ -#ifndef SQLITE_OMIT_PAGER_PRAGMAS SQLITE_PRIVATE void sqlite3PagerSetFlags( Pager *pPager, /* The pager to set safety level for */ unsigned pgFlags /* Various flags */ @@ -49396,20 +60824,17 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( } if( pPager->noSync ){ pPager->syncFlags = 0; - pPager->ckptSyncFlags = 0; }else if( pgFlags & PAGER_FULLFSYNC ){ pPager->syncFlags = SQLITE_SYNC_FULL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; - }else if( pgFlags & PAGER_CKPT_FULLFSYNC ){ - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_FULL; }else{ pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; } - pPager->walSyncFlags = pPager->syncFlags; + pPager->walSyncFlags = (pPager->syncFlags<<2); if( pPager->fullSync ){ - pPager->walSyncFlags |= WAL_SYNC_TRANSACTIONS; + pPager->walSyncFlags |= pPager->syncFlags; + } + if( (pgFlags & PAGER_CKPT_FULLFSYNC) && !pPager->noSync ){ + pPager->walSyncFlags |= (SQLITE_SYNC_FULL<<2); } if( pgFlags & PAGER_CACHESPILL ){ pPager->doNotSpill &= ~SPILLFLAG_OFF; @@ -49417,12 +60842,11 @@ SQLITE_PRIVATE void sqlite3PagerSetFlags( pPager->doNotSpill |= SPILLFLAG_OFF; } } -#endif /* ** The following global variable is incremented whenever the library ** attempts to open a temporary file. This information is used for -** testing and analysis only. +** testing and analysis only. */ #ifdef SQLITE_TEST SQLITE_API int sqlite3_opentemp_count = 0; @@ -49431,8 +60855,8 @@ SQLITE_API int sqlite3_opentemp_count = 0; /* ** Open a temporary file. ** -** Write the file descriptor into *pFile. Return SQLITE_OK on success -** or some other error code if we fail. The OS will automatically +** Write the file descriptor into *pFile. Return SQLITE_OK on success +** or some other error code if we fail. The OS will automatically ** delete the temporary file when it is closed. ** ** The flags passed to the VFS layer xOpen() call are those specified @@ -49464,9 +60888,9 @@ static int pagerOpentemp( /* ** Set the busy handler function. ** -** The pager invokes the busy-handler if sqlite3OsLock() returns +** The pager invokes the busy-handler if sqlite3OsLock() returns ** SQLITE_BUSY when trying to upgrade from no-lock to a SHARED lock, -** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE +** or when trying to upgrade from a RESERVED lock to an EXCLUSIVE ** lock. It does *not* invoke the busy handler when upgrading from ** SHARED to RESERVED, or when upgrading from SHARED to EXCLUSIVE ** (which occurs during hot-journal rollback). Summary: @@ -49478,37 +60902,35 @@ static int pagerOpentemp( ** SHARED_LOCK -> EXCLUSIVE_LOCK | No ** RESERVED_LOCK -> EXCLUSIVE_LOCK | Yes ** -** If the busy-handler callback returns non-zero, the lock is +** If the busy-handler callback returns non-zero, the lock is ** retried. If it returns zero, then the SQLITE_BUSY error is ** returned to the caller of the pager API function. */ -SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( +SQLITE_PRIVATE void sqlite3PagerSetBusyHandler( Pager *pPager, /* Pager object */ int (*xBusyHandler)(void *), /* Pointer to busy-handler function */ void *pBusyHandlerArg /* Argument to pass to xBusyHandler */ ){ + void **ap; pPager->xBusyHandler = xBusyHandler; pPager->pBusyHandlerArg = pBusyHandlerArg; - - if( isOpen(pPager->fd) ){ - void **ap = (void **)&pPager->xBusyHandler; - assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); - assert( ap[1]==pBusyHandlerArg ); - sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); - } + ap = (void **)&pPager->xBusyHandler; + assert( ((int(*)(void *))(ap[0]))==xBusyHandler ); + assert( ap[1]==pBusyHandlerArg ); + sqlite3OsFileControlHint(pPager->fd, SQLITE_FCNTL_BUSYHANDLER, (void *)ap); } /* -** Change the page size used by the Pager object. The new page size +** Change the page size used by the Pager object. The new page size ** is passed in *pPageSize. ** ** If the pager is in the error state when this function is called, it -** is a no-op. The value returned is the error state error code (i.e. +** is a no-op. The value returned is the error state error code (i.e. ** one of SQLITE_IOERR, an SQLITE_IOERR_xxx sub-code or SQLITE_FULL). ** ** Otherwise, if all of the following are true: ** -** * the new page size (value of *pPageSize) is valid (a power +** * the new page size (value of *pPageSize) is valid (a power ** of two between 512 and SQLITE_MAX_PAGE_SIZE, inclusive), and ** ** * there are no outstanding page references, and @@ -49518,14 +60940,14 @@ SQLITE_PRIVATE void sqlite3PagerSetBusyhandler( ** ** then the pager object page size is set to *pPageSize. ** -** If the page size is changed, then this function uses sqlite3PagerMalloc() -** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt -** fails, SQLITE_NOMEM is returned and the page size remains unchanged. +** If the page size is changed, then this function uses sqlite3PagerMalloc() +** to obtain a new Pager.pTmpSpace buffer. If this allocation attempt +** fails, SQLITE_NOMEM is returned and the page size remains unchanged. ** In all other cases, SQLITE_OK is returned. ** ** If the page size is not changed, either because one of the enumerated ** conditions above is not true, the pager was in error state when this -** function was called, or because the memory allocation attempt failed, +** function was called, or because the memory allocation attempt failed, ** then *pPageSize is set to the old, retained page size before returning. */ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nReserve){ @@ -49535,7 +60957,7 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR ** function may be called from within PagerOpen(), before the state ** of the Pager object is internally consistent. ** - ** At one point this function returned an error if the pager was in + ** At one point this function returned an error if the pager was in ** PAGER_ERROR state. But since PAGER_ERROR state guarantees that ** there is at least one outstanding page reference, this function ** is a no-op for that case anyhow. @@ -49544,8 +60966,8 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR u32 pageSize = *pPageSize; assert( pageSize==0 || (pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE) ); if( (pPager->memDb==0 || pPager->dbSize==0) - && sqlite3PcacheRefCount(pPager->pPCache)==0 - && pageSize && pageSize!=(u32)pPager->pageSize + && sqlite3PcacheRefCount(pPager->pPCache)==0 + && pageSize && pageSize!=(u32)pPager->pageSize ){ char *pNew = NULL; /* New temp space */ i64 nByte = 0; @@ -49554,8 +60976,14 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR rc = sqlite3OsFileSize(pPager->fd, &nByte); } if( rc==SQLITE_OK ){ - pNew = (char *)sqlite3PageMalloc(pageSize); - if( !pNew ) rc = SQLITE_NOMEM_BKPT; + /* 8 bytes of zeroed overrun space is sufficient so that the b-tree + * cell header parser will never run off the end of the allocation */ + pNew = (char *)sqlite3PageMalloc(pageSize+8); + if( !pNew ){ + rc = SQLITE_NOMEM_BKPT; + }else{ + memset(pNew+pageSize, 0, 8); + } } if( rc==SQLITE_OK ){ @@ -49567,6 +60995,7 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR pPager->pTmpSpace = pNew; pPager->dbSize = (Pgno)((nByte+pageSize-1)/pageSize); pPager->pageSize = pageSize; + pPager->lckPgno = (Pgno)(PENDING_BYTE/pageSize) + 1; }else{ sqlite3PageFree(pNew); } @@ -49577,7 +61006,6 @@ SQLITE_PRIVATE int sqlite3PagerSetPagesize(Pager *pPager, u32 *pPageSize, int nR if( nReserve<0 ) nReserve = pPager->nReserve; assert( nReserve>=0 && nReserve<1000 ); pPager->nReserve = (i16)nReserve; - pagerReportSize(pPager); pagerFixMaplimit(pPager); } return rc; @@ -49596,18 +61024,21 @@ SQLITE_PRIVATE void *sqlite3PagerTempSpace(Pager *pPager){ } /* -** Attempt to set the maximum database page count if mxPage is positive. +** Attempt to set the maximum database page count if mxPage is positive. ** Make no changes if mxPage is zero or negative. And never reduce the ** maximum page count below the current size of the database. ** ** Regardless of mxPage, return the current maximum page count. */ -SQLITE_PRIVATE int sqlite3PagerMaxPageCount(Pager *pPager, int mxPage){ +SQLITE_PRIVATE Pgno sqlite3PagerMaxPageCount(Pager *pPager, Pgno mxPage){ if( mxPage>0 ){ pPager->mxPgno = mxPage; } assert( pPager->eState!=PAGER_OPEN ); /* Called only by OP_MaxPgcnt */ - assert( pPager->mxPgno>=pPager->dbSize ); /* OP_MaxPgcnt enforces this */ + /* assert( pPager->mxPgno>=pPager->dbSize ); */ + /* OP_MaxPgcnt ensures that the parameter passed to this function is not + ** less than the total number of valid pages in the database. But this + ** may be less than Pager.dbSize, and so the assert() above is not valid */ return pPager->mxPgno; } @@ -49637,11 +61068,11 @@ void enable_simulated_io_errors(void){ /* ** Read the first N bytes from the beginning of the file into memory -** that pDest points to. +** that pDest points to. ** ** If the pager was opened on a transient file (zFilename==""), or ** opened on a file less than N bytes in size, the output buffer is -** zeroed and SQLITE_OK returned. The rationale for this is that this +** zeroed and SQLITE_OK returned. The rationale for this is that this ** function is used to read database headers, and a new transient or ** zero sized database has a header than consists entirely of zeroes. ** @@ -49674,7 +61105,7 @@ SQLITE_PRIVATE int sqlite3PagerReadFileheader(Pager *pPager, int N, unsigned cha ** This function may only be called when a read-transaction is open on ** the pager. It returns the total number of pages in the database. ** -** However, if the file is between 1 and bytes in size, then +** However, if the file is between 1 and bytes in size, then ** this is considered a 1 page file. */ SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ @@ -49689,19 +61120,19 @@ SQLITE_PRIVATE void sqlite3PagerPagecount(Pager *pPager, int *pnPage){ ** a similar or greater lock is already held, this function is a no-op ** (returning SQLITE_OK immediately). ** -** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke -** the busy callback if the lock is currently not available. Repeat -** until the busy callback returns false or until the attempt to +** Otherwise, attempt to obtain the lock using sqlite3OsLock(). Invoke +** the busy callback if the lock is currently not available. Repeat +** until the busy callback returns false or until the attempt to ** obtain the lock succeeds. ** ** Return SQLITE_OK on success and an error code if we cannot obtain -** the lock. If the lock is obtained successfully, set the Pager.state +** the lock. If the lock is obtained successfully, set the Pager.state ** variable to locktype before returning. */ static int pager_wait_on_lock(Pager *pPager, int locktype){ int rc; /* Return code */ - /* Check that this is either a no-op (because the requested lock is + /* Check that this is either a no-op (because the requested lock is ** already held), or one of the transitions that the busy-handler ** may be invoked during, according to the comment above ** sqlite3PagerSetBusyhandler(). @@ -49718,15 +61149,14 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ } /* -** Function assertTruncateConstraint(pPager) checks that one of the +** Function assertTruncateConstraint(pPager) checks that one of the ** following is true for all dirty pages currently in the page-cache: ** -** a) The page number is less than or equal to the size of the +** a) The page number is less than or equal to the size of the ** current database image, in pages, OR ** ** b) if the page content were written at this time, it would not -** be necessary to write the current content out to the sub-journal -** (as determined by function subjRequiresPage()). +** be necessary to write the current content out to the sub-journal. ** ** If the condition asserted by this function were not true, and the ** dirty page were to be discarded from the cache via the pagerStress() @@ -49734,15 +61164,23 @@ static int pager_wait_on_lock(Pager *pPager, int locktype){ ** the database file. If a savepoint transaction were rolled back after ** this happened, the correct behavior would be to restore the current ** content of the page. However, since this content is not present in either -** the database file or the portion of the rollback journal and +** the database file or the portion of the rollback journal and ** sub-journal rolled back the content could not be restored and the -** database image would become corrupt. It is therefore fortunate that +** database image would become corrupt. It is therefore fortunate that ** this circumstance cannot arise. */ #if defined(SQLITE_DEBUG) static void assertTruncateConstraintCb(PgHdr *pPg){ + Pager *pPager = pPg->pPager; assert( pPg->flags&PGHDR_DIRTY ); - assert( !subjRequiresPage(pPg) || pPg->pgno<=pPg->pPager->dbSize ); + if( pPg->pgno>pPager->dbSize ){ /* if (a) is false */ + Pgno pgno = pPg->pgno; + int i; + for(i=0; ipPager->nSavepoint; i++){ + PagerSavepoint *p = &pPager->aSavepoint[i]; + assert( p->nOrigpInSavepoint,pgno) ); + } + } } static void assertTruncateConstraint(Pager *pPager){ sqlite3PcacheIterateDirty(pPager->pPCache, assertTruncateConstraintCb); @@ -49752,9 +61190,9 @@ static void assertTruncateConstraint(Pager *pPager){ #endif /* -** Truncate the in-memory database file image to nPage pages. This -** function does not actually modify the database file on disk. It -** just sets the internal state of the pager object so that the +** Truncate the in-memory database file image to nPage pages. This +** function does not actually modify the database file on disk. It +** just sets the internal state of the pager object so that the ** truncation will be done when the current transaction is committed. ** ** This function is only called right before committing a transaction. @@ -49763,17 +61201,17 @@ static void assertTruncateConstraint(Pager *pPager){ ** then continue writing to the database. */ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ - assert( pPager->dbSize>=nPage ); + assert( pPager->dbSize>=nPage || CORRUPT_DB ); assert( pPager->eState>=PAGER_WRITER_CACHEMOD ); pPager->dbSize = nPage; /* At one point the code here called assertTruncateConstraint() to ** ensure that all pages being truncated away by this operation are, - ** if one or more savepoints are open, present in the savepoint + ** if one or more savepoints are open, present in the savepoint ** journal so that they can be restored if the savepoint is rolled ** back. This is no longer necessary as this function is now only - ** called right before committing a transaction. So although the - ** Pager object may still have open savepoints (Pager.nSavepoint!=0), + ** called right before committing a transaction. So although the + ** Pager object may still have open savepoints (Pager.nSavepoint!=0), ** they cannot be rolled back. So the assertTruncateConstraint() call ** is no longer correct. */ } @@ -49785,12 +61223,12 @@ SQLITE_PRIVATE void sqlite3PagerTruncateImage(Pager *pPager, Pgno nPage){ ** size of the journal file so that the pager_playback() routine knows ** that the entire journal file has been synced. ** -** Syncing a hot-journal to disk before attempting to roll it back ensures +** Syncing a hot-journal to disk before attempting to roll it back ensures ** that if a power-failure occurs during the rollback, the process that ** attempts rollback following system recovery sees the same journal ** content as this process. ** -** If everything goes as planned, SQLITE_OK is returned. Otherwise, +** If everything goes as planned, SQLITE_OK is returned. Otherwise, ** an SQLite error code. */ static int pagerSyncHotJournal(Pager *pPager){ @@ -49804,8 +61242,9 @@ static int pagerSyncHotJournal(Pager *pPager){ return rc; } +#if SQLITE_MAX_MMAP_SIZE>0 /* -** Obtain a reference to a memory mapped page object for page number pgno. +** Obtain a reference to a memory mapped page object for page number pgno. ** The new object will use the pointer pData, obtained from xFetch(). ** If successful, set *ppPage to point to the new page reference ** and return SQLITE_OK. Otherwise, return an SQLite error code and set @@ -49821,12 +61260,13 @@ static int pagerAcquireMapPage( PgHdr **ppPage /* OUT: Acquired page object */ ){ PgHdr *p; /* Memory mapped page to return */ - + if( pPager->pMmapFreelist ){ *ppPage = p = pPager->pMmapFreelist; pPager->pMmapFreelist = p->pDirty; p->pDirty = 0; - memset(p->pExtra, 0, pPager->nExtra); + assert( pPager->nExtra>=8 ); + memset(p->pExtra, 0, 8); }else{ *ppPage = p = (PgHdr *)sqlite3MallocZero(sizeof(PgHdr) + pPager->nExtra); if( p==0 ){ @@ -49834,6 +61274,7 @@ static int pagerAcquireMapPage( return SQLITE_NOMEM_BKPT; } p->pExtra = (void *)&p[1]; + assert( EIGHT_BYTE_ALIGNMENT( p->pExtra ) ); p->flags = PGHDR_MMAP; p->nRef = 1; p->pPager = pPager; @@ -49851,9 +61292,10 @@ static int pagerAcquireMapPage( return SQLITE_OK; } +#endif /* -** Release a reference to page pPg. pPg must have been returned by an +** Release a reference to page pPg. pPg must have been returned by an ** earlier call to pagerAcquireMapPage(). */ static void pagerReleaseMapPage(PgHdr *pPg){ @@ -49878,6 +61320,30 @@ static void pagerFreeMapHdrs(Pager *pPager){ } } +/* Verify that the database file has not be deleted or renamed out from +** under the pager. Return SQLITE_OK if the database is still where it ought +** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error +** code from sqlite3OsAccess()) if the database has gone missing. +*/ +static int databaseIsUnmoved(Pager *pPager){ + int bHasMoved = 0; + int rc; + + if( pPager->tempFile ) return SQLITE_OK; + if( pPager->dbSize==0 ) return SQLITE_OK; + assert( pPager->zFilename && pPager->zFilename[0] ); + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); + if( rc==SQLITE_NOTFOUND ){ + /* If the HAS_MOVED file-control is unimplemented, assume that the file + ** has not been moved. That is the historical behavior of SQLite: prior to + ** version 3.8.3, it never checked */ + rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bHasMoved ){ + rc = SQLITE_READONLY_DBMOVED; + } + return rc; +} + /* ** Shutdown the page cache. Free all memory and close all files. @@ -49889,13 +61355,13 @@ static void pagerFreeMapHdrs(Pager *pPager){ ** result in a coredump. ** ** This function always succeeds. If a transaction is active an attempt -** is made to roll it back. If an error occurs during the rollback +** is made to roll it back. If an error occurs during the rollback ** a hot journal may be left in the filesystem but no error is returned ** to the caller. */ -SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ - u8 *pTmp = (u8 *)pPager->pTmpSpace; - +SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ + u8 *pTmp = (u8*)pPager->pTmpSpace; + assert( db || pagerUseWal(pPager)==0 ); assert( assert_pager_state(pPager) ); disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); @@ -49903,16 +61369,25 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ /* pPager->errCode = 0; */ pPager->exclusiveMode = 0; #ifndef SQLITE_OMIT_WAL - sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, pPager->pageSize, pTmp); - pPager->pWal = 0; + { + u8 *a = 0; + assert( db || pPager->pWal==0 ); + if( db && 0==(db->flags & SQLITE_NoCkptOnClose) + && SQLITE_OK==databaseIsUnmoved(pPager) + ){ + a = pTmp; + } + sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize,a); + pPager->pWal = 0; + } #endif pager_reset(pPager); if( MEMDB ){ pager_unlock(pPager); }else{ /* If it is open, sync the journal file before calling UnlockAndRollback. - ** If this is not done, then an unsynced portion of the open journal - ** file may be played back into the database. If a power failure occurs + ** If this is not done, then an unsynced portion of the open journal + ** file may be played back into the database. If a power failure occurs ** while this is happening, the database could become corrupt. ** ** If an error occurs while trying to sync the journal, shift the pager @@ -49934,11 +61409,6 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager){ sqlite3OsClose(pPager->fd); sqlite3PageFree(pTmp); sqlite3PcacheClose(pPager->pPCache); - -#ifdef SQLITE_HAS_CODEC - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); -#endif - assert( !pPager->aSavepoint && !pPager->pInJournal ); assert( !isOpen(pPager->jfd) && !isOpen(pPager->sjfd) ); @@ -49968,7 +61438,7 @@ SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ ** disk and can be restored in the event of a hot-journal rollback. ** ** If the Pager.noSync flag is set, then this function is a no-op. -** Otherwise, the actions required depend on the journal-mode and the +** Otherwise, the actions required depend on the journal-mode and the ** device characteristics of the file-system, as follows: ** ** * If the journal file is an in-memory journal file, no action need @@ -49980,7 +61450,7 @@ SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ ** been written following it. If the pager is operating in full-sync ** mode, then the journal file is synced before this field is updated. ** -** * If the device does not support the SEQUENTIAL property, then +** * If the device does not support the SEQUENTIAL property, then ** journal file is synced. ** ** Or, in pseudo-code: @@ -49989,11 +61459,11 @@ SQLITE_PRIVATE void sqlite3PagerRef(DbPage *pPg){ ** if( NOT SAFE_APPEND ){ ** if( ) xSync(); ** -** } +** } ** if( NOT SEQUENTIAL ) xSync(); ** } ** -** If successful, this routine clears the PGHDR_NEED_SYNC flag of every +** If successful, this routine clears the PGHDR_NEED_SYNC flag of every ** page currently held in memory before returning SQLITE_OK. If an IO ** error is encountered, then the IO error code is returned to the caller. */ @@ -50021,10 +61491,10 @@ static int syncJournal(Pager *pPager, int newHdr){ ** mode, then the journal file may at this point actually be larger ** than Pager.journalOff bytes. If the next thing in the journal ** file happens to be a journal-header (written as part of the - ** previous connection's transaction), and a crash or power-failure - ** occurs after nRec is updated but before this connection writes - ** anything else to the journal file (or commits/rolls back its - ** transaction), then SQLite may become confused when doing the + ** previous connection's transaction), and a crash or power-failure + ** occurs after nRec is updated but before this connection writes + ** anything else to the journal file (or commits/rolls back its + ** transaction), then SQLite may become confused when doing the ** hot-journal rollback following recovery. It may roll back all ** of this connections data, then proceed to rolling back the old, ** out-of-date data that follows it. Database corruption. @@ -50034,7 +61504,7 @@ static int syncJournal(Pager *pPager, int newHdr){ ** byte to the start of it to prevent it from being recognized. ** ** Variable iNextHdrOffset is set to the offset at which this - ** problematic header will occur, if it exists. aMagic is used + ** problematic header will occur, if it exists. aMagic is used ** as a temporary buffer to inspect the first couple of bytes of ** the potential journal header. */ @@ -50061,7 +61531,7 @@ static int syncJournal(Pager *pPager, int newHdr){ ** it as a candidate for rollback. ** ** This is not required if the persistent media supports the - ** SAFE_APPEND property. Because in this case it is not possible + ** SAFE_APPEND property. Because in this case it is not possible ** for garbage data to be appended to the file, the nRec field ** is populated with 0xFFFFFFFF when the journal header is written ** and never needs to be updated. @@ -50081,7 +61551,7 @@ static int syncJournal(Pager *pPager, int newHdr){ if( 0==(iDc&SQLITE_IOCAP_SEQUENTIAL) ){ PAGERTRACE(("SYNC journal of %d\n", PAGERID(pPager))); IOTRACE(("JSYNC %p\n", pPager)) - rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| + rc = sqlite3OsSync(pPager->jfd, pPager->syncFlags| (pPager->syncFlags==SQLITE_SYNC_FULL?SQLITE_SYNC_DATAONLY:0) ); if( rc!=SQLITE_OK ) return rc; @@ -50098,8 +61568,8 @@ static int syncJournal(Pager *pPager, int newHdr){ } } - /* Unless the pager is in noSync mode, the journal file was just - ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on + /* Unless the pager is in noSync mode, the journal file was just + ** successfully synced. Either way, clear the PGHDR_NEED_SYNC flag on ** all pages. */ sqlite3PcacheClearSyncFlags(pPager->pPCache); @@ -50119,9 +61589,9 @@ static int syncJournal(Pager *pPager, int newHdr){ ** is called. Before writing anything to the database file, this lock ** is upgraded to an EXCLUSIVE lock. If the lock cannot be obtained, ** SQLITE_BUSY is returned and no data is written to the database file. -** +** ** If the pager is a temp-file pager and the actual file-system file -** is not yet open, it is created and opened before any data is +** is not yet open, it is created and opened before any data is ** written out. ** ** Once the lock has been upgraded and, if necessary, the file opened, @@ -50136,7 +61606,7 @@ static int syncJournal(Pager *pPager, int newHdr){ ** in Pager.dbFileVers[] is updated to match the new value stored in ** the database file. ** -** If everything is successful, SQLITE_OK is returned. If an IO error +** If everything is successful, SQLITE_OK is returned. If an IO error ** occurs, an IO error code is returned. Or, if the EXCLUSIVE lock cannot ** be obtained, SQLITE_BUSY is returned. */ @@ -50162,7 +61632,7 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ ** file size will be. */ assert( rc!=SQLITE_OK || isOpen(pPager->fd) ); - if( rc==SQLITE_OK + if( rc==SQLITE_OK && pPager->dbHintSizedbSize && (pList->pDirty || pList->pgno>pPager->dbHintSize) ){ @@ -50184,20 +61654,19 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ */ if( pgno<=pPager->dbSize && 0==(pList->flags&PGHDR_DONT_WRITE) ){ i64 offset = (pgno-1)*(i64)pPager->pageSize; /* Offset to write */ - char *pData; /* Data to write */ + char *pData; /* Data to write */ assert( (pList->flags&PGHDR_NEED_SYNC)==0 ); if( pList->pgno==1 ) pager_write_changecounter(pList); - /* Encode the database */ - CODEC2(pPager, pList->pData, pgno, 6, return SQLITE_NOMEM_BKPT, pData); + pData = pList->pData; /* Write out the page data. */ rc = sqlite3OsWrite(pPager->fd, pData, pPager->pageSize, offset); /* If page 1 was just written, update Pager.dbFileVers to match - ** the value now stored in the database file. If writing this - ** page caused the database file to grow, update dbFileSize. + ** the value now stored in the database file. If writing this + ** page caused the database file to grow, update dbFileSize. */ if( pgno==1 ){ memcpy(&pPager->dbFileVers, &pData[24], sizeof(pPager->dbFileVers)); @@ -50225,18 +61694,18 @@ static int pager_write_pagelist(Pager *pPager, PgHdr *pList){ } /* -** Ensure that the sub-journal file is open. If it is already open, this +** Ensure that the sub-journal file is open. If it is already open, this ** function is a no-op. ** -** SQLITE_OK is returned if everything goes according to plan. An -** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() +** SQLITE_OK is returned if everything goes according to plan. An +** SQLITE_IOERR_XXX error code is returned if a call to sqlite3OsOpen() ** fails. */ static int openSubJournal(Pager *pPager){ int rc = SQLITE_OK; if( !isOpen(pPager->sjfd) ){ - const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE - | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE + const int flags = SQLITE_OPEN_SUBJOURNAL | SQLITE_OPEN_READWRITE + | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_DELETEONCLOSE; int nStmtSpill = sqlite3Config.nStmtSpill; if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY || pPager->subjInMemory ){ @@ -50248,13 +61717,13 @@ static int openSubJournal(Pager *pPager){ } /* -** Append a record of the current state of page pPg to the sub-journal. +** Append a record of the current state of page pPg to the sub-journal. ** ** If successful, set the bit corresponding to pPg->pgno in the bitvecs ** for all open savepoints before returning. ** ** This function returns SQLITE_OK if everything is successful, an IO -** error code if the attempt to write to the sub-journal fails, or +** error code if the attempt to write to the sub-journal fails, or ** SQLITE_NOMEM if a malloc fails while setting a bit in a savepoint ** bitvec. */ @@ -50267,9 +61736,9 @@ static int subjournalPage(PgHdr *pPg){ assert( pPager->useJournal ); assert( isOpen(pPager->jfd) || pagerUseWal(pPager) ); assert( isOpen(pPager->sjfd) || pPager->nSubRec==0 ); - assert( pagerUseWal(pPager) - || pageInJournal(pPager, pPg) - || pPg->pgno>pPager->dbOrigSize + assert( pagerUseWal(pPager) + || pageInJournal(pPager, pPg) + || pPg->pgno>pPager->dbOrigSize ); rc = openSubJournal(pPager); @@ -50279,8 +61748,7 @@ static int subjournalPage(PgHdr *pPg){ void *pData = pPg->pData; i64 offset = (i64)pPager->nSubRec*(4+pPager->pageSize); char *pData2; - - CODEC2(pPager, pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); + pData2 = pData; PAGERTRACE(("STMT-JOURNAL %d page %d\n", PAGERID(pPager), pPg->pgno)); rc = write32bits(pPager->sjfd, offset, pPg->pgno); if( rc==SQLITE_OK ){ @@ -50307,14 +61775,14 @@ static int subjournalPageIfRequired(PgHdr *pPg){ ** This function is called by the pcache layer when it has reached some ** soft memory limit. The first argument is a pointer to a Pager object ** (cast as a void*). The pager is always 'purgeable' (not an in-memory -** database). The second argument is a reference to a page that is +** database). The second argument is a reference to a page that is ** currently dirty but has no outstanding references. The page -** is always associated with the Pager object passed as the first +** is always associated with the Pager object passed as the first ** argument. ** ** The job of this function is to make pPg clean by writing its contents ** out to the database file, if possible. This may involve syncing the -** journal file. +** journal file. ** ** If successful, sqlite3PcacheMakeClean() is called on the page and ** SQLITE_OK returned. If an IO error occurs while trying to make the @@ -50339,7 +61807,7 @@ static int pagerStress(void *p, PgHdr *pPg){ ** a rollback or by user request, respectively. ** ** Spilling is also prohibited when in an error state since that could - ** lead to database corruption. In the current implementation it + ** lead to database corruption. In the current implementation it ** is impossible for sqlite3PcacheFetch() to be called with createFlag==3 ** while in the error state, hence it is impossible for this routine to ** be called in the error state. Nevertheless, we include a NEVER() @@ -50356,22 +61824,30 @@ static int pagerStress(void *p, PgHdr *pPg){ return SQLITE_OK; } + pPager->aStat[PAGER_STAT_SPILL]++; pPg->pDirty = 0; if( pagerUseWal(pPager) ){ /* Write a single frame for this page to the log. */ - rc = subjournalPageIfRequired(pPg); + rc = subjournalPageIfRequired(pPg); if( rc==SQLITE_OK ){ rc = pagerWalFrames(pPager, pPg, 0, 0); } }else{ - + +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( pPager->tempFile==0 ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) return pager_error(pPager, rc); + } +#endif + /* Sync the journal file if required. */ - if( pPg->flags&PGHDR_NEED_SYNC + if( pPg->flags&PGHDR_NEED_SYNC || pPager->eState==PAGER_WRITER_CACHEMOD ){ rc = syncJournal(pPager, 1); } - + /* Write the contents of the page out to the database file. */ if( rc==SQLITE_OK ){ assert( (pPg->flags&PGHDR_NEED_SYNC)==0 ); @@ -50385,7 +61861,7 @@ static int pagerStress(void *p, PgHdr *pPg){ sqlite3PcacheMakeClean(pPg); } - return pager_error(pPager, rc); + return pager_error(pPager, rc); } /* @@ -50416,26 +61892,28 @@ SQLITE_PRIVATE int sqlite3PagerFlush(Pager *pPager){ ** The zFilename argument is the path to the database file to open. ** If zFilename is NULL then a randomly-named temporary file is created ** and used as the file to be cached. Temporary files are be deleted -** automatically when they are closed. If zFilename is ":memory:" then -** all information is held in cache. It is never written to disk. +** automatically when they are closed. If zFilename is ":memory:" then +** all information is held in cache. It is never written to disk. ** This can be used to implement an in-memory database. ** ** The nExtra parameter specifies the number of bytes of space allocated ** along with each page reference. This space is available to the user -** via the sqlite3PagerGetExtra() API. +** via the sqlite3PagerGetExtra() API. When a new page is allocated, the +** first 8 bytes of this space are zeroed but the remainder is uninitialized. +** (The extra space is used by btree as the MemPage object.) ** ** The flags argument is used to specify properties that affect the ** operation of the pager. It should be passed some bitwise combination ** of the PAGER_* flags. ** ** The vfsFlags parameter is a bitmask to pass to the flags parameter -** of the xOpen() method of the supplied VFS when opening files. +** of the xOpen() method of the supplied VFS when opening files. ** -** If the pager object is allocated and the specified file opened +** If the pager object is allocated and the specified file opened ** successfully, SQLITE_OK is returned and *ppPager set to point to ** the new pager object. If an error occurs, *ppPager is set to NULL ** and error code returned. This function may return SQLITE_NOMEM -** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or +** (sqlite3Malloc() is used to allocate memory), SQLITE_CANTOPEN or ** various SQLITE_IO_XXX errors. */ SQLITE_PRIVATE int sqlite3PagerOpen( @@ -50452,6 +61930,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int rc = SQLITE_OK; /* Return code */ int tempFile = 0; /* True for temp files (incl. in-memory files) */ int memDb = 0; /* True if this is an in-memory file */ + int memJM = 0; /* Memory journal mode */ int readOnly = 0; /* True if this is a read-only file */ int journalFileSize; /* Bytes to allocate for each journal fd */ char *zPathname = 0; /* Full path to database file */ @@ -50460,7 +61939,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int pcacheSize = sqlite3PcacheSize(); /* Bytes to allocate for PCache */ u32 szPageDflt = SQLITE_DEFAULT_PAGE_SIZE; /* Default page size */ const char *zUri = 0; /* URI args to copy */ - int nUri = 0; /* Number of bytes of URI args at *zUri */ + int nUriByte = 1; /* Number of bytes of URI args at *zUri */ /* Figure out how much space is required for each journal file-handle ** (there are two of them, the main journal and the sub-journal). */ @@ -50494,14 +61973,23 @@ SQLITE_PRIVATE int sqlite3PagerOpen( } zPathname[0] = 0; /* Make sure initialized even if FullPathname() fails */ rc = sqlite3OsFullPathname(pVfs, zFilename, nPathname, zPathname); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_OK_SYMLINK ){ + if( vfsFlags & SQLITE_OPEN_NOFOLLOW ){ + rc = SQLITE_CANTOPEN_SYMLINK; + }else{ + rc = SQLITE_OK; + } + } + } nPathname = sqlite3Strlen30(zPathname); z = zUri = &zFilename[sqlite3Strlen30(zFilename)+1]; while( *z ){ - z += sqlite3Strlen30(z)+1; - z += sqlite3Strlen30(z)+1; + z += strlen(z)+1; + z += strlen(z)+1; } - nUri = (int)(&z[1] - zUri); - assert( nUri>=0 ); + nUriByte = (int)(&z[1] - zUri); + assert( nUriByte>=1 ); if( rc==SQLITE_OK && nPathname+8>pVfs->mxPathname ){ /* This branch is taken when the journal path required by ** the database being opened will be more than pVfs->mxPathname @@ -50518,7 +62006,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( } /* Allocate memory for the Pager structure, PCache object, the - ** three file descriptors, the database file name and the journal + ** three file descriptors, the database file name and the journal ** file name. The layout in memory is as follows: ** ** Pager object (sizeof(Pager) bytes) @@ -50526,50 +62014,113 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** Database file handle (pVfs->szOsFile bytes) ** Sub-journal file handle (journalFileSize bytes) ** Main journal file handle (journalFileSize bytes) + ** Ptr back to the Pager (sizeof(Pager*) bytes) + ** \0\0\0\0 database prefix (4 bytes) ** Database file name (nPathname+1 bytes) - ** Journal file name (nPathname+8+1 bytes) + ** URI query parameters (nUriByte bytes) + ** Journal filename (nPathname+8+1 bytes) + ** WAL filename (nPathname+4+1 bytes) + ** \0\0\0 terminator (3 bytes) + ** + ** Some 3rd-party software, over which we have no control, depends on + ** the specific order of the filenames and the \0 separators between them + ** so that it can (for example) find the database filename given the WAL + ** filename without using the sqlite3_filename_database() API. This is a + ** misuse of SQLite and a bug in the 3rd-party software, but the 3rd-party + ** software is in widespread use, so we try to avoid changing the filename + ** order and formatting if possible. In particular, the details of the + ** filename format expected by 3rd-party software should be as follows: + ** + ** - Main Database Path + ** - \0 + ** - Multiple URI components consisting of: + ** - Key + ** - \0 + ** - Value + ** - \0 + ** - \0 + ** - Journal Path + ** - \0 + ** - WAL Path (zWALName) + ** - \0 + ** + ** The sqlite3_create_filename() interface and the databaseFilename() utility + ** that is used by sqlite3_filename_database() and kin also depend on the + ** specific formatting and order of the various filenames, so if the format + ** changes here, be sure to change it there as well. */ + assert( SQLITE_PTRSIZE==sizeof(Pager*) ); pPtr = (u8 *)sqlite3MallocZero( - ROUND8(sizeof(*pPager)) + /* Pager structure */ - ROUND8(pcacheSize) + /* PCache object */ - ROUND8(pVfs->szOsFile) + /* The main db file */ - journalFileSize * 2 + /* The two journal files */ - nPathname + 1 + nUri + /* zFilename */ - nPathname + 8 + 2 /* zJournal */ + ROUND8(sizeof(*pPager)) + /* Pager structure */ + ROUND8(pcacheSize) + /* PCache object */ + ROUND8(pVfs->szOsFile) + /* The main db file */ + journalFileSize * 2 + /* The two journal files */ + SQLITE_PTRSIZE + /* Space to hold a pointer */ + 4 + /* Database prefix */ + nPathname + 1 + /* database filename */ + nUriByte + /* query parameters */ + nPathname + 8 + 1 + /* Journal filename */ #ifndef SQLITE_OMIT_WAL - + nPathname + 4 + 2 /* zWal */ + nPathname + 4 + 1 + /* WAL filename */ #endif + 3 /* Terminator */ ); assert( EIGHT_BYTE_ALIGNMENT(SQLITE_INT_TO_PTR(journalFileSize)) ); if( !pPtr ){ sqlite3DbFree(0, zPathname); return SQLITE_NOMEM_BKPT; } - pPager = (Pager*)(pPtr); - pPager->pPCache = (PCache*)(pPtr += ROUND8(sizeof(*pPager))); - pPager->fd = (sqlite3_file*)(pPtr += ROUND8(pcacheSize)); - pPager->sjfd = (sqlite3_file*)(pPtr += ROUND8(pVfs->szOsFile)); - pPager->jfd = (sqlite3_file*)(pPtr += journalFileSize); - pPager->zFilename = (char*)(pPtr += journalFileSize); + pPager = (Pager*)pPtr; pPtr += ROUND8(sizeof(*pPager)); + pPager->pPCache = (PCache*)pPtr; pPtr += ROUND8(pcacheSize); + pPager->fd = (sqlite3_file*)pPtr; pPtr += ROUND8(pVfs->szOsFile); + pPager->sjfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; + pPager->jfd = (sqlite3_file*)pPtr; pPtr += journalFileSize; assert( EIGHT_BYTE_ALIGNMENT(pPager->jfd) ); + memcpy(pPtr, &pPager, SQLITE_PTRSIZE); pPtr += SQLITE_PTRSIZE; - /* Fill in the Pager.zFilename and Pager.zJournal buffers, if required. */ - if( zPathname ){ - assert( nPathname>0 ); - pPager->zJournal = (char*)(pPtr += nPathname + 1 + nUri); - memcpy(pPager->zFilename, zPathname, nPathname); - if( nUri ) memcpy(&pPager->zFilename[nPathname+1], zUri, nUri); - memcpy(pPager->zJournal, zPathname, nPathname); - memcpy(&pPager->zJournal[nPathname], "-journal\000", 8+2); - sqlite3FileSuffix3(pPager->zFilename, pPager->zJournal); -#ifndef SQLITE_OMIT_WAL - pPager->zWal = &pPager->zJournal[nPathname+8+1]; - memcpy(pPager->zWal, zPathname, nPathname); - memcpy(&pPager->zWal[nPathname], "-wal\000", 4+1); - sqlite3FileSuffix3(pPager->zFilename, pPager->zWal); -#endif - sqlite3DbFree(0, zPathname); + /* Fill in the Pager.zFilename and pPager.zQueryParam fields */ + pPtr += 4; /* Skip zero prefix */ + pPager->zFilename = (char*)pPtr; + if( nPathname>0 ){ + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname + 1; + if( zUri ){ + memcpy(pPtr, zUri, nUriByte); pPtr += nUriByte; + }else{ + pPtr++; + } } + + + /* Fill in Pager.zJournal */ + if( nPathname>0 ){ + pPager->zJournal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-journal",8); pPtr += 8 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename,pPager->zJournal); + pPtr = (u8*)(pPager->zJournal + sqlite3Strlen30(pPager->zJournal)+1); +#endif + }else{ + pPager->zJournal = 0; + } + +#ifndef SQLITE_OMIT_WAL + /* Fill in Pager.zWal */ + if( nPathname>0 ){ + pPager->zWal = (char*)pPtr; + memcpy(pPtr, zPathname, nPathname); pPtr += nPathname; + memcpy(pPtr, "-wal", 4); pPtr += 4 + 1; +#ifdef SQLITE_ENABLE_8_3_NAMES + sqlite3FileSuffix3(zFilename, pPager->zWal); + pPtr = (u8*)(pPager->zWal + sqlite3Strlen30(pPager->zWal)+1); +#endif + }else{ + pPager->zWal = 0; + } +#endif + (void)pPtr; /* Suppress warning about unused pPtr value */ + + if( nPathname ) sqlite3DbFree(0, zPathname); pPager->pVfs = pVfs; pPager->vfsFlags = vfsFlags; @@ -50579,7 +62130,8 @@ SQLITE_PRIVATE int sqlite3PagerOpen( int fout = 0; /* VFS flags returned by xOpen() */ rc = sqlite3OsOpen(pVfs, pPager->zFilename, pPager->fd, vfsFlags, &fout); assert( !memDb ); - readOnly = (fout&SQLITE_OPEN_READONLY); + pPager->memVfs = memJM = (fout&SQLITE_OPEN_MEMORY)!=0; + readOnly = (fout&SQLITE_OPEN_READONLY)!=0; /* If the file was successfully opened for read/write access, ** choose a default page size in case we have to create the @@ -50615,9 +62167,9 @@ SQLITE_PRIVATE int sqlite3PagerOpen( } #endif } - pPager->noLock = sqlite3_uri_boolean(zFilename, "nolock", 0); + pPager->noLock = sqlite3_uri_boolean(pPager->zFilename, "nolock", 0); if( (iDc & SQLITE_IOCAP_IMMUTABLE)!=0 - || sqlite3_uri_boolean(zFilename, "immutable", 0) ){ + || sqlite3_uri_boolean(pPager->zFilename, "immutable", 0) ){ vfsFlags |= SQLITE_OPEN_READONLY; goto act_like_temp_file; } @@ -50632,7 +62184,7 @@ SQLITE_PRIVATE int sqlite3PagerOpen( ** disk and uses an in-memory rollback journal. ** ** This branch also runs for files marked as immutable. - */ + */ act_like_temp_file: tempFile = 1; pPager->eState = PAGER_READER; /* Pretend we already have a lock */ @@ -50641,7 +62193,7 @@ act_like_temp_file: readOnly = (vfsFlags&SQLITE_OPEN_READONLY); } - /* The following call to PagerSetPagesize() serves to set the value of + /* The following call to PagerSetPagesize() serves to set the value of ** Pager.pageSize and to allocate the Pager.pTmpSpace buffer. */ if( rc==SQLITE_OK ){ @@ -50652,8 +62204,8 @@ act_like_temp_file: /* Initialize the PCache object. */ if( rc==SQLITE_OK ){ - assert( nExtra<1000 ); nExtra = ROUND8(nExtra); + assert( nExtra>=8 && nExtra<1000 ); rc = sqlite3PcacheOpen(szPageDflt, nExtra, !memDb, !memDb?pagerStress:0, (void *)pPager, pPager->pPCache); } @@ -50681,28 +62233,15 @@ act_like_temp_file: /* pPager->state = PAGER_UNLOCK; */ /* pPager->errMask = 0; */ pPager->tempFile = (u8)tempFile; - assert( tempFile==PAGER_LOCKINGMODE_NORMAL + assert( tempFile==PAGER_LOCKINGMODE_NORMAL || tempFile==PAGER_LOCKINGMODE_EXCLUSIVE ); assert( PAGER_LOCKINGMODE_EXCLUSIVE==1 ); - pPager->exclusiveMode = (u8)tempFile; + pPager->exclusiveMode = (u8)tempFile; pPager->changeCountDone = pPager->tempFile; pPager->memDb = (u8)memDb; pPager->readOnly = (u8)readOnly; assert( useJournal || pPager->tempFile ); - pPager->noSync = pPager->tempFile; - if( pPager->noSync ){ - assert( pPager->fullSync==0 ); - assert( pPager->extraSync==0 ); - assert( pPager->syncFlags==0 ); - assert( pPager->walSyncFlags==0 ); - assert( pPager->ckptSyncFlags==0 ); - }else{ - pPager->fullSync = 1; - pPager->extraSync = 0; - pPager->syncFlags = SQLITE_SYNC_NORMAL; - pPager->walSyncFlags = SQLITE_SYNC_NORMAL | WAL_SYNC_TRANSACTIONS; - pPager->ckptSyncFlags = SQLITE_SYNC_NORMAL; - } + sqlite3PagerSetFlags(pPager, (SQLITE_DEFAULT_SYNCHRONOUS+1)|PAGER_CACHESPILL); /* pPager->pFirst = 0; */ /* pPager->pFirstSynced = 0; */ /* pPager->pLast = 0; */ @@ -50712,12 +62251,13 @@ act_like_temp_file: setSectorSize(pPager); if( !useJournal ){ pPager->journalMode = PAGER_JOURNALMODE_OFF; - }else if( memDb ){ + }else if( memDb || memJM ){ pPager->journalMode = PAGER_JOURNALMODE_MEMORY; } /* pPager->xBusyHandler = 0; */ /* pPager->pBusyHandlerArg = 0; */ pPager->xReiniter = xReinit; + setGetterMethod(pPager); /* memset(pPager->aHash, 0, sizeof(pPager->aHash)); */ /* pPager->szMmap = SQLITE_DEFAULT_MMAP_SIZE // will be set by btree.c */ @@ -50725,36 +62265,28 @@ act_like_temp_file: return SQLITE_OK; } - -/* Verify that the database file has not be deleted or renamed out from -** under the pager. Return SQLITE_OK if the database is still were it ought -** to be on disk. Return non-zero (SQLITE_READONLY_DBMOVED or some other error -** code from sqlite3OsAccess()) if the database has gone missing. +/* +** Return the sqlite3_file for the main database given the name +** of the corresponding WAL or Journal name as passed into +** xOpen. */ -static int databaseIsUnmoved(Pager *pPager){ - int bHasMoved = 0; - int rc; - - if( pPager->tempFile ) return SQLITE_OK; - if( pPager->dbSize==0 ) return SQLITE_OK; - assert( pPager->zFilename && pPager->zFilename[0] ); - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_HAS_MOVED, &bHasMoved); - if( rc==SQLITE_NOTFOUND ){ - /* If the HAS_MOVED file-control is unimplemented, assume that the file - ** has not been moved. That is the historical behavior of SQLite: prior to - ** version 3.8.3, it never checked */ - rc = SQLITE_OK; - }else if( rc==SQLITE_OK && bHasMoved ){ - rc = SQLITE_READONLY_DBMOVED; +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char *zName){ + Pager *pPager; + const char *p; + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; } - return rc; + p = zName - 4 - sizeof(Pager*); + assert( EIGHT_BYTE_ALIGNMENT(p) ); + pPager = *(Pager**)p; + return pPager->fd; } /* ** This function is called after transitioning from PAGER_UNLOCK to ** PAGER_SHARED state. It tests if there is a hot journal present in -** the file-system for the given pager. A hot journal is one that +** the file-system for the given pager. A hot journal is one that ** needs to be played back. According to this function, a hot-journal ** file exists if the following criteria are met: ** @@ -50769,14 +62301,14 @@ static int databaseIsUnmoved(Pager *pPager){ ** just deleted using OsDelete, *pExists is set to 0 and SQLITE_OK ** is returned. ** -** This routine does not check if there is a master journal filename -** at the end of the file. If there is, and that master journal file +** This routine does not check if there is a super-journal filename +** at the end of the file. If there is, and that super-journal file ** does not exist, then the journal file is not really hot. In this ** case this routine will return a false-positive. The pager_playback() -** routine will discover that the journal file is not really hot and -** will not roll it back. +** routine will discover that the journal file is not really hot and +** will not roll it back. ** -** If a hot-journal file is found to exist, *pExists is set to 1 and +** If a hot-journal file is found to exist, *pExists is set to 1 and ** SQLITE_OK returned. If no hot-journal file is present, *pExists is ** set to 0 and SQLITE_OK returned. If an IO error occurs while trying ** to determine whether or not a hot-journal file exists, the IO error @@ -50804,7 +62336,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ int locked = 0; /* True if some process holds a RESERVED lock */ /* Race condition here: Another process might have been holding the - ** the RESERVED lock and have a journal open at the sqlite3OsAccess() + ** the RESERVED lock and have a journal open at the sqlite3OsAccess() ** call above, but then delete the journal and drop the lock before ** we get to the following sqlite3OsCheckReservedLock() call. If that ** is the case, this routine might think there is a hot journal when @@ -50837,7 +62369,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ /* The journal file exists and no other connection has a reserved ** or greater lock on the database file. Now check that there is ** at least one non-zero bytes at the start of the journal file. - ** If there is, then we consider this journal to be hot. If not, + ** If there is, then we consider this journal to be hot. If not, ** it can be ignored. */ if( !jrnlOpen ){ @@ -50887,7 +62419,7 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** on the database file), then an attempt is made to obtain a ** SHARED lock on the database file. Immediately after obtaining ** the SHARED lock, the file-system is checked for a hot-journal, -** which is played back if present. Following any hot-journal +** which is played back if present. Following any hot-journal ** rollback, the contents of the cache are validated by checking ** the 'change-counter' field of the database file header and ** discarded if they are found to be invalid. @@ -50898,8 +62430,8 @@ static int hasHotJournal(Pager *pPager, int *pExists){ ** the contents of the page cache and rolling back any open journal ** file. ** -** If everything is successful, SQLITE_OK is returned. If an IO error -** occurs while locking the database, checking for a hot-journal file or +** If everything is successful, SQLITE_OK is returned. If an IO error +** occurs while locking the database, checking for a hot-journal file or ** rolling back a journal file, the IO error code is returned. */ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ @@ -50907,7 +62439,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ /* This routine is only called from b-tree and only when there are no ** outstanding pages. This implies that the pager state should either - ** be OPEN or READER. READER is only possible if the pager is or was in + ** be OPEN or READER. READER is only possible if the pager is or was in ** exclusive access mode. */ assert( sqlite3PcacheRefCount(pPager->pPCache)==0 ); assert( assert_pager_state(pPager) ); @@ -50945,12 +62477,12 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** important that a RESERVED lock is not obtained on the way to the ** EXCLUSIVE lock. If it were, another process might open the ** database file, detect the RESERVED lock, and conclude that the - ** database is safe to read while this process is still rolling the + ** database is safe to read while this process is still rolling the ** hot-journal back. - ** + ** ** Because the intermediate RESERVED lock is not requested, any - ** other process attempting to access the database file will get to - ** this point in the code and fail to obtain its own EXCLUSIVE lock + ** other process attempting to access the database file will get to + ** this point in the code and fail to obtain its own EXCLUSIVE lock ** on the database file. ** ** Unless the pager is in locking_mode=exclusive mode, the lock is @@ -50960,21 +62492,21 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ if( rc!=SQLITE_OK ){ goto failed; } - - /* If it is not already open and the file exists on disk, open the - ** journal for read/write access. Write access is required because - ** in exclusive-access mode the file descriptor will be kept open - ** and possibly used for a transaction later on. Also, write-access - ** is usually required to finalize the journal in journal_mode=persist + + /* If it is not already open and the file exists on disk, open the + ** journal for read/write access. Write access is required because + ** in exclusive-access mode the file descriptor will be kept open + ** and possibly used for a transaction later on. Also, write-access + ** is usually required to finalize the journal in journal_mode=persist ** mode (and also for journal_mode=truncate on some systems). ** - ** If the journal does not exist, it usually means that some - ** other connection managed to get in and roll it back before - ** this connection obtained the exclusive lock above. Or, it + ** If the journal does not exist, it usually means that some + ** other connection managed to get in and roll it back before + ** this connection obtained the exclusive lock above. Or, it ** may mean that the pager was in the error-state when this ** function was called and the journal file does not exist. */ - if( !isOpen(pPager->jfd) ){ + if( !isOpen(pPager->jfd) && pPager->journalMode!=PAGER_JOURNALMODE_OFF ){ sqlite3_vfs * const pVfs = pPager->pVfs; int bExists; /* True if journal file exists */ rc = sqlite3OsAccess( @@ -50991,7 +62523,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ } } } - + /* Playback and delete the journal. Drop the database write ** lock and reacquire the read lock. Purge the cache before ** playing back the hot-journal so that we don't end up with @@ -51016,8 +62548,8 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** or roll back a hot-journal while holding an EXCLUSIVE lock. The ** pager_unlock() routine will be called before returning to unlock ** the file. If the unlock attempt fails, then Pager.eLock must be - ** set to UNKNOWN_LOCK (see the comment above the #define for - ** UNKNOWN_LOCK above for an explanation). + ** set to UNKNOWN_LOCK (see the comment above the #define for + ** UNKNOWN_LOCK above for an explanation). ** ** In order to get pager_unlock() to do this, set Pager.eState to ** PAGER_ERROR now. This is not actually counted as a transition @@ -51025,7 +62557,7 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** since we know that the same call to pager_unlock() will very ** shortly transition the pager object to the OPEN state. Calling ** assert_pager_state() would fail now, as it should not be possible - ** to be in ERROR state when there are zero outstanding page + ** to be in ERROR state when there are zero outstanding page ** references. */ pager_error(pPager, rc); @@ -51050,24 +62582,19 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** a 32-bit counter that is incremented with each change. The ** other bytes change randomly with each file change when ** a codec is in use. - ** - ** There is a vanishingly small chance that a change will not be + ** + ** There is a vanishingly small chance that a change will not be ** detected. The chance of an undetected change is so small that ** it can be neglected. */ - Pgno nPage = 0; char dbFileVers[sizeof(pPager->dbFileVers)]; - rc = pagerPagecount(pPager, &nPage); - if( rc ) goto failed; - - if( nPage>0 ){ - IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); - rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); - if( rc!=SQLITE_OK && rc!=SQLITE_IOERR_SHORT_READ ){ + IOTRACE(("CKVERS %p %d\n", pPager, sizeof(dbFileVers))); + rc = sqlite3OsRead(pPager->fd, &dbFileVers, sizeof(dbFileVers), 24); + if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_IOERR_SHORT_READ ){ goto failed; } - }else{ memset(dbFileVers, 0, sizeof(dbFileVers)); } @@ -51123,37 +62650,45 @@ SQLITE_PRIVATE int sqlite3PagerSharedLock(Pager *pPager){ ** Except, in locking_mode=EXCLUSIVE when there is nothing to in ** the rollback journal, the unlock is not performed and there is ** nothing to rollback, so this routine is a no-op. -*/ +*/ static void pagerUnlockIfUnused(Pager *pPager){ - if( pPager->nMmapOut==0 && (sqlite3PcacheRefCount(pPager->pPCache)==0) ){ + if( sqlite3PcacheRefCount(pPager->pPCache)==0 ){ + assert( pPager->nMmapOut==0 ); /* because page1 is never memory mapped */ pagerUnlockAndRollback(pPager); } } /* -** Acquire a reference to page number pgno in pager pPager (a page -** reference has type DbPage*). If the requested reference is +** The page getter methods each try to acquire a reference to a +** page with page number pgno. If the requested reference is ** successfully obtained, it is copied to *ppPage and SQLITE_OK returned. ** -** If the requested page is already in the cache, it is returned. +** There are different implementations of the getter method depending +** on the current state of the pager. +** +** getPageNormal() -- The normal getter +** getPageError() -- Used if the pager is in an error state +** getPageMmap() -- Used if memory-mapped I/O is enabled +** +** If the requested page is already in the cache, it is returned. ** Otherwise, a new page object is allocated and populated with data ** read from the database file. In some cases, the pcache module may ** choose not to allocate a new page object and may reuse an existing ** object with no outstanding references. ** -** The extra data appended to a page is always initialized to zeros the -** first time a page is loaded into memory. If the page requested is +** The extra data appended to a page is always initialized to zeros the +** first time a page is loaded into memory. If the page requested is ** already in the cache when this function is called, then the extra ** data is left as it was when the page object was last used. ** -** If the database image is smaller than the requested page or if a -** non-zero value is passed as the noContent parameter and the -** requested page is not already stored in the cache, then no -** actual disk read occurs. In this case the memory image of the -** page is initialized to all zeros. +** If the database image is smaller than the requested page or if +** the flags parameter contains the PAGER_GET_NOCONTENT bit and the +** requested page is not already stored in the cache, then no +** actual disk read occurs. In this case the memory image of the +** page is initialized to all zeros. ** -** If noContent is true, it means that we do not care about the contents -** of the page. This occurs in two scenarios: +** If PAGER_GET_NOCONTENT is true, it means that we do not care about +** the contents of the page. This occurs in two scenarios: ** ** a) When reading a free-list leaf page from the database, and ** @@ -51161,8 +62696,8 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** a new page into the cache to be filled with the data read ** from the savepoint journal. ** -** If noContent is true, then the data returned is zeroed instead of -** being read from the database. Additionally, the bits corresponding +** If PAGER_GET_NOCONTENT is true, then the data returned is zeroed instead +** of being read from the database. Additionally, the bits corresponding ** to pgno in Pager.pInJournal (bitvec of pages already written to the ** journal file) and the PagerSavepoint.pInSavepoint bitvecs of any open ** savepoints are set. This means if the page is made writable at any @@ -51180,137 +62715,75 @@ static void pagerUnlockIfUnused(Pager *pPager){ ** Since Lookup() never goes to disk, it never has to deal with locks ** or journal files. */ -SQLITE_PRIVATE int sqlite3PagerGet( +static int getPageNormal( Pager *pPager, /* The pager open on the database file */ Pgno pgno, /* Page number to fetch */ DbPage **ppPage, /* Write a pointer to the page here */ int flags /* PAGER_GET_XXX flags */ ){ int rc = SQLITE_OK; - PgHdr *pPg = 0; - u32 iFrame = 0; /* Frame to read from WAL file */ - const int noContent = (flags & PAGER_GET_NOCONTENT); + PgHdr *pPg; + u8 noContent; /* True if PAGER_GET_NOCONTENT is set */ + sqlite3_pcache_page *pBase; - /* It is acceptable to use a read-only (mmap) page for any page except - ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY - ** flag was specified by the caller. And so long as the db is not a - ** temporary or in-memory database. */ - const int bMmapOk = (pgno>1 && USEFETCH(pPager) - && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) -#ifdef SQLITE_HAS_CODEC - && pPager->xCodec==0 -#endif - ); - - /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here - ** allows the compiler optimizer to reuse the results of the "pgno>1" - ** test in the previous statement, and avoid testing pgno==0 in the - ** common case where pgno is large. */ - if( pgno<=1 && pgno==0 ){ - return SQLITE_CORRUPT_BKPT; - } + assert( pPager->errCode==SQLITE_OK ); assert( pPager->eState>=PAGER_READER ); assert( assert_pager_state(pPager) ); - assert( noContent==0 || bMmapOk==0 ); - assert( pPager->hasHeldSharedLock==1 ); - /* If the pager is in the error state, return an error immediately. - ** Otherwise, request the page from the PCache layer. */ - if( pPager->errCode!=SQLITE_OK ){ - rc = pPager->errCode; - }else{ - if( bMmapOk && pagerUseWal(pPager) ){ - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - } - - if( bMmapOk && iFrame==0 ){ - void *pData = 0; - - rc = sqlite3OsFetch(pPager->fd, - (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData - ); - - if( rc==SQLITE_OK && pData ){ - if( pPager->eState>PAGER_READER || pPager->tempFile ){ - pPg = sqlite3PagerLookup(pPager, pgno); - } - if( pPg==0 ){ - rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); - }else{ - sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); - } - if( pPg ){ - assert( rc==SQLITE_OK ); - *ppPage = pPg; - return SQLITE_OK; - } - } - if( rc!=SQLITE_OK ){ - goto pager_acquire_err; - } - } - - { - sqlite3_pcache_page *pBase; - pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); - if( pBase==0 ){ - rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - if( pBase==0 ){ - pPg = *ppPage = 0; - rc = SQLITE_NOMEM_BKPT; - goto pager_acquire_err; - } - } - pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); - assert( pPg!=0 ); - } - } - - if( rc!=SQLITE_OK ){ - /* Either the call to sqlite3PcacheFetch() returned an error or the - ** pager was already in the error-state when this function was called. - ** Set pPg to 0 and jump to the exception handler. */ + if( pgno==0 ) return SQLITE_CORRUPT_BKPT; + pBase = sqlite3PcacheFetch(pPager->pPCache, pgno, 3); + if( pBase==0 ){ pPg = 0; - goto pager_acquire_err; + rc = sqlite3PcacheFetchStress(pPager->pPCache, pgno, &pBase); + if( rc!=SQLITE_OK ) goto pager_acquire_err; + if( pBase==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto pager_acquire_err; + } } + pPg = *ppPage = sqlite3PcacheFetchFinish(pPager->pPCache, pgno, pBase); assert( pPg==(*ppPage) ); assert( pPg->pgno==pgno ); assert( pPg->pPager==pPager || pPg->pPager==0 ); + noContent = (flags & PAGER_GET_NOCONTENT)!=0; if( pPg->pPager && !noContent ){ /* In this case the pcache already contains an initialized copy of ** the page. Return without further ado. */ - assert( pgno<=PAGER_MAX_PGNO && pgno!=PAGER_MJ_PGNO(pPager) ); + assert( pgno!=PAGER_SJ_PGNO(pPager) ); pPager->aStat[PAGER_STAT_HIT]++; return SQLITE_OK; }else{ - /* The pager cache has created a new page. Its content needs to - ** be initialized. */ - - pPg->pPager = pPager; - - /* The maximum page number is 2^31. Return SQLITE_CORRUPT if a page - ** number greater than this, or the unused locking-page, is requested. */ - if( pgno>PAGER_MAX_PGNO || pgno==PAGER_MJ_PGNO(pPager) ){ + /* The pager cache has created a new page. Its content needs to + ** be initialized. But first some error checks: + ** + ** (*) obsolete. Was: maximum page number is 2^31 + ** (2) Never try to fetch the locking page + */ + if( pgno==PAGER_SJ_PGNO(pPager) ){ rc = SQLITE_CORRUPT_BKPT; goto pager_acquire_err; } + pPg->pPager = pPager; + assert( !isOpen(pPager->fd) || !MEMDB ); if( !isOpen(pPager->fd) || pPager->dbSizepPager->mxPgno ){ rc = SQLITE_FULL; + if( pgno<=pPager->dbSize ){ + sqlite3PcacheRelease(pPg); + pPg = 0; + } goto pager_acquire_err; } if( noContent ){ /* Failure to set the bits in the InJournal bit-vectors is benign. - ** It merely means that we might do some extra work to journal a - ** page that does not need to be journaled. Nevertheless, be sure - ** to test the case where a malloc error occurs while trying to set + ** It merely means that we might do some extra work to journal a + ** page that does not need to be journaled. Nevertheless, be sure + ** to test the case where a malloc error occurs while trying to set ** a bit in a bit vector. */ sqlite3BeginBenignMalloc(); @@ -51325,20 +62798,15 @@ SQLITE_PRIVATE int sqlite3PagerGet( memset(pPg->pData, 0, pPager->pageSize); IOTRACE(("ZERO %p %d\n", pPager, pgno)); }else{ - if( pagerUseWal(pPager) && bMmapOk==0 ){ - rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); - if( rc!=SQLITE_OK ) goto pager_acquire_err; - } assert( pPg->pPager==pPager ); pPager->aStat[PAGER_STAT_MISS]++; - rc = readDbPage(pPg, iFrame); + rc = readDbPage(pPg); if( rc!=SQLITE_OK ){ goto pager_acquire_err; } } pager_set_pagehash(pPg); } - return SQLITE_OK; pager_acquire_err: @@ -51347,20 +62815,128 @@ pager_acquire_err: sqlite3PcacheDrop(pPg); } pagerUnlockIfUnused(pPager); - *ppPage = 0; return rc; } +#if SQLITE_MAX_MMAP_SIZE>0 +/* The page getter for when memory-mapped I/O is enabled */ +static int getPageMMap( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + int rc = SQLITE_OK; + PgHdr *pPg = 0; + u32 iFrame = 0; /* Frame to read from WAL file */ + + /* It is acceptable to use a read-only (mmap) page for any page except + ** page 1 if there is no write-transaction open or the ACQUIRE_READONLY + ** flag was specified by the caller. And so long as the db is not a + ** temporary or in-memory database. */ + const int bMmapOk = (pgno>1 + && (pPager->eState==PAGER_READER || (flags & PAGER_GET_READONLY)) + ); + + assert( USEFETCH(pPager) ); + + /* Optimization note: Adding the "pgno<=1" term before "pgno==0" here + ** allows the compiler optimizer to reuse the results of the "pgno>1" + ** test in the previous statement, and avoid testing pgno==0 in the + ** common case where pgno is large. */ + if( pgno<=1 && pgno==0 ){ + return SQLITE_CORRUPT_BKPT; + } + assert( pPager->eState>=PAGER_READER ); + assert( assert_pager_state(pPager) ); + assert( pPager->hasHeldSharedLock==1 ); + assert( pPager->errCode==SQLITE_OK ); + + if( bMmapOk && pagerUseWal(pPager) ){ + rc = sqlite3WalFindFrame(pPager->pWal, pgno, &iFrame); + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + if( bMmapOk && iFrame==0 ){ + void *pData = 0; + rc = sqlite3OsFetch(pPager->fd, + (i64)(pgno-1) * pPager->pageSize, pPager->pageSize, &pData + ); + if( rc==SQLITE_OK && pData ){ + if( pPager->eState>PAGER_READER || pPager->tempFile ){ + pPg = sqlite3PagerLookup(pPager, pgno); + } + if( pPg==0 ){ + rc = pagerAcquireMapPage(pPager, pgno, pData, &pPg); + }else{ + sqlite3OsUnfetch(pPager->fd, (i64)(pgno-1)*pPager->pageSize, pData); + } + if( pPg ){ + assert( rc==SQLITE_OK ); + *ppPage = pPg; + return SQLITE_OK; + } + } + if( rc!=SQLITE_OK ){ + *ppPage = 0; + return rc; + } + } + return getPageNormal(pPager, pgno, ppPage, flags); +} +#endif /* SQLITE_MAX_MMAP_SIZE>0 */ + +/* The page getter method for when the pager is an error state */ +static int getPageError( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ + UNUSED_PARAMETER(pgno); + UNUSED_PARAMETER(flags); + assert( pPager->errCode!=SQLITE_OK ); + *ppPage = 0; + return pPager->errCode; +} + + +/* Dispatch all page fetch requests to the appropriate getter method. +*/ +SQLITE_PRIVATE int sqlite3PagerGet( + Pager *pPager, /* The pager open on the database file */ + Pgno pgno, /* Page number to fetch */ + DbPage **ppPage, /* Write a pointer to the page here */ + int flags /* PAGER_GET_XXX flags */ +){ +#if 0 /* Trace page fetch by setting to 1 */ + int rc; + printf("PAGE %u\n", pgno); + fflush(stdout); + rc = pPager->xGet(pPager, pgno, ppPage, flags); + if( rc ){ + printf("PAGE %u failed with 0x%02x\n", pgno, rc); + fflush(stdout); + } + return rc; +#else + /* Normal, high-speed version of sqlite3PagerGet() */ + return pPager->xGet(pPager, pgno, ppPage, flags); +#endif +} + /* ** Acquire a page if it is already in the in-memory cache. Do ** not read the page from disk. Return a pointer to the page, -** or 0 if the page is not in cache. +** or 0 if the page is not in cache. ** ** See also sqlite3PagerGet(). The difference between this routine ** and sqlite3PagerGet() is that _get() will go to the disk and read ** in the page if the page is not already in cache. This routine -** returns NULL if the page is not in cache or if a disk I/O error +** returns NULL if the page is not in cache or if a disk I/O error ** has ever happened. */ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ @@ -51377,46 +62953,62 @@ SQLITE_PRIVATE DbPage *sqlite3PagerLookup(Pager *pPager, Pgno pgno){ /* ** Release a page reference. ** -** If the number of references to the page drop to zero, then the -** page is added to the LRU list. When all references to all pages -** are released, a rollback occurs and the lock on the database is -** removed. +** The sqlite3PagerUnref() and sqlite3PagerUnrefNotNull() may only be used +** if we know that the page being released is not the last reference to page1. +** The btree layer always holds page1 open until the end, so these first +** two routines can be used to release any page other than BtShared.pPage1. +** The assert() at tag-20230419-2 proves that this constraint is always +** honored. +** +** Use sqlite3PagerUnrefPageOne() to release page1. This latter routine +** checks the total number of outstanding pages and if the number of +** pages reaches zero it drops the database lock. */ SQLITE_PRIVATE void sqlite3PagerUnrefNotNull(DbPage *pPg){ - Pager *pPager; + TESTONLY( Pager *pPager = pPg->pPager; ) assert( pPg!=0 ); - pPager = pPg->pPager; if( pPg->flags & PGHDR_MMAP ){ + assert( pPg->pgno!=1 ); /* Page1 is never memory mapped */ pagerReleaseMapPage(pPg); }else{ sqlite3PcacheRelease(pPg); } - pagerUnlockIfUnused(pPager); + /* Do not use this routine to release the last reference to page1 */ + assert( sqlite3PcacheRefCount(pPager->pPCache)>0 ); /* tag-20230419-2 */ } SQLITE_PRIVATE void sqlite3PagerUnref(DbPage *pPg){ if( pPg ) sqlite3PagerUnrefNotNull(pPg); } +SQLITE_PRIVATE void sqlite3PagerUnrefPageOne(DbPage *pPg){ + Pager *pPager; + assert( pPg!=0 ); + assert( pPg->pgno==1 ); + assert( (pPg->flags & PGHDR_MMAP)==0 ); /* Page1 is never memory mapped */ + pPager = pPg->pPager; + sqlite3PcacheRelease(pPg); + pagerUnlockIfUnused(pPager); +} /* ** This function is called at the start of every write transaction. -** There must already be a RESERVED or EXCLUSIVE lock on the database +** There must already be a RESERVED or EXCLUSIVE lock on the database ** file when this routine is called. ** ** Open the journal file for pager pPager and write a journal header ** to the start of it. If there are active savepoints, open the sub-journal -** as well. This function is only used when the journal file is being -** opened to write a rollback log for a transaction. It is not used +** as well. This function is only used when the journal file is being +** opened to write a rollback log for a transaction. It is not used ** when opening a hot journal file to roll it back. ** ** If the journal file is already open (as it may be in exclusive mode), ** then this function just writes a journal header to the start of the -** already open file. +** already open file. ** ** Whether or not the journal file is opened by this function, the ** Pager.pInJournal bitvec structure is allocated. ** -** Return SQLITE_OK if everything is successful. Otherwise, return -** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or +** Return SQLITE_OK if everything is successful. Otherwise, return +** SQLITE_NOMEM if the attempt to allocate Pager.pInJournal fails, or ** an IO error code if opening or writing the journal file fails. */ static int pager_open_journal(Pager *pPager){ @@ -51426,7 +63018,7 @@ static int pager_open_journal(Pager *pPager){ assert( pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); assert( pPager->pInJournal==0 ); - + /* If already in the error state, this function is a no-op. But on ** the other hand, this routine is never called if we are already in ** an error state. */ @@ -51437,7 +63029,7 @@ static int pager_open_journal(Pager *pPager){ if( pPager->pInJournal==0 ){ return SQLITE_NOMEM_BKPT; } - + /* Open the journal file if it is not already open. */ if( !isOpen(pPager->jfd) ){ if( pPager->journalMode==PAGER_JOURNALMODE_MEMORY ){ @@ -51448,12 +63040,13 @@ static int pager_open_journal(Pager *pPager){ if( pPager->tempFile ){ flags |= (SQLITE_OPEN_DELETEONCLOSE|SQLITE_OPEN_TEMP_JOURNAL); + flags |= SQLITE_OPEN_EXCLUSIVE; nSpill = sqlite3Config.nStmtSpill; }else{ flags |= SQLITE_OPEN_MAIN_JOURNAL; nSpill = jrnlBufferSize(pPager); } - + /* Verify that the database still has the same name as it did when ** it was originally opened. */ rc = databaseIsUnmoved(pPager); @@ -51465,16 +63058,16 @@ static int pager_open_journal(Pager *pPager){ } assert( rc!=SQLITE_OK || isOpen(pPager->jfd) ); } - - - /* Write the first journal header to the journal file and open + + + /* Write the first journal header to the journal file and open ** the sub-journal if necessary. */ if( rc==SQLITE_OK ){ /* TODO: Check if all of these are really required. */ pPager->nRec = 0; pPager->journalOff = 0; - pPager->setMaster = 0; + pPager->setSuper = 0; pPager->journalHdr = 0; rc = writeJournalHdr(pPager); } @@ -51483,6 +63076,7 @@ static int pager_open_journal(Pager *pPager){ if( rc!=SQLITE_OK ){ sqlite3BitvecDestroy(pPager->pInJournal); pPager->pInJournal = 0; + pPager->journalOff = 0; }else{ assert( pPager->eState==PAGER_WRITER_LOCKED ); pPager->eState = PAGER_WRITER_CACHEMOD; @@ -51492,12 +63086,12 @@ static int pager_open_journal(Pager *pPager){ } /* -** Begin a write-transaction on the specified pager object. If a +** Begin a write-transaction on the specified pager object. If a ** write-transaction has already been opened, this function is a no-op. ** ** If the exFlag argument is false, then acquire at least a RESERVED ** lock on the database file. If exFlag is true, then acquire at least -** an EXCLUSIVE lock. If such a lock is already held, no locking +** an EXCLUSIVE lock. If such a lock is already held, no locking ** functions need be called. ** ** If the subjInMemory argument is non-zero, then any sub-journal opened @@ -51505,7 +63099,7 @@ static int pager_open_journal(Pager *pPager){ ** has no effect if the sub-journal is already opened (as it may be when ** running in exclusive mode) or if the transaction does not require a ** sub-journal. If the subjInMemory argument is zero, then any required -** sub-journal is implemented in-memory if pPager is an in-memory database, +** sub-journal is implemented in-memory if pPager is an in-memory database, ** or using a temporary file otherwise. */ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory){ @@ -51515,7 +63109,7 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory assert( pPager->eState>=PAGER_READER && pPager->eStatesubjInMemory = (u8)subjInMemory; - if( ALWAYS(pPager->eState==PAGER_READER) ){ + if( pPager->eState==PAGER_READER ){ assert( pPager->pInJournal==0 ); if( pagerUseWal(pPager) ){ @@ -51553,9 +63147,9 @@ SQLITE_PRIVATE int sqlite3PagerBegin(Pager *pPager, int exFlag, int subjInMemory ** ** WAL mode sets Pager.eState to PAGER_WRITER_LOCKED or CACHEMOD ** when it has an open transaction, but never to DBMOD or FINISHED. - ** This is because in those states the code to roll back savepoint - ** transactions may copy data from the sub-journal into the database - ** file as well as into the page cache. Which would be incorrect in + ** This is because in those states the code to roll back savepoint + ** transactions may copy data from the sub-journal into the database + ** file as well as into the page cache. Which would be incorrect in ** WAL mode. */ pPager->eState = PAGER_WRITER_LOCKED; @@ -51587,10 +63181,10 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ /* We should never write to the journal file the page that ** contains the database locks. The following assert verifies ** that we do not. */ - assert( pPg->pgno!=PAGER_MJ_PGNO(pPager) ); + assert( pPg->pgno!=PAGER_SJ_PGNO(pPager) ); assert( pPager->journalHdr<=pPager->journalOff ); - CODEC2(pPager, pPg->pData, pPg->pgno, 7, return SQLITE_NOMEM_BKPT, pData2); + pData2 = pPg->pData; cksum = pager_cksum(pPager, (u8*)pData2); /* Even if an IO or diskfull error occurs while journalling the @@ -51609,11 +63203,11 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ rc = write32bits(pPager->jfd, iOff+pPager->pageSize+4, cksum); if( rc!=SQLITE_OK ) return rc; - IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, + IOTRACE(("JOUT %p %d %lld %d\n", pPager, pPg->pgno, pPager->journalOff, pPager->pageSize)); PAGER_INCR(sqlite3_pager_writej_count); PAGERTRACE(("JOURNAL %d page %d needSync=%d hash(%08x)\n", - PAGERID(pPager), pPg->pgno, + PAGERID(pPager), pPg->pgno, ((pPg->flags&PGHDR_NEED_SYNC)?1:0), pager_pagehash(pPg))); pPager->journalOff += 8 + pPager->pageSize; @@ -51628,9 +63222,9 @@ static SQLITE_NOINLINE int pagerAddPageToRollbackJournal(PgHdr *pPg){ } /* -** Mark a single data page as writeable. The page is written into the +** Mark a single data page as writeable. The page is written into the ** main journal or sub-journal as required. If the page is written into -** one of the journals, the corresponding bit is set in the +** one of the journals, the corresponding bit is set in the ** Pager.pInJournal bitvec and the PagerSavepoint.pInSavepoint bitvecs ** of any open savepoints as appropriate. */ @@ -51638,7 +63232,7 @@ static int pager_write(PgHdr *pPg){ Pager *pPager = pPg->pPager; int rc = SQLITE_OK; - /* This routine is not called unless a write-transaction has already + /* This routine is not called unless a write-transaction has already ** been started. The journal file may or may not be open at this point. ** It is never called in the ERROR state. */ @@ -51655,7 +63249,7 @@ static int pager_write(PgHdr *pPg){ ** obtained the necessary locks to begin the write-transaction, but the ** rollback journal might not yet be open. Open it now if this is the case. ** - ** This is done before calling sqlite3PcacheMakeDirty() on the page. + ** This is done before calling sqlite3PcacheMakeDirty() on the page. ** Otherwise, if it were done after calling sqlite3PcacheMakeDirty(), then ** an error might occur and the pager would end up in WRITER_LOCKED state ** with pages marked as dirty in the cache. @@ -51700,7 +63294,7 @@ static int pager_write(PgHdr *pPg){ ** PGHDR_WRITEABLE bit that indicates that the page can be safely modified. */ pPg->flags |= PGHDR_WRITEABLE; - + /* If the statement journal is open and the page is not in it, ** then write the page into the statement journal. */ @@ -51766,7 +63360,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ Pgno pg = pg1+ii; PgHdr *pPage; if( pg==pPg->pgno || !sqlite3BitvecTest(pPager->pInJournal, pg) ){ - if( pg!=PAGER_MJ_PGNO(pPager) ){ + if( pg!=PAGER_SJ_PGNO(pPager) ){ rc = sqlite3PagerGet(pPager, pg, &pPage, 0); if( rc==SQLITE_OK ){ rc = pager_write(pPage); @@ -51784,7 +63378,7 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ } } - /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages + /* If the PGHDR_NEED_SYNC flag is set for any of the nPage pages ** starting at pg1, then it needs to be set for all of them. Because ** writing to any of these nPage pages may damage the others, the ** journal file must contain sync()ed copies of all of them @@ -51807,9 +63401,9 @@ static SQLITE_NOINLINE int pagerWriteLargeSector(PgHdr *pPg){ } /* -** Mark a data page as writeable. This routine must be called before -** making changes to a page. The caller must check the return value -** of this function and be careful not to change any page data unless +** Mark a data page as writeable. This routine must be called before +** making changes to a page. The caller must check the return value +** of this function and be careful not to change any page data unless ** this routine returns SQLITE_OK. ** ** The difference between this function and pager_write() is that this @@ -51825,11 +63419,11 @@ SQLITE_PRIVATE int sqlite3PagerWrite(PgHdr *pPg){ assert( (pPg->flags & PGHDR_MMAP)==0 ); assert( pPager->eState>=PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); - if( pPager->errCode ){ - return pPager->errCode; - }else if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ + if( (pPg->flags & PGHDR_WRITEABLE)!=0 && pPager->dbSize>=pPg->pgno ){ if( pPager->nSavepoint ) return subjournalPageIfRequired(pPg); return SQLITE_OK; + }else if( pPager->errCode ){ + return pPager->errCode; }else if( pPager->sectorSize > (u32)pPager->pageSize ){ assert( pPager->tempFile==0 ); return pagerWriteLargeSector(pPg); @@ -51860,13 +63454,13 @@ SQLITE_PRIVATE int sqlite3PagerIswriteable(DbPage *pPg){ ** on the given page is unused. The pager marks the page as clean so ** that it does not get written to disk. ** -** Tests show that this optimization can quadruple the speed of large +** Tests show that this optimization can quadruple the speed of large ** DELETE operations. ** ** This optimization cannot be used with a temp-file, as the page may ** have been dirty at the start of the transaction. In that case, if -** memory pressure forces page pPg out of the cache, the data does need -** to be written out to disk so that it may be read back in if the +** memory pressure forces page pPg out of the cache, the data does need +** to be written out to disk so that it may be read back in if the ** current transaction is rolled back. */ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ @@ -51882,17 +63476,17 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ } /* -** This routine is called to increment the value of the database file -** change-counter, stored as a 4-byte big-endian integer starting at +** This routine is called to increment the value of the database file +** change-counter, stored as a 4-byte big-endian integer starting at ** byte offset 24 of the pager file. The secondary change counter at ** 92 is also updated, as is the SQLite version number at offset 96. ** ** But this only happens if the pPager->changeCountDone flag is false. ** To avoid excess churning of page 1, the update only happens once. -** See also the pager_write_changecounter() routine that does an +** See also the pager_write_changecounter() routine that does an ** unconditional update of the change counters. ** -** If the isDirectMode flag is zero, then this is done by calling +** If the isDirectMode flag is zero, then this is done by calling ** sqlite3PagerWrite() on page 1, then modifying the contents of the ** page data. In this case the file will be updated when the current ** transaction is committed. @@ -51900,7 +63494,7 @@ SQLITE_PRIVATE void sqlite3PagerDontWrite(PgHdr *pPg){ ** The isDirectMode flag may only be non-zero if the library was compiled ** with the SQLITE_ENABLE_ATOMIC_WRITE macro defined. In this case, ** if isDirect is non-zero, then the database file is updated directly -** by writing an updated version of page 1 using a call to the +** by writing an updated version of page 1 using a call to the ** sqlite3OsWrite() function. */ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ @@ -51929,7 +63523,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ # define DIRECT_MODE isDirectMode #endif - if( !pPager->changeCountDone && ALWAYS(pPager->dbSize>0) ){ + if( !pPager->changeCountDone && pPager->dbSize>0 ){ PgHdr *pPgHdr; /* Reference to page 1 */ assert( !pPager->tempFile && isOpen(pPager->fd) ); @@ -51939,7 +63533,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ assert( pPgHdr==0 || rc==SQLITE_OK ); /* If page one was fetched successfully, and this function is not - ** operating in direct-mode, make page 1 writable. When not in + ** operating in direct-mode, make page 1 writable. When not in ** direct mode, page 1 is always held in cache and hence the PagerGet() ** above is always successful - hence the ALWAYS on rc==SQLITE_OK. */ @@ -51955,7 +63549,7 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ if( DIRECT_MODE ){ const void *zBuf; assert( pPager->dbFileSize>0 ); - CODEC2(pPager, pPgHdr->pData, 1, 6, rc=SQLITE_NOMEM_BKPT, zBuf); + zBuf = pPgHdr->pData; if( rc==SQLITE_OK ){ rc = sqlite3OsWrite(pPager->fd, zBuf, pPager->pageSize, 0); pPager->aStat[PAGER_STAT_WRITE]++; @@ -51986,14 +63580,11 @@ static int pager_incr_changecounter(Pager *pPager, int isDirectMode){ ** If successful, or if called on a pager for which it is a no-op, this ** function returns SQLITE_OK. Otherwise, an IO error code is returned. */ -SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster){ +SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zSuper){ int rc = SQLITE_OK; - - if( isOpen(pPager->fd) ){ - void *pArg = (void*)zMaster; - rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg); - if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; - } + void *pArg = (void*)zSuper; + rc = sqlite3OsFileControl(pPager->fd, SQLITE_FCNTL_SYNC, pArg); + if( rc==SQLITE_NOTFOUND ) rc = SQLITE_OK; if( rc==SQLITE_OK && !pPager->noSync ){ assert( !MEMDB ); rc = sqlite3OsSync(pPager->fd, pPager->syncFlags); @@ -52003,22 +63594,22 @@ SQLITE_PRIVATE int sqlite3PagerSync(Pager *pPager, const char *zMaster){ /* ** This function may only be called while a write-transaction is active in -** rollback. If the connection is in WAL mode, this call is a no-op. -** Otherwise, if the connection does not already have an EXCLUSIVE lock on +** rollback. If the connection is in WAL mode, this call is a no-op. +** Otherwise, if the connection does not already have an EXCLUSIVE lock on ** the database file, an attempt is made to obtain one. ** ** If the EXCLUSIVE lock is already held or the attempt to obtain it is ** successful, or the connection is in WAL mode, SQLITE_OK is returned. -** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is +** Otherwise, either SQLITE_BUSY or an SQLITE_IOERR_XXX error code is ** returned. */ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ int rc = pPager->errCode; assert( assert_pager_state(pPager) ); if( rc==SQLITE_OK ){ - assert( pPager->eState==PAGER_WRITER_CACHEMOD - || pPager->eState==PAGER_WRITER_DBMOD - || pPager->eState==PAGER_WRITER_LOCKED + assert( pPager->eState==PAGER_WRITER_CACHEMOD + || pPager->eState==PAGER_WRITER_DBMOD + || pPager->eState==PAGER_WRITER_LOCKED ); assert( assert_pager_state(pPager) ); if( 0==pagerUseWal(pPager) ){ @@ -52029,24 +63620,24 @@ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ } /* -** Sync the database file for the pager pPager. zMaster points to the name -** of a master journal file that should be written into the individual -** journal file. zMaster may be NULL, which is interpreted as no master -** journal (a single database transaction). +** Sync the database file for the pager pPager. zSuper points to the name +** of a super-journal file that should be written into the individual +** journal file. zSuper may be NULL, which is interpreted as no +** super-journal (a single database transaction). ** ** This routine ensures that: ** ** * The database file change-counter is updated, ** * the journal is synced (unless the atomic-write optimization is used), -** * all dirty pages are written to the database file, +** * all dirty pages are written to the database file, ** * the database file is truncated (if required), and -** * the database file synced. +** * the database file synced. ** -** The only thing that remains to commit the transaction is to finalize -** (delete, truncate or zero the first part of) the journal file (or -** delete the master journal file if specified). +** The only thing that remains to commit the transaction is to finalize +** (delete, truncate or zero the first part of) the journal file (or +** delete the super-journal file if specified). ** -** Note that if zMaster==NULL, this does not overwrite a previous value +** Note that if zSuper==NULL, this does not overwrite a previous value ** passed to an sqlite3PagerCommitPhaseOne() call. ** ** If the final parameter - noSync - is true, then the database file itself @@ -52056,7 +63647,7 @@ SQLITE_PRIVATE int sqlite3PagerExclusiveLock(Pager *pPager){ */ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( Pager *pPager, /* Pager object */ - const char *zMaster, /* If not NULL, the master journal name */ + const char *zSuper, /* If not NULL, the super-journal name */ int noSync /* True to omit the xSync on the db file */ ){ int rc = SQLITE_OK; /* Return code */ @@ -52074,8 +63665,8 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( /* Provide the ability to easily simulate an I/O error during testing */ if( sqlite3FaultSim(400) ) return SQLITE_IOERR; - PAGERTRACE(("DATABASE SYNC: File=%s zMaster=%s nSize=%d\n", - pPager->zFilename, zMaster, pPager->dbSize)); + PAGERTRACE(("DATABASE SYNC: File=%s zSuper=%s nSize=%d\n", + pPager->zFilename, zSuper, pPager->dbSize)); /* If no database changes have been made, return early. */ if( pPager->eStatepBackup); }else{ + PgHdr *pList; if( pagerUseWal(pPager) ){ - PgHdr *pList = sqlite3PcacheDirtyList(pPager->pPCache); PgHdr *pPageOne = 0; + pList = sqlite3PcacheDirtyList(pPager->pPCache); if( pList==0 ){ /* Must have at least one page for the WAL commit flag. ** Ticket [2d1a5c67dfc2363e44f29d9bbd57f] 2011-05-18 */ @@ -52107,13 +63699,28 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( sqlite3PcacheCleanAll(pPager->pPCache); } }else{ + /* The bBatch boolean is true if the batch-atomic-write commit method + ** should be used. No rollback journal is created if batch-atomic-write + ** is enabled. + */ +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + sqlite3_file *fd = pPager->fd; + int bBatch = zSuper==0 /* An SQLITE_IOCAP_BATCH_ATOMIC commit */ + && (sqlite3OsDeviceCharacteristics(fd) & SQLITE_IOCAP_BATCH_ATOMIC) + && !pPager->noSync + && sqlite3JournalIsInMemory(pPager->jfd); +#else +# define bBatch 0 +#endif + +#ifdef SQLITE_ENABLE_ATOMIC_WRITE /* The following block updates the change-counter. Exactly how it ** does this depends on whether or not the atomic-update optimization - ** was enabled at compile time, and if this transaction meets the - ** runtime criteria to use the operation: + ** was enabled at compile time, and if this transaction meets the + ** runtime criteria to use the operation: ** ** * The file-system supports the atomic-write property for - ** blocks of size page-size, and + ** blocks of size page-size, and ** * This commit is not part of a multi-file transaction, and ** * Exactly one page has been modified and store in the journal file. ** @@ -52123,86 +63730,130 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseOne( ** is not applicable to this transaction, call sqlite3JournalCreate() ** to make sure the journal file has actually been created, then call ** pager_incr_changecounter() to update the change-counter in indirect - ** mode. + ** mode. ** ** Otherwise, if the optimization is both enabled and applicable, ** then call pager_incr_changecounter() to update the change-counter ** in 'direct' mode. In this case the journal file will never be ** created for this transaction. */ - #ifdef SQLITE_ENABLE_ATOMIC_WRITE - PgHdr *pPg; - assert( isOpen(pPager->jfd) - || pPager->journalMode==PAGER_JOURNALMODE_OFF - || pPager->journalMode==PAGER_JOURNALMODE_WAL - ); - if( !zMaster && isOpen(pPager->jfd) - && pPager->journalOff==jrnlBufferSize(pPager) - && pPager->dbSize>=pPager->dbOrigSize - && (0==(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) - ){ - /* Update the db file change counter via the direct-write method. The - ** following call will modify the in-memory representation of page 1 - ** to include the updated change counter and then write page 1 - ** directly to the database file. Because of the atomic-write - ** property of the host file-system, this is safe. - */ - rc = pager_incr_changecounter(pPager, 1); - }else{ - rc = sqlite3JournalCreate(pPager->jfd); - if( rc==SQLITE_OK ){ - rc = pager_incr_changecounter(pPager, 0); + if( bBatch==0 ){ + PgHdr *pPg; + assert( isOpen(pPager->jfd) + || pPager->journalMode==PAGER_JOURNALMODE_OFF + || pPager->journalMode==PAGER_JOURNALMODE_WAL + ); + if( !zSuper && isOpen(pPager->jfd) + && pPager->journalOff==jrnlBufferSize(pPager) + && pPager->dbSize>=pPager->dbOrigSize + && (!(pPg = sqlite3PcacheDirtyList(pPager->pPCache)) || 0==pPg->pDirty) + ){ + /* Update the db file change counter via the direct-write method. The + ** following call will modify the in-memory representation of page 1 + ** to include the updated change counter and then write page 1 + ** directly to the database file. Because of the atomic-write + ** property of the host file-system, this is safe. + */ + rc = pager_incr_changecounter(pPager, 1); + }else{ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc==SQLITE_OK ){ + rc = pager_incr_changecounter(pPager, 0); + } } } - #else +#else /* SQLITE_ENABLE_ATOMIC_WRITE */ +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( zSuper ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ) goto commit_phase_one_exit; + assert( bBatch==0 ); + } +#endif rc = pager_incr_changecounter(pPager, 0); - #endif +#endif /* !SQLITE_ENABLE_ATOMIC_WRITE */ if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - /* Write the master journal name into the journal file. If a master - ** journal file name has already been written to the journal file, - ** or if zMaster is NULL (no master journal), then this call is a no-op. + + /* Write the super-journal name into the journal file. If a + ** super-journal file name has already been written to the journal file, + ** or if zSuper is NULL (no super-journal), then this call is a no-op. */ - rc = writeMasterJournal(pPager, zMaster); + rc = writeSuperJournal(pPager, zSuper); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - + /* Sync the journal file and write all dirty pages to the database. - ** If the atomic-update optimization is being used, this sync will not + ** If the atomic-update optimization is being used, this sync will not ** create the journal file or perform any real IO. ** ** Because the change-counter page was just modified, unless the ** atomic-update optimization is used it is almost certain that the ** journal requires a sync here. However, in locking_mode=exclusive - ** on a system under memory pressure it is just possible that this is + ** on a system under memory pressure it is just possible that this is ** not the case. In this case it is likely enough that the redundant - ** xSync() call will be changed to a no-op by the OS anyhow. + ** xSync() call will be changed to a no-op by the OS anyhow. */ rc = syncJournal(pPager, 0); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; - - rc = pager_write_pagelist(pPager,sqlite3PcacheDirtyList(pPager->pPCache)); + + pList = sqlite3PcacheDirtyList(pPager->pPCache); +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + if( bBatch ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_BEGIN_ATOMIC_WRITE, 0); + if( rc==SQLITE_OK ){ + rc = pager_write_pagelist(pPager, pList); + if( rc==SQLITE_OK && pPager->dbSize>pPager->dbFileSize ){ + char *pTmp = pPager->pTmpSpace; + int szPage = (int)pPager->pageSize; + memset(pTmp, 0, szPage); + rc = sqlite3OsWrite(pPager->fd, pTmp, szPage, + ((i64)pPager->dbSize*pPager->pageSize)-szPage); + } + if( rc==SQLITE_OK ){ + rc = sqlite3OsFileControl(fd, SQLITE_FCNTL_COMMIT_ATOMIC_WRITE, 0); + } + if( rc!=SQLITE_OK ){ + sqlite3OsFileControlHint(fd, SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE, 0); + } + } + + if( (rc&0xFF)==SQLITE_IOERR && rc!=SQLITE_IOERR_NOMEM ){ + rc = sqlite3JournalCreate(pPager->jfd); + if( rc!=SQLITE_OK ){ + sqlite3OsClose(pPager->jfd); + goto commit_phase_one_exit; + } + bBatch = 0; + }else{ + sqlite3OsClose(pPager->jfd); + } + } +#endif /* SQLITE_ENABLE_BATCH_ATOMIC_WRITE */ + + if( bBatch==0 ){ + rc = pager_write_pagelist(pPager, pList); + } if( rc!=SQLITE_OK ){ assert( rc!=SQLITE_IOERR_BLOCKED ); goto commit_phase_one_exit; } sqlite3PcacheCleanAll(pPager->pPCache); - /* If the file on disk is smaller than the database image, use + /* If the file on disk is smaller than the database image, use ** pager_truncate to grow the file here. This can happen if the database ** image was extended as part of the current transaction and then the ** last page in the db image moved to the free-list. In this case the ** last page is never written out to disk, leaving the database file ** undersized. Fix this now if it is the case. */ if( pPager->dbSize>pPager->dbFileSize ){ - Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_MJ_PGNO(pPager)); + Pgno nNew = pPager->dbSize - (pPager->dbSize==PAGER_SJ_PGNO(pPager)); assert( pPager->eState==PAGER_WRITER_DBMOD ); rc = pager_truncate(pPager, nNew); if( rc!=SQLITE_OK ) goto commit_phase_one_exit; } - + /* Finally, sync the database file. */ if( !noSync ){ - rc = sqlite3PagerSync(pPager, zMaster); + rc = sqlite3PagerSync(pPager, zSuper); } IOTRACE(("DBSYNC %p\n", pPager)) } @@ -52219,12 +63870,12 @@ commit_phase_one_exit: /* ** When this function is called, the database file has been completely ** updated to reflect the changes made by the current transaction and -** synced to disk. The journal file still exists in the file-system +** synced to disk. The journal file still exists in the file-system ** though, and if a failure occurs at this point it will eventually ** be used as a hot-journal and the current transaction rolled back. ** -** This function finalizes the journal file, either by deleting, -** truncating or partially zeroing it, so that it cannot be used +** This function finalizes the journal file, either by deleting, +** truncating or partially zeroing it, so that it cannot be used ** for hot-journal rollback. Once this is done the transaction is ** irrevocably committed. ** @@ -52238,6 +63889,7 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** But if (due to a coding error elsewhere in the system) it does get ** called, just return the same error code without doing anything. */ if( NEVER(pPager->errCode) ) return pPager->errCode; + pPager->iDataVersion++; assert( pPager->eState==PAGER_WRITER_LOCKED || pPager->eState==PAGER_WRITER_FINISHED @@ -52249,15 +63901,15 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** this transaction, the pager is running in exclusive-mode and is ** using persistent journals, then this function is a no-op. ** - ** The start of the journal file currently contains a single journal + ** The start of the journal file currently contains a single journal ** header with the nRec field set to 0. If such a journal is used as ** a hot-journal during hot-journal rollback, 0 changes will be made - ** to the database file. So there is no need to zero the journal + ** to the database file. So there is no need to zero the journal ** header. Since the pager is in exclusive mode, there is no need ** to drop any locks either. */ - if( pPager->eState==PAGER_WRITER_LOCKED - && pPager->exclusiveMode + if( pPager->eState==PAGER_WRITER_LOCKED + && pPager->exclusiveMode && pPager->journalMode==PAGER_JOURNALMODE_PERSIST ){ assert( pPager->journalOff==JOURNAL_HDR_SZ(pPager) || !pPager->journalOff ); @@ -52266,13 +63918,12 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ } PAGERTRACE(("COMMIT %d\n", PAGERID(pPager))); - pPager->iDataVersion++; - rc = pager_end_transaction(pPager, pPager->setMaster, 1); + rc = pager_end_transaction(pPager, pPager->setSuper, 1); return pager_error(pPager, rc); } /* -** If a write transaction is open, then all changes made within the +** If a write transaction is open, then all changes made within the ** transaction are reverted and the current write-transaction is closed. ** The pager falls back to PAGER_READER state if successful, or PAGER_ERROR ** state if an error occurs. @@ -52282,14 +63933,14 @@ SQLITE_PRIVATE int sqlite3PagerCommitPhaseTwo(Pager *pPager){ ** ** Otherwise, in rollback mode, this function performs two functions: ** -** 1) It rolls back the journal file, restoring all database file and +** 1) It rolls back the journal file, restoring all database file and ** in-memory cache pages to the state they were in when the transaction ** was opened, and ** ** 2) It finalizes the journal file, so that it is not used for hot ** rollback at any point in the future. ** -** Finalization of the journal file (task 2) is only performed if the +** Finalization of the journal file (task 2) is only performed if the ** rollback is successful. ** ** In WAL mode, all cache-entries containing data modified within the @@ -52302,7 +63953,7 @@ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ PAGERTRACE(("ROLLBACK %d\n", PAGERID(pPager))); /* PagerRollback() is a no-op if called in READER or OPEN state. If - ** the pager is already in the ERROR state, the rollback is not + ** the pager is already in the ERROR state, the rollback is not ** attempted here. Instead, the error code is returned to the caller. */ assert( assert_pager_state(pPager) ); @@ -52312,18 +63963,19 @@ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ if( pagerUseWal(pPager) ){ int rc2; rc = sqlite3PagerSavepoint(pPager, SAVEPOINT_ROLLBACK, -1); - rc2 = pager_end_transaction(pPager, pPager->setMaster, 0); + rc2 = pager_end_transaction(pPager, pPager->setSuper, 0); if( rc==SQLITE_OK ) rc = rc2; }else if( !isOpen(pPager->jfd) || pPager->eState==PAGER_WRITER_LOCKED ){ int eState = pPager->eState; rc = pager_end_transaction(pPager, 0, 0); if( !MEMDB && eState>PAGER_WRITER_LOCKED ){ - /* This can happen using journal_mode=off. Move the pager to the error + /* This can happen using journal_mode=off. Move the pager to the error ** state to indicate that the contents of the cache may not be trusted. ** Any active readers will get SQLITE_ABORT. */ pPager->errCode = SQLITE_ABORT; pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); return rc; } }else{ @@ -52332,7 +63984,7 @@ SQLITE_PRIVATE int sqlite3PagerRollback(Pager *pPager){ assert( pPager->eState==PAGER_READER || rc!=SQLITE_OK ); assert( rc==SQLITE_OK || rc==SQLITE_FULL || rc==SQLITE_CORRUPT - || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR + || rc==SQLITE_NOMEM || (rc&0xFF)==SQLITE_IOERR || rc==SQLITE_CANTOPEN ); @@ -52364,8 +64016,8 @@ SQLITE_PRIVATE int sqlite3PagerRefcount(Pager *pPager){ ** used by the pager and its associated cache. */ SQLITE_PRIVATE int sqlite3PagerMemUsed(Pager *pPager){ - int perPageSize = pPager->pageSize + pPager->nExtra + sizeof(PgHdr) - + 5*sizeof(void*); + int perPageSize = pPager->pageSize + pPager->nExtra + + (int)(sizeof(PgHdr) + 5*sizeof(void*)); return perPageSize*sqlite3PcachePagecount(pPager->pPCache) + sqlite3MallocSize(pPager) + pPager->pageSize; @@ -52390,36 +64042,43 @@ SQLITE_PRIVATE int *sqlite3PagerStats(Pager *pPager){ a[3] = pPager->eState==PAGER_OPEN ? -1 : (int) pPager->dbSize; a[4] = pPager->eState; a[5] = pPager->errCode; - a[6] = pPager->aStat[PAGER_STAT_HIT]; - a[7] = pPager->aStat[PAGER_STAT_MISS]; + a[6] = (int)pPager->aStat[PAGER_STAT_HIT] & 0x7fffffff; + a[7] = (int)pPager->aStat[PAGER_STAT_MISS] & 0x7fffffff; a[8] = 0; /* Used to be pPager->nOvfl */ a[9] = pPager->nRead; - a[10] = pPager->aStat[PAGER_STAT_WRITE]; + a[10] = (int)pPager->aStat[PAGER_STAT_WRITE] & 0x7fffffff; return a; } #endif /* -** Parameter eStat must be either SQLITE_DBSTATUS_CACHE_HIT or -** SQLITE_DBSTATUS_CACHE_MISS. Before returning, *pnVal is incremented by the -** current cache hit or miss count, according to the value of eStat. If the -** reset parameter is non-zero, the cache hit or miss count is zeroed before +** Parameter eStat must be one of SQLITE_DBSTATUS_CACHE_HIT, _MISS, _WRITE, +** or _WRITE+1. The SQLITE_DBSTATUS_CACHE_WRITE+1 case is a translation +** of SQLITE_DBSTATUS_CACHE_SPILL. The _SPILL case is not contiguous because +** it was added later. +** +** Before returning, *pnVal is incremented by the +** current cache hit or miss count, according to the value of eStat. If the +** reset parameter is non-zero, the cache hit or miss count is zeroed before ** returning. */ -SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, int *pnVal){ +SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, u64 *pnVal){ assert( eStat==SQLITE_DBSTATUS_CACHE_HIT || eStat==SQLITE_DBSTATUS_CACHE_MISS || eStat==SQLITE_DBSTATUS_CACHE_WRITE + || eStat==SQLITE_DBSTATUS_CACHE_WRITE+1 ); assert( SQLITE_DBSTATUS_CACHE_HIT+1==SQLITE_DBSTATUS_CACHE_MISS ); assert( SQLITE_DBSTATUS_CACHE_HIT+2==SQLITE_DBSTATUS_CACHE_WRITE ); - assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 && PAGER_STAT_WRITE==2 ); + assert( PAGER_STAT_HIT==0 && PAGER_STAT_MISS==1 + && PAGER_STAT_WRITE==2 && PAGER_STAT_SPILL==3 ); - *pnVal += pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT]; + eStat -= SQLITE_DBSTATUS_CACHE_HIT; + *pnVal += pPager->aStat[eStat]; if( reset ){ - pPager->aStat[eStat - SQLITE_DBSTATUS_CACHE_HIT] = 0; + pPager->aStat[eStat] = 0; } } @@ -52427,7 +64086,7 @@ SQLITE_PRIVATE void sqlite3PagerCacheStat(Pager *pPager, int eStat, int reset, i ** Return true if this is an in-memory or temp-file backed pager. */ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ - return pPager->tempFile; + return pPager->tempFile || pPager->memVfs; } /* @@ -52436,7 +64095,7 @@ SQLITE_PRIVATE int sqlite3PagerIsMemdb(Pager *pPager){ ** to make up the difference. If the number of savepoints is already ** equal to nSavepoint, then this function is a no-op. ** -** If a memory allocation fails, SQLITE_NOMEM is returned. If an error +** If a memory allocation fails, SQLITE_NOMEM is returned. If an error ** occurs while opening the sub-journal file, then an IO error code is ** returned. Otherwise, SQLITE_OK. */ @@ -52451,7 +64110,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ assert( nSavepoint>nCurrent && pPager->useJournal ); /* Grow the Pager.aSavepoint array using realloc(). Return SQLITE_NOMEM - ** if the allocation fails. Otherwise, zero the new portion in case a + ** if the allocation fails. Otherwise, zero the new portion in case a ** malloc failure occurs while populating it in the for(...) loop below. */ aNew = (PagerSavepoint *)sqlite3Realloc( @@ -52473,6 +64132,7 @@ static SQLITE_NOINLINE int pagerOpenSavepoint(Pager *pPager, int nSavepoint){ } aNew[ii].iSubRec = pPager->nSubRec; aNew[ii].pInSavepoint = sqlite3BitvecCreate(pPager->dbSize); + aNew[ii].bTruncateOnRelease = 1; if( !aNew[ii].pInSavepoint ){ return SQLITE_NOMEM_BKPT; } @@ -52499,7 +64159,7 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ /* ** This function is called to rollback or release (commit) a savepoint. -** The savepoint to release or rollback need not be the most recently +** The savepoint to release or rollback need not be the most recently ** created savepoint. ** ** Parameter op is always either SAVEPOINT_ROLLBACK or SAVEPOINT_RELEASE. @@ -52507,28 +64167,32 @@ SQLITE_PRIVATE int sqlite3PagerOpenSavepoint(Pager *pPager, int nSavepoint){ ** index iSavepoint. If it is SAVEPOINT_ROLLBACK, then rollback all changes ** that have occurred since the specified savepoint was created. ** -** The savepoint to rollback or release is identified by parameter +** The savepoint to rollback or release is identified by parameter ** iSavepoint. A value of 0 means to operate on the outermost savepoint ** (the first created). A value of (Pager.nSavepoint-1) means operate ** on the most recently created savepoint. If iSavepoint is greater than ** (Pager.nSavepoint-1), then this function is a no-op. ** ** If a negative value is passed to this function, then the current -** transaction is rolled back. This is different to calling +** transaction is rolled back. This is different to calling ** sqlite3PagerRollback() because this function does not terminate -** the transaction or unlock the database, it just restores the -** contents of the database to its original state. +** the transaction or unlock the database, it just restores the +** contents of the database to its original state. ** -** In any case, all savepoints with an index greater than iSavepoint +** In any case, all savepoints with an index greater than iSavepoint ** are destroyed. If this is a release operation (op==SAVEPOINT_RELEASE), ** then savepoint iSavepoint is also destroyed. ** ** This function may return SQLITE_NOMEM if a memory allocation fails, -** or an IO error code if an IO error occurs while rolling back a +** or an IO error code if an IO error occurs while rolling back a ** savepoint. If no errors occur, SQLITE_OK is returned. -*/ +*/ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ - int rc = pPager->errCode; /* Return code */ + int rc = pPager->errCode; + +#ifdef SQLITE_ENABLE_ZIPVFS + if( op==SAVEPOINT_RELEASE ) rc = SQLITE_OK; +#endif assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || op==SAVEPOINT_ROLLBACK ); @@ -52538,7 +64202,7 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ int nNew; /* Number of remaining savepoints after this op. */ /* Figure out how many savepoints will still be active after this - ** operation. Store this value in nNew. Then free resources associated + ** operation. Store this value in nNew. Then free resources associated ** with any savepoints that are destroyed by this operation. */ nNew = iSavepoint + (( op==SAVEPOINT_RELEASE ) ? 0 : 1); @@ -52547,16 +64211,18 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ } pPager->nSavepoint = nNew; - /* If this is a release of the outermost savepoint, truncate - ** the sub-journal to zero bytes in size. */ + /* Truncate the sub-journal so that it only includes the parts + ** that are still in use. */ if( op==SAVEPOINT_RELEASE ){ - if( nNew==0 && isOpen(pPager->sjfd) ){ + PagerSavepoint *pRel = &pPager->aSavepoint[nNew]; + if( pRel->bTruncateOnRelease && isOpen(pPager->sjfd) ){ /* Only truncate if it is an in-memory sub-journal. */ if( sqlite3JournalIsInMemory(pPager->sjfd) ){ - rc = sqlite3OsTruncate(pPager->sjfd, 0); + i64 sz = (pPager->pageSize+4)*(i64)pRel->iSubRec; + rc = sqlite3OsTruncate(pPager->sjfd, sz); assert( rc==SQLITE_OK ); } - pPager->nSubRec = 0; + pPager->nSubRec = pRel->iSubRec; } } /* Else this is a rollback operation, playback the specified savepoint. @@ -52569,6 +64235,21 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ rc = pagerPlaybackSavepoint(pPager, pSavepoint); assert(rc!=SQLITE_DONE); } + +#ifdef SQLITE_ENABLE_ZIPVFS + /* If the cache has been modified but the savepoint cannot be rolled + ** back journal_mode=off, put the pager in the error state. This way, + ** if the VFS used by this pager includes ZipVFS, the entire transaction + ** can be rolled back at the ZipVFS level. */ + else if( + pPager->journalMode==PAGER_JOURNALMODE_OFF + && pPager->eState>=PAGER_WRITER_CACHEMOD + ){ + pPager->errCode = SQLITE_ABORT; + pPager->eState = PAGER_ERROR; + setGetterMethod(pPager); + } +#endif } return rc; @@ -52583,9 +64264,17 @@ SQLITE_PRIVATE int sqlite3PagerSavepoint(Pager *pPager, int op, int iSavepoint){ ** behavior. But when the Btree needs to know the filename for matching to ** shared cache, it uses nullIfMemDb==0 so that in-memory databases can ** participate in shared-cache. +** +** The return value to this routine is always safe to use with +** sqlite3_uri_parameter() and sqlite3_filename_database() and friends. */ -SQLITE_PRIVATE const char *sqlite3PagerFilename(Pager *pPager, int nullIfMemDb){ - return (nullIfMemDb && pPager->memDb) ? "" : pPager->zFilename; +SQLITE_PRIVATE const char *sqlite3PagerFilename(const Pager *pPager, int nullIfMemDb){ + static const char zFake[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + if( nullIfMemDb && (pPager->memDb || sqlite3IsMemdb(pPager->pVfs)) ){ + return &zFake[4]; + }else{ + return pPager->zFilename; + } } /* @@ -52609,7 +64298,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3PagerFile(Pager *pPager){ ** This will be either the rollback journal or the WAL file. */ SQLITE_PRIVATE sqlite3_file *sqlite3PagerJrnlFile(Pager *pPager){ -#if SQLITE_OMIT_WAL +#ifdef SQLITE_OMIT_WAL return pPager->jfd; #else return pPager->pWal ? sqlite3WalFile(pPager->pWal) : pPager->jfd; @@ -52623,49 +64312,6 @@ SQLITE_PRIVATE const char *sqlite3PagerJournalname(Pager *pPager){ return pPager->zJournal; } -#ifdef SQLITE_HAS_CODEC -/* -** Set or retrieve the codec for this pager -*/ -SQLITE_PRIVATE void sqlite3PagerSetCodec( - Pager *pPager, - void *(*xCodec)(void*,void*,Pgno,int), - void (*xCodecSizeChng)(void*,int,int), - void (*xCodecFree)(void*), - void *pCodec -){ - if( pPager->xCodecFree ) pPager->xCodecFree(pPager->pCodec); - pPager->xCodec = pPager->memDb ? 0 : xCodec; - pPager->xCodecSizeChng = xCodecSizeChng; - pPager->xCodecFree = xCodecFree; - pPager->pCodec = pCodec; - pagerReportSize(pPager); -} -SQLITE_PRIVATE void *sqlite3PagerGetCodec(Pager *pPager){ - return pPager->pCodec; -} - -/* -** This function is called by the wal module when writing page content -** into the log file. -** -** This function returns a pointer to a buffer containing the encrypted -** page content. If a malloc fails, this function may return NULL. -*/ -SQLITE_PRIVATE void *sqlite3PagerCodec(PgHdr *pPg){ - void *aData = 0; - CODEC2(pPg->pPager, pPg->pData, pPg->pgno, 6, return 0, aData); - return aData; -} - -/* -** Return the current pager state -*/ -SQLITE_PRIVATE int sqlite3PagerState(Pager *pPager){ - return pPager->eState; -} -#endif /* SQLITE_HAS_CODEC */ - #ifndef SQLITE_OMIT_AUTOVACUUM /* ** Move the page pPg to location pgno in the file. @@ -52685,8 +64331,8 @@ SQLITE_PRIVATE int sqlite3PagerState(Pager *pPager){ ** transaction is active). ** ** If the fourth argument, isCommit, is non-zero, then this page is being -** moved as part of a database reorganization just before the transaction -** is being committed. In this case, it is guaranteed that the database page +** moved as part of a database reorganization just before the transaction +** is being committed. In this case, it is guaranteed that the database page ** pPg refers to will not be written to again within this transaction. ** ** This function may return SQLITE_NOMEM or an IO error code if an error @@ -52714,7 +64360,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } /* If the page being moved is dirty and has not been saved by the latest - ** savepoint, then save the current contents of the page into the + ** savepoint, then save the current contents of the page into the ** sub-journal now. This is required to handle the following scenario: ** ** BEGIN; @@ -52737,7 +64383,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i return rc; } - PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", + PAGERTRACE(("MOVE %d page %d (needSync=%d) moves to %d\n", PAGERID(pPager), pPg->pgno, (pPg->flags&PGHDR_NEED_SYNC)?1:0, pgno)); IOTRACE(("MOVE %p %d %d\n", pPager, pPg->pgno, pgno)) @@ -52745,7 +64391,7 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i ** be written to, store pPg->pgno in local variable needSyncPgno. ** ** If the isCommit flag is set, there is no need to remember that - ** the journal needs to be sync()ed before database page pPg->pgno + ** the journal needs to be sync()ed before database page pPg->pgno ** can be written to. The caller has already promised not to write to it. */ if( (pPg->flags&PGHDR_NEED_SYNC) && !isCommit ){ @@ -52756,14 +64402,18 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } /* If the cache contains a page with page-number pgno, remove it - ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for - ** page pgno before the 'move' operation, it needs to be retained + ** from its hash chain. Also, if the PGHDR_NEED_SYNC flag was set for + ** page pgno before the 'move' operation, it needs to be retained ** for the page moved there. */ pPg->flags &= ~PGHDR_NEED_SYNC; pPgOld = sqlite3PagerLookup(pPager, pgno); - assert( !pPgOld || pPgOld->nRef==1 ); + assert( !pPgOld || pPgOld->nRef==1 || CORRUPT_DB ); if( pPgOld ){ + if( NEVER(pPgOld->nRef>1) ){ + sqlite3PagerUnrefNotNull(pPgOld); + return SQLITE_CORRUPT_BKPT; + } pPg->flags |= (pPgOld->flags&PGHDR_NEED_SYNC); if( pPager->tempFile ){ /* Do not discard pages from an in-memory database since we might @@ -52788,9 +64438,9 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i } if( needSyncPgno ){ - /* If needSyncPgno is non-zero, then the journal file needs to be + /* If needSyncPgno is non-zero, then the journal file needs to be ** sync()ed before any data is written to database file page needSyncPgno. - ** Currently, no such page exists in the page-cache and the + ** Currently, no such page exists in the page-cache and the ** "is journaled" bitvec flag has been set. This needs to be remedied by ** loading the page into the pager-cache and setting the PGHDR_NEED_SYNC ** flag. @@ -52821,9 +64471,9 @@ SQLITE_PRIVATE int sqlite3PagerMovepage(Pager *pPager, DbPage *pPg, Pgno pgno, i #endif /* -** The page handle passed as the first argument refers to a dirty page -** with a page number other than iNew. This function changes the page's -** page number to iNew and sets the value of the PgHdr.flags field to +** The page handle passed as the first argument refers to a dirty page +** with a page number other than iNew. This function changes the page's +** page number to iNew and sets the value of the PgHdr.flags field to ** the value passed as the third parameter. */ SQLITE_PRIVATE void sqlite3PagerRekey(DbPage *pPg, Pgno iNew, u16 flags){ @@ -52841,7 +64491,7 @@ SQLITE_PRIVATE void *sqlite3PagerGetData(DbPage *pPg){ } /* -** Return a pointer to the Pager.nExtra bytes of "extra" space +** Return a pointer to the Pager.nExtra bytes of "extra" space ** allocated along with the specified page. */ SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *pPg){ @@ -52850,7 +64500,7 @@ SQLITE_PRIVATE void *sqlite3PagerGetExtra(DbPage *pPg){ /* ** Get/set the locking-mode for this pager. Parameter eMode must be one -** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or +** of PAGER_LOCKINGMODE_QUERY, PAGER_LOCKINGMODE_NORMAL or ** PAGER_LOCKINGMODE_EXCLUSIVE. If the parameter is not _QUERY, then ** the locking-mode is set to the value specified. ** @@ -52894,20 +64544,13 @@ SQLITE_PRIVATE int sqlite3PagerLockingMode(Pager *pPager, int eMode){ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ u8 eOld = pPager->journalMode; /* Prior journalmode */ -#ifdef SQLITE_DEBUG - /* The print_pager_state() routine is intended to be used by the debugger - ** only. We invoke it once here to suppress a compiler warning. */ - print_pager_state(pPager); -#endif - - /* The eMode parameter is always valid */ - assert( eMode==PAGER_JOURNALMODE_DELETE - || eMode==PAGER_JOURNALMODE_TRUNCATE - || eMode==PAGER_JOURNALMODE_PERSIST - || eMode==PAGER_JOURNALMODE_OFF - || eMode==PAGER_JOURNALMODE_WAL - || eMode==PAGER_JOURNALMODE_MEMORY ); + assert( eMode==PAGER_JOURNALMODE_DELETE /* 0 */ + || eMode==PAGER_JOURNALMODE_PERSIST /* 1 */ + || eMode==PAGER_JOURNALMODE_OFF /* 2 */ + || eMode==PAGER_JOURNALMODE_TRUNCATE /* 3 */ + || eMode==PAGER_JOURNALMODE_MEMORY /* 4 */ + || eMode==PAGER_JOURNALMODE_WAL /* 5 */ ); /* This routine is only called from the OP_JournalMode opcode, and ** the logic there will never allow a temporary file to be changed @@ -52931,7 +64574,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( pPager->eState!=PAGER_ERROR ); pPager->journalMode = (u8)eMode; - /* When transistioning from TRUNCATE or PERSIST to any other journal + /* When transitioning from TRUNCATE or PERSIST to any other journal ** mode except WAL, unless the pager is in locking_mode=exclusive mode, ** delete the journal file. */ @@ -52944,7 +64587,6 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ assert( isOpen(pPager->fd) || pPager->exclusiveMode ); if( !pPager->exclusiveMode && (eOld & 5)==1 && (eMode & 1)==0 ){ - /* In this case we would like to delete the journal file. If it is ** not possible, then that is not a problem. Deleting the journal file ** here is an optimization only. @@ -52977,7 +64619,7 @@ SQLITE_PRIVATE int sqlite3PagerSetJournalMode(Pager *pPager, int eMode){ } assert( state==pPager->eState ); } - }else if( eMode==PAGER_JOURNALMODE_OFF ){ + }else if( eMode==PAGER_JOURNALMODE_OFF || eMode==PAGER_JOURNALMODE_MEMORY ){ sqlite3OsClose(pPager->jfd); } } @@ -53039,6 +64681,7 @@ SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ } #endif + #ifndef SQLITE_OMIT_WAL /* ** This function is called when the user invokes "PRAGMA wal_checkpoint", @@ -53047,13 +64690,31 @@ SQLITE_PRIVATE void sqlite3PagerClearCache(Pager *pPager){ ** ** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. */ -SQLITE_PRIVATE int sqlite3PagerCheckpoint(Pager *pPager, int eMode, int *pnLog, int *pnCkpt){ +SQLITE_PRIVATE int sqlite3PagerCheckpoint( + Pager *pPager, /* Checkpoint on this pager */ + sqlite3 *db, /* Db handle used to check for interrupts */ + int eMode, /* Type of checkpoint */ + int *pnLog, /* OUT: Final number of frames in log */ + int *pnCkpt /* OUT: Final number of checkpointed frames */ +){ int rc = SQLITE_OK; + if( pPager->pWal==0 && pPager->journalMode==PAGER_JOURNALMODE_WAL ){ + /* This only happens when a database file is zero bytes in size opened and + ** then "PRAGMA journal_mode=WAL" is run and then sqlite3_wal_checkpoint() + ** is invoked without any intervening transactions. We need to start + ** a transaction to initialize pWal. The PRAGMA table_list statement is + ** used for this since it starts transactions on every database file, + ** including all ATTACHed databases. This seems expensive for a single + ** sqlite3_wal_checkpoint() call, but it happens very rarely. + ** https://sqlite.org/forum/forumpost/fd0f19d229156939 + */ + sqlite3_exec(db, "PRAGMA table_list",0,0,0); + } if( pPager->pWal ){ - rc = sqlite3WalCheckpoint(pPager->pWal, eMode, + rc = sqlite3WalCheckpoint(pPager->pWal, db, eMode, (eMode==SQLITE_CHECKPOINT_PASSIVE ? 0 : pPager->xBusyHandler), pPager->pBusyHandlerArg, - pPager->ckptSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, + pPager->walSyncFlags, pPager->pageSize, (u8 *)pPager->pTmpSpace, pnLog, pnCkpt ); } @@ -53080,20 +64741,22 @@ SQLITE_PRIVATE int sqlite3PagerWalSupported(Pager *pPager){ */ static int pagerExclusiveLock(Pager *pPager){ int rc; /* Return code */ + u8 eOrigLock; /* Original lock */ - assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); + assert( pPager->eLock>=SHARED_LOCK ); + eOrigLock = pPager->eLock; rc = pagerLockDb(pPager, EXCLUSIVE_LOCK); if( rc!=SQLITE_OK ){ - /* If the attempt to grab the exclusive lock failed, release the + /* If the attempt to grab the exclusive lock failed, release the ** pending lock that may have been obtained instead. */ - pagerUnlockDb(pPager, SHARED_LOCK); + pagerUnlockDb(pPager, eOrigLock); } return rc; } /* -** Call sqlite3WalOpen() to open the WAL handle. If the pager is in +** Call sqlite3WalOpen() to open the WAL handle. If the pager is in ** exclusive-locking mode when this function is called, take an EXCLUSIVE ** lock on the database file and use heap-memory to store the wal-index ** in. Otherwise, use the normal shared-memory. @@ -53104,8 +64767,8 @@ static int pagerOpenWal(Pager *pPager){ assert( pPager->pWal==0 && pPager->tempFile==0 ); assert( pPager->eLock==SHARED_LOCK || pPager->eLock==EXCLUSIVE_LOCK ); - /* If the pager is already in exclusive-mode, the WAL module will use - ** heap-memory for the wal-index instead of the VFS shared-memory + /* If the pager is already in exclusive-mode, the WAL module will use + ** heap-memory for the wal-index instead of the VFS shared-memory ** implementation. Take the exclusive lock now, before opening the WAL ** file, to make sure this is safe. */ @@ -53113,7 +64776,7 @@ static int pagerOpenWal(Pager *pPager){ rc = pagerExclusiveLock(pPager); } - /* Open the connection to the log file. If this operation fails, + /* Open the connection to the log file. If this operation fails, ** (e.g. due to malloc() failure), return an error code. */ if( rc==SQLITE_OK ){ @@ -53135,7 +64798,7 @@ static int pagerOpenWal(Pager *pPager){ ** If the pager passed as the first argument is open on a real database ** file (not a temp file or an in-memory database), and the WAL file ** is not already open, make an attempt to open it now. If successful, -** return SQLITE_OK. If an error occurs or the VFS used by the pager does +** return SQLITE_OK. If an error occurs or the VFS used by the pager does ** not support the xShmXXX() methods, return an error code. *pbOpen is ** not modified in either case. ** @@ -53177,12 +64840,12 @@ SQLITE_PRIVATE int sqlite3PagerOpenWal( ** This function is called to close the connection to the log file prior ** to switching from WAL to rollback mode. ** -** Before closing the log file, this function attempts to take an +** Before closing the log file, this function attempts to take an ** EXCLUSIVE lock on the database file. If this cannot be obtained, an ** error (SQLITE_BUSY) is returned and the log connection is not closed. ** If successful, the EXCLUSIVE lock is not released before returning. */ -SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ +SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager, sqlite3 *db){ int rc = SQLITE_OK; assert( pPager->journalMode==PAGER_JOURNALMODE_WAL ); @@ -53203,14 +64866,14 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ rc = pagerOpenWal(pPager); } } - + /* Checkpoint and close the log. Because an EXCLUSIVE lock is held on ** the database file, the log and log-summary files will be deleted. */ if( rc==SQLITE_OK && pPager->pWal ){ rc = pagerExclusiveLock(pPager); if( rc==SQLITE_OK ){ - rc = sqlite3WalClose(pPager->pWal, pPager->ckptSyncFlags, + rc = sqlite3WalClose(pPager->pWal, db, pPager->walSyncFlags, pPager->pageSize, (u8*)pPager->pTmpSpace); pPager->pWal = 0; pagerFixMaplimit(pPager); @@ -53220,6 +64883,32 @@ SQLITE_PRIVATE int sqlite3PagerCloseWal(Pager *pPager){ return rc; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +/* +** If pager pPager is a wal-mode database not in exclusive locking mode, +** invoke the sqlite3WalWriteLock() function on the associated Wal object +** with the same db and bLock parameters as were passed to this function. +** Return an SQLite error code if an error occurs, or SQLITE_OK otherwise. +*/ +SQLITE_PRIVATE int sqlite3PagerWalWriteLock(Pager *pPager, int bLock){ + int rc = SQLITE_OK; + if( pagerUseWal(pPager) && pPager->exclusiveMode==0 ){ + rc = sqlite3WalWriteLock(pPager->pWal, bLock); + } + return rc; +} + +/* +** Set the database handle used by the wal layer to determine if +** blocking locks are required. +*/ +SQLITE_PRIVATE void sqlite3PagerWalDb(Pager *pPager, sqlite3 *db){ + if( pagerUseWal(pPager) ){ + sqlite3WalDb(pPager->pWal, db); + } +} +#endif + #ifdef SQLITE_ENABLE_SNAPSHOT /* ** If this is a WAL database, obtain a snapshot handle for the snapshot @@ -53235,10 +64924,13 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotGet(Pager *pPager, sqlite3_snapshot **ppS /* ** If this is a WAL database, store a pointer to pSnapshot. Next time a -** read transaction is opened, attempt to read from the snapshot it +** read transaction is opened, attempt to read from the snapshot it ** identifies. If this is not a WAL database, return an error. */ -SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSnapshot){ +SQLITE_PRIVATE int sqlite3PagerSnapshotOpen( + Pager *pPager, + sqlite3_snapshot *pSnapshot +){ int rc = SQLITE_OK; if( pPager->pWal ){ sqlite3WalSnapshotOpen(pPager->pWal, pSnapshot); @@ -53247,6 +64939,52 @@ SQLITE_PRIVATE int sqlite3PagerSnapshotOpen(Pager *pPager, sqlite3_snapshot *pSn } return rc; } + +/* +** If this is a WAL database, call sqlite3WalSnapshotRecover(). If this +** is not a WAL database, return an error. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotRecover(Pager *pPager){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotRecover(pPager->pWal); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** The caller currently has a read transaction open on the database. +** If this is not a WAL database, SQLITE_ERROR is returned. Otherwise, +** this function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3PagerSnapshotCheck(Pager *pPager, sqlite3_snapshot *pSnapshot){ + int rc; + if( pPager->pWal ){ + rc = sqlite3WalSnapshotCheck(pPager->pWal, pSnapshot); + }else{ + rc = SQLITE_ERROR; + } + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3PagerSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3PagerSnapshotUnlock(Pager *pPager){ + assert( pPager->pWal ); + sqlite3WalSnapshotUnlock(pPager->pWal); +} + #endif /* SQLITE_ENABLE_SNAPSHOT */ #endif /* !SQLITE_OMIT_WAL */ @@ -53264,6 +65002,11 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ } #endif +#if defined(SQLITE_USE_SEH) && !defined(SQLITE_OMIT_WAL) +SQLITE_PRIVATE int sqlite3PagerWalSystemErrno(Pager *pPager){ + return sqlite3WalSystemErrno(pPager->pWal); +} +#endif #endif /* SQLITE_OMIT_DISKIO */ @@ -53281,7 +65024,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** ************************************************************************* ** -** This file contains the implementation of a write-ahead log (WAL) used in +** This file contains the implementation of a write-ahead log (WAL) used in ** "journal_mode=WAL" mode. ** ** WRITE-AHEAD LOG (WAL) FILE FORMAT @@ -53290,7 +65033,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** Each frame records the revised content of a single page from the ** database file. All changes to the database are recorded by writing ** frames into the WAL. Transactions commit when a frame is written that -** contains a commit marker. A single WAL can and usually does record +** contains a commit marker. A single WAL can and usually does record ** multiple transactions. Periodically, the content of the WAL is ** transferred back into the database file in an operation called a ** "checkpoint". @@ -53315,12 +65058,12 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** 28: Checksum-2 (second part of checksum for first 24 bytes of header). ** ** Immediately following the wal-header are zero or more frames. Each -** frame consists of a 24-byte frame-header followed by a bytes -** of page data. The frame-header is six big-endian 32-bit unsigned +** frame consists of a 24-byte frame-header followed by bytes +** of page data. The frame-header is six big-endian 32-bit unsigned ** integer values, as follows: ** ** 0: Page number. -** 4: For commit records, the size of the database image in pages +** 4: For commit records, the size of the database image in pages ** after the commit. For all other records, zero. ** 8: Salt-1 (copied from the header) ** 12: Salt-2 (copied from the header) @@ -53346,7 +65089,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** the checksum. The checksum is computed by interpreting the input as ** an even number of unsigned 32-bit integers: x[0] through x[N]. The ** algorithm used for the checksum is as follows: -** +** ** for i from 0 to n-1 step 2: ** s0 += x[i] + s1; ** s1 += x[i+1] + s0; @@ -53354,7 +65097,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** ** Note that s0 and s1 are both weighted checksums using fibonacci weights ** in reverse order (the largest fibonacci weight occurs on the first element -** of the sequence being summed.) The s1 value spans all 32-bit +** of the sequence being summed.) The s1 value spans all 32-bit ** terms of the sequence whereas s0 omits the final term. ** ** On a checkpoint, the WAL is first VFS.xSync-ed, then valid content of the @@ -53387,22 +65130,26 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** multiple concurrent readers to view different versions of the database ** content simultaneously. ** -** The reader algorithm in the previous paragraphs works correctly, but +** The reader algorithm in the previous paragraphs works correctly, but ** because frames for page P can appear anywhere within the WAL, the ** reader has to scan the entire WAL looking for page P frames. If the ** WAL is large (multiple megabytes is typical) that scan can be slow, ** and read performance suffers. To overcome this problem, a separate ** data structure called the wal-index is maintained to expedite the ** search for frames of a particular page. -** +** ** WAL-INDEX FORMAT ** ** Conceptually, the wal-index is shared memory, though VFS implementations ** might choose to implement the wal-index using a mmapped file. Because -** the wal-index is shared memory, SQLite does not support journal_mode=WAL +** the wal-index is shared memory, SQLite does not support journal_mode=WAL ** on a network filesystem. All users of the database must be able to ** share memory. ** +** In the default unix and windows implementation, the wal-index is a mmapped +** file whose name is the database name with a "-shm" suffix added. For that +** reason, the wal-index is sometimes called the "shm" file. +** ** The wal-index is transient. After a crash, the wal-index can (and should ** be) reconstructed from the original WAL file. In fact, the VFS is required ** to either truncate or zero the header of the wal-index when the last @@ -53413,28 +65160,31 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** byte order of the host computer. ** ** The purpose of the wal-index is to answer this question quickly: Given -** a page number P and a maximum frame index M, return the index of the +** a page number P and a maximum frame index M, return the index of the ** last frame in the wal before frame M for page P in the WAL, or return ** NULL if there are no frames for page P in the WAL prior to M. ** ** The wal-index consists of a header region, followed by an one or -** more index blocks. +** more index blocks. ** ** The wal-index header contains the total number of frames within the WAL ** in the mxFrame field. ** -** Each index block except for the first contains information on +** Each index block except for the first contains information on ** HASHTABLE_NPAGE frames. The first index block contains information on -** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and +** HASHTABLE_NPAGE_ONE frames. The values of HASHTABLE_NPAGE_ONE and ** HASHTABLE_NPAGE are selected so that together the wal-index header and ** first index block are the same size as all other index blocks in the -** wal-index. +** wal-index. The values are: +** +** HASHTABLE_NPAGE 4096 +** HASHTABLE_NPAGE_ONE 4062 ** ** Each index block contains two sections, a page-mapping that contains the -** database page number associated with each wal frame, and a hash-table +** database page number associated with each wal frame, and a hash-table ** that allows readers to query an index block for a specific page number. ** The page-mapping is an array of HASHTABLE_NPAGE (or HASHTABLE_NPAGE_ONE -** for the first index block) 32-bit page numbers. The first entry in the +** for the first index block) 32-bit page numbers. The first entry in the ** first index-block contains the database page number corresponding to the ** first frame in the WAL file. The first entry in the second index block ** in the WAL file corresponds to the (HASHTABLE_NPAGE_ONE+1)th frame in @@ -53455,8 +65205,8 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** ** The hash table consists of HASHTABLE_NSLOT 16-bit unsigned integers. ** HASHTABLE_NSLOT = 2*HASHTABLE_NPAGE, and there is one entry in the -** hash table for each page number in the mapping section, so the hash -** table is never more than half full. The expected number of collisions +** hash table for each page number in the mapping section, so the hash +** table is never more than half full. The expected number of collisions ** prior to finding a match is 1. Each entry of the hash table is an ** 1-based index of an entry in the mapping section of the same ** index block. Let K be the 1-based index of the largest entry in @@ -53475,12 +65225,12 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** reached) until an unused hash slot is found. Let the first unused slot ** be at index iUnused. (iUnused might be less than iKey if there was ** wrap-around.) Because the hash table is never more than half full, -** the search is guaranteed to eventually hit an unused entry. Let +** the search is guaranteed to eventually hit an unused entry. Let ** iMax be the value between iKey and iUnused, closest to iUnused, ** where aHash[iMax]==P. If there is no iMax entry (if there exists ** no hash slot such that aHash[i]==p) then page P is not in the ** current index block. Otherwise the iMax-th mapping entry of the -** current index block corresponds to the last entry that references +** current index block corresponds to the last entry that references ** page P. ** ** A hash search begins with the last index block and moves toward the @@ -53505,7 +65255,7 @@ SQLITE_PRIVATE int sqlite3PagerWalFramesize(Pager *pPager){ ** if no values greater than K0 had ever been inserted into the hash table ** in the first place - which is what reader one wants. Meanwhile, the ** second reader using K1 will see additional values that were inserted -** later, which is exactly what reader two wants. +** later, which is exactly what reader two wants. ** ** When a rollback occurs, the value of K is decreased. Hash table entries ** that correspond to frames greater than the new K value are removed @@ -53533,7 +65283,7 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; ** values in the wal-header are correct and (b) the version field is not ** WAL_MAX_VERSION, recovery fails and SQLite returns SQLITE_CANTOPEN. ** -** Similarly, if a client successfully reads a wal-index header (i.e. the +** Similarly, if a client successfully reads a wal-index header (i.e. the ** checksum test is successful) and finds that the version field is not ** WALINDEX_MAX_VERSION, then no read-transaction is opened and SQLite ** returns SQLITE_CANTOPEN. @@ -53542,9 +65292,18 @@ SQLITE_PRIVATE int sqlite3WalTrace = 0; #define WALINDEX_MAX_VERSION 3007000 /* -** Indices of various locking bytes. WAL_NREADER is the number +** Index numbers for various locking bytes. WAL_NREADER is the number ** of available reader locks and should be at least 3. The default ** is SQLITE_SHM_NLOCK==8 and WAL_NREADER==5. +** +** Technically, the various VFSes are free to implement these locks however +** they see fit. However, compatibility is encouraged so that VFSes can +** interoperate. The standard implementation used on both unix and windows +** is for the index number to indicate a byte offset into the +** WalCkptInfo.aLock[] array in the wal-index header. In other words, all +** locks are on the shm file. The WALINDEX_LOCK_OFFSET constant (which +** should be 120) is the location in the shm file for the first locking +** byte. */ #define WAL_WRITE_LOCK 0 #define WAL_ALL_BUT_WRITE 1 @@ -53571,7 +65330,7 @@ typedef struct WalCkptInfo WalCkptInfo; ** ** The szPage value can be any power of 2 between 512 and 32768, inclusive. ** Or it can be 1 to represent a 65536-byte page. The latter case was -** added in 3.7.1 when support for 64K pages was added. +** added in 3.7.1 when support for 64K pages was added. */ struct WalIndexHdr { u32 iVersion; /* Wal-index version */ @@ -53613,9 +65372,9 @@ struct WalIndexHdr { ** There is one entry in aReadMark[] for each reader lock. If a reader ** holds read-lock K, then the value in aReadMark[K] is no greater than ** the mxFrame for that reader. The value READMARK_NOT_USED (0xffffffff) -** for any aReadMark[] means that entry is unused. aReadMark[0] is +** for any aReadMark[] means that entry is unused. aReadMark[0] is ** a special case; its value is never used and it exists as a place-holder -** to avoid having to offset aReadMark[] indexs by one. Readers holding +** to avoid having to offset aReadMark[] indexes by one. Readers holding ** WAL_READ_LOCK(0) always ignore the entire WAL and read all content ** directly from the database. ** @@ -53633,7 +65392,7 @@ struct WalIndexHdr { ** previous sentence is when nBackfill equals mxFrame (meaning that everything ** in the WAL has been backfilled into the database) then new readers ** will choose aReadMark[0] which has value 0 and hence such reader will -** get all their all content directly from the database file and ignore +** get all their all content directly from the database file and ignore ** the WAL. ** ** Writers normally append new frames to the end of the WAL. However, @@ -53655,6 +65414,70 @@ struct WalCkptInfo { }; #define READMARK_NOT_USED 0xffffffff +/* +** This is a schematic view of the complete 136-byte header of the +** wal-index file (also known as the -shm file): +** +** +-----------------------------+ +** 0: | iVersion | \ +** +-----------------------------+ | +** 4: | (unused padding) | | +** +-----------------------------+ | +** 8: | iChange | | +** +-------+-------+-------------+ | +** 12: | bInit | bBig | szPage | | +** +-------+-------+-------------+ | +** 16: | mxFrame | | First copy of the +** +-----------------------------+ | WalIndexHdr object +** 20: | nPage | | +** +-----------------------------+ | +** 24: | aFrameCksum | | +** | | | +** +-----------------------------+ | +** 32: | aSalt | | +** | | | +** +-----------------------------+ | +** 40: | aCksum | | +** | | / +** +-----------------------------+ +** 48: | iVersion | \ +** +-----------------------------+ | +** 52: | (unused padding) | | +** +-----------------------------+ | +** 56: | iChange | | +** +-------+-------+-------------+ | +** 60: | bInit | bBig | szPage | | +** +-------+-------+-------------+ | Second copy of the +** 64: | mxFrame | | WalIndexHdr +** +-----------------------------+ | +** 68: | nPage | | +** +-----------------------------+ | +** 72: | aFrameCksum | | +** | | | +** +-----------------------------+ | +** 80: | aSalt | | +** | | | +** +-----------------------------+ | +** 88: | aCksum | | +** | | / +** +-----------------------------+ +** 96: | nBackfill | +** +-----------------------------+ +** 100: | 5 read marks | +** | | +** | | +** | | +** | | +** +-------+-------+------+------+ +** 120: | Write | Ckpt | Rcvr | Rd0 | \ +** +-------+-------+------+------+ ) 8 lock bytes +** | Read1 | Read2 | Rd3 | Rd4 | / +** +-------+-------+------+------+ +** 128: | nBackfillAttempted | +** +-----------------------------+ +** 132: | (unused padding) | +** +-----------------------------+ +*/ /* A block of WALINDEX_LOCK_RESERVED bytes beginning at ** WALINDEX_LOCK_OFFSET is reserved for locks. Since some systems @@ -53668,7 +65491,6 @@ struct WalCkptInfo { #define WAL_FRAME_HDRSIZE 24 /* Size of write ahead log header, including checksum. */ -/* #define WAL_HDRSIZE 24 */ #define WAL_HDRSIZE 32 /* WAL magic value. Either this value, or the same value with the least @@ -53676,14 +65498,14 @@ struct WalCkptInfo { ** big-endian format in the first 4 bytes of a WAL file. ** ** If the LSB is set, then the checksums for each frame within the WAL -** file are calculated by treating all data as an array of 32-bit -** big-endian words. Otherwise, they are calculated by interpreting +** file are calculated by treating all data as an array of 32-bit +** big-endian words. Otherwise, they are calculated by interpreting ** all data as 32-bit little-endian words. */ #define WAL_MAGIC 0x377f0682 /* -** Return the offset of frame iFrame in the write-ahead log file, +** Return the offset of frame iFrame in the write-ahead log file, ** assuming a database page size of szPage bytes. The offset returned ** is to the start of the write-ahead log frame-header. */ @@ -53714,16 +65536,29 @@ struct Wal { u8 truncateOnCommit; /* True to truncate WAL file on commit */ u8 syncHeader; /* Fsync the WAL header if true */ u8 padToSectorBoundary; /* Pad transactions out to the next sector */ + u8 bShmUnreliable; /* SHM content is read-only and unreliable */ WalIndexHdr hdr; /* Wal-index header for current transaction */ u32 minFrame; /* Ignore wal frames before this one */ u32 iReCksum; /* On commit, recalculate checksums from here */ const char *zWalName; /* Name of WAL file */ u32 nCkpt; /* Checkpoint sequence counter in the wal-header */ +#ifdef SQLITE_USE_SEH + u32 lockMask; /* Mask of locks held */ + void *pFree; /* Pointer to sqlite3_free() if exception thrown */ + u32 *pWiValue; /* Value to write into apWiData[iWiPg] */ + int iWiPg; /* Write pWiValue into apWiData[iWiPg] */ + int iSysErrno; /* System error code following exception */ +#endif #ifdef SQLITE_DEBUG + int nSehTry; /* Number of nested SEH_TRY{} blocks */ u8 lockError; /* True if a locking error has occurred */ #endif #ifdef SQLITE_ENABLE_SNAPSHOT WalIndexHdr *pSnapshot; /* Start transaction here if not NULL */ + int bGetSnapshot; /* Transaction opened for sqlite3_get_snapshot() */ +#endif +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + sqlite3 *db; #endif }; @@ -53731,7 +65566,7 @@ struct Wal { ** Candidate values for Wal.exclusiveMode. */ #define WAL_NORMAL_MODE 0 -#define WAL_EXCLUSIVE_MODE 1 +#define WAL_EXCLUSIVE_MODE 1 #define WAL_HEAPMEMORY_MODE 2 /* @@ -53750,7 +65585,7 @@ typedef u16 ht_slot; /* ** This structure is used to implement an iterator that loops through ** all frames in the WAL in database page order. Where two or more frames -** correspond to the same database page, the iterator visits only the +** correspond to the same database page, the iterator visits only the ** frame most recently written to the WAL (in other words, the frame with ** the largest index). ** @@ -53763,7 +65598,7 @@ typedef u16 ht_slot; ** This functionality is used by the checkpoint code (see walCheckpoint()). */ struct WalIterator { - int iPrior; /* Last result returned from the iterator */ + u32 iPrior; /* Last result returned from the iterator */ int nSegment; /* Number of entries in aSegment[] */ struct WalSegment { int iNext; /* Next slot in aIndex[] not yet returned */ @@ -53786,7 +65621,7 @@ struct WalIterator { #define HASHTABLE_HASH_1 383 /* Should be prime */ #define HASHTABLE_NSLOT (HASHTABLE_NPAGE*2) /* Must be a power of 2 */ -/* +/* ** The block of page numbers associated with the first hash-table in a ** wal-index is smaller than usual. This is so that there is a complete ** hash-table on each aligned 32KB page of the wal-index. @@ -53798,23 +65633,143 @@ struct WalIterator { sizeof(ht_slot)*HASHTABLE_NSLOT + HASHTABLE_NPAGE*sizeof(u32) \ ) +/* +** Structured Exception Handling (SEH) is a Windows-specific technique +** for catching exceptions raised while accessing memory-mapped files. +** +** The -DSQLITE_USE_SEH compile-time option means to use SEH to catch and +** deal with system-level errors that arise during WAL -shm file processing. +** Without this compile-time option, any system-level faults that appear +** while accessing the memory-mapped -shm file will cause a process-wide +** signal to be deliver, which will more than likely cause the entire +** process to exit. +*/ +#ifdef SQLITE_USE_SEH +#include + +/* Beginning of a block of code in which an exception might occur */ +# define SEH_TRY __try { \ + assert( walAssertLockmask(pWal) && pWal->nSehTry==0 ); \ + VVA_ONLY(pWal->nSehTry++); + +/* The end of a block of code in which an exception might occur */ +# define SEH_EXCEPT(X) \ + VVA_ONLY(pWal->nSehTry--); \ + assert( pWal->nSehTry==0 ); \ + } __except( sehExceptionFilter(pWal, GetExceptionCode(), GetExceptionInformation() ) ){ X } + +/* Simulate a memory-mapping fault in the -shm file for testing purposes */ +# define SEH_INJECT_FAULT sehInjectFault(pWal) + +/* +** The second argument is the return value of GetExceptionCode() for the +** current exception. Return EXCEPTION_EXECUTE_HANDLER if the exception code +** indicates that the exception may have been caused by accessing the *-shm +** file mapping. Or EXCEPTION_CONTINUE_SEARCH otherwise. +*/ +static int sehExceptionFilter(Wal *pWal, int eCode, EXCEPTION_POINTERS *p){ + VVA_ONLY(pWal->nSehTry--); + if( eCode==EXCEPTION_IN_PAGE_ERROR ){ + if( p && p->ExceptionRecord && p->ExceptionRecord->NumberParameters>=3 ){ + /* From MSDN: For this type of exception, the first element of the + ** ExceptionInformation[] array is a read-write flag - 0 if the exception + ** was thrown while reading, 1 if while writing. The second element is + ** the virtual address being accessed. The "third array element specifies + ** the underlying NTSTATUS code that resulted in the exception". */ + pWal->iSysErrno = (int)p->ExceptionRecord->ExceptionInformation[2]; + } + return EXCEPTION_EXECUTE_HANDLER; + } + return EXCEPTION_CONTINUE_SEARCH; +} + +/* +** If one is configured, invoke the xTestCallback callback with 650 as +** the argument. If it returns true, throw the same exception that is +** thrown by the system if the *-shm file mapping is accessed after it +** has been invalidated. +*/ +static void sehInjectFault(Wal *pWal){ + int res; + assert( pWal->nSehTry>0 ); + + res = sqlite3FaultSim(650); + if( res!=0 ){ + ULONG_PTR aArg[3]; + aArg[0] = 0; + aArg[1] = 0; + aArg[2] = (ULONG_PTR)res; + RaiseException(EXCEPTION_IN_PAGE_ERROR, 0, 3, (const ULONG_PTR*)aArg); + } +} + +/* +** There are two ways to use this macro. To set a pointer to be freed +** if an exception is thrown: +** +** SEH_FREE_ON_ERROR(0, pPtr); +** +** and to cancel the same: +** +** SEH_FREE_ON_ERROR(pPtr, 0); +** +** In the first case, there must not already be a pointer registered to +** be freed. In the second case, pPtr must be the registered pointer. +*/ +#define SEH_FREE_ON_ERROR(X,Y) \ + assert( (X==0 || Y==0) && pWal->pFree==X ); pWal->pFree = Y + +/* +** There are two ways to use this macro. To arrange for pWal->apWiData[iPg] +** to be set to pValue if an exception is thrown: +** +** SEH_SET_ON_ERROR(iPg, pValue); +** +** and to cancel the same: +** +** SEH_SET_ON_ERROR(0, 0); +*/ +#define SEH_SET_ON_ERROR(X,Y) pWal->iWiPg = X; pWal->pWiValue = Y + +#else +# define SEH_TRY VVA_ONLY(pWal->nSehTry++); +# define SEH_EXCEPT(X) VVA_ONLY(pWal->nSehTry--); assert( pWal->nSehTry==0 ); +# define SEH_INJECT_FAULT assert( pWal->nSehTry>0 ); +# define SEH_FREE_ON_ERROR(X,Y) +# define SEH_SET_ON_ERROR(X,Y) +#endif /* ifdef SQLITE_USE_SEH */ + + /* ** Obtain a pointer to the iPage'th page of the wal-index. The wal-index ** is broken into pages of WALINDEX_PGSZ bytes. Wal-index pages are ** numbered from zero. ** -** If this call is successful, *ppPage is set to point to the wal-index -** page and SQLITE_OK is returned. If an error (an OOM or VFS error) occurs, -** then an SQLite error code is returned and *ppPage is set to 0. +** If the wal-index is currently smaller the iPage pages then the size +** of the wal-index might be increased, but only if it is safe to do +** so. It is safe to enlarge the wal-index if pWal->writeLock is true +** or pWal->exclusiveMode==WAL_HEAPMEMORY_MODE. +** +** Three possible result scenarios: +** +** (1) rc==SQLITE_OK and *ppPage==Requested-Wal-Index-Page +** (2) rc>=SQLITE_ERROR and *ppPage==NULL +** (3) rc==SQLITE_OK and *ppPage==NULL // only if iPage==0 +** +** Scenario (3) can only occur when pWal->writeLock is false and iPage==0 */ -static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ +static SQLITE_NOINLINE int walIndexPageRealloc( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ int rc = SQLITE_OK; /* Enlarge the pWal->apWiData[] array if required */ if( pWal->nWiData<=iPage ){ - int nByte = sizeof(u32*)*(iPage+1); + sqlite3_int64 nByte = sizeof(u32*)*(iPage+1); volatile u32 **apNew; - apNew = (volatile u32 **)sqlite3_realloc64((void *)pWal->apWiData, nByte); + apNew = (volatile u32 **)sqlite3Realloc((void *)pWal->apWiData, nByte); if( !apNew ){ *ppPage = 0; return SQLITE_NOMEM_BKPT; @@ -53826,16 +65781,23 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ } /* Request a pointer to the required page from the VFS */ - if( pWal->apWiData[iPage]==0 ){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ - pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); - if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; - }else{ - rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, - pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] - ); + assert( pWal->apWiData[iPage]==0 ); + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + pWal->apWiData[iPage] = (u32 volatile *)sqlite3MallocZero(WALINDEX_PGSZ); + if( !pWal->apWiData[iPage] ) rc = SQLITE_NOMEM_BKPT; + }else{ + rc = sqlite3OsShmMap(pWal->pDbFd, iPage, WALINDEX_PGSZ, + pWal->writeLock, (void volatile **)&pWal->apWiData[iPage] + ); + assert( pWal->apWiData[iPage]!=0 + || rc!=SQLITE_OK + || (pWal->writeLock==0 && iPage==0) ); + testcase( pWal->apWiData[iPage]==0 && rc==SQLITE_OK ); + if( rc==SQLITE_OK ){ + if( iPage>0 && sqlite3FaultSim(600) ) rc = SQLITE_NOMEM; + }else if( (rc&0xff)==SQLITE_READONLY ){ + pWal->readOnly |= WAL_SHM_RDONLY; if( rc==SQLITE_READONLY ){ - pWal->readOnly |= WAL_SHM_RDONLY; rc = SQLITE_OK; } } @@ -53845,12 +65807,24 @@ static int walIndexPage(Wal *pWal, int iPage, volatile u32 **ppPage){ assert( iPage==0 || *ppPage || rc!=SQLITE_OK ); return rc; } +static int walIndexPage( + Wal *pWal, /* The WAL context */ + int iPage, /* The page we seek */ + volatile u32 **ppPage /* Write the page pointer here */ +){ + SEH_INJECT_FAULT; + if( pWal->nWiData<=iPage || (*ppPage = pWal->apWiData[iPage])==0 ){ + return walIndexPageRealloc(pWal, iPage, ppPage); + } + return SQLITE_OK; +} /* ** Return a pointer to the WalCkptInfo structure in the wal-index. */ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ assert( pWal->nWiData>0 && pWal->apWiData[0] ); + SEH_INJECT_FAULT; return (volatile WalCkptInfo*)&(pWal->apWiData[0][sizeof(WalIndexHdr)/2]); } @@ -53859,6 +65833,7 @@ static volatile WalCkptInfo *walCkptInfo(Wal *pWal){ */ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ assert( pWal->nWiData>0 && pWal->apWiData[0] ); + SEH_INJECT_FAULT; return (volatile WalIndexHdr*)pWal->apWiData[0]; } @@ -53875,7 +65850,7 @@ static volatile WalIndexHdr *walIndexHdr(Wal *pWal){ ) /* -** Generate or extend an 8 byte checksum based on the data in +** Generate or extend an 8 byte checksum based on the data in ** array aByte[] and the initial values of aIn[0] and aIn[1] (or ** initial values of 0 and 0 if aIn==NULL). ** @@ -53903,36 +65878,75 @@ static void walChecksumBytes( assert( nByte>=8 ); assert( (nByte&0x00000007)==0 ); + assert( nByte<=65536 ); + assert( nByte%4==0 ); - if( nativeCksum ){ - do { - s1 += *aData++ + s2; - s2 += *aData++ + s1; - }while( aDataexclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmBarrier(pWal->pDbFd); } } +/* +** Add the SQLITE_NO_TSAN as part of the return-type of a function +** definition as a hint that the function contains constructs that +** might give false-positive TSAN warnings. +** +** See tag-20200519-1. +*/ +#if defined(__clang__) && !defined(SQLITE_NO_TSAN) +# define SQLITE_NO_TSAN __attribute__((no_sanitize_thread)) +#else +# define SQLITE_NO_TSAN +#endif + /* ** Write the header information in pWal->hdr into the wal-index. ** ** The checksum on pWal->hdr is updated before it is written. */ -static void walIndexWriteHdr(Wal *pWal){ +static SQLITE_NO_TSAN void walIndexWriteHdr(Wal *pWal){ volatile WalIndexHdr *aHdr = walIndexHdr(pWal); const int nCksum = offsetof(WalIndexHdr, aCksum); @@ -53940,6 +65954,7 @@ static void walIndexWriteHdr(Wal *pWal){ pWal->hdr.isInit = 1; pWal->hdr.iVersion = WALINDEX_MAX_VERSION; walChecksumBytes(1, (u8*)&pWal->hdr, nCksum, 0, pWal->hdr.aCksum); + /* Possible TSAN false-positive. See tag-20200519-1 */ memcpy((void*)&aHdr[1], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); walShmBarrier(pWal); memcpy((void*)&aHdr[0], (const void*)&pWal->hdr, sizeof(WalIndexHdr)); @@ -53947,11 +65962,11 @@ static void walIndexWriteHdr(Wal *pWal){ /* ** This function encodes a single frame header and writes it to a buffer -** supplied by the caller. A frame-header is made up of a series of +** supplied by the caller. A frame-header is made up of a series of ** 4-byte big-endian integers, as follows: ** ** 0: Page number. -** 4: For commit records, the size of the database image in pages +** 4: For commit records, the size of the database image in pages ** after the commit. For all other records, zero. ** 8: Salt-1 (copied from the wal-header) ** 12: Salt-2 (copied from the wal-header) @@ -54002,13 +66017,13 @@ static int walDecodeFrame( assert( WAL_FRAME_HDRSIZE==24 ); /* A frame is only valid if the salt values in the frame-header - ** match the salt values in the wal-header. + ** match the salt values in the wal-header. */ if( memcmp(&pWal->hdr.aSalt, &aFrame[8], 8)!=0 ){ return 0; } - /* A frame is only valid if the page number is creater than zero. + /* A frame is only valid if the page number is greater than zero. */ pgno = sqlite3Get4byte(&aFrame[0]); if( pgno==0 ){ @@ -54016,15 +66031,15 @@ static int walDecodeFrame( } /* A frame is only valid if a checksum of the WAL header, - ** all prior frams, the first 16 bytes of this frame-header, - ** and the frame-data matches the checksum in the last 8 + ** all prior frames, the first 16 bytes of this frame-header, + ** and the frame-data matches the checksum in the last 8 ** bytes of this frame-header. */ nativeCksum = (pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN); walChecksumBytes(nativeCksum, aFrame, 8, aCksum, aCksum); walChecksumBytes(nativeCksum, aData, pWal->szPage, aCksum, aCksum); - if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) - || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) + if( aCksum[0]!=sqlite3Get4byte(&aFrame[16]) + || aCksum[1]!=sqlite3Get4byte(&aFrame[20]) ){ /* Checksum failed. */ return 0; @@ -54059,7 +66074,7 @@ static const char *walLockName(int lockIdx){ } } #endif /*defined(SQLITE_TEST) || defined(SQLITE_DEBUG) */ - + /* ** Set or release locks on the WAL. Locks are either shared or exclusive. @@ -54075,13 +66090,19 @@ static int walLockShared(Wal *pWal, int lockIdx){ SQLITE_SHM_LOCK | SQLITE_SHM_SHARED); WALTRACE(("WAL%p: acquire SHARED-%s %s\n", pWal, walLockName(lockIdx), rc ? "failed" : "ok")); - VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_OK ) pWal->lockMask |= (1 << lockIdx); +#endif return rc; } static void walUnlockShared(Wal *pWal, int lockIdx){ if( pWal->exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, 1, SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED); +#ifdef SQLITE_USE_SEH + pWal->lockMask &= ~(1 << lockIdx); +#endif WALTRACE(("WAL%p: release SHARED-%s\n", pWal, walLockName(lockIdx))); } static int walLockExclusive(Wal *pWal, int lockIdx, int n){ @@ -54091,13 +66112,21 @@ static int walLockExclusive(Wal *pWal, int lockIdx, int n){ SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE); WALTRACE(("WAL%p: acquire EXCLUSIVE-%s cnt=%d %s\n", pWal, walLockName(lockIdx), n, rc ? "failed" : "ok")); - VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && rc!=SQLITE_BUSY); ) + VVA_ONLY( pWal->lockError = (u8)(rc!=SQLITE_OK && (rc&0xFF)!=SQLITE_BUSY); ) +#ifdef SQLITE_USE_SEH + if( rc==SQLITE_OK ){ + pWal->lockMask |= (((1<exclusiveMode ) return; (void)sqlite3OsShmLock(pWal->pDbFd, lockIdx, n, SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE); +#ifdef SQLITE_USE_SEH + pWal->lockMask &= ~(((1<aHash to point to the start of the hash table +** in the wal-index file. Set pLoc->iZero to one less than the frame ** number of the first frame indexed by this hash table. If a -** slot in the hash table is set to N, it refers to frame number -** (*piZero+N) in the log. +** slot in the hash table is set to N, it refers to frame number +** (pLoc->iZero+N) in the log. ** -** Finally, set *paPgno so that *paPgno[1] is the page number of the -** first frame indexed by the hash table, frame (*piZero+1). +** Finally, set pLoc->aPgno so that pLoc->aPgno[0] is the page number of the +** first frame indexed by the hash table, frame (pLoc->iZero). */ static int walHashGet( Wal *pWal, /* WAL handle */ int iHash, /* Find the iHash'th table */ - volatile ht_slot **paHash, /* OUT: Pointer to hash index */ - volatile u32 **paPgno, /* OUT: Pointer to page number array */ - u32 *piZero /* OUT: Frame associated with *paPgno[0] */ + WalHashLoc *pLoc /* OUT: Hash table location */ ){ int rc; /* Return code */ - volatile u32 *aPgno; - rc = walIndexPage(pWal, iHash, &aPgno); + rc = walIndexPage(pWal, iHash, &pLoc->aPgno); assert( rc==SQLITE_OK || iHash>0 ); - if( rc==SQLITE_OK ){ - u32 iZero; - volatile ht_slot *aHash; - - aHash = (volatile ht_slot *)&aPgno[HASHTABLE_NPAGE]; + if( pLoc->aPgno ){ + pLoc->aHash = (volatile ht_slot *)&pLoc->aPgno[HASHTABLE_NPAGE]; if( iHash==0 ){ - aPgno = &aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; - iZero = 0; + pLoc->aPgno = &pLoc->aPgno[WALINDEX_HDR_SIZE/sizeof(u32)]; + pLoc->iZero = 0; }else{ - iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; + pLoc->iZero = HASHTABLE_NPAGE_ONE + (iHash-1)*HASHTABLE_NPAGE; } - - *paPgno = &aPgno[-1]; - *paHash = aHash; - *piZero = iZero; + }else if( NEVER(rc==SQLITE_OK) ){ + rc = SQLITE_ERROR; } return rc; } @@ -54165,7 +66198,7 @@ static int walHashGet( /* ** Return the number of the wal-index page that contains the hash-table ** and page-number array that contain entries corresponding to WAL frame -** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages +** iFrame. The wal-index is broken up into 32KB pages. Wal-index pages ** are numbered starting from 0. */ static int walFramePage(u32 iFrame){ @@ -54176,6 +66209,7 @@ static int walFramePage(u32 iFrame){ && (iHash>=2 || iFrame<=HASHTABLE_NPAGE_ONE+HASHTABLE_NPAGE) && (iHash<=2 || iFrame>(HASHTABLE_NPAGE_ONE+2*HASHTABLE_NPAGE)) ); + assert( iHash>=0 ); return iHash; } @@ -54184,6 +66218,7 @@ static int walFramePage(u32 iFrame){ */ static u32 walFramePgno(Wal *pWal, u32 iFrame){ int iHash = walFramePage(iFrame); + SEH_INJECT_FAULT; if( iHash==0 ){ return pWal->apWiData[0][WALINDEX_HDR_SIZE/sizeof(u32) + iFrame - 1]; } @@ -54203,9 +66238,7 @@ static u32 walFramePgno(Wal *pWal, u32 iFrame){ ** actually needed. */ static void walCleanupHash(Wal *pWal){ - volatile ht_slot *aHash = 0; /* Pointer to hash table to clear */ - volatile u32 *aPgno = 0; /* Page number array for hash table */ - u32 iZero = 0; /* frame == (aHash[x]+iZero) */ + WalHashLoc sLoc; /* Hash table location */ int iLimit = 0; /* Zero values greater than this */ int nByte; /* Number of bytes to zero in aPgno[] */ int i; /* Used to iterate through aHash[] */ @@ -54217,30 +66250,32 @@ static void walCleanupHash(Wal *pWal){ if( pWal->hdr.mxFrame==0 ) return; - /* Obtain pointers to the hash-table and page-number array containing + /* Obtain pointers to the hash-table and page-number array containing ** the entry that corresponds to frame pWal->hdr.mxFrame. It is guaranteed - ** that the page said hash-table and array reside on is already mapped. + ** that the page said hash-table and array reside on is already mapped.(1) */ assert( pWal->nWiData>walFramePage(pWal->hdr.mxFrame) ); assert( pWal->apWiData[walFramePage(pWal->hdr.mxFrame)] ); - walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &aHash, &aPgno, &iZero); + i = walHashGet(pWal, walFramePage(pWal->hdr.mxFrame), &sLoc); + if( NEVER(i) ) return; /* Defense-in-depth, in case (1) above is wrong */ /* Zero all hash-table entries that correspond to frame numbers greater ** than pWal->hdr.mxFrame. */ - iLimit = pWal->hdr.mxFrame - iZero; + iLimit = pWal->hdr.mxFrame - sLoc.iZero; assert( iLimit>0 ); for(i=0; iiLimit ){ - aHash[i] = 0; + if( sLoc.aHash[i]>iLimit ){ + sLoc.aHash[i] = 0; } } - + /* Zero the entries in the aPgno array that correspond to frames with - ** frame numbers greater than pWal->hdr.mxFrame. + ** frame numbers greater than pWal->hdr.mxFrame. */ - nByte = (int)((char *)aHash - (char *)&aPgno[iLimit+1]); - memset((void *)&aPgno[iLimit+1], 0, nByte); + nByte = (int)((char *)sLoc.aHash - (char *)&sLoc.aPgno[iLimit]); + assert( nByte>=0 ); + memset((void *)&sLoc.aPgno[iLimit], 0, nByte); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the every entry in the mapping region is still reachable @@ -54249,11 +66284,11 @@ static void walCleanupHash(Wal *pWal){ if( iLimit ){ int j; /* Loop counter */ int iKey; /* Hash key */ - for(j=1; j<=iLimit; j++){ - for(iKey=walHash(aPgno[j]); aHash[iKey]; iKey=walNextHash(iKey)){ - if( aHash[iKey]==j ) break; + for(j=0; j=0 ); + memset((void*)sLoc.aPgno, 0, nByte); } /* If the entry in aPgno[] is already set, then the previous writer ** must have exited unexpectedly in the middle of a transaction (after - ** writing one or more dirty pages to the WAL to free up memory). - ** Remove the remnants of that writers uncommitted transaction from + ** writing one or more dirty pages to the WAL to free up memory). + ** Remove the remnants of that writers uncommitted transaction from ** the hash-table before writing any new entries. */ - if( aPgno[idx] ){ + if( sLoc.aPgno[idx-1] ){ walCleanupHash(pWal); - assert( !aPgno[idx] ); + assert( !sLoc.aPgno[idx-1] ); } /* Write the aPgno[] array entry and the hash-table slot. */ nCollide = idx; - for(iKey=walHash(iPage); aHash[iKey]; iKey=walNextHash(iKey)){ + for(iKey=walHash(iPage); sLoc.aHash[iKey]; iKey=walNextHash(iKey)){ if( (nCollide--)==0 ) return SQLITE_CORRUPT_BKPT; } - aPgno[idx] = iPage; - aHash[iKey] = (ht_slot)idx; + sLoc.aPgno[idx-1] = iPage; + AtomicStore(&sLoc.aHash[iKey], (ht_slot)idx); #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT /* Verify that the number of entries in the hash table exactly equals @@ -54317,7 +66351,7 @@ static int walIndexAppend(Wal *pWal, u32 iFrame, u32 iPage){ { int i; /* Loop counter */ int nEntry = 0; /* Number of entries in the hash table */ - for(i=0; iwriteLock ); iLock = WAL_ALL_BUT_WRITE + pWal->ckptLock; - nLock = SQLITE_SHM_NLOCK - iLock; - rc = walLockExclusive(pWal, iLock, nLock); + rc = walLockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); if( rc ){ return rc; } + WALTRACE(("WAL%p: recovery begin...\n", pWal)); memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); @@ -54387,15 +66421,16 @@ static int walIndexRecover(Wal *pWal){ if( nSize>WAL_HDRSIZE ){ u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u32 *aPrivate = 0; /* Heap copy of *-shm hash being populated */ u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ int szFrame; /* Number of bytes in buffer aFrame[] */ u8 *aData; /* Pointer to data part of aFrame buffer */ - int iFrame; /* Index of last frame read */ - i64 iOffset; /* Next offset to read from log file */ int szPage; /* Page size according to the log */ u32 magic; /* Magic value read from WAL header */ u32 version; /* Magic value read from WAL header */ int isValid; /* True if this frame is valid */ + u32 iPg; /* Current 32KB wal-index page */ + u32 iLastFrame; /* Last frame in wal, based on nSize alone */ /* Read in the WAL header. */ rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); @@ -54404,16 +66439,16 @@ static int walIndexRecover(Wal *pWal){ } /* If the database page size is not a power of two, or is greater than - ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid + ** SQLITE_MAX_PAGE_SIZE, conclude that the WAL file contains no valid ** data. Similarly, if the 'magic' value is invalid, ignore the whole ** WAL file. */ magic = sqlite3Get4byte(&aBuf[0]); szPage = sqlite3Get4byte(&aBuf[8]); - if( (magic&0xFFFFFFFE)!=WAL_MAGIC - || szPage&(szPage-1) - || szPage>SQLITE_MAX_PAGE_SIZE - || szPage<512 + if( (magic&0xFFFFFFFE)!=WAL_MAGIC + || szPage&(szPage-1) + || szPage>SQLITE_MAX_PAGE_SIZE + || szPage<512 ){ goto finished; } @@ -54423,7 +66458,7 @@ static int walIndexRecover(Wal *pWal){ memcpy(&pWal->hdr.aSalt, &aBuf[16], 8); /* Verify that the WAL header checksum is correct */ - walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, + walChecksumBytes(pWal->hdr.bigEndCksum==SQLITE_BIGENDIAN, aBuf, WAL_HDRSIZE-2*4, 0, pWal->hdr.aFrameCksum ); if( pWal->hdr.aFrameCksum[0]!=sqlite3Get4byte(&aBuf[24]) @@ -54442,40 +66477,90 @@ static int walIndexRecover(Wal *pWal){ /* Malloc a buffer to read frames into. */ szFrame = szPage + WAL_FRAME_HDRSIZE; - aFrame = (u8 *)sqlite3_malloc64(szFrame); + aFrame = (u8 *)sqlite3_malloc64(szFrame + WALINDEX_PGSZ); + SEH_FREE_ON_ERROR(0, aFrame); if( !aFrame ){ rc = SQLITE_NOMEM_BKPT; goto recovery_error; } aData = &aFrame[WAL_FRAME_HDRSIZE]; + aPrivate = (u32*)&aData[szPage]; /* Read all frames from the log file. */ - iFrame = 0; - for(iOffset=WAL_HDRSIZE; (iOffset+szFrame)<=nSize; iOffset+=szFrame){ - u32 pgno; /* Database page number for frame */ - u32 nTruncate; /* dbsize field from frame header */ + iLastFrame = (nSize - WAL_HDRSIZE) / szFrame; + for(iPg=0; iPg<=(u32)walFramePage(iLastFrame); iPg++){ + u32 *aShare; + u32 iFrame; /* Index of last frame read */ + u32 iLast = MIN(iLastFrame, HASHTABLE_NPAGE_ONE+iPg*HASHTABLE_NPAGE); + u32 iFirst = 1 + (iPg==0?0:HASHTABLE_NPAGE_ONE+(iPg-1)*HASHTABLE_NPAGE); + u32 nHdr, nHdr32; + rc = walIndexPage(pWal, iPg, (volatile u32**)&aShare); + assert( aShare!=0 || rc!=SQLITE_OK ); + if( aShare==0 ) break; + SEH_SET_ON_ERROR(iPg, aShare); + pWal->apWiData[iPg] = aPrivate; - /* Read and decode the next log frame. */ - iFrame++; - rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); - if( rc!=SQLITE_OK ) break; - isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); - if( !isValid ) break; - rc = walIndexAppend(pWal, iFrame, pgno); - if( rc!=SQLITE_OK ) break; + for(iFrame=iFirst; iFrame<=iLast; iFrame++){ + i64 iOffset = walFrameOffset(iFrame, szPage); + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ - /* If nTruncate is non-zero, this is a commit record. */ - if( nTruncate ){ - pWal->hdr.mxFrame = iFrame; - pWal->hdr.nPage = nTruncate; - pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); - testcase( szPage<=32768 ); - testcase( szPage>=65536 ); - aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; - aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + isValid = walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame); + if( !isValid ) break; + rc = walIndexAppend(pWal, iFrame, pgno); + if( NEVER(rc!=SQLITE_OK) ) break; + + /* If nTruncate is non-zero, this is a commit record. */ + if( nTruncate ){ + pWal->hdr.mxFrame = iFrame; + pWal->hdr.nPage = nTruncate; + pWal->hdr.szPage = (u16)((szPage&0xff00) | (szPage>>16)); + testcase( szPage<=32768 ); + testcase( szPage>=65536 ); + aFrameCksum[0] = pWal->hdr.aFrameCksum[0]; + aFrameCksum[1] = pWal->hdr.aFrameCksum[1]; + } } + pWal->apWiData[iPg] = aShare; + SEH_SET_ON_ERROR(0,0); + nHdr = (iPg==0 ? WALINDEX_HDR_SIZE : 0); + nHdr32 = nHdr / sizeof(u32); +#ifndef SQLITE_SAFER_WALINDEX_RECOVERY + /* Memcpy() should work fine here, on all reasonable implementations. + ** Technically, memcpy() might change the destination to some + ** intermediate value before setting to the final value, and that might + ** cause a concurrent reader to malfunction. Memcpy() is allowed to + ** do that, according to the spec, but no memcpy() implementation that + ** we know of actually does that, which is why we say that memcpy() + ** is safe for this. Memcpy() is certainly a lot faster. + */ + memcpy(&aShare[nHdr32], &aPrivate[nHdr32], WALINDEX_PGSZ-nHdr); +#else + /* In the event that some platform is found for which memcpy() + ** changes the destination to some intermediate value before + ** setting the final value, this alternative copy routine is + ** provided. + */ + { + int i; + for(i=nHdr32; ihdr.aFrameCksum[1] = aFrameCksum[1]; walIndexWriteHdr(pWal); - /* Reset the checkpoint-header. This is safe because this thread is - ** currently holding locks that exclude all other readers, writers and - ** checkpointers. + /* Reset the checkpoint-header. This is safe because this thread is + ** currently holding locks that exclude all other writers and + ** checkpointers. Then set the values of read-mark slots 1 through N. */ pInfo = walCkptInfo(pWal); pInfo->nBackfill = 0; pInfo->nBackfillAttempted = pWal->hdr.mxFrame; pInfo->aReadMark[0] = 0; - for(i=1; iaReadMark[i] = READMARK_NOT_USED; - if( pWal->hdr.mxFrame ) pInfo->aReadMark[1] = pWal->hdr.mxFrame; + for(i=1; ihdr.mxFrame ){ + pInfo->aReadMark[i] = pWal->hdr.mxFrame; + }else{ + pInfo->aReadMark[i] = READMARK_NOT_USED; + } + SEH_INJECT_FAULT; + walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); + }else if( rc!=SQLITE_BUSY ){ + goto recovery_error; + } + } /* If more than one frame was recovered from the log file, report an ** event via sqlite3_log(). This is to help with identifying performance @@ -54513,7 +66610,7 @@ finished: recovery_error: WALTRACE(("WAL%p: recovery %s\n", pWal, rc ? "failed" : "ok")); - walUnlockExclusive(pWal, iLock, nLock); + walUnlockExclusive(pWal, iLock, WAL_READ_LOCK(0)-iLock); return rc; } @@ -54521,19 +66618,20 @@ recovery_error: ** Close an open wal-index. */ static void walIndexClose(Wal *pWal, int isDelete){ - if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ){ + if( pWal->exclusiveMode==WAL_HEAPMEMORY_MODE || pWal->bShmUnreliable ){ int i; for(i=0; inWiData; i++){ sqlite3_free((void *)pWal->apWiData[i]); pWal->apWiData[i] = 0; } - }else{ + } + if( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE ){ sqlite3OsShmUnmap(pWal->pDbFd, isDelete); } } -/* -** Open a connection to the WAL file zWalName. The database file must +/* +** Open a connection to the WAL file zWalName. The database file must ** already be opened on connection pDbFd. The buffer that zWalName points ** to must remain valid for the lifetime of the returned Wal* handle. ** @@ -54543,7 +66641,7 @@ static void walIndexClose(Wal *pWal, int isDelete){ ** were to do this just after this client opened one of these files, the ** system would be badly broken. ** -** If the log file is successfully opened, SQLITE_OK is returned and +** If the log file is successfully opened, SQLITE_OK is returned and ** *ppWal is set to point to a new WAL handle. If an error occurs, ** an SQLite error code is returned and *ppWal is left unmodified. */ @@ -54562,14 +66660,43 @@ SQLITE_PRIVATE int sqlite3WalOpen( assert( zWalName && zWalName[0] ); assert( pDbFd ); + /* Verify the values of various constants. Any changes to the values + ** of these constants would result in an incompatible on-disk format + ** for the -shm file. Any change that causes one of these asserts to + ** fail is a backward compatibility problem, even if the change otherwise + ** works. + ** + ** This table also serves as a helpful cross-reference when trying to + ** interpret hex dumps of the -shm file. + */ + assert( 48 == sizeof(WalIndexHdr) ); + assert( 40 == sizeof(WalCkptInfo) ); + assert( 120 == WALINDEX_LOCK_OFFSET ); + assert( 136 == WALINDEX_HDR_SIZE ); + assert( 4096 == HASHTABLE_NPAGE ); + assert( 4062 == HASHTABLE_NPAGE_ONE ); + assert( 8192 == HASHTABLE_NSLOT ); + assert( 383 == HASHTABLE_HASH_1 ); + assert( 32768 == WALINDEX_PGSZ ); + assert( 8 == SQLITE_SHM_NLOCK ); + assert( 5 == WAL_NREADER ); + assert( 24 == WAL_FRAME_HDRSIZE ); + assert( 32 == WAL_HDRSIZE ); + assert( 120 == WALINDEX_LOCK_OFFSET + WAL_WRITE_LOCK ); + assert( 121 == WALINDEX_LOCK_OFFSET + WAL_CKPT_LOCK ); + assert( 122 == WALINDEX_LOCK_OFFSET + WAL_RECOVER_LOCK ); + assert( 123 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(0) ); + assert( 124 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(1) ); + assert( 125 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(2) ); + assert( 126 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(3) ); + assert( 127 == WALINDEX_LOCK_OFFSET + WAL_READ_LOCK(4) ); + /* In the amalgamation, the os_unix.c and os_win.c source files come before ** this source file. Verify that the #defines of the locking byte offsets ** in os_unix.c and os_win.c agree with the WALINDEX_LOCK_OFFSET value. ** For that matter, if the lock offset ever changes from its initial design ** value of 120, we need to know that so there is an assert() to check it. */ - assert( 120==WALINDEX_LOCK_OFFSET ); - assert( 136==WALINDEX_HDR_SIZE ); #ifdef WIN_SHM_BASE assert( WIN_SHM_BASE==WALINDEX_LOCK_OFFSET ); #endif @@ -54619,7 +66746,7 @@ SQLITE_PRIVATE int sqlite3WalOpen( } /* -** Change the size to which the WAL file is trucated on each reset. +** Change the size to which the WAL file is truncated on each reset. */ SQLITE_PRIVATE void sqlite3WalLimit(Wal *pWal, i64 iLimit){ if( pWal ) pWal->mxWalSize = iLimit; @@ -54707,7 +66834,7 @@ static void walMerge( ht_slot logpage; Pgno dbpage; - if( (iLeft=nRight || aContent[aLeft[iLeft]]HASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) + ); if( !p ){ return SQLITE_NOMEM_BKPT; } memset(p, 0, nByte); p->nSegment = nSegment; + aTmp = (ht_slot*)&(((u8*)p)[nByte]); + SEH_FREE_ON_ERROR(0, p); + for(i=walFramePage(nBackfill+1); rc==SQLITE_OK && iHASHTABLE_NPAGE?HASHTABLE_NPAGE:iLast) - ); - if( !aTmp ){ - rc = SQLITE_NOMEM_BKPT; - } - - for(i=0; rc==SQLITE_OK && iaSegment[p->nSegment])[iZero]; - iZero++; - + aIndex = &((ht_slot *)&p->aSegment[p->nSegment])[sLoc.iZero]; + sLoc.iZero++; + for(j=0; jaSegment[i].iZero = iZero; + walMergesort((u32 *)sLoc.aPgno, aTmp, aIndex, &nEntry); + p->aSegment[i].iZero = sLoc.iZero; p->aSegment[i].nEntry = nEntry; p->aSegment[i].aIndex = aIndex; - p->aSegment[i].aPgno = (u32 *)aPgno; + p->aSegment[i].aPgno = (u32 *)sLoc.aPgno; } } - sqlite3_free(aTmp); - if( rc!=SQLITE_OK ){ + SEH_FREE_ON_ERROR(p, 0); walIteratorFree(p); + p = 0; } *pp = p; return rc; } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + + +/* +** Attempt to enable blocking locks that block for nMs ms. Return 1 if +** blocking locks are successfully enabled, or 0 otherwise. +*/ +static int walEnableBlockingMs(Wal *pWal, int nMs){ + int rc = sqlite3OsFileControl( + pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&nMs + ); + return (rc==SQLITE_OK); +} + +/* +** Attempt to enable blocking locks. Blocking locks are enabled only if (a) +** they are supported by the VFS, and (b) the database handle is configured +** with a busy-timeout. Return 1 if blocking locks are successfully enabled, +** or 0 otherwise. +*/ +static int walEnableBlocking(Wal *pWal){ + int res = 0; + if( pWal->db ){ + int tmout = pWal->db->busyTimeout; + if( tmout ){ + res = walEnableBlockingMs(pWal, tmout); + } + } + return res; +} + +/* +** Disable blocking locks. +*/ +static void walDisableBlocking(Wal *pWal){ + int tmout = 0; + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_LOCK_TIMEOUT, (void*)&tmout); +} + +/* +** If parameter bLock is true, attempt to enable blocking locks, take +** the WRITER lock, and then disable blocking locks. If blocking locks +** cannot be enabled, no attempt to obtain the WRITER lock is made. Return +** an SQLite error code if an error occurs, or SQLITE_OK otherwise. It is not +** an error if blocking locks can not be enabled. +** +** If the bLock parameter is false and the WRITER lock is held, release it. +*/ +SQLITE_PRIVATE int sqlite3WalWriteLock(Wal *pWal, int bLock){ + int rc = SQLITE_OK; + assert( pWal->readLock<0 || bLock==0 ); + if( bLock ){ + assert( pWal->db ); + if( walEnableBlocking(pWal) ){ + rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + } + walDisableBlocking(pWal); + } + }else if( pWal->writeLock ){ + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); + pWal->writeLock = 0; + } + return rc; +} + +/* +** Set the database handle used to determine if blocking locks are required. +*/ +SQLITE_PRIVATE void sqlite3WalDb(Wal *pWal, sqlite3 *db){ + pWal->db = db; +} + +#else +# define walEnableBlocking(x) 0 +# define walDisableBlocking(x) +# define walEnableBlockingMs(pWal, ms) 0 +# define sqlite3WalDb(pWal, db) +#endif /* ifdef SQLITE_ENABLE_SETLK_TIMEOUT */ + + /* ** Attempt to obtain the exclusive WAL lock defined by parameters lockIdx and ** n. If the attempt fails and parameter xBusy is not NULL, then it is a @@ -54917,6 +67117,12 @@ static int walBusyLock( do { rc = walLockExclusive(pWal, lockIdx, n); }while( xBusy && rc==SQLITE_BUSY && xBusy(pBusyArg) ); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ){ + walDisableBlocking(pWal); + rc = SQLITE_BUSY; + } +#endif return rc; } @@ -54941,8 +67147,8 @@ static int walPagesize(Wal *pWal){ ** client to write to the database (which may be this one) does so by ** writing frames into the start of the log file. ** -** The value of parameter salt1 is used as the aSalt[1] value in the -** new wal-index header. It should be passed a pseudo-random value (i.e. +** The value of parameter salt1 is used as the aSalt[1] value in the +** new wal-index header. It should be passed a pseudo-random value (i.e. ** one obtained from sqlite3_randomness()). */ static void walRestartHdr(Wal *pWal, u32 salt1){ @@ -54954,7 +67160,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ sqlite3Put4byte((u8*)&aSalt[0], 1 + sqlite3Get4byte((u8*)&aSalt[0])); memcpy(&pWal->hdr.aSalt[1], &salt1, 4); walIndexWriteHdr(pWal); - pInfo->nBackfill = 0; + AtomicStore(&pInfo->nBackfill, 0); pInfo->nBackfillAttempted = 0; pInfo->aReadMark[1] = 0; for(i=2; iaReadMark[i] = READMARK_NOT_USED; @@ -54970,8 +67176,8 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ ** that a concurrent reader might be using. ** ** All I/O barrier operations (a.k.a fsyncs) occur in this routine when -** SQLite is in WAL-mode in synchronous=NORMAL. That means that if -** checkpoints are always run by a background thread or background +** SQLite is in WAL-mode in synchronous=NORMAL. That means that if +** checkpoints are always run by a background thread or background ** process, foreground threads will never block on a lengthy fsync call. ** ** Fsync is called on the WAL before writing content out of the WAL and @@ -54984,7 +67190,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ ** database file. ** ** This routine uses and updates the nBackfill field of the wal-index header. -** This is the only routine that will increase the value of nBackfill. +** This is the only routine that will increase the value of nBackfill. ** (A WAL reset or recovery will revert nBackfill to zero, but not increase ** its value.) ** @@ -54994,6 +67200,7 @@ static void walRestartHdr(Wal *pWal, u32 salt1){ */ static int walCheckpoint( Wal *pWal, /* Wal connection */ + sqlite3 *db, /* Check for interrupts on this handle */ int eMode, /* One of PASSIVE, FULL or RESTART */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ @@ -55016,13 +67223,6 @@ static int walCheckpoint( pInfo = walCkptInfo(pWal); if( pInfo->nBackfillhdr.mxFrame ){ - /* Allocate the iterator */ - rc = walIteratorInit(pWal, &pIter); - if( rc!=SQLITE_OK ){ - return rc; - } - assert( pIter ); - /* EVIDENCE-OF: R-62920-47450 The busy-handler callback is never invoked ** in the SQLITE_CHECKPOINT_PASSIVE mode. */ assert( eMode!=SQLITE_CHECKPOINT_PASSIVE || xBusy==0 ); @@ -55035,20 +67235,13 @@ static int walCheckpoint( mxSafeFrame = pWal->hdr.mxFrame; mxPage = pWal->hdr.nPage; for(i=1; iaReadMark[i]; + u32 y = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; if( mxSafeFrame>y ){ assert( y<=pWal->hdr.mxFrame ); rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_READ_LOCK(i), 1); if( rc==SQLITE_OK ){ - pInfo->aReadMark[i] = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + u32 iMark = (i==1 ? mxSafeFrame : READMARK_NOT_USED); + AtomicStore(pInfo->aReadMark+i, iMark); SEH_INJECT_FAULT; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); }else if( rc==SQLITE_BUSY ){ mxSafeFrame = y; @@ -55059,35 +67252,52 @@ static int walCheckpoint( } } - if( pInfo->nBackfillnBackfill; + /* Allocate the iterator */ + if( pInfo->nBackfillnBackfill, &pIter); + assert( rc==SQLITE_OK || pIter==0 ); + } - pInfo->nBackfillAttempted = mxSafeFrame; + if( pIter + && (rc = walBusyLock(pWal,xBusy,pBusyArg,WAL_READ_LOCK(0),1))==SQLITE_OK + ){ + u32 nBackfill = pInfo->nBackfill; + pInfo->nBackfillAttempted = mxSafeFrame; SEH_INJECT_FAULT; /* Sync the WAL to disk */ - if( sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags); - } + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); /* If the database may grow as a result of this checkpoint, hint ** about the eventual size of the db file to the VFS layer. */ if( rc==SQLITE_OK ){ i64 nReq = ((i64)mxPage * szPage); + i64 nSize; /* Current size of database file */ + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_START, 0); rc = sqlite3OsFileSize(pWal->pDbFd, &nSize); if( rc==SQLITE_OK && nSizepDbFd, SQLITE_FCNTL_SIZE_HINT, &nReq); + if( (nSize+65536+(i64)pWal->hdr.mxFrame*szPage)pDbFd, SQLITE_FCNTL_SIZE_HINT,&nReq); + } } - } + } /* Iterate through the contents of the WAL, copying data to the db file */ while( rc==SQLITE_OK && 0==walIteratorNext(pIter, &iDbpage, &iFrame) ){ i64 iOffset; assert( walFramePgno(pWal, iFrame)==iDbpage ); + SEH_INJECT_FAULT; + if( AtomicLoad(&db->u1.isInterrupted) ){ + rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; + break; + } if( iFrame<=nBackfill || iFrame>mxSafeFrame || iDbpage>mxPage ){ continue; } @@ -55100,6 +67310,7 @@ static int walCheckpoint( rc = sqlite3OsWrite(pWal->pDbFd, zBuf, szPage, iOffset); if( rc!=SQLITE_OK ) break; } + sqlite3OsFileControl(pWal->pDbFd, SQLITE_FCNTL_CKPT_DONE, 0); /* If work was actually accomplished... */ if( rc==SQLITE_OK ){ @@ -55107,12 +67318,12 @@ static int walCheckpoint( i64 szDb = pWal->hdr.nPage*(i64)szPage; testcase( IS_BIG_INT(szDb) ); rc = sqlite3OsTruncate(pWal->pDbFd, szDb); - if( rc==SQLITE_OK && sync_flags ){ - rc = sqlite3OsSync(pWal->pDbFd, sync_flags); + if( rc==SQLITE_OK ){ + rc = sqlite3OsSync(pWal->pDbFd, CKPT_SYNC_FLAGS(sync_flags)); } } if( rc==SQLITE_OK ){ - pInfo->nBackfill = mxSafeFrame; + AtomicStore(&pInfo->nBackfill, mxSafeFrame); SEH_INJECT_FAULT; } } @@ -55128,12 +67339,13 @@ static int walCheckpoint( } /* If this is an SQLITE_CHECKPOINT_RESTART or TRUNCATE operation, and the - ** entire wal file has been copied into the database file, then block - ** until all readers have finished using the wal file. This ensures that + ** entire wal file has been copied into the database file, then block + ** until all readers have finished using the wal file. This ensures that ** the next process to write to the database restarts the wal file. */ if( rc==SQLITE_OK && eMode!=SQLITE_CHECKPOINT_PASSIVE ){ assert( pWal->writeLock ); + SEH_INJECT_FAULT; if( pInfo->nBackfillhdr.mxFrame ){ rc = SQLITE_BUSY; }else if( eMode>=SQLITE_CHECKPOINT_RESTART ){ @@ -55153,7 +67365,7 @@ static int walCheckpoint( ** writer clients should see that the entire log file has been ** checkpointed and behave accordingly. This seems unsafe though, ** as it would leave the system in a state where the contents of - ** the wal-index header do not match the contents of the + ** the wal-index header do not match the contents of the ** file-system. To avoid this, update the wal-index header to ** indicate that the log file contains zero valid frames. */ walRestartHdr(pWal, salt1); @@ -55165,6 +67377,7 @@ static int walCheckpoint( } walcheckpoint_out: + SEH_FREE_ON_ERROR(pIter, 0); walIteratorFree(pIter); return rc; } @@ -55187,11 +67400,99 @@ static void walLimitSize(Wal *pWal, i64 nMax){ } } +#ifdef SQLITE_USE_SEH +/* +** This is the "standard" exception handler used in a few places to handle +** an exception thrown by reading from the *-shm mapping after it has become +** invalid in SQLITE_USE_SEH builds. It is used as follows: +** +** SEH_TRY { ... } +** SEH_EXCEPT( rc = walHandleException(pWal); ) +** +** This function does three things: +** +** 1) Determines the locks that should be held, based on the contents of +** the Wal.readLock, Wal.writeLock and Wal.ckptLock variables. All other +** held locks are assumed to be transient locks that would have been +** released had the exception not been thrown and are dropped. +** +** 2) Frees the pointer at Wal.pFree, if any, using sqlite3_free(). +** +** 3) Set pWal->apWiData[pWal->iWiPg] to pWal->pWiValue if not NULL +** +** 4) Returns SQLITE_IOERR. +*/ +static int walHandleException(Wal *pWal){ + if( pWal->exclusiveMode==0 ){ + static const int S = 1; + static const int E = (1<lockMask & ~( + (pWal->readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) + | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) + | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) + ); + for(ii=0; iipFree); + pWal->pFree = 0; + if( pWal->pWiValue ){ + pWal->apWiData[pWal->iWiPg] = pWal->pWiValue; + pWal->pWiValue = 0; + } + return SQLITE_IOERR_IN_PAGE; +} + +/* +** Assert that the Wal.lockMask mask, which indicates the locks held +** by the connection, is consistent with the Wal.readLock, Wal.writeLock +** and Wal.ckptLock variables. To be used as: +** +** assert( walAssertLockmask(pWal) ); +*/ +static int walAssertLockmask(Wal *pWal){ + if( pWal->exclusiveMode==0 ){ + static const int S = 1; + static const int E = (1<readLock<0 ? 0 : (S << WAL_READ_LOCK(pWal->readLock))) + | (pWal->writeLock ? (E << WAL_WRITE_LOCK) : 0) + | (pWal->ckptLock ? (E << WAL_CKPT_LOCK) : 0) +#ifdef SQLITE_ENABLE_SNAPSHOT + | (pWal->pSnapshot ? (pWal->lockMask & (1 << WAL_CKPT_LOCK)) : 0) +#endif + ); + assert( mExpect==pWal->lockMask ); + } + return 1; +} + +/* +** Return and zero the "system error" field set when an +** EXCEPTION_IN_PAGE_ERROR exception is caught. +*/ +SQLITE_PRIVATE int sqlite3WalSystemErrno(Wal *pWal){ + int iRet = 0; + if( pWal ){ + iRet = pWal->iSysErrno; + pWal->iSysErrno = 0; + } + return iRet; +} + +#else +# define walAssertLockmask(x) 1 +#endif /* ifdef SQLITE_USE_SEH */ + /* ** Close a connection to a log file. */ SQLITE_PRIVATE int sqlite3WalClose( Wal *pWal, /* Wal to close */ + sqlite3 *db, /* For interrupt flag */ int sync_flags, /* Flags to pass to OsSync() (or 0) */ int nBuf, u8 *zBuf /* Buffer of at least nBuf bytes */ @@ -55200,6 +67501,8 @@ SQLITE_PRIVATE int sqlite3WalClose( if( pWal ){ int isDelete = 0; /* True to unlink wal and wal-index files */ + assert( walAssertLockmask(pWal) ); + /* If an EXCLUSIVE lock can be obtained on the database file (using the ** ordinary, rollback-mode locking methods, this guarantees that the ** connection associated with this log file is the only connection to @@ -55208,13 +67511,14 @@ SQLITE_PRIVATE int sqlite3WalClose( ** ** The EXCLUSIVE lock is not released before returning. */ - rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE); - if( rc==SQLITE_OK ){ + if( zBuf!=0 + && SQLITE_OK==(rc = sqlite3OsLock(pWal->pDbFd, SQLITE_LOCK_EXCLUSIVE)) + ){ if( pWal->exclusiveMode==WAL_NORMAL_MODE ){ pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = sqlite3WalCheckpoint( - pWal, SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 + rc = sqlite3WalCheckpoint(pWal, db, + SQLITE_CHECKPOINT_PASSIVE, 0, 0, sync_flags, nBuf, zBuf, 0, 0 ); if( rc==SQLITE_OK ){ int bPersist = -1; @@ -55223,7 +67527,7 @@ SQLITE_PRIVATE int sqlite3WalClose( ); if( bPersist!=1 ){ /* Try to delete the WAL file if the checkpoint completed and - ** fsyned (rc==SQLITE_OK) and if we are not in persistent-wal + ** fsynced (rc==SQLITE_OK) and if we are not in persistent-wal ** mode (!bPersist) */ isDelete = 1; }else if( pWal->mxWalSize>=0 ){ @@ -55269,7 +67573,7 @@ SQLITE_PRIVATE int sqlite3WalClose( ** If the checksum cannot be verified return non-zero. If the header ** is read successfully and the checksum verified, return zero. */ -static int walIndexTryHdr(Wal *pWal, int *pChanged){ +static SQLITE_NO_TSAN int walIndexTryHdr(Wal *pWal, int *pChanged){ u32 aCksum[2]; /* Checksum on the header content */ WalIndexHdr h1, h2; /* Two copies of the header content */ WalIndexHdr volatile *aHdr; /* Header in shared memory */ @@ -55282,19 +67586,25 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** meaning it is possible that an inconsistent snapshot is read ** from the file. If this happens, return non-zero. ** + ** tag-20200519-1: ** There are two copies of the header at the beginning of the wal-index. ** When reading, read [0] first then [1]. Writes are in the reverse order. ** Memory barriers are used to prevent the compiler or the hardware from - ** reordering the reads and writes. + ** reordering the reads and writes. TSAN and similar tools can sometimes + ** give false-positive warnings about these accesses because the tools do not + ** account for the double-read and the memory barrier. The use of mutexes + ** here would be problematic as the memory being accessed is potentially + ** shared among multiple processes and not all mutex implementations work + ** reliably in that environment. */ aHdr = walIndexHdr(pWal); - memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); + memcpy(&h1, (void *)&aHdr[0], sizeof(h1)); /* Possible TSAN false-positive */ walShmBarrier(pWal); memcpy(&h2, (void *)&aHdr[1], sizeof(h2)); if( memcmp(&h1, &h2, sizeof(h1))!=0 ){ return 1; /* Dirty read */ - } + } if( h1.isInit==0 ){ return 1; /* Malformed header - probably all zeros */ } @@ -55315,6 +67625,12 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ return 0; } +/* +** This is the value that walTryBeginRead returns when it needs to +** be retried. +*/ +#define WAL_RETRY (-1) + /* ** Read the wal-index header from the wal-index and into pWal->hdr. ** If the wal-header appears to be corrupt, try to reconstruct the @@ -55324,7 +67640,7 @@ static int walIndexTryHdr(Wal *pWal, int *pChanged){ ** changed by this operation. If pWal->hdr is unchanged, set *pChanged ** to 0. ** -** If the wal-index header is successfully read, return SQLITE_OK. +** If the wal-index header is successfully read, return SQLITE_OK. ** Otherwise an SQLite error code. */ static int walIndexReadHdr(Wal *pWal, int *pChanged){ @@ -55332,19 +67648,39 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ int badHdr; /* True if a header read failed */ volatile u32 *page0; /* Chunk of wal-index containing header */ - /* Ensure that page 0 of the wal-index (the page that contains the + /* Ensure that page 0 of the wal-index (the page that contains the ** wal-index header) is mapped. Return early if an error occurs here. */ assert( pChanged ); rc = walIndexPage(pWal, 0, &page0); if( rc!=SQLITE_OK ){ - return rc; - }; - assert( page0 || pWal->writeLock==0 ); + assert( rc!=SQLITE_READONLY ); /* READONLY changed to OK in walIndexPage */ + if( rc==SQLITE_READONLY_CANTINIT ){ + /* The SQLITE_READONLY_CANTINIT return means that the shared-memory + ** was openable but is not writable, and this thread is unable to + ** confirm that another write-capable connection has the shared-memory + ** open, and hence the content of the shared-memory is unreliable, + ** since the shared-memory might be inconsistent with the WAL file + ** and there is no writer on hand to fix it. */ + assert( page0==0 ); + assert( pWal->writeLock==0 ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + pWal->bShmUnreliable = 1; + pWal->exclusiveMode = WAL_HEAPMEMORY_MODE; + *pChanged = 1; + }else{ + return rc; /* Any other non-OK return is just an error */ + } + }else{ + /* page0 can be NULL if the SHM is zero bytes in size and pWal->writeLock + ** is zero, which prevents the SHM from growing */ + testcase( page0!=0 ); + } + assert( page0!=0 || pWal->writeLock==0 ); /* If the first page of the wal-index has been mapped, try to read the ** wal-index header immediately, without holding any lock. This usually - ** works, but may fail if the wal-index header is corrupt or currently + ** works, but may fail if the wal-index header is corrupt or currently ** being modified by another thread or process. */ badHdr = (page0 ? walIndexTryHdr(pWal, pChanged) : 1); @@ -55352,28 +67688,35 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ /* If the first attempt failed, it might have been due to a race ** with a writer. So get a WRITE lock and try again. */ - assert( badHdr==0 || pWal->writeLock==0 ); if( badHdr ){ - if( pWal->readOnly & WAL_SHM_RDONLY ){ + if( pWal->bShmUnreliable==0 && (pWal->readOnly & WAL_SHM_RDONLY) ){ if( SQLITE_OK==(rc = walLockShared(pWal, WAL_WRITE_LOCK)) ){ walUnlockShared(pWal, WAL_WRITE_LOCK); rc = SQLITE_READONLY_RECOVERY; } - }else if( SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) ){ - pWal->writeLock = 1; - if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ - badHdr = walIndexTryHdr(pWal, pChanged); - if( badHdr ){ - /* If the wal-index header is still malformed even while holding - ** a WRITE lock, it can only mean that the header is corrupted and - ** needs to be reconstructed. So run recovery to do exactly that. - */ - rc = walIndexRecover(pWal); - *pChanged = 1; + }else{ + int bWriteLock = pWal->writeLock; + if( bWriteLock + || SQLITE_OK==(rc = walLockExclusive(pWal, WAL_WRITE_LOCK, 1)) + ){ + pWal->writeLock = 1; + if( SQLITE_OK==(rc = walIndexPage(pWal, 0, &page0)) ){ + badHdr = walIndexTryHdr(pWal, pChanged); + if( badHdr ){ + /* If the wal-index header is still malformed even while holding + ** a WRITE lock, it can only mean that the header is corrupted and + ** needs to be reconstructed. So run recovery to do exactly that. + ** Disable blocking locks first. */ + walDisableBlocking(pWal); + rc = walIndexRecover(pWal); + *pChanged = 1; + } + } + if( bWriteLock==0 ){ + pWal->writeLock = 0; + walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } } - pWal->writeLock = 0; - walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); } } @@ -55384,15 +67727,226 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ if( badHdr==0 && pWal->hdr.iVersion!=WALINDEX_MAX_VERSION ){ rc = SQLITE_CANTOPEN_BKPT; } + if( pWal->bShmUnreliable ){ + if( rc!=SQLITE_OK ){ + walIndexClose(pWal, 0); + pWal->bShmUnreliable = 0; + assert( pWal->nWiData>0 && pWal->apWiData[0]==0 ); + /* walIndexRecover() might have returned SHORT_READ if a concurrent + ** writer truncated the WAL out from under it. If that happens, it + ** indicates that a writer has fixed the SHM file for us, so retry */ + if( rc==SQLITE_IOERR_SHORT_READ ) rc = WAL_RETRY; + } + pWal->exclusiveMode = WAL_NORMAL_MODE; + } return rc; } /* -** This is the value that walTryBeginRead returns when it needs to -** be retried. +** Open a transaction in a connection where the shared-memory is read-only +** and where we cannot verify that there is a separate write-capable connection +** on hand to keep the shared-memory up-to-date with the WAL file. +** +** This can happen, for example, when the shared-memory is implemented by +** memory-mapping a *-shm file, where a prior writer has shut down and +** left the *-shm file on disk, and now the present connection is trying +** to use that database but lacks write permission on the *-shm file. +** Other scenarios are also possible, depending on the VFS implementation. +** +** Precondition: +** +** The *-wal file has been read and an appropriate wal-index has been +** constructed in pWal->apWiData[] using heap memory instead of shared +** memory. +** +** If this function returns SQLITE_OK, then the read transaction has +** been successfully opened. In this case output variable (*pChanged) +** is set to true before returning if the caller should discard the +** contents of the page cache before proceeding. Or, if it returns +** WAL_RETRY, then the heap memory wal-index has been discarded and +** the caller should retry opening the read transaction from the +** beginning (including attempting to map the *-shm file). +** +** If an error occurs, an SQLite error code is returned. */ -#define WAL_RETRY (-1) +static int walBeginShmUnreliable(Wal *pWal, int *pChanged){ + i64 szWal; /* Size of wal file on disk in bytes */ + i64 iOffset; /* Current offset when reading wal file */ + u8 aBuf[WAL_HDRSIZE]; /* Buffer to load WAL header into */ + u8 *aFrame = 0; /* Malloc'd buffer to load entire frame */ + int szFrame; /* Number of bytes in buffer aFrame[] */ + u8 *aData; /* Pointer to data part of aFrame buffer */ + volatile void *pDummy; /* Dummy argument for xShmMap */ + int rc; /* Return code */ + u32 aSaveCksum[2]; /* Saved copy of pWal->hdr.aFrameCksum */ + + assert( pWal->bShmUnreliable ); + assert( pWal->readOnly & WAL_SHM_RDONLY ); + assert( pWal->nWiData>0 && pWal->apWiData[0] ); + + /* Take WAL_READ_LOCK(0). This has the effect of preventing any + ** writers from running a checkpoint, but does not stop them + ** from running recovery. */ + rc = walLockShared(pWal, WAL_READ_LOCK(0)); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_BUSY ) rc = WAL_RETRY; + goto begin_unreliable_shm_out; + } + pWal->readLock = 0; + + /* Check to see if a separate writer has attached to the shared-memory area, + ** thus making the shared-memory "reliable" again. Do this by invoking + ** the xShmMap() routine of the VFS and looking to see if the return + ** is SQLITE_READONLY instead of SQLITE_READONLY_CANTINIT. + ** + ** If the shared-memory is now "reliable" return WAL_RETRY, which will + ** cause the heap-memory WAL-index to be discarded and the actual + ** shared memory to be used in its place. + ** + ** This step is important because, even though this connection is holding + ** the WAL_READ_LOCK(0) which prevents a checkpoint, a writer might + ** have already checkpointed the WAL file and, while the current + ** is active, wrap the WAL and start overwriting frames that this + ** process wants to use. + ** + ** Once sqlite3OsShmMap() has been called for an sqlite3_file and has + ** returned any SQLITE_READONLY value, it must return only SQLITE_READONLY + ** or SQLITE_READONLY_CANTINIT or some error for all subsequent invocations, + ** even if some external agent does a "chmod" to make the shared-memory + ** writable by us, until sqlite3OsShmUnmap() has been called. + ** This is a requirement on the VFS implementation. + */ + rc = sqlite3OsShmMap(pWal->pDbFd, 0, WALINDEX_PGSZ, 0, &pDummy); + assert( rc!=SQLITE_OK ); /* SQLITE_OK not possible for read-only connection */ + if( rc!=SQLITE_READONLY_CANTINIT ){ + rc = (rc==SQLITE_READONLY ? WAL_RETRY : rc); + goto begin_unreliable_shm_out; + } + + /* We reach this point only if the real shared-memory is still unreliable. + ** Assume the in-memory WAL-index substitute is correct and load it + ** into pWal->hdr. + */ + memcpy(&pWal->hdr, (void*)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + /* Make sure some writer hasn't come in and changed the WAL file out + ** from under us, then disconnected, while we were not looking. + */ + rc = sqlite3OsFileSize(pWal->pWalFd, &szWal); + if( rc!=SQLITE_OK ){ + goto begin_unreliable_shm_out; + } + if( szWalhdr.mxFrame==0 ? SQLITE_OK : WAL_RETRY); + goto begin_unreliable_shm_out; + } + + /* Check the salt keys at the start of the wal file still match. */ + rc = sqlite3OsRead(pWal->pWalFd, aBuf, WAL_HDRSIZE, 0); + if( rc!=SQLITE_OK ){ + goto begin_unreliable_shm_out; + } + if( memcmp(&pWal->hdr.aSalt, &aBuf[16], 8) ){ + /* Some writer has wrapped the WAL file while we were not looking. + ** Return WAL_RETRY which will cause the in-memory WAL-index to be + ** rebuilt. */ + rc = WAL_RETRY; + goto begin_unreliable_shm_out; + } + + /* Allocate a buffer to read frames into */ + assert( (pWal->szPage & (pWal->szPage-1))==0 ); + assert( pWal->szPage>=512 && pWal->szPage<=65536 ); + szFrame = pWal->szPage + WAL_FRAME_HDRSIZE; + aFrame = (u8 *)sqlite3_malloc64(szFrame); + if( aFrame==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto begin_unreliable_shm_out; + } + aData = &aFrame[WAL_FRAME_HDRSIZE]; + + /* Check to see if a complete transaction has been appended to the + ** wal file since the heap-memory wal-index was created. If so, the + ** heap-memory wal-index is discarded and WAL_RETRY returned to + ** the caller. */ + aSaveCksum[0] = pWal->hdr.aFrameCksum[0]; + aSaveCksum[1] = pWal->hdr.aFrameCksum[1]; + for(iOffset=walFrameOffset(pWal->hdr.mxFrame+1, pWal->szPage); + iOffset+szFrame<=szWal; + iOffset+=szFrame + ){ + u32 pgno; /* Database page number for frame */ + u32 nTruncate; /* dbsize field from frame header */ + + /* Read and decode the next log frame. */ + rc = sqlite3OsRead(pWal->pWalFd, aFrame, szFrame, iOffset); + if( rc!=SQLITE_OK ) break; + if( !walDecodeFrame(pWal, &pgno, &nTruncate, aData, aFrame) ) break; + + /* If nTruncate is non-zero, then a complete transaction has been + ** appended to this wal file. Set rc to WAL_RETRY and break out of + ** the loop. */ + if( nTruncate ){ + rc = WAL_RETRY; + break; + } + } + pWal->hdr.aFrameCksum[0] = aSaveCksum[0]; + pWal->hdr.aFrameCksum[1] = aSaveCksum[1]; + + begin_unreliable_shm_out: + sqlite3_free(aFrame); + if( rc!=SQLITE_OK ){ + int i; + for(i=0; inWiData; i++){ + sqlite3_free((void*)pWal->apWiData[i]); + pWal->apWiData[i] = 0; + } + pWal->bShmUnreliable = 0; + sqlite3WalEndReadTransaction(pWal); + *pChanged = 1; + } + return rc; +} + +/* +** The final argument passed to walTryBeginRead() is of type (int*). The +** caller should invoke walTryBeginRead as follows: +** +** int cnt = 0; +** do { +** rc = walTryBeginRead(..., &cnt); +** }while( rc==WAL_RETRY ); +** +** The final value of "cnt" is of no use to the caller. It is used by +** the implementation of walTryBeginRead() as follows: +** +** + Each time walTryBeginRead() is called, it is incremented. Once +** it reaches WAL_RETRY_PROTOCOL_LIMIT - indicating that walTryBeginRead() +** has many times been invoked and failed with WAL_RETRY - walTryBeginRead() +** returns SQLITE_PROTOCOL. +** +** + If SQLITE_ENABLE_SETLK_TIMEOUT is defined and walTryBeginRead() failed +** because a blocking lock timed out (SQLITE_BUSY_TIMEOUT from the OS +** layer), the WAL_RETRY_BLOCKED_MASK bit is set in "cnt". In this case +** the next invocation of walTryBeginRead() may omit an expected call to +** sqlite3OsSleep(). There has already been a delay when the previous call +** waited on a lock. +*/ +#define WAL_RETRY_PROTOCOL_LIMIT 100 +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT +# define WAL_RETRY_BLOCKED_MASK 0x10000000 +#else +# define WAL_RETRY_BLOCKED_MASK 0 +#endif /* ** Attempt to start a read transaction. This might fail due to a race or @@ -55405,10 +67959,10 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** ** The useWal parameter is true to force the use of the WAL and disable ** the case where the WAL is bypassed because it has been completely -** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() -** to make a copy of the wal-index header into pWal->hdr. If the -** wal-index header has changed, *pChanged is set to 1 (as an indication -** to the caller that the local paget cache is obsolete and needs to be +** checkpointed. If useWal==0 then this routine calls walIndexReadHdr() +** to make a copy of the wal-index header into pWal->hdr. If the +** wal-index header has changed, *pChanged is set to 1 (as an indication +** to the caller that the local page cache is obsolete and needs to be ** flushed.) When useWal==1, the wal-index header is assumed to already ** be loaded and the pChanged parameter is unused. ** @@ -55423,7 +67977,7 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** bad luck when there is lots of contention for the wal-index, but that ** possibility is so small that it can be safely neglected, we believe. ** -** On success, this routine obtains a read lock on +** On success, this routine obtains a read lock on ** WAL_READ_LOCK(pWal->readLock). The pWal->readLock integer is ** in the range 0 <= pWal->readLock < WAL_NREADER. If pWal->readLock==(-1) ** that means the Wal does not hold any read lock. The reader must not @@ -55444,45 +67998,81 @@ static int walIndexReadHdr(Wal *pWal, int *pChanged){ ** so it takes care to hold an exclusive lock on the corresponding ** WAL_READ_LOCK() while changing values. */ -static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ +static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int *pCnt){ volatile WalCkptInfo *pInfo; /* Checkpoint information in wal-index */ u32 mxReadMark; /* Largest aReadMark[] value */ int mxI; /* Index of largest aReadMark[] value */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ u32 mxFrame; /* Wal frame to lock to */ +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + int nBlockTmout = 0; +#endif assert( pWal->readLock<0 ); /* Not currently locked */ + /* useWal may only be set for read/write connections */ + assert( (pWal->readOnly & WAL_SHM_RDONLY)==0 || useWal==0 ); + /* Take steps to avoid spinning forever if there is a protocol error. ** ** Circumstances that cause a RETRY should only last for the briefest ** instances of time. No I/O or other system calls are done while the - ** locks are held, so the locks should not be held for very long. But + ** locks are held, so the locks should not be held for very long. But ** if we are unlucky, another process that is holding a lock might get - ** paged out or take a page-fault that is time-consuming to resolve, + ** paged out or take a page-fault that is time-consuming to resolve, ** during the few nanoseconds that it is holding the lock. In that case, ** it might take longer than normal for the lock to free. ** ** After 5 RETRYs, we begin calling sqlite3OsSleep(). The first few ** calls to sqlite3OsSleep() have a delay of 1 microsecond. Really this ** is more of a scheduler yield than an actual delay. But on the 10th - ** an subsequent retries, the delays start becoming longer and longer, + ** an subsequent retries, the delays start becoming longer and longer, ** so that on the 100th (and last) RETRY we delay for 323 milliseconds. ** The total delay time before giving up is less than 10 seconds. */ - if( cnt>5 ){ + (*pCnt)++; + if( *pCnt>5 ){ int nDelay = 1; /* Pause time in microseconds */ - if( cnt>100 ){ + int cnt = (*pCnt & ~WAL_RETRY_BLOCKED_MASK); + if( cnt>WAL_RETRY_PROTOCOL_LIMIT ){ VVA_ONLY( pWal->lockError = 1; ) return SQLITE_PROTOCOL; } - if( cnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; + if( *pCnt>=10 ) nDelay = (cnt-9)*(cnt-9)*39; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* In SQLITE_ENABLE_SETLK_TIMEOUT builds, configure the file-descriptor + ** to block for locks for approximately nDelay us. This affects three + ** locks: (a) the shared lock taken on the DMS slot in os_unix.c (if + ** using os_unix.c), (b) the WRITER lock taken in walIndexReadHdr() if the + ** first attempted read fails, and (c) the shared lock taken on the + ** read-mark. + ** + ** If the previous call failed due to an SQLITE_BUSY_TIMEOUT error, + ** then sleep for the minimum of 1us. The previous call already provided + ** an extra delay while it was blocking on the lock. + */ + nBlockTmout = (nDelay+998) / 1000; + if( !useWal && walEnableBlockingMs(pWal, nBlockTmout) ){ + if( *pCnt & WAL_RETRY_BLOCKED_MASK ) nDelay = 1; + } +#endif sqlite3OsSleep(pWal->pVfs, nDelay); + *pCnt &= ~WAL_RETRY_BLOCKED_MASK; } if( !useWal ){ - rc = walIndexReadHdr(pWal, pChanged); + assert( rc==SQLITE_OK ); + if( pWal->bShmUnreliable==0 ){ + rc = walIndexReadHdr(pWal, pChanged); + } +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + walDisableBlocking(pWal); + if( rc==SQLITE_BUSY_TIMEOUT ){ + rc = SQLITE_BUSY; + *pCnt |= WAL_RETRY_BLOCKED_MASK; + } +#endif if( rc==SQLITE_BUSY ){ /* If there is not a recovery running in another thread or process ** then convert BUSY errors to WAL_RETRY. If recovery is known to @@ -55495,9 +68085,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( pWal->apWiData[0]==0 ){ /* This branch is taken when the xShmMap() method returns SQLITE_BUSY. ** We assume this is a transient condition, so return WAL_RETRY. The - ** xShmMap() implementation used by the default unix and win32 VFS - ** modules may return SQLITE_BUSY due to a race condition in the - ** code that determines whether or not the shared-memory region + ** xShmMap() implementation used by the default unix and win32 VFS + ** modules may return SQLITE_BUSY due to a race condition in the + ** code that determines whether or not the shared-memory region ** must be zeroed before the requested page is returned. */ rc = WAL_RETRY; @@ -55511,13 +68101,18 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ if( rc!=SQLITE_OK ){ return rc; } + else if( pWal->bShmUnreliable ){ + return walBeginShmUnreliable(pWal, pChanged); + } } + assert( pWal->nWiData>0 ); + assert( pWal->apWiData[0]!=0 ); pInfo = walCkptInfo(pWal); - if( !useWal && pInfo->nBackfill==pWal->hdr.mxFrame + SEH_INJECT_FAULT; + if( !useWal && AtomicLoad(&pInfo->nBackfill)==pWal->hdr.mxFrame #ifdef SQLITE_ENABLE_SNAPSHOT - && (pWal->pSnapshot==0 || pWal->hdr.mxFrame==0 - || 0==memcmp(&pWal->hdr, pWal->pSnapshot, sizeof(WalIndexHdr))) + && ((pWal->bGetSnapshot==0 && pWal->pSnapshot==0) || pWal->hdr.mxFrame==0) #endif ){ /* The WAL has been completely backfilled (or it is empty). @@ -55534,7 +68129,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** snapshot. Since holding READ_LOCK(0) prevents a checkpoint from ** happening, this is usually correct. ** - ** However, if frames have been appended to the log (or if the log + ** However, if frames have been appended to the log (or if the log ** is wrapped and written for that matter) before the READ_LOCK(0) ** is obtained, that is not necessarily true. A checkpointer may ** have started to backfill the appended frames but crashed before @@ -55564,7 +68159,7 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } #endif for(i=1; iaReadMark[i]; + u32 thisMark = AtomicLoad(pInfo->aReadMark+i); SEH_INJECT_FAULT; if( mxReadMark<=thisMark && thisMark<=mxFrame ){ assert( thisMark!=READMARK_NOT_USED ); mxReadMark = thisMark; @@ -55577,7 +68172,8 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ for(i=1; iaReadMark[i] = mxFrame; + AtomicStore(pInfo->aReadMark+i,mxFrame); + mxReadMark = mxFrame; mxI = i; walUnlockExclusive(pWal, WAL_READ_LOCK(i), 1); break; @@ -55588,12 +68184,22 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ } if( mxI==0 ){ assert( rc==SQLITE_BUSY || (pWal->readOnly & WAL_SHM_RDONLY)!=0 ); - return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTLOCK; + return rc==SQLITE_BUSY ? WAL_RETRY : SQLITE_READONLY_CANTINIT; } + (void)walEnableBlockingMs(pWal, nBlockTmout); rc = walLockShared(pWal, WAL_READ_LOCK(mxI)); + walDisableBlocking(pWal); if( rc ){ - return rc==SQLITE_BUSY ? WAL_RETRY : rc; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ){ + *pCnt |= WAL_RETRY_BLOCKED_MASK; + } +#else + assert( rc!=SQLITE_BUSY_TIMEOUT ); +#endif + assert( (rc&0xFF)!=SQLITE_BUSY||rc==SQLITE_BUSY||rc==SQLITE_BUSY_TIMEOUT ); + return (rc&0xFF)==SQLITE_BUSY ? WAL_RETRY : rc; } /* Now that the read-lock has been obtained, check that neither the ** value in the aReadMark[] array or the contents of the wal-index @@ -55615,9 +68221,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** to read any frames earlier than minFrame from the wal file - they ** can be safely read directly from the database file. ** - ** Because a ShmBarrier() call is made between taking the copy of + ** Because a ShmBarrier() call is made between taking the copy of ** nBackfill and checking that the wal-header in shared-memory still - ** matches the one cached in pWal->hdr, it is guaranteed that the + ** matches the one cached in pWal->hdr, it is guaranteed that the ** checkpointer that set nBackfill was not working with a wal-index ** header newer than that cached in pWal->hdr. If it were, that could ** cause a problem. The checkpointer could omit to checkpoint @@ -55629,9 +68235,9 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ ** we can guarantee that the checkpointer that set nBackfill could not ** see any pages past pWal->hdr.mxFrame, this problem does not come up. */ - pWal->minFrame = pInfo->nBackfill+1; + pWal->minFrame = AtomicLoad(&pInfo->nBackfill)+1; SEH_INJECT_FAULT; walShmBarrier(pWal); - if( pInfo->aReadMark[mxI]!=mxReadMark + if( AtomicLoad(pInfo->aReadMark+mxI)!=mxReadMark || memcmp((void *)walIndexHdr(pWal), &pWal->hdr, sizeof(WalIndexHdr)) ){ walUnlockShared(pWal, WAL_READ_LOCK(mxI)); @@ -55643,34 +68249,145 @@ static int walTryBeginRead(Wal *pWal, int *pChanged, int useWal, int cnt){ return rc; } +#ifdef SQLITE_ENABLE_SNAPSHOT /* -** Begin a read transaction on the database. -** -** This routine used to be called sqlite3OpenSnapshot() and with good reason: -** it takes a snapshot of the state of the WAL and wal-index for the current -** instant in time. The current thread will continue to use this snapshot. -** Other threads might append new content to the WAL and wal-index but -** that extra content is ignored by the current thread. -** -** If the database contents have changes since the previous read -** transaction, then *pChanged is set to 1 before returning. The -** Pager layer will use this to know that is cache is stale and -** needs to be flushed. +** This function does the work of sqlite3WalSnapshotRecover(). */ -SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ +static int walSnapshotRecover( + Wal *pWal, /* WAL handle */ + void *pBuf1, /* Temp buffer pWal->szPage bytes in size */ + void *pBuf2 /* Temp buffer pWal->szPage bytes in size */ +){ + int szPage = (int)pWal->szPage; + int rc; + i64 szDb; /* Size of db file in bytes */ + + rc = sqlite3OsFileSize(pWal->pDbFd, &szDb); + if( rc==SQLITE_OK ){ + volatile WalCkptInfo *pInfo = walCkptInfo(pWal); + u32 i = pInfo->nBackfillAttempted; + for(i=pInfo->nBackfillAttempted; i>AtomicLoad(&pInfo->nBackfill); i--){ + WalHashLoc sLoc; /* Hash table location */ + u32 pgno; /* Page number in db file */ + i64 iDbOff; /* Offset of db file entry */ + i64 iWalOff; /* Offset of wal file entry */ + + rc = walHashGet(pWal, walFramePage(i), &sLoc); + if( rc!=SQLITE_OK ) break; + assert( i - sLoc.iZero - 1 >=0 ); + pgno = sLoc.aPgno[i-sLoc.iZero-1]; + iDbOff = (i64)(pgno-1) * szPage; + + if( iDbOff+szPage<=szDb ){ + iWalOff = walFrameOffset(i, szPage) + WAL_FRAME_HDRSIZE; + rc = sqlite3OsRead(pWal->pWalFd, pBuf1, szPage, iWalOff); + + if( rc==SQLITE_OK ){ + rc = sqlite3OsRead(pWal->pDbFd, pBuf2, szPage, iDbOff); + } + + if( rc!=SQLITE_OK || 0==memcmp(pBuf1, pBuf2, szPage) ){ + break; + } + } + + pInfo->nBackfillAttempted = i-1; + } + } + + return rc; +} + +/* +** Attempt to reduce the value of the WalCkptInfo.nBackfillAttempted +** variable so that older snapshots can be accessed. To do this, loop +** through all wal frames from nBackfillAttempted to (nBackfill+1), +** comparing their content to the corresponding page with the database +** file, if any. Set nBackfillAttempted to the frame number of the +** first frame for which the wal file content matches the db file. +** +** This is only really safe if the file-system is such that any page +** writes made by earlier checkpointers were atomic operations, which +** is not always true. It is also possible that nBackfillAttempted +** may be left set to a value larger than expected, if a wal frame +** contains content that duplicate of an earlier version of the same +** page. +** +** SQLITE_OK is returned if successful, or an SQLite error code if an +** error occurs. It is not an error if nBackfillAttempted cannot be +** decreased at all. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotRecover(Wal *pWal){ + int rc; + + assert( pWal->readLock>=0 ); + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + if( rc==SQLITE_OK ){ + void *pBuf1 = sqlite3_malloc(pWal->szPage); + void *pBuf2 = sqlite3_malloc(pWal->szPage); + if( pBuf1==0 || pBuf2==0 ){ + rc = SQLITE_NOMEM; + }else{ + pWal->ckptLock = 1; + SEH_TRY { + rc = walSnapshotRecover(pWal, pBuf1, pBuf2); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + pWal->ckptLock = 0; + } + + sqlite3_free(pBuf1); + sqlite3_free(pBuf2); + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + } + + return rc; +} +#endif /* SQLITE_ENABLE_SNAPSHOT */ + +/* +** This function does the work of sqlite3WalBeginReadTransaction() (see +** below). That function simply calls this one inside an SEH_TRY{...} block. +*/ +static int walBeginReadTransaction(Wal *pWal, int *pChanged){ int rc; /* Return code */ int cnt = 0; /* Number of TryBeginRead attempts */ - #ifdef SQLITE_ENABLE_SNAPSHOT + int ckptLock = 0; int bChanged = 0; WalIndexHdr *pSnapshot = pWal->pSnapshot; - if( pSnapshot && memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ - bChanged = 1; +#endif + + assert( pWal->ckptLock==0 ); + assert( pWal->nSehTry>0 ); + +#ifdef SQLITE_ENABLE_SNAPSHOT + if( pSnapshot ){ + if( memcmp(pSnapshot, &pWal->hdr, sizeof(WalIndexHdr))!=0 ){ + bChanged = 1; + } + + /* It is possible that there is a checkpointer thread running + ** concurrent with this code. If this is the case, it may be that the + ** checkpointer has already determined that it will checkpoint + ** snapshot X, where X is later in the wal file than pSnapshot, but + ** has not yet set the pInfo->nBackfillAttempted variable to indicate + ** its intent. To avoid the race condition this leads to, ensure that + ** there is no checkpointer process by taking a shared CKPT lock + ** before checking pInfo->nBackfillAttempted. */ + (void)walEnableBlocking(pWal); + rc = walLockShared(pWal, WAL_CKPT_LOCK); + walDisableBlocking(pWal); + + if( rc!=SQLITE_OK ){ + return rc; + } + ckptLock = 1; } #endif do{ - rc = walTryBeginRead(pWal, pChanged, 0, ++cnt); + rc = walTryBeginRead(pWal, pChanged, 0, &cnt); }while( rc==WAL_RETRY ); testcase( (rc&0xff)==SQLITE_BUSY ); testcase( (rc&0xff)==SQLITE_IOERR ); @@ -55698,47 +68415,68 @@ SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ assert( pWal->readLock>0 || pWal->hdr.mxFrame==0 ); assert( pInfo->aReadMark[pWal->readLock]<=pSnapshot->mxFrame ); - /* It is possible that there is a checkpointer thread running - ** concurrent with this code. If this is the case, it may be that the - ** checkpointer has already determined that it will checkpoint - ** snapshot X, where X is later in the wal file than pSnapshot, but - ** has not yet set the pInfo->nBackfillAttempted variable to indicate - ** its intent. To avoid the race condition this leads to, ensure that - ** there is no checkpointer process by taking a shared CKPT lock - ** before checking pInfo->nBackfillAttempted. */ - rc = walLockShared(pWal, WAL_CKPT_LOCK); - - if( rc==SQLITE_OK ){ - /* Check that the wal file has not been wrapped. Assuming that it has - ** not, also check that no checkpointer has attempted to checkpoint any - ** frames beyond pSnapshot->mxFrame. If either of these conditions are - ** true, return SQLITE_BUSY_SNAPSHOT. Otherwise, overwrite pWal->hdr - ** with *pSnapshot and set *pChanged as appropriate for opening the - ** snapshot. */ - if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) - && pSnapshot->mxFrame>=pInfo->nBackfillAttempted - ){ - assert( pWal->readLock>0 ); - memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); - *pChanged = bChanged; - }else{ - rc = SQLITE_BUSY_SNAPSHOT; - } - - /* Release the shared CKPT lock obtained above. */ - walUnlockShared(pWal, WAL_CKPT_LOCK); + /* Check that the wal file has not been wrapped. Assuming that it has + ** not, also check that no checkpointer has attempted to checkpoint any + ** frames beyond pSnapshot->mxFrame. If either of these conditions are + ** true, return SQLITE_ERROR_SNAPSHOT. Otherwise, overwrite pWal->hdr + ** with *pSnapshot and set *pChanged as appropriate for opening the + ** snapshot. */ + if( !memcmp(pSnapshot->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + && pSnapshot->mxFrame>=pInfo->nBackfillAttempted + ){ + assert( pWal->readLock>0 ); + memcpy(&pWal->hdr, pSnapshot, sizeof(WalIndexHdr)); + *pChanged = bChanged; + }else{ + rc = SQLITE_ERROR_SNAPSHOT; } + /* A client using a non-current snapshot may not ignore any frames + ** from the start of the wal file. This is because, for a system + ** where (minFrame < iSnapshot < maxFrame), a checkpointer may + ** have omitted to checkpoint a frame earlier than minFrame in + ** the file because there exists a frame after iSnapshot that + ** is the same database page. */ + pWal->minFrame = 1; if( rc!=SQLITE_OK ){ sqlite3WalEndReadTransaction(pWal); } } } + + /* Release the shared CKPT lock obtained above. */ + if( ckptLock ){ + assert( pSnapshot ); + walUnlockShared(pWal, WAL_CKPT_LOCK); + } #endif return rc; } +/* +** Begin a read transaction on the database. +** +** This routine used to be called sqlite3OpenSnapshot() and with good reason: +** it takes a snapshot of the state of the WAL and wal-index for the current +** instant in time. The current thread will continue to use this snapshot. +** Other threads might append new content to the WAL and wal-index but +** that extra content is ignored by the current thread. +** +** If the database contents have changes since the previous read +** transaction, then *pChanged is set to 1 before returning. The +** Pager layer will use this to know that its cache is stale and +** needs to be flushed. +*/ +SQLITE_PRIVATE int sqlite3WalBeginReadTransaction(Wal *pWal, int *pChanged){ + int rc; + SEH_TRY { + rc = walBeginReadTransaction(pWal, pChanged); + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + /* ** Finish with a read transaction. All this does is release the ** read-lock. @@ -55759,7 +68497,7 @@ SQLITE_PRIVATE void sqlite3WalEndReadTransaction(Wal *pWal){ ** Return SQLITE_OK if successful, or an error code if an error occurs. If an ** error does occur, the final value of *piRead is undefined. */ -SQLITE_PRIVATE int sqlite3WalFindFrame( +static int walFindFrame( Wal *pWal, /* WAL handle */ Pgno pgno, /* Database page number to read data for */ u32 *piRead /* OUT: Frame number (or zero) */ @@ -55774,11 +68512,11 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( /* If the "last page" field of the wal-index header snapshot is 0, then ** no data will be read from the wal under any circumstances. Return early - ** in this case as an optimization. Likewise, if pWal->readLock==0, - ** then the WAL is ignored by the reader so return early, as if the + ** in this case as an optimization. Likewise, if pWal->readLock==0, + ** then the WAL is ignored by the reader so return early, as if the ** WAL were empty. */ - if( iLast==0 || pWal->readLock==0 ){ + if( iLast==0 || (pWal->readLock==0 && pWal->bShmUnreliable==0) ){ *piRead = 0; return SQLITE_OK; } @@ -55788,9 +68526,9 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** hash table (each hash table indexes up to HASHTABLE_NPAGE frames). ** ** This code might run concurrently to the code in walIndexAppend() - ** that adds entries to the wal-index (and possibly to this hash - ** table). This means the value just read from the hash - ** slot (aHash[iKey]) may have been added before or after the + ** that adds entries to the wal-index (and possibly to this hash + ** table). This means the value just read from the hash + ** slot (aHash[iKey]) may have been added before or after the ** current read transaction was opened. Values added after the ** read transaction was opened may have been written incorrectly - ** i.e. these slots may contain garbage data. However, we assume @@ -55798,40 +68536,44 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( ** opened remain unmodified. ** ** For the reasons above, the if(...) condition featured in the inner - ** loop of the following block is more stringent that would be required + ** loop of the following block is more stringent that would be required ** if we had exclusive access to the hash-table: ** - ** (aPgno[iFrame]==pgno): + ** (aPgno[iFrame]==pgno): ** This condition filters out normal hash-table collisions. ** - ** (iFrame<=iLast): + ** (iFrame<=iLast): ** This condition filters out entries that were added to the hash ** table after the current read-transaction had started. */ iMinHash = walFramePage(pWal->minFrame); - for(iHash=walFramePage(iLast); iHash>=iMinHash && iRead==0; iHash--){ - volatile ht_slot *aHash; /* Pointer to hash table */ - volatile u32 *aPgno; /* Pointer to array of page numbers */ - u32 iZero; /* Frame number corresponding to aPgno[0] */ + for(iHash=walFramePage(iLast); iHash>=iMinHash; iHash--){ + WalHashLoc sLoc; /* Hash table location */ int iKey; /* Hash slot index */ int nCollide; /* Number of hash collisions remaining */ int rc; /* Error code */ + u32 iH; - rc = walHashGet(pWal, iHash, &aHash, &aPgno, &iZero); + rc = walHashGet(pWal, iHash, &sLoc); if( rc!=SQLITE_OK ){ return rc; } nCollide = HASHTABLE_NSLOT; - for(iKey=walHash(pgno); aHash[iKey]; iKey=walNextHash(iKey)){ - u32 iFrame = aHash[iKey] + iZero; - if( iFrame<=iLast && iFrame>=pWal->minFrame && aPgno[aHash[iKey]]==pgno ){ + iKey = walHash(pgno); + SEH_INJECT_FAULT; + while( (iH = AtomicLoad(&sLoc.aHash[iKey]))!=0 ){ + u32 iFrame = iH + sLoc.iZero; + if( iFrame<=iLast && iFrame>=pWal->minFrame && sLoc.aPgno[iH-1]==pgno ){ assert( iFrame>iRead || CORRUPT_DB ); iRead = iFrame; } if( (nCollide--)==0 ){ + *piRead = 0; return SQLITE_CORRUPT_BKPT; } + iKey = walNextHash(iKey); } + if( iRead ) break; } #ifdef SQLITE_ENABLE_EXPENSIVE_ASSERT @@ -55841,8 +68583,8 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( { u32 iRead2 = 0; u32 iTest; - assert( pWal->minFrame>0 ); - for(iTest=iLast; iTest>=pWal->minFrame; iTest--){ + assert( pWal->bShmUnreliable || pWal->minFrame>0 ); + for(iTest=iLast; iTest>=pWal->minFrame && iTest>0; iTest--){ if( walFramePgno(pWal, iTest)==pgno ){ iRead2 = iTest; break; @@ -55856,6 +68598,30 @@ SQLITE_PRIVATE int sqlite3WalFindFrame( return SQLITE_OK; } +/* +** Search the wal file for page pgno. If found, set *piRead to the frame that +** contains the page. Otherwise, if pgno is not in the wal file, set *piRead +** to zero. +** +** Return SQLITE_OK if successful, or an error code if an error occurs. If an +** error does occur, the final value of *piRead is undefined. +** +** The difference between this function and walFindFrame() is that this +** function wraps walFindFrame() in an SEH_TRY{...} block. +*/ +SQLITE_PRIVATE int sqlite3WalFindFrame( + Wal *pWal, /* WAL handle */ + Pgno pgno, /* Database page number to read data for */ + u32 *piRead /* OUT: Frame number (or zero) */ +){ + int rc; + SEH_TRY { + rc = walFindFrame(pWal, pgno, piRead); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + return rc; +} + /* ** Read the contents of frame iRead from the wal file into buffer pOut ** (which is nOut bytes in size). Return SQLITE_OK if successful, or an @@ -55878,7 +68644,7 @@ SQLITE_PRIVATE int sqlite3WalReadFrame( return sqlite3OsRead(pWal->pWalFd, pOut, (nOut>sz ? sz : nOut), iOffset); } -/* +/* ** Return the size of the database in pages (or zero, if unknown). */ SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ @@ -55889,7 +68655,7 @@ SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ } -/* +/* ** This function starts a write transaction on the WAL. ** ** A read transaction must have already been started by a prior call @@ -55905,6 +68671,16 @@ SQLITE_PRIVATE Pgno sqlite3WalDbsize(Wal *pWal){ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ int rc; +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If the write-lock is already held, then it was obtained before the + ** read-transaction was even opened, making this call a no-op. + ** Return early. */ + if( pWal->writeLock ){ + assert( !memcmp(&pWal->hdr,(void *)walIndexHdr(pWal),sizeof(WalIndexHdr)) ); + return SQLITE_OK; + } +#endif + /* Cannot start a write transaction without first holding a read ** transaction. */ assert( pWal->readLock>=0 ); @@ -55927,12 +68703,17 @@ SQLITE_PRIVATE int sqlite3WalBeginWriteTransaction(Wal *pWal){ ** time the read transaction on this connection was started, then ** the write is disallowed. */ - if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + SEH_TRY { + if( memcmp(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr))!=0 ){ + rc = SQLITE_BUSY_SNAPSHOT; + } + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) + + if( rc!=SQLITE_OK ){ walUnlockExclusive(pWal, WAL_WRITE_LOCK, 1); pWal->writeLock = 0; - rc = SQLITE_BUSY_SNAPSHOT; } - return rc; } @@ -55967,39 +68748,42 @@ SQLITE_PRIVATE int sqlite3WalUndo(Wal *pWal, int (*xUndo)(void *, Pgno), void *p if( ALWAYS(pWal->writeLock) ){ Pgno iMax = pWal->hdr.mxFrame; Pgno iFrame; - - /* Restore the clients cache of the wal-index header to the state it - ** was in before the client began writing to the database. - */ - memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); - for(iFrame=pWal->hdr.mxFrame+1; - ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; - iFrame++ - ){ - /* This call cannot fail. Unless the page for which the page number - ** is passed as the second argument is (a) in the cache and - ** (b) has an outstanding reference, then xUndo is either a no-op - ** (if (a) is false) or simply expels the page from the cache (if (b) - ** is false). - ** - ** If the upper layer is doing a rollback, it is guaranteed that there - ** are no outstanding references to any page other than page 1. And - ** page 1 is never written to the log until the transaction is - ** committed. As a result, the call to xUndo may not fail. + SEH_TRY { + /* Restore the clients cache of the wal-index header to the state it + ** was in before the client began writing to the database. */ - assert( walFramePgno(pWal, iFrame)!=1 ); - rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + memcpy(&pWal->hdr, (void *)walIndexHdr(pWal), sizeof(WalIndexHdr)); + + for(iFrame=pWal->hdr.mxFrame+1; + ALWAYS(rc==SQLITE_OK) && iFrame<=iMax; + iFrame++ + ){ + /* This call cannot fail. Unless the page for which the page number + ** is passed as the second argument is (a) in the cache and + ** (b) has an outstanding reference, then xUndo is either a no-op + ** (if (a) is false) or simply expels the page from the cache (if (b) + ** is false). + ** + ** If the upper layer is doing a rollback, it is guaranteed that there + ** are no outstanding references to any page other than page 1. And + ** page 1 is never written to the log until the transaction is + ** committed. As a result, the call to xUndo may not fail. + */ + assert( walFramePgno(pWal, iFrame)!=1 ); + rc = xUndo(pUndoCtx, walFramePgno(pWal, iFrame)); + } + if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); } - if( iMax!=pWal->hdr.mxFrame ) walCleanupHash(pWal); + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; } -/* -** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 -** values. This function populates the array with values required to -** "rollback" the write position of the WAL handle back to the current +/* +** Argument aWalData must point to an array of WAL_SAVEPOINT_NDATA u32 +** values. This function populates the array with values required to +** "rollback" the write position of the WAL handle back to the current ** point in the event of a savepoint rollback (via WalSavepointUndo()). */ SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ @@ -56010,7 +68794,7 @@ SQLITE_PRIVATE void sqlite3WalSavepoint(Wal *pWal, u32 *aWalData){ aWalData[3] = pWal->nCkpt; } -/* +/* ** Move the write position of the WAL back to the point identified by ** the values in the aWalData[] array. aWalData must point to an array ** of WAL_SAVEPOINT_NDATA u32 values that has been previously populated @@ -56035,7 +68819,10 @@ SQLITE_PRIVATE int sqlite3WalSavepointUndo(Wal *pWal, u32 *aWalData){ pWal->hdr.mxFrame = aWalData[0]; pWal->hdr.aFrameCksum[0] = aWalData[1]; pWal->hdr.aFrameCksum[1] = aWalData[2]; - walCleanupHash(pWal); + SEH_TRY { + walCleanupHash(pWal); + } + SEH_EXCEPT( rc = SQLITE_IOERR_IN_PAGE; ) } return rc; @@ -56085,7 +68872,7 @@ static int walRestartLog(Wal *pWal){ cnt = 0; do{ int notUsed; - rc = walTryBeginRead(pWal, ¬Used, 1, ++cnt); + rc = walTryBeginRead(pWal, ¬Used, 1, &cnt); }while( rc==WAL_RETRY ); assert( (rc&0xff)!=SQLITE_BUSY ); /* BUSY not possible when useWal==1 */ testcase( (rc&0xff)==SQLITE_IOERR ); @@ -56130,8 +68917,8 @@ static int walWriteToLog( iOffset += iFirstAmt; iAmt -= iFirstAmt; pContent = (void*)(iFirstAmt + (char*)pContent); - assert( p->syncFlags & (SQLITE_SYNC_NORMAL|SQLITE_SYNC_FULL) ); - rc = sqlite3OsSync(p->pFd, p->syncFlags & SQLITE_SYNC_MASK); + assert( WAL_SYNC_FLAGS(p->syncFlags)!=0 ); + rc = sqlite3OsSync(p->pFd, WAL_SYNC_FLAGS(p->syncFlags)); if( iAmt==0 || rc ) return rc; } rc = sqlite3OsWrite(p->pFd, pContent, iAmt, iOffset); @@ -56150,11 +68937,7 @@ static int walWriteOneFrame( int rc; /* Result code from subfunctions */ void *pData; /* Data actually written */ u8 aFrame[WAL_FRAME_HDRSIZE]; /* Buffer to assemble frame-header in */ -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(pPage))==0 ) return SQLITE_NOMEM_BKPT; -#else pData = pPage->pData; -#endif walEncodeFrame(p->pWal, pPage->pgno, nTruncate, pData, aFrame); rc = walWriteToLog(p, aFrame, sizeof(aFrame), iOffset); if( rc ) return rc; @@ -56216,11 +68999,11 @@ static int walRewriteChecksums(Wal *pWal, u32 iLast){ return rc; } -/* +/* ** Write a set of frames to the log. The caller must hold the write-lock ** on the log file (obtained using sqlite3WalBeginWriteTransaction()). */ -SQLITE_PRIVATE int sqlite3WalFrames( +static int walFrames( Wal *pWal, /* Wal handle to write to */ int szPage, /* Database page-size in bytes */ PgHdr *pList, /* List of dirty pages to write */ @@ -56283,7 +69066,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( walChecksumBytes(1, aWalHdr, WAL_HDRSIZE-2*4, 0, aCksum); sqlite3Put4byte(&aWalHdr[24], aCksum[0]); sqlite3Put4byte(&aWalHdr[28], aCksum[1]); - + pWal->szPage = szPage; pWal->hdr.bigEndCksum = SQLITE_BIGENDIAN; pWal->hdr.aFrameCksum[0] = aCksum[0]; @@ -56301,14 +69084,16 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** an out-of-order write following a WAL restart could result in ** database corruption. See the ticket: ** - ** http://localhost:591/sqlite/info/ff5be73dee + ** https://sqlite.org/src/info/ff5be73dee */ - if( pWal->syncHeader && sync_flags ){ - rc = sqlite3OsSync(pWal->pWalFd, sync_flags & SQLITE_SYNC_MASK); + if( pWal->syncHeader ){ + rc = sqlite3OsSync(pWal->pWalFd, CKPT_SYNC_FLAGS(sync_flags)); if( rc ) return rc; } } - assert( (int)pWal->szPage==szPage ); + if( (int)pWal->szPage!=szPage ){ + return SQLITE_CORRUPT_BKPT; /* TH3 test case: cov1/corrupt155.test */ + } /* Setup information needed to write frames into the WAL */ w.pWal = pWal; @@ -56325,11 +69110,11 @@ SQLITE_PRIVATE int sqlite3WalFrames( /* Check if this page has already been written into the wal file by ** the current transaction. If so, overwrite the existing frame and - ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that + ** set Wal.writeLock to WAL_WRITELOCK_RECKSUM - indicating that ** checksums must be recomputed when the transaction is committed. */ if( iFirst && (p->pDirty || isCommit==0) ){ u32 iWrite = 0; - VVA_ONLY(rc =) sqlite3WalFindFrame(pWal, p->pgno, &iWrite); + VVA_ONLY(rc =) walFindFrame(pWal, p->pgno, &iWrite); assert( rc==SQLITE_OK || iWrite==0 ); if( iWrite>=iFirst ){ i64 iOff = walFrameOffset(iWrite, szPage) + WAL_FRAME_HDRSIZE; @@ -56337,11 +69122,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( if( pWal->iReCksum==0 || iWriteiReCksum ){ pWal->iReCksum = iWrite; } -#if defined(SQLITE_HAS_CODEC) - if( (pData = sqlite3PagerCodec(p))==0 ) return SQLITE_NOMEM; -#else pData = p->pData; -#endif rc = sqlite3OsWrite(pWal->pWalFd, pData, szPage, iOff); if( rc ) return rc; p->flags &= ~PGHDR_WAL_APPEND; @@ -56379,18 +69160,24 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** sector boundary is synced; the part of the last frame that extends ** past the sector boundary is written after the sync. */ - if( isCommit && (sync_flags & WAL_SYNC_TRANSACTIONS)!=0 ){ + if( isCommit && WAL_SYNC_FLAGS(sync_flags)!=0 ){ + int bSync = 1; if( pWal->padToSectorBoundary ){ int sectorSize = sqlite3SectorSize(pWal->pWalFd); w.iSyncPoint = ((iOffset+sectorSize-1)/sectorSize)*sectorSize; + bSync = (w.iSyncPoint==iOffset); + testcase( bSync ); while( iOffsettruncateOnCommit = 0; } - /* Append data to the wal-index. It is not necessary to lock the + /* Append data to the wal-index. It is not necessary to lock the ** wal-index to do this as the SQLITE_SHM_WRITE lock held on the wal-index ** guarantees that there are no other writers, and no data that may ** be in use by existing readers is being overwritten. @@ -56418,6 +69205,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( iFrame++; rc = walIndexAppend(pWal, iFrame, p->pgno); } + assert( pLast!=0 || nExtra==0 ); while( rc==SQLITE_OK && nExtra>0 ){ iFrame++; nExtra--; @@ -56445,7 +69233,30 @@ SQLITE_PRIVATE int sqlite3WalFrames( return rc; } -/* +/* +** Write a set of frames to the log. The caller must hold the write-lock +** on the log file (obtained using sqlite3WalBeginWriteTransaction()). +** +** The difference between this function and walFrames() is that this +** function wraps walFrames() in an SEH_TRY{...} block. +*/ +SQLITE_PRIVATE int sqlite3WalFrames( + Wal *pWal, /* Wal handle to write to */ + int szPage, /* Database page-size in bytes */ + PgHdr *pList, /* List of dirty pages to write */ + Pgno nTruncate, /* Database size after this commit */ + int isCommit, /* True if this is a commit */ + int sync_flags /* Flags to pass to OsSync() (or 0) */ +){ + int rc; + SEH_TRY { + rc = walFrames(pWal, szPage, pList, nTruncate, isCommit, sync_flags); + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + +/* ** This routine is called to implement sqlite3_wal_checkpoint() and ** related interfaces. ** @@ -56457,6 +69268,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( */ SQLITE_PRIVATE int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ + sqlite3 *db, /* Check this handle's interrupt flag */ int eMode, /* PASSIVE, FULL, RESTART, or TRUNCATE */ int (*xBusy)(void*), /* Function to call when busy */ void *pBusyArg, /* Context argument for xBusyHandler */ @@ -56481,68 +69293,82 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( if( pWal->readOnly ) return SQLITE_READONLY; WALTRACE(("WAL%p: checkpoint begins\n", pWal)); - /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive - ** "checkpoint" lock on the database file. */ - rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); - if( rc ){ - /* EVIDENCE-OF: R-10421-19736 If any other process is running a - ** checkpoint operation at the same time, the lock cannot be obtained and - ** SQLITE_BUSY is returned. - ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, - ** it will not be invoked in this case. - */ - testcase( rc==SQLITE_BUSY ); - testcase( xBusy!=0 ); - return rc; - } - pWal->ckptLock = 1; + /* Enable blocking locks, if possible. */ + sqlite3WalDb(pWal, db); + if( xBusy2 ) (void)walEnableBlocking(pWal); - /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and - ** TRUNCATE modes also obtain the exclusive "writer" lock on the database - ** file. - ** - ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained - ** immediately, and a busy-handler is configured, it is invoked and the - ** writer lock retried until either the busy-handler returns 0 or the - ** lock is successfully obtained. + /* IMPLEMENTATION-OF: R-62028-47212 All calls obtain an exclusive + ** "checkpoint" lock on the database file. + ** EVIDENCE-OF: R-10421-19736 If any other process is running a + ** checkpoint operation at the same time, the lock cannot be obtained and + ** SQLITE_BUSY is returned. + ** EVIDENCE-OF: R-53820-33897 Even if there is a busy-handler configured, + ** it will not be invoked in this case. */ - if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ - rc = walBusyLock(pWal, xBusy, pBusyArg, WAL_WRITE_LOCK, 1); - if( rc==SQLITE_OK ){ - pWal->writeLock = 1; - }else if( rc==SQLITE_BUSY ){ - eMode2 = SQLITE_CHECKPOINT_PASSIVE; - xBusy2 = 0; - rc = SQLITE_OK; + rc = walLockExclusive(pWal, WAL_CKPT_LOCK, 1); + testcase( rc==SQLITE_BUSY ); + testcase( rc!=SQLITE_OK && xBusy2!=0 ); + if( rc==SQLITE_OK ){ + pWal->ckptLock = 1; + + /* IMPLEMENTATION-OF: R-59782-36818 The SQLITE_CHECKPOINT_FULL, RESTART and + ** TRUNCATE modes also obtain the exclusive "writer" lock on the database + ** file. + ** + ** EVIDENCE-OF: R-60642-04082 If the writer lock cannot be obtained + ** immediately, and a busy-handler is configured, it is invoked and the + ** writer lock retried until either the busy-handler returns 0 or the + ** lock is successfully obtained. + */ + if( eMode!=SQLITE_CHECKPOINT_PASSIVE ){ + rc = walBusyLock(pWal, xBusy2, pBusyArg, WAL_WRITE_LOCK, 1); + if( rc==SQLITE_OK ){ + pWal->writeLock = 1; + }else if( rc==SQLITE_BUSY ){ + eMode2 = SQLITE_CHECKPOINT_PASSIVE; + xBusy2 = 0; + rc = SQLITE_OK; + } } } + /* Read the wal-index header. */ - if( rc==SQLITE_OK ){ - rc = walIndexReadHdr(pWal, &isChanged); - if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ - sqlite3OsUnfetch(pWal->pDbFd, 0, 0); - } - } - - /* Copy data from the log to the database file. */ - if( rc==SQLITE_OK ){ - - if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ - rc = SQLITE_CORRUPT_BKPT; - }else{ - rc = walCheckpoint(pWal, eMode2, xBusy2, pBusyArg, sync_flags, zBuf); - } - - /* If no error occurred, set the output variables. */ - if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ - if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; - if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + SEH_TRY { + if( rc==SQLITE_OK ){ + /* For a passive checkpoint, do not re-enable blocking locks after + ** reading the wal-index header. A passive checkpoint should not block + ** or invoke the busy handler. The only lock such a checkpoint may + ** attempt to obtain is a lock on a read-slot, and it should give up + ** immediately and do a partial checkpoint if it cannot obtain it. */ + walDisableBlocking(pWal); + rc = walIndexReadHdr(pWal, &isChanged); + if( eMode2!=SQLITE_CHECKPOINT_PASSIVE ) (void)walEnableBlocking(pWal); + if( isChanged && pWal->pDbFd->pMethods->iVersion>=3 ){ + sqlite3OsUnfetch(pWal->pDbFd, 0, 0); + } + } + + /* Copy data from the log to the database file. */ + if( rc==SQLITE_OK ){ + if( pWal->hdr.mxFrame && walPagesize(pWal)!=nBuf ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = walCheckpoint(pWal, db, eMode2, xBusy2, pBusyArg, sync_flags,zBuf); + } + + /* If no error occurred, set the output variables. */ + if( rc==SQLITE_OK || rc==SQLITE_BUSY ){ + if( pnLog ) *pnLog = (int)pWal->hdr.mxFrame; + SEH_INJECT_FAULT; + if( pnCkpt ) *pnCkpt = (int)(walCkptInfo(pWal)->nBackfill); + } } } + SEH_EXCEPT( rc = walHandleException(pWal); ) if( isChanged ){ - /* If a new wal-index header was loaded before the checkpoint was + /* If a new wal-index header was loaded before the checkpoint was ** performed, then the pager-cache associated with pWal is now ** out of date. So zero the cached wal-index header to ensure that ** next time the pager opens a snapshot on this database it knows that @@ -56551,11 +69377,19 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( memset(&pWal->hdr, 0, sizeof(WalIndexHdr)); } + walDisableBlocking(pWal); + sqlite3WalDb(pWal, 0); + /* Release the locks. */ sqlite3WalEndWriteTransaction(pWal); - walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); - pWal->ckptLock = 0; + if( pWal->ckptLock ){ + walUnlockExclusive(pWal, WAL_CKPT_LOCK, 1); + pWal->ckptLock = 0; + } WALTRACE(("WAL%p: checkpoint %s\n", pWal, rc ? "failed" : "ok")); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif return (rc==SQLITE_OK && eMode!=eMode2 ? SQLITE_BUSY : rc); } @@ -56585,7 +69419,7 @@ SQLITE_PRIVATE int sqlite3WalCallback(Wal *pWal){ ** operation must occur while the pager is still holding the exclusive ** lock on the main database file. ** -** If op is one, then change from locking_mode=NORMAL into +** If op is one, then change from locking_mode=NORMAL into ** locking_mode=EXCLUSIVE. This means that the pWal->readLock must ** be released. Return 1 if the transition is made and 0 if the ** WAL is already in exclusive-locking mode - meaning that this @@ -56602,42 +69436,44 @@ SQLITE_PRIVATE int sqlite3WalExclusiveMode(Wal *pWal, int op){ assert( pWal->writeLock==0 ); assert( pWal->exclusiveMode!=WAL_HEAPMEMORY_MODE || op==-1 ); - /* pWal->readLock is usually set, but might be -1 if there was a - ** prior error while attempting to acquire are read-lock. This cannot + /* pWal->readLock is usually set, but might be -1 if there was a + ** prior error while attempting to acquire are read-lock. This cannot ** happen if the connection is actually in exclusive mode (as no xShmLock ** locks are taken in this case). Nor should the pager attempt to ** upgrade to exclusive-mode following such an error. */ +#ifndef SQLITE_USE_SEH assert( pWal->readLock>=0 || pWal->lockError ); +#endif assert( pWal->readLock>=0 || (op<=0 && pWal->exclusiveMode==0) ); if( op==0 ){ - if( pWal->exclusiveMode ){ - pWal->exclusiveMode = 0; + if( pWal->exclusiveMode!=WAL_NORMAL_MODE ){ + pWal->exclusiveMode = WAL_NORMAL_MODE; if( walLockShared(pWal, WAL_READ_LOCK(pWal->readLock))!=SQLITE_OK ){ - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; } - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; }else{ /* Already in locking_mode=NORMAL */ rc = 0; } }else if( op>0 ){ - assert( pWal->exclusiveMode==0 ); + assert( pWal->exclusiveMode==WAL_NORMAL_MODE ); assert( pWal->readLock>=0 ); walUnlockShared(pWal, WAL_READ_LOCK(pWal->readLock)); - pWal->exclusiveMode = 1; + pWal->exclusiveMode = WAL_EXCLUSIVE_MODE; rc = 1; }else{ - rc = pWal->exclusiveMode==0; + rc = pWal->exclusiveMode==WAL_NORMAL_MODE; } return rc; } -/* +/* ** Return true if the argument is non-NULL and the WAL module is using ** heap-memory for the wal-index. Otherwise, if the argument is NULL or the -** WAL module is using shared-memory, return false. +** WAL module is using shared-memory, return false. */ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ return (pWal && pWal->exclusiveMode==WAL_HEAPMEMORY_MODE ); @@ -56651,9 +69487,14 @@ SQLITE_PRIVATE int sqlite3WalHeapMemory(Wal *pWal){ SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapshot){ int rc = SQLITE_OK; WalIndexHdr *pRet; + static const u32 aZero[4] = { 0, 0, 0, 0 }; assert( pWal->readLock>=0 && pWal->writeLock==0 ); + if( memcmp(&pWal->hdr.aFrameCksum[0],aZero,16)==0 ){ + *ppSnapshot = 0; + return SQLITE_ERROR; + } pRet = (WalIndexHdr*)sqlite3_malloc(sizeof(WalIndexHdr)); if( pRet==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -56667,15 +69508,31 @@ SQLITE_PRIVATE int sqlite3WalSnapshotGet(Wal *pWal, sqlite3_snapshot **ppSnapsho /* Try to open on pSnapshot when the next read-transaction starts */ -SQLITE_PRIVATE void sqlite3WalSnapshotOpen(Wal *pWal, sqlite3_snapshot *pSnapshot){ - pWal->pSnapshot = (WalIndexHdr*)pSnapshot; +SQLITE_PRIVATE void sqlite3WalSnapshotOpen( + Wal *pWal, + sqlite3_snapshot *pSnapshot +){ + if( pSnapshot && ((WalIndexHdr*)pSnapshot)->iVersion==0 ){ + /* iVersion==0 means that this is a call to sqlite3_snapshot_get(). In + ** this case set the bGetSnapshot flag so that if the call to + ** sqlite3_snapshot_get() is about to read transaction on this wal + ** file, it does not take read-lock 0 if the wal file has been completely + ** checkpointed. Taking read-lock 0 would work, but then it would be + ** possible for a subsequent writer to destroy the snapshot even while + ** this connection is holding its read-transaction open. This is contrary + ** to user expectations, so we avoid it by not taking read-lock 0. */ + pWal->bGetSnapshot = 1; + }else{ + pWal->pSnapshot = (WalIndexHdr*)pSnapshot; + pWal->bGetSnapshot = 0; + } } -/* +/* ** Return a +ve value if snapshot p1 is newer than p2. A -ve value if ** p1 is older than p2 and zero if p1 and p2 are the same snapshot. */ -SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ +SQLITE_API int sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3_snapshot *p2){ WalIndexHdr *pHdr1 = (WalIndexHdr*)p1; WalIndexHdr *pHdr2 = (WalIndexHdr*)p2; @@ -56687,6 +69544,46 @@ SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_cmp(sqlite3_snapshot *p1, sqlite3 if( pHdr1->mxFrame>pHdr2->mxFrame ) return +1; return 0; } + +/* +** The caller currently has a read transaction open on the database. +** This function takes a SHARED lock on the CHECKPOINTER slot and then +** checks if the snapshot passed as the second argument is still +** available. If so, SQLITE_OK is returned. +** +** If the snapshot is not available, SQLITE_ERROR is returned. Or, if +** the CHECKPOINTER lock cannot be obtained, SQLITE_BUSY. If any error +** occurs (any value other than SQLITE_OK is returned), the CHECKPOINTER +** lock is released before returning. +*/ +SQLITE_PRIVATE int sqlite3WalSnapshotCheck(Wal *pWal, sqlite3_snapshot *pSnapshot){ + int rc; + SEH_TRY { + rc = walLockShared(pWal, WAL_CKPT_LOCK); + if( rc==SQLITE_OK ){ + WalIndexHdr *pNew = (WalIndexHdr*)pSnapshot; + if( memcmp(pNew->aSalt, pWal->hdr.aSalt, sizeof(pWal->hdr.aSalt)) + || pNew->mxFramenBackfillAttempted + ){ + rc = SQLITE_ERROR_SNAPSHOT; + walUnlockShared(pWal, WAL_CKPT_LOCK); + } + } + } + SEH_EXCEPT( rc = walHandleException(pWal); ) + return rc; +} + +/* +** Release a lock obtained by an earlier successful call to +** sqlite3WalSnapshotCheck(). +*/ +SQLITE_PRIVATE void sqlite3WalSnapshotUnlock(Wal *pWal){ + assert( pWal ); + walUnlockShared(pWal, WAL_CKPT_LOCK); +} + + #endif /* SQLITE_ENABLE_SNAPSHOT */ #ifdef SQLITE_ENABLE_ZIPVFS @@ -56761,16 +69658,16 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** on Ptr(N) and its subpages have values greater than Key(N-1). And ** so forth. ** -** Finding a particular key requires reading O(log(M)) pages from the +** Finding a particular key requires reading O(log(M)) pages from the ** disk where M is the number of entries in the tree. ** -** In this implementation, a single file can hold one or more separate +** In this implementation, a single file can hold one or more separate ** BTrees. Each BTree is identified by the index of its root page. The ** key and data for any entry are combined to form the "payload". A ** fixed amount of payload can be carried directly on the database ** page. If the payload is larger than the preset amount then surplus ** bytes are stored on overflow pages. The payload for an entry -** and the preceding pointer are combined to form a "Cell". Each +** and the preceding pointer are combined to form a "Cell". Each ** page has a small header which contains the Ptr(N) pointer and other ** information such as the size of key and data. ** @@ -56796,7 +69693,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** 22 1 Min embedded payload fraction (must be 32) ** 23 1 Min leaf payload fraction (must be 32) ** 24 4 File change counter -** 28 4 Reserved for future use +** 28 4 The size of the database in pages ** 32 4 First freelist page ** 36 4 Number of freelist pages in the file ** 40 60 15 4-byte meta values passed to higher layers @@ -56900,11 +69797,11 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** contiguous or in order, but cell pointers are contiguous and in order. ** ** Cell content makes use of variable length integers. A variable -** length integer is 1 to 9 bytes where the lower 7 bits of each +** length integer is 1 to 9 bytes where the lower 7 bits of each ** byte are used. The integer consists of all bytes that have bit 8 set and ** the first byte with bit 8 clear. The most significant byte of the integer ** appears first. A variable-length integer may not be more than 9 bytes long. -** As a special case, all 8 bytes of the 9th byte are used as data. This +** As a special case, all 8 bits of the 9th byte are used as data. This ** allows a 64-bit integer to be encoded in 9 bytes. ** ** 0x00 becomes 0x00000000 @@ -56912,7 +69809,7 @@ SQLITE_PRIVATE sqlite3_file *sqlite3WalFile(Wal *pWal){ ** 0x81 0x00 becomes 0x00000080 ** 0x82 0x00 becomes 0x00000100 ** 0x80 0x7f becomes 0x0000007f -** 0x8a 0x91 0xd1 0xac 0x78 becomes 0x12345678 +** 0x81 0x91 0xd1 0xac 0x78 becomes 0x12345678 ** 0x81 0x81 0x81 0x81 0x01 becomes 0x10204081 ** ** Variable length integers are used for rowids and to hold the number of @@ -56973,7 +69870,7 @@ typedef struct CellInfo CellInfo; ** -DSQLITE_FILE_HEADER="..." on the compiler command-line. The ** header must be exactly 16 bytes including the zero-terminator so ** the string itself should be 15 characters long. If you change -** the header, then your custom library will not be able to read +** the header, then your custom library will not be able to read ** databases generated by the standard tools and the standard tools ** will not be able to read databases created by your custom library. */ @@ -56991,58 +69888,53 @@ typedef struct CellInfo CellInfo; #define PTF_LEAF 0x08 /* -** As each page of the file is loaded into memory, an instance of the following -** structure is appended and initialized to zero. This structure stores -** information about the page that is decoded from the raw file page. +** An instance of this object stores information about each a single database +** page that has been loaded into memory. The information in this object +** is derived from the raw on-disk page content. ** -** The pParent field points back to the parent page. This allows us to -** walk up the BTree from any leaf to the root. Care must be taken to -** unref() the parent page pointer when this page is no longer referenced. -** The pageDestructor() routine handles that chore. +** As each database page is loaded into memory, the pager allocates an +** instance of this object and zeros the first 8 bytes. (This is the +** "extra" information associated with each page of the pager.) ** ** Access to all fields of this structure is controlled by the mutex ** stored in MemPage.pBt->mutex. */ struct MemPage { u8 isInit; /* True if previously initialized. MUST BE FIRST! */ - u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u8 intKey; /* True if table b-trees. False for index b-trees */ u8 intKeyLeaf; /* True if the leaf of an intKey table */ + Pgno pgno; /* Page number for this page */ + /* Only the first 8 bytes (above) are zeroed by pager.c when a new page + ** is allocated. All fields that follow must be initialized before use */ u8 leaf; /* True if a leaf page */ u8 hdrOffset; /* 100 for page 1. 0 otherwise */ u8 childPtrSize; /* 0 if leaf==1. 4 if leaf==0 */ u8 max1bytePayload; /* min(maxLocal,127) */ - u8 bBusy; /* Prevent endless loops on corrupt database files */ + u8 nOverflow; /* Number of overflow cell bodies in aCell[] */ u16 maxLocal; /* Copy of BtShared.maxLocal or BtShared.maxLeaf */ u16 minLocal; /* Copy of BtShared.minLocal or BtShared.minLeaf */ u16 cellOffset; /* Index in aData of first cell pointer */ - u16 nFree; /* Number of free bytes on the page */ + int nFree; /* Number of free bytes on the page. -1 for unknown */ u16 nCell; /* Number of cells on this page, local and ovfl */ u16 maskPage; /* Mask for page offset */ - u16 aiOvfl[5]; /* Insert the i-th overflow cell before the aiOvfl-th + u16 aiOvfl[4]; /* Insert the i-th overflow cell before the aiOvfl-th ** non-overflow cell */ - u8 *apOvfl[5]; /* Pointers to the body of overflow cells */ + u8 *apOvfl[4]; /* Pointers to the body of overflow cells */ BtShared *pBt; /* Pointer to BtShared that this page is part of */ u8 *aData; /* Pointer to disk image of the page data */ - u8 *aDataEnd; /* One byte past the end of usable data */ + u8 *aDataEnd; /* One byte past the end of the entire page - not just + ** the usable space, the entire page. Used to prevent + ** corruption-induced buffer overflow. */ u8 *aCellIdx; /* The cell index area */ u8 *aDataOfst; /* Same as aData for leaves. aData+4 for interior */ DbPage *pDbPage; /* Pager page handle */ u16 (*xCellSize)(MemPage*,u8*); /* cellSizePtr method */ void (*xParseCell)(MemPage*,u8*,CellInfo*); /* btreeParseCell method */ - Pgno pgno; /* Page number for this page */ }; -/* -** The in-memory image of a disk page has the auxiliary information appended -** to the end. EXTRA_SIZE is the number of bytes of space needed to hold -** that extra information. -*/ -#define EXTRA_SIZE sizeof(MemPage) - /* ** A linked list of the following structures is stored at BtShared.pLock. -** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor +** Locks are added (or upgraded from READ_LOCK to WRITE_LOCK) when a cursor ** is opened on the table with root page BtShared.iTable. Locks are removed ** from this list when a transaction is committed or rolled back, or when ** a btree handle is closed. @@ -57066,7 +69958,7 @@ struct BtLock { ** see the internals of this structure and only deals with pointers to ** this structure. ** -** For some database files, the same underlying database cache might be +** For some database files, the same underlying database cache might be ** shared between multiple connections. In that case, each connection ** has it own instance of this object. But each instance of this object ** points to the same BtShared object. The database cache and the @@ -57074,7 +69966,7 @@ struct BtLock { ** the BtShared object. ** ** All fields in this structure are accessed under sqlite3.mutex. -** The pBt pointer itself may not be changed while there exists cursors +** The pBt pointer itself may not be changed while there exists cursors ** in the referenced BtShared that point back to this Btree since those ** cursors have to go through this Btree to find their BtShared and ** they often do so without holding sqlite3.mutex. @@ -57088,9 +69980,12 @@ struct Btree { u8 hasIncrblobCur; /* True if there are one or more Incrblob cursors */ int wantToLock; /* Number of nested calls to sqlite3BtreeEnter() */ int nBackup; /* Number of backup operations reading this btree */ - u32 iDataVersion; /* Combines with pBt->pPager->iDataVersion */ + u32 iBDataVersion; /* Combines with pBt->pPager->iDataVersion */ Btree *pNext; /* List of other sharable Btrees from the same db */ Btree *pPrev; /* Back pointer of the same list */ +#ifdef SQLITE_DEBUG + u64 nSeek; /* Calls to sqlite3BtreeMovetoUnpacked() */ +#endif #ifndef SQLITE_OMIT_SHARED_CACHE BtLock lock; /* Object used to lock page 1 */ #endif @@ -57102,14 +69997,28 @@ struct Btree { ** If the shared-data extension is enabled, there may be multiple users ** of the Btree structure. At most one of these may open a write transaction, ** but any number may have active read transactions. +** +** These values must match SQLITE_TXN_NONE, SQLITE_TXN_READ, and +** SQLITE_TXN_WRITE */ #define TRANS_NONE 0 #define TRANS_READ 1 #define TRANS_WRITE 2 +#if TRANS_NONE!=SQLITE_TXN_NONE +# error wrong numeric code for no-transaction +#endif +#if TRANS_READ!=SQLITE_TXN_READ +# error wrong numeric code for read-transaction +#endif +#if TRANS_WRITE!=SQLITE_TXN_WRITE +# error wrong numeric code for write-transaction +#endif + + /* ** An instance of this object represents a single database file. -** +** ** A single database file can be in use at the same time by two ** or more database connections. When two or more connections are ** sharing the same database file, each connection has it own @@ -57119,7 +70028,7 @@ struct Btree { ** ** Fields in this structure are accessed under the BtShared.mutex ** mutex, except for nRef and pNext which are accessed under the -** global SQLITE_MUTEX_STATIC_MASTER mutex. The pPager field +** global SQLITE_MUTEX_STATIC_MAIN mutex. The pPager field ** may not be modified once it is initially set as long as nRef>0. ** The pSchema field may be set once under BtShared.mutex and ** thereafter is unchanged as long as nRef>0. @@ -57155,9 +70064,7 @@ struct BtShared { #endif u8 inTransaction; /* Transaction state */ u8 max1bytePayload; /* Maximum first byte of cell for a 1-byte payload */ -#ifdef SQLITE_HAS_CODEC - u8 optimalReserve; /* Desired amount of reserved space per page */ -#endif + u8 nReserveWanted; /* Desired number of extra bytes per page */ u16 btsFlags; /* Boolean parameters. See BTS_* macros below */ u16 maxLocal; /* Maximum local payload in non-LEAFDATA tables */ u16 minLocal; /* Minimum local payload in non-LEAFDATA tables */ @@ -57178,6 +70085,7 @@ struct BtShared { Btree *pWriter; /* Btree with currently open write transaction */ #endif u8 *pTmpSpace; /* Temp space sufficient to hold a single cell */ + int nPreformatSize; /* Size of last cell written by TransferRow() */ }; /* @@ -57186,10 +70094,12 @@ struct BtShared { #define BTS_READ_ONLY 0x0001 /* Underlying file is readonly */ #define BTS_PAGESIZE_FIXED 0x0002 /* Page size can no longer be changed */ #define BTS_SECURE_DELETE 0x0004 /* PRAGMA secure_delete is enabled */ -#define BTS_INITIALLY_EMPTY 0x0008 /* Database was empty at trans start */ -#define BTS_NO_WAL 0x0010 /* Do not open write-ahead-log files */ -#define BTS_EXCLUSIVE 0x0020 /* pWriter has an exclusive lock */ -#define BTS_PENDING 0x0040 /* Waiting for read-locks to clear */ +#define BTS_OVERWRITE 0x0008 /* Overwrite deleted content with zeros */ +#define BTS_FAST_SECURE 0x000c /* Combination of the previous two */ +#define BTS_INITIALLY_EMPTY 0x0010 /* Database was empty at trans start */ +#define BTS_NO_WAL 0x0020 /* Do not open write-ahead-log files */ +#define BTS_EXCLUSIVE 0x0040 /* pWriter has an exclusive lock */ +#define BTS_PENDING 0x0080 /* Waiting for read-locks to clear */ /* ** An instance of the following structure is used to hold information @@ -57227,38 +70137,46 @@ struct CellInfo { ** particular database connection identified BtCursor.pBtree.db. ** ** Fields in this structure are accessed under the BtShared.mutex -** found at self->pBt->mutex. +** found at self->pBt->mutex. ** ** skipNext meaning: -** eState==SKIPNEXT && skipNext>0: Next sqlite3BtreeNext() is no-op. -** eState==SKIPNEXT && skipNext<0: Next sqlite3BtreePrevious() is no-op. -** eState==FAULT: Cursor fault with skipNext as error code. +** The meaning of skipNext depends on the value of eState: +** +** eState Meaning of skipNext +** VALID skipNext is meaningless and is ignored +** INVALID skipNext is meaningless and is ignored +** SKIPNEXT sqlite3BtreeNext() is a no-op if skipNext>0 and +** sqlite3BtreePrevious() is no-op if skipNext<0. +** REQUIRESEEK restoreCursorPosition() restores the cursor to +** eState=SKIPNEXT if skipNext!=0 +** FAULT skipNext holds the cursor fault error code. */ struct BtCursor { - Btree *pBtree; /* The Btree to which this cursor belongs */ - BtShared *pBt; /* The BtShared this cursor points to */ - BtCursor *pNext; /* Forms a linked list of all cursors */ - Pgno *aOverflow; /* Cache of overflow page locations */ - CellInfo info; /* A parse of the cell we are pointing at */ - i64 nKey; /* Size of pKey, or last integer key */ - void *pKey; /* Saved key that was cursor last known position */ - Pgno pgnoRoot; /* The root page of this tree */ - int nOvflAlloc; /* Allocated size of aOverflow[] array */ - int skipNext; /* Prev() is noop if negative. Next() is noop if positive. - ** Error code if eState==CURSOR_FAULT */ + u8 eState; /* One of the CURSOR_XXX constants (see below) */ u8 curFlags; /* zero or more BTCF_* flags defined below */ u8 curPagerFlags; /* Flags to send to sqlite3PagerGet() */ - u8 eState; /* One of the CURSOR_XXX constants (see below) */ u8 hints; /* As configured by CursorSetHints() */ + int skipNext; /* Prev() is noop if negative. Next() is noop if positive. + ** Error code if eState==CURSOR_FAULT */ + Btree *pBtree; /* The Btree to which this cursor belongs */ + Pgno *aOverflow; /* Cache of overflow page locations */ + void *pKey; /* Saved key that was cursor last known position */ /* All fields above are zeroed when the cursor is allocated. See ** sqlite3BtreeCursorZero(). Fields that follow must be manually ** initialized. */ +#define BTCURSOR_FIRST_UNINIT pBt /* Name of first uninitialized field */ + BtShared *pBt; /* The BtShared this cursor points to */ + BtCursor *pNext; /* Forms a linked list of all cursors */ + CellInfo info; /* A parse of the cell we are pointing at */ + i64 nKey; /* Size of pKey, or last integer key */ + Pgno pgnoRoot; /* The root page of this tree */ i8 iPage; /* Index of current page in apPage */ u8 curIntKey; /* Value of apPage[0]->intKey */ - struct KeyInfo *pKeyInfo; /* Argument passed to comparison function */ - void *padding1; /* Make object size a multiple of 16 */ - u16 aiIdx[BTCURSOR_MAX_DEPTH]; /* Current index in apPage[i] */ - MemPage *apPage[BTCURSOR_MAX_DEPTH]; /* Pages from root to current page */ + u16 ix; /* Current index for apPage[iPage] */ + u16 aiIdx[BTCURSOR_MAX_DEPTH-1]; /* Current index in apPage[i] */ + struct KeyInfo *pKeyInfo; /* Arg passed to comparison function */ + MemPage *pPage; /* Current page */ + MemPage *apPage[BTCURSOR_MAX_DEPTH-1]; /* Stack of parents of current page */ }; /* @@ -57267,15 +70185,16 @@ struct BtCursor { #define BTCF_WriteFlag 0x01 /* True if a write cursor */ #define BTCF_ValidNKey 0x02 /* True if info.nKey is valid */ #define BTCF_ValidOvfl 0x04 /* True if aOverflow is valid */ -#define BTCF_AtLast 0x08 /* Cursor is pointing ot the last entry */ +#define BTCF_AtLast 0x08 /* Cursor is pointing to the last entry */ #define BTCF_Incrblob 0x10 /* True if an incremental I/O handle */ #define BTCF_Multiple 0x20 /* Maybe another cursor on the same btree */ +#define BTCF_Pinned 0x40 /* Cursor is busy and cannot be moved */ /* ** Potential values for BtCursor.eState. ** ** CURSOR_INVALID: -** Cursor does not point to a valid entry. This can happen (for example) +** Cursor does not point to a valid entry. This can happen (for example) ** because the table is empty or because BtreeCursorFirst() has not been ** called. ** @@ -57288,9 +70207,9 @@ struct BtCursor { ** operation should be a no-op. ** ** CURSOR_REQUIRESEEK: -** The table that this cursor was opened on still exists, but has been +** The table that this cursor was opened on still exists, but has been ** modified since the cursor was last used. The cursor position is saved -** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in +** in variables BtCursor.pKey and BtCursor.nKey. When a cursor is in ** this state, restoreCursorPosition() can be called to attempt to ** seek the cursor to the saved position. ** @@ -57301,19 +70220,19 @@ struct BtCursor { ** Do nothing else with this cursor. Any attempt to use the cursor ** should return the error code stored in BtCursor.skipNext */ -#define CURSOR_INVALID 0 -#define CURSOR_VALID 1 +#define CURSOR_VALID 0 +#define CURSOR_INVALID 1 #define CURSOR_SKIPNEXT 2 #define CURSOR_REQUIRESEEK 3 #define CURSOR_FAULT 4 -/* +/* ** The database page the PENDING_BYTE occupies. This page is never used. */ -# define PENDING_BYTE_PAGE(pBt) PAGER_MJ_PGNO(pBt) +#define PENDING_BYTE_PAGE(pBt) ((Pgno)((PENDING_BYTE/((pBt)->pageSize))+1)) /* -** These macros define the location of the pointer-map entry for a +** These macros define the location of the pointer-map entry for a ** database page. The first argument to each is the number of usable ** bytes on each page of the database (often 1024). The second is the ** page number to look up in the pointer map. @@ -57348,10 +70267,10 @@ struct BtCursor { ** PTRMAP_ROOTPAGE: The database page is a root-page. The page-number is not ** used in this case. ** -** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number +** PTRMAP_FREEPAGE: The database page is an unused (free) page. The page-number ** is not used in this case. ** -** PTRMAP_OVERFLOW1: The database page is the first page in a list of +** PTRMAP_OVERFLOW1: The database page is the first page in a list of ** overflow pages. The page number identifies the page that ** contains the cell with a pointer to this overflow page. ** @@ -57373,31 +70292,31 @@ struct BtCursor { */ #define btreeIntegrity(p) \ assert( p->pBt->inTransaction!=TRANS_NONE || p->pBt->nTransaction==0 ); \ - assert( p->pBt->inTransaction>=p->inTrans ); + assert( p->pBt->inTransaction>=p->inTrans ); /* ** The ISAUTOVACUUM macro is used within balance_nonroot() to determine ** if the database supports auto-vacuum or not. Because it is used -** within an expression that is an argument to another macro +** within an expression that is an argument to another macro ** (sqliteMallocRaw), it is not possible to use conditional compilation. ** So, this macro is defined instead. */ #ifndef SQLITE_OMIT_AUTOVACUUM -#define ISAUTOVACUUM (pBt->autoVacuum) +#define ISAUTOVACUUM(pBt) (pBt->autoVacuum) #else -#define ISAUTOVACUUM 0 +#define ISAUTOVACUUM(pBt) 0 #endif /* -** This structure is passed around through all the sanity checking routines -** in order to keep track of some global state information. +** This structure is passed around through all the PRAGMA integrity_check +** checking routines in order to keep track of some global state information. ** ** The aRef[] array is allocated so that there is 1 bit for each page in ** the database. As the integrity-check proceeds, for each page used in -** the database the corresponding bit is set. This allows integrity-check to -** detect pages that are used twice and orphaned pages (both of which +** the database the corresponding bit is set. This allows integrity-check to +** detect pages that are used twice and orphaned pages (both of which ** indicate corruption). */ typedef struct IntegrityCk IntegrityCk; @@ -57405,14 +70324,19 @@ struct IntegrityCk { BtShared *pBt; /* The tree being checked out */ Pager *pPager; /* The associated pager. Also accessible by pBt->pPager */ u8 *aPgRef; /* 1 bit per page in the db (see above) */ - Pgno nPage; /* Number of pages in the database */ + Pgno nCkPage; /* Pages in the database. 0 for partial check */ int mxErr; /* Stop accumulating errors when this reaches zero */ int nErr; /* Number of messages written to zErrMsg so far */ - int mallocFailed; /* A memory allocation error has occurred */ + int rc; /* SQLITE_OK, SQLITE_NOMEM, or SQLITE_INTERRUPT */ + u32 nStep; /* Number of steps into the integrity_check process */ const char *zPfx; /* Error message prefix */ - int v1, v2; /* Values for up to two %d fields in zPfx */ + Pgno v0; /* Value for first %u substitution in zPfx (root page) */ + Pgno v1; /* Value for second %u substitution in zPfx (current pg) */ + int v2; /* Value for third %d substitution in zPfx */ StrAccum errMsg; /* Accumulate the error message text here */ u32 *heap; /* Min-heap used for analyzing cell coverage */ + sqlite3 *db; /* Database connection running the check */ + i64 nRow; /* Number of rows visited in current tree */ }; /* @@ -57425,16 +70349,14 @@ struct IntegrityCk { /* ** get2byteAligned(), unlike get2byte(), requires that its argument point to a -** two-byte aligned address. get2bytea() is only used for accessing the +** two-byte aligned address. get2byteAligned() is only used for accessing the ** cell addresses in a btree header. */ #if SQLITE_BYTEORDER==4321 # define get2byteAligned(x) (*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && GCC_VERSION>=4008000 +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4008000 # define get2byteAligned(x) __builtin_bswap16(*(u16*)(x)) -#elif SQLITE_BYTEORDER==1234 && !defined(SQLITE_DISABLE_INTRINSIC) \ - && defined(_MSC_VER) && _MSC_VER>=1300 +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 # define get2byteAligned(x) _byteswap_ushort(*(u16*)(x)) #else # define get2byteAligned(x) ((x)[0]<<8 | (x)[1]) @@ -57604,21 +70526,29 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsMutex(Btree *p){ ** ** There is a corresponding leave-all procedures. ** -** Enter the mutexes in accending order by BtShared pointer address +** Enter the mutexes in ascending order by BtShared pointer address ** to avoid the possibility of deadlock when two threads with ** two or more btrees in common both try to lock all their btrees ** at the same instant. */ -SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ +static void SQLITE_NOINLINE btreeEnterAll(sqlite3 *db){ int i; + int skipOk = 1; Btree *p; assert( sqlite3_mutex_held(db->mutex) ); for(i=0; inDb; i++){ p = db->aDb[i].pBt; - if( p ) sqlite3BtreeEnter(p); + if( p && p->sharable ){ + sqlite3BtreeEnter(p); + skipOk = 0; + } } + db->noSharedCache = skipOk; } -SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ + if( db->noSharedCache==0 ) btreeEnterAll(db); +} +static void SQLITE_NOINLINE btreeLeaveAll(sqlite3 *db){ int i; Btree *p; assert( sqlite3_mutex_held(db->mutex) ); @@ -57627,6 +70557,9 @@ SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ if( p ) sqlite3BtreeLeave(p); } } +SQLITE_PRIVATE void sqlite3BtreeLeaveAll(sqlite3 *db){ + if( db->noSharedCache==0 ) btreeLeaveAll(db); +} #ifndef NDEBUG /* @@ -57667,6 +70600,7 @@ SQLITE_PRIVATE int sqlite3BtreeHoldsAllMutexes(sqlite3 *db){ SQLITE_PRIVATE int sqlite3SchemaMutexHeld(sqlite3 *db, int iDb, Schema *pSchema){ Btree *p; assert( db!=0 ); + if( db->pVfs==0 && db->nDb==0 ) return 1; if( pSchema ) iDb = sqlite3SchemaToIndex(db, pSchema); assert( iDb>=0 && iDbnDb ); if( !sqlite3_mutex_held(db->mutex) ) return 0; @@ -57704,10 +70638,10 @@ SQLITE_PRIVATE void sqlite3BtreeEnterAll(sqlite3 *db){ #ifndef SQLITE_OMIT_INCRBLOB /* -** Enter a mutex on a Btree given a cursor owned by that Btree. +** Enter a mutex on a Btree given a cursor owned by that Btree. ** -** These entry points are used by incremental I/O only. Enter() is required -** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not +** These entry points are used by incremental I/O only. Enter() is required +** any time OMIT_SHARED_CACHE is not defined, regardless of whether or not ** the build is threadsafe. Leave() is only required by threadsafe builds. */ SQLITE_PRIVATE void sqlite3BtreeEnterCursor(BtCursor *pCur){ @@ -57777,7 +70711,7 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ #define BTALLOC_LE 2 /* Allocate any page <= the parameter */ /* -** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not +** Macro IfNotOmitAV(x) returns (x) if SQLITE_OMIT_AUTOVACUUM is not ** defined, or 0 if it is. For example: ** ** bIncrVacuum = IfNotOmitAV(pBtShared->incrVacuum); @@ -57792,10 +70726,10 @@ int sqlite3BtreeTrace=1; /* True to enable tracing */ /* ** A list of BtShared objects that are eligible for participation ** in shared cache. This variable has file scope during normal builds, -** but the test harness needs to access it so we make it global for +** but the test harness needs to access it so we make it global for ** test builds. ** -** Access to this variable is protected by SQLITE_MUTEX_STATIC_MASTER. +** Access to this variable is protected by SQLITE_MUTEX_STATIC_MAIN. */ #ifdef SQLITE_TEST SQLITE_PRIVATE BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; @@ -57812,7 +70746,7 @@ static BtShared *SQLITE_WSD sqlite3SharedCacheList = 0; ** The shared cache setting effects only future calls to ** sqlite3_open(), sqlite3_open16(), or sqlite3_open_v2(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int enable){ +SQLITE_API int sqlite3_enable_shared_cache(int enable){ sqlite3GlobalConfig.sharedCacheEnabled = enable; return SQLITE_OK; } @@ -57827,7 +70761,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int enable){ ** manipulate entries in the BtShared.pLock linked list used to store ** shared-cache table level locks. If the library is compiled with the ** shared-cache feature disabled, then there is only ever one user - ** of each BtShared structure and so this locking is not necessary. + ** of each BtShared structure and so this locking is not necessary. ** So define the lock related functions as no-ops. */ #define querySharedCacheTableLock(a,b,c) SQLITE_OK @@ -57838,21 +70772,99 @@ SQLITE_API int SQLITE_STDCALL sqlite3_enable_shared_cache(int enable){ #define hasReadConflicts(a, b) 0 #endif +#ifdef SQLITE_DEBUG +/* +** Return and reset the seek counter for a Btree object. +*/ +SQLITE_PRIVATE sqlite3_uint64 sqlite3BtreeSeekCount(Btree *pBt){ + u64 n = pBt->nSeek; + pBt->nSeek = 0; + return n; +} +#endif + +/* +** Implementation of the SQLITE_CORRUPT_PAGE() macro. Takes a single +** (MemPage*) as an argument. The (MemPage*) must not be NULL. +** +** If SQLITE_DEBUG is not defined, then this macro is equivalent to +** SQLITE_CORRUPT_BKPT. Or, if SQLITE_DEBUG is set, then the log message +** normally produced as a side-effect of SQLITE_CORRUPT_BKPT is augmented +** with the page number and filename associated with the (MemPage*). +*/ +#ifdef SQLITE_DEBUG +int corruptPageError(int lineno, MemPage *p){ + char *zMsg; + sqlite3BeginBenignMalloc(); + zMsg = sqlite3_mprintf("database corruption page %u of %s", + p->pgno, sqlite3PagerFilename(p->pBt->pPager, 0) + ); + sqlite3EndBenignMalloc(); + if( zMsg ){ + sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); + } + sqlite3_free(zMsg); + return SQLITE_CORRUPT_BKPT; +} +# define SQLITE_CORRUPT_PAGE(pMemPage) corruptPageError(__LINE__, pMemPage) +#else +# define SQLITE_CORRUPT_PAGE(pMemPage) SQLITE_CORRUPT_PGNO(pMemPage->pgno) +#endif + +/* Default value for SHARED_LOCK_TRACE macro if shared-cache is disabled +** or if the lock tracking is disabled. This is always the value for +** release builds. +*/ +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) /*no-op*/ + #ifndef SQLITE_OMIT_SHARED_CACHE +#if 0 +/* ^---- Change to 1 and recompile to enable shared-lock tracing +** for debugging purposes. +** +** Print all shared-cache locks on a BtShared. Debugging use only. +*/ +static void sharedLockTrace( + BtShared *pBt, + const char *zMsg, + int iRoot, + int eLockType +){ + BtLock *pLock; + if( iRoot>0 ){ + printf("%s-%p %u%s:", zMsg, pBt, iRoot, eLockType==READ_LOCK?"R":"W"); + }else{ + printf("%s-%p:", zMsg, pBt); + } + for(pLock=pBt->pLock; pLock; pLock=pLock->pNext){ + printf(" %p/%u%s", pLock->pBtree, pLock->iTable, + pLock->eLock==READ_LOCK ? "R" : "W"); + while( pLock->pNext && pLock->pBtree==pLock->pNext->pBtree ){ + pLock = pLock->pNext; + printf(",%u%s", pLock->iTable, pLock->eLock==READ_LOCK ? "R" : "W"); + } + } + printf("\n"); + fflush(stdout); +} +#undef SHARED_LOCK_TRACE +#define SHARED_LOCK_TRACE(X,MSG,TAB,TYPE) sharedLockTrace(X,MSG,TAB,TYPE) +#endif /* Shared-lock tracing */ + #ifdef SQLITE_DEBUG /* **** This function is only used as part of an assert() statement. *** ** -** Check to see if pBtree holds the required locks to read or write to the +** Check to see if pBtree holds the required locks to read or write to the ** table with root page iRoot. Return 1 if it does and 0 if not. ** -** For example, when writing to a table with root-page iRoot via +** For example, when writing to a table with root-page iRoot via ** Btree connection pBtree: ** ** assert( hasSharedCacheTableLock(pBtree, iRoot, 0, WRITE_LOCK) ); ** -** When writing to an index that resides in a sharable database, the +** When writing to an index that resides in a sharable database, the ** caller should have first obtained a lock specifying the root page of ** the corresponding table. This makes things a bit more complicated, ** as this module treats each table as a separate structure. To determine @@ -57874,11 +70886,11 @@ static int hasSharedCacheTableLock( BtLock *pLock; /* If this database is not shareable, or if the client is reading - ** and has the read-uncommitted flag set, then no lock is required. + ** and has the read-uncommitted flag set, then no lock is required. ** Return true immediately. */ if( (pBtree->sharable==0) - || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommitted)) + || (eLockType==READ_LOCK && (pBtree->db->flags & SQLITE_ReadUncommit)) ){ return 1; } @@ -57898,29 +70910,33 @@ static int hasSharedCacheTableLock( ** table. */ if( isIndex ){ HashElem *p; + int bSeen = 0; for(p=sqliteHashFirst(&pSchema->idxHash); p; p=sqliteHashNext(p)){ Index *pIdx = (Index *)sqliteHashData(p); - if( pIdx->tnum==(int)iRoot ){ - if( iTab ){ + if( pIdx->tnum==iRoot ){ + if( bSeen ){ /* Two or more indexes share the same root page. There must ** be imposter tables. So just return true. The assert is not ** useful in that case. */ return 1; } iTab = pIdx->pTable->tnum; + bSeen = 1; } } }else{ iTab = iRoot; } - /* Search for the required lock. Either a write-lock on root-page iTab, a + SHARED_LOCK_TRACE(pBtree->pBt,"hasLock",iRoot,eLockType); + + /* Search for the required lock. Either a write-lock on root-page iTab, a ** write-lock on the schema table, or (if the client is reading) a ** read-lock on iTab will suffice. Return 1 if any of these are found. */ for(pLock=pBtree->pBt->pLock; pLock; pLock=pLock->pNext){ - if( pLock->pBtree==pBtree + if( pLock->pBtree==pBtree && (pLock->iTable==iTab || (pLock->eLock==WRITE_LOCK && pLock->iTable==1)) - && pLock->eLock>=eLockType + && pLock->eLock>=eLockType ){ return 1; } @@ -57953,9 +70969,9 @@ static int hasSharedCacheTableLock( static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ BtCursor *p; for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - if( p->pgnoRoot==iRoot + if( p->pgnoRoot==iRoot && p->pBtree!=pBtree - && 0==(p->pBtree->db->flags & SQLITE_ReadUncommitted) + && 0==(p->pBtree->db->flags & SQLITE_ReadUncommit) ){ return 1; } @@ -57965,7 +70981,7 @@ static int hasReadConflicts(Btree *pBtree, Pgno iRoot){ #endif /* #ifdef SQLITE_DEBUG */ /* -** Query to see if Btree handle p may obtain a lock of type eLock +** Query to see if Btree handle p may obtain a lock of type eLock ** (READ_LOCK or WRITE_LOCK) on the table with root-page iTab. Return ** SQLITE_OK if the lock may be obtained (by calling ** setSharedCacheTableLock()), or SQLITE_LOCKED if not. @@ -57977,15 +70993,15 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); - assert( !(p->db->flags&SQLITE_ReadUncommitted)||eLock==WRITE_LOCK||iTab==1 ); - + assert( !(p->db->flags&SQLITE_ReadUncommit)||eLock==WRITE_LOCK||iTab==1 ); + /* If requesting a write-lock, then the Btree must have an open write - ** transaction on this file. And, obviously, for this to be so there + ** transaction on this file. And, obviously, for this to be so there ** must be an open write transaction on the file itself. */ assert( eLock==READ_LOCK || (p==pBt->pWriter && p->inTrans==TRANS_WRITE) ); assert( eLock==READ_LOCK || pBt->inTransaction==TRANS_WRITE ); - + /* This routine is a no-op if the shared-cache is not enabled */ if( !p->sharable ){ return SQLITE_OK; @@ -58000,7 +71016,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ } for(pIter=pBt->pLock; pIter; pIter=pIter->pNext){ - /* The condition (pIter->eLock!=eLock) in the following if(...) + /* The condition (pIter->eLock!=eLock) in the following if(...) ** statement is a simplification of: ** ** (eLock==WRITE_LOCK || pIter->eLock==WRITE_LOCK) @@ -58027,7 +71043,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ #ifndef SQLITE_OMIT_SHARED_CACHE /* ** Add a lock on the table with root-page iTable to the shared-btree used -** by Btree handle p. Parameter eLock must be either READ_LOCK or +** by Btree handle p. Parameter eLock must be either READ_LOCK or ** WRITE_LOCK. ** ** This function assumes the following: @@ -58039,7 +71055,7 @@ static int querySharedCacheTableLock(Btree *p, Pgno iTab, u8 eLock){ ** with the requested lock (i.e. querySharedCacheTableLock() has ** already been called and returned SQLITE_OK). ** -** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM +** SQLITE_OK is returned if the lock is added successfully. SQLITE_NOMEM ** is returned if a malloc attempt fails. */ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ @@ -58047,17 +71063,19 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ BtLock *pLock = 0; BtLock *pIter; + SHARED_LOCK_TRACE(pBt,"setLock", iTable, eLock); + assert( sqlite3BtreeHoldsMutex(p) ); assert( eLock==READ_LOCK || eLock==WRITE_LOCK ); assert( p->db!=0 ); /* A connection with the read-uncommitted flag set will never try to ** obtain a read-lock using this function. The only read-lock obtained - ** by a connection in read-uncommitted mode is on the sqlite_master + ** by a connection in read-uncommitted mode is on the sqlite_schema ** table, and that lock is obtained in BtreeBeginTrans(). */ - assert( 0==(p->db->flags&SQLITE_ReadUncommitted) || eLock==WRITE_LOCK ); + assert( 0==(p->db->flags&SQLITE_ReadUncommit) || eLock==WRITE_LOCK ); - /* This function should only be called on a sharable b-tree after it + /* This function should only be called on a sharable b-tree after it ** has been determined that no other b-tree holds a conflicting lock. */ assert( p->sharable ); assert( SQLITE_OK==querySharedCacheTableLock(p, iTable, eLock) ); @@ -58102,7 +71120,7 @@ static int setSharedCacheTableLock(Btree *p, Pgno iTable, u8 eLock){ ** Release all the table locks (locks obtained via calls to ** the setSharedCacheTableLock() procedure) held by Btree object p. ** -** This function assumes that Btree p has an open read or write +** This function assumes that Btree p has an open read or write ** transaction. If it does not, then the BTS_PENDING flag ** may be incorrectly cleared. */ @@ -58114,6 +71132,8 @@ static void clearAllSharedCacheTableLocks(Btree *p){ assert( p->sharable || 0==*ppIter ); assert( p->inTrans>0 ); + SHARED_LOCK_TRACE(pBt, "clearAllLocks", 0, 0); + while( *ppIter ){ BtLock *pLock = *ppIter; assert( (pBt->btsFlags & BTS_EXCLUSIVE)==0 || pBt->pWriter==pLock->pBtree ); @@ -58134,7 +71154,7 @@ static void clearAllSharedCacheTableLocks(Btree *p){ pBt->pWriter = 0; pBt->btsFlags &= ~(BTS_EXCLUSIVE|BTS_PENDING); }else if( pBt->nTransaction==2 ){ - /* This function is called when Btree p is concluding its + /* This function is called when Btree p is concluding its ** transaction. If there currently exists a writer, and p is not ** that writer, then the number of locks held by connections other ** than the writer must be about to drop to zero. In this case @@ -58152,6 +71172,9 @@ static void clearAllSharedCacheTableLocks(Btree *p){ */ static void downgradeAllSharedCacheTableLocks(Btree *p){ BtShared *pBt = p->pBt; + + SHARED_LOCK_TRACE(pBt, "downgradeLocks", 0, 0); + if( pBt->pWriter==p ){ BtLock *pLock; pBt->pWriter = 0; @@ -58165,7 +71188,9 @@ static void downgradeAllSharedCacheTableLocks(Btree *p){ #endif /* SQLITE_OMIT_SHARED_CACHE */ -static void releasePage(MemPage *pPage); /* Forward reference */ +static void releasePage(MemPage *pPage); /* Forward reference */ +static void releasePageOne(MemPage *pPage); /* Forward reference */ +static void releasePageNotNull(MemPage *pPage); /* Forward reference */ /* ***** This routine is used inside of assert() only **** @@ -58176,6 +71201,15 @@ static void releasePage(MemPage *pPage); /* Forward reference */ static int cursorHoldsMutex(BtCursor *p){ return sqlite3_mutex_held(p->pBt->mutex); } + +/* Verify that the cursor and the BtShared agree about what is the current +** database connetion. This is important in shared-cache mode. If the database +** connection pointers get out-of-sync, it is possible for routines like +** btreeInitPage() to reference an stale connection pointer that references a +** a connection that has already closed. This routine is used inside assert() +** statements only and for the purpose of double-checking that the btree code +** does keep the database connection pointers up-to-date. +*/ static int cursorOwnsBtShared(BtCursor *p){ assert( cursorHoldsMutex(p) ); return (p->pBtree->db==p->pBt->db); @@ -58216,17 +71250,18 @@ static void invalidateAllOverflowCache(BtShared *pBt){ */ static void invalidateIncrblobCursors( Btree *pBtree, /* The database file to check */ + Pgno pgnoRoot, /* The table that might be changing */ i64 iRow, /* The rowid that might be changing */ int isClearTable /* True if all rows are being deleted */ ){ BtCursor *p; - if( pBtree->hasIncrblobCur==0 ) return; + assert( pBtree->hasIncrblobCur ); assert( sqlite3BtreeHoldsMutex(pBtree) ); pBtree->hasIncrblobCur = 0; for(p=pBtree->pBt->pCursor; p; p=p->pNext){ if( (p->curFlags & BTCF_Incrblob)!=0 ){ pBtree->hasIncrblobCur = 1; - if( isClearTable || p->info.nKey==iRow ){ + if( p->pgnoRoot==pgnoRoot && (isClearTable || p->info.nKey==iRow) ){ p->eState = CURSOR_INVALID; } } @@ -58235,12 +71270,12 @@ static void invalidateIncrblobCursors( #else /* Stub function when INCRBLOB is omitted */ - #define invalidateIncrblobCursors(x,y,z) + #define invalidateIncrblobCursors(w,x,y,z) #endif /* SQLITE_OMIT_INCRBLOB */ /* -** Set bit pgno of the BtShared.pHasContent bitvec. This is called -** when a page that previously contained data becomes a free-list leaf +** Set bit pgno of the BtShared.pHasContent bitvec. This is called +** when a page that previously contained data becomes a free-list leaf ** page. ** ** The BtShared.pHasContent bitvec exists to work around an obscure @@ -58266,7 +71301,7 @@ static void invalidateIncrblobCursors( ** may be lost. In the event of a rollback, it may not be possible ** to restore the database to its original configuration. ** -** The solution is the BtShared.pHasContent bitvec. Whenever a page is +** The solution is the BtShared.pHasContent bitvec. Whenever a page is ** moved to become a free-list leaf page, the corresponding bit is ** set in the bitvec. Whenever a leaf page is extracted from the free-list, ** optimization 2 above is omitted if the corresponding bit is already @@ -58297,7 +71332,7 @@ static int btreeSetHasContent(BtShared *pBt, Pgno pgno){ */ static int btreeGetHasContent(BtShared *pBt, Pgno pgno){ Bitvec *p = pBt->pHasContent; - return (p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTest(p, pgno))); + return p && (pgno>sqlite3BitvecSize(p) || sqlite3BitvecTestNotNull(p, pgno)); } /* @@ -58314,45 +71349,51 @@ static void btreeClearHasContent(BtShared *pBt){ */ static void btreeReleaseAllCursorPages(BtCursor *pCur){ int i; - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - pCur->apPage[i] = 0; + if( pCur->iPage>=0 ){ + for(i=0; iiPage; i++){ + releasePageNotNull(pCur->apPage[i]); + } + releasePageNotNull(pCur->pPage); + pCur->iPage = -1; } - pCur->iPage = -1; } /* ** The cursor passed as the only argument must point to a valid entry ** when this function is called (i.e. have eState==CURSOR_VALID). This ** function saves the current cursor key in variables pCur->nKey and -** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error +** pCur->pKey. SQLITE_OK is returned if successful or an SQLite error ** code otherwise. ** ** If the cursor is open on an intkey table, then the integer key ** (the rowid) is stored in pCur->nKey and pCur->pKey is left set to -** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is -** set to point to a malloced buffer pCur->nKey bytes in size containing +** NULL. If the cursor is open on a non-intkey table, then pCur->pKey is +** set to point to a malloced buffer pCur->nKey bytes in size containing ** the key. */ static int saveCursorKey(BtCursor *pCur){ - int rc; + int rc = SQLITE_OK; assert( CURSOR_VALID==pCur->eState ); assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); - rc = sqlite3BtreeKeySize(pCur, &pCur->nKey); - assert( rc==SQLITE_OK ); /* KeySize() cannot fail */ - - /* If this is an intKey table, then the above call to BtreeKeySize() - ** stores the integer key in pCur->nKey. In this case this value is - ** all that is required. Otherwise, if pCur is not open on an intKey - ** table, then malloc space for and store the pCur->nKey bytes of key - ** data. */ - if( 0==pCur->curIntKey ){ - void *pKey = sqlite3Malloc( pCur->nKey ); + if( pCur->curIntKey ){ + /* Only the rowid is required for a table btree */ + pCur->nKey = sqlite3BtreeIntegerKey(pCur); + }else{ + /* For an index btree, save the complete key content. It is possible + ** that the current key is corrupt. In that case, it is possible that + ** the sqlite3VdbeRecordUnpack() function may overread the buffer by + ** up to the size of 1 varint plus 1 8-byte value when the cursor + ** position is restored. Hence the 17 bytes of padding allocated + ** below. */ + void *pKey; + pCur->nKey = sqlite3BtreePayloadSize(pCur); + pKey = sqlite3Malloc( pCur->nKey + 9 + 8 ); if( pKey ){ - rc = sqlite3BtreeKey(pCur, 0, (int)pCur->nKey, pKey); + rc = sqlite3BtreePayload(pCur, 0, (int)pCur->nKey, pKey); if( rc==SQLITE_OK ){ + memset(((u8*)pKey)+pCur->nKey, 0, 9+8); pCur->pKey = pKey; }else{ sqlite3_free(pKey); @@ -58366,11 +71407,11 @@ static int saveCursorKey(BtCursor *pCur){ } /* -** Save the current cursor position in the variables BtCursor.nKey +** Save the current cursor position in the variables BtCursor.nKey ** and BtCursor.pKey. The cursor's state is set to CURSOR_REQUIRESEEK. ** ** The caller must ensure that the cursor is valid (has eState==CURSOR_VALID) -** prior to calling this routine. +** prior to calling this routine. */ static int saveCursorPosition(BtCursor *pCur){ int rc; @@ -58379,6 +71420,9 @@ static int saveCursorPosition(BtCursor *pCur){ assert( 0==pCur->pKey ); assert( cursorHoldsMutex(pCur) ); + if( pCur->curFlags & BTCF_Pinned ){ + return SQLITE_CONSTRAINT_PINNED; + } if( pCur->eState==CURSOR_SKIPNEXT ){ pCur->eState = CURSOR_VALID; }else{ @@ -58406,7 +71450,7 @@ static int SQLITE_NOINLINE saveCursorsOnList(BtCursor*,Pgno,BtCursor*); ** routine is called just before cursor pExcept is used to modify the ** table, for example in BtreeDelete() or BtreeInsert(). ** -** If there are two or more cursors on the same btree, then all such +** If there are two or more cursors on the same btree, then all such ** cursors should have their BTCF_Multiple flag set. The btreeCursor() ** routine enforces that rule. This routine only needs to be called in ** the uncommon case when pExpect has the BTCF_Multiple flag set. @@ -58449,7 +71493,7 @@ static int SQLITE_NOINLINE saveCursorsOnList( return rc; } }else{ - testcase( p->iPage>0 ); + testcase( p->iPage>=0 ); btreeReleaseAllCursorPages(p); } } @@ -58471,7 +71515,7 @@ SQLITE_PRIVATE void sqlite3BtreeClearCursor(BtCursor *pCur){ /* ** In this version of BtreeMoveto, pKey is a packed index record ** such as is generated by the OP_MakeRecord opcode. Unpack the -** record and then call BtreeMovetoUnpacked() to do the work. +** record and then call sqlite3BtreeIndexMoveto() to do the work. */ static int btreeMoveto( BtCursor *pCur, /* Cursor open on the btree to be searched */ @@ -58482,52 +71526,52 @@ static int btreeMoveto( ){ int rc; /* Status code */ UnpackedRecord *pIdxKey; /* Unpacked index key */ - char aSpace[200]; /* Temp space for pIdxKey - to avoid a malloc */ - char *pFree = 0; if( pKey ){ + KeyInfo *pKeyInfo = pCur->pKeyInfo; assert( nKey==(i64)(int)nKey ); - pIdxKey = sqlite3VdbeAllocUnpackedRecord( - pCur->pKeyInfo, aSpace, sizeof(aSpace), &pFree - ); + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pIdxKey==0 ) return SQLITE_NOMEM_BKPT; - sqlite3VdbeRecordUnpack(pCur->pKeyInfo, (int)nKey, pKey, pIdxKey); - if( pIdxKey->nField==0 ){ - sqlite3DbFree(pCur->pKeyInfo->db, pFree); - return SQLITE_CORRUPT_BKPT; + sqlite3VdbeRecordUnpack(pKeyInfo, (int)nKey, pKey, pIdxKey); + if( pIdxKey->nField==0 || pIdxKey->nField>pKeyInfo->nAllField ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = sqlite3BtreeIndexMoveto(pCur, pIdxKey, pRes); } + sqlite3DbFree(pCur->pKeyInfo->db, pIdxKey); }else{ pIdxKey = 0; - } - rc = sqlite3BtreeMovetoUnpacked(pCur, pIdxKey, nKey, bias, pRes); - if( pFree ){ - sqlite3DbFree(pCur->pKeyInfo->db, pFree); + rc = sqlite3BtreeTableMoveto(pCur, nKey, bias, pRes); } return rc; } /* ** Restore the cursor to the position it was in (or as close to as possible) -** when saveCursorPosition() was called. Note that this call deletes the +** when saveCursorPosition() was called. Note that this call deletes the ** saved position info stored by saveCursorPosition(), so there can be -** at most one effective restoreCursorPosition() call after each +** at most one effective restoreCursorPosition() call after each ** saveCursorPosition(). */ static int btreeRestoreCursorPosition(BtCursor *pCur){ int rc; - int skipNext; + int skipNext = 0; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState>=CURSOR_REQUIRESEEK ); if( pCur->eState==CURSOR_FAULT ){ return pCur->skipNext; } pCur->eState = CURSOR_INVALID; - rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + if( sqlite3FaultSim(410) ){ + rc = SQLITE_IOERR; + }else{ + rc = btreeMoveto(pCur, pCur->pKey, pCur->nKey, 0, &skipNext); + } if( rc==SQLITE_OK ){ sqlite3_free(pCur->pKey); pCur->pKey = 0; assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_INVALID ); - pCur->skipNext |= skipNext; + if( skipNext ) pCur->skipNext = skipNext; if( pCur->skipNext && pCur->eState==CURSOR_VALID ){ pCur->eState = CURSOR_SKIPNEXT; } @@ -58553,13 +71597,28 @@ static int btreeRestoreCursorPosition(BtCursor *pCur){ ** back to where it ought to be if this routine returns true. */ SQLITE_PRIVATE int sqlite3BtreeCursorHasMoved(BtCursor *pCur){ - return pCur->eState!=CURSOR_VALID; + assert( EIGHT_BYTE_ALIGNMENT(pCur) + || pCur==sqlite3BtreeFakeValidCursor() ); + assert( offsetof(BtCursor, eState)==0 ); + assert( sizeof(pCur->eState)==1 ); + return CURSOR_VALID != *(u8*)pCur; +} + +/* +** Return a pointer to a fake BtCursor object that will always answer +** false to the sqlite3BtreeCursorHasMoved() routine above. The fake +** cursor returned must not be used with any other Btree interface. +*/ +SQLITE_PRIVATE BtCursor *sqlite3BtreeFakeValidCursor(void){ + static u8 fakeCursor = CURSOR_VALID; + assert( offsetof(BtCursor, eState)==0 ); + return (BtCursor*)&fakeCursor; } /* ** This routine restores a cursor back to its original position after it ** has been moved by some outside activity (such as a btree rebalance or -** a row having been deleted out from under the cursor). +** a row having been deleted out from under the cursor). ** ** On success, the *pDifferentRow parameter is false if the cursor is left ** pointing at exactly the same row. *pDifferntRow is the row the cursor @@ -58582,7 +71641,6 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) if( pCur->eState!=CURSOR_VALID ){ *pDifferentRow = 1; }else{ - assert( pCur->skipNext==0 ); *pDifferentRow = 0; } return SQLITE_OK; @@ -58596,8 +71654,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorRestore(BtCursor *pCur, int *pDifferentRow) */ SQLITE_PRIVATE void sqlite3BtreeCursorHint(BtCursor *pCur, int eHintType, ...){ /* Used only by system that substitute their own storage engine */ +#ifdef SQLITE_DEBUG + if( ALWAYS(eHintType==BTREE_HINT_RANGE) ){ + va_list ap; + Expr *pExpr; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3CursorRangeHintExprCheck; + va_start(ap, eHintType); + pExpr = va_arg(ap, Expr*); + w.u.aMem = va_arg(ap, Mem*); + va_end(ap); + assert( pExpr!=0 ); + assert( w.u.aMem!=0 ); + sqlite3WalkExpr(&w, pExpr); + } +#endif /* SQLITE_DEBUG */ } -#endif +#endif /* SQLITE_ENABLE_CURSOR_HINTS */ + /* ** Provide flag hints to the cursor. @@ -58625,7 +71700,7 @@ static Pgno ptrmapPageno(BtShared *pBt, Pgno pgno){ if( pgno<2 ) return 0; nPagesPerMapPage = (pBt->usableSize/5)+1; iPtrMap = (pgno-2)/nPagesPerMapPage; - ret = (iPtrMap*nPagesPerMapPage) + 2; + ret = (iPtrMap*nPagesPerMapPage) + 2; if( ret==PENDING_BYTE_PAGE(pBt) ){ ret++; } @@ -58652,7 +71727,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ if( *pRC ) return; assert( sqlite3_mutex_held(pBt->mutex) ); - /* The master-journal page number must never be used as a pointer map page */ + /* The super-journal page number must never be used as a pointer map page */ assert( 0==PTRMAP_ISPAGE(pBt, PENDING_BYTE_PAGE(pBt)) ); assert( pBt->autoVacuum ); @@ -58666,6 +71741,13 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ *pRC = rc; return; } + if( ((char*)sqlite3PagerGetExtra(pDbPage))[0]!=0 ){ + /* The first byte of the extra data is the MemPage.isInit byte. + ** If that byte is set, it means this page is also being used + ** as a btree page. */ + *pRC = SQLITE_CORRUPT_BKPT; + goto ptrmap_exit; + } offset = PTRMAP_PTROFFSET(iPtrmap, key); if( offset<0 ){ *pRC = SQLITE_CORRUPT_BKPT; @@ -58675,7 +71757,7 @@ static void ptrmapPut(BtShared *pBt, Pgno key, u8 eType, Pgno parent, int *pRC){ pPtrmap = (u8 *)sqlite3PagerGetData(pDbPage); if( eType!=pPtrmap[offset] || get4byte(&pPtrmap[offset+1])!=parent ){ - TRACE(("PTRMAP_UPDATE: %d->(%d,%d)\n", key, eType, parent)); + TRACE(("PTRMAP_UPDATE: %u->(%u,%u)\n", key, eType, parent)); *pRC= rc = sqlite3PagerWrite(pDbPage); if( rc==SQLITE_OK ){ pPtrmap[offset] = eType; @@ -58721,14 +71803,14 @@ static int ptrmapGet(BtShared *pBt, Pgno key, u8 *pEType, Pgno *pPgno){ if( pPgno ) *pPgno = get4byte(&pPtrmap[offset+1]); sqlite3PagerUnref(pDbPage); - if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_BKPT; + if( *pEType<1 || *pEType>5 ) return SQLITE_CORRUPT_PGNO(iPtrmap); return SQLITE_OK; } #else /* if defined SQLITE_OMIT_AUTOVACUUM */ #define ptrmapPut(w,x,y,z,rc) #define ptrmapGet(w,x,y,z) SQLITE_OK - #define ptrmapPutOvflPtr(x, y, rc) + #define ptrmapPutOvflPtr(x, y, z, rc) #endif /* @@ -58784,6 +71866,24 @@ static SQLITE_NOINLINE void btreeParseCellAdjustSizeForOverflow( pInfo->nSize = (u16)(&pInfo->pPayload[pInfo->nLocal] - pCell) + 4; } +/* +** Given a record with nPayload bytes of payload stored within btree +** page pPage, return the number of bytes of payload stored locally. +*/ +static int btreePayloadToLocal(MemPage *pPage, i64 nPayload){ + int maxLocal; /* Maximum amount of payload held locally */ + maxLocal = pPage->maxLocal; + if( nPayload<=maxLocal ){ + return nPayload; + }else{ + int minLocal; /* Minimum amount of payload held locally */ + int surplus; /* Overflow payload available for local storage */ + minLocal = pPage->minLocal; + surplus = minLocal + (nPayload - minLocal)%(pPage->pBt->usableSize-4); + return ( surplus <= maxLocal ) ? surplus : minLocal; + } +} + /* ** The following routines are implementations of the MemPage.xParseCell() ** method. @@ -58850,19 +71950,37 @@ static void btreeParseCellPtr( ** ** pIter += getVarint(pIter, (u64*)&pInfo->nKey); ** - ** The code is inlined to avoid a function call. + ** The code is inlined and the loop is unrolled for performance. + ** This routine is a high-runner. */ iKey = *pIter; if( iKey>=0x80 ){ - u8 *pEnd = &pIter[7]; - iKey &= 0x7f; - while(1){ - iKey = (iKey<<7) | (*++pIter & 0x7f); - if( (*pIter)<0x80 ) break; - if( pIter>=pEnd ){ - iKey = (iKey<<8) | *++pIter; - break; + u8 x; + iKey = (iKey<<7) ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x10204000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<7) ^ 0x4000 ^ (x = *++pIter); + if( x>=0x80 ){ + iKey = (iKey<<8) ^ 0x8000 ^ (*++pIter); + } + } + } + } + } + }else{ + iKey ^= 0x204000; } + }else{ + iKey ^= 0x4000; } } pIter++; @@ -58871,7 +71989,7 @@ static void btreeParseCellPtr( pInfo->nPayload = nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); - testcase( nPayload==pPage->maxLocal+1 ); + testcase( nPayload==(u32)pPage->maxLocal+1 ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. @@ -58908,7 +72026,7 @@ static void btreeParseCellPtrIndex( pInfo->nPayload = nPayload; pInfo->pPayload = pIter; testcase( nPayload==pPage->maxLocal ); - testcase( nPayload==pPage->maxLocal+1 ); + testcase( nPayload==(u32)pPage->maxLocal+1 ); if( nPayload<=pPage->maxLocal ){ /* This is the (easy) common case where the entire payload fits ** on the local page. No overflow is required. @@ -58938,10 +72056,12 @@ static void btreeParseCell( ** the space used by the cell pointer. ** ** cellSizePtrNoPayload() => table internal nodes -** cellSizePtr() => all index nodes & table leaf nodes +** cellSizePtrTableLeaf() => table leaf nodes +** cellSizePtr() => index internal nodes +** cellSizeIdxLeaf() => index leaf nodes */ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ - u8 *pIter = pCell + pPage->childPtrSize; /* For looping over bytes of pCell */ + u8 *pIter = pCell + 4; /* For looping over bytes of pCell */ u8 *pEnd; /* End mark for a varint */ u32 nSize; /* Size value to return */ @@ -58954,6 +72074,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ pPage->xParseCell(pPage, pCell, &debuginfo); #endif + assert( pPage->childPtrSize==4 ); nSize = *pIter; if( nSize>=0x80 ){ pEnd = &pIter[8]; @@ -58963,15 +72084,50 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ }while( *(pIter)>=0x80 && pIterintKey ){ - /* pIter now points at the 64-bit integer key value, a variable length - ** integer. The following block moves pIter to point at the first byte - ** past the end of the key value. */ - pEnd = &pIter[9]; - while( (*pIter++)&0x80 && pItermaxLocal ); - testcase( nSize==pPage->maxLocal+1 ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + assert( nSize>4 ); + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} +static u16 cellSizePtrIdxLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + assert( pPage->childPtrSize==0 ); + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pItermaxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); if( nSize<=pPage->maxLocal ){ nSize += (u32)(pIter - pCell); if( nSize<4 ) nSize = 4; @@ -58979,7 +72135,7 @@ static u16 cellSizePtr(MemPage *pPage, u8 *pCell){ int minLocal = pPage->minLocal; nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); testcase( nSize==pPage->maxLocal ); - testcase( nSize==pPage->maxLocal+1 ); + testcase( nSize==(u32)pPage->maxLocal+1 ); if( nSize>pPage->maxLocal ){ nSize = minLocal; } @@ -59009,6 +72165,58 @@ static u16 cellSizePtrNoPayload(MemPage *pPage, u8 *pCell){ assert( debuginfo.nSize==(u16)(pIter - pCell) || CORRUPT_DB ); return (u16)(pIter - pCell); } +static u16 cellSizePtrTableLeaf(MemPage *pPage, u8 *pCell){ + u8 *pIter = pCell; /* For looping over bytes of pCell */ + u8 *pEnd; /* End mark for a varint */ + u32 nSize; /* Size value to return */ + +#ifdef SQLITE_DEBUG + /* The value returned by this function should always be the same as + ** the (CellInfo.nSize) value found by doing a full parse of the + ** cell. If SQLITE_DEBUG is defined, an assert() at the bottom of + ** this function verifies that this invariant is not violated. */ + CellInfo debuginfo; + pPage->xParseCell(pPage, pCell, &debuginfo); +#endif + + nSize = *pIter; + if( nSize>=0x80 ){ + pEnd = &pIter[8]; + nSize &= 0x7f; + do{ + nSize = (nSize<<7) | (*++pIter & 0x7f); + }while( *(pIter)>=0x80 && pItermaxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize<=pPage->maxLocal ){ + nSize += (u32)(pIter - pCell); + if( nSize<4 ) nSize = 4; + }else{ + int minLocal = pPage->minLocal; + nSize = minLocal + (nSize - minLocal) % (pPage->pBt->usableSize - 4); + testcase( nSize==pPage->maxLocal ); + testcase( nSize==(u32)pPage->maxLocal+1 ); + if( nSize>pPage->maxLocal ){ + nSize = minLocal; + } + nSize += 4 + (u16)(pIter - pCell); + } + assert( nSize==debuginfo.nSize || CORRUPT_DB ); + return (u16)nSize; +} #ifdef SQLITE_DEBUG @@ -59021,17 +72229,24 @@ static u16 cellSize(MemPage *pPage, int iCell){ #ifndef SQLITE_OMIT_AUTOVACUUM /* -** If the cell pCell, part of page pPage contains a pointer -** to an overflow page, insert an entry into the pointer-map -** for the overflow page. +** The cell pCell is currently part of page pSrc but will ultimately be part +** of pPage. (pSrc and pPage are often the same.) If pCell contains a +** pointer to an overflow page, insert an entry into the pointer-map for +** the overflow page that will be valid after pCell has been moved to pPage. */ -static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ +static void ptrmapPutOvflPtr(MemPage *pPage, MemPage *pSrc, u8 *pCell,int *pRC){ CellInfo info; if( *pRC ) return; assert( pCell!=0 ); pPage->xParseCell(pPage, pCell, &info); if( info.nLocalaDataEnd, pCell, pCell+info.nLocal) ){ + testcase( pSrc!=pPage ); + *pRC = SQLITE_CORRUPT_BKPT; + return; + } + ovfl = get4byte(&pCell[info.nSize-4]); ptrmapPut(pPage->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, pRC); } } @@ -59039,17 +72254,18 @@ static void ptrmapPutOvflPtr(MemPage *pPage, u8 *pCell, int *pRC){ /* -** Defragment the page given. All Cells are moved to the -** end of the page and all free space is collected into one -** big FreeBlk that occurs in between the header and cell -** pointer array and the cell content area. +** Defragment the page given. This routine reorganizes cells within the +** page so that there are no free-blocks on the free-block list. +** +** Parameter nMaxFrag is the maximum amount of fragmented space that may be +** present in the page after this routine returns. ** ** EVIDENCE-OF: R-44582-60138 SQLite may from time to time reorganize a ** b-tree page so that there are no freeblocks or fragment bytes, all ** unused bytes are contained in the unallocated space region, and all ** cells are packed tightly at the end of the page. */ -static int defragmentPage(MemPage *pPage){ +static int defragmentPage(MemPage *pPage, int nMaxFrag){ int i; /* Loop counter */ int pc; /* Address of the i-th cell */ int hdr; /* Offset to the page header */ @@ -59063,65 +72279,109 @@ static int defragmentPage(MemPage *pPage){ unsigned char *src; /* Source of content */ int iCellFirst; /* First allowable cell index */ int iCellLast; /* Last possible cell index */ - + int iCellStart; /* First cell offset in input */ assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt!=0 ); assert( pPage->pBt->usableSize <= SQLITE_MAX_PAGE_SIZE ); assert( pPage->nOverflow==0 ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - temp = 0; - src = data = pPage->aData; + data = pPage->aData; hdr = pPage->hdrOffset; cellOffset = pPage->cellOffset; nCell = pPage->nCell; - assert( nCell==get2byte(&data[hdr+3]) ); - usableSize = pPage->pBt->usableSize; - cbrk = usableSize; + assert( nCell==get2byte(&data[hdr+3]) || CORRUPT_DB ); iCellFirst = cellOffset + 2*nCell; + usableSize = pPage->pBt->usableSize; + + /* This block handles pages with two or fewer free blocks and nMaxFrag + ** or fewer fragmented bytes. In this case it is faster to move the + ** two (or one) blocks of cells using memmove() and add the required + ** offsets to each pointer in the cell-pointer array than it is to + ** reconstruct the entire page. */ + if( (int)data[hdr+7]<=nMaxFrag ){ + int iFree = get2byte(&data[hdr+1]); + if( iFree>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( iFree ){ + int iFree2 = get2byte(&data[iFree]); + if( iFree2>usableSize-4 ) return SQLITE_CORRUPT_PAGE(pPage); + if( 0==iFree2 || (data[iFree2]==0 && data[iFree2+1]==0) ){ + u8 *pEnd = &data[cellOffset + nCell*2]; + u8 *pAddr; + int sz2 = 0; + int sz = get2byte(&data[iFree+2]); + int top = get2byte(&data[hdr+5]); + if( top>=iFree ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( iFree2 ){ + if( iFree+sz>iFree2 ) return SQLITE_CORRUPT_PAGE(pPage); + sz2 = get2byte(&data[iFree2+2]); + if( iFree2+sz2 > usableSize ) return SQLITE_CORRUPT_PAGE(pPage); + memmove(&data[iFree+sz+sz2], &data[iFree+sz], iFree2-(iFree+sz)); + sz += sz2; + }else if( iFree+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + + cbrk = top+sz; + assert( cbrk+(iFree-top) <= usableSize ); + memmove(&data[cbrk], &data[top], iFree-top); + for(pAddr=&data[cellOffset]; pAddriCellLast ){ - return SQLITE_CORRUPT_BKPT; + iCellStart = get2byte(&data[hdr+5]); + if( nCell>0 ){ + temp = sqlite3PagerTempSpace(pPage->pBt->pPager); + memcpy(temp, data, usableSize); + src = temp; + for(i=0; iiCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pc>=0 && pc<=iCellLast ); + size = pPage->xCellSize(pPage, &src[pc]); + cbrk -= size; + if( cbrkusableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( cbrk+size<=usableSize && cbrk>=iCellStart ); + testcase( cbrk+size==usableSize ); + testcase( pc+size==usableSize ); + put2byte(pAddr, cbrk); + memcpy(&data[cbrk], &src[pc], size); } - assert( pc>=iCellFirst && pc<=iCellLast ); - size = pPage->xCellSize(pPage, &src[pc]); - cbrk -= size; - if( cbrkusableSize ){ - return SQLITE_CORRUPT_BKPT; - } - assert( cbrk+size<=usableSize && cbrk>=iCellFirst ); - testcase( cbrk+size==usableSize ); - testcase( pc+size==usableSize ); - put2byte(pAddr, cbrk); - if( temp==0 ){ - int x; - if( cbrk==pc ) continue; - temp = sqlite3PagerTempSpace(pPage->pBt->pPager); - x = get2byte(&data[hdr+5]); - memcpy(&temp[x], &data[x], (cbrk+size) - x); - src = temp; - } - memcpy(&data[cbrk], &src[pc], size); + } + data[hdr+7] = 0; + +defragment_out: + assert( pPage->nFree>=0 ); + if( data[hdr+7]+cbrk-iCellFirst!=pPage->nFree ){ + return SQLITE_CORRUPT_PAGE(pPage); } assert( cbrk>=iCellFirst ); put2byte(&data[hdr+5], cbrk); data[hdr+1] = 0; data[hdr+2] = 0; - data[hdr+7] = 0; memset(&data[iCellFirst], 0, cbrk-iCellFirst); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); - if( cbrk-iCellFirst!=pPage->nFree ){ - return SQLITE_CORRUPT_BKPT; - } return SQLITE_OK; } @@ -59140,33 +72400,26 @@ static int defragmentPage(MemPage *pPage){ ** causes the fragmentation count to exceed 60. */ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ - const int hdr = pPg->hdrOffset; - u8 * const aData = pPg->aData; - int iAddr = hdr + 1; - int pc = get2byte(&aData[iAddr]); - int x; - int usableSize = pPg->pBt->usableSize; + const int hdr = pPg->hdrOffset; /* Offset to page header */ + u8 * const aData = pPg->aData; /* Page data */ + int iAddr = hdr + 1; /* Address of ptr to pc */ + u8 *pTmp = &aData[iAddr]; /* Temporary ptr into aData[] */ + int pc = get2byte(pTmp); /* Address of a free slot */ + int x; /* Excess size of the slot */ + int maxPC = pPg->pBt->usableSize - nByte; /* Max address for a usable slot */ + int size; /* Size of the free slot */ assert( pc>0 ); - do{ - int size; /* Size of the free slot */ - /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of - ** increasing offset. */ - if( pc>usableSize-4 || pc=0 ){ testcase( x==4 ); testcase( x==3 ); - if( pc < pPg->cellOffset+2*pPg->nCell || size+pc > usableSize ){ - *pRc = SQLITE_CORRUPT_BKPT; - return 0; - }else if( x<4 ){ + if( x<4 ){ /* EVIDENCE-OF: R-11498-58022 In a well-formed b-tree page, the total ** number of bytes in fragments may not exceed 60. */ if( aData[hdr+7]>57 ) return 0; @@ -59175,17 +72428,33 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** fragmented bytes within the page. */ memcpy(&aData[iAddr], &aData[pc], 2); aData[hdr+7] += (u8)x; + return &aData[pc]; + }else if( x+pc > maxPC ){ + /* This slot extends off the end of the usable part of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + return 0; }else{ /* The slot remains on the free-list. Reduce its size to account - ** for the portion used by the new allocation. */ + ** for the portion used by the new allocation. */ put2byte(&aData[pc+2], x); } return &aData[pc + x]; } iAddr = pc; - pc = get2byte(&aData[pc]); - }while( pc ); - + pTmp = &aData[pc]; + pc = get2byte(pTmp); + if( pc<=iAddr ){ + if( pc ){ + /* The next slot in the chain comes before the current slot */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } + return 0; + } + } + if( pc>maxPC+nByte-4 ){ + /* The free slot chain extends off the end of the page */ + *pRc = SQLITE_CORRUPT_PAGE(pPg); + } return 0; } @@ -59202,13 +72471,14 @@ static u8 *pageFindSlot(MemPage *pPg, int nByte, int *pRc){ ** allocation is being made in order to insert a new cell, so we will ** also end up needing a new cell pointer. */ -static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ +static SQLITE_INLINE int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ const int hdr = pPage->hdrOffset; /* Local cache of pPage->hdrOffset */ u8 * const data = pPage->aData; /* Local cache of pPage->aData */ int top; /* First byte of cell content area */ int rc = SQLITE_OK; /* Integer return code */ + u8 *pTmp; /* Temp ptr into data[] */ int gap; /* First byte of gap between cell pointers and cell content */ - + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( pPage->pBt ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); @@ -59225,19 +72495,21 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** then the cell content offset of an empty page wants to be 65536. ** However, that integer is too large to be stored in a 2-byte unsigned ** integer, so a value of 0 is used in its place. */ - top = get2byte(&data[hdr+5]); - assert( top<=(int)pPage->pBt->usableSize ); /* Prevent by getAndInitPage() */ + pTmp = &data[hdr+5]; + top = get2byte(pTmp); if( gap>top ){ if( top==0 && pPage->pBt->usableSize==65536 ){ top = 65536; }else{ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } + }else if( top>(int)pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); } - /* If there is enough space between gap and top for one more cell pointer - ** array entry offset, and if the freelist is not empty, then search the - ** freelist looking for a free slot big enough to satisfy the request. + /* If there is enough space between gap and top for one more cell pointer, + ** and if the freelist is not empty, then search the + ** freelist looking for a slot big enough to satisfy the request. */ testcase( gap+2==top ); testcase( gap+1==top ); @@ -59245,9 +72517,14 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ if( (data[hdr+2] || data[hdr+1]) && gap+2<=top ){ u8 *pSpace = pageFindSlot(pPage, nByte, &rc); if( pSpace ){ - assert( pSpace>=data && (pSpace - data)<65536 ); - *pIdx = (int)(pSpace - data); - return SQLITE_OK; + int g2; + assert( pSpace+nByte<=data+pPage->pBt->usableSize ); + *pIdx = g2 = (int)(pSpace-data); + if( g2<=gap ){ + return SQLITE_CORRUPT_PAGE(pPage); + }else{ + return SQLITE_OK; + } }else if( rc ){ return rc; } @@ -59259,15 +72536,16 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ testcase( gap+2+nByte==top ); if( gap+2+nByte>top ){ assert( pPage->nCell>0 || CORRUPT_DB ); - rc = defragmentPage(pPage); + assert( pPage->nFree>=0 ); + rc = defragmentPage(pPage, MIN(4, pPage->nFree - (2+nByte))); if( rc ) return rc; top = get2byteNotZero(&data[hdr+5]); - assert( gap+nByte<=top ); + assert( gap+2+nByte<=top ); } /* Allocate memory from the gap in between the cell pointer array - ** and the cell content area. The btreeInitPage() call has already + ** and the cell content area. The btreeComputeFreeSpace() call has already ** validated the freelist. Given that the freelist is valid, there ** is no way that the allocation can extend off the end of the page. ** The assert() below verifies the previous sentence. @@ -59286,9 +72564,9 @@ static int allocateSpace(MemPage *pPage, int nByte, int *pIdx){ ** ** Adjacent freeblocks are coalesced. ** -** Note that even though the freeblock list was checked by btreeInitPage(), +** Even though the freeblock list was checked by btreeComputeFreeSpace(), ** that routine will not detect overlap between cells or freeblocks. Nor -** does it detect cells or freeblocks that encrouch into the reserved bytes +** does it detect cells or freeblocks that encroach into the reserved bytes ** at the end of the page. So do additional corruption checks inside this ** routine and return SQLITE_CORRUPT if any problems are found. */ @@ -59298,9 +72576,10 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ u8 hdr; /* Page header size. 0 or 100 */ u8 nFrag = 0; /* Reduction in fragmentation */ u16 iOrigSize = iSize; /* Original value of iSize */ - u32 iLast = pPage->pBt->usableSize-4; /* Largest possible freeblock offset */ + u16 x; /* Offset to cell content area */ u32 iEnd = iStart + iSize; /* First byte past the iStart buffer */ unsigned char *data = pPage->aData; /* Page content */ + u8 *pTmp; /* Temporary ptr into data[] */ assert( pPage->pBt!=0 ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); @@ -59308,15 +72587,9 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ assert( CORRUPT_DB || iEnd <= pPage->pBt->usableSize ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); assert( iSize>=4 ); /* Minimum cell size is 4 */ - assert( iStart<=iLast ); + assert( CORRUPT_DB || iStart<=pPage->pBt->usableSize-4 ); - /* Overwrite deleted information with zeros when the secure_delete - ** option is enabled */ - if( pPage->pBt->btsFlags & BTS_SECURE_DELETE ){ - memset(&data[iStart], 0, iSize); - } - - /* The list of freeblocks must be in ascending order. Find the + /* The list of freeblocks must be in ascending order. Find the ** spot on the list where iStart should be inserted. */ hdr = pPage->hdrOffset; @@ -59324,13 +72597,18 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( data[iPtr+1]==0 && data[iPtr]==0 ){ iFreeBlk = 0; /* Shortcut for the case when the freelist is empty */ }else{ - while( (iFreeBlk = get2byte(&data[iPtr]))>0 && iFreeBlkiLast ) return SQLITE_CORRUPT_BKPT; - assert( iFreeBlk>iPtr || iFreeBlk==0 ); - + if( iFreeBlk>pPage->pBt->usableSize-4 ){ /* TH3: corrupt081.100 */ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( iFreeBlk>iPtr || iFreeBlk==0 || CORRUPT_DB ); + /* At this point: ** iFreeBlk: First freeblock after iStart, or zero if none ** iPtr: The address of a pointer to iFreeBlk @@ -59339,13 +72617,15 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ */ if( iFreeBlk && iEnd+3>=iFreeBlk ){ nFrag = iFreeBlk - iEnd; - if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_BKPT; + if( iEnd>iFreeBlk ) return SQLITE_CORRUPT_PAGE(pPage); iEnd = iFreeBlk + get2byte(&data[iFreeBlk+2]); - if( iEnd > pPage->pBt->usableSize ) return SQLITE_CORRUPT_BKPT; + if( iEnd > pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } iSize = iEnd - iStart; iFreeBlk = get2byte(&data[iFreeBlk]); } - + /* If iPtr is another freeblock (that is, if iPtr is not the freelist ** pointer in the page header) then check to see if iStart should be ** coalesced onto the end of iPtr. @@ -59353,20 +72633,28 @@ static int freeSpace(MemPage *pPage, u16 iStart, u16 iSize){ if( iPtr>hdr+1 ){ int iPtrEnd = iPtr + get2byte(&data[iPtr+2]); if( iPtrEnd+3>=iStart ){ - if( iPtrEnd>iStart ) return SQLITE_CORRUPT_BKPT; + if( iPtrEnd>iStart ) return SQLITE_CORRUPT_PAGE(pPage); nFrag += iStart - iPtrEnd; iSize = iEnd - iPtr; iStart = iPtr; } } - if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_BKPT; + if( nFrag>data[hdr+7] ) return SQLITE_CORRUPT_PAGE(pPage); data[hdr+7] -= nFrag; } - if( iStart==get2byte(&data[hdr+5]) ){ + pTmp = &data[hdr+5]; + x = get2byte(pTmp); + if( pPage->pBt->btsFlags & BTS_FAST_SECURE ){ + /* Overwrite deleted information with zeros when the secure_delete + ** option is enabled */ + memset(&data[iStart], 0, iSize); + } + if( iStart<=x ){ /* The new freeblock is at the beginning of the cell content area, ** so just extend the cell content area rather than create another ** freelist entry */ - if( iPtr!=hdr+1 ) return SQLITE_CORRUPT_BKPT; + if( iStartpBt */ assert( pPage->hdrOffset==(pPage->pgno==1 ? 100 : 0) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->leaf = (u8)(flagByte>>3); assert( PTF_LEAF == 1<<3 ); - flagByte &= ~PTF_LEAF; - pPage->childPtrSize = 4-4*pPage->leaf; - pPage->xCellSize = cellSizePtr; pBt = pPage->pBt; - if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ - /* EVIDENCE-OF: R-07291-35328 A value of 5 (0x05) means the page is an - ** interior table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY)==5 ); - /* EVIDENCE-OF: R-26900-09176 A value of 13 (0x0d) means the page is a - ** leaf table b-tree page. */ - assert( (PTF_LEAFDATA|PTF_INTKEY|PTF_LEAF)==13 ); - pPage->intKey = 1; - if( pPage->leaf ){ + pPage->max1bytePayload = pBt->max1bytePayload; + if( flagByte>=(PTF_ZERODATA | PTF_LEAF) ){ + pPage->childPtrSize = 0; + pPage->leaf = 1; + if( flagByte==(PTF_LEAFDATA | PTF_INTKEY | PTF_LEAF) ){ pPage->intKeyLeaf = 1; + pPage->xCellSize = cellSizePtrTableLeaf; pPage->xParseCell = btreeParseCellPtr; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else if( flagByte==(PTF_ZERODATA | PTF_LEAF) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtrIdxLeaf; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); + } + }else{ + pPage->childPtrSize = 4; + pPage->leaf = 0; + if( flagByte==(PTF_ZERODATA) ){ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + pPage->maxLocal = pBt->maxLocal; + pPage->minLocal = pBt->minLocal; + }else if( flagByte==(PTF_LEAFDATA | PTF_INTKEY) ){ pPage->intKeyLeaf = 0; pPage->xCellSize = cellSizePtrNoPayload; pPage->xParseCell = btreeParseCellPtrNoPayload; + pPage->intKey = 1; + pPage->maxLocal = pBt->maxLeaf; + pPage->minLocal = pBt->minLeaf; + }else{ + pPage->intKey = 0; + pPage->intKeyLeaf = 0; + pPage->xCellSize = cellSizePtr; + pPage->xParseCell = btreeParseCellPtrIndex; + return SQLITE_CORRUPT_PAGE(pPage); } - pPage->maxLocal = pBt->maxLeaf; - pPage->minLocal = pBt->minLeaf; - }else if( flagByte==PTF_ZERODATA ){ - /* EVIDENCE-OF: R-43316-37308 A value of 2 (0x02) means the page is an - ** interior index b-tree page. */ - assert( (PTF_ZERODATA)==2 ); - /* EVIDENCE-OF: R-59615-42828 A value of 10 (0x0a) means the page is a - ** leaf index b-tree page. */ - assert( (PTF_ZERODATA|PTF_LEAF)==10 ); - pPage->intKey = 0; - pPage->intKeyLeaf = 0; - pPage->xParseCell = btreeParseCellPtrIndex; - pPage->maxLocal = pBt->maxLocal; - pPage->minLocal = pBt->minLocal; - }else{ - /* EVIDENCE-OF: R-47608-56469 Any other value for the b-tree page type is - ** an error. */ - return SQLITE_CORRUPT_BKPT; } - pPage->max1bytePayload = pBt->max1bytePayload; return SQLITE_OK; } /* -** Initialize the auxiliary information for a disk block. -** -** Return SQLITE_OK on success. If we see that the page does -** not contain a well-formed database page, then return -** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not -** guarantee that the page is well-formed. It only shows that -** we failed to detect any corruption. +** Compute the amount of freespace on the page. In other words, fill +** in the pPage->nFree field. */ -static int btreeInitPage(MemPage *pPage){ +static int btreeComputeFreeSpace(MemPage *pPage){ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 hdr; /* Offset to beginning of page header */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Amount of usable space on each page */ + int nFree; /* Number of unused bytes on the page */ + int top; /* First byte of the cell content area */ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ assert( pPage->pBt!=0 ); assert( pPage->pBt->db!=0 ); @@ -59457,121 +72758,158 @@ static int btreeInitPage(MemPage *pPage){ assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==1 ); + assert( pPage->nFree<0 ); - if( !pPage->isInit ){ - u16 pc; /* Address of a freeblock within pPage->aData[] */ - u8 hdr; /* Offset to beginning of page header */ - u8 *data; /* Equal to pPage->aData */ - BtShared *pBt; /* The main btree structure */ - int usableSize; /* Amount of usable space on each page */ - u16 cellOffset; /* Offset from start of page to first cell pointer */ - int nFree; /* Number of unused bytes on the page */ - int top; /* First byte of the cell content area */ - int iCellFirst; /* First allowable cell or freeblock offset */ - int iCellLast; /* Last possible cell or freeblock offset */ + usableSize = pPage->pBt->usableSize; + hdr = pPage->hdrOffset; + data = pPage->aData; + /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates + ** the start of the cell content area. A zero value for this integer is + ** interpreted as 65536. */ + top = get2byteNotZero(&data[hdr+5]); + iCellFirst = hdr + 8 + pPage->childPtrSize + 2*pPage->nCell; + iCellLast = usableSize - 4; - pBt = pPage->pBt; - - hdr = pPage->hdrOffset; - data = pPage->aData; - /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating - ** the b-tree page type. */ - if( decodeFlags(pPage, data[hdr]) ) return SQLITE_CORRUPT_BKPT; - assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); - pPage->maskPage = (u16)(pBt->pageSize - 1); - pPage->nOverflow = 0; - usableSize = pBt->usableSize; - pPage->cellOffset = cellOffset = hdr + 8 + pPage->childPtrSize; - pPage->aDataEnd = &data[usableSize]; - pPage->aCellIdx = &data[cellOffset]; - pPage->aDataOfst = &data[pPage->childPtrSize]; - /* EVIDENCE-OF: R-58015-48175 The two-byte integer at offset 5 designates - ** the start of the cell content area. A zero value for this integer is - ** interpreted as 65536. */ - top = get2byteNotZero(&data[hdr+5]); - /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the - ** number of cells on the page. */ - pPage->nCell = get2byte(&data[hdr+3]); - if( pPage->nCell>MX_CELL(pBt) ){ - /* To many cells for a single page. The page must be corrupt */ - return SQLITE_CORRUPT_BKPT; + /* Compute the total free space on the page + ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the + ** start of the first freeblock on the page, or is zero if there are no + ** freeblocks. */ + pc = get2byte(&data[hdr+1]); + nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ + if( pc>0 ){ + u32 next, size; + if( pcnCell==MX_CELL(pBt) ); - /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only - ** possible for a root page of a table that contains no rows) then the - ** offset to the cell content area will equal the page size minus the - ** bytes of reserved space. */ - assert( pPage->nCell>0 || top==usableSize || CORRUPT_DB ); - - /* A malformed database page might cause us to read past the end - ** of page when parsing a cell. - ** - ** The following block of code checks early to see if a cell extends - ** past the end of a page boundary and causes SQLITE_CORRUPT to be - ** returned if it does. - */ - iCellFirst = cellOffset + 2*pPage->nCell; - iCellLast = usableSize - 4; - if( pBt->db->flags & SQLITE_CellSizeCk ){ - int i; /* Index into the cell pointer array */ - int sz; /* Size of a cell */ - - if( !pPage->leaf ) iCellLast--; - for(i=0; inCell; i++){ - pc = get2byteAligned(&data[cellOffset+i*2]); - testcase( pc==iCellFirst ); - testcase( pc==iCellLast ); - if( pciCellLast ){ - return SQLITE_CORRUPT_BKPT; - } - sz = pPage->xCellSize(pPage, &data[pc]); - testcase( pc+sz==usableSize ); - if( pc+sz>usableSize ){ - return SQLITE_CORRUPT_BKPT; - } - } - if( !pPage->leaf ) iCellLast++; - } - - /* Compute the total free space on the page - ** EVIDENCE-OF: R-23588-34450 The two-byte integer at offset 1 gives the - ** start of the first freeblock on the page, or is zero if there are no - ** freeblocks. */ - pc = get2byte(&data[hdr+1]); - nFree = data[hdr+7] + top; /* Init nFree to non-freeblock free space */ - while( pc>0 ){ - u16 next, size; - if( pciCellLast ){ - /* EVIDENCE-OF: R-55530-52930 In a well-formed b-tree page, there will - ** always be at least one cell before the first freeblock. - ** - ** Or, the freeblock is off the end of the page - */ - return SQLITE_CORRUPT_BKPT; + while( 1 ){ + if( pc>iCellLast ){ + /* Freeblock off the end of the page */ + return SQLITE_CORRUPT_PAGE(pPage); } next = get2byte(&data[pc]); size = get2byte(&data[pc+2]); - if( (next>0 && next<=pc+size+3) || pc+size>usableSize ){ - /* Free blocks must be in ascending order. And the last byte of - ** the free-block must lie on the database page. */ - return SQLITE_CORRUPT_BKPT; - } nFree = nFree + size; + if( next<=pc+size+3 ) break; pc = next; } - - /* At this point, nFree contains the sum of the offset to the start - ** of the cell-content area plus the number of free bytes within - ** the cell-content area. If this is greater than the usable-size - ** of the page, then the page must be corrupted. This check also - ** serves to verify that the offset to the start of the cell-content - ** area, according to the page header, lies within the page. - */ - if( nFree>usableSize ){ - return SQLITE_CORRUPT_BKPT; + if( next>0 ){ + /* Freeblock not in ascending order */ + return SQLITE_CORRUPT_PAGE(pPage); } - pPage->nFree = (u16)(nFree - iCellFirst); - pPage->isInit = 1; + if( pc+size>(unsigned int)usableSize ){ + /* Last freeblock extends past page end */ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + + /* At this point, nFree contains the sum of the offset to the start + ** of the cell-content area plus the number of free bytes within + ** the cell-content area. If this is greater than the usable-size + ** of the page, then the page must be corrupted. This check also + ** serves to verify that the offset to the start of the cell-content + ** area, according to the page header, lies within the page. + */ + if( nFree>usableSize || nFreenFree = (u16)(nFree - iCellFirst); + return SQLITE_OK; +} + +/* +** Do additional sanity check after btreeInitPage() if +** PRAGMA cell_size_check=ON +*/ +static SQLITE_NOINLINE int btreeCellSizeCheck(MemPage *pPage){ + int iCellFirst; /* First allowable cell or freeblock offset */ + int iCellLast; /* Last possible cell or freeblock offset */ + int i; /* Index into the cell pointer array */ + int sz; /* Size of a cell */ + int pc; /* Address of a freeblock within pPage->aData[] */ + u8 *data; /* Equal to pPage->aData */ + int usableSize; /* Maximum usable space on the page */ + int cellOffset; /* Start of cell content area */ + + iCellFirst = pPage->cellOffset + 2*pPage->nCell; + usableSize = pPage->pBt->usableSize; + iCellLast = usableSize - 4; + data = pPage->aData; + cellOffset = pPage->cellOffset; + if( !pPage->leaf ) iCellLast--; + for(i=0; inCell; i++){ + pc = get2byteAligned(&data[cellOffset+i*2]); + testcase( pc==iCellFirst ); + testcase( pc==iCellLast ); + if( pciCellLast ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + sz = pPage->xCellSize(pPage, &data[pc]); + testcase( pc+sz==usableSize ); + if( pc+sz>usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + } + return SQLITE_OK; +} + +/* +** Initialize the auxiliary information for a disk block. +** +** Return SQLITE_OK on success. If we see that the page does +** not contain a well-formed database page, then return +** SQLITE_CORRUPT. Note that a return of SQLITE_OK does not +** guarantee that the page is well-formed. It only shows that +** we failed to detect any corruption. +*/ +static int btreeInitPage(MemPage *pPage){ + u8 *data; /* Equal to pPage->aData */ + BtShared *pBt; /* The main btree structure */ + + assert( pPage->pBt!=0 ); + assert( pPage->pBt->db!=0 ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->pgno==sqlite3PagerPagenumber(pPage->pDbPage) ); + assert( pPage == sqlite3PagerGetExtra(pPage->pDbPage) ); + assert( pPage->aData == sqlite3PagerGetData(pPage->pDbPage) ); + assert( pPage->isInit==0 ); + + pBt = pPage->pBt; + data = pPage->aData + pPage->hdrOffset; + /* EVIDENCE-OF: R-28594-02890 The one-byte flag at offset 0 indicating + ** the b-tree page type. */ + if( decodeFlags(pPage, data[0]) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + assert( pBt->pageSize>=512 && pBt->pageSize<=65536 ); + pPage->maskPage = (u16)(pBt->pageSize - 1); + pPage->nOverflow = 0; + pPage->cellOffset = pPage->hdrOffset + 8 + pPage->childPtrSize; + pPage->aCellIdx = data + pPage->childPtrSize + 8; + pPage->aDataEnd = pPage->aData + pBt->pageSize; + pPage->aDataOfst = pPage->aData + pPage->childPtrSize; + /* EVIDENCE-OF: R-37002-32774 The two-byte integer at offset 3 gives the + ** number of cells on the page. */ + pPage->nCell = get2byte(&data[3]); + if( pPage->nCell>MX_CELL(pBt) ){ + /* To many cells for a single page. The page must be corrupt */ + return SQLITE_CORRUPT_PAGE(pPage); + } + testcase( pPage->nCell==MX_CELL(pBt) ); + /* EVIDENCE-OF: R-24089-57979 If a page contains no cells (which is only + ** possible for a root page of a table that contains no rows) then the + ** offset to the cell content area will equal the page size minus the + ** bytes of reserved space. */ + assert( pPage->nCell>0 + || get2byteNotZero(&data[5])==(int)pBt->usableSize + || CORRUPT_DB ); + pPage->nFree = -1; /* Indicate that this value is yet uncomputed */ + pPage->isInit = 1; + if( pBt->db->flags & SQLITE_CellSizeCk ){ + return btreeCellSizeCheck(pPage); } return SQLITE_OK; } @@ -59586,12 +72924,12 @@ static void zeroPage(MemPage *pPage, int flags){ u8 hdr = pPage->hdrOffset; u16 first; - assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno ); + assert( sqlite3PagerPagenumber(pPage->pDbPage)==pPage->pgno || CORRUPT_DB ); assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); assert( sqlite3PagerGetData(pPage->pDbPage) == data ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pBt->mutex) ); - if( pBt->btsFlags & BTS_SECURE_DELETE ){ + if( pBt->btsFlags & BTS_FAST_SECURE ){ memset(&data[hdr], 0, pBt->usableSize - hdr); } data[hdr] = (char)flags; @@ -59602,7 +72940,7 @@ static void zeroPage(MemPage *pPage, int flags){ pPage->nFree = (u16)(pBt->usableSize - first); decodeFlags(pPage, flags); pPage->cellOffset = first; - pPage->aDataEnd = &data[pBt->usableSize]; + pPage->aDataEnd = &data[pBt->pageSize]; pPage->aCellIdx = &data[first]; pPage->aDataOfst = &data[pPage->childPtrSize]; pPage->nOverflow = 0; @@ -59627,7 +72965,7 @@ static MemPage *btreePageFromDbPage(DbPage *pDbPage, Pgno pgno, BtShared *pBt){ pPage->hdrOffset = pgno==1 ? 100 : 0; } assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); - return pPage; + return pPage; } /* @@ -59680,78 +73018,55 @@ static MemPage *btreePageLookup(BtShared *pBt, Pgno pgno){ static Pgno btreePagecount(BtShared *pBt){ return pBt->nPage; } -SQLITE_PRIVATE u32 sqlite3BtreeLastPage(Btree *p){ +SQLITE_PRIVATE Pgno sqlite3BtreeLastPage(Btree *p){ assert( sqlite3BtreeHoldsMutex(p) ); - assert( ((p->pBt->nPage)&0x8000000)==0 ); return btreePagecount(p->pBt); } /* ** Get a page from the pager and initialize it. -** -** If pCur!=0 then the page is being fetched as part of a moveToChild() -** call. Do additional sanity checking on the page in this case. -** And if the fetch fails, this routine must decrement pCur->iPage. -** -** The page is fetched as read-write unless pCur is not NULL and is -** a read-only cursor. -** -** If an error occurs, then *ppPage is undefined. It -** may remain unchanged, or it may be set to an invalid value. */ static int getAndInitPage( BtShared *pBt, /* The database file */ Pgno pgno, /* Number of the page to get */ MemPage **ppPage, /* Write the page pointer here */ - BtCursor *pCur, /* Cursor to receive the page, or NULL */ int bReadOnly /* True for a read-only page */ ){ int rc; DbPage *pDbPage; + MemPage *pPage; assert( sqlite3_mutex_held(pBt->mutex) ); - assert( pCur==0 || ppPage==&pCur->apPage[pCur->iPage] ); - assert( pCur==0 || bReadOnly==pCur->curPagerFlags ); - assert( pCur==0 || pCur->iPage>0 ); if( pgno>btreePagecount(pBt) ){ - rc = SQLITE_CORRUPT_BKPT; - goto getAndInitPage_error; + *ppPage = 0; + return SQLITE_CORRUPT_BKPT; } rc = sqlite3PagerGet(pBt->pPager, pgno, (DbPage**)&pDbPage, bReadOnly); if( rc ){ - goto getAndInitPage_error; + *ppPage = 0; + return rc; } - *ppPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); - if( (*ppPage)->isInit==0 ){ + pPage = (MemPage*)sqlite3PagerGetExtra(pDbPage); + if( pPage->isInit==0 ){ btreePageFromDbPage(pDbPage, pgno, pBt); - rc = btreeInitPage(*ppPage); + rc = btreeInitPage(pPage); if( rc!=SQLITE_OK ){ - releasePage(*ppPage); - goto getAndInitPage_error; + releasePage(pPage); + *ppPage = 0; + return rc; } } - assert( (*ppPage)->pgno==pgno ); - assert( (*ppPage)->aData==sqlite3PagerGetData(pDbPage) ); - - /* If obtaining a child page for a cursor, we must verify that the page is - ** compatible with the root page. */ - if( pCur && ((*ppPage)->nCell<1 || (*ppPage)->intKey!=pCur->curIntKey) ){ - rc = SQLITE_CORRUPT_BKPT; - releasePage(*ppPage); - goto getAndInitPage_error; - } + assert( pPage->pgno==pgno || CORRUPT_DB ); + assert( pPage->aData==sqlite3PagerGetData(pDbPage) ); + *ppPage = pPage; return SQLITE_OK; - -getAndInitPage_error: - if( pCur ) pCur->iPage--; - testcase( pgno==0 ); - assert( pgno!=0 || rc==SQLITE_CORRUPT ); - return rc; } /* ** Release a MemPage. This should be called once for each prior ** call to btreeGetPage. +** +** Page1 is a special case and must be released using releasePageOne(). */ static void releasePageNotNull(MemPage *pPage){ assert( pPage->aData ); @@ -59765,6 +73080,16 @@ static void releasePageNotNull(MemPage *pPage){ static void releasePage(MemPage *pPage){ if( pPage ) releasePageNotNull(pPage); } +static void releasePageOne(MemPage *pPage){ + assert( pPage!=0 ); + assert( pPage->aData ); + assert( pPage->pBt ); + assert( pPage->pDbPage!=0 ); + assert( sqlite3PagerGetExtra(pPage->pDbPage) == (void*)pPage ); + assert( sqlite3PagerGetData(pPage->pDbPage)==pPage->aData ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + sqlite3PagerUnrefPageOne(pPage->pDbPage); +} /* ** Get an unused page. @@ -59817,7 +73142,7 @@ static void pageReinit(DbPage *pData){ ** call to btreeInitPage() will likely return SQLITE_CORRUPT. ** But no harm is done by this. And it is very important that ** btreeInitPage() be called on every btree page so we make - ** the call for every page that comes in for re-initing. */ + ** the call for every page that comes in for re-initializing. */ btreeInitPage(pPage); } } @@ -59835,11 +73160,11 @@ static int btreeInvokeBusyHandler(void *pArg){ /* ** Open a database file. -** +** ** zFilename is the name of the database file. If zFilename is NULL ** then an ephemeral database is created. The ephemeral database might ** be exclusively in memory, or it might use a disk-based memory cache. -** Either way, the ephemeral database will be automatically deleted +** Either way, the ephemeral database will be automatically deleted ** when sqlite3BtreeClose() is called. ** ** If zFilename is ":memory:" then an in-memory database is created @@ -59872,7 +73197,7 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( /* True if opening an ephemeral, temporary database */ const int isTempDb = zFilename==0 || zFilename[0]==0; - /* Set the variable isMemdb to true for an in-memory database, or + /* Set the variable isMemdb to true for an in-memory database, or ** false for a file-based database. */ #ifdef SQLITE_OMIT_MEMORYDB @@ -59934,15 +73259,19 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( rc = sqlite3OsFullPathname(pVfs, zFilename, nFullPathname, zFullPathname); if( rc ){ - sqlite3_free(zFullPathname); - sqlite3_free(p); - return rc; + if( rc==SQLITE_OK_SYMLINK ){ + rc = SQLITE_OK; + }else{ + sqlite3_free(zFullPathname); + sqlite3_free(p); + return rc; + } } } #if SQLITE_THREADSAFE mutexOpen = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_OPEN); sqlite3_mutex_enter(mutexOpen); - mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); sqlite3_mutex_enter(mutexShared); #endif for(pBt=GLOBAL(BtShared*,sqlite3SharedCacheList); pBt; pBt=pBt->pNext){ @@ -59991,14 +73320,17 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( assert( sizeof(u32)==4 ); assert( sizeof(u16)==2 ); assert( sizeof(Pgno)==4 ); - + + /* Suppress false-positive compiler warning from PVS-Studio */ + memset(&zDbHeader[16], 0, 8); + pBt = sqlite3MallocZero( sizeof(*pBt) ); if( pBt==0 ){ rc = SQLITE_NOMEM_BKPT; goto btree_open_out; } rc = sqlite3PagerOpen(pVfs, &pBt->pPager, zFilename, - EXTRA_SIZE, flags, vfsFlags, pageReinit); + sizeof(MemPage), flags, vfsFlags, pageReinit); if( rc==SQLITE_OK ){ sqlite3PagerSetMmapLimit(pBt->pPager, db->szMmap); rc = sqlite3PagerReadFileheader(pBt->pPager,sizeof(zDbHeader),zDbHeader); @@ -60008,14 +73340,16 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( } pBt->openFlags = (u8)flags; pBt->db = db; - sqlite3PagerSetBusyhandler(pBt->pPager, btreeInvokeBusyHandler, pBt); + sqlite3PagerSetBusyHandler(pBt->pPager, btreeInvokeBusyHandler, pBt); p->pBt = pBt; - + pBt->pCursor = 0; pBt->pPage1 = 0; if( sqlite3PagerIsreadonly(pBt->pPager) ) pBt->btsFlags |= BTS_READ_ONLY; -#ifdef SQLITE_SECURE_DELETE +#if defined(SQLITE_SECURE_DELETE) pBt->btsFlags |= BTS_SECURE_DELETE; +#elif defined(SQLITE_FAST_SECURE_DELETE) + pBt->btsFlags |= BTS_OVERWRITE; #endif /* EVIDENCE-OF: R-51873-39618 The page size for a database file is ** determined by the 2-byte integer located at an offset of 16 bytes from @@ -60052,14 +73386,14 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( if( rc ) goto btree_open_out; pBt->usableSize = pBt->pageSize - nReserve; assert( (pBt->pageSize & 7)==0 ); /* 8-byte alignment of pageSize */ - + #if !defined(SQLITE_OMIT_SHARED_CACHE) && !defined(SQLITE_OMIT_DISKIO) /* Add the new BtShared object to the linked list sharable BtShareds. */ + pBt->nRef = 1; if( p->sharable ){ MUTEX_LOGIC( sqlite3_mutex *mutexShared; ) - pBt->nRef = 1; - MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER);) + MUTEX_LOGIC( mutexShared = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN);) if( SQLITE_THREADSAFE && sqlite3GlobalConfig.bCoreMutex ){ pBt->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_FAST); if( pBt->mutex==0 ){ @@ -60111,24 +73445,32 @@ SQLITE_PRIVATE int sqlite3BtreeOpen( btree_open_out: if( rc!=SQLITE_OK ){ if( pBt && pBt->pPager ){ - sqlite3PagerClose(pBt->pPager); + sqlite3PagerClose(pBt->pPager, 0); } sqlite3_free(pBt); sqlite3_free(p); *ppBtree = 0; }else{ + sqlite3_file *pFile; + /* If the B-Tree was successfully opened, set the pager-cache size to the ** default value. Except, when opening on an existing shared pager-cache, ** do not change the pager-cache size. */ if( sqlite3BtreeSchema(p, 0, 0)==0 ){ - sqlite3PagerSetCachesize(p->pBt->pPager, SQLITE_DEFAULT_CACHE_SIZE); + sqlite3BtreeSetCacheSize(p, SQLITE_DEFAULT_CACHE_SIZE); + } + + pFile = sqlite3PagerFile(pBt->pPager); + if( pFile->pMethods ){ + sqlite3OsFileControlHint(pFile, SQLITE_FCNTL_PDB, (void*)&pBt->db); } } if( mutexOpen ){ assert( sqlite3_mutex_held(mutexOpen) ); sqlite3_mutex_leave(mutexOpen); } + assert( rc!=SQLITE_OK || sqlite3BtreeConnectionCount(*ppBtree)>0 ); return rc; } @@ -60140,13 +73482,13 @@ btree_open_out: */ static int removeFromSharingList(BtShared *pBt){ #ifndef SQLITE_OMIT_SHARED_CACHE - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) BtShared *pList; int removed = 0; assert( sqlite3_mutex_notheld(pBt->mutex) ); - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - sqlite3_mutex_enter(pMaster); + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(pMainMtx); pBt->nRef--; if( pBt->nRef<=0 ){ if( GLOBAL(BtShared*,sqlite3SharedCacheList)==pBt ){ @@ -60165,7 +73507,7 @@ static int removeFromSharingList(BtShared *pBt){ } removed = 1; } - sqlite3_mutex_leave(pMaster); + sqlite3_mutex_leave(pMainMtx); return removed; #else return 1; @@ -60173,34 +73515,42 @@ static int removeFromSharingList(BtShared *pBt){ } /* -** Make sure pBt->pTmpSpace points to an allocation of +** Make sure pBt->pTmpSpace points to an allocation of ** MX_CELL_SIZE(pBt) bytes with a 4-byte prefix for a left-child ** pointer. */ -static void allocateTempSpace(BtShared *pBt){ - if( !pBt->pTmpSpace ){ - pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); - - /* One of the uses of pBt->pTmpSpace is to format cells before - ** inserting them into a leaf page (function fillInCell()). If - ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes - ** by the various routines that manipulate binary cells. Which - ** can mean that fillInCell() only initializes the first 2 or 3 - ** bytes of pTmpSpace, but that the first 4 bytes are copied from - ** it into a database page. This is not actually a problem, but it - ** does cause a valgrind error when the 1 or 2 bytes of unitialized - ** data is passed to system call write(). So to avoid this error, - ** zero the first 4 bytes of temp space here. - ** - ** Also: Provide four bytes of initialized space before the - ** beginning of pTmpSpace as an area available to prepend the - ** left-child pointer to the beginning of a cell. - */ - if( pBt->pTmpSpace ){ - memset(pBt->pTmpSpace, 0, 8); - pBt->pTmpSpace += 4; - } +static SQLITE_NOINLINE int allocateTempSpace(BtShared *pBt){ + assert( pBt!=0 ); + assert( pBt->pTmpSpace==0 ); + /* This routine is called only by btreeCursor() when allocating the + ** first write cursor for the BtShared object */ + assert( pBt->pCursor!=0 && (pBt->pCursor->curFlags & BTCF_WriteFlag)!=0 ); + pBt->pTmpSpace = sqlite3PageMalloc( pBt->pageSize ); + if( pBt->pTmpSpace==0 ){ + BtCursor *pCur = pBt->pCursor; + pBt->pCursor = pCur->pNext; /* Unlink the cursor */ + memset(pCur, 0, sizeof(*pCur)); + return SQLITE_NOMEM_BKPT; } + + /* One of the uses of pBt->pTmpSpace is to format cells before + ** inserting them into a leaf page (function fillInCell()). If + ** a cell is less than 4 bytes in size, it is rounded up to 4 bytes + ** by the various routines that manipulate binary cells. Which + ** can mean that fillInCell() only initializes the first 2 or 3 + ** bytes of pTmpSpace, but that the first 4 bytes are copied from + ** it into a database page. This is not actually a problem, but it + ** does cause a valgrind error when the 1 or 2 bytes of uninitialized + ** data is passed to system call write(). So to avoid this error, + ** zero the first 4 bytes of temp space here. + ** + ** Also: Provide four bytes of initialized space before the + ** beginning of pTmpSpace as an area available to prepend the + ** left-child pointer to the beginning of a cell. + */ + memset(pBt->pTmpSpace, 0, 8); + pBt->pTmpSpace += 4; + return SQLITE_OK; } /* @@ -60219,19 +73569,23 @@ static void freeTempSpace(BtShared *pBt){ */ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ BtShared *pBt = p->pBt; - BtCursor *pCur; /* Close all cursors opened via this handle. */ assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - pCur = pBt->pCursor; - while( pCur ){ - BtCursor *pTmp = pCur; - pCur = pCur->pNext; - if( pTmp->pBtree==p ){ - sqlite3BtreeCloseCursor(pTmp); + + /* Verify that no other cursors have this Btree open */ +#ifdef SQLITE_DEBUG + { + BtCursor *pCur = pBt->pCursor; + while( pCur ){ + BtCursor *pTmp = pCur; + pCur = pCur->pNext; + assert( pTmp->pBtree!=p ); + } } +#endif /* Rollback any active transaction and free the handle structure. ** The call to sqlite3BtreeRollback() drops any table-locks held by @@ -60241,7 +73595,7 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ sqlite3BtreeLeave(p); /* If there are still other outstanding references to the shared-btree - ** structure, return now. The remainder of this procedure cleans + ** structure, return now. The remainder of this procedure cleans ** up the shared-btree. */ assert( p->wantToLock==0 && p->locked==0 ); @@ -60252,7 +73606,7 @@ SQLITE_PRIVATE int sqlite3BtreeClose(Btree *p){ ** Clean out and delete the BtShared object. */ assert( !pBt->pCursor ); - sqlite3PagerClose(pBt->pPager); + sqlite3PagerClose(pBt->pPager, p->db); if( pBt->xFreeSchema && pBt->pSchema ){ pBt->xFreeSchema(pBt->pSchema); } @@ -60347,7 +73701,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( /* ** Change the default pages size and the number of reserved bytes per page. -** Or, if the page size has already been fixed, return SQLITE_READONLY +** Or, if the page size has already been fixed, return SQLITE_READONLY ** without changing anything. ** ** The page size must be a power of 2 between 512 and 65536. If the page @@ -60367,24 +73721,23 @@ SQLITE_PRIVATE int sqlite3BtreeSetPagerFlags( */ SQLITE_PRIVATE int sqlite3BtreeSetPageSize(Btree *p, int pageSize, int nReserve, int iFix){ int rc = SQLITE_OK; + int x; BtShared *pBt = p->pBt; - assert( nReserve>=-1 && nReserve<=255 ); + assert( nReserve>=0 && nReserve<=255 ); sqlite3BtreeEnter(p); -#if SQLITE_HAS_CODEC - if( nReserve>pBt->optimalReserve ) pBt->optimalReserve = (u8)nReserve; -#endif + pBt->nReserveWanted = nReserve; + x = pBt->pageSize - pBt->usableSize; + if( nReservebtsFlags & BTS_PAGESIZE_FIXED ){ sqlite3BtreeLeave(p); return SQLITE_READONLY; } - if( nReserve<0 ){ - nReserve = pBt->pageSize - pBt->usableSize; - } assert( nReserve>=0 && nReserve<=255 ); if( pageSize>=512 && pageSize<=SQLITE_MAX_PAGE_SIZE && ((pageSize-1)&pageSize)==0 ){ assert( (pageSize & 7)==0 ); assert( !pBt->pCursor ); + if( nReserve>32 && pageSize==512 ) pageSize = 1024; pBt->pageSize = (u32)pageSize; freeTempSpace(pBt); } @@ -60408,7 +73761,7 @@ SQLITE_PRIVATE int sqlite3BtreeGetPageSize(Btree *p){ ** held. ** ** This is useful in one special case in the backup API code where it is -** known that the shared b-tree mutex is held, but the mutex on the +** known that the shared b-tree mutex is held, but the mutex on the ** database handle that owns *p is not. In this case if sqlite3BtreeEnter() ** were to be called, it might collide with some other operation on the ** database handle that owns *p, causing undefined behavior. @@ -60422,22 +73775,20 @@ SQLITE_PRIVATE int sqlite3BtreeGetReserveNoMutex(Btree *p){ /* ** Return the number of bytes of space at the end of every page that -** are intentually left unused. This is the "reserved" space that is +** are intentionally left unused. This is the "reserved" space that is ** sometimes used by extensions. ** -** If SQLITE_HAS_MUTEX is defined then the number returned is the -** greater of the current reserved space and the maximum requested -** reserve space. +** The value returned is the larger of the current reserve size and +** the latest reserve size requested by SQLITE_FILECTRL_RESERVE_BYTES. +** The amount of reserve can only grow - never shrink. */ -SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree *p){ - int n; +SQLITE_PRIVATE int sqlite3BtreeGetRequestedReserve(Btree *p){ + int n1, n2; sqlite3BtreeEnter(p); - n = sqlite3BtreeGetReserveNoMutex(p); -#ifdef SQLITE_HAS_CODEC - if( npBt->optimalReserve ) n = p->pBt->optimalReserve; -#endif + n1 = (int)p->pBt->nReserveWanted; + n2 = sqlite3BtreeGetReserveNoMutex(p); sqlite3BtreeLeave(p); - return n; + return n1>n2 ? n1 : n2; } @@ -60446,8 +73797,8 @@ SQLITE_PRIVATE int sqlite3BtreeGetOptimalReserve(Btree *p){ ** No changes are made if mxPage is 0 or negative. ** Regardless of the value of mxPage, return the maximum page count. */ -SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ - int n; +SQLITE_PRIVATE Pgno sqlite3BtreeMaxPageCount(Btree *p, Pgno mxPage){ + Pgno n; sqlite3BtreeEnter(p); n = sqlite3PagerMaxPageCount(p->pBt->pPager, mxPage); sqlite3BtreeLeave(p); @@ -60455,19 +73806,34 @@ SQLITE_PRIVATE int sqlite3BtreeMaxPageCount(Btree *p, int mxPage){ } /* -** Set the BTS_SECURE_DELETE flag if newFlag is 0 or 1. If newFlag is -1, -** then make no changes. Always return the value of the BTS_SECURE_DELETE -** setting after the change. +** Change the values for the BTS_SECURE_DELETE and BTS_OVERWRITE flags: +** +** newFlag==0 Both BTS_SECURE_DELETE and BTS_OVERWRITE are cleared +** newFlag==1 BTS_SECURE_DELETE set and BTS_OVERWRITE is cleared +** newFlag==2 BTS_SECURE_DELETE cleared and BTS_OVERWRITE is set +** newFlag==(-1) No changes +** +** This routine acts as a query if newFlag is less than zero +** +** With BTS_OVERWRITE set, deleted content is overwritten by zeros, but +** freelist leaf pages are not written back to the database. Thus in-page +** deleted content is cleared, but freelist deleted content is not. +** +** With BTS_SECURE_DELETE, operation is like BTS_OVERWRITE with the addition +** that freelist leaf pages are written back into the database, increasing +** the amount of disk I/O. */ SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ int b; if( p==0 ) return 0; sqlite3BtreeEnter(p); + assert( BTS_OVERWRITE==BTS_SECURE_DELETE*2 ); + assert( BTS_FAST_SECURE==(BTS_OVERWRITE|BTS_SECURE_DELETE) ); if( newFlag>=0 ){ - p->pBt->btsFlags &= ~BTS_SECURE_DELETE; - if( newFlag ) p->pBt->btsFlags |= BTS_SECURE_DELETE; - } - b = (p->pBt->btsFlags & BTS_SECURE_DELETE)!=0; + p->pBt->btsFlags &= ~BTS_FAST_SECURE; + p->pBt->btsFlags |= BTS_SECURE_DELETE*newFlag; + } + b = (p->pBt->btsFlags & BTS_FAST_SECURE)/BTS_SECURE_DELETE; sqlite3BtreeLeave(p); return b; } @@ -60475,7 +73841,7 @@ SQLITE_PRIVATE int sqlite3BtreeSecureDelete(Btree *p, int newFlag){ /* ** Change the 'auto-vacuum' property of the database. If the 'autoVacuum' ** parameter is non-zero, then auto-vacuum mode is enabled. If zero, it -** is disabled. The default value for the auto-vacuum property is +** is disabled. The default value for the auto-vacuum property is ** determined by the SQLITE_DEFAULT_AUTOVACUUM macro. */ SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ @@ -60499,7 +73865,7 @@ SQLITE_PRIVATE int sqlite3BtreeSetAutoVacuum(Btree *p, int autoVacuum){ } /* -** Return the value of the 'auto-vacuum' property. If auto-vacuum is +** Return the value of the 'auto-vacuum' property. If auto-vacuum is ** enabled 1 is returned. Otherwise 0. */ SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ @@ -60518,6 +73884,36 @@ SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ #endif } +/* +** If the user has not set the safety-level for this database connection +** using "PRAGMA synchronous", and if the safety-level is not already +** set to the value passed to this function as the second parameter, +** set it so. +*/ +#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS \ + && !defined(SQLITE_OMIT_WAL) +static void setDefaultSyncFlag(BtShared *pBt, u8 safety_level){ + sqlite3 *db; + Db *pDb; + if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ + while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } + if( pDb->bSyncSet==0 + && pDb->safety_level!=safety_level + && pDb!=&db->aDb[1] + ){ + pDb->safety_level = safety_level; + sqlite3PagerSetFlags(pBt->pPager, + pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); + } + } +} +#else +# define setDefaultSyncFlag(pBt,safety_level) +#endif + +/* Forward declaration */ +static int newDatabase(BtShared*); + /* ** Get a reference to pPage1 of the database file. This will @@ -60526,14 +73922,13 @@ SQLITE_PRIVATE int sqlite3BtreeGetAutoVacuum(Btree *p){ ** SQLITE_OK is returned on success. If the file is not a ** well-formed database file, then SQLITE_CORRUPT is returned. ** SQLITE_BUSY is returned if the database is locked. SQLITE_NOMEM -** is returned if we run out of memory. +** is returned if we run out of memory. */ static int lockBtree(BtShared *pBt){ int rc; /* Result code from subfunctions */ MemPage *pPage1; /* Page 1 of the database file */ - int nPage; /* Number of pages in the database */ - int nPageFile = 0; /* Number of pages in the database file */ - int nPageHeader; /* Number of pages in the database according to hdr */ + u32 nPage; /* Number of pages in the database */ + u32 nPageFile = 0; /* Number of pages in the database file */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( pBt->pPage1==0 ); @@ -60543,13 +73938,16 @@ static int lockBtree(BtShared *pBt){ if( rc!=SQLITE_OK ) return rc; /* Do some checking to help insure the file we opened really is - ** a valid database file. + ** a valid database file. */ - nPage = nPageHeader = get4byte(28+(u8*)pPage1->aData); - sqlite3PagerPagecount(pBt->pPager, &nPageFile); + nPage = get4byte(28+(u8*)pPage1->aData); + sqlite3PagerPagecount(pBt->pPager, (int*)&nPageFile); if( nPage==0 || memcmp(24+(u8*)pPage1->aData, 92+(u8*)pPage1->aData,4)!=0 ){ nPage = nPageFile; } + if( (pBt->db->flags & SQLITE_ResetDatabase)!=0 ){ + nPage = 0; + } if( nPage>0 ){ u32 pageSize; u32 usableSize; @@ -60577,8 +73975,8 @@ static int lockBtree(BtShared *pBt){ goto page1_init_failed; } - /* If the write version is set to 2, this database should be accessed - ** in WAL mode. If the log is not already open, open it now. Then + /* If the read version is set to 2, this database should be accessed + ** in WAL mode. If the log is not already open, open it now. Then ** return SQLITE_OK and return without populating BtShared.pPage1. ** The caller detects this and calls this function again. This is ** required as the version of page 1 currently in the page1 buffer @@ -60591,26 +73989,15 @@ static int lockBtree(BtShared *pBt){ if( rc!=SQLITE_OK ){ goto page1_init_failed; }else{ -#if SQLITE_DEFAULT_SYNCHRONOUS!=SQLITE_DEFAULT_WAL_SYNCHRONOUS - sqlite3 *db; - Db *pDb; - if( (db=pBt->db)!=0 && (pDb=db->aDb)!=0 ){ - while( pDb->pBt==0 || pDb->pBt->pBt!=pBt ){ pDb++; } - if( pDb->bSyncSet==0 - && pDb->safety_level==SQLITE_DEFAULT_SYNCHRONOUS+1 - ){ - pDb->safety_level = SQLITE_DEFAULT_WAL_SYNCHRONOUS+1; - sqlite3PagerSetFlags(pBt->pPager, - pDb->safety_level | (db->flags & PAGER_FLAGS_MASK)); - } - } -#endif + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_WAL_SYNCHRONOUS+1); if( isOpen==0 ){ - releasePage(pPage1); + releasePageOne(pPage1); return SQLITE_OK; } } rc = SQLITE_NOTADB; + }else{ + setDefaultSyncFlag(pBt, SQLITE_DEFAULT_SYNCHRONOUS+1); } #endif @@ -60630,15 +74017,15 @@ static int lockBtree(BtShared *pBt){ /* EVIDENCE-OF: R-25008-21688 The size of a page is a power of two ** between 512 and 65536 inclusive. */ if( ((pageSize-1)&pageSize)!=0 - || pageSize>SQLITE_MAX_PAGE_SIZE - || pageSize<=256 + || pageSize>SQLITE_MAX_PAGE_SIZE + || pageSize<=256 ){ goto page1_init_failed; } assert( (pageSize & 7)==0 ); /* EVIDENCE-OF: R-59310-51205 The "reserved space" size in the 1-byte ** integer at offset 20 is the number of bytes of space at the end of - ** each page to reserve for extensions. + ** each page to reserve for extensions. ** ** EVIDENCE-OF: R-37497-42412 The size of the reserved region is ** determined by the one-byte unsigned integer found at an offset of 20 @@ -60651,17 +74038,22 @@ static int lockBtree(BtShared *pBt){ ** zero and return SQLITE_OK. The caller will call this function ** again with the correct page-size. */ - releasePage(pPage1); + releasePageOne(pPage1); pBt->usableSize = usableSize; pBt->pageSize = pageSize; + pBt->btsFlags |= BTS_PAGESIZE_FIXED; freeTempSpace(pBt); rc = sqlite3PagerSetPagesize(pBt->pPager, &pBt->pageSize, pageSize-usableSize); return rc; } - if( (pBt->db->flags & SQLITE_RecoveryMode)==0 && nPage>nPageFile ){ - rc = SQLITE_CORRUPT_BKPT; - goto page1_init_failed; + if( nPage>nPageFile ){ + if( sqlite3WritableSchema(pBt->db)==0 ){ + rc = SQLITE_CORRUPT_BKPT; + goto page1_init_failed; + }else{ + nPage = nPageFile; + } } /* EVIDENCE-OF: R-28312-64704 However, the usable size is not allowed to ** be less than 480. In other words, if the page size is 512, then the @@ -60669,6 +74061,7 @@ static int lockBtree(BtShared *pBt){ if( usableSize<480 ){ goto page1_init_failed; } + pBt->btsFlags |= BTS_PAGESIZE_FIXED; pBt->pageSize = pageSize; pBt->usableSize = usableSize; #ifndef SQLITE_OMIT_AUTOVACUUM @@ -60705,7 +74098,7 @@ static int lockBtree(BtShared *pBt){ return SQLITE_OK; page1_init_failed: - releasePage(pPage1); + releasePageOne(pPage1); pBt->pPage1 = 0; return rc; } @@ -60728,7 +74121,7 @@ static int countValidCursors(BtShared *pBt, int wrOnly){ int r = 0; for(pCur=pBt->pCursor; pCur; pCur=pCur->pNext){ if( (wrOnly==0 || (pCur->curFlags & BTCF_WriteFlag)!=0) - && pCur->eState!=CURSOR_FAULT ) r++; + && pCur->eState!=CURSOR_FAULT ) r++; } return r; } @@ -60737,7 +74130,7 @@ static int countValidCursors(BtShared *pBt, int wrOnly){ /* ** If there are no outstanding cursors and we are not in the middle ** of a transaction but there is a read lock on the database, then -** this routine unrefs the first page of the database file which +** this routine unrefs the first page of the database file which ** has the effect of releasing the read lock. ** ** If there is a transaction in progress, this routine is a no-op. @@ -60750,7 +74143,7 @@ static void unlockBtreeIfUnused(BtShared *pBt){ assert( pPage1->aData ); assert( sqlite3PagerRefcount(pBt->pPager)==1 ); pBt->pPage1 = 0; - releasePageNotNull(pPage1); + releasePageOne(pPage1); } } @@ -60821,8 +74214,8 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** upgraded to exclusive by calling this routine a second time - the ** exclusivity flag only works for a new transaction. ** -** A write-transaction must be started before attempting any -** changes to the database. None of the following routines +** A write-transaction must be started before attempting any +** changes to the database. None of the following routines ** will work unless a transaction is started first: ** ** sqlite3BtreeCreateTable() @@ -60836,7 +74229,7 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** If an initial attempt to acquire the lock fails because of lock contention ** and the database was previously unlocked, then invoke the busy handler ** if there is one. But if there was previously a read-lock, do not -** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is +** invoke the busy handler - just return SQLITE_BUSY. SQLITE_BUSY is ** returned when there is already a read-lock in order to avoid a deadlock. ** ** Suppose there are two processes A and B. A has a read lock and B has @@ -60847,8 +74240,13 @@ SQLITE_PRIVATE int sqlite3BtreeNewDb(Btree *p){ ** when A already has a read lock, we encourage A to give up and let B ** proceed. */ -SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ +static SQLITE_NOINLINE int btreeBeginTrans( + Btree *p, /* The btree in which to start the transaction */ + int wrflag, /* True to start a write transaction */ + int *pSchemaVersion /* Put schema version number here, if not NULL */ +){ BtShared *pBt = p->pBt; + Pager *pPager = pBt->pPager; int rc = SQLITE_OK; sqlite3BtreeEnter(p); @@ -60863,6 +74261,12 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } assert( pBt->inTransaction==TRANS_WRITE || IfNotOmitAV(pBt->bDoTruncate)==0 ); + if( (p->db->flags & SQLITE_ResetDatabase) + && sqlite3PagerIsreadonly(pPager)==0 + ){ + pBt->btsFlags &= ~BTS_READ_ONLY; + } + /* Write transactions are not possible on a read-only database */ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 && wrflag ){ rc = SQLITE_READONLY; @@ -60872,7 +74276,7 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ #ifndef SQLITE_OMIT_SHARED_CACHE { sqlite3 *pBlock = 0; - /* If another database handle has already opened a write transaction + /* If another database handle has already opened a write transaction ** on this shared-btree structure and a second write transaction is ** requested, return SQLITE_LOCKED. */ @@ -60897,19 +74301,31 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } #endif - /* Any read-only or read-write transaction implies a read-lock on - ** page 1. So if some other shared-cache client already has a write-lock + /* Any read-only or read-write transaction implies a read-lock on + ** page 1. So if some other shared-cache client already has a write-lock ** on page 1, the transaction cannot be opened. */ - rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); + rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); if( SQLITE_OK!=rc ) goto trans_begun; pBt->btsFlags &= ~BTS_INITIALLY_EMPTY; if( pBt->nPage==0 ) pBt->btsFlags |= BTS_INITIALLY_EMPTY; do { + sqlite3PagerWalDb(pPager, p->db); + +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + /* If transitioning from no transaction directly to a write transaction, + ** block for the WRITER lock first if possible. */ + if( pBt->pPage1==0 && wrflag ){ + assert( pBt->inTransaction==TRANS_NONE ); + rc = sqlite3PagerWalWriteLock(pPager, 1); + if( rc!=SQLITE_BUSY && rc!=SQLITE_OK ) break; + } +#endif + /* Call lockBtree() until either pBt->pPage1 is populated or ** lockBtree() returns something other than SQLITE_OK. lockBtree() ** may return SQLITE_OK but leave pBt->pPage1 set to 0 if after - ** reading page 1 it discovers that the page-size of the database + ** reading page 1 it discovers that the page-size of the database ** file is not pBt->pageSize. In this case lockBtree() will update ** pBt->pageSize to the page-size of the file on disk. */ @@ -60919,18 +74335,28 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ if( (pBt->btsFlags & BTS_READ_ONLY)!=0 ){ rc = SQLITE_READONLY; }else{ - rc = sqlite3PagerBegin(pBt->pPager,wrflag>1,sqlite3TempInMemory(p->db)); + rc = sqlite3PagerBegin(pPager, wrflag>1, sqlite3TempInMemory(p->db)); if( rc==SQLITE_OK ){ rc = newDatabase(pBt); + }else if( rc==SQLITE_BUSY_SNAPSHOT && pBt->inTransaction==TRANS_NONE ){ + /* if there was no transaction opened when this function was + ** called and SQLITE_BUSY_SNAPSHOT is returned, change the error + ** code to SQLITE_BUSY. */ + rc = SQLITE_BUSY; } } } - + if( rc!=SQLITE_OK ){ + (void)sqlite3PagerWalWriteLock(pPager, 0); unlockBtreeIfUnused(pBt); } }while( (rc&0xFF)==SQLITE_BUSY && pBt->inTransaction==TRANS_NONE && btreeInvokeBusyHandler(pBt) ); + sqlite3PagerWalDb(pPager, 0); +#ifdef SQLITE_ENABLE_SETLK_TIMEOUT + if( rc==SQLITE_BUSY_TIMEOUT ) rc = SQLITE_BUSY; +#endif if( rc==SQLITE_OK ){ if( p->inTrans==TRANS_NONE ){ @@ -60959,7 +74385,7 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ /* If the db-size header field is incorrect (as it may be if an old ** client has been writing the database file), update it now. Doing - ** this sooner rather than later means the database size can safely + ** this sooner rather than later means the database size can safely ** re-read the database size from page 1 if a savepoint or transaction ** rollback occurs within the transaction. */ @@ -60972,20 +74398,46 @@ SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag){ } } - trans_begun: - if( rc==SQLITE_OK && wrflag ){ - /* This call makes sure that the pager has the correct number of - ** open savepoints. If the second parameter is greater than 0 and - ** the sub-journal is not already open, then it will be opened here. - */ - rc = sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + if( rc==SQLITE_OK ){ + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + rc = sqlite3PagerOpenSavepoint(pPager, p->db->nSavepoint); + } } btreeIntegrity(p); sqlite3BtreeLeave(p); return rc; } +SQLITE_PRIVATE int sqlite3BtreeBeginTrans(Btree *p, int wrflag, int *pSchemaVersion){ + BtShared *pBt; + if( p->sharable + || p->inTrans==TRANS_NONE + || (p->inTrans==TRANS_READ && wrflag!=0) + ){ + return btreeBeginTrans(p,wrflag,pSchemaVersion); + } + pBt = p->pBt; + if( pSchemaVersion ){ + *pSchemaVersion = get4byte(&pBt->pPage1->aData[40]); + } + if( wrflag ){ + /* This call makes sure that the pager has the correct number of + ** open savepoints. If the second parameter is greater than 0 and + ** the sub-journal is not already open, then it will be opened here. + */ + return sqlite3PagerOpenSavepoint(pBt->pPager, p->db->nSavepoint); + }else{ + return SQLITE_OK; + } +} #ifndef SQLITE_OMIT_AUTOVACUUM @@ -60999,20 +74451,17 @@ static int setChildPtrmaps(MemPage *pPage){ int nCell; /* Number of cells in page pPage */ int rc; /* Return code */ BtShared *pBt = pPage->pBt; - u8 isInitOrig = pPage->isInit; Pgno pgno = pPage->pgno; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - rc = btreeInitPage(pPage); - if( rc!=SQLITE_OK ){ - goto set_child_ptrmaps_out; - } + rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage); + if( rc!=SQLITE_OK ) return rc; nCell = pPage->nCell; for(i=0; ileaf ){ Pgno childPgno = get4byte(pCell); @@ -61025,8 +74474,6 @@ static int setChildPtrmaps(MemPage *pPage){ ptrmapPut(pBt, childPgno, PTRMAP_BTREE, pgno, &rc); } -set_child_ptrmaps_out: - pPage->isInit = isInitOrig; return rc; } @@ -61035,7 +74482,7 @@ set_child_ptrmaps_out: ** that it points to iTo. Parameter eType describes the type of pointer to ** be modified, as follows: ** -** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child +** PTRMAP_BTREE: pPage is a btree-page. The pointer points at a child ** page of pPage. ** ** PTRMAP_OVERFLOW1: pPage is a btree-page. The pointer points at an overflow @@ -61050,16 +74497,15 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW2 ){ /* The pointer is always the first 4 bytes of the page in this case. */ if( get4byte(pPage->aData)!=iFrom ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } put4byte(pPage->aData, iTo); }else{ - u8 isInitOrig = pPage->isInit; int i; int nCell; int rc; - rc = btreeInitPage(pPage); + rc = pPage->isInit ? SQLITE_OK : btreeInitPage(pPage); if( rc ) return rc; nCell = pPage->nCell; @@ -61068,41 +74514,44 @@ static int modifyPagePointer(MemPage *pPage, Pgno iFrom, Pgno iTo, u8 eType){ if( eType==PTRMAP_OVERFLOW1 ){ CellInfo info; pPage->xParseCell(pPage, pCell, &info); - if( info.nLocalaData+pPage->maskPage - && iFrom==get4byte(pCell+info.nSize-4) - ){ - put4byte(pCell+info.nSize-4, iTo); - break; + if( info.nLocal pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( iFrom==get4byte(pCell+info.nSize-4) ){ + put4byte(pCell+info.nSize-4, iTo); + break; + } } }else{ + if( pCell+4 > pPage->aData+pPage->pBt->usableSize ){ + return SQLITE_CORRUPT_PAGE(pPage); + } if( get4byte(pCell)==iFrom ){ put4byte(pCell, iTo); break; } } } - + if( i==nCell ){ - if( eType!=PTRMAP_BTREE || + if( eType!=PTRMAP_BTREE || get4byte(&pPage->aData[pPage->hdrOffset+8])!=iFrom ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } put4byte(&pPage->aData[pPage->hdrOffset+8], iTo); } - - pPage->isInit = isInitOrig; } return SQLITE_OK; } /* -** Move the open database page pDbPage to location iFreePage in the +** Move the open database page pDbPage to location iFreePage in the ** database. The pDbPage reference remains valid. ** ** The isCommit flag indicates that there is no need to remember that -** the journal needs to be sync()ed before database page pDbPage->pgno +** the journal needs to be sync()ed before database page pDbPage->pgno ** can be written to. The caller has already promised not to write to that ** page. */ @@ -61119,13 +74568,14 @@ static int relocatePage( Pager *pPager = pBt->pPager; int rc; - assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || + assert( eType==PTRMAP_OVERFLOW2 || eType==PTRMAP_OVERFLOW1 || eType==PTRMAP_BTREE || eType==PTRMAP_ROOTPAGE ); assert( sqlite3_mutex_held(pBt->mutex) ); assert( pDbPage->pBt==pBt ); + if( iDbPage<3 ) return SQLITE_CORRUPT_BKPT; /* Move page iDbPage from its current location to page number iFreePage */ - TRACE(("AUTOVACUUM: Moving %d to free page %d (ptr page %d type %d)\n", + TRACE(("AUTOVACUUM: Moving %u to free page %u (ptr page %u type %u)\n", iDbPage, iFreePage, iPtrPage, eType)); rc = sqlite3PagerMovepage(pPager, pDbPage->pDbPage, iFreePage, isCommit); if( rc!=SQLITE_OK ){ @@ -61184,19 +74634,19 @@ static int allocateBtreePage(BtShared *, MemPage **, Pgno *, Pgno, u8); /* ** Perform a single step of an incremental-vacuum. If successful, return -** SQLITE_OK. If there is no work to do (and therefore no point in -** calling this function again), return SQLITE_DONE. Or, if an error +** SQLITE_OK. If there is no work to do (and therefore no point in +** calling this function again), return SQLITE_DONE. Or, if an error ** occurs, return some other error code. ** -** More specifically, this function attempts to re-organize the database so +** More specifically, this function attempts to re-organize the database so ** that the last page of the file currently in use is no longer in use. ** ** Parameter nFin is the number of pages that this database would contain ** were this function called until it returns SQLITE_DONE. ** -** If the bCommit parameter is non-zero, this function assumes that the -** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE -** or an error. bCommit is passed true for an auto-vacuum-on-commit +** If the bCommit parameter is non-zero, this function assumes that the +** caller will keep calling incrVacuumStep() until it returns SQLITE_DONE +** or an error. bCommit is passed true for an auto-vacuum-on-commit ** operation, or false for an incremental vacuum. */ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ @@ -61227,7 +74677,7 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ if( bCommit==0 ){ /* Remove the page from the files free-list. This is not required ** if bCommit is non-zero. In that case, the free-list will be - ** truncated to zero after this function returns, so it doesn't + ** truncated to zero after this function returns, so it doesn't ** matter if it still contains some garbage entries. */ Pgno iFreePg; @@ -61263,15 +74713,20 @@ static int incrVacuumStep(BtShared *pBt, Pgno nFin, Pgno iLastPg, int bCommit){ } do { MemPage *pFreePg; + Pgno dbSize = btreePagecount(pBt); rc = allocateBtreePage(pBt, &pFreePg, &iFreePg, iNear, eMode); if( rc!=SQLITE_OK ){ releasePage(pLastPg); return rc; } releasePage(pFreePg); + if( iFreePg>dbSize ){ + releasePage(pLastPg); + return SQLITE_CORRUPT_BKPT; + } }while( bCommit && iFreePg>nFin ); assert( iFreePgpPage1->aData[36]); Pgno nFin = finalDbSize(pBt, nOrig, nFree); - if( nOrig=nOrig ){ rc = SQLITE_CORRUPT_BKPT; }else if( nFree>0 ){ rc = saveAllCursors(pBt, 0, 0); @@ -61357,16 +74812,18 @@ SQLITE_PRIVATE int sqlite3BtreeIncrVacuum(Btree *p){ /* ** This routine is called prior to sqlite3PagerCommit when a transaction ** is committed for an auto-vacuum database. -** -** If SQLITE_OK is returned, then *pnTrunc is set to the number of pages -** the database file should be truncated to during the commit process. -** i.e. the database has been reorganized so that only the first *pnTrunc -** pages are in use. */ -static int autoVacuumCommit(BtShared *pBt){ +static int autoVacuumCommit(Btree *p){ int rc = SQLITE_OK; - Pager *pPager = pBt->pPager; - VVA_ONLY( int nRef = sqlite3PagerRefcount(pPager); ) + Pager *pPager; + BtShared *pBt; + sqlite3 *db; + VVA_ONLY( int nRef ); + + assert( p!=0 ); + pBt = p->pBt; + pPager = pBt->pPager; + VVA_ONLY( nRef = sqlite3PagerRefcount(pPager); ) assert( sqlite3_mutex_held(pBt->mutex) ); invalidateAllOverflowCache(pBt); @@ -61374,6 +74831,7 @@ static int autoVacuumCommit(BtShared *pBt){ if( !pBt->incrVacuum ){ Pgno nFin; /* Number of pages in database after autovacuuming */ Pgno nFree; /* Number of pages on the freelist initially */ + Pgno nVac; /* Number of pages to vacuum */ Pgno iFree; /* The next page to be freed */ Pgno nOrig; /* Database size before freeing */ @@ -61387,18 +74845,42 @@ static int autoVacuumCommit(BtShared *pBt){ } nFree = get4byte(&pBt->pPage1->aData[36]); - nFin = finalDbSize(pBt, nOrig, nFree); + db = p->db; + if( db->xAutovacPages ){ + int iDb; + for(iDb=0; ALWAYS(iDbnDb); iDb++){ + if( db->aDb[iDb].pBt==p ) break; + } + nVac = db->xAutovacPages( + db->pAutovacPagesArg, + db->aDb[iDb].zDbSName, + nOrig, + nFree, + pBt->pageSize + ); + if( nVac>nFree ){ + nVac = nFree; + } + if( nVac==0 ){ + return SQLITE_OK; + } + }else{ + nVac = nFree; + } + nFin = finalDbSize(pBt, nOrig, nVac); if( nFin>nOrig ) return SQLITE_CORRUPT_BKPT; if( nFinnFin && rc==SQLITE_OK; iFree--){ - rc = incrVacuumStep(pBt, nFin, iFree, 1); + rc = incrVacuumStep(pBt, nFin, iFree, nVac==nFree); } if( (rc==SQLITE_DONE || rc==SQLITE_OK) && nFree>0 ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); - put4byte(&pBt->pPage1->aData[32], 0); - put4byte(&pBt->pPage1->aData[36], 0); + if( nVac==nFree ){ + put4byte(&pBt->pPage1->aData[32], 0); + put4byte(&pBt->pPage1->aData[36], 0); + } put4byte(&pBt->pPage1->aData[28], nFin); pBt->bDoTruncate = 1; pBt->nPage = nFin; @@ -61431,25 +74913,25 @@ static int autoVacuumCommit(BtShared *pBt){ ** ** This call is a no-op if no write-transaction is currently active on pBt. ** -** Otherwise, sync the database file for the btree pBt. zMaster points to -** the name of a master journal file that should be written into the -** individual journal file, or is NULL, indicating no master journal file +** Otherwise, sync the database file for the btree pBt. zSuperJrnl points to +** the name of a super-journal file that should be written into the +** individual journal file, or is NULL, indicating no super-journal file ** (single database transaction). ** -** When this is called, the master journal should already have been +** When this is called, the super-journal should already have been ** created, populated with this journal pointer and synced to disk. ** ** Once this is routine has returned, the only thing required to commit ** the write-transaction for this database file is to delete the journal. */ -SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ +SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zSuperJrnl){ int rc = SQLITE_OK; if( p->inTrans==TRANS_WRITE ){ BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - rc = autoVacuumCommit(pBt); + rc = autoVacuumCommit(p); if( rc!=SQLITE_OK ){ sqlite3BtreeLeave(p); return rc; @@ -61459,7 +74941,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseOne(Btree *p, const char *zMaster){ sqlite3PagerTruncateImage(pBt->pPager, pBt->nPage); } #endif - rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zMaster, 0); + rc = sqlite3PagerCommitPhaseOne(pBt->pPager, zSuperJrnl, 0); sqlite3BtreeLeave(p); } return rc; @@ -61484,8 +74966,8 @@ static void btreeEndTransaction(Btree *p){ downgradeAllSharedCacheTableLocks(p); p->inTrans = TRANS_READ; }else{ - /* If the handle had any kind of transaction open, decrement the - ** transaction count of the shared btree. If the transaction count + /* If the handle had any kind of transaction open, decrement the + ** transaction count of the shared btree. If the transaction count ** reaches 0, set the shared state to TRANS_NONE. The unlockBtreeIfUnused() ** call below will unlock the pager. */ if( p->inTrans!=TRANS_NONE ){ @@ -61496,7 +74978,7 @@ static void btreeEndTransaction(Btree *p){ } } - /* Set the current transaction state to TRANS_NONE and unlock the + /* Set the current transaction state to TRANS_NONE and unlock the ** pager if this call closed the only read or write transaction. */ p->inTrans = TRANS_NONE; unlockBtreeIfUnused(pBt); @@ -61517,12 +74999,12 @@ static void btreeEndTransaction(Btree *p){ ** the rollback journal (which causes the transaction to commit) and ** drop locks. ** -** Normally, if an error occurs while the pager layer is attempting to +** Normally, if an error occurs while the pager layer is attempting to ** finalize the underlying journal file, this function returns an error and ** the upper layer will attempt a rollback. However, if the second argument -** is non-zero then this b-tree transaction is part of a multi-file -** transaction. In this case, the transaction has already been committed -** (by deleting a master journal file) and the caller will ignore this +** is non-zero then this b-tree transaction is part of a multi-file +** transaction. In this case, the transaction has already been committed +** (by deleting a super-journal file) and the caller will ignore this ** functions return code. So, even if an error occurs in the pager layer, ** reset the b-tree objects internal state to indicate that the write ** transaction has been closed. This is quite safe, as the pager will have @@ -61537,7 +75019,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeEnter(p); btreeIntegrity(p); - /* If the handle has a write-transaction open, commit the shared-btrees + /* If the handle has a write-transaction open, commit the shared-btrees ** transaction and set the shared state to TRANS_READ. */ if( p->inTrans==TRANS_WRITE ){ @@ -61550,7 +75032,7 @@ SQLITE_PRIVATE int sqlite3BtreeCommitPhaseTwo(Btree *p, int bCleanup){ sqlite3BtreeLeave(p); return rc; } - p->iDataVersion--; /* Compensate for pPager->iDataVersion++; */ + p->iBDataVersion--; /* Compensate for pPager->iDataVersion++; */ pBt->inTransaction = TRANS_READ; btreeClearHasContent(pBt); } @@ -61586,15 +75068,15 @@ SQLITE_PRIVATE int sqlite3BtreeCommit(Btree *p){ ** ** This routine gets called when a rollback occurs. If the writeOnly ** flag is true, then only write-cursors need be tripped - read-only -** cursors save their current positions so that they may continue -** following the rollback. Or, if writeOnly is false, all cursors are +** cursors save their current positions so that they may continue +** following the rollback. Or, if writeOnly is false, all cursors are ** tripped. In general, writeOnly is false if the transaction being ** rolled back modified the database schema. In this case b-tree root ** pages may be moved or deleted from the database altogether, making ** it unsafe for read cursors to continue. ** -** If the writeOnly flag is true and an error is encountered while -** saving the current position of a read-only cursor, all cursors, +** If the writeOnly flag is true and an error is encountered while +** saving the current position of a read-only cursor, all cursors, ** including all read-cursors are tripped. ** ** SQLITE_OK is returned if successful, or if an error occurs while @@ -61608,7 +75090,6 @@ SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int wr if( pBtree ){ sqlite3BtreeEnter(pBtree); for(p=pBtree->pBt->pCursor; p; p=p->pNext){ - int i; if( writeOnly && (p->curFlags & BTCF_WriteFlag)==0 ){ if( p->eState==CURSOR_VALID || p->eState==CURSOR_SKIPNEXT ){ rc = saveCursorPosition(p); @@ -61622,16 +75103,25 @@ SQLITE_PRIVATE int sqlite3BtreeTripAllCursors(Btree *pBtree, int errCode, int wr p->eState = CURSOR_FAULT; p->skipNext = errCode; } - for(i=0; i<=p->iPage; i++){ - releasePage(p->apPage[i]); - p->apPage[i] = 0; - } + btreeReleaseAllCursorPages(p); } sqlite3BtreeLeave(pBtree); } return rc; } +/* +** Set the pBt->nPage field correctly, according to the current +** state of the database. Assume pBt->pPage1 is valid. +*/ +static void btreeSetNPage(BtShared *pBt, MemPage *pPage1){ + int nPage = get4byte(&pPage1->aData[28]); + testcase( nPage==0 ); + if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); + testcase( pBt->nPage!=(u32)nPage ); + pBt->nPage = nPage; +} + /* ** Rollback the transaction in progress. ** @@ -61677,12 +75167,8 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ ** call btreeGetPage() on page 1 again to make ** sure pPage1->aData is set correctly. */ if( btreeGetPage(pBt, 1, &pPage1, 0)==SQLITE_OK ){ - int nPage = get4byte(28+(u8*)pPage1->aData); - testcase( nPage==0 ); - if( nPage==0 ) sqlite3PagerPagecount(pBt->pPager, &nPage); - testcase( pBt->nPage!=nPage ); - pBt->nPage = nPage; - releasePage(pPage1); + btreeSetNPage(pBt, pPage1); + releasePageOne(pPage1); } assert( countValidCursors(pBt, 1)==0 ); pBt->inTransaction = TRANS_READ; @@ -61696,8 +75182,8 @@ SQLITE_PRIVATE int sqlite3BtreeRollback(Btree *p, int tripCode, int writeOnly){ /* ** Start a statement subtransaction. The subtransaction can be rolled -** back independently of the main transaction. You must start a transaction -** before starting a subtransaction. The subtransaction is ended automatically +** back independently of the main transaction. You must start a transaction +** before starting a subtransaction. The subtransaction is ended automatically ** if the main transaction commits or rolls back. ** ** Statement subtransactions are used around individual SQL statements @@ -61734,11 +75220,11 @@ SQLITE_PRIVATE int sqlite3BtreeBeginStmt(Btree *p, int iStatement){ /* ** The second argument to this function, op, is always SAVEPOINT_ROLLBACK ** or SAVEPOINT_RELEASE. This function either releases or rolls back the -** savepoint identified by parameter iSavepoint, depending on the value +** savepoint identified by parameter iSavepoint, depending on the value ** of op. ** ** Normally, iSavepoint is greater than or equal to zero. However, if op is -** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the +** SAVEPOINT_ROLLBACK, then iSavepoint may also be -1. In this case the ** contents of the entire transaction are rolled back. This is different ** from a normal transaction rollback, as no locks are released and the ** transaction remains open. @@ -61750,18 +75236,22 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ assert( op==SAVEPOINT_RELEASE || op==SAVEPOINT_ROLLBACK ); assert( iSavepoint>=0 || (iSavepoint==-1 && op==SAVEPOINT_ROLLBACK) ); sqlite3BtreeEnter(p); - rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + if( op==SAVEPOINT_ROLLBACK ){ + rc = saveAllCursors(pBt, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSavepoint(pBt->pPager, op, iSavepoint); + } if( rc==SQLITE_OK ){ if( iSavepoint<0 && (pBt->btsFlags & BTS_INITIALLY_EMPTY)!=0 ){ pBt->nPage = 0; } rc = newDatabase(pBt); - pBt->nPage = get4byte(28 + pBt->pPage1->aData); + btreeSetNPage(pBt, pBt->pPage1); - /* The database size was written into the offset 28 of the header - ** when the transaction started, so we know that the value at offset - ** 28 is nonzero. */ - assert( pBt->nPage>0 ); + /* pBt->nPage might be zero if the database was corrupt when + ** the transaction was started. Otherwise, it must be at least 1. */ + assert( CORRUPT_DB || pBt->nPage>0 ); } sqlite3BtreeLeave(p); } @@ -61797,10 +75287,10 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ ** is set. If FORDELETE is set, that is a hint to the implementation that ** this cursor will only be used to seek to and delete entries of an index ** as part of a larger DELETE statement. The FORDELETE hint is not used by -** this implementation. But in a hypothetical alternative storage engine +** this implementation. But in a hypothetical alternative storage engine ** in which index entries are automatically deleted when corresponding table ** rows are deleted, the FORDELETE flag is a hint that all SEEK and DELETE -** operations on this cursor can be no-ops and all READ operations can +** operations on this cursor can be no-ops and all READ operations can ** return a null row (2-bytes: 0x01 0x00). ** ** No checking is done to make sure that page iTable really is the @@ -61812,7 +75302,7 @@ SQLITE_PRIVATE int sqlite3BtreeSavepoint(Btree *p, int op, int iSavepoint){ */ static int btreeCursor( Btree *p, /* The btree */ - int iTable, /* Root page of table to open */ + Pgno iTable, /* Root page of table to open */ int wrFlag, /* 1 to write. 0 read-only */ struct KeyInfo *pKeyInfo, /* First arg to comparison function */ BtCursor *pCur /* Space for new cursor */ @@ -61821,16 +75311,17 @@ static int btreeCursor( BtCursor *pX; /* Looping over other all cursors */ assert( sqlite3BtreeHoldsMutex(p) ); - assert( wrFlag==0 - || wrFlag==BTREE_WRCSR - || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) + assert( wrFlag==0 + || wrFlag==BTREE_WRCSR + || wrFlag==(BTREE_WRCSR|BTREE_FORDELETE) ); - /* The following assert statements verify that if this is a sharable - ** b-tree database, the connection is holding the required table locks, - ** and that no other connection has any open cursor that conflicts with - ** this lock. */ - assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) ); + /* The following assert statements verify that if this is a sharable + ** b-tree database, the connection is holding the required table locks, + ** and that no other connection has any open cursor that conflicts with + ** this lock. The iTable<1 term disables the check for corrupt schemas. */ + assert( hasSharedCacheTableLock(p, iTable, pKeyInfo!=0, (wrFlag?2:1)) + || iTable<1 ); assert( wrFlag==0 || !hasReadConflicts(p, iTable) ); /* Assert that the caller has opened the required transaction. */ @@ -61839,53 +75330,68 @@ static int btreeCursor( assert( pBt->pPage1 && pBt->pPage1->aData ); assert( wrFlag==0 || (pBt->btsFlags & BTS_READ_ONLY)==0 ); - if( wrFlag ){ - allocateTempSpace(pBt); - if( pBt->pTmpSpace==0 ) return SQLITE_NOMEM_BKPT; - } - if( iTable==1 && btreePagecount(pBt)==0 ){ - assert( wrFlag==0 ); - iTable = 0; + if( iTable<=1 ){ + if( iTable<1 ){ + return SQLITE_CORRUPT_BKPT; + }else if( btreePagecount(pBt)==0 ){ + assert( wrFlag==0 ); + iTable = 0; + } } /* Now that no other errors can occur, finish filling in the BtCursor ** variables and link the cursor into the BtShared list. */ - pCur->pgnoRoot = (Pgno)iTable; + pCur->pgnoRoot = iTable; pCur->iPage = -1; pCur->pKeyInfo = pKeyInfo; pCur->pBtree = p; pCur->pBt = pBt; - pCur->curFlags = wrFlag ? BTCF_WriteFlag : 0; - pCur->curPagerFlags = wrFlag ? 0 : PAGER_GET_READONLY; + pCur->curFlags = 0; /* If there are two or more cursors on the same btree, then all such ** cursors *must* have the BTCF_Multiple flag set. */ for(pX=pBt->pCursor; pX; pX=pX->pNext){ - if( pX->pgnoRoot==(Pgno)iTable ){ + if( pX->pgnoRoot==iTable ){ pX->curFlags |= BTCF_Multiple; - pCur->curFlags |= BTCF_Multiple; + pCur->curFlags = BTCF_Multiple; } } + pCur->eState = CURSOR_INVALID; pCur->pNext = pBt->pCursor; pBt->pCursor = pCur; - pCur->eState = CURSOR_INVALID; + if( wrFlag ){ + pCur->curFlags |= BTCF_WriteFlag; + pCur->curPagerFlags = 0; + if( pBt->pTmpSpace==0 ) return allocateTempSpace(pBt); + }else{ + pCur->curPagerFlags = PAGER_GET_READONLY; + } return SQLITE_OK; } +static int btreeCursorWithLock( + Btree *p, /* The btree */ + Pgno iTable, /* Root page of table to open */ + int wrFlag, /* 1 to write. 0 read-only */ + struct KeyInfo *pKeyInfo, /* First arg to comparison function */ + BtCursor *pCur /* Space for new cursor */ +){ + int rc; + sqlite3BtreeEnter(p); + rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); + sqlite3BtreeLeave(p); + return rc; +} SQLITE_PRIVATE int sqlite3BtreeCursor( Btree *p, /* The btree */ - int iTable, /* Root page of table to open */ + Pgno iTable, /* Root page of table to open */ int wrFlag, /* 1 to write. 0 read-only */ struct KeyInfo *pKeyInfo, /* First arg to xCompare() */ BtCursor *pCur /* Write new cursor here */ ){ - int rc; - if( iTable<1 ){ - rc = SQLITE_CORRUPT_BKPT; + if( p->sharable ){ + return btreeCursorWithLock(p, iTable, wrFlag, pKeyInfo, pCur); }else{ - sqlite3BtreeEnter(p); - rc = btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); - sqlite3BtreeLeave(p); + return btreeCursor(p, iTable, wrFlag, pKeyInfo, pCur); } - return rc; } /* @@ -61900,6 +75406,25 @@ SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){ return ROUND8(sizeof(BtCursor)); } +#ifdef SQLITE_DEBUG +/* +** Return true if and only if the Btree object will be automatically +** closed with the BtCursor closes. This is used within assert() statements +** only. +*/ +SQLITE_PRIVATE int sqlite3BtreeClosesWithCursor( + Btree *pBtree, /* the btree object */ + BtCursor *pCur /* Corresponding cursor */ +){ + BtShared *pBt = pBtree->pBt; + if( (pBt->openFlags & BTREE_SINGLE)==0 ) return 0; + if( pBt->pCursor!=pCur ) return 0; + if( pCur->pNext!=0 ) return 0; + if( pCur->pBtree!=pBtree ) return 0; + return 1; +} +#endif + /* ** Initialize memory that will be converted into a BtCursor object. ** @@ -61909,7 +75434,7 @@ SQLITE_PRIVATE int sqlite3BtreeCursorSize(void){ ** of run-time by skipping the initialization of those elements. */ SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ - memset(p, 0, offsetof(BtCursor, iPage)); + memset(p, 0, offsetof(BtCursor, BTCURSOR_FIRST_UNINIT)); } /* @@ -61919,10 +75444,8 @@ SQLITE_PRIVATE void sqlite3BtreeCursorZero(BtCursor *p){ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ Btree *pBtree = pCur->pBtree; if( pBtree ){ - int i; BtShared *pBt = pCur->pBt; sqlite3BtreeEnter(pBtree); - sqlite3BtreeClearCursor(pCur); assert( pBt->pCursor!=0 ); if( pBt->pCursor==pCur ){ pBt->pCursor = pCur->pNext; @@ -61936,13 +75459,19 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ pPrev = pPrev->pNext; }while( ALWAYS(pPrev) ); } - for(i=0; i<=pCur->iPage; i++){ - releasePage(pCur->apPage[i]); - } + btreeReleaseAllCursorPages(pCur); unlockBtreeIfUnused(pBt); sqlite3_free(pCur->aOverflow); - /* sqlite3_free(pCur); */ - sqlite3BtreeLeave(pBtree); + sqlite3_free(pCur->pKey); + if( (pBt->openFlags & BTREE_SINGLE) && pBt->pCursor==0 ){ + /* Since the BtShared is not sharable, there is no need to + ** worry about the missing sqlite3BtreeLeave() call here. */ + assert( pBtree->sharable==0 ); + sqlite3BtreeClose(pBtree); + }else{ + sqlite3BtreeLeave(pBtree); + } + pCur->pBtree = 0; } return SQLITE_OK; } @@ -61956,21 +75485,27 @@ SQLITE_PRIVATE int sqlite3BtreeCloseCursor(BtCursor *pCur){ ** Using this cache reduces the number of calls to btreeParseCell(). */ #ifndef NDEBUG + static int cellInfoEqual(CellInfo *a, CellInfo *b){ + if( a->nKey!=b->nKey ) return 0; + if( a->pPayload!=b->pPayload ) return 0; + if( a->nPayload!=b->nPayload ) return 0; + if( a->nLocal!=b->nLocal ) return 0; + if( a->nSize!=b->nSize ) return 0; + return 1; + } static void assertCellInfo(BtCursor *pCur){ CellInfo info; - int iPage = pCur->iPage; memset(&info, 0, sizeof(info)); - btreeParseCell(pCur->apPage[iPage], pCur->aiIdx[iPage], &info); - assert( CORRUPT_DB || memcmp(&info, &pCur->info, sizeof(info))==0 ); + btreeParseCell(pCur->pPage, pCur->ix, &info); + assert( CORRUPT_DB || cellInfoEqual(&info, &pCur->info) ); } #else #define assertCellInfo(x) #endif static SQLITE_NOINLINE void getCellInfo(BtCursor *pCur){ if( pCur->info.nSize==0 ){ - int iPage = pCur->iPage; pCur->curFlags |= BTCF_ValidNKey; - btreeParseCell(pCur->apPage[iPage],pCur->aiIdx[iPage],&pCur->info); + btreeParseCell(pCur->pPage,pCur->ix,&pCur->info); }else{ assertCellInfo(pCur); } @@ -61986,61 +75521,95 @@ SQLITE_PRIVATE int sqlite3BtreeCursorIsValid(BtCursor *pCur){ return pCur && pCur->eState==CURSOR_VALID; } #endif /* NDEBUG */ - -/* -** Set *pSize to the size of the buffer needed to hold the value of -** the key for the current entry. If the cursor is not pointing -** to a valid entry, *pSize is set to 0. -** -** For a table with the INTKEY flag set, this routine returns the key -** itself, not the number of bytes in the key. -** -** The caller must position the cursor prior to invoking this routine. -** -** This routine cannot fail. It always returns SQLITE_OK. -*/ -SQLITE_PRIVATE int sqlite3BtreeKeySize(BtCursor *pCur, i64 *pSize){ - assert( cursorHoldsMutex(pCur) ); - assert( pCur->eState==CURSOR_VALID ); - getCellInfo(pCur); - *pSize = pCur->info.nKey; - return SQLITE_OK; +SQLITE_PRIVATE int sqlite3BtreeCursorIsValidNN(BtCursor *pCur){ + assert( pCur!=0 ); + return pCur->eState==CURSOR_VALID; } /* -** Set *pSize to the number of bytes of data in the entry the -** cursor currently points to. +** Return the value of the integer key or "rowid" for a table btree. +** This routine is only valid for a cursor that is pointing into a +** ordinary table btree. If the cursor points to an index btree or +** is invalid, the result of this routine is undefined. +*/ +SQLITE_PRIVATE i64 sqlite3BtreeIntegerKey(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->curIntKey ); + getCellInfo(pCur); + return pCur->info.nKey; +} + +/* +** Pin or unpin a cursor. +*/ +SQLITE_PRIVATE void sqlite3BtreeCursorPin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)==0 ); + pCur->curFlags |= BTCF_Pinned; +} +SQLITE_PRIVATE void sqlite3BtreeCursorUnpin(BtCursor *pCur){ + assert( (pCur->curFlags & BTCF_Pinned)!=0 ); + pCur->curFlags &= ~BTCF_Pinned; +} + +/* +** Return the offset into the database file for the start of the +** payload to which the cursor is pointing. +*/ +SQLITE_PRIVATE i64 sqlite3BtreeOffset(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + getCellInfo(pCur); + return (i64)pCur->pBt->pageSize*((i64)pCur->pPage->pgno - 1) + + (i64)(pCur->info.pPayload - pCur->pPage->aData); +} + +/* +** Return the number of bytes of payload for the entry that pCur is +** currently pointing to. For table btrees, this will be the amount +** of data. For index btrees, this will be the size of the key. ** ** The caller must guarantee that the cursor is pointing to a non-NULL ** valid entry. In other words, the calling procedure must guarantee ** that the cursor has Cursor.eState==CURSOR_VALID. -** -** Failure is not possible. This function always returns SQLITE_OK. -** It might just as well be a procedure (returning void) but we continue -** to return an integer result code for historical reasons. */ -SQLITE_PRIVATE int sqlite3BtreeDataSize(BtCursor *pCur, u32 *pSize){ - assert( cursorOwnsBtShared(pCur) ); +SQLITE_PRIVATE u32 sqlite3BtreePayloadSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 ); - assert( pCur->iPageapPage[pCur->iPage]->intKeyLeaf==1 ); getCellInfo(pCur); - *pSize = pCur->info.nPayload; - return SQLITE_OK; + return pCur->info.nPayload; +} + +/* +** Return an upper bound on the size of any record for the table +** that the cursor is pointing into. +** +** This is an optimization. Everything will still work if this +** routine always returns 2147483647 (which is the largest record +** that SQLite can handle) or more. But returning a smaller value might +** prevent large memory allocations when trying to interpret a +** corrupt database. +** +** The current implementation merely returns the size of the underlying +** database file. +*/ +SQLITE_PRIVATE sqlite3_int64 sqlite3BtreeMaxRecordSize(BtCursor *pCur){ + assert( cursorHoldsMutex(pCur) ); + assert( pCur->eState==CURSOR_VALID ); + return pCur->pBt->pageSize * (sqlite3_int64)pCur->pBt->nPage; } /* ** Given the page number of an overflow page in the database (parameter -** ovfl), this function finds the page number of the next page in the +** ovfl), this function finds the page number of the next page in the ** linked list of overflow pages. If possible, it uses the auto-vacuum -** pointer-map data instead of reading the content of page ovfl to do so. +** pointer-map data instead of reading the content of page ovfl to do so. ** ** If an error occurs an SQLite error code is returned. Otherwise: ** -** The page number of the next overflow page in the linked list is -** written to *pPgnoNext. If page ovfl is the last page in its linked -** list, *pPgnoNext is set to zero. +** The page number of the next overflow page in the linked list is +** written to *pPgnoNext. If page ovfl is the last page in its linked +** list, *pPgnoNext is set to zero. ** ** If ppPage is not NULL, and a reference to the MemPage object corresponding ** to page number pOvfl was obtained, then *ppPage is set to point to that @@ -62064,9 +75633,9 @@ static int getOverflowPage( #ifndef SQLITE_OMIT_AUTOVACUUM /* Try to find the next page in the overflow list using the - ** autovacuum pointer-map pages. Guess that the next page in - ** the overflow list is page number (ovfl+1). If that guess turns - ** out to be wrong, fall back to loading the data of page + ** autovacuum pointer-map pages. Guess that the next page in + ** the overflow list is page number (ovfl+1). If that guess turns + ** out to be wrong, fall back to loading the data of page ** number ovfl to determine the next page number. */ if( pBt->autoVacuum ){ @@ -62145,7 +75714,6 @@ static int copyPayload( ** ** 0: The operation is a read. Populate the overflow cache. ** 1: The operation is a write. Populate the overflow cache. -** 2: The operation is a read. Do not populate the overflow cache. ** ** A total of "amt" bytes are read or written beginning at "offset". ** Data is read to or from the buffer pBuf. @@ -62153,13 +75721,13 @@ static int copyPayload( ** The content being read or written might appear on the main page ** or be scattered out on multiple overflow pages. ** -** If the current cursor entry uses one or more overflow pages and the -** eOp argument is not 2, this function may allocate space for and lazily -** populates the overflow page-list cache array (BtCursor.aOverflow). -** Subsequent calls use this cache to make seeking to the supplied offset +** If the current cursor entry uses one or more overflow pages +** this function may allocate space for and lazily populate +** the overflow page-list cache array (BtCursor.aOverflow). +** Subsequent calls use this cache to make seeking to the supplied offset ** more efficient. ** -** Once an overflow page-list cache has been allocated, it may be +** Once an overflow page-list cache has been allocated, it must be ** invalidated if some other cursor writes to the same table, or if ** the cursor is moved to a different row. Additionally, in auto-vacuum ** mode, the following events may invalidate an overflow page-list cache. @@ -62172,30 +75740,28 @@ static int accessPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 offset, /* Begin reading this far into payload */ u32 amt, /* Read this many bytes */ - unsigned char *pBuf, /* Write the bytes into this buffer */ + unsigned char *pBuf, /* Write the bytes into this buffer */ int eOp /* zero to read. non-zero to write. */ ){ unsigned char *aPayload; int rc = SQLITE_OK; int iIdx = 0; - MemPage *pPage = pCur->apPage[pCur->iPage]; /* Btree page of current entry */ + MemPage *pPage = pCur->pPage; /* Btree page of current entry */ BtShared *pBt = pCur->pBt; /* Btree this cursor belongs to */ #ifdef SQLITE_DIRECT_OVERFLOW_READ - unsigned char * const pBufStart = pBuf; - int bEnd; /* True if reading to end of data */ + unsigned char * const pBufStart = pBuf; /* Start of original out buffer */ #endif assert( pPage ); + assert( eOp==0 || eOp==1 ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->aiIdx[pCur->iPage]nCell ); + if( pCur->ix>=pPage->nCell ){ + return SQLITE_CORRUPT_PAGE(pPage); + } assert( cursorHoldsMutex(pCur) ); - assert( eOp!=2 || offset==0 ); /* Always start from beginning for eOp==2 */ getCellInfo(pCur); aPayload = pCur->info.pPayload; -#ifdef SQLITE_DIRECT_OVERFLOW_READ - bEnd = offset+amt==pCur->info.nPayload; -#endif assert( offset+amt <= pCur->info.nPayload ); assert( aPayload > pPage->aData ); @@ -62205,7 +75771,7 @@ static int accessPayload( ** &aPayload[pCur->info.nLocal] > &pPage->aData[pBt->usableSize] ** but is recast into its current form to avoid integer overflow problems */ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pPage); } /* Check if data must be read/written to/from the btree page itself. */ @@ -62214,7 +75780,7 @@ static int accessPayload( if( a+offset>pCur->info.nLocal ){ a = pCur->info.nLocal - offset; } - rc = copyPayload(&aPayload[offset], pBuf, a, (eOp & 0x01), pPage->pDbPage); + rc = copyPayload(&aPayload[offset], pBuf, a, eOp, pPage->pDbPage); offset = 0; pBuf += a; amt -= a; @@ -62230,53 +75796,57 @@ static int accessPayload( nextPage = get4byte(&aPayload[pCur->info.nLocal]); /* If the BtCursor.aOverflow[] has not been allocated, allocate it now. - ** Except, do not allocate aOverflow[] for eOp==2. ** ** The aOverflow[] array is sized at one entry for each overflow page ** in the overflow chain. The page number of the first overflow page is ** stored in aOverflow[0], etc. A value of 0 in the aOverflow[] array ** means "not yet known" (the cache is lazily populated). */ - if( eOp!=2 && (pCur->curFlags & BTCF_ValidOvfl)==0 ){ + if( (pCur->curFlags & BTCF_ValidOvfl)==0 ){ int nOvfl = (pCur->info.nPayload-pCur->info.nLocal+ovflSize-1)/ovflSize; - if( nOvfl>pCur->nOvflAlloc ){ - Pgno *aNew = (Pgno*)sqlite3Realloc( - pCur->aOverflow, nOvfl*2*sizeof(Pgno) - ); - if( aNew==0 ){ - rc = SQLITE_NOMEM_BKPT; + if( pCur->aOverflow==0 + || nOvfl*(int)sizeof(Pgno) > sqlite3MallocSize(pCur->aOverflow) + ){ + Pgno *aNew; + if( sqlite3FaultSim(413) ){ + aNew = 0; + }else{ + aNew = (Pgno*)sqlite3Realloc(pCur->aOverflow, nOvfl*2*sizeof(Pgno)); + } + if( aNew==0 ){ + return SQLITE_NOMEM_BKPT; }else{ - pCur->nOvflAlloc = nOvfl*2; pCur->aOverflow = aNew; } } - if( rc==SQLITE_OK ){ - memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); - pCur->curFlags |= BTCF_ValidOvfl; + memset(pCur->aOverflow, 0, nOvfl*sizeof(Pgno)); + pCur->curFlags |= BTCF_ValidOvfl; + }else{ + /* Sanity check the validity of the overflow page cache */ + assert( pCur->aOverflow[0]==nextPage + || pCur->aOverflow[0]==0 + || CORRUPT_DB ); + assert( pCur->aOverflow[0]!=0 || pCur->aOverflow[offset/ovflSize]==0 ); + + /* If the overflow page-list cache has been allocated and the + ** entry for the first required overflow page is valid, skip + ** directly to it. + */ + if( pCur->aOverflow[offset/ovflSize] ){ + iIdx = (offset/ovflSize); + nextPage = pCur->aOverflow[iIdx]; + offset = (offset%ovflSize); } } - /* If the overflow page-list cache has been allocated and the - ** entry for the first required overflow page is valid, skip - ** directly to it. - */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 - && pCur->aOverflow[offset/ovflSize] - ){ - iIdx = (offset/ovflSize); - nextPage = pCur->aOverflow[iIdx]; - offset = (offset%ovflSize); - } - - for( ; rc==SQLITE_OK && amt>0 && nextPage; iIdx++){ - + assert( rc==SQLITE_OK && amt>0 ); + while( nextPage ){ /* If required, populate the overflow page-list cache. */ - if( (pCur->curFlags & BTCF_ValidOvfl)!=0 ){ - assert( pCur->aOverflow[iIdx]==0 - || pCur->aOverflow[iIdx]==nextPage - || CORRUPT_DB ); - pCur->aOverflow[iIdx] = nextPage; - } + if( nextPage > pBt->nPage ) return SQLITE_CORRUPT_BKPT; + assert( pCur->aOverflow[iIdx]==0 + || pCur->aOverflow[iIdx]==nextPage + || CORRUPT_DB ); + pCur->aOverflow[iIdx] = nextPage; if( offset>=ovflSize ){ /* The only reason to read this page is to obtain the page @@ -62284,11 +75854,7 @@ static int accessPayload( ** data is not required. So first try to lookup the overflow ** page-list cache, if any, then fall back to the getOverflowPage() ** function. - ** - ** Note that the aOverflow[] array must be allocated because eOp!=2 - ** here. If eOp==2, then offset==0 and this branch is never taken. */ - assert( eOp!=2 ); assert( pCur->curFlags & BTCF_ValidOvfl ); assert( pCur->pBtree->db==pBt->db ); if( pCur->aOverflow[iIdx+1] ){ @@ -62301,9 +75867,6 @@ static int accessPayload( /* Need to read this page properly. It contains some of the ** range of data that is being read (eOp==0) or written (eOp!=0). */ -#ifdef SQLITE_DIRECT_OVERFLOW_READ - sqlite3_file *fd; -#endif int a = amt; if( a + offset > ovflSize ){ a = ovflSize - offset; @@ -62312,29 +75875,26 @@ static int accessPayload( #ifdef SQLITE_DIRECT_OVERFLOW_READ /* If all the following are true: ** - ** 1) this is a read operation, and + ** 1) this is a read operation, and ** 2) data is required from the start of this overflow page, and - ** 3) the database is file-backed, and - ** 4) there is no open write-transaction, and - ** 5) the database is not a WAL database, - ** 6) all data from the page is being read. - ** 7) at least 4 bytes have already been read into the output buffer + ** 3) there are no dirty pages in the page-cache + ** 4) the database is file-backed, and + ** 5) the page is not in the WAL file + ** 6) at least 4 bytes have already been read into the output buffer ** ** then data can be read directly from the database file into the ** output buffer, bypassing the page-cache altogether. This speeds ** up loading large records that span many overflow pages. */ - if( (eOp&0x01)==0 /* (1) */ + if( eOp==0 /* (1) */ && offset==0 /* (2) */ - && (bEnd || a==ovflSize) /* (6) */ - && pBt->inTransaction==TRANS_READ /* (4) */ - && (fd = sqlite3PagerFile(pBt->pPager))->pMethods /* (3) */ - && pBt->pPage1->aData[19]==0x01 /* (5) */ - && &pBuf[-4]>=pBufStart /* (7) */ + && sqlite3PagerDirectReadOk(pBt->pPager, nextPage) /* (3,4,5) */ + && &pBuf[-4]>=pBufStart /* (6) */ ){ + sqlite3_file *fd = sqlite3PagerFile(pBt->pPager); u8 aSave[4]; u8 *aWrite = &pBuf[-4]; - assert( aWrite>=pBufStart ); /* hence (7) */ + assert( aWrite>=pBufStart ); /* due to (6) */ memcpy(aSave, aWrite, 4); rc = sqlite3OsRead(fd, aWrite, a+4, (i64)pBt->pageSize*(nextPage-1)); nextPage = get4byte(aWrite); @@ -62345,79 +75905,88 @@ static int accessPayload( { DbPage *pDbPage; rc = sqlite3PagerGet(pBt->pPager, nextPage, &pDbPage, - ((eOp&0x01)==0 ? PAGER_GET_READONLY : 0) + (eOp==0 ? PAGER_GET_READONLY : 0) ); if( rc==SQLITE_OK ){ aPayload = sqlite3PagerGetData(pDbPage); nextPage = get4byte(aPayload); - rc = copyPayload(&aPayload[offset+4], pBuf, a, (eOp&0x01), pDbPage); + rc = copyPayload(&aPayload[offset+4], pBuf, a, eOp, pDbPage); sqlite3PagerUnref(pDbPage); offset = 0; } } amt -= a; + if( amt==0 ) return rc; pBuf += a; } + if( rc ) break; + iIdx++; } } if( rc==SQLITE_OK && amt>0 ){ - return SQLITE_CORRUPT_BKPT; + /* Overflow chain ends prematurely */ + return SQLITE_CORRUPT_PAGE(pPage); } return rc; } /* -** Read part of the key associated with cursor pCur. Exactly -** "amt" bytes will be transferred into pBuf[]. The transfer +** Read part of the payload for the row at which that cursor pCur is currently +** pointing. "amt" bytes will be transferred into pBuf[]. The transfer ** begins at "offset". ** -** The caller must ensure that pCur is pointing to a valid row -** in the table. +** pCur can be pointing to either a table or an index b-tree. +** If pointing to a table btree, then the content section is read. If +** pCur is pointing to an index b-tree then the key section is read. +** +** For sqlite3BtreePayload(), the caller must ensure that pCur is pointing +** to a valid row in the table. For sqlite3BtreePayloadChecked(), the +** cursor might be invalid or might need to be restored before being read. ** ** Return SQLITE_OK on success or an error code if anything goes ** wrong. An error is returned if "offset+amt" is larger than ** the available payload. */ -SQLITE_PRIVATE int sqlite3BtreeKey(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ +SQLITE_PRIVATE int sqlite3BtreePayload(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ assert( cursorHoldsMutex(pCur) ); assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->iPage>=0 && pCur->pPage ); return accessPayload(pCur, offset, amt, (unsigned char*)pBuf, 0); } /* -** Read part of the data associated with cursor pCur. Exactly -** "amt" bytes will be transfered into pBuf[]. The transfer -** begins at "offset". -** -** Return SQLITE_OK on success or an error code if anything goes -** wrong. An error is returned if "offset+amt" is larger than -** the available payload. +** This variant of sqlite3BtreePayload() works even if the cursor has not +** in the CURSOR_VALID state. It is only used by the sqlite3_blob_read() +** interface. */ -SQLITE_PRIVATE int sqlite3BtreeData(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ - int rc; - #ifndef SQLITE_OMIT_INCRBLOB +static SQLITE_NOINLINE int accessPayloadChecked( + BtCursor *pCur, + u32 offset, + u32 amt, + void *pBuf +){ + int rc; if ( pCur->eState==CURSOR_INVALID ){ return SQLITE_ABORT; } -#endif - assert( cursorOwnsBtShared(pCur) ); - rc = restoreCursorPosition(pCur); - if( rc==SQLITE_OK ){ - assert( pCur->eState==CURSOR_VALID ); - assert( pCur->iPage>=0 && pCur->apPage[pCur->iPage] ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - rc = accessPayload(pCur, offset, amt, pBuf, 0); - } - return rc; + rc = btreeRestoreCursorPosition(pCur); + return rc ? rc : accessPayload(pCur, offset, amt, pBuf, 0); } +SQLITE_PRIVATE int sqlite3BtreePayloadChecked(BtCursor *pCur, u32 offset, u32 amt, void *pBuf){ + if( pCur->eState==CURSOR_VALID ){ + assert( cursorOwnsBtShared(pCur) ); + return accessPayload(pCur, offset, amt, pBuf, 0); + }else{ + return accessPayloadChecked(pCur, offset, amt, pBuf); + } +} +#endif /* SQLITE_OMIT_INCRBLOB */ /* -** Return a pointer to payload information from the entry that the +** Return a pointer to payload information from the entry that the ** pCur cursor is pointing to. The pointer is to the beginning of ** the key if index btrees (pPage->intKey==0) and is the data for ** table btrees (pPage->intKey==1). The number of bytes of available @@ -62439,18 +76008,23 @@ static const void *fetchPayload( BtCursor *pCur, /* Cursor pointing to entry to read from */ u32 *pAmt /* Write the number of available bytes here */ ){ - u32 amt; - assert( pCur!=0 && pCur->iPage>=0 && pCur->apPage[pCur->iPage]); + int amt; + assert( pCur!=0 && pCur->iPage>=0 && pCur->pPage); assert( pCur->eState==CURSOR_VALID ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( cursorOwnsBtShared(pCur) ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); + assert( pCur->ixpPage->nCell || CORRUPT_DB ); assert( pCur->info.nSize>0 ); - assert( pCur->info.pPayload>pCur->apPage[pCur->iPage]->aData || CORRUPT_DB ); - assert( pCur->info.pPayloadapPage[pCur->iPage]->aDataEnd ||CORRUPT_DB); - amt = (int)(pCur->apPage[pCur->iPage]->aDataEnd - pCur->info.pPayload); - if( pCur->info.nLocalinfo.nLocal; - *pAmt = amt; + assert( pCur->info.pPayload>pCur->pPage->aData || CORRUPT_DB ); + assert( pCur->info.pPayloadpPage->aDataEnd ||CORRUPT_DB); + amt = pCur->info.nLocal; + if( amt>(int)(pCur->pPage->aDataEnd - pCur->info.pPayload) ){ + /* There is too little space on the page for the expected amount + ** of local content. Database must be corrupt. */ + assert( CORRUPT_DB ); + amt = MAX(0, (int)(pCur->pPage->aDataEnd - pCur->info.pPayload)); + } + *pAmt = (u32)amt; return (void*)pCur->info.pPayload; } @@ -62469,10 +76043,7 @@ static const void *fetchPayload( ** These routines is used to get quick access to key and data ** in the common case where no overflow pages are used. */ -SQLITE_PRIVATE const void *sqlite3BtreeKeyFetch(BtCursor *pCur, u32 *pAmt){ - return fetchPayload(pCur, pAmt); -} -SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ +SQLITE_PRIVATE const void *sqlite3BtreePayloadFetch(BtCursor *pCur, u32 *pAmt){ return fetchPayload(pCur, pAmt); } @@ -62487,8 +76058,7 @@ SQLITE_PRIVATE const void *sqlite3BtreeDataFetch(BtCursor *pCur, u32 *pAmt){ ** vice-versa). */ static int moveToChild(BtCursor *pCur, u32 newPgno){ - BtShared *pBt = pCur->pBt; - + int rc; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPageinfo.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + pCur->aiIdx[pCur->iPage] = pCur->ix; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; pCur->iPage++; - pCur->aiIdx[pCur->iPage] = 0; - return getAndInitPage(pBt, newPgno, &pCur->apPage[pCur->iPage], - pCur, pCur->curPagerFlags); + rc = getAndInitPage(pCur->pBt, newPgno, &pCur->pPage, pCur->curPagerFlags); + assert( pCur->pPage!=0 || rc!=SQLITE_OK ); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(newPgno); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + } + return rc; } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* -** Page pParent is an internal (non-leaf) tree page. This function +** Page pParent is an internal (non-leaf) tree page. This function ** asserts that page number iChild is the left-child if the iIdx'th ** cell in page pParent. Or, if iIdx is equal to the total number of ** cells in pParent, that page number iChild is the right-child of @@ -62523,7 +76105,7 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ } } #else -# define assertParentIndex(x,y,z) +# define assertParentIndex(x,y,z) #endif /* @@ -62535,19 +76117,23 @@ static void assertParentIndex(MemPage *pParent, int iIdx, Pgno iChild){ ** the largest cell index. */ static void moveToParent(BtCursor *pCur){ + MemPage *pLeaf; assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); assert( pCur->iPage>0 ); - assert( pCur->apPage[pCur->iPage] ); + assert( pCur->pPage ); assertParentIndex( - pCur->apPage[pCur->iPage-1], - pCur->aiIdx[pCur->iPage-1], - pCur->apPage[pCur->iPage]->pgno + pCur->apPage[pCur->iPage-1], + pCur->aiIdx[pCur->iPage-1], + pCur->pPage->pgno ); testcase( pCur->aiIdx[pCur->iPage-1] > pCur->apPage[pCur->iPage-1]->nCell ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - releasePageNotNull(pCur->apPage[pCur->iPage--]); + pCur->ix = pCur->aiIdx[pCur->iPage-1]; + pLeaf = pCur->pPage; + pCur->pPage = pCur->apPage[--pCur->iPage]; + releasePageNotNull(pLeaf); } /* @@ -62555,19 +76141,19 @@ static void moveToParent(BtCursor *pCur){ ** ** If the table has a virtual root page, then the cursor is moved to point ** to the virtual root page instead of the actual root page. A table has a -** virtual root page when the actual root page contains no cells and a +** virtual root page when the actual root page contains no cells and a ** single child page. This can only happen with the table rooted at page 1. ** -** If the b-tree structure is empty, the cursor state is set to -** CURSOR_INVALID. Otherwise, the cursor is set to point to the first -** cell located on the root (or virtual root) page and the cursor state -** is set to CURSOR_VALID. +** If the b-tree structure is empty, the cursor state is set to +** CURSOR_INVALID and this routine returns SQLITE_EMPTY. Otherwise, +** the cursor is set to point to the first cell located on the root +** (or virtual root) page and the cursor state is set to CURSOR_VALID. ** ** If this function returns successfully, it may be assumed that the -** page-header flags indicate that the [virtual] root-page is the expected +** page-header flags indicate that the [virtual] root-page is the expected ** kind of b-tree page (i.e. if when opening the cursor the caller did not ** specify a KeyInfo structure the flags byte is set to 0x05 or 0x0D, -** indicating a table b-tree, or if the caller did specify a KeyInfo +** indicating a table b-tree, or if the caller did specify a KeyInfo ** structure the flags byte is set to 0x02 or 0x0A, indicating an index ** b-tree). */ @@ -62579,52 +76165,59 @@ static int moveToRoot(BtCursor *pCur){ assert( CURSOR_INVALID < CURSOR_REQUIRESEEK ); assert( CURSOR_VALID < CURSOR_REQUIRESEEK ); assert( CURSOR_FAULT > CURSOR_REQUIRESEEK ); - if( pCur->eState>=CURSOR_REQUIRESEEK ){ - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; - } - sqlite3BtreeClearCursor(pCur); - } + assert( pCur->eState < CURSOR_REQUIRESEEK || pCur->iPage<0 ); + assert( pCur->pgnoRoot>0 || pCur->iPage<0 ); if( pCur->iPage>=0 ){ - while( pCur->iPage ){ - assert( pCur->apPage[pCur->iPage]!=0 ); - releasePageNotNull(pCur->apPage[pCur->iPage--]); + if( pCur->iPage ){ + releasePageNotNull(pCur->pPage); + while( --pCur->iPage ){ + releasePageNotNull(pCur->apPage[pCur->iPage]); + } + pRoot = pCur->pPage = pCur->apPage[0]; + goto skip_init; } }else if( pCur->pgnoRoot==0 ){ pCur->eState = CURSOR_INVALID; - return SQLITE_OK; + return SQLITE_EMPTY; }else{ assert( pCur->iPage==(-1) ); - rc = getAndInitPage(pCur->pBtree->pBt, pCur->pgnoRoot, &pCur->apPage[0], - 0, pCur->curPagerFlags); + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + if( pCur->eState==CURSOR_FAULT ){ + assert( pCur->skipNext!=SQLITE_OK ); + return pCur->skipNext; + } + sqlite3BtreeClearCursor(pCur); + } + rc = getAndInitPage(pCur->pBt, pCur->pgnoRoot, &pCur->pPage, + pCur->curPagerFlags); if( rc!=SQLITE_OK ){ pCur->eState = CURSOR_INVALID; return rc; } pCur->iPage = 0; - pCur->curIntKey = pCur->apPage[0]->intKey; + pCur->curIntKey = pCur->pPage->intKey; } - pRoot = pCur->apPage[0]; - assert( pRoot->pgno==pCur->pgnoRoot ); + pRoot = pCur->pPage; + assert( pRoot->pgno==pCur->pgnoRoot || CORRUPT_DB ); /* If pCur->pKeyInfo is not NULL, then the caller that opened this cursor ** expected to open it on an index b-tree. Otherwise, if pKeyInfo is ** NULL, the caller expects a table b-tree. If this is not the case, - ** return an SQLITE_CORRUPT error. + ** return an SQLITE_CORRUPT error. ** ** Earlier versions of SQLite assumed that this test could not fail ** if the root page was already loaded when this function was called (i.e. - ** if pCur->iPage>=0). But this is not so if the database is corrupted - ** in such a way that page pRoot is linked into a second b-tree table + ** if pCur->iPage>=0). But this is not so if the database is corrupted + ** in such a way that page pRoot is linked into a second b-tree table ** (or the freelist). */ assert( pRoot->intKey==1 || pRoot->intKey==0 ); if( pRoot->isInit==0 || (pCur->pKeyInfo==0)!=pRoot->intKey ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PAGE(pCur->pPage); } - pCur->aiIdx[0] = 0; +skip_init: + pCur->ix = 0; pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidNKey|BTCF_ValidOvfl); @@ -62638,6 +76231,7 @@ static int moveToRoot(BtCursor *pCur){ rc = moveToChild(pCur, subpage); }else{ pCur->eState = CURSOR_INVALID; + rc = SQLITE_EMPTY; } return rc; } @@ -62656,9 +76250,9 @@ static int moveToLeftmost(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( rc==SQLITE_OK && !(pPage = pCur->apPage[pCur->iPage])->leaf ){ - assert( pCur->aiIdx[pCur->iPage]nCell ); - pgno = get4byte(findCell(pPage, pCur->aiIdx[pCur->iPage])); + while( rc==SQLITE_OK && !(pPage = pCur->pPage)->leaf ){ + assert( pCur->ixnCell ); + pgno = get4byte(findCell(pPage, pCur->ix)); rc = moveToChild(pCur, pgno); } return rc; @@ -62681,13 +76275,13 @@ static int moveToRightmost(BtCursor *pCur){ assert( cursorOwnsBtShared(pCur) ); assert( pCur->eState==CURSOR_VALID ); - while( !(pPage = pCur->apPage[pCur->iPage])->leaf ){ + while( !(pPage = pCur->pPage)->leaf ){ pgno = get4byte(&pPage->aData[pPage->hdrOffset+8]); - pCur->aiIdx[pCur->iPage] = pPage->nCell; + pCur->ix = pPage->nCell; rc = moveToChild(pCur, pgno); if( rc ) return rc; } - pCur->aiIdx[pCur->iPage] = pPage->nCell-1; + pCur->ix = pPage->nCell-1; assert( pCur->info.nSize==0 ); assert( (pCur->curFlags & BTCF_ValidNKey)==0 ); return SQLITE_OK; @@ -62704,69 +76298,71 @@ SQLITE_PRIVATE int sqlite3BtreeFirst(BtCursor *pCur, int *pRes){ assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); rc = moveToRoot(pCur); if( rc==SQLITE_OK ){ - if( pCur->eState==CURSOR_INVALID ){ - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); - *pRes = 1; - }else{ - assert( pCur->apPage[pCur->iPage]->nCell>0 ); - *pRes = 0; - rc = moveToLeftmost(pCur); - } + assert( pCur->pPage->nCell>0 ); + *pRes = 0; + rc = moveToLeftmost(pCur); + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || (pCur->pPage!=0 && pCur->pPage->nCell==0) ); + *pRes = 1; + rc = SQLITE_OK; } return rc; } +#ifdef SQLITE_DEBUG +/* The cursors is CURSOR_VALID and has BTCF_AtLast set. Verify that +** this flags are true for a consistent database. +** +** This routine is is called from within assert() statements only. +** It is an internal verification routine and does not appear in production +** builds. +*/ +static int cursorIsAtLastEntry(BtCursor *pCur){ + int ii; + for(ii=0; iiiPage; ii++){ + if( pCur->aiIdx[ii]!=pCur->apPage[ii]->nCell ) return 0; + } + return pCur->ix==pCur->pPage->nCell-1 && pCur->pPage->leaf!=0; +} +#endif + /* Move the cursor to the last entry in the table. Return SQLITE_OK ** on success. Set *pRes to 0 if the cursor actually points to something ** or set *pRes to 1 if the table is empty. */ +static SQLITE_NOINLINE int btreeLast(BtCursor *pCur, int *pRes){ + int rc = moveToRoot(pCur); + if( rc==SQLITE_OK ){ + assert( pCur->eState==CURSOR_VALID ); + *pRes = 0; + rc = moveToRightmost(pCur); + if( rc==SQLITE_OK ){ + pCur->curFlags |= BTCF_AtLast; + }else{ + pCur->curFlags &= ~BTCF_AtLast; + } + }else if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = 1; + rc = SQLITE_OK; + } + return rc; +} SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ - int rc; - assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); /* If the cursor already points to the last entry, this is a no-op. */ if( CURSOR_VALID==pCur->eState && (pCur->curFlags & BTCF_AtLast)!=0 ){ -#ifdef SQLITE_DEBUG - /* This block serves to assert() that the cursor really does point - ** to the last entry in the b-tree. */ - int ii; - for(ii=0; iiiPage; ii++){ - assert( pCur->aiIdx[ii]==pCur->apPage[ii]->nCell ); - } - assert( pCur->aiIdx[pCur->iPage]==pCur->apPage[pCur->iPage]->nCell-1 ); - assert( pCur->apPage[pCur->iPage]->leaf ); -#endif + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); + *pRes = 0; return SQLITE_OK; } - - rc = moveToRoot(pCur); - if( rc==SQLITE_OK ){ - if( CURSOR_INVALID==pCur->eState ){ - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); - *pRes = 1; - }else{ - assert( pCur->eState==CURSOR_VALID ); - *pRes = 0; - rc = moveToRightmost(pCur); - if( rc==SQLITE_OK ){ - pCur->curFlags |= BTCF_AtLast; - }else{ - pCur->curFlags &= ~BTCF_AtLast; - } - - } - } - return rc; + return btreeLast(pCur, pRes); } -/* Move the cursor so that it points to an entry near the key -** specified by pIdxKey or intKey. Return a success code. -** -** For INTKEY tables, the intKey parameter is used. pIdxKey -** must be NULL. For index tables, pIdxKey is used and intKey -** is ignored. +/* Move the cursor so that it points to an entry in a table (a.k.a INTKEY) +** table near the key intKey. Return a success code. ** ** If an exact match is not found, then the cursor is always ** left pointing at a leaf page which would hold the entry if it @@ -62774,82 +76370,90 @@ SQLITE_PRIVATE int sqlite3BtreeLast(BtCursor *pCur, int *pRes){ ** before or after the key. ** ** An integer is written into *pRes which is the result of -** comparing the key with the entry to which the cursor is +** comparing the key with the entry to which the cursor is ** pointing. The meaning of the integer written into ** *pRes is as follows: ** ** *pRes<0 The cursor is left pointing at an entry that -** is smaller than intKey/pIdxKey or if the table is empty +** is smaller than intKey or if the table is empty ** and the cursor is therefore left point to nothing. ** ** *pRes==0 The cursor is left pointing at an entry that -** exactly matches intKey/pIdxKey. +** exactly matches intKey. ** ** *pRes>0 The cursor is left pointing at an entry that -** is larger than intKey/pIdxKey. -** -** For index tables, the pIdxKey->eqSeen field is set to 1 if there -** exists an entry in the table that exactly matches pIdxKey. +** is larger than intKey. */ -SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( +SQLITE_PRIVATE int sqlite3BtreeTableMoveto( BtCursor *pCur, /* The cursor to be moved */ - UnpackedRecord *pIdxKey, /* Unpacked index key */ i64 intKey, /* The table key */ int biasRight, /* If true, bias the search to the high end */ int *pRes /* Write search results here */ ){ int rc; - RecordCompare xRecordCompare; assert( cursorOwnsBtShared(pCur) ); assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); assert( pRes ); - assert( (pIdxKey==0)==(pCur->pKeyInfo==0) ); + assert( pCur->pKeyInfo==0 ); + assert( pCur->eState!=CURSOR_VALID || pCur->curIntKey!=0 ); /* If the cursor is already positioned at the point we are trying ** to move to, then just return without doing any work */ - if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 - && pCur->curIntKey - ){ + if( pCur->eState==CURSOR_VALID && (pCur->curFlags & BTCF_ValidNKey)!=0 ){ if( pCur->info.nKey==intKey ){ *pRes = 0; return SQLITE_OK; } - if( (pCur->curFlags & BTCF_AtLast)!=0 && pCur->info.nKeyinfo.nKeycurFlags & BTCF_AtLast)!=0 ){ + assert( cursorIsAtLastEntry(pCur) || CORRUPT_DB ); + *pRes = -1; + return SQLITE_OK; + } + /* If the requested key is one more than the previous key, then + ** try to get there using sqlite3BtreeNext() rather than a full + ** binary search. This is an optimization only. The correct answer + ** is still obtained without this case, only a little more slowly. */ + if( pCur->info.nKey+1==intKey ){ + *pRes = 0; + rc = sqlite3BtreeNext(pCur, 0); + if( rc==SQLITE_OK ){ + getCellInfo(pCur); + if( pCur->info.nKey==intKey ){ + return SQLITE_OK; + } + }else if( rc!=SQLITE_DONE ){ + return rc; + } + } } } - if( pIdxKey ){ - xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); - pIdxKey->errCode = 0; - assert( pIdxKey->default_rc==1 - || pIdxKey->default_rc==0 - || pIdxKey->default_rc==-1 - ); - }else{ - xRecordCompare = 0; /* All keys are integers */ - } +#ifdef SQLITE_DEBUG + pCur->pBtree->nSeek++; /* Performance measurement during testing */ +#endif rc = moveToRoot(pCur); if( rc ){ + if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = -1; + return SQLITE_OK; + } return rc; } - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage] ); - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->isInit ); - assert( pCur->eState==CURSOR_INVALID || pCur->apPage[pCur->iPage]->nCell>0 ); - if( pCur->eState==CURSOR_INVALID ){ - *pRes = -1; - assert( pCur->pgnoRoot==0 || pCur->apPage[pCur->iPage]->nCell==0 ); - return SQLITE_OK; - } - assert( pCur->apPage[0]->intKey==pCur->curIntKey ); - assert( pCur->curIntKey || pIdxKey ); + assert( pCur->pPage ); + assert( pCur->pPage->isInit ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->pPage->nCell > 0 ); + assert( pCur->iPage==0 || pCur->apPage[0]->intKey==pCur->curIntKey ); + assert( pCur->curIntKey ); + for(;;){ int lwr, upr, idx, c; Pgno chldPg; - MemPage *pPage = pCur->apPage[pCur->iPage]; + MemPage *pPage = pCur->pPage; u8 *pCell; /* Pointer to current cell in pPage */ /* pPage->nCell must be greater than zero. If this is the root-page @@ -62859,151 +76463,386 @@ SQLITE_PRIVATE int sqlite3BtreeMovetoUnpacked( ** be the right kind (index or table) of b-tree page. Otherwise ** a moveToChild() or moveToRoot() call would have detected corruption. */ assert( pPage->nCell>0 ); - assert( pPage->intKey==(pIdxKey==0) ); + assert( pPage->intKey ); lwr = 0; upr = pPage->nCell-1; assert( biasRight==0 || biasRight==1 ); idx = upr>>(1-biasRight); /* idx = biasRight ? upr : (lwr+upr)/2; */ - pCur->aiIdx[pCur->iPage] = (u16)idx; - if( xRecordCompare==0 ){ - for(;;){ - i64 nCellKey; - pCell = findCellPastPtr(pPage, idx); - if( pPage->intKeyLeaf ){ - while( 0x80 <= *(pCell++) ){ - if( pCell>=pPage->aDataEnd ) return SQLITE_CORRUPT_BKPT; + for(;;){ + i64 nCellKey; + pCell = findCellPastPtr(pPage, idx); + if( pPage->intKeyLeaf ){ + while( 0x80 <= *(pCell++) ){ + if( pCell>=pPage->aDataEnd ){ + return SQLITE_CORRUPT_PAGE(pPage); } } - getVarint(pCell, (u64*)&nCellKey); - if( nCellKeyupr ){ c = -1; break; } - }else if( nCellKey>intKey ){ - upr = idx-1; - if( lwr>upr ){ c = +1; break; } + } + getVarint(pCell, (u64*)&nCellKey); + if( nCellKeyupr ){ c = -1; break; } + }else if( nCellKey>intKey ){ + upr = idx-1; + if( lwr>upr ){ c = +1; break; } + }else{ + assert( nCellKey==intKey ); + pCur->ix = (u16)idx; + if( !pPage->leaf ){ + lwr = idx; + goto moveto_table_next_layer; }else{ - assert( nCellKey==intKey ); pCur->curFlags |= BTCF_ValidNKey; pCur->info.nKey = nCellKey; - pCur->aiIdx[pCur->iPage] = (u16)idx; - if( !pPage->leaf ){ - lwr = idx; - goto moveto_next_layer; - }else{ - *pRes = 0; - rc = SQLITE_OK; - goto moveto_finish; - } - } - assert( lwr+upr>=0 ); - idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2; */ - } - }else{ - for(;;){ - int nCell; /* Size of the pCell cell in bytes */ - pCell = findCellPastPtr(pPage, idx); - - /* The maximum supported page-size is 65536 bytes. This means that - ** the maximum number of record bytes stored on an index B-Tree - ** page is less than 16384 bytes and may be stored as a 2-byte - ** varint. This information is used to attempt to avoid parsing - ** the entire cell by checking for the cases where the record is - ** stored entirely within the b-tree page by inspecting the first - ** 2 bytes of the cell. - */ - nCell = pCell[0]; - if( nCell<=pPage->max1bytePayload ){ - /* This branch runs if the record-size field of the cell is a - ** single byte varint and the record fits entirely on the main - ** b-tree page. */ - testcase( pCell+nCell+1==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); - }else if( !(pCell[1] & 0x80) - && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal - ){ - /* The record-size field is a 2 byte varint and the record - ** fits entirely on the main b-tree page. */ - testcase( pCell+nCell+2==pPage->aDataEnd ); - c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); - }else{ - /* The record flows over onto one or more overflow pages. In - ** this case the whole cell needs to be parsed, a buffer allocated - ** and accessPayload() used to retrieve the record into the - ** buffer before VdbeRecordCompare() can be called. - ** - ** If the record is corrupt, the xRecordCompare routine may read - ** up to two varints past the end of the buffer. An extra 18 - ** bytes of padding is allocated at the end of the buffer in - ** case this happens. */ - void *pCellKey; - u8 * const pCellBody = pCell - pPage->childPtrSize; - pPage->xParseCell(pPage, pCellBody, &pCur->info); - nCell = (int)pCur->info.nKey; - testcase( nCell<0 ); /* True if key size is 2^32 or more */ - testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ - testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ - testcase( nCell==2 ); /* Minimum legal index key size */ - if( nCell<2 ){ - rc = SQLITE_CORRUPT_BKPT; - goto moveto_finish; - } - pCellKey = sqlite3Malloc( nCell+18 ); - if( pCellKey==0 ){ - rc = SQLITE_NOMEM_BKPT; - goto moveto_finish; - } - pCur->aiIdx[pCur->iPage] = (u16)idx; - rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 2); - if( rc ){ - sqlite3_free(pCellKey); - goto moveto_finish; - } - c = xRecordCompare(nCell, pCellKey, pIdxKey); - sqlite3_free(pCellKey); - } - assert( - (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) - && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) - ); - if( c<0 ){ - lwr = idx+1; - }else if( c>0 ){ - upr = idx-1; - }else{ - assert( c==0 ); + pCur->info.nSize = 0; *pRes = 0; - rc = SQLITE_OK; - pCur->aiIdx[pCur->iPage] = (u16)idx; - if( pIdxKey->errCode ) rc = SQLITE_CORRUPT; - goto moveto_finish; + return SQLITE_OK; } - if( lwr>upr ) break; - assert( lwr+upr>=0 ); - idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ } + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2; */ } - assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); + assert( lwr==upr+1 || !pPage->leaf ); assert( pPage->isInit ); if( pPage->leaf ){ - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - pCur->aiIdx[pCur->iPage] = (u16)idx; + assert( pCur->ixpPage->nCell ); + pCur->ix = (u16)idx; *pRes = c; rc = SQLITE_OK; - goto moveto_finish; + goto moveto_table_finish; } -moveto_next_layer: +moveto_table_next_layer: if( lwr>=pPage->nCell ){ chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); }else{ chldPg = get4byte(findCell(pPage, lwr)); } - pCur->aiIdx[pCur->iPage] = (u16)lwr; + pCur->ix = (u16)lwr; rc = moveToChild(pCur, chldPg); if( rc ) break; } -moveto_finish: +moveto_table_finish: pCur->info.nSize = 0; - pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); + return rc; +} + +/* +** Compare the "idx"-th cell on the page the cursor pCur is currently +** pointing to to pIdxKey using xRecordCompare. Return negative or +** zero if the cell is less than or equal pIdxKey. Return positive +** if unknown. +** +** Return value negative: Cell at pCur[idx] less than pIdxKey +** +** Return value is zero: Cell at pCur[idx] equals pIdxKey +** +** Return value positive: Nothing is known about the relationship +** of the cell at pCur[idx] and pIdxKey. +** +** This routine is part of an optimization. It is always safe to return +** a positive value as that will cause the optimization to be skipped. +*/ +static int indexCellCompare( + BtCursor *pCur, + int idx, + UnpackedRecord *pIdxKey, + RecordCompare xRecordCompare +){ + MemPage *pPage = pCur->pPage; + int c; + int nCell; /* Size of the pCell cell in bytes */ + u8 *pCell = findCellPastPtr(pPage, idx); + + nCell = pCell[0]; + if( nCell<=pPage->max1bytePayload ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + }else{ + /* If the record extends into overflow pages, do not attempt + ** the optimization. */ + c = 99; + } + return c; +} + +/* +** Return true (non-zero) if pCur is current pointing to the last +** page of a table. +*/ +static int cursorOnLastPage(BtCursor *pCur){ + int i; + assert( pCur->eState==CURSOR_VALID ); + for(i=0; iiPage; i++){ + MemPage *pPage = pCur->apPage[i]; + if( pCur->aiIdx[i]nCell ) return 0; + } + return 1; +} + +/* Move the cursor so that it points to an entry in an index table +** near the key pIdxKey. Return a success code. +** +** If an exact match is not found, then the cursor is always +** left pointing at a leaf page which would hold the entry if it +** were present. The cursor might point to an entry that comes +** before or after the key. +** +** An integer is written into *pRes which is the result of +** comparing the key with the entry to which the cursor is +** pointing. The meaning of the integer written into +** *pRes is as follows: +** +** *pRes<0 The cursor is left pointing at an entry that +** is smaller than pIdxKey or if the table is empty +** and the cursor is therefore left point to nothing. +** +** *pRes==0 The cursor is left pointing at an entry that +** exactly matches pIdxKey. +** +** *pRes>0 The cursor is left pointing at an entry that +** is larger than pIdxKey. +** +** The pIdxKey->eqSeen field is set to 1 if there +** exists an entry in the table that exactly matches pIdxKey. +*/ +SQLITE_PRIVATE int sqlite3BtreeIndexMoveto( + BtCursor *pCur, /* The cursor to be moved */ + UnpackedRecord *pIdxKey, /* Unpacked index key */ + int *pRes /* Write search results here */ +){ + int rc; + RecordCompare xRecordCompare; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + assert( pRes ); + assert( pCur->pKeyInfo!=0 ); + +#ifdef SQLITE_DEBUG + pCur->pBtree->nSeek++; /* Performance measurement during testing */ +#endif + + xRecordCompare = sqlite3VdbeFindCompare(pIdxKey); + pIdxKey->errCode = 0; + assert( pIdxKey->default_rc==1 + || pIdxKey->default_rc==0 + || pIdxKey->default_rc==-1 + ); + + + /* Check to see if we can skip a lot of work. Two cases: + ** + ** (1) If the cursor is already pointing to the very last cell + ** in the table and the pIdxKey search key is greater than or + ** equal to that last cell, then no movement is required. + ** + ** (2) If the cursor is on the last page of the table and the first + ** cell on that last page is less than or equal to the pIdxKey + ** search key, then we can start the search on the current page + ** without needing to go back to root. + */ + if( pCur->eState==CURSOR_VALID + && pCur->pPage->leaf + && cursorOnLastPage(pCur) + ){ + int c; + if( pCur->ix==pCur->pPage->nCell-1 + && (c = indexCellCompare(pCur, pCur->ix, pIdxKey, xRecordCompare))<=0 + && pIdxKey->errCode==SQLITE_OK + ){ + *pRes = c; + return SQLITE_OK; /* Cursor already pointing at the correct spot */ + } + if( pCur->iPage>0 + && indexCellCompare(pCur, 0, pIdxKey, xRecordCompare)<=0 + && pIdxKey->errCode==SQLITE_OK + ){ + pCur->curFlags &= ~(BTCF_ValidOvfl|BTCF_AtLast); + if( !pCur->pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } + goto bypass_moveto_root; /* Start search on the current page */ + } + pIdxKey->errCode = SQLITE_OK; + } + + rc = moveToRoot(pCur); + if( rc ){ + if( rc==SQLITE_EMPTY ){ + assert( pCur->pgnoRoot==0 || pCur->pPage->nCell==0 ); + *pRes = -1; + return SQLITE_OK; + } + return rc; + } + +bypass_moveto_root: + assert( pCur->pPage ); + assert( pCur->pPage->isInit ); + assert( pCur->eState==CURSOR_VALID ); + assert( pCur->pPage->nCell > 0 ); + assert( pCur->curIntKey==0 ); + assert( pIdxKey!=0 ); + for(;;){ + int lwr, upr, idx, c; + Pgno chldPg; + MemPage *pPage = pCur->pPage; + u8 *pCell; /* Pointer to current cell in pPage */ + + /* pPage->nCell must be greater than zero. If this is the root-page + ** the cursor would have been INVALID above and this for(;;) loop + ** not run. If this is not the root-page, then the moveToChild() routine + ** would have already detected db corruption. Similarly, pPage must + ** be the right kind (index or table) of b-tree page. Otherwise + ** a moveToChild() or moveToRoot() call would have detected corruption. */ + assert( pPage->nCell>0 ); + assert( pPage->intKey==0 ); + lwr = 0; + upr = pPage->nCell-1; + idx = upr>>1; /* idx = (lwr+upr)/2; */ + for(;;){ + int nCell; /* Size of the pCell cell in bytes */ + pCell = findCellPastPtr(pPage, idx); + + /* The maximum supported page-size is 65536 bytes. This means that + ** the maximum number of record bytes stored on an index B-Tree + ** page is less than 16384 bytes and may be stored as a 2-byte + ** varint. This information is used to attempt to avoid parsing + ** the entire cell by checking for the cases where the record is + ** stored entirely within the b-tree page by inspecting the first + ** 2 bytes of the cell. + */ + nCell = pCell[0]; + if( nCell<=pPage->max1bytePayload ){ + /* This branch runs if the record-size field of the cell is a + ** single byte varint and the record fits entirely on the main + ** b-tree page. */ + testcase( pCell+nCell+1==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[1], pIdxKey); + }else if( !(pCell[1] & 0x80) + && (nCell = ((nCell&0x7f)<<7) + pCell[1])<=pPage->maxLocal + ){ + /* The record-size field is a 2 byte varint and the record + ** fits entirely on the main b-tree page. */ + testcase( pCell+nCell+2==pPage->aDataEnd ); + c = xRecordCompare(nCell, (void*)&pCell[2], pIdxKey); + }else{ + /* The record flows over onto one or more overflow pages. In + ** this case the whole cell needs to be parsed, a buffer allocated + ** and accessPayload() used to retrieve the record into the + ** buffer before VdbeRecordCompare() can be called. + ** + ** If the record is corrupt, the xRecordCompare routine may read + ** up to two varints past the end of the buffer. An extra 18 + ** bytes of padding is allocated at the end of the buffer in + ** case this happens. */ + void *pCellKey; + u8 * const pCellBody = pCell - pPage->childPtrSize; + const int nOverrun = 18; /* Size of the overrun padding */ + pPage->xParseCell(pPage, pCellBody, &pCur->info); + nCell = (int)pCur->info.nKey; + testcase( nCell<0 ); /* True if key size is 2^32 or more */ + testcase( nCell==0 ); /* Invalid key size: 0x80 0x80 0x00 */ + testcase( nCell==1 ); /* Invalid key size: 0x80 0x80 0x01 */ + testcase( nCell==2 ); /* Minimum legal index key size */ + if( nCell<2 || nCell/pCur->pBt->usableSize>pCur->pBt->nPage ){ + rc = SQLITE_CORRUPT_PAGE(pPage); + goto moveto_index_finish; + } + pCellKey = sqlite3Malloc( nCell+nOverrun ); + if( pCellKey==0 ){ + rc = SQLITE_NOMEM_BKPT; + goto moveto_index_finish; + } + pCur->ix = (u16)idx; + rc = accessPayload(pCur, 0, nCell, (unsigned char*)pCellKey, 0); + memset(((u8*)pCellKey)+nCell,0,nOverrun); /* Fix uninit warnings */ + pCur->curFlags &= ~BTCF_ValidOvfl; + if( rc ){ + sqlite3_free(pCellKey); + goto moveto_index_finish; + } + c = sqlite3VdbeRecordCompare(nCell, pCellKey, pIdxKey); + sqlite3_free(pCellKey); + } + assert( + (pIdxKey->errCode!=SQLITE_CORRUPT || c==0) + && (pIdxKey->errCode!=SQLITE_NOMEM || pCur->pBtree->db->mallocFailed) + ); + if( c<0 ){ + lwr = idx+1; + }else if( c>0 ){ + upr = idx-1; + }else{ + assert( c==0 ); + *pRes = 0; + rc = SQLITE_OK; + pCur->ix = (u16)idx; + if( pIdxKey->errCode ) rc = SQLITE_CORRUPT_BKPT; + goto moveto_index_finish; + } + if( lwr>upr ) break; + assert( lwr+upr>=0 ); + idx = (lwr+upr)>>1; /* idx = (lwr+upr)/2 */ + } + assert( lwr==upr+1 || (pPage->intKey && !pPage->leaf) ); + assert( pPage->isInit ); + if( pPage->leaf ){ + assert( pCur->ixpPage->nCell || CORRUPT_DB ); + pCur->ix = (u16)idx; + *pRes = c; + rc = SQLITE_OK; + goto moveto_index_finish; + } + if( lwr>=pPage->nCell ){ + chldPg = get4byte(&pPage->aData[pPage->hdrOffset+8]); + }else{ + chldPg = get4byte(findCell(pPage, lwr)); + } + + /* This block is similar to an in-lined version of: + ** + ** pCur->ix = (u16)lwr; + ** rc = moveToChild(pCur, chldPg); + ** if( rc ) break; + */ + pCur->info.nSize = 0; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); + if( pCur->iPage>=(BTCURSOR_MAX_DEPTH-1) ){ + return SQLITE_CORRUPT_BKPT; + } + pCur->aiIdx[pCur->iPage] = (u16)lwr; + pCur->apPage[pCur->iPage] = pCur->pPage; + pCur->ix = 0; + pCur->iPage++; + rc = getAndInitPage(pCur->pBt, chldPg, &pCur->pPage, pCur->curPagerFlags); + if( rc==SQLITE_OK + && (pCur->pPage->nCell<1 || pCur->pPage->intKey!=pCur->curIntKey) + ){ + releasePage(pCur->pPage); + rc = SQLITE_CORRUPT_PGNO(chldPg); + } + if( rc ){ + pCur->pPage = pCur->apPage[--pCur->iPage]; + break; + } + /* + ***** End of in-lined moveToChild() call */ + } +moveto_index_finish: + pCur->info.nSize = 0; + assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); return rc; } @@ -63024,10 +76863,37 @@ SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ } /* -** Advance the cursor to the next entry in the database. If -** successful then set *pRes=0. If the cursor -** was already pointing to the last entry in the database before -** this routine was called, then set *pRes=1. +** Return an estimate for the number of rows in the table that pCur is +** pointing to. Return a negative number if no estimate is currently +** available. +*/ +SQLITE_PRIVATE i64 sqlite3BtreeRowCountEst(BtCursor *pCur){ + i64 n; + u8 i; + + assert( cursorOwnsBtShared(pCur) ); + assert( sqlite3_mutex_held(pCur->pBtree->db->mutex) ); + + /* Currently this interface is only called by the OP_IfSizeBetween + ** opcode and the OP_Count opcode with P3=1. In either case, + ** the cursor will always be valid unless the btree is empty. */ + if( pCur->eState!=CURSOR_VALID ) return 0; + if( NEVER(pCur->pPage->leaf==0) ) return -1; + + n = pCur->pPage->nCell; + for(i=0; iiPage; i++){ + n *= pCur->apPage[i]->nCell; + } + return n; +} + +/* +** Advance the cursor to the next entry in the database. +** Return value: +** +** SQLITE_OK success +** SQLITE_DONE cursor is already pointing at the last element +** otherwise some kind of error occurred ** ** The main entry point is sqlite3BtreeNext(). That routine is optimized ** for the common case of merely incrementing the cell counter BtCursor.aiIdx @@ -63035,23 +76901,18 @@ SQLITE_PRIVATE int sqlite3BtreeEof(BtCursor *pCur){ ** routine is called when it is necessary to move to a different page or ** to restore the cursor. ** -** The calling function will set *pRes to 0 or 1. The initial *pRes value -** will be 1 if the cursor being stepped corresponds to an SQL index and -** if this routine could have been skipped if that SQL index had been -** a unique index. Otherwise the caller will have set *pRes to zero. -** Zero is the common case. The btree implementation is free to use the -** initial *pRes value as a hint to improve performance, but the current -** SQLite btree implementation does not. (Note that the comdb2 btree -** implementation does use this hint, however.) +** If bit 0x01 of the F argument in sqlite3BtreeNext(C,F) is 1, then the +** cursor corresponds to an SQL index and this routine could have been +** skipped if the SQL index had been a unique index. The F argument +** is a hint to the implement. SQLite btree implementation does not use +** this hint, but COMDB2 does. */ -static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreeNext(BtCursor *pCur){ int rc; int idx; MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - assert( *pRes==0 ); if( pCur->eState!=CURSOR_VALID ){ assert( (pCur->curFlags & BTCF_ValidOvfl)==0 ); rc = restoreCursorPosition(pCur); @@ -63059,30 +76920,20 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ return rc; } if( CURSOR_INVALID==pCur->eState ){ - *pRes = 1; - return SQLITE_OK; + return SQLITE_DONE; } - if( pCur->skipNext ){ - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + if( pCur->eState==CURSOR_SKIPNEXT ){ pCur->eState = CURSOR_VALID; - if( pCur->skipNext>0 ){ - pCur->skipNext = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; + if( pCur->skipNext>0 ) return SQLITE_OK; } } - pPage = pCur->apPage[pCur->iPage]; - idx = ++pCur->aiIdx[pCur->iPage]; - assert( pPage->isInit ); - - /* If the database file is corrupt, it is possible for the value of idx - ** to be invalid here. This can only occur if a second cursor modifies - ** the page while cursor pCur is holding a reference to it. Which can - ** only happen if the database is corrupt in such a way as to link the - ** page into more than one b-tree structure. */ - testcase( idx>pPage->nCell ); + pPage = pCur->pPage; + idx = ++pCur->ix; + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } if( idx>=pPage->nCell ){ if( !pPage->leaf ){ @@ -63092,15 +76943,14 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ } do{ if( pCur->iPage==0 ){ - *pRes = 1; pCur->eState = CURSOR_INVALID; - return SQLITE_OK; + return SQLITE_DONE; } moveToParent(pCur); - pPage = pCur->apPage[pCur->iPage]; - }while( pCur->aiIdx[pCur->iPage]>=pPage->nCell ); + pPage = pCur->pPage; + }while( pCur->ix>=pPage->nCell ); if( pPage->intKey ){ - return sqlite3BtreeNext(pCur, pRes); + return sqlite3BtreeNext(pCur, 0); }else{ return SQLITE_OK; } @@ -63111,20 +76961,18 @@ static SQLITE_NOINLINE int btreeNext(BtCursor *pCur, int *pRes){ return moveToLeftmost(pCur); } } -SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ +SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int flags){ MemPage *pPage; + UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ assert( cursorOwnsBtShared(pCur) ); - assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); + assert( flags==0 || flags==1 ); pCur->info.nSize = 0; pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); - *pRes = 0; - if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur, pRes); - pPage = pCur->apPage[pCur->iPage]; - if( (++pCur->aiIdx[pCur->iPage])>=pPage->nCell ){ - pCur->aiIdx[pCur->iPage]--; - return btreeNext(pCur, pRes); + if( pCur->eState!=CURSOR_VALID ) return btreeNext(pCur); + pPage = pCur->pPage; + if( (++pCur->ix)>=pPage->nCell ){ + pCur->ix--; + return btreeNext(pCur); } if( pPage->leaf ){ return SQLITE_OK; @@ -63134,10 +76982,12 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ } /* -** Step the cursor to the back to the previous entry in the database. If -** successful then set *pRes=0. If the cursor -** was already pointing to the first entry in the database before -** this routine was called, then set *pRes=1. +** Step the cursor to the back to the previous entry in the database. +** Return values: +** +** SQLITE_OK success +** SQLITE_DONE the cursor is already on the first element of the table +** otherwise some kind of error occurred ** ** The main entry point is sqlite3BtreePrevious(). That routine is optimized ** for the common case of merely decrementing the cell counter BtCursor.aiIdx @@ -63145,23 +76995,17 @@ SQLITE_PRIVATE int sqlite3BtreeNext(BtCursor *pCur, int *pRes){ ** helper routine is called when it is necessary to move to a different page ** or to restore the cursor. ** -** The calling function will set *pRes to 0 or 1. The initial *pRes value -** will be 1 if the cursor being stepped corresponds to an SQL index and -** if this routine could have been skipped if that SQL index had been -** a unique index. Otherwise the caller will have set *pRes to zero. -** Zero is the common case. The btree implementation is free to use the -** initial *pRes value as a hint to improve performance, but the current -** SQLite btree implementation does not. (Note that the comdb2 btree -** implementation does use this hint, however.) +** If bit 0x01 of the F argument to sqlite3BtreePrevious(C,F) is 1, then +** the cursor corresponds to an SQL index and this routine could have been +** skipped if the SQL index had been a unique index. The F argument is a +** hint to the implement. The native SQLite btree implementation does not +** use this hint, but COMDB2 does. */ -static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ +static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur){ int rc; MemPage *pPage; assert( cursorOwnsBtShared(pCur) ); - assert( pRes!=0 ); - assert( *pRes==0 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); assert( (pCur->curFlags & (BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey))==0 ); assert( pCur->info.nSize==0 ); if( pCur->eState!=CURSOR_VALID ){ @@ -63170,64 +77014,58 @@ static SQLITE_NOINLINE int btreePrevious(BtCursor *pCur, int *pRes){ return rc; } if( CURSOR_INVALID==pCur->eState ){ - *pRes = 1; - return SQLITE_OK; + return SQLITE_DONE; } - if( pCur->skipNext ){ - assert( pCur->eState==CURSOR_VALID || pCur->eState==CURSOR_SKIPNEXT ); + if( CURSOR_SKIPNEXT==pCur->eState ){ pCur->eState = CURSOR_VALID; - if( pCur->skipNext<0 ){ - pCur->skipNext = 0; - return SQLITE_OK; - } - pCur->skipNext = 0; + if( pCur->skipNext<0 ) return SQLITE_OK; } } - pPage = pCur->apPage[pCur->iPage]; - assert( pPage->isInit ); + pPage = pCur->pPage; + if( sqlite3FaultSim(412) ) pPage->isInit = 0; + if( !pPage->isInit ){ + return SQLITE_CORRUPT_BKPT; + } if( !pPage->leaf ){ - int idx = pCur->aiIdx[pCur->iPage]; + int idx = pCur->ix; rc = moveToChild(pCur, get4byte(findCell(pPage, idx))); if( rc ) return rc; rc = moveToRightmost(pCur); }else{ - while( pCur->aiIdx[pCur->iPage]==0 ){ + while( pCur->ix==0 ){ if( pCur->iPage==0 ){ pCur->eState = CURSOR_INVALID; - *pRes = 1; - return SQLITE_OK; + return SQLITE_DONE; } moveToParent(pCur); } assert( pCur->info.nSize==0 ); - assert( (pCur->curFlags & (BTCF_ValidNKey|BTCF_ValidOvfl))==0 ); + assert( (pCur->curFlags & (BTCF_ValidOvfl))==0 ); - pCur->aiIdx[pCur->iPage]--; - pPage = pCur->apPage[pCur->iPage]; + pCur->ix--; + pPage = pCur->pPage; if( pPage->intKey && !pPage->leaf ){ - rc = sqlite3BtreePrevious(pCur, pRes); + rc = sqlite3BtreePrevious(pCur, 0); }else{ rc = SQLITE_OK; } } return rc; } -SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ +SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int flags){ assert( cursorOwnsBtShared(pCur) ); - assert( pRes!=0 ); - assert( *pRes==0 || *pRes==1 ); - assert( pCur->skipNext==0 || pCur->eState!=CURSOR_VALID ); - *pRes = 0; + assert( flags==0 || flags==1 ); + UNUSED_PARAMETER( flags ); /* Used in COMDB2 but not native SQLite */ pCur->curFlags &= ~(BTCF_AtLast|BTCF_ValidOvfl|BTCF_ValidNKey); pCur->info.nSize = 0; if( pCur->eState!=CURSOR_VALID - || pCur->aiIdx[pCur->iPage]==0 - || pCur->apPage[pCur->iPage]->leaf==0 + || pCur->ix==0 + || pCur->pPage->leaf==0 ){ - return btreePrevious(pCur, pRes); + return btreePrevious(pCur); } - pCur->aiIdx[pCur->iPage]--; + pCur->ix--; return SQLITE_OK; } @@ -63242,7 +77080,7 @@ SQLITE_PRIVATE int sqlite3BtreePrevious(BtCursor *pCur, int *pRes){ ** SQLITE_OK is returned on success. Any other return value indicates ** an error. *ppPage is set to NULL in the event of an error. ** -** If the "nearby" parameter is not 0, then an effort is made to +** If the "nearby" parameter is not 0, then an effort is made to ** locate a page close to the page number "nearby". This can be used in an ** attempt to keep related pages close to each other in the database file, ** which in turn can make database access faster. @@ -63272,8 +77110,8 @@ static int allocateBtreePage( assert( eMode==BTALLOC_ANY || (nearby>0 && IfNotOmitAV(pBt->autoVacuum)) ); pPage1 = pBt->pPage1; mxPage = btreePagecount(pBt); - /* EVIDENCE-OF: R-05119-02637 The 4-byte big-endian integer at offset 36 - ** stores stores the total number of pages on the freelist. */ + /* EVIDENCE-OF: R-21003-45125 The 4-byte big-endian integer at offset 36 + ** stores the total number of pages on the freelist. */ n = get4byte(&pPage1->aData[36]); testcase( n==mxPage-1 ); if( n>=mxPage ){ @@ -63284,7 +77122,7 @@ static int allocateBtreePage( Pgno iTrunk; u8 searchList = 0; /* If the free-list must be searched for 'nearby' */ u32 nSearch = 0; /* Count of the number of search attempts */ - + /* If eMode==BTALLOC_EXACT and a query of the pointer-map ** shows that the page 'nearby' is somewhere on the free-list, then ** the entire-list will be searched for that page. @@ -63333,7 +77171,7 @@ static int allocateBtreePage( } testcase( iTrunk==mxPage ); if( iTrunk>mxPage || nSearch++ > n ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pPrevTrunk ? pPrevTrunk->pgno : 1); }else{ rc = btreeGetUnusedPage(pBt, iTrunk, &pTrunk, 0); } @@ -63347,8 +77185,8 @@ static int allocateBtreePage( ** is the number of leaf page pointers to follow. */ k = get4byte(&pTrunk->aData[4]); if( k==0 && !searchList ){ - /* The trunk has no leaves and the list is not being searched. - ** So extract the trunk page itself and use it as the newly + /* The trunk has no leaves and the list is not being searched. + ** So extract the trunk page itself and use it as the newly ** allocated page */ assert( pPrevTrunk==0 ); rc = sqlite3PagerWrite(pTrunk->pDbPage); @@ -63359,14 +77197,14 @@ static int allocateBtreePage( memcpy(&pPage1->aData[32], &pTrunk->aData[0], 4); *ppPage = pTrunk; pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); }else if( k>(u32)(pBt->usableSize/4 - 2) ){ /* Value of k is out of range. Database corruption */ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(iTrunk); goto end_allocate_page; #ifndef SQLITE_OMIT_AUTOVACUUM - }else if( searchList - && (nearby==iTrunk || (iTrunkaData[0], &pTrunk->aData[0], 4); } }else{ - /* The trunk page is required by the caller but it contains + /* The trunk page is required by the caller but it contains ** pointers to free-list leaves. The first leaf becomes a trunk ** page in this case. */ MemPage *pNewTrunk; Pgno iNewTrunk = get4byte(&pTrunk->aData[8]); - if( iNewTrunk>mxPage ){ - rc = SQLITE_CORRUPT_BKPT; + if( iNewTrunk>mxPage ){ + rc = SQLITE_CORRUPT_PGNO(iTrunk); goto end_allocate_page; } testcase( iNewTrunk==mxPage ); @@ -63425,7 +77263,7 @@ static int allocateBtreePage( } } pTrunk = 0; - TRACE(("ALLOCATE: %d trunk - %d free pages left\n", *pPgno, n-1)); + TRACE(("ALLOCATE: %u trunk - %u free pages left\n", *pPgno, n-1)); #endif }else if( k>0 ){ /* Extract a leaf from the trunk */ @@ -63460,18 +77298,18 @@ static int allocateBtreePage( iPage = get4byte(&aData[8+closest*4]); testcase( iPage==mxPage ); - if( iPage>mxPage ){ - rc = SQLITE_CORRUPT_BKPT; + if( iPage>mxPage || iPage<2 ){ + rc = SQLITE_CORRUPT_PGNO(iTrunk); goto end_allocate_page; } testcase( iPage==mxPage ); - if( !searchList - || (iPage==nearby || (iPagepgno, n-1)); rc = sqlite3PagerWrite(pTrunk->pDbPage); if( rc ) goto end_allocate_page; @@ -63507,7 +77345,7 @@ static int allocateBtreePage( ** not set the no-content flag. This causes the pager to load and journal ** the current page content before overwriting it. ** - ** Note that the pager will not actually attempt to load or journal + ** Note that the pager will not actually attempt to load or journal ** content for any page that really does lie past the end of the database ** file on disk. So the effects of disabling the no-content optimization ** here are confined to those pages that lie between the end of the @@ -63527,7 +77365,7 @@ static int allocateBtreePage( ** becomes a new pointer-map page, the second is used by the caller. */ MemPage *pPg = 0; - TRACE(("ALLOCATE: %d from end of file (pointer-map page)\n", pBt->nPage)); + TRACE(("ALLOCATE: %u from end of file (pointer-map page)\n", pBt->nPage)); assert( pBt->nPage!=PENDING_BYTE_PAGE(pBt) ); rc = btreeGetUnusedPage(pBt, pBt->nPage, &pPg, bNoContent); if( rc==SQLITE_OK ){ @@ -63550,10 +77388,10 @@ static int allocateBtreePage( releasePage(*ppPage); *ppPage = 0; } - TRACE(("ALLOCATE: %d from end of file\n", *pPgno)); + TRACE(("ALLOCATE: %u from end of file\n", *pPgno)); } - assert( *pPgno!=PENDING_BYTE_PAGE(pBt) ); + assert( CORRUPT_DB || *pPgno!=PENDING_BYTE_PAGE(pBt) ); end_allocate_page: releasePage(pTrunk); @@ -63564,12 +77402,12 @@ end_allocate_page: } /* -** This function is used to add page iPage to the database file free-list. +** This function is used to add page iPage to the database file free-list. ** It is assumed that the page is not already a part of the free-list. ** ** The value passed as the second argument to this function is optional. -** If the caller happens to have a pointer to the MemPage object -** corresponding to page iPage handy, it may pass it as the second value. +** If the caller happens to have a pointer to the MemPage object +** corresponding to page iPage handy, it may pass it as the second value. ** Otherwise, it may pass NULL. ** ** If a pointer to a MemPage object is passed as the second argument, @@ -63577,17 +77415,19 @@ end_allocate_page: */ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ MemPage *pTrunk = 0; /* Free-list trunk page */ - Pgno iTrunk = 0; /* Page number of free-list trunk page */ + Pgno iTrunk = 0; /* Page number of free-list trunk page */ MemPage *pPage1 = pBt->pPage1; /* Local reference to page 1 */ MemPage *pPage; /* Page being freed. May be NULL. */ int rc; /* Return Code */ - int nFree; /* Initial number of pages on free-list */ + u32 nFree; /* Initial number of pages on free-list */ assert( sqlite3_mutex_held(pBt->mutex) ); assert( CORRUPT_DB || iPage>1 ); assert( !pMemPage || pMemPage->pgno==iPage ); - if( iPage<2 ) return SQLITE_CORRUPT_BKPT; + if( iPage<2 || iPage>pBt->nPage ){ + return SQLITE_CORRUPT_BKPT; + } if( pMemPage ){ pPage = pMemPage; sqlite3PagerRef(pPage->pDbPage); @@ -63616,7 +77456,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ /* If the database supports auto-vacuum, write an entry in the pointer-map ** to indicate that the page is free. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, iPage, PTRMAP_FREEPAGE, 0, &rc); if( rc ) goto freepage_out; } @@ -63632,6 +77472,10 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ u32 nLeaf; /* Initial number of leaf cells on trunk page */ iTrunk = get4byte(&pPage1->aData[32]); + if( iTrunk>btreePagecount(pBt) ){ + rc = SQLITE_CORRUPT_BKPT; + goto freepage_out; + } rc = btreeGetPage(pBt, iTrunk, &pTrunk, 0); if( rc!=SQLITE_OK ){ goto freepage_out; @@ -63672,14 +77516,14 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ } rc = btreeSetHasContent(pBt, iPage); } - TRACE(("FREE-PAGE: %d leaf on trunk page %d\n",pPage->pgno,pTrunk->pgno)); + TRACE(("FREE-PAGE: %u leaf on trunk page %u\n",pPage->pgno,pTrunk->pgno)); goto freepage_out; } } /* If control flows to this point, then it was not possible to add the ** the page being freed as a leaf page of the first trunk in the free-list. - ** Possibly because the free-list is empty, or possibly because the + ** Possibly because the free-list is empty, or possibly because the ** first trunk in the free-list is full. Either way, the page being freed ** will become the new first trunk page in the free-list. */ @@ -63693,7 +77537,7 @@ static int freePage2(BtShared *pBt, MemPage *pMemPage, Pgno iPage){ put4byte(pPage->aData, iTrunk); put4byte(&pPage->aData[4], 0); put4byte(&pPage1->aData[32], iPage); - TRACE(("FREE-PAGE: %d new trunk page replacing %d\n", pPage->pgno, iTrunk)); + TRACE(("FREE-PAGE: %u new trunk page replacing %u\n", pPage->pgno, iTrunk)); freepage_out: if( pPage ){ @@ -63710,44 +77554,41 @@ static void freePage(MemPage *pPage, int *pRC){ } /* -** Free any overflow pages associated with the given Cell. Write the -** local Cell size (the number of bytes on the original page, omitting -** overflow) into *pnSize. +** Free the overflow pages associated with the given Cell. */ -static int clearCell( +static SQLITE_NOINLINE int clearCellOverflow( MemPage *pPage, /* The page that contains the Cell */ unsigned char *pCell, /* First byte of the Cell */ - u16 *pnSize /* Write the size of the Cell here */ + CellInfo *pInfo /* Size information about the cell */ ){ - BtShared *pBt = pPage->pBt; - CellInfo info; + BtShared *pBt; Pgno ovflPgno; int rc; int nOvfl; u32 ovflPageSize; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - pPage->xParseCell(pPage, pCell, &info); - *pnSize = info.nSize; - if( info.nLocal==info.nPayload ){ - return SQLITE_OK; /* No overflow pages. Return without doing anything */ + assert( pInfo->nLocal!=pInfo->nPayload ); + testcase( pCell + pInfo->nSize == pPage->aDataEnd ); + testcase( pCell + (pInfo->nSize-1) == pPage->aDataEnd ); + if( pCell + pInfo->nSize > pPage->aDataEnd ){ + /* Cell extends past end of page */ + return SQLITE_CORRUPT_PAGE(pPage); } - if( pCell+info.nSize-1 > pPage->aData+pPage->maskPage ){ - return SQLITE_CORRUPT_BKPT; /* Cell extends past end of page */ - } - ovflPgno = get4byte(pCell + info.nSize - 4); + ovflPgno = get4byte(pCell + pInfo->nSize - 4); + pBt = pPage->pBt; assert( pBt->usableSize > 4 ); ovflPageSize = pBt->usableSize - 4; - nOvfl = (info.nPayload - info.nLocal + ovflPageSize - 1)/ovflPageSize; - assert( nOvfl>0 || - (CORRUPT_DB && (info.nPayload + ovflPageSize)nPayload - pInfo->nLocal + ovflPageSize - 1)/ovflPageSize; + assert( nOvfl>0 || + (CORRUPT_DB && (pInfo->nPayload + ovflPageSize)btreePagecount(pBt) ){ - /* 0 is not a legal page number and page 1 cannot be an - ** overflow page. Therefore if ovflPgno<2 or past the end of the + /* 0 is not a legal page number and page 1 cannot be an + ** overflow page. Therefore if ovflPgno<2 or past the end of the ** file the database must be corrupt. */ return SQLITE_CORRUPT_BKPT; } @@ -63759,11 +77600,11 @@ static int clearCell( if( ( pOvfl || ((pOvfl = btreePageLookup(pBt, ovflPgno))!=0) ) && sqlite3PagerPageRefcount(pOvfl->pDbPage)!=1 ){ - /* There is no reason any cursor should have an outstanding reference + /* There is no reason any cursor should have an outstanding reference ** to an overflow page belonging to a cell that is being deleted/updated. - ** So if there exists more than one reference to this page, then it - ** must not really be an overflow page and the database must be corrupt. - ** It is helpful to detect this before calling freePage2(), as + ** So if there exists more than one reference to this page, then it + ** must not really be an overflow page and the database must be corrupt. + ** It is helpful to detect this before calling freePage2(), as ** freePage2() may zero the page contents if secure-delete mode is ** enabled. If this 'overflow' page happens to be a page that the ** caller is iterating through or using in some other way, this @@ -63783,6 +77624,21 @@ static int clearCell( return SQLITE_OK; } +/* Call xParseCell to compute the size of a cell. If the cell contains +** overflow, then invoke cellClearOverflow to clear out that overflow. +** Store the result code (SQLITE_OK or some error code) in rc. +** +** Implemented as macro to force inlining for performance. +*/ +#define BTREE_CLEAR_CELL(rc, pPage, pCell, sInfo) \ + pPage->xParseCell(pPage, pCell, &sInfo); \ + if( sInfo.nLocal!=sInfo.nPayload ){ \ + rc = clearCellOverflow(pPage, pCell, &sInfo); \ + }else{ \ + rc = SQLITE_OK; \ + } + + /* ** Create the byte sequence used to represent a cell on page pPage ** and write that byte sequence into pCell[]. Overflow pages are @@ -63798,71 +77654,77 @@ static int clearCell( static int fillInCell( MemPage *pPage, /* The page that contains the cell */ unsigned char *pCell, /* Complete text of the cell */ - const void *pKey, i64 nKey, /* The key */ - const void *pData,int nData, /* The data */ - int nZero, /* Extra zero bytes to append to pData */ + const BtreePayload *pX, /* Payload with which to construct the cell */ int *pnSize /* Write cell size here */ ){ int nPayload; const u8 *pSrc; - int nSrc, n, rc; + int nSrc, n, rc, mn; int spaceLeft; - MemPage *pOvfl = 0; - MemPage *pToRelease = 0; + MemPage *pToRelease; unsigned char *pPrior; unsigned char *pPayload; - BtShared *pBt = pPage->pBt; - Pgno pgnoOvfl = 0; + BtShared *pBt; + Pgno pgnoOvfl; int nHeader; assert( sqlite3_mutex_held(pPage->pBt->mutex) ); /* pPage is not necessarily writeable since pCell might be auxiliary ** buffer space that is separate from the pPage buffer area */ - assert( pCellaData || pCell>=&pPage->aData[pBt->pageSize] + assert( pCellaData || pCell>=&pPage->aData[pPage->pBt->pageSize] || sqlite3PagerIswriteable(pPage->pDbPage) ); /* Fill in the header. */ nHeader = pPage->childPtrSize; - nPayload = nData + nZero; - if( pPage->intKeyLeaf ){ - nHeader += putVarint32(&pCell[nHeader], nPayload); - }else{ - assert( nData==0 ); - assert( nZero==0 ); - } - nHeader += putVarint(&pCell[nHeader], *(u64*)&nKey); - - /* Fill in the payload size */ if( pPage->intKey ){ - pSrc = pData; - nSrc = nData; - nData = 0; - }else{ - assert( nKey<=0x7fffffff && pKey!=0 ); - nPayload = (int)nKey; - pSrc = pKey; - nSrc = (int)nKey; + nPayload = pX->nData + pX->nZero; + pSrc = pX->pData; + nSrc = pX->nData; + assert( pPage->intKeyLeaf ); /* fillInCell() only called for leaves */ + nHeader += putVarint32(&pCell[nHeader], nPayload); + nHeader += putVarint(&pCell[nHeader], *(u64*)&pX->nKey); + }else{ + assert( pX->nKey<=0x7fffffff && pX->pKey!=0 ); + nSrc = nPayload = (int)pX->nKey; + pSrc = pX->pKey; + nHeader += putVarint32(&pCell[nHeader], nPayload); } + + /* Fill in the payload */ + pPayload = &pCell[nHeader]; if( nPayload<=pPage->maxLocal ){ + /* This is the common case where everything fits on the btree page + ** and no overflow pages are required. */ n = nHeader + nPayload; testcase( n==3 ); testcase( n==4 ); - if( n<4 ) n = 4; + if( n<4 ){ + n = 4; + pPayload[nPayload] = 0; + } *pnSize = n; - spaceLeft = nPayload; - pPrior = pCell; - }else{ - int mn = pPage->minLocal; - n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); - testcase( n==pPage->maxLocal ); - testcase( n==pPage->maxLocal+1 ); - if( n > pPage->maxLocal ) n = mn; - spaceLeft = n; - *pnSize = n + nHeader + 4; - pPrior = &pCell[nHeader+n]; + assert( nSrc<=nPayload ); + testcase( nSrcminLocal; + n = mn + (nPayload - mn) % (pPage->pBt->usableSize - 4); + testcase( n==pPage->maxLocal ); + testcase( n==pPage->maxLocal+1 ); + if( n > pPage->maxLocal ) n = mn; + spaceLeft = n; + *pnSize = n + nHeader + 4; + pPrior = &pCell[nHeader+n]; + pToRelease = 0; + pgnoOvfl = 0; + pBt = pPage->pBt; /* At this point variables should be set as follows: ** @@ -63876,27 +77738,54 @@ static int fillInCell( ** Use a call to btreeParseCellPtr() to verify that the values above ** were computed correctly. */ -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG { CellInfo info; pPage->xParseCell(pPage, pCell, &info); assert( nHeader==(int)(info.pPayload - pCell) ); - assert( info.nKey==nKey ); + assert( info.nKey==pX->nKey ); assert( *pnSize == info.nSize ); assert( spaceLeft == info.nLocal ); } #endif /* Write the payload into the local Cell and any extra into overflow pages */ - while( nPayload>0 ){ + while( 1 ){ + n = nPayload; + if( n>spaceLeft ) n = spaceLeft; + + /* If pToRelease is not zero than pPayload points into the data area + ** of pToRelease. Make sure pToRelease is still writeable. */ + assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); + + /* If pPayload is part of the data area of pPage, then make sure pPage + ** is still writeable */ + assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] + || sqlite3PagerIswriteable(pPage->pDbPage) ); + + if( nSrc>=n ){ + memcpy(pPayload, pSrc, n); + }else if( nSrc>0 ){ + n = nSrc; + memcpy(pPayload, pSrc, n); + }else{ + memset(pPayload, 0, n); + } + nPayload -= n; + if( nPayload<=0 ) break; + pPayload += n; + pSrc += n; + nSrc -= n; + spaceLeft -= n; if( spaceLeft==0 ){ + MemPage *pOvfl = 0; #ifndef SQLITE_OMIT_AUTOVACUUM Pgno pgnoPtrmap = pgnoOvfl; /* Overflow page pointer-map entry page */ if( pBt->autoVacuum ){ do{ pgnoOvfl++; - } while( - PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) + } while( + PTRMAP_ISPAGE(pBt, pgnoOvfl) || pgnoOvfl==PENDING_BYTE_PAGE(pBt) ); } #endif @@ -63904,9 +77793,9 @@ static int fillInCell( #ifndef SQLITE_OMIT_AUTOVACUUM /* If the database supports auto-vacuum, and the second or subsequent ** overflow page is being allocated, add an entry to the pointer-map - ** for that page now. + ** for that page now. ** - ** If this is the first overflow page, then write a partial entry + ** If this is the first overflow page, then write a partial entry ** to the pointer-map. If we write nothing to this pointer-map slot, ** then the optimistic overflow chain processing in clearCell() ** may misinterpret the uninitialized values and delete the @@ -63942,34 +77831,6 @@ static int fillInCell( pPayload = &pOvfl->aData[4]; spaceLeft = pBt->usableSize - 4; } - n = nPayload; - if( n>spaceLeft ) n = spaceLeft; - - /* If pToRelease is not zero than pPayload points into the data area - ** of pToRelease. Make sure pToRelease is still writeable. */ - assert( pToRelease==0 || sqlite3PagerIswriteable(pToRelease->pDbPage) ); - - /* If pPayload is part of the data area of pPage, then make sure pPage - ** is still writeable */ - assert( pPayloadaData || pPayload>=&pPage->aData[pBt->pageSize] - || sqlite3PagerIswriteable(pPage->pDbPage) ); - - if( nSrc>0 ){ - if( n>nSrc ) n = nSrc; - assert( pSrc ); - memcpy(pPayload, pSrc, n); - }else{ - memset(pPayload, 0, n); - } - nPayload -= n; - pPayload += n; - pSrc += n; - nSrc -= n; - spaceLeft -= n; - if( nSrc==0 ){ - nSrc = nData; - pSrc = pData; - } } releasePage(pToRelease); return SQLITE_OK; @@ -63991,18 +77852,20 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ int hdr; /* Beginning of the header. 0 most pages. 100 page 1 */ if( *pRC ) return; - - assert( idx>=0 && idxnCell ); + assert( idx>=0 ); + assert( idxnCell ); assert( CORRUPT_DB || sz==cellSize(pPage, idx) ); assert( sqlite3PagerIswriteable(pPage->pDbPage) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( pPage->nFree>=0 ); data = pPage->aData; ptr = &pPage->aCellIdx[2*idx]; + assert( pPage->pBt->usableSize > (u32)(ptr-data) ); pc = get2byte(ptr); hdr = pPage->hdrOffset; - testcase( pc==get2byte(&data[hdr+5]) ); + testcase( pc==(u32)get2byte(&data[hdr+5]) ); testcase( pc+sz==pPage->pBt->usableSize ); - if( pc < (u32)get2byte(&data[hdr+5]) || pc+sz > pPage->pBt->usableSize ){ + if( pc+sz > pPage->pBt->usableSize ){ *pRC = SQLITE_CORRUPT_BKPT; return; } @@ -64033,48 +77896,141 @@ static void dropCell(MemPage *pPage, int idx, int sz, int *pRC){ ** will not fit, then make a copy of the cell content into pTemp if ** pTemp is not null. Regardless of pTemp, allocate a new entry ** in pPage->apOvfl[] and make it point to the cell content (either -** in pTemp or the original pCell) and also record its index. -** Allocating a new entry in pPage->aCell[] implies that +** in pTemp or the original pCell) and also record its index. +** Allocating a new entry in pPage->aCell[] implies that ** pPage->nOverflow is incremented. +** +** The insertCellFast() routine below works exactly the same as +** insertCell() except that it lacks the pTemp and iChild parameters +** which are assumed zero. Other than that, the two routines are the +** same. +** +** Fixes or enhancements to this routine should be reflected in +** insertCellFast()! */ -static void insertCell( +static int insertCell( MemPage *pPage, /* Page into which we are copying */ int i, /* New cell becomes the i-th cell of the page */ u8 *pCell, /* Content of the new cell */ int sz, /* Bytes of content in pCell */ u8 *pTemp, /* Temp storage space for pCell, if needed */ - Pgno iChild, /* If non-zero, replace first 4 bytes with this value */ - int *pRC /* Read and write return code from here */ + Pgno iChild /* If non-zero, replace first 4 bytes with this value */ ){ int idx = 0; /* Where to write new cell content in data[] */ int j; /* Loop counter */ u8 *data; /* The content of the whole page */ u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ - if( *pRC ) return; - assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); assert( MX_CELL(pPage->pBt)<=10921 ); assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); assert( sqlite3_mutex_held(pPage->pBt->mutex) ); - /* The cell should normally be sized correctly. However, when moving a - ** malformed cell from a leaf page to an interior page, if the cell size - ** wanted to be less than 4 but got rounded up to 4 on the leaf, then size - ** might be less than 8 (leaf-size + pointer) on the interior node. Hence - ** the term after the || in the following assert(). */ - assert( sz==pPage->xCellSize(pPage, pCell) || (sz==8 && iChild>0) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + assert( iChild>0 ); if( pPage->nOverflow || sz+2>pPage->nFree ){ if( pTemp ){ memcpy(pTemp, pCell, sz); pCell = pTemp; } - if( iChild ){ - put4byte(pCell, iChild); - } + put4byte(pCell, iChild); j = pPage->nOverflow++; - assert( j<(int)(sizeof(pPage->apOvfl)/sizeof(pPage->apOvfl[0])) ); + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); + pPage->apOvfl[j] = pCell; + pPage->aiOvfl[j] = (u16)i; + + /* When multiple overflows occur, they are always sequential and in + ** sorted order. This invariants arise because multiple overflows can + ** only occur when inserting divider cells into the parent page during + ** balancing, and the dividers are adjacent and sorted. + */ + assert( j==0 || pPage->aiOvfl[j-1]<(u16)i ); /* Overflows in sorted order */ + assert( j==0 || i==pPage->aiOvfl[j-1]+1 ); /* Overflows are sequential */ + }else{ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( NEVER(rc!=SQLITE_OK) ){ + return rc; + } + assert( sqlite3PagerIswriteable(pPage->pDbPage) ); + data = pPage->aData; + assert( &data[pPage->cellOffset]==pPage->aCellIdx ); + rc = allocateSpace(pPage, sz, &idx); + if( rc ){ return rc; } + /* The allocateSpace() routine guarantees the following properties + ** if it returns successfully */ + assert( idx >= 0 ); + assert( idx >= pPage->cellOffset+2*pPage->nCell+2 || CORRUPT_DB ); + assert( idx+sz <= (int)pPage->pBt->usableSize ); + pPage->nFree -= (u16)(2 + sz); + /* In a corrupt database where an entry in the cell index section of + ** a btree page has a value of 3 or less, the pCell value might point + ** as many as 4 bytes in front of the start of the aData buffer for + ** the source page. Make sure this does not cause problems by not + ** reading the first 4 bytes */ + memcpy(&data[idx+4], pCell+4, sz-4); + put4byte(&data[idx], iChild); + pIns = pPage->aCellIdx + i*2; + memmove(pIns+2, pIns, 2*(pPage->nCell - i)); + put2byte(pIns, idx); + pPage->nCell++; + /* increment the cell count */ + if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); +#ifndef SQLITE_OMIT_AUTOVACUUM + if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; + /* The cell may contain a pointer to an overflow page. If so, write + ** the entry for the overflow page into the pointer map. + */ + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; + } +#endif + } + return SQLITE_OK; +} + +/* +** This variant of insertCell() assumes that the pTemp and iChild +** parameters are both zero. Use this variant in sqlite3BtreeInsert() +** for performance improvement, and also so that this variant is only +** called from that one place, and is thus inlined, and thus runs must +** faster. +** +** Fixes or enhancements to this routine should be reflected into +** the insertCell() routine. +*/ +static int insertCellFast( + MemPage *pPage, /* Page into which we are copying */ + int i, /* New cell becomes the i-th cell of the page */ + u8 *pCell, /* Content of the new cell */ + int sz /* Bytes of content in pCell */ +){ + int idx = 0; /* Where to write new cell content in data[] */ + int j; /* Loop counter */ + u8 *data; /* The content of the whole page */ + u8 *pIns; /* The point in pPage->aCellIdx[] where no cell inserted */ + + assert( i>=0 && i<=pPage->nCell+pPage->nOverflow ); + assert( MX_CELL(pPage->pBt)<=10921 ); + assert( pPage->nCell<=MX_CELL(pPage->pBt) || CORRUPT_DB ); + assert( pPage->nOverflow<=ArraySize(pPage->apOvfl) ); + assert( ArraySize(pPage->apOvfl)==ArraySize(pPage->aiOvfl) ); + assert( sqlite3_mutex_held(pPage->pBt->mutex) ); + assert( sz==pPage->xCellSize(pPage, pCell) || CORRUPT_DB ); + assert( pPage->nFree>=0 ); + assert( pPage->nOverflow==0 ); + if( sz+2>pPage->nFree ){ + j = pPage->nOverflow++; + /* Comparison against ArraySize-1 since we hold back one extra slot + ** as a contingency. In other words, never need more than 3 overflow + ** slots but 4 are allocated, just to be safe. */ + assert( j < ArraySize(pPage->apOvfl)-1 ); pPage->apOvfl[j] = pCell; pPage->aiOvfl[j] = (u16)i; @@ -64088,14 +78044,13 @@ static void insertCell( }else{ int rc = sqlite3PagerWrite(pPage->pDbPage); if( rc!=SQLITE_OK ){ - *pRC = rc; - return; + return rc; } assert( sqlite3PagerIswriteable(pPage->pDbPage) ); data = pPage->aData; assert( &data[pPage->cellOffset]==pPage->aCellIdx ); rc = allocateSpace(pPage, sz, &idx); - if( rc ){ *pRC = rc; return; } + if( rc ){ return rc; } /* The allocateSpace() routine guarantees the following properties ** if it returns successfully */ assert( idx >= 0 ); @@ -64103,30 +78058,109 @@ static void insertCell( assert( idx+sz <= (int)pPage->pBt->usableSize ); pPage->nFree -= (u16)(2 + sz); memcpy(&data[idx], pCell, sz); - if( iChild ){ - put4byte(&data[idx], iChild); - } pIns = pPage->aCellIdx + i*2; memmove(pIns+2, pIns, 2*(pPage->nCell - i)); put2byte(pIns, idx); pPage->nCell++; /* increment the cell count */ if( (++data[pPage->hdrOffset+4])==0 ) data[pPage->hdrOffset+3]++; - assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell ); + assert( get2byte(&data[pPage->hdrOffset+3])==pPage->nCell || CORRUPT_DB ); #ifndef SQLITE_OMIT_AUTOVACUUM if( pPage->pBt->autoVacuum ){ + int rc2 = SQLITE_OK; /* The cell may contain a pointer to an overflow page. If so, write ** the entry for the overflow page into the pointer map. */ - ptrmapPutOvflPtr(pPage, pCell, pRC); + ptrmapPutOvflPtr(pPage, pPage, pCell, &rc2); + if( rc2 ) return rc2; } #endif } + return SQLITE_OK; } +/* +** The following parameters determine how many adjacent pages get involved +** in a balancing operation. NN is the number of neighbors on either side +** of the page that participate in the balancing operation. NB is the +** total number of pages that participate, including the target page and +** NN neighbors on either side. +** +** The minimum value of NN is 1 (of course). Increasing NN above 1 +** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance +** in exchange for a larger degradation in INSERT and UPDATE performance. +** The value of NN appears to give the best results overall. +** +** (Later:) The description above makes it seem as if these values are +** tunable - as if you could change them and recompile and it would all work. +** But that is unlikely. NB has been 3 since the inception of SQLite and +** we have never tested any other value. +*/ +#define NN 1 /* Number of neighbors on either side of pPage */ +#define NB 3 /* (NN*2+1): Total pages involved in the balance */ + /* ** A CellArray object contains a cache of pointers and sizes for a -** consecutive sequence of cells that might be held multiple pages. +** consecutive sequence of cells that might be held on multiple pages. +** +** The cells in this array are the divider cell or cells from the pParent +** page plus up to three child pages. There are a total of nCell cells. +** +** pRef is a pointer to one of the pages that contributes cells. This is +** used to access information such as MemPage.intKey and MemPage.pBt->pageSize +** which should be common to all pages that contribute cells to this array. +** +** apCell[] and szCell[] hold, respectively, pointers to the start of each +** cell and the size of each cell. Some of the apCell[] pointers might refer +** to overflow cells. In other words, some apCel[] pointers might not point +** to content area of the pages. +** +** A szCell[] of zero means the size of that cell has not yet been computed. +** +** The cells come from as many as four different pages: +** +** ----------- +** | Parent | +** ----------- +** / | \ +** / | \ +** --------- --------- --------- +** |Child-1| |Child-2| |Child-3| +** --------- --------- --------- +** +** The order of cells is in the array is for an index btree is: +** +** 1. All cells from Child-1 in order +** 2. The first divider cell from Parent +** 3. All cells from Child-2 in order +** 4. The second divider cell from Parent +** 5. All cells from Child-3 in order +** +** For a table-btree (with rowids) the items 2 and 4 are empty because +** content exists only in leaves and there are no divider cells. +** +** For an index btree, the apEnd[] array holds pointer to the end of page +** for Child-1, the Parent, Child-2, the Parent (again), and Child-3, +** respectively. The ixNx[] array holds the number of cells contained in +** each of these 5 stages, and all stages to the left. Hence: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 plus 1 for first divider. +** ixNx[2] = Number of cells in Child-1 and Child-2 + 1 for 1st divider. +** ixNx[3] = Number of cells in Child-1 and Child-2 + both divider cells +** ixNx[4] = Total number of cells. +** +** For a table-btree, the concept is similar, except only apEnd[0]..apEnd[2] +** are used and they point to the leaf pages only, and the ixNx value are: +** +** ixNx[0] = Number of cells in Child-1. +** ixNx[1] = Number of cells in Child-1 and Child-2. +** ixNx[2] = Total number of cells. +** +** Sometimes when deleting, a child page can have zero cells. In those +** cases, ixNx[] entries with higher indexes, and the corresponding apEnd[] +** entries, shift down. The end result is that each ixNx[] entry should +** be larger than the previous */ typedef struct CellArray CellArray; struct CellArray { @@ -64134,6 +78168,8 @@ struct CellArray { MemPage *pRef; /* Reference page */ u8 **apCell; /* All cells begin balanced */ u16 *szCell; /* Local size of all cells in apCell[] */ + u8 *apEnd[NB*2]; /* MemPage.aDataEnd values */ + int ixNx[NB*2]; /* Index of at which we move to the next apEnd[] */ }; /* @@ -64141,14 +78177,16 @@ struct CellArray { ** computed. */ static void populateCellCache(CellArray *p, int idx, int N){ + MemPage *pRef = p->pRef; + u16 *szCell = p->szCell; assert( idx>=0 && idx+N<=p->nCell ); while( N>0 ){ assert( p->apCell[idx]!=0 ); - if( p->szCell[idx]==0 ){ - p->szCell[idx] = p->pRef->xCellSize(p->pRef, p->apCell[idx]); + if( szCell[idx]==0 ){ + szCell[idx] = pRef->xCellSize(pRef, p->apCell[idx]); }else{ assert( CORRUPT_DB || - p->szCell[idx]==p->pRef->xCellSize(p->pRef, p->apCell[idx]) ); + szCell[idx]==pRef->xCellSize(pRef, p->apCell[idx]) ); } idx++; N--; @@ -64171,49 +78209,73 @@ static u16 cachedCellSize(CellArray *p, int N){ } /* -** Array apCell[] contains pointers to nCell b-tree page cells. The +** Array apCell[] contains pointers to nCell b-tree page cells. The ** szCell[] array contains the size in bytes of each cell. This function ** replaces the current contents of page pPg with the contents of the cell ** array. ** ** Some of the cells in apCell[] may currently be stored in pPg. This -** function works around problems caused by this by making a copy of any +** function works around problems caused by this by making a copy of any ** such cells before overwriting the page data. ** -** The MemPage.nFree field is invalidated by this function. It is the +** The MemPage.nFree field is invalidated by this function. It is the ** responsibility of the caller to set it correctly. */ static int rebuildPage( - MemPage *pPg, /* Edit this page */ + CellArray *pCArray, /* Content to be added to page pPg */ + int iFirst, /* First cell in pCArray to use */ int nCell, /* Final number of cells on page */ - u8 **apCell, /* Array of cells */ - u16 *szCell /* Array of cell sizes */ + MemPage *pPg /* The page to be reconstructed */ ){ const int hdr = pPg->hdrOffset; /* Offset of header on pPg */ u8 * const aData = pPg->aData; /* Pointer to data for pPg */ const int usableSize = pPg->pBt->usableSize; u8 * const pEnd = &aData[usableSize]; - int i; + int i = iFirst; /* Which cell to copy from pCArray*/ + u32 j; /* Start of cell content area */ + int iEnd = i+nCell; /* Loop terminator */ u8 *pCellptr = pPg->aCellIdx; u8 *pTmp = sqlite3PagerTempSpace(pPg->pBt->pPager); u8 *pData; + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pSrcEnd; /* Current pCArray->apEnd[k] value */ - i = get2byte(&aData[hdr+5]); - memcpy(&pTmp[i], &aData[i], usableSize - i); + assert( nCell>0 ); + assert( i(u32)usableSize ){ j = 0; } + memcpy(&pTmp[j], &aData[j], usableSize - j); + + assert( pCArray->ixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i; k++){} + pSrcEnd = pCArray->apEnd[k]; pData = pEnd; - for(i=0; iapCell[i]; + u16 sz = pCArray->szCell[i]; + assert( sz>0 ); + if( SQLITE_WITHIN(pCell,aData+j,pEnd) ){ + if( ((uptr)(pCell+sz))>(uptr)pEnd ) return SQLITE_CORRUPT_BKPT; pCell = &pTmp[pCell - aData]; + }else if( (uptr)(pCell+sz)>(uptr)pSrcEnd + && (uptr)(pCell)<(uptr)pSrcEnd + ){ + return SQLITE_CORRUPT_BKPT; } - pData -= szCell[i]; + + pData -= sz; put2byte(pCellptr, (pData - aData)); pCellptr += 2; if( pData < pCellptr ) return SQLITE_CORRUPT_BKPT; - memcpy(pData, pCell, szCell[i]); - assert( szCell[i]==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); - testcase( szCell[i]!=pPg->xCellSize(pPg,pCell) ); + memmove(pData, pCell, sz); + assert( sz==pPg->xCellSize(pPg, pCell) || CORRUPT_DB ); + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pSrcEnd = pCArray->apEnd[k]; + } } /* The pPg->nFree field is now set incorrectly. The caller will fix it. */ @@ -64228,12 +78290,11 @@ static int rebuildPage( } /* -** Array apCell[] contains nCell pointers to b-tree cells. Array szCell -** contains the size in bytes of each such cell. This function attempts to -** add the cells stored in the array to page pPg. If it cannot (because -** the page needs to be defragmented before the cells will fit), non-zero -** is returned. Otherwise, if the cells are added successfully, zero is -** returned. +** The pCArray objects contains pointers to b-tree cells and the cell sizes. +** This function attempts to add the cells stored in the array to page pPg. +** If it cannot (because the page needs to be defragmented before the cells +** will fit), non-zero is returned. Otherwise, if the cells are added +** successfully, zero is returned. ** ** Argument pCellptr points to the first entry in the cell-pointer array ** (part of page pPg) to populate. After cell apCell[0] is written to the @@ -64241,7 +78302,7 @@ static int rebuildPage( ** cell in the array. It is the responsibility of the caller to ensure ** that it is safe to overwrite this part of the cell-pointer array. ** -** When this function is called, *ppData points to the start of the +** When this function is called, *ppData points to the start of the ** content area on page pPg. If the size of the content area is extended, ** *ppData is updated to point to the new start of the content area ** before returning. @@ -64249,27 +78310,34 @@ static int rebuildPage( ** Finally, argument pBegin points to the byte immediately following the ** end of the space required by this page for the cell-pointer area (for ** all cells - not just those inserted by the current call). If the content -** area must be extended to before this point in order to accomodate all +** area must be extended to before this point in order to accommodate all ** cells in apCell[], then the cells do not fit and non-zero is returned. */ static int pageInsertArray( MemPage *pPg, /* Page to add cells to */ u8 *pBegin, /* End of cell-pointer array */ - u8 **ppData, /* IN/OUT: Page content -area pointer */ + u8 **ppData, /* IN/OUT: Page content-area pointer */ u8 *pCellptr, /* Pointer to cell-pointer area */ int iFirst, /* Index of first cell to add */ int nCell, /* Number of cells to add to pPg */ CellArray *pCArray /* Array of cells */ ){ - int i; - u8 *aData = pPg->aData; - u8 *pData = *ppData; - int iEnd = iFirst + nCell; + int i = iFirst; /* Loop counter - cell index to insert */ + u8 *aData = pPg->aData; /* Complete page */ + u8 *pData = *ppData; /* Content area. A subset of aData[] */ + int iEnd = iFirst + nCell; /* End of loop. One past last cell to ins */ + int k; /* Current slot in pCArray->apEnd[] */ + u8 *pEnd; /* Maximum extent of cell data */ assert( CORRUPT_DB || pPg->hdrOffset==0 ); /* Never called on page 1 */ - for(i=iFirst; iixNx[NB*2-1]>i ); + for(k=0; pCArray->ixNx[k]<=i ; k++){} + pEnd = pCArray->apEnd[k]; + while( 1 /*Exit by break*/ ){ int sz, rc; u8 *pSlot; - sz = cachedCellSize(pCArray, i); + assert( pCArray->szCell[i]!=0 ); + sz = pCArray->szCell[i]; if( (aData[1]==0 && aData[2]==0) || (pSlot = pageFindSlot(pPg,sz,&rc))==0 ){ if( (pData - pBegin)apCell[i] || pSlot>=(pCArray->apCell[i]+sz) || CORRUPT_DB ); + if( (uptr)(pCArray->apCell[i]+sz)>(uptr)pEnd + && (uptr)(pCArray->apCell[i])<(uptr)pEnd + ){ + assert( CORRUPT_DB ); + (void)SQLITE_CORRUPT_BKPT; + return 1; + } memmove(pSlot, pCArray->apCell[i], sz); put2byte(pCellptr, (pSlot - aData)); pCellptr += 2; + i++; + if( i>=iEnd ) break; + if( pCArray->ixNx[k]<=i ){ + k++; + pEnd = pCArray->apEnd[k]; + } } *ppData = pData; return 0; } /* -** Array apCell[] contains nCell pointers to b-tree cells. Array szCell -** contains the size in bytes of each such cell. This function adds the -** space associated with each cell in the array that is currently stored -** within the body of pPg to the pPg free-list. The cell-pointers and other -** fields of the page are not updated. +** The pCArray object contains pointers to b-tree cells and their sizes. +** +** This function adds the space associated with each cell in the array +** that is currently stored within the body of pPg to the pPg free-list. +** The cell-pointers and other fields of the page are not updated. ** ** This function returns the total number of cells added to the free-list. */ @@ -64308,45 +78389,58 @@ static int pageFreeArray( u8 * const pEnd = &aData[pPg->pBt->usableSize]; u8 * const pStart = &aData[pPg->hdrOffset + 8 + pPg->childPtrSize]; int nRet = 0; - int i; + int i, j; int iEnd = iFirst + nCell; - u8 *pFree = 0; - int szFree = 0; + int nFree = 0; + int aOfst[10]; + int aAfter[10]; for(i=iFirst; iapCell[i]; if( SQLITE_WITHIN(pCell, pStart, pEnd) ){ int sz; + int iAfter; + int iOfst; /* No need to use cachedCellSize() here. The sizes of all cells that ** are to be freed have already been computing while deciding which ** cells need freeing */ sz = pCArray->szCell[i]; assert( sz>0 ); - if( pFree!=(pCell + sz) ){ - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + iOfst = (u16)(pCell - aData); + iAfter = iOfst+sz; + for(j=0; jpEnd ) return 0; - }else{ - pFree = pCell; - szFree += sz; + } + if( j>=nFree ){ + if( nFree>=(int)(sizeof(aOfst)/sizeof(aOfst[0])) ){ + for(j=0; jpEnd ) return 0; + nFree++; } nRet++; } } - if( pFree ){ - assert( pFree>aData && (pFree - aData)<65536 ); - freeSpace(pPg, (u16)(pFree - aData), szFree); + for(j=0; jnCell cells starting -** with apCell[iOld]. After balancing, this page should hold nNew cells +** pCArray contains pointers to and sizes of all cells in the page being +** balanced. The current page, pPg, has pPg->nCell cells starting with +** pCArray->apCell[iOld]. After balancing, this page should hold nNew cells ** starting at apCell[iNew]. ** ** This routine makes the necessary adjustments to pPg so that it contains @@ -64378,22 +78472,28 @@ static int editPage( #endif /* Remove cells from the start and end of the page */ + assert( nCell>=0 ); if( iOldnCell) ) return SQLITE_CORRUPT_BKPT; memmove(pPg->aCellIdx, &pPg->aCellIdx[nShift*2], nCell*2); nCell -= nShift; } if( iNewEnd < iOldEnd ){ - nCell -= pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + int nTail = pageFreeArray(pPg, iNewEnd, iOldEnd - iNewEnd, pCArray); + assert( nCell>=nTail ); + nCell -= nTail; } - pData = &aData[get2byteNotZero(&aData[hdr+5])]; + pData = &aData[get2byte(&aData[hdr+5])]; if( pDatapPg->aDataEnd) ) goto editpage_fail; /* Add cells to the start of the page */ if( iNew=0 ); pCellptr = pPg->aCellIdx; memmove(&pCellptr[nAdd*2], pCellptr, nCell*2); if( pageInsertArray( @@ -64408,8 +78508,11 @@ static int editPage( int iCell = (iOld + pPg->aiOvfl[i]) - iNew; if( iCell>=0 && iCellaCellIdx[iCell * 2]; - memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + if( nCell>iCell ){ + memmove(&pCellptr[2], pCellptr, (nCell - iCell) * 2); + } nCell++; + cachedCellSize(pCArray, iCell+iNew); if( pageInsertArray( pPg, pBegin, &pData, pCellptr, iCell+iNew, 1, pCArray @@ -64418,6 +78521,7 @@ static int editPage( } /* Append cells to the end of the page */ + assert( nCell>=0 ); pCellptr = &pPg->aCellIdx[nCell*2]; if( pageInsertArray( pPg, pBegin, &pData, pCellptr, @@ -64445,25 +78549,11 @@ static int editPage( return SQLITE_OK; editpage_fail: /* Unable to edit this page. Rebuild it from scratch instead. */ + if( nNew<1 ) return SQLITE_CORRUPT_BKPT; populateCellCache(pCArray, iNew, nNew); - return rebuildPage(pPg, nNew, &pCArray->apCell[iNew], &pCArray->szCell[iNew]); + return rebuildPage(pCArray, iNew, nNew, pPg); } -/* -** The following parameters determine how many adjacent pages get involved -** in a balancing operation. NN is the number of neighbors on either side -** of the page that participate in the balancing operation. NB is the -** total number of pages that participate, including the target page and -** NN neighbors on either side. -** -** The minimum value of NN is 1 (of course). Increasing NN above 1 -** (to 2 or 3) gives a modest improvement in SELECT and DELETE performance -** in exchange for a larger degradation in INSERT and UPDATE performance. -** The value of NN appears to give the best results overall. -*/ -#define NN 1 /* Number of neighbors on either side of pPage */ -#define NB (NN*2+1) /* Total pages involved in the balance */ - #ifndef SQLITE_OMIT_QUICKBALANCE /* @@ -64499,10 +78589,11 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ assert( sqlite3PagerIswriteable(pParent->pDbPage) ); assert( pPage->nOverflow==1 ); - /* This error condition is now caught prior to reaching this function */ - if( NEVER(pPage->nCell==0) ) return SQLITE_CORRUPT_BKPT; + if( pPage->nCell==0 ) return SQLITE_CORRUPT_BKPT; /* dbfuzz001.test */ + assert( pPage->nFree>=0 ); + assert( pParent->nFree>=0 ); - /* Allocate a new page. This page will become the right-sibling of + /* Allocate a new page. This page will become the right-sibling of ** pPage. Make the parent page writable, so that the new divider cell ** may be inserted. If both these operations are successful, proceed. */ @@ -64514,37 +78605,48 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ u8 *pCell = pPage->apOvfl[0]; u16 szCell = pPage->xCellSize(pPage, pCell); u8 *pStop; + CellArray b; assert( sqlite3PagerIswriteable(pNew->pDbPage) ); - assert( pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); + assert( CORRUPT_DB || pPage->aData[0]==(PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF) ); zeroPage(pNew, PTF_INTKEY|PTF_LEAFDATA|PTF_LEAF); - rc = rebuildPage(pNew, 1, &pCell, &szCell); - if( NEVER(rc) ) return rc; + b.nCell = 1; + b.pRef = pPage; + b.apCell = &pCell; + b.szCell = &szCell; + b.apEnd[0] = pPage->aDataEnd; + b.ixNx[0] = 2; + b.ixNx[NB*2-1] = 0x7fffffff; + rc = rebuildPage(&b, 0, 1, pNew); + if( NEVER(rc) ){ + releasePage(pNew); + return rc; + } pNew->nFree = pBt->usableSize - pNew->cellOffset - 2 - szCell; /* If this is an auto-vacuum database, update the pointer map - ** with entries for the new page, and any pointer from the + ** with entries for the new page, and any pointer from the ** cell on the page to an overflow page. If either of these ** operations fails, the return code is set, but the contents - ** of the parent page are still manipulated by thh code below. + ** of the parent page are still manipulated by the code below. ** That is Ok, at this point the parent page is guaranteed to ** be marked as dirty. Returning an error code will cause a ** rollback, undoing any changes made to the parent page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoNew, PTRMAP_BTREE, pParent->pgno, &rc); if( szCell>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); + ptrmapPutOvflPtr(pNew, pNew, pCell, &rc); } } - + /* Create a divider cell to insert into pParent. The divider cell ** consists of a 4-byte page number (the page number of pPage) and ** a variable length key value (which must be the same value as the ** largest key on pPage). ** - ** To find the largest key value on pPage, first find the right-most - ** cell on pPage. The first two fields of this cell are the + ** To find the largest key value on pPage, first find the right-most + ** cell on pPage. The first two fields of this cell are the ** record-length (a variable length integer at most 32-bits in size) ** and the key value (a variable length integer, may have any value). ** The first of the while(...) loops below skips over the record-length @@ -64558,12 +78660,14 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ while( ((*(pOut++) = *(pCell++))&0x80) && pCellnCell, pSpace, (int)(pOut-pSpace), - 0, pPage->pgno, &rc); + if( rc==SQLITE_OK ){ + rc = insertCell(pParent, pParent->nCell, pSpace, (int)(pOut-pSpace), + 0, pPage->pgno); + } /* Set the right-child pointer of pParent to point to the new page. */ put4byte(&pParent->aData[pParent->hdrOffset+8], pgnoNew); - + /* Release the reference to the new page. */ releasePage(pNew); } @@ -64575,7 +78679,7 @@ static int balance_quick(MemPage *pParent, MemPage *pPage, u8 *pSpace){ #if 0 /* ** This function does not contribute anything to the operation of SQLite. -** it is sometimes activated temporarily while debugging code responsible +** it is sometimes activated temporarily while debugging code responsible ** for setting pointer-map entries. */ static int ptrmapCheckPages(MemPage **apPage, int nPage){ @@ -64590,7 +78694,7 @@ static int ptrmapCheckPages(MemPage **apPage, int nPage){ for(j=0; jnCell; j++){ CellInfo info; u8 *z; - + z = findCell(pPage, j); pPage->xParseCell(pPage, z, &info); if( info.nLocalpgno==1) ? 100 : 0); int rc; int iData; - - + + assert( pFrom->isInit ); assert( pFrom->nFree>=iToHdr ); assert( get2byte(&aFrom[iFromHdr+5]) <= (int)pBt->usableSize ); - + /* Copy the b-tree node content from page pFrom to page pTo. */ iData = get2byte(&aFrom[iFromHdr+5]); memcpy(&aTo[iData], &aFrom[iData], pBt->usableSize-iData); memcpy(&aTo[iToHdr], &aFrom[iFromHdr], pFrom->cellOffset + 2*pFrom->nCell); - + /* Reinitialize page pTo so that the contents of the MemPage structure ** match the new data. The initialization of pTo can actually fail under - ** fairly obscure circumstances, even though it is a copy of initialized + ** fairly obscure circumstances, even though it is a copy of initialized ** page pFrom. */ pTo->isInit = 0; rc = btreeInitPage(pTo); + if( rc==SQLITE_OK ) rc = btreeComputeFreeSpace(pTo); if( rc!=SQLITE_OK ){ *pRC = rc; return; } - + /* If this is an auto-vacuum database, update the pointer-map entries ** for any b-tree or overflow pages that pTo now contains the pointers to. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ *pRC = setChildPtrmaps(pTo); } } @@ -64677,13 +78782,13 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** (hereafter "the page") and up to 2 siblings so that all pages have about the ** same amount of free space. Usually a single sibling on either side of the ** page are used in the balancing, though both siblings might come from one -** side if the page is the first or last child of its parent. If the page +** side if the page is the first or last child of its parent. If the page ** has fewer than 2 siblings (something which can only happen if the page ** is a root page or a child of a root page) then all available siblings ** participate in the balancing. ** -** The number of siblings of the page might be increased or decreased by -** one or two in an effort to keep pages nearly full but not over full. +** The number of siblings of the page might be increased or decreased by +** one or two in an effort to keep pages nearly full but not over full. ** ** Note that when this routine is called, some of the cells on the page ** might not actually be stored in MemPage.aData[]. This can happen @@ -64694,7 +78799,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** inserted into or removed from the parent page (pParent). Doing so ** may cause the parent page to become overfull or underfull. If this ** happens, it is the responsibility of the caller to invoke the correct -** balancing routine to fix this problem (see the balance() routine). +** balancing routine to fix this problem (see the balance() routine). ** ** If this routine fails for any reason, it might leave the database ** in a corrupted state. So if this routine fails, the database should @@ -64709,7 +78814,7 @@ static void copyNodeContent(MemPage *pFrom, MemPage *pTo, int *pRC){ ** of the page-size, the aOvflSpace[] buffer is guaranteed to be large ** enough for all overflow cells. ** -** If aOvflSpace is set to a null pointer, this function returns +** If aOvflSpace is set to a null pointer, this function returns ** SQLITE_NOMEM. */ static int balance_nonroot( @@ -64744,23 +78849,18 @@ static int balance_nonroot( Pgno pgno; /* Temp var to store a page number in */ u8 abDone[NB+2]; /* True after i'th new page is populated */ Pgno aPgno[NB+2]; /* Page numbers of new pages before shuffling */ - Pgno aPgOrder[NB+2]; /* Copy of aPgno[] used for sorting pages */ - u16 aPgFlags[NB+2]; /* flags field of new pages before shuffling */ - CellArray b; /* Parsed information on cells being balanced */ + CellArray b; /* Parsed information on cells being balanced */ memset(abDone, 0, sizeof(abDone)); - b.nCell = 0; - b.apCell = 0; + assert( sizeof(b) - sizeof(b.ixNx) == offsetof(CellArray,ixNx) ); + memset(&b, 0, sizeof(b)-sizeof(b.ixNx[0])); + b.ixNx[NB*2-1] = 0x7fffffff; pBt = pParent->pBt; assert( sqlite3_mutex_held(pBt->mutex) ); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); -#if 0 - TRACE(("BALANCE: begin page %d child of %d\n", pPage->pgno, pParent->pgno)); -#endif - /* At this point pParent may have at most one overflow cell. And if - ** this overflow cell is present, it must be the cell with + ** this overflow cell is present, it must be the cell with ** index iParentIdx. This scenario comes about when this function ** is called (indirectly) from sqlite3BtreeDelete(). */ @@ -64770,12 +78870,13 @@ static int balance_nonroot( if( !aOvflSpace ){ return SQLITE_NOMEM_BKPT; } + assert( pParent->nFree>=0 ); - /* Find the sibling pages to balance. Also locate the cells in pParent - ** that divide the siblings. An attempt is made to find NN siblings on - ** either side of pPage. More siblings are taken from one side, however, + /* Find the sibling pages to balance. Also locate the cells in pParent + ** that divide the siblings. An attempt is made to find NN siblings on + ** either side of pPage. More siblings are taken from one side, however, ** if there are fewer than NN siblings on the other side. If pParent - ** has NB or fewer children then all children of pParent are taken. + ** has NB or fewer children then all children of pParent are taken. ** ** This loop also drops the divider cells from the parent page. This ** way, the remainder of the function does not have to deal with any @@ -64787,7 +78888,7 @@ static int balance_nonroot( nxDiv = 0; }else{ assert( bBulk==0 || bBulk==1 ); - if( iParentIdx==0 ){ + if( iParentIdx==0 ){ nxDiv = 0; }else if( iParentIdx==i ){ nxDiv = i-2+bBulk; @@ -64804,15 +78905,24 @@ static int balance_nonroot( } pgno = get4byte(pRight); while( 1 ){ - rc = getAndInitPage(pBt, pgno, &apOld[i], 0, 0); + if( rc==SQLITE_OK ){ + rc = getAndInitPage(pBt, pgno, &apOld[i], 0); + } if( rc ){ memset(apOld, 0, (i+1)*sizeof(MemPage*)); goto balance_cleanup; } - nMaxCells += 1+apOld[i]->nCell+apOld[i]->nOverflow; + if( apOld[i]->nFree<0 ){ + rc = btreeComputeFreeSpace(apOld[i]); + if( rc ){ + memset(apOld, 0, (i)*sizeof(MemPage*)); + goto balance_cleanup; + } + } + nMaxCells += apOld[i]->nCell + ArraySize(pParent->apOvfl); if( (i--)==0 ) break; - if( i+nxDiv==pParent->aiOvfl[0] && pParent->nOverflow ){ + if( pParent->nOverflow && i+nxDiv==pParent->aiOvfl[0] ){ apDiv[i] = pParent->apOvfl[0]; pgno = get4byte(apDiv[i]); szNew[i] = pParent->xCellSize(pParent, apDiv[i]); @@ -64827,22 +78937,20 @@ static int balance_nonroot( ** This is safe because dropping a cell only overwrites the first ** four bytes of it, and this function does not need the first ** four bytes of the divider cell. So the pointer is safe to use - ** later on. + ** later on. ** ** But not if we are in secure-delete mode. In secure-delete mode, ** the dropCell() routine will overwrite the entire cell with zeroes. ** In this case, temporarily copy the cell into the aOvflSpace[] ** buffer. It will be copied out again as soon as the aSpace[] buffer ** is allocated. */ - if( pBt->btsFlags & BTS_SECURE_DELETE ){ + if( pBt->btsFlags & BTS_FAST_SECURE ){ int iOff; + /* If the following if() condition is not true, the db is corrupted. + ** The call to dropCell() below will detect this. */ iOff = SQLITE_PTR_TO_INT(apDiv[i]) - SQLITE_PTR_TO_INT(pParent->aData); - if( (iOff+szNew[i])>(int)pBt->usableSize ){ - rc = SQLITE_CORRUPT_BKPT; - memset(apOld, 0, (i+1)*sizeof(MemPage*)); - goto balance_cleanup; - }else{ + if( (iOff+szNew[i])<=(int)pBt->usableSize ){ memcpy(&aOvflSpace[iOff], apDiv[i], szNew[i]); apDiv[i] = &aOvflSpace[apDiv[i]-pParent->aData]; } @@ -64863,10 +78971,8 @@ static int balance_nonroot( + nMaxCells*sizeof(u16) /* b.szCell */ + pBt->pageSize; /* aSpace1 */ - /* EVIDENCE-OF: R-28375-38319 SQLite will never request a scratch buffer - ** that is more than 6 times the database page size. */ - assert( szScratch<=6*(int)pBt->pageSize ); - b.apCell = sqlite3ScratchMalloc( szScratch ); + assert( szScratch<=7*(int)pBt->pageSize ); + b.apCell = sqlite3StackAllocRaw(0, szScratch ); if( b.apCell==0 ){ rc = SQLITE_NOMEM_BKPT; goto balance_cleanup; @@ -64901,17 +79007,18 @@ static int balance_nonroot( u16 maskPage = pOld->maskPage; u8 *piCell = aData + pOld->cellOffset; u8 *piEnd; + VVA_ONLY( int nCellAtStart = b.nCell; ) /* Verify that all sibling pages are of the same "type" (table-leaf, ** table-interior, index-leaf, or index-interior). */ if( pOld->aData[0]!=apOld[0]->aData[0] ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PAGE(pOld); goto balance_cleanup; } /* Load b.apCell[] with pointers to all cells in pOld. If pOld - ** constains overflow cells, include them in the b.apCell[] array + ** contains overflow cells, include them in the b.apCell[] array ** in the correct spot. ** ** Note that when there are multiple overflow cells, it is always the @@ -64929,6 +79036,10 @@ static int balance_nonroot( */ memset(&b.szCell[b.nCell], 0, sizeof(b.szCell[0])*(limit+pOld->nOverflow)); if( pOld->nOverflow>0 ){ + if( NEVER(limitaiOvfl[0]) ){ + rc = SQLITE_CORRUPT_PAGE(pOld); + goto balance_cleanup; + } limit = pOld->aiOvfl[0]; for(j=0; jnCell+pOld->nOverflow) ); cntOld[i] = b.nCell; if( ileaf ){ assert( leafCorrection==0 ); - assert( pOld->hdrOffset==0 ); + assert( pOld->hdrOffset==0 || CORRUPT_DB ); /* The right pointer of the child page pOld becomes the left ** pointer of the divider cell */ memcpy(b.apCell[b.nCell], &pOld->aData[8], 4); @@ -64988,7 +79100,7 @@ static int balance_nonroot( ** Figure out the number of pages needed to hold all b.nCell cells. ** Store this number in "k". Also compute szNew[] which is the total ** size of all cells on the i-th page and cntNew[] which is the index - ** in b.apCell[] of the cell that divides page i from page i+1. + ** in b.apCell[] of the cell that divides page i from page i+1. ** cntNew[k] should equal b.nCell. ** ** Values computed by this block: @@ -64998,13 +79110,23 @@ static int balance_nonroot( ** cntNew[i]: Index in b.apCell[] and b.szCell[] for the first cell to ** the right of the i-th sibling page. ** usableSpace: Number of bytes of space available on each sibling. - ** + ** */ usableSpace = pBt->usableSize - 12 + leafCorrection; - for(i=0; iaDataEnd; + b.ixNx[k] = cntOld[i]; + if( k && b.ixNx[k]==b.ixNx[k-1] ){ + k--; /* Omit b.ixNx[] entry for child pages with no cells */ + } + if( !leafData ){ + k++; + b.apEnd[k] = pParent->aDataEnd; + b.ixNx[k] = cntOld[i]+1; + } + assert( p->nFree>=0 ); szNew[i] = usableSpace - p->nFree; - if( szNew[i]<0 ){ rc = SQLITE_CORRUPT_BKPT; goto balance_cleanup; } for(j=0; jnOverflow; j++){ szNew[i] += 2 + p->xCellSize(p, p->apOvfl[j]); } @@ -65075,15 +79197,17 @@ static int balance_nonroot( d = r + 1 - leafData; (void)cachedCellSize(&b, d); do{ + int szR, szD; assert( d szLeft-(b.szCell[r]+2)) ){ + && (bBulk || szRight+szD+2 > szLeft-(szR+(i==k-1?0:2)))){ break; } - szRight += b.szCell[d] + 2; - szLeft -= b.szCell[r] + 2; + szRight += szD + 2; + szLeft -= szR + 2; cntNew[i-1] = r; r--; d--; @@ -65096,7 +79220,7 @@ static int balance_nonroot( } } - /* Sanity check: For a non-corrupt database file one of the follwing + /* Sanity check: For a non-corrupt database file one of the following ** must be true: ** (1) We found one or more cells (cntNew[0])>0), or ** (2) pPage is a virtual root page. A virtual root page is when @@ -65104,7 +79228,7 @@ static int balance_nonroot( ** that page. */ assert( cntNew[0]>0 || (pParent->pgno==1 && pParent->nCell==0) || CORRUPT_DB); - TRACE(("BALANCE: old: %d(nc=%d) %d(nc=%d) %d(nc=%d)\n", + TRACE(("BALANCE: old: %u(nc=%u) %u(nc=%u) %u(nc=%u)\n", apOld[0]->pgno, apOld[0]->nCell, nOld>=2 ? apOld[1]->pgno : 0, nOld>=2 ? apOld[1]->nCell : 0, nOld>=3 ? apOld[2]->pgno : 0, nOld>=3 ? apOld[2]->nCell : 0 @@ -65121,6 +79245,11 @@ static int balance_nonroot( apOld[i] = 0; rc = sqlite3PagerWrite(pNew->pDbPage); nNew++; + if( sqlite3PagerPageRefcount(pNew->pDbPage)!=1+(i==(iParentIdx-nxDiv)) + && rc==SQLITE_OK + ){ + rc = SQLITE_CORRUPT_BKPT; + } if( rc ) goto balance_cleanup; }else{ assert( i>0 ); @@ -65132,7 +79261,7 @@ static int balance_nonroot( cntOld[i] = b.nCell; /* Set the pointer-map entry for the new sibling page. */ - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pNew->pgno, PTRMAP_BTREE, pParent->pgno, &rc); if( rc!=SQLITE_OK ){ goto balance_cleanup; @@ -65142,52 +79271,49 @@ static int balance_nonroot( } /* - ** Reassign page numbers so that the new pages are in ascending order. + ** Reassign page numbers so that the new pages are in ascending order. ** This helps to keep entries in the disk file in order so that a scan - ** of the table is closer to a linear scan through the file. That in turn + ** of the table is closer to a linear scan through the file. That in turn ** helps the operating system to deliver pages from the disk more rapidly. ** - ** An O(n^2) insertion sort algorithm is used, but since n is never more - ** than (NB+2) (a small constant), that should not be a problem. + ** An O(N*N) sort algorithm is used, but since N is never more than NB+2 + ** (5), that is not a performance concern. ** - ** When NB==3, this one optimization makes the database about 25% faster + ** When NB==3, this one optimization makes the database about 25% faster ** for large insertions and deletions. */ for(i=0; ipgno; - aPgFlags[i] = apNew[i]->pDbPage->flags; - for(j=0; jpgno; + assert( apNew[i]->pDbPage->flags & PGHDR_WRITEABLE ); + assert( apNew[i]->pDbPage->flags & PGHDR_DIRTY ); } - for(i=0; ipgno < apNew[iB]->pgno ) iB = j; } - pgno = aPgOrder[iBest]; - aPgOrder[iBest] = 0xffffffff; - if( iBest!=i ){ - if( iBest>i ){ - sqlite3PagerRekey(apNew[iBest]->pDbPage, pBt->nPage+iBest+1, 0); - } - sqlite3PagerRekey(apNew[i]->pDbPage, pgno, aPgFlags[iBest]); - apNew[i]->pgno = pgno; + + /* If apNew[i] has a page number that is bigger than any of the + ** subsequence apNew[i] entries, then swap apNew[i] with the subsequent + ** entry that has the smallest page number (which we know to be + ** entry apNew[iB]). + */ + if( iB!=i ){ + Pgno pgnoA = apNew[i]->pgno; + Pgno pgnoB = apNew[iB]->pgno; + Pgno pgnoTemp = (PENDING_BYTE/pBt->pageSize)+1; + u16 fgA = apNew[i]->pDbPage->flags; + u16 fgB = apNew[iB]->pDbPage->flags; + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoTemp, fgB); + sqlite3PagerRekey(apNew[iB]->pDbPage, pgnoA, fgA); + sqlite3PagerRekey(apNew[i]->pDbPage, pgnoB, fgB); + apNew[i]->pgno = pgnoB; + apNew[iB]->pgno = pgnoA; } } - TRACE(("BALANCE: new: %d(%d nc=%d) %d(%d nc=%d) %d(%d nc=%d) " - "%d(%d nc=%d) %d(%d nc=%d)\n", + TRACE(("BALANCE: new: %u(%u nc=%u) %u(%u nc=%u) %u(%u nc=%u) " + "%u(%u nc=%u) %u(%u nc=%u)\n", apNew[0]->pgno, szNew[0], cntNew[0], nNew>=2 ? apNew[1]->pgno : 0, nNew>=2 ? szNew[1] : 0, nNew>=2 ? cntNew[1] - cntNew[0] - !leafData : 0, @@ -65200,17 +79326,19 @@ static int balance_nonroot( )); assert( sqlite3PagerIswriteable(pParent->pDbPage) ); + assert( nNew>=1 && nNew<=ArraySize(apNew) ); + assert( apNew[nNew-1]!=0 ); put4byte(pRight, apNew[nNew-1]->pgno); /* If the sibling pages are not leaves, ensure that the right-child pointer - ** of the right-most new sibling page is set to the value that was + ** of the right-most new sibling page is set to the value that was ** originally in the same field of the right-most old sibling page. */ if( (pageFlags & PTF_LEAF)==0 && nOld!=nNew ){ MemPage *pOld = (nNew>nOld ? apNew : apOld)[nOld-1]; memcpy(&apNew[nNew-1]->aData[8], &pOld->aData[8], 4); } - /* Make any required updates to pointer map entries associated with + /* Make any required updates to pointer map entries associated with ** cells stored on sibling pages following the balance operation. Pointer ** map entries associated with divider cells are set by the insertCell() ** routine. The associated pointer map entries are: @@ -65221,25 +79349,26 @@ static int balance_nonroot( ** b) if the sibling pages are not leaves, the child page associated ** with the cell. ** - ** If the sibling pages are not leaves, then the pointer map entry - ** associated with the right-child of each sibling may also need to be - ** updated. This happens below, after the sibling pages have been + ** If the sibling pages are not leaves, then the pointer map entry + ** associated with the right-child of each sibling may also need to be + ** updated. This happens below, after the sibling pages have been ** populated, not here. */ - if( ISAUTOVACUUM ){ - MemPage *pNew = apNew[0]; - u8 *aOld = pNew->aData; + if( ISAUTOVACUUM(pBt) ){ + MemPage *pOld; + MemPage *pNew = pOld = apNew[0]; int cntOldNext = pNew->nCell + pNew->nOverflow; - int usableSize = pBt->usableSize; int iNew = 0; int iOld = 0; for(i=0; i=0 && iOldnCell + pOld->nOverflow + !leafData; - aOld = pOld->aData; } if( i==cntNew[iNew] ){ pNew = apNew[++iNew]; @@ -65247,20 +79376,20 @@ static int balance_nonroot( } /* Cell pCell is destined for new sibling page pNew. Originally, it - ** was either part of sibling page iOld (possibly an overflow cell), + ** was either part of sibling page iOld (possibly an overflow cell), ** or else the divider cell to the left of sibling page iOld. So, ** if sibling page iOld had the same page number as pNew, and if ** pCell really was a part of sibling page iOld (not a divider or ** overflow cell), we can skip updating the pointer map entries. */ if( iOld>=nNew || pNew->pgno!=aPgno[iOld] - || !SQLITE_WITHIN(pCell,aOld,&aOld[usableSize]) + || !SQLITE_WITHIN(pCell,pOld->aData,pOld->aDataEnd) ){ if( !leafCorrection ){ ptrmapPut(pBt, get4byte(pCell), PTRMAP_BTREE, pNew->pgno, &rc); } if( cachedCellSize(&b,i)>pNew->minLocal ){ - ptrmapPutOvflPtr(pNew, pCell, &rc); + ptrmapPutOvflPtr(pNew, pOld, pCell, &rc); } if( rc ) goto balance_cleanup; } @@ -65272,6 +79401,7 @@ static int balance_nonroot( u8 *pCell; u8 *pTemp; int sz; + u8 *pSrcEnd; MemPage *pNew = apNew[i]; j = cntNew[i]; @@ -65283,9 +79413,9 @@ static int balance_nonroot( if( !pNew->leaf ){ memcpy(&pNew->aData[8], pCell, 4); }else if( leafData ){ - /* If the tree is a leaf-data tree, and the siblings are leaves, - ** then there is no divider cell in b.apCell[]. Instead, the divider - ** cell consists of the integer key for the right-most cell of + /* If the tree is a leaf-data tree, and the siblings are leaves, + ** then there is no divider cell in b.apCell[]. Instead, the divider + ** cell consists of the integer key for the right-most cell of ** the sibling-page assembled above only. */ CellInfo info; @@ -65298,9 +79428,9 @@ static int balance_nonroot( pCell -= 4; /* Obscure case for non-leaf-data trees: If the cell at pCell was ** previously stored on a leaf node, and its reported size was 4 - ** bytes, then it may actually be smaller than this + ** bytes, then it may actually be smaller than this ** (see btreeParseCellPtr(), 4 bytes is the minimum size of - ** any cell). But it is important to pass the correct size to + ** any cell). But it is important to pass the correct size to ** insertCell(), so reparse the cell now. ** ** This can only happen for b-trees used to evaluate "IN (SELECT ...)" @@ -65315,7 +79445,14 @@ static int balance_nonroot( iOvflSpace += sz; assert( sz<=pBt->maxLocal+23 ); assert( iOvflSpace <= (int)pBt->pageSize ); - insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno, &rc); + assert( b.ixNx[NB*2-1]>j ); + for(k=0; b.ixNx[k]<=j; k++){} + pSrcEnd = b.apEnd[k]; + if( SQLITE_OVERFLOW(pSrcEnd, pCell, pCell+sz) ){ + rc = SQLITE_CORRUPT_BKPT; + goto balance_cleanup; + } + rc = insertCell(pParent, nxDiv+i, pCell, sz, pTemp, pNew->pgno); if( rc!=SQLITE_OK ) goto balance_cleanup; assert( sqlite3PagerIswriteable(pParent->pDbPage) ); } @@ -65345,6 +79482,8 @@ static int balance_nonroot( for(i=1-nNew; i=0 && iPg=1 || i>=0 ); + assert( iPg=0 /* On the upwards pass, or... */ || cntOld[iPg-1]>=cntNew[iPg-1] /* Condition (1) is true */ @@ -65392,8 +79531,8 @@ static int balance_nonroot( ** b-tree structure by one. This is described as the "balance-shallower" ** sub-algorithm in some documentation. ** - ** If this is an auto-vacuum database, the call to copyNodeContent() - ** sets all pointer-map entries corresponding to database image pages + ** If this is an auto-vacuum database, the call to copyNodeContent() + ** sets all pointer-map entries corresponding to database image pages ** for which the pointer is stored within the content being copied. ** ** It is critical that the child page be defragmented before being @@ -65402,15 +79541,16 @@ static int balance_nonroot( ** free space needs to be up front. */ assert( nNew==1 || CORRUPT_DB ); - rc = defragmentPage(apNew[0]); + rc = defragmentPage(apNew[0], -1); testcase( rc!=SQLITE_OK ); - assert( apNew[0]->nFree == - (get2byte(&apNew[0]->aData[5])-apNew[0]->cellOffset-apNew[0]->nCell*2) + assert( apNew[0]->nFree == + (get2byteNotZero(&apNew[0]->aData[5]) - apNew[0]->cellOffset + - apNew[0]->nCell*2) || rc!=SQLITE_OK ); copyNodeContent(apNew[0], pParent, &rc); freePage(apNew[0], &rc); - }else if( ISAUTOVACUUM && !leafCorrection ){ + }else if( ISAUTOVACUUM(pBt) && !leafCorrection ){ /* Fix the pointer map entries associated with the right-child of each ** sibling page. All other pointer map entries have already been taken ** care of. */ @@ -65421,7 +79561,7 @@ static int balance_nonroot( } assert( pParent->isInit ); - TRACE(("BALANCE: finished: old=%d new=%d cells=%d\n", + TRACE(("BALANCE: finished: old=%u new=%u cells=%u\n", nOld, nNew, b.nCell)); /* Free any old pages that were not reused as new pages. @@ -65431,9 +79571,9 @@ static int balance_nonroot( } #if 0 - if( ISAUTOVACUUM && rc==SQLITE_OK && apNew[0]->isInit ){ + if( ISAUTOVACUUM(pBt) && rc==SQLITE_OK && apNew[0]->isInit ){ /* The ptrmapCheckPages() contains assert() statements that verify that - ** all pointer map pages are set correctly. This is helpful while + ** all pointer map pages are set correctly. This is helpful while ** debugging. This is usually disabled because a corrupt database may ** cause an assert() statement to fail. */ ptrmapCheckPages(apNew, nNew); @@ -65445,7 +79585,7 @@ static int balance_nonroot( ** Cleanup before returning. */ balance_cleanup: - sqlite3ScratchFree(b.apCell); + sqlite3StackFree(0, b.apCell); for(i=0; inOverflow>0 ); assert( sqlite3_mutex_held(pBt->mutex) ); - /* Make pRoot, the root page of the b-tree, writable. Allocate a new + /* Make pRoot, the root page of the b-tree, writable. Allocate a new ** page that will become the new right-child of pPage. Copy the contents ** of the node stored on pRoot into the new child page. */ @@ -65493,7 +79633,7 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ if( rc==SQLITE_OK ){ rc = allocateBtreePage(pBt,&pChild,&pgnoChild,pRoot->pgno,0); copyNodeContent(pRoot, pChild, &rc); - if( ISAUTOVACUUM ){ + if( ISAUTOVACUUM(pBt) ){ ptrmapPut(pBt, pgnoChild, PTRMAP_BTREE, pRoot->pgno, &rc); } } @@ -65504,9 +79644,9 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ } assert( sqlite3PagerIswriteable(pChild->pDbPage) ); assert( sqlite3PagerIswriteable(pRoot->pDbPage) ); - assert( pChild->nCell==pRoot->nCell ); + assert( pChild->nCell==pRoot->nCell || CORRUPT_DB ); - TRACE(("BALANCE: copy root %d into %d\n", pRoot->pgno, pChild->pgno)); + TRACE(("BALANCE: copy root %u into %u\n", pRoot->pgno, pChild->pgno)); /* Copy the overflow cells from pRoot to pChild */ memcpy(pChild->aiOvfl, pRoot->aiOvfl, @@ -65523,10 +79663,34 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ return SQLITE_OK; } +/* +** Return SQLITE_CORRUPT if any cursor other than pCur is currently valid +** on the same B-tree as pCur. +** +** This can occur if a database is corrupt with two or more SQL tables +** pointing to the same b-tree. If an insert occurs on one SQL table +** and causes a BEFORE TRIGGER to do a secondary insert on the other SQL +** table linked to the same b-tree. If the secondary insert causes a +** rebalance, that can change content out from under the cursor on the +** first SQL table, violating invariants on the first insert. +*/ +static int anotherValidCursor(BtCursor *pCur){ + BtCursor *pOther; + for(pOther=pCur->pBt->pCursor; pOther; pOther=pOther->pNext){ + if( pOther!=pCur + && pOther->eState==CURSOR_VALID + && pOther->pPage==pCur->pPage + ){ + return SQLITE_CORRUPT_PAGE(pCur->pPage); + } + } + return SQLITE_OK; +} + /* ** The page that pCur currently points to has just been modified in ** some way. This function figures out if this modification means the -** tree needs to be balanced, and if so calls the appropriate balancing +** tree needs to be balanced, and if so calls the appropriate balancing ** routine. Balancing routines are: ** ** balance_quick() @@ -65535,7 +79699,6 @@ static int balance_deeper(MemPage *pRoot, MemPage **ppChild){ */ static int balance(BtCursor *pCur){ int rc = SQLITE_OK; - const int nMin = pCur->pBt->usableSize * 2 / 3; u8 aBalanceQuickSpace[13]; u8 *pFree = 0; @@ -65543,35 +79706,50 @@ static int balance(BtCursor *pCur){ VVA_ONLY( int balance_deeper_called = 0 ); do { - int iPage = pCur->iPage; - MemPage *pPage = pCur->apPage[iPage]; + int iPage; + MemPage *pPage = pCur->pPage; - if( iPage==0 ){ - if( pPage->nOverflow ){ + if( NEVER(pPage->nFree<0) && btreeComputeFreeSpace(pPage) ) break; + if( pPage->nOverflow==0 && pPage->nFree*3<=(int)pCur->pBt->usableSize*2 ){ + /* No rebalance required as long as: + ** (1) There are no overflow cells + ** (2) The amount of free space on the page is less than 2/3rds of + ** the total usable space on the page. */ + break; + }else if( (iPage = pCur->iPage)==0 ){ + if( pPage->nOverflow && (rc = anotherValidCursor(pCur))==SQLITE_OK ){ /* The root page of the b-tree is overfull. In this case call the ** balance_deeper() function to create a new child for the root-page ** and copy the current contents of the root-page to it. The ** next iteration of the do-loop will balance the child page. - */ + */ assert( balance_deeper_called==0 ); VVA_ONLY( balance_deeper_called++ ); rc = balance_deeper(pPage, &pCur->apPage[1]); if( rc==SQLITE_OK ){ pCur->iPage = 1; + pCur->ix = 0; pCur->aiIdx[0] = 0; - pCur->aiIdx[1] = 0; - assert( pCur->apPage[1]->nOverflow ); + pCur->apPage[0] = pPage; + pCur->pPage = pCur->apPage[1]; + assert( pCur->pPage->nOverflow ); } }else{ break; } - }else if( pPage->nOverflow==0 && pPage->nFree<=nMin ){ - break; + }else if( sqlite3PagerPageRefcount(pPage->pDbPage)>1 ){ + /* The page being written is not a root page, and there is currently + ** more than one reference to it. This only happens if the page is one + ** of its own ancestor pages. Corruption. */ + rc = SQLITE_CORRUPT_PAGE(pPage); }else{ MemPage * const pParent = pCur->apPage[iPage-1]; int const iIdx = pCur->aiIdx[iPage-1]; rc = sqlite3PagerWrite(pParent->pDbPage); + if( rc==SQLITE_OK && pParent->nFree<0 ){ + rc = btreeComputeFreeSpace(pParent); + } if( rc==SQLITE_OK ){ #ifndef SQLITE_OMIT_QUICKBALANCE if( pPage->intKeyLeaf @@ -65583,17 +79761,17 @@ static int balance(BtCursor *pCur){ /* Call balance_quick() to create a new sibling of pPage on which ** to store the overflow cell. balance_quick() inserts a new cell ** into pParent, which may cause pParent overflow. If this - ** happens, the next iteration of the do-loop will balance pParent + ** happens, the next iteration of the do-loop will balance pParent ** use either balance_nonroot() or balance_deeper(). Until this ** happens, the overflow cell is stored in the aBalanceQuickSpace[] - ** buffer. + ** buffer. ** ** The purpose of the following assert() is to check that only a ** single call to balance_quick() is made for each call to this ** function. If this were not verified, a subtle bug involving reuse ** of the aBalanceQuickSpace[] might sneak in. */ - assert( balance_quick_called==0 ); + assert( balance_quick_called==0 ); VVA_ONLY( balance_quick_called++ ); rc = balance_quick(pParent, pPage, aBalanceQuickSpace); }else @@ -65604,15 +79782,15 @@ static int balance(BtCursor *pCur){ ** modifying the contents of pParent, which may cause pParent to ** become overfull or underfull. The next iteration of the do-loop ** will balance the parent page to correct this. - ** + ** ** If the parent page becomes overfull, the overflow cell or cells - ** are stored in the pSpace buffer allocated immediately below. + ** are stored in the pSpace buffer allocated immediately below. ** A subsequent iteration of the do-loop will deal with this by ** calling balance_nonroot() (balance_deeper() may be called first, ** but it doesn't deal with overflow cells - just moves them to a - ** different page). Once this subsequent call to balance_nonroot() + ** different page). Once this subsequent call to balance_nonroot() ** has completed, it is safe to release the pSpace buffer used by - ** the previous call, as the overflow cell data will have been + ** the previous call, as the overflow cell data will have been ** copied either into the body of a database page or into the new ** pSpace buffer passed to the latter call to balance_nonroot(). */ @@ -65620,9 +79798,9 @@ static int balance(BtCursor *pCur){ rc = balance_nonroot(pParent, iIdx, pSpace, iPage==1, pCur->hints&BTREE_BULKLOAD); if( pFree ){ - /* If pFree is not NULL, it points to the pSpace buffer used + /* If pFree is not NULL, it points to the pSpace buffer used ** by a previous call to balance_nonroot(). Its contents are - ** now stored either on real database pages or within the + ** now stored either on real database pages or within the ** new pSpace buffer, so it may be safely freed here. */ sqlite3PageFree(pFree); } @@ -65640,6 +79818,7 @@ static int balance(BtCursor *pCur){ releasePage(pPage); pCur->iPage--; assert( pCur->iPage>=0 ); + pCur->pPage = pCur->apPage[pCur->iPage]; } }while( rc==SQLITE_OK ); @@ -65649,36 +79828,159 @@ static int balance(BtCursor *pCur){ return rc; } +/* Overwrite content from pX into pDest. Only do the write if the +** content is different from what is already there. +*/ +static int btreeOverwriteContent( + MemPage *pPage, /* MemPage on which writing will occur */ + u8 *pDest, /* Pointer to the place to start writing */ + const BtreePayload *pX, /* Source of data to write */ + int iOffset, /* Offset of first byte to write */ + int iAmt /* Number of bytes to be written */ +){ + int nData = pX->nData - iOffset; + if( nData<=0 ){ + /* Overwriting with zeros */ + int i; + for(i=0; ipDbPage); + if( rc ) return rc; + memset(pDest + i, 0, iAmt - i); + } + }else{ + if( nDatapData) + iOffset, iAmt)!=0 ){ + int rc = sqlite3PagerWrite(pPage->pDbPage); + if( rc ) return rc; + /* In a corrupt database, it is possible for the source and destination + ** buffers to overlap. This is harmless since the database is already + ** corrupt but it does cause valgrind and ASAN warnings. So use + ** memmove(). */ + memmove(pDest, ((u8*)pX->pData) + iOffset, iAmt); + } + } + return SQLITE_OK; +} /* -** Insert a new record into the BTree. The key is given by (pKey,nKey) -** and the data is given by (pData,nData). The cursor is used only to -** define what table the record should be inserted into. The cursor -** is left pointing at a random location. +** Overwrite the cell that cursor pCur is pointing to with fresh content +** contained in pX. In this variant, pCur is pointing to an overflow +** cell. +*/ +static SQLITE_NOINLINE int btreeOverwriteOverflowCell( + BtCursor *pCur, /* Cursor pointing to cell to overwrite */ + const BtreePayload *pX /* Content to write into the cell */ +){ + int iOffset; /* Next byte of pX->pData to write */ + int nTotal = pX->nData + pX->nZero; /* Total bytes of to write */ + int rc; /* Return code */ + MemPage *pPage = pCur->pPage; /* Page being written */ + BtShared *pBt; /* Btree */ + Pgno ovflPgno; /* Next overflow page to write */ + u32 ovflPageSize; /* Size to write on overflow page */ + + assert( pCur->info.nLocalinfo.pPayload, pX, + 0, pCur->info.nLocal); + if( rc ) return rc; + + /* Now overwrite the overflow pages */ + iOffset = pCur->info.nLocal; + assert( nTotal>=0 ); + assert( iOffset>=0 ); + ovflPgno = get4byte(pCur->info.pPayload + iOffset); + pBt = pPage->pBt; + ovflPageSize = pBt->usableSize - 4; + do{ + rc = btreeGetPage(pBt, ovflPgno, &pPage, 0); + if( rc ) return rc; + if( sqlite3PagerPageRefcount(pPage->pDbPage)!=1 || pPage->isInit ){ + rc = SQLITE_CORRUPT_PAGE(pPage); + }else{ + if( iOffset+ovflPageSize<(u32)nTotal ){ + ovflPgno = get4byte(pPage->aData); + }else{ + ovflPageSize = nTotal - iOffset; + } + rc = btreeOverwriteContent(pPage, pPage->aData+4, pX, + iOffset, ovflPageSize); + } + sqlite3PagerUnref(pPage->pDbPage); + if( rc ) return rc; + iOffset += ovflPageSize; + }while( iOffsetnData + pX->nZero; /* Total bytes of to write */ + MemPage *pPage = pCur->pPage; /* Page being written */ + + if( pCur->info.pPayload + pCur->info.nLocal > pPage->aDataEnd + || pCur->info.pPayload < pPage->aData + pPage->cellOffset + ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( pCur->info.nLocal==nTotal ){ + /* The entire cell is local */ + return btreeOverwriteContent(pPage, pCur->info.pPayload, pX, + 0, pCur->info.nLocal); + }else{ + /* The cell contains overflow content */ + return btreeOverwriteOverflowCell(pCur, pX); + } +} + + +/* +** Insert a new record into the BTree. The content of the new record +** is described by the pX object. The pCur cursor is used only to +** define what table the record should be inserted into, and is left +** pointing at a random location. ** -** For an INTKEY table, only the nKey value of the key is used. pKey is -** ignored. For a ZERODATA table, the pData and nData are both ignored. +** For a table btree (used for rowid tables), only the pX.nKey value of +** the key is used. The pX.pKey value must be NULL. The pX.nKey is the +** rowid or INTEGER PRIMARY KEY of the row. The pX.nData,pData,nZero fields +** hold the content of the row. +** +** For an index btree (used for indexes and WITHOUT ROWID tables), the +** key is an arbitrary byte sequence stored in pX.pKey,nKey. The +** pX.pData,nData,nZero fields must be zero. ** ** If the seekResult parameter is non-zero, then a successful call to -** MovetoUnpacked() to seek cursor pCur to (pKey, nKey) has already -** been performed. seekResult is the search result returned (a negative -** number if pCur points at an entry that is smaller than (pKey, nKey), or -** a positive value if pCur points at an entry that is larger than -** (pKey, nKey)). +** sqlite3BtreeIndexMoveto() to seek cursor pCur to (pKey,nKey) has already +** been performed. In other words, if seekResult!=0 then the cursor +** is currently pointing to a cell that will be adjacent to the cell +** to be inserted. If seekResult<0 then pCur points to a cell that is +** smaller then (pKey,nKey). If seekResult>0 then pCur points to a cell +** that is larger than (pKey,nKey). ** -** If the seekResult parameter is non-zero, then the caller guarantees that -** cursor pCur is pointing at the existing copy of a row that is to be -** overwritten. If the seekResult parameter is 0, then cursor pCur may -** point to any entry or to no entry at all and so this function has to seek -** the cursor before the new key can be inserted. +** If seekResult==0, that means pCur is pointing at some unknown location. +** In that case, this routine must seek the cursor to the correct insertion +** point for (pKey,nKey) before doing the insertion. For index btrees, +** if pX->nMem is non-zero, then pX->aMem contains pointers to the unpacked +** key values and pX->aMem can be used instead of pX->pKey to avoid having +** to decode the key. */ SQLITE_PRIVATE int sqlite3BtreeInsert( BtCursor *pCur, /* Insert data into the table of this cursor */ - const void *pKey, i64 nKey, /* The key of the new record */ - const void *pData, int nData, /* The data of the new record */ - int nZero, /* Number of extra 0 bytes to append to data */ - int appendBias, /* True if this is likely an append */ - int seekResult /* Result of prior MovetoUnpacked() call */ + const BtreePayload *pX, /* Content of the row to be inserted */ + int flags, /* True if this is likely an append */ + int seekResult /* Result of prior IndexMoveto() call */ ){ int rc; int loc = seekResult; /* -1: before desired location +1: after */ @@ -65686,19 +79988,50 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( int idx; MemPage *pPage; Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; unsigned char *oldCell; unsigned char *newCell = 0; - if( pCur->eState==CURSOR_FAULT ){ - assert( pCur->skipNext!=SQLITE_OK ); - return pCur->skipNext; + assert( (flags & (BTREE_SAVEPOSITION|BTREE_APPEND|BTREE_PREFORMAT))==flags ); + assert( (flags & BTREE_PREFORMAT)==0 || seekResult || pCur->pKeyInfo==0 ); + + /* Save the positions of any other cursors open on this table. + ** + ** In some cases, the call to btreeMoveto() below is a no-op. For + ** example, when inserting data into a table with auto-generated integer + ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the + ** integer key to use. It then calls this function to actually insert the + ** data into the intkey B-Tree. In this case btreeMoveto() recognizes + ** that the cursor is already where it needs to be and returns without + ** doing any work. To avoid thwarting these optimizations, it is important + ** not to clear the cursor here. + */ + if( pCur->curFlags & BTCF_Multiple ){ + rc = saveAllCursors(p->pBt, pCur->pgnoRoot, pCur); + if( rc ) return rc; + if( loc && pCur->iPage<0 ){ + /* This can only happen if the schema is corrupt such that there is more + ** than one table or index with the same root page as used by the cursor. + ** Which can only happen if the SQLITE_NoSchemaError flag was set when + ** the schema was loaded. This cannot be asserted though, as a user might + ** set the flag, load the schema, and then unset the flag. */ + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); + } + } + + /* Ensure that the cursor is not in the CURSOR_FAULT state and that it + ** points to a valid cell. + */ + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + testcase( pCur->eState==CURSOR_REQUIRESEEK ); + testcase( pCur->eState==CURSOR_FAULT ); + rc = moveToRoot(pCur); + if( rc && rc!=SQLITE_EMPTY ) return rc; } assert( cursorOwnsBtShared(pCur) ); assert( (pCur->curFlags & BTCF_WriteFlag)!=0 - && pBt->inTransaction==TRANS_WRITE - && (pBt->btsFlags & BTS_READ_ONLY)==0 ); + && p->pBt->inTransaction==TRANS_WRITE + && (p->pBt->btsFlags & BTS_READ_ONLY)==0 ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); /* Assert that the caller has been consistent. If this cursor was opened @@ -65706,64 +80039,148 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** keys with no associated data. If the cursor was opened expecting an ** intkey table, the caller should be inserting integer keys with a ** blob of associated data. */ - assert( (pKey==0)==(pCur->pKeyInfo==0) ); - - /* Save the positions of any other cursors open on this table. - ** - ** In some cases, the call to btreeMoveto() below is a no-op. For - ** example, when inserting data into a table with auto-generated integer - ** keys, the VDBE layer invokes sqlite3BtreeLast() to figure out the - ** integer key to use. It then calls this function to actually insert the - ** data into the intkey B-Tree. In this case btreeMoveto() recognizes - ** that the cursor is already where it needs to be and returns without - ** doing any work. To avoid thwarting these optimizations, it is important - ** not to clear the cursor here. - */ - if( pCur->curFlags & BTCF_Multiple ){ - rc = saveAllCursors(pBt, pCur->pgnoRoot, pCur); - if( rc ) return rc; - } + assert( (flags & BTREE_PREFORMAT) || (pX->pKey==0)==(pCur->pKeyInfo==0) ); if( pCur->pKeyInfo==0 ){ - assert( pKey==0 ); - /* If this is an insert into a table b-tree, invalidate any incrblob + assert( pX->pKey==0 ); + /* If this is an insert into a table b-tree, invalidate any incrblob ** cursors open on the row being replaced */ - invalidateIncrblobCursors(p, nKey, 0); + if( p->hasIncrblobCur ){ + invalidateIncrblobCursors(p, pCur->pgnoRoot, pX->nKey, 0); + } - /* If the cursor is currently on the last row and we are appending a - ** new row onto the end, set the "loc" to avoid an unnecessary - ** btreeMoveto() call */ - if( (pCur->curFlags&BTCF_ValidNKey)!=0 && nKey>0 - && pCur->info.nKey==nKey-1 ){ - loc = -1; + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. + */ +#ifdef SQLITE_DEBUG + if( flags & BTREE_SAVEPOSITION ){ + assert( pCur->curFlags & BTCF_ValidNKey ); + assert( pX->nKey==pCur->info.nKey ); + assert( loc==0 ); + } +#endif + + /* On the other hand, BTREE_SAVEPOSITION==0 does not imply + ** that the cursor is not pointing to a row to be overwritten. + ** So do a complete check. + */ + if( (pCur->curFlags&BTCF_ValidNKey)!=0 && pX->nKey==pCur->info.nKey ){ + /* The cursor is pointing to the entry that is to be + ** overwritten */ + assert( pX->nData>=0 && pX->nZero>=0 ); + if( pCur->info.nSize!=0 + && pCur->info.nPayload==(u32)pX->nData+pX->nZero + ){ + /* New entry is the same size as the old. Do an overwrite */ + return btreeOverwriteCell(pCur, pX); + } + assert( loc==0 ); }else if( loc==0 ){ - rc = sqlite3BtreeMovetoUnpacked(pCur, 0, nKey, appendBias, &loc); + /* The cursor is *not* pointing to the cell to be overwritten, nor + ** to an adjacent cell. Move the cursor so that it is pointing either + ** to the cell to be overwritten or an adjacent cell. + */ + rc = sqlite3BtreeTableMoveto(pCur, pX->nKey, + (flags & BTREE_APPEND)!=0, &loc); if( rc ) return rc; } - }else if( loc==0 ){ - rc = btreeMoveto(pCur, pKey, nKey, appendBias, &loc); + }else{ + /* This is an index or a WITHOUT ROWID table */ + + /* If BTREE_SAVEPOSITION is set, the cursor must already be pointing + ** to a row with the same key as the new entry being inserted. + */ + assert( (flags & BTREE_SAVEPOSITION)==0 || loc==0 ); + + /* If the cursor is not already pointing either to the cell to be + ** overwritten, or if a new cell is being inserted, if the cursor is + ** not pointing to an immediately adjacent cell, then move the cursor + ** so that it does. + */ + if( loc==0 && (flags & BTREE_SAVEPOSITION)==0 ){ + if( pX->nMem ){ + UnpackedRecord r; + r.pKeyInfo = pCur->pKeyInfo; + r.aMem = pX->aMem; + r.nField = pX->nMem; + r.default_rc = 0; + r.eqSeen = 0; + rc = sqlite3BtreeIndexMoveto(pCur, &r, &loc); + }else{ + rc = btreeMoveto(pCur, pX->pKey, pX->nKey, + (flags & BTREE_APPEND)!=0, &loc); + } + if( rc ) return rc; + } + + /* If the cursor is currently pointing to an entry to be overwritten + ** and the new content is the same as as the old, then use the + ** overwrite optimization. + */ + if( loc==0 ){ + getCellInfo(pCur); + if( pCur->info.nKey==pX->nKey ){ + BtreePayload x2; + x2.pData = pX->pKey; + x2.nData = pX->nKey; + x2.nZero = 0; + return btreeOverwriteCell(pCur, &x2); + } + } + } + assert( pCur->eState==CURSOR_VALID + || (pCur->eState==CURSOR_INVALID && loc) || CORRUPT_DB ); + + pPage = pCur->pPage; + assert( pPage->intKey || pX->nKey>=0 || (flags & BTREE_PREFORMAT) ); + assert( pPage->leaf || !pPage->intKey ); + if( pPage->nFree<0 ){ + if( NEVER(pCur->eState>CURSOR_INVALID) ){ + /* ^^^^^--- due to the moveToRoot() call above */ + rc = SQLITE_CORRUPT_PAGE(pPage); + }else{ + rc = btreeComputeFreeSpace(pPage); + } if( rc ) return rc; } - assert( pCur->eState==CURSOR_VALID || (pCur->eState==CURSOR_INVALID && loc) ); - pPage = pCur->apPage[pCur->iPage]; - assert( pPage->intKey || nKey>=0 ); - assert( pPage->leaf || !pPage->intKey ); - - TRACE(("INSERT: table=%d nkey=%lld ndata=%d page=%d %s\n", - pCur->pgnoRoot, nKey, nData, pPage->pgno, + TRACE(("INSERT: table=%u nkey=%lld ndata=%u page=%u %s\n", + pCur->pgnoRoot, pX->nKey, pX->nData, pPage->pgno, loc==0 ? "overwrite" : "new entry")); - assert( pPage->isInit ); - newCell = pBt->pTmpSpace; + assert( pPage->isInit || CORRUPT_DB ); + newCell = p->pBt->pTmpSpace; assert( newCell!=0 ); - rc = fillInCell(pPage, newCell, pKey, nKey, pData, nData, nZero, &szNew); - if( rc ) goto end_insert; + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); + if( flags & BTREE_PREFORMAT ){ + rc = SQLITE_OK; + szNew = p->pBt->nPreformatSize; + if( szNew<4 ){ + szNew = 4; + newCell[3] = 0; + } + if( ISAUTOVACUUM(p->pBt) && szNew>pPage->maxLocal ){ + CellInfo info; + pPage->xParseCell(pPage, newCell, &info); + if( info.nPayload!=info.nLocal ){ + Pgno ovfl = get4byte(&newCell[szNew-4]); + ptrmapPut(p->pBt, ovfl, PTRMAP_OVERFLOW1, pPage->pgno, &rc); + if( NEVER(rc) ) goto end_insert; + } + } + }else{ + rc = fillInCell(pPage, newCell, pX, &szNew); + if( rc ) goto end_insert; + } assert( szNew==pPage->xCellSize(pPage, newCell) ); - assert( szNew <= MX_CELL_SIZE(pBt) ); - idx = pCur->aiIdx[pCur->iPage]; + assert( szNew <= MX_CELL_SIZE(p->pBt) ); + idx = pCur->ix; + pCur->info.nSize = 0; if( loc==0 ){ - u16 szOld; - assert( idxnCell ); + CellInfo info; + assert( idx>=0 ); + if( idx>=pPage->nCell ){ + return SQLITE_CORRUPT_PAGE(pPage); + } rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ){ goto end_insert; @@ -65772,19 +80189,45 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( if( !pPage->leaf ){ memcpy(newCell, oldCell, 4); } - rc = clearCell(pPage, oldCell, &szOld); - dropCell(pPage, idx, szOld, &rc); + BTREE_CLEAR_CELL(rc, pPage, oldCell, info); + testcase( pCur->curFlags & BTCF_ValidOvfl ); + invalidateOverflowCache(pCur); + if( info.nSize==szNew && info.nLocal==info.nPayload + && (!ISAUTOVACUUM(p->pBt) || szNewminLocal) + ){ + /* Overwrite the old cell with the new if they are the same size. + ** We could also try to do this if the old cell is smaller, then add + ** the leftover space to the free list. But experiments show that + ** doing that is no faster then skipping this optimization and just + ** calling dropCell() and insertCell(). + ** + ** This optimization cannot be used on an autovacuum database if the + ** new entry uses overflow pages, as the insertCell() call below is + ** necessary to add the PTRMAP_OVERFLOW1 pointer-map entry. */ + assert( rc==SQLITE_OK ); /* clearCell never fails when nLocal==nPayload */ + if( oldCell < pPage->aData+pPage->hdrOffset+10 ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( oldCell+szNew > pPage->aDataEnd ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + memcpy(oldCell, newCell, szNew); + return SQLITE_OK; + } + dropCell(pPage, idx, info.nSize, &rc); if( rc ) goto end_insert; }else if( loc<0 && pPage->nCell>0 ){ assert( pPage->leaf ); - idx = ++pCur->aiIdx[pCur->iPage]; + idx = ++pCur->ix; + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); }else{ assert( pPage->leaf ); } - insertCell(pPage, idx, newCell, szNew, 0, 0, &rc); + rc = insertCellFast(pPage, idx, newCell, szNew); + assert( pPage->nOverflow==0 || rc==SQLITE_OK ); assert( rc!=SQLITE_OK || pPage->nCell>0 || pPage->nOverflow>0 ); - /* If no error has occurred and pPage has an overflow cell, call balance() + /* If no error has occurred and pPage has an overflow cell, call balance() ** to redistribute the cells within the tree. Since balance() may move ** the cursor, zero the BtCursor.info.nSize and BTCF_ValidNKey ** variables. @@ -65804,26 +80247,152 @@ SQLITE_PRIVATE int sqlite3BtreeInsert( ** larger than the largest existing key, it is possible to insert the ** row without seeking the cursor. This can be a big performance boost. */ - pCur->info.nSize = 0; - if( rc==SQLITE_OK && pPage->nOverflow ){ - pCur->curFlags &= ~(BTCF_ValidNKey); + if( pPage->nOverflow ){ + assert( rc==SQLITE_OK ); + pCur->curFlags &= ~(BTCF_ValidNKey|BTCF_ValidOvfl); rc = balance(pCur); /* Must make sure nOverflow is reset to zero even if the balance() - ** fails. Internal data structure corruption will result otherwise. + ** fails. Internal data structure corruption will result otherwise. ** Also, set the cursor state to invalid. This stops saveCursorPosition() ** from trying to save the current position of the cursor. */ - pCur->apPage[pCur->iPage]->nOverflow = 0; + pCur->pPage->nOverflow = 0; pCur->eState = CURSOR_INVALID; + if( (flags & BTREE_SAVEPOSITION) && rc==SQLITE_OK ){ + btreeReleaseAllCursorPages(pCur); + if( pCur->pKeyInfo ){ + assert( pCur->pKey==0 ); + pCur->pKey = sqlite3Malloc( pX->nKey ); + if( pCur->pKey==0 ){ + rc = SQLITE_NOMEM; + }else{ + memcpy(pCur->pKey, pX->pKey, pX->nKey); + } + } + pCur->eState = CURSOR_REQUIRESEEK; + pCur->nKey = pX->nKey; + } } - assert( pCur->apPage[pCur->iPage]->nOverflow==0 ); + assert( pCur->iPage<0 || pCur->pPage->nOverflow==0 ); end_insert: return rc; } /* -** Delete the entry that the cursor is pointing to. +** This function is used as part of copying the current row from cursor +** pSrc into cursor pDest. If the cursors are open on intkey tables, then +** parameter iKey is used as the rowid value when the record is copied +** into pDest. Otherwise, the record is copied verbatim. +** +** This function does not actually write the new value to cursor pDest. +** Instead, it creates and populates any required overflow pages and +** writes the data for the new cell into the BtShared.pTmpSpace buffer +** for the destination database. The size of the cell, in bytes, is left +** in BtShared.nPreformatSize. The caller completes the insertion by +** calling sqlite3BtreeInsert() with the BTREE_PREFORMAT flag specified. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +*/ +SQLITE_PRIVATE int sqlite3BtreeTransferRow(BtCursor *pDest, BtCursor *pSrc, i64 iKey){ + BtShared *pBt = pDest->pBt; + u8 *aOut = pBt->pTmpSpace; /* Pointer to next output buffer */ + const u8 *aIn; /* Pointer to next input buffer */ + u32 nIn; /* Size of input buffer aIn[] */ + u32 nRem; /* Bytes of data still to copy */ + + getCellInfo(pSrc); + if( pSrc->info.nPayload<0x80 ){ + *(aOut++) = pSrc->info.nPayload; + }else{ + aOut += sqlite3PutVarint(aOut, pSrc->info.nPayload); + } + if( pDest->pKeyInfo==0 ) aOut += putVarint(aOut, iKey); + nIn = pSrc->info.nLocal; + aIn = pSrc->info.pPayload; + if( aIn+nIn>pSrc->pPage->aDataEnd ){ + return SQLITE_CORRUPT_PAGE(pSrc->pPage); + } + nRem = pSrc->info.nPayload; + if( nIn==nRem && nInpPage->maxLocal ){ + memcpy(aOut, aIn, nIn); + pBt->nPreformatSize = nIn + (aOut - pBt->pTmpSpace); + return SQLITE_OK; + }else{ + int rc = SQLITE_OK; + Pager *pSrcPager = pSrc->pBt->pPager; + u8 *pPgnoOut = 0; + Pgno ovflIn = 0; + DbPage *pPageIn = 0; + MemPage *pPageOut = 0; + u32 nOut; /* Size of output buffer aOut[] */ + + nOut = btreePayloadToLocal(pDest->pPage, pSrc->info.nPayload); + pBt->nPreformatSize = nOut + (aOut - pBt->pTmpSpace); + if( nOutinfo.nPayload ){ + pPgnoOut = &aOut[nOut]; + pBt->nPreformatSize += 4; + } + + if( nRem>nIn ){ + if( aIn+nIn+4>pSrc->pPage->aDataEnd ){ + return SQLITE_CORRUPT_PAGE(pSrc->pPage); + } + ovflIn = get4byte(&pSrc->info.pPayload[nIn]); + } + + do { + nRem -= nOut; + do{ + assert( nOut>0 ); + if( nIn>0 ){ + int nCopy = MIN(nOut, nIn); + memcpy(aOut, aIn, nCopy); + nOut -= nCopy; + nIn -= nCopy; + aOut += nCopy; + aIn += nCopy; + } + if( nOut>0 ){ + sqlite3PagerUnref(pPageIn); + pPageIn = 0; + rc = sqlite3PagerGet(pSrcPager, ovflIn, &pPageIn, PAGER_GET_READONLY); + if( rc==SQLITE_OK ){ + aIn = (const u8*)sqlite3PagerGetData(pPageIn); + ovflIn = get4byte(aIn); + aIn += 4; + nIn = pSrc->pBt->usableSize - 4; + } + } + }while( rc==SQLITE_OK && nOut>0 ); + + if( rc==SQLITE_OK && nRem>0 && ALWAYS(pPgnoOut) ){ + Pgno pgnoNew; + MemPage *pNew = 0; + rc = allocateBtreePage(pBt, &pNew, &pgnoNew, 0, 0); + put4byte(pPgnoOut, pgnoNew); + if( ISAUTOVACUUM(pBt) && pPageOut ){ + ptrmapPut(pBt, pgnoNew, PTRMAP_OVERFLOW2, pPageOut->pgno, &rc); + } + releasePage(pPageOut); + pPageOut = pNew; + if( pPageOut ){ + pPgnoOut = pPageOut->aData; + put4byte(pPgnoOut, 0); + aOut = &pPgnoOut[4]; + nOut = MIN(pBt->usableSize - 4, nRem); + } + } + }while( nRem>0 && rc==SQLITE_OK ); + + releasePage(pPageOut); + sqlite3PagerUnref(pPageIn); + return rc; + } +} + +/* +** Delete the entry that the cursor is pointing to. ** ** If the BTREE_SAVEPOSITION bit of the flags parameter is zero, then ** the cursor is left pointing at an arbitrary location after the delete. @@ -65841,15 +80410,14 @@ end_insert: */ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ Btree *p = pCur->pBtree; - BtShared *pBt = p->pBt; - int rc; /* Return code */ - MemPage *pPage; /* Page to delete cell from */ - unsigned char *pCell; /* Pointer to cell to delete */ - int iCellIdx; /* Index of cell to delete */ - int iCellDepth; /* Depth of node containing pCell */ - u16 szCell; /* Size of the cell being deleted */ - int bSkipnext = 0; /* Leaf cursor in SKIPNEXT state */ - u8 bPreserve = flags & BTREE_SAVEPOSITION; /* Keep cursor valid */ + BtShared *pBt = p->pBt; + int rc; /* Return code */ + MemPage *pPage; /* Page to delete cell from */ + unsigned char *pCell; /* Pointer to cell to delete */ + int iCellIdx; /* Index of cell to delete */ + int iCellDepth; /* Depth of node containing pCell */ + CellInfo info; /* Size of the cell being deleted */ + u8 bPreserve; /* Keep cursor valid. 2 for CURSOR_SKIPNEXT */ assert( cursorOwnsBtShared(pCur) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -65857,34 +80425,61 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ assert( pCur->curFlags & BTCF_WriteFlag ); assert( hasSharedCacheTableLock(p, pCur->pgnoRoot, pCur->pKeyInfo!=0, 2) ); assert( !hasReadConflicts(p, pCur->pgnoRoot) ); - assert( pCur->aiIdx[pCur->iPage]apPage[pCur->iPage]->nCell ); - assert( pCur->eState==CURSOR_VALID ); assert( (flags & ~(BTREE_SAVEPOSITION | BTREE_AUXDELETE))==0 ); + if( pCur->eState!=CURSOR_VALID ){ + if( pCur->eState>=CURSOR_REQUIRESEEK ){ + rc = btreeRestoreCursorPosition(pCur); + assert( rc!=SQLITE_OK || CORRUPT_DB || pCur->eState==CURSOR_VALID ); + if( rc || pCur->eState!=CURSOR_VALID ) return rc; + }else{ + return SQLITE_CORRUPT_PGNO(pCur->pgnoRoot); + } + } + assert( pCur->eState==CURSOR_VALID ); iCellDepth = pCur->iPage; - iCellIdx = pCur->aiIdx[iCellDepth]; - pPage = pCur->apPage[iCellDepth]; + iCellIdx = pCur->ix; + pPage = pCur->pPage; + if( pPage->nCell<=iCellIdx ){ + return SQLITE_CORRUPT_PAGE(pPage); + } pCell = findCell(pPage, iCellIdx); + if( pPage->nFree<0 && btreeComputeFreeSpace(pPage) ){ + return SQLITE_CORRUPT_PAGE(pPage); + } + if( pCell<&pPage->aCellIdx[pPage->nCell] ){ + return SQLITE_CORRUPT_PAGE(pPage); + } - /* If the bPreserve flag is set to true, then the cursor position must + /* If the BTREE_SAVEPOSITION bit is on, then the cursor position must ** be preserved following this delete operation. If the current delete ** will cause a b-tree rebalance, then this is done by saving the cursor - ** key and leaving the cursor in CURSOR_REQUIRESEEK state before - ** returning. + ** key and leaving the cursor in CURSOR_REQUIRESEEK state before + ** returning. ** - ** Or, if the current delete will not cause a rebalance, then the cursor + ** If the current delete will not cause a rebalance, then the cursor ** will be left in CURSOR_SKIPNEXT state pointing to the entry immediately - ** before or after the deleted entry. In this case set bSkipnext to true. */ + ** before or after the deleted entry. + ** + ** The bPreserve value records which path is required: + ** + ** bPreserve==0 Not necessary to save the cursor position + ** bPreserve==1 Use CURSOR_REQUIRESEEK to save the cursor position + ** bPreserve==2 Cursor won't move. Set CURSOR_SKIPNEXT. + */ + bPreserve = (flags & BTREE_SAVEPOSITION)!=0; if( bPreserve ){ - if( !pPage->leaf - || (pPage->nFree+cellSizePtr(pPage,pCell)+2)>(int)(pBt->usableSize*2/3) + if( !pPage->leaf + || (pPage->nFree+pPage->xCellSize(pPage,pCell)+2) > + (int)(pBt->usableSize*2/3) + || pPage->nCell==1 /* See dbfuzz001.test for a test case */ ){ /* A b-tree rebalance will be required after deleting this entry. ** Save the cursor key. */ rc = saveCursorKey(pCur); if( rc ) return rc; }else{ - bSkipnext = 1; + bPreserve = 2; } } @@ -65896,8 +80491,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** sub-tree headed by the child page of the cell being deleted. This makes ** balancing the tree following the delete operation easier. */ if( !pPage->leaf ){ - int notUsed = 0; - rc = sqlite3BtreePrevious(pCur, ¬Used); + rc = sqlite3BtreePrevious(pCur, 0); + assert( rc!=SQLITE_DONE ); if( rc ) return rc; } @@ -65910,8 +80505,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ /* If this is a delete operation to remove a row from a table b-tree, ** invalidate any incrblob cursors open on the row being deleted. */ - if( pCur->pKeyInfo==0 ){ - invalidateIncrblobCursors(p, pCur->info.nKey, 0); + if( pCur->pKeyInfo==0 && p->hasIncrblobCur ){ + invalidateIncrblobCursors(p, pCur->pgnoRoot, pCur->info.nKey, 0); } /* Make the page containing the entry to be deleted writable. Then free any @@ -65919,8 +80514,8 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** itself from within the page. */ rc = sqlite3PagerWrite(pPage->pDbPage); if( rc ) return rc; - rc = clearCell(pPage, pCell, &szCell); - dropCell(pPage, iCellIdx, szCell, &rc); + BTREE_CLEAR_CELL(rc, pPage, pCell, info); + dropCell(pPage, iCellIdx, info.nSize, &rc); if( rc ) return rc; /* If the cell deleted was not located on a leaf page, then the cursor @@ -65929,19 +80524,30 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** node. The cell from the leaf node needs to be moved to the internal ** node to replace the deleted cell. */ if( !pPage->leaf ){ - MemPage *pLeaf = pCur->apPage[pCur->iPage]; + MemPage *pLeaf = pCur->pPage; int nCell; - Pgno n = pCur->apPage[iCellDepth+1]->pgno; + Pgno n; unsigned char *pTmp; + if( pLeaf->nFree<0 ){ + rc = btreeComputeFreeSpace(pLeaf); + if( rc ) return rc; + } + if( iCellDepthiPage-1 ){ + n = pCur->apPage[iCellDepth+1]->pgno; + }else{ + n = pCur->pPage->pgno; + } pCell = findCell(pLeaf, pLeaf->nCell-1); - if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_BKPT; + if( pCell<&pLeaf->aData[4] ) return SQLITE_CORRUPT_PAGE(pLeaf); nCell = pLeaf->xCellSize(pLeaf, pCell); assert( MX_CELL_SIZE(pBt) >= nCell ); pTmp = pBt->pTmpSpace; assert( pTmp!=0 ); rc = sqlite3PagerWrite(pLeaf->pDbPage); - insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n, &rc); + if( rc==SQLITE_OK ){ + rc = insertCell(pPage, iCellIdx, pCell-4, nCell+4, pTmp, n); + } dropCell(pLeaf, pLeaf->nCell-1, nCell, &rc); if( rc ) return rc; } @@ -65959,33 +80565,46 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** on the leaf node first. If the balance proceeds far enough up the ** tree that we can be sure that any problem in the internal node has ** been corrected, so be it. Otherwise, after balancing the leaf node, - ** walk the cursor up the tree to the internal node and balance it as + ** walk the cursor up the tree to the internal node and balance it as ** well. */ - rc = balance(pCur); + assert( pCur->pPage->nOverflow==0 ); + assert( pCur->pPage->nFree>=0 ); + if( pCur->pPage->nFree*3<=(int)pCur->pBt->usableSize*2 ){ + /* Optimization: If the free space is less than 2/3rds of the page, + ** then balance() will always be a no-op. No need to invoke it. */ + rc = SQLITE_OK; + }else{ + rc = balance(pCur); + } if( rc==SQLITE_OK && pCur->iPage>iCellDepth ){ + releasePageNotNull(pCur->pPage); + pCur->iPage--; while( pCur->iPage>iCellDepth ){ releasePage(pCur->apPage[pCur->iPage--]); } + pCur->pPage = pCur->apPage[pCur->iPage]; rc = balance(pCur); } if( rc==SQLITE_OK ){ - if( bSkipnext ){ - assert( bPreserve && (pCur->iPage==iCellDepth || CORRUPT_DB) ); - assert( pPage==pCur->apPage[pCur->iPage] || CORRUPT_DB ); + if( bPreserve>1 ){ + assert( (pCur->iPage==iCellDepth || CORRUPT_DB) ); + assert( pPage==pCur->pPage || CORRUPT_DB ); assert( (pPage->nCell>0 || CORRUPT_DB) && iCellIdx<=pPage->nCell ); pCur->eState = CURSOR_SKIPNEXT; if( iCellIdx>=pPage->nCell ){ pCur->skipNext = -1; - pCur->aiIdx[iCellDepth] = pPage->nCell-1; + pCur->ix = pPage->nCell-1; }else{ pCur->skipNext = 1; } }else{ rc = moveToRoot(pCur); if( bPreserve ){ + btreeReleaseAllCursorPages(pCur); pCur->eState = CURSOR_REQUIRESEEK; } + if( rc==SQLITE_EMPTY ) rc = SQLITE_OK; } } return rc; @@ -66002,12 +80621,12 @@ SQLITE_PRIVATE int sqlite3BtreeDelete(BtCursor *pCur, u8 flags){ ** BTREE_INTKEY|BTREE_LEAFDATA Used for SQL tables with rowid keys ** BTREE_ZERODATA Used for SQL indices */ -static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ +static int btreeCreateTable(Btree *p, Pgno *piTable, int createTabFlags){ BtShared *pBt = p->pBt; MemPage *pRoot; Pgno pgnoRoot; int rc; - int ptfFlags; /* Page-type flage for the root page of new table */ + int ptfFlags; /* Page-type flags for the root page of new table */ assert( sqlite3BtreeHoldsMutex(p) ); assert( pBt->inTransaction==TRANS_WRITE ); @@ -66035,6 +80654,9 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ ** created so far, so the new root-page is (meta[3]+1). */ sqlite3BtreeGetMeta(p, BTREE_LARGEST_ROOT_PAGE, &pgnoRoot); + if( pgnoRoot>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_PGNO(pgnoRoot); + } pgnoRoot++; /* The new root-page may not be allocated on a pointer-map page, or the @@ -66044,8 +80666,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ pgnoRoot==PENDING_BYTE_PAGE(pBt) ){ pgnoRoot++; } - assert( pgnoRoot>=3 || CORRUPT_DB ); - testcase( pgnoRoot<3 ); + assert( pgnoRoot>=3 ); /* Allocate a page. The page that currently resides at pgnoRoot will ** be moved to the allocated page (unless the allocated page happens @@ -66082,7 +80703,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ } rc = ptrmapGet(pBt, pgnoRoot, &eType, &iPtrPage); if( eType==PTRMAP_ROOTPAGE || eType==PTRMAP_FREEPAGE ){ - rc = SQLITE_CORRUPT_BKPT; + rc = SQLITE_CORRUPT_PGNO(pgnoRoot); } if( rc!=SQLITE_OK ){ releasePage(pRoot); @@ -66108,7 +80729,7 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ } }else{ pRoot = pPageMove; - } + } /* Update the pointer-map and meta-data with the new root-page number. */ ptrmapPut(pBt, pgnoRoot, PTRMAP_ROOTPAGE, 0, &rc); @@ -66142,10 +80763,10 @@ static int btreeCreateTable(Btree *p, int *piTable, int createTabFlags){ zeroPage(pRoot, ptfFlags); sqlite3PagerUnref(pRoot->pDbPage); assert( (pBt->openFlags & BTREE_SINGLE)==0 || pgnoRoot==2 ); - *piTable = (int)pgnoRoot; + *piTable = pgnoRoot; return SQLITE_OK; } -SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, int *piTable, int flags){ +SQLITE_PRIVATE int sqlite3BtreeCreateTable(Btree *p, Pgno *piTable, int flags){ int rc; sqlite3BtreeEnter(p); rc = btreeCreateTable(p, piTable, flags); @@ -66161,26 +80782,27 @@ static int clearDatabasePage( BtShared *pBt, /* The BTree that contains the table */ Pgno pgno, /* Page number to clear */ int freePageFlag, /* Deallocate page if true */ - int *pnChange /* Add number of Cells freed to this counter */ + i64 *pnChange /* Add number of Cells freed to this counter */ ){ MemPage *pPage; int rc; unsigned char *pCell; int i; int hdr; - u16 szCell; + CellInfo info; assert( sqlite3_mutex_held(pBt->mutex) ); if( pgno>btreePagecount(pBt) ){ - return SQLITE_CORRUPT_BKPT; + return SQLITE_CORRUPT_PGNO(pgno); } - rc = getAndInitPage(pBt, pgno, &pPage, 0, 0); + rc = getAndInitPage(pBt, pgno, &pPage, 0); if( rc ) return rc; - if( pPage->bBusy ){ - rc = SQLITE_CORRUPT_BKPT; + if( (pBt->openFlags & BTREE_SINGLE)==0 + && sqlite3PagerPageRefcount(pPage->pDbPage) != (1 + (pgno==1)) + ){ + rc = SQLITE_CORRUPT_PAGE(pPage); goto cleardatabasepage_out; } - pPage->bBusy = 1; hdr = pPage->hdrOffset; for(i=0; inCell; i++){ pCell = findCell(pPage, i); @@ -66188,14 +80810,15 @@ static int clearDatabasePage( rc = clearDatabasePage(pBt, get4byte(pCell), 1, pnChange); if( rc ) goto cleardatabasepage_out; } - rc = clearCell(pPage, pCell, &szCell); + BTREE_CLEAR_CELL(rc, pPage, pCell, info); if( rc ) goto cleardatabasepage_out; } if( !pPage->leaf ){ rc = clearDatabasePage(pBt, get4byte(&pPage->aData[hdr+8]), 1, pnChange); if( rc ) goto cleardatabasepage_out; - }else if( pnChange ){ - assert( pPage->intKey || CORRUPT_DB ); + if( pPage->intKey ) pnChange = 0; + } + if( pnChange ){ testcase( !pPage->intKey ); *pnChange += pPage->nCell; } @@ -66206,7 +80829,6 @@ static int clearDatabasePage( } cleardatabasepage_out: - pPage->bBusy = 0; releasePage(pPage); return rc; } @@ -66220,11 +80842,10 @@ cleardatabasepage_out: ** read cursors on the table. Open write cursors are moved to the ** root of the table. ** -** If pnChange is not NULL, then table iTable must be an intkey table. The -** integer value pointed to by pnChange is incremented by the number of -** entries in the table. +** If pnChange is not NULL, then the integer value pointed to by pnChange +** is incremented by the number of entries in the table. */ -SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ +SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, i64 *pnChange){ int rc; BtShared *pBt = p->pBt; sqlite3BtreeEnter(p); @@ -66236,7 +80857,9 @@ SQLITE_PRIVATE int sqlite3BtreeClearTable(Btree *p, int iTable, int *pnChange){ /* Invalidate all incrblob cursors open on table iTable (assuming iTable ** is the root of a table b-tree - if it is not, the following call is ** a no-op). */ - invalidateIncrblobCursors(p, 0, 1); + if( p->hasIncrblobCur ){ + invalidateIncrblobCursors(p, (Pgno)iTable, 0, 1); + } rc = clearDatabasePage(pBt, (Pgno)iTable, 0, pnChange); } sqlite3BtreeLeave(p); @@ -66261,12 +80884,12 @@ SQLITE_PRIVATE int sqlite3BtreeClearTableOfCursor(BtCursor *pCur){ ** cursors on the table. ** ** If AUTOVACUUM is enabled and the page at iTable is not the last -** root page in the database file, then the last root page +** root page in the database file, then the last root page ** in the database file is moved into the slot formerly occupied by ** iTable and that last slot formerly occupied by the last root page ** is added to the freelist instead of iTable. In this say, all ** root pages are kept at the beginning of the database file, which -** is necessary for AUTOVACUUM to work right. *piMoved is set to the +** is necessary for AUTOVACUUM to work right. *piMoved is set to the ** page number that used to be the last root page in the file before ** the move. If no page gets moved, *piMoved is set to 0. ** The last root page is recorded in meta[3] and the value of @@ -66279,32 +80902,15 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ assert( sqlite3BtreeHoldsMutex(p) ); assert( p->inTrans==TRANS_WRITE ); - - /* It is illegal to drop a table if any cursors are open on the - ** database. This is because in auto-vacuum mode the backend may - ** need to move another root-page to fill a gap left by the deleted - ** root page. If an open cursor was using this page a problem would - ** occur. - ** - ** This error is caught long before control reaches this point. - */ - if( NEVER(pBt->pCursor) ){ - sqlite3ConnectionBlocked(p->db, pBt->pCursor->pBtree->db); - return SQLITE_LOCKED_SHAREDCACHE; + assert( iTable>=2 ); + if( iTable>btreePagecount(pBt) ){ + return SQLITE_CORRUPT_PGNO(iTable); } - /* - ** It is illegal to drop the sqlite_master table on page 1. But again, - ** this error is caught long before reaching this point. - */ - if( NEVER(iTable<2) ){ - return SQLITE_CORRUPT_BKPT; - } - - rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); - if( rc ) return rc; rc = sqlite3BtreeClearTable(p, iTable, 0); - if( rc ){ + if( rc ) return rc; + rc = btreeGetPage(pBt, (Pgno)iTable, &pPage, 0); + if( NEVER(rc) ){ releasePage(pPage); return rc; } @@ -66321,7 +80927,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ if( iTable==maxRootPgno ){ /* If the table being dropped is the table with the largest root-page - ** number in the database, put the root page on the free list. + ** number in the database, put the root page on the free list. */ freePage(pPage, &rc); releasePage(pPage); @@ -66330,7 +80936,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ } }else{ /* The table being dropped does not have the largest root-page - ** number in the database. So move the page that does into the + ** number in the database. So move the page that does into the ** gap left by the deleted root-page. */ MemPage *pMove; @@ -66372,7 +80978,7 @@ static int btreeDropTable(Btree *p, Pgno iTable, int *piMoved){ releasePage(pPage); } #endif - return rc; + return rc; } SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ int rc; @@ -66391,7 +80997,7 @@ SQLITE_PRIVATE int sqlite3BtreeDropTable(Btree *p, int iTable, int *piMoved){ ** is the number of free pages currently in the database. Meta[1] ** through meta[15] are available for use by higher layers. Meta[0] ** is read-only, the others are read/write. -** +** ** The schema layer numbers meta values differently. At the schema ** layer (and the SetCookie and ReadCookie opcodes) the number of ** free pages is not visible. So Cookie[0] is the same as Meta[1]. @@ -66408,12 +81014,12 @@ SQLITE_PRIVATE void sqlite3BtreeGetMeta(Btree *p, int idx, u32 *pMeta){ sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE ); - assert( SQLITE_OK==querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK) ); + assert( SQLITE_OK==querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK) ); assert( pBt->pPage1 ); assert( idx>=0 && idx<=15 ); if( idx==BTREE_DATA_VERSION ){ - *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iDataVersion; + *pMeta = sqlite3PagerDataVersion(pBt->pPager) + p->iBDataVersion; }else{ *pMeta = get4byte(&pBt->pPage1->aData[36 + idx*4]); } @@ -66457,42 +81063,41 @@ SQLITE_PRIVATE int sqlite3BtreeUpdateMeta(Btree *p, int idx, u32 iMeta){ return rc; } -#ifndef SQLITE_OMIT_BTREECOUNT /* ** The first argument, pCur, is a cursor opened on some b-tree. Count the ** number of entries in the b-tree and write the result to *pnEntry. ** -** SQLITE_OK is returned if the operation is successfully executed. +** SQLITE_OK is returned if the operation is successfully executed. ** Otherwise, if an error is encountered (i.e. an IO error or database ** corruption) an SQLite error code is returned. */ -SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ +SQLITE_PRIVATE int sqlite3BtreeCount(sqlite3 *db, BtCursor *pCur, i64 *pnEntry){ i64 nEntry = 0; /* Value to return in *pnEntry */ int rc; /* Return code */ - if( pCur->pgnoRoot==0 ){ + rc = moveToRoot(pCur); + if( rc==SQLITE_EMPTY ){ *pnEntry = 0; return SQLITE_OK; } - rc = moveToRoot(pCur); /* Unless an error occurs, the following loop runs one iteration for each - ** page in the B-Tree structure (not including overflow pages). + ** page in the B-Tree structure (not including overflow pages). */ - while( rc==SQLITE_OK ){ + while( rc==SQLITE_OK && !AtomicLoad(&db->u1.isInterrupted) ){ int iIdx; /* Index of child node in parent */ MemPage *pPage; /* Current page of the b-tree */ - /* If this is a leaf page or the tree is not an int-key tree, then + /* If this is a leaf page or the tree is not an int-key tree, then ** this page contains countable entries. Increment the entry counter ** accordingly. */ - pPage = pCur->apPage[pCur->iPage]; + pPage = pCur->pPage; if( pPage->leaf || !pPage->intKey ){ nEntry += pPage->nCell; } - /* pPage is a leaf node. This loop navigates the cursor so that it + /* pPage is a leaf node. This loop navigates the cursor so that it ** points to the first interior cell that it points to the parent of ** the next page in the tree that has not yet been visited. The ** pCur->aiIdx[pCur->iPage] value is set to the index of the parent cell @@ -66510,16 +81115,16 @@ SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ return moveToRoot(pCur); } moveToParent(pCur); - }while ( pCur->aiIdx[pCur->iPage]>=pCur->apPage[pCur->iPage]->nCell ); + }while ( pCur->ix>=pCur->pPage->nCell ); - pCur->aiIdx[pCur->iPage]++; - pPage = pCur->apPage[pCur->iPage]; + pCur->ix++; + pPage = pCur->pPage; } - /* Descend to the child node of the cell that the cursor currently + /* Descend to the child node of the cell that the cursor currently ** points at. This is the right-child if (iIdx==pPage->nCell). */ - iIdx = pCur->aiIdx[pCur->iPage]; + iIdx = pCur->ix; if( iIdx==pPage->nCell ){ rc = moveToChild(pCur, get4byte(&pPage->aData[pPage->hdrOffset+8])); }else{ @@ -66530,7 +81135,6 @@ SQLITE_PRIVATE int sqlite3BtreeCount(BtCursor *pCur, i64 *pnEntry){ /* An error has occurred. Return an error code. */ return rc; } -#endif /* ** Return the pager associated with a BTree. This routine is used for @@ -66541,6 +81145,41 @@ SQLITE_PRIVATE Pager *sqlite3BtreePager(Btree *p){ } #ifndef SQLITE_OMIT_INTEGRITY_CHECK +/* +** Record an OOM error during integrity_check +*/ +static void checkOom(IntegrityCk *pCheck){ + pCheck->rc = SQLITE_NOMEM; + pCheck->mxErr = 0; /* Causes integrity_check processing to stop */ + if( pCheck->nErr==0 ) pCheck->nErr++; +} + +/* +** Invoke the progress handler, if appropriate. Also check for an +** interrupt. +*/ +static void checkProgress(IntegrityCk *pCheck){ + sqlite3 *db = pCheck->db; + if( AtomicLoad(&db->u1.isInterrupted) ){ + pCheck->rc = SQLITE_INTERRUPT; + pCheck->nErr++; + pCheck->mxErr = 0; + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + assert( db->nProgressOps>0 ); + pCheck->nStep++; + if( (pCheck->nStep % db->nProgressOps)==0 + && db->xProgress(db->pProgressArg) + ){ + pCheck->rc = SQLITE_INTERRUPT; + pCheck->nErr++; + pCheck->mxErr = 0; + } + } +#endif +} + /* ** Append a message to the error message string. */ @@ -66550,20 +81189,22 @@ static void checkAppendMsg( ... ){ va_list ap; + checkProgress(pCheck); if( !pCheck->mxErr ) return; pCheck->mxErr--; pCheck->nErr++; va_start(ap, zFormat); if( pCheck->errMsg.nChar ){ - sqlite3StrAccumAppend(&pCheck->errMsg, "\n", 1); + sqlite3_str_append(&pCheck->errMsg, "\n", 1); } if( pCheck->zPfx ){ - sqlite3XPrintf(&pCheck->errMsg, pCheck->zPfx, pCheck->v1, pCheck->v2); + sqlite3_str_appendf(&pCheck->errMsg, pCheck->zPfx, + pCheck->v0, pCheck->v1, pCheck->v2); } - sqlite3VXPrintf(&pCheck->errMsg, zFormat, ap); + sqlite3_str_vappendf(&pCheck->errMsg, zFormat, ap); va_end(ap); - if( pCheck->errMsg.accError==STRACCUM_NOMEM ){ - pCheck->mallocFailed = 1; + if( pCheck->errMsg.accError==SQLITE_NOMEM ){ + checkOom(pCheck); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -66575,7 +81216,8 @@ static void checkAppendMsg( ** corresponds to page iPg is already set. */ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); return (pCheck->aPgRef[iPg/8] & (1 << (iPg & 0x07))); } @@ -66583,7 +81225,8 @@ static int getPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Set the bit in the IntegrityCk.aPgRef[] array that corresponds to page iPg. */ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ - assert( iPg<=pCheck->nPage && sizeof(pCheck->aPgRef[0])==1 ); + assert( pCheck->aPgRef!=0 ); + assert( iPg<=pCheck->nCkPage && sizeof(pCheck->aPgRef[0])==1 ); pCheck->aPgRef[iPg/8] |= (1 << (iPg & 0x07)); } @@ -66597,13 +81240,12 @@ static void setPageReferenced(IntegrityCk *pCheck, Pgno iPg){ ** Also check that the page number is in bounds. */ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ - if( iPage==0 ) return 1; - if( iPage>pCheck->nPage ){ - checkAppendMsg(pCheck, "invalid page number %d", iPage); + if( iPage>pCheck->nCkPage || iPage==0 ){ + checkAppendMsg(pCheck, "invalid page number %u", iPage); return 1; } if( getPageReferenced(pCheck, iPage) ){ - checkAppendMsg(pCheck, "2nd reference to page %d", iPage); + checkAppendMsg(pCheck, "2nd reference to page %u", iPage); return 1; } setPageReferenced(pCheck, iPage); @@ -66612,7 +81254,7 @@ static int checkRef(IntegrityCk *pCheck, Pgno iPage){ #ifndef SQLITE_OMIT_AUTOVACUUM /* -** Check that the entry in the pointer-map for page iChild maps to +** Check that the entry in the pointer-map for page iChild maps to ** page iParent, pointer type ptrType. If not, append an error message ** to pCheck. */ @@ -66628,14 +81270,14 @@ static void checkPtrmap( rc = ptrmapGet(pCheck->pBt, iChild, &ePtrmapType, &iPtrmapParent); if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) pCheck->mallocFailed = 1; - checkAppendMsg(pCheck, "Failed to read ptrmap key=%d", iChild); + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ) checkOom(pCheck); + checkAppendMsg(pCheck, "Failed to read ptrmap key=%u", iChild); return; } if( ePtrmapType!=eType || iPtrmapParent!=iParent ){ checkAppendMsg(pCheck, - "Bad ptr map entry key=%d expected=(%d,%d) got=(%d,%d)", + "Bad ptr map entry key=%u expected=(%u,%u) got=(%u,%u)", iChild, eType, iParent, ePtrmapType, iPtrmapParent); } } @@ -66648,40 +81290,35 @@ static void checkPtrmap( static void checkList( IntegrityCk *pCheck, /* Integrity checking context */ int isFreeList, /* True for a freelist. False for overflow page list */ - int iPage, /* Page number for first page in the list */ - int N /* Expected number of pages in the list */ + Pgno iPage, /* Page number for first page in the list */ + u32 N /* Expected number of pages in the list */ ){ int i; - int expected = N; - int iFirst = iPage; - while( N-- > 0 && pCheck->mxErr ){ + u32 expected = N; + int nErrAtStart = pCheck->nErr; + while( iPage!=0 && pCheck->mxErr ){ DbPage *pOvflPage; unsigned char *pOvflData; - if( iPage<1 ){ - checkAppendMsg(pCheck, - "%d of %d pages missing from overflow list starting at %d", - N+1, expected, iFirst); - break; - } if( checkRef(pCheck, iPage) ) break; + N--; if( sqlite3PagerGet(pCheck->pPager, (Pgno)iPage, &pOvflPage, 0) ){ - checkAppendMsg(pCheck, "failed to get page %d", iPage); + checkAppendMsg(pCheck, "failed to get page %u", iPage); break; } pOvflData = (unsigned char *)sqlite3PagerGetData(pOvflPage); if( isFreeList ){ - int n = get4byte(&pOvflData[4]); + u32 n = (u32)get4byte(&pOvflData[4]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pCheck->pBt->autoVacuum ){ checkPtrmap(pCheck, iPage, PTRMAP_FREEPAGE, 0); } #endif - if( n>(int)pCheck->pBt->usableSize/4-2 ){ + if( n>pCheck->pBt->usableSize/4-2 ){ checkAppendMsg(pCheck, - "freelist leaf count too big on page %d", iPage); + "freelist leaf count too big on page %u", iPage); N--; }else{ - for(i=0; ipBt->autoVacuum ){ @@ -66707,10 +81344,12 @@ static void checkList( #endif iPage = get4byte(pOvflData); sqlite3PagerUnref(pOvflPage); - - if( isFreeList && N<(iPage!=0) ){ - checkAppendMsg(pCheck, "free-page count in header is too small"); - } + } + if( N && nErrAtStart==pCheck->nErr ){ + checkAppendMsg(pCheck, + "%s is %u but should be %u", + isFreeList ? "size" : "overflow list length", + expected-N, expected); } } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -66733,12 +81372,14 @@ static void checkList( ** property. ** ** This heap is used for cell overlap and coverage testing. Each u32 -** entry represents the span of a cell or freeblock on a btree page. +** entry represents the span of a cell or freeblock on a btree page. ** The upper 16 bits are the index of the first byte of a range and the ** lower 16 bits are the index of the last byte of that range. */ static void btreeHeapInsert(u32 *aHeap, u32 x){ - u32 j, i = ++aHeap[0]; + u32 j, i; + assert( aHeap!=0 ); + i = ++aHeap[0]; aHeap[i] = x; while( (j = i/2)>0 && aHeap[j]>aHeap[i] ){ x = aHeap[j]; @@ -66763,7 +81404,7 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ aHeap[j] = x; i = j; } - return 1; + return 1; } #ifndef SQLITE_OMIT_INTEGRITY_CHECK @@ -66771,7 +81412,7 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ ** Do various sanity checks on a single page of a tree. Return ** the tree depth. Root pages return 0. Parents of root pages ** return 1, and so forth. -** +** ** These checks are done: ** ** 1. Make sure that cells and freeblocks do not overlap @@ -66783,7 +81424,7 @@ static int btreeHeapPull(u32 *aHeap, u32 *pOut){ */ static int checkTreePage( IntegrityCk *pCheck, /* Context for the sanity check */ - int iPage, /* Page number of the page to check */ + Pgno iPage, /* Page number of the page to check */ i64 *piMinKey, /* Write minimum integer primary key here */ i64 maxKey /* Error if integer primary key greater than this */ ){ @@ -66815,15 +81456,18 @@ static int checkTreePage( /* Check that the page exists */ + checkProgress(pCheck); + if( pCheck->mxErr==0 ) goto end_of_check; pBt = pCheck->pBt; usableSize = pBt->usableSize; if( iPage==0 ) return 0; if( checkRef(pCheck, iPage) ) return 0; - pCheck->zPfx = "Page %d: "; + pCheck->zPfx = "Tree %u page %u: "; pCheck->v1 = iPage; - if( (rc = btreeGetPage(pBt, (Pgno)iPage, &pPage, 0))!=0 ){ + if( (rc = btreeGetPage(pBt, iPage, &pPage, 0))!=0 ){ checkAppendMsg(pCheck, "unable to get the page. error code=%d", rc); + if( rc==SQLITE_IOERR_NOMEM ) pCheck->rc = SQLITE_NOMEM; goto end_of_check; } @@ -66837,11 +81481,16 @@ static int checkTreePage( "btreeInitPage() returns error code %d", rc); goto end_of_check; } + if( (rc = btreeComputeFreeSpace(pPage))!=0 ){ + assert( rc==SQLITE_CORRUPT ); + checkAppendMsg(pCheck, "free space corruption", rc); + goto end_of_check; + } data = pPage->aData; hdr = pPage->hdrOffset; /* Set up for cell analysis */ - pCheck->zPfx = "On tree page %d cell %d: "; + pCheck->zPfx = "Tree %u page %u cell %u: "; contentOffset = get2byteNotZero(&data[hdr+5]); assert( contentOffset<=usableSize ); /* Enforced by btreeInitPage() */ @@ -66849,6 +81498,9 @@ static int checkTreePage( ** number of cells on the page. */ nCell = get2byte(&data[hdr+3]); assert( pPage->nCell==nCell ); + if( pPage->leaf || pPage->intKey==0 ){ + pCheck->nRow += nCell; + } /* EVIDENCE-OF: R-23882-45353 The cell pointer array of a b-tree page ** immediately follows the b-tree page header. */ @@ -66861,7 +81513,7 @@ static int checkTreePage( pgno = get4byte(&data[hdr+8]); #ifndef SQLITE_OMIT_AUTOVACUUM if( pBt->autoVacuum ){ - pCheck->zPfx = "On page %d at right child: "; + pCheck->zPfx = "Tree %u page %u right child: "; checkPtrmap(pCheck, pgno, PTRMAP_BTREE, iPage); } #endif @@ -66885,7 +81537,7 @@ static int checkTreePage( pc = get2byteAligned(pCellIdx); pCellIdx -= 2; if( pcusableSize-4 ){ - checkAppendMsg(pCheck, "Offset %d out of range %d..%d", + checkAppendMsg(pCheck, "Offset %u out of range %u..%u", pc, contentOffset, usableSize-4); doCoverageCheck = 0; continue; @@ -66904,11 +81556,12 @@ static int checkTreePage( checkAppendMsg(pCheck, "Rowid %lld out of order", info.nKey); } maxKey = info.nKey; + keyCanBeEqual = 0; /* Only the first key on the page may ==maxKey */ } /* Check the content overflow list */ if( info.nPayload>info.nLocal ){ - int nPage; /* Number of pages on the overflow chain */ + u32 nPage; /* Number of pages on the overflow chain */ Pgno pgnoOvfl; /* First page of the overflow chain */ assert( pc + info.nSize - 4 <= usableSize ); nPage = (info.nPayload - info.nLocal + usableSize - 5)/(usableSize - 4); @@ -66959,18 +81612,19 @@ static int checkTreePage( btreeHeapInsert(heap, (pc<<16)|(pc+size-1)); } } + assert( heap!=0 ); /* Add the freeblocks to the min-heap ** ** EVIDENCE-OF: R-20690-50594 The second field of the b-tree page header ** is the offset of the first freeblock, or zero if there are no - ** freeblocks on the page. + ** freeblocks on the page. */ i = get2byte(&data[hdr+1]); while( i>0 ){ int size, j; - assert( (u32)i<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( (u32)i<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ size = get2byte(&data[i+2]); - assert( (u32)(i+size)<=usableSize ); /* Enforced by btreeInitPage() */ + assert( (u32)(i+size)<=usableSize ); /* due to btreeComputeFreeSpace() */ btreeHeapInsert(heap, (((u32)i)<<16)|(i+size-1)); /* EVIDENCE-OF: R-58208-19414 The first 2 bytes of a freeblock are a ** big-endian integer which is the offset in the b-tree page of the next @@ -66979,17 +81633,17 @@ static int checkTreePage( j = get2byte(&data[i]); /* EVIDENCE-OF: R-06866-39125 Freeblocks are always connected in order of ** increasing offset. */ - assert( j==0 || j>i+size ); /* Enforced by btreeInitPage() */ - assert( (u32)j<=usableSize-4 ); /* Enforced by btreeInitPage() */ + assert( j==0 || j>i+size ); /* Enforced by btreeComputeFreeSpace() */ + assert( (u32)j<=usableSize-4 ); /* Enforced by btreeComputeFreeSpace() */ i = j; } - /* Analyze the min-heap looking for overlap between cells and/or + /* Analyze the min-heap looking for overlap between cells and/or ** freeblocks, and counting the number of untracked bytes in nFrag. - ** + ** ** Each min-heap entry is of the form: (start_address<<16)|end_address. ** There is an implied first entry the covers the page header, the cell ** pointer index, and the gap between the cell pointer index and the start - ** of cell content. + ** of cell content. ** ** The loop below pulls entries from the min-heap in order and compares ** the start_address against the previous end_address. If there is an @@ -67001,7 +81655,7 @@ static int checkTreePage( while( btreeHeapPull(heap,&x) ){ if( (prev&0xffff)>=(x>>16) ){ checkAppendMsg(pCheck, - "Multiple uses for byte %u of page %d", x>>16, iPage); + "Multiple uses for byte %u of page %u", x>>16, iPage); break; }else{ nFrag += (x>>16) - (prev&0xffff) - 1; @@ -67016,7 +81670,7 @@ static int checkTreePage( */ if( heap[0]==0 && nFrag!=data[hdr+7] ){ checkAppendMsg(pCheck, - "Fragmentation of %d bytes reported as %d on page %d", + "Fragmentation of %u bytes reported as %u on page %u", nFrag, data[hdr+7], iPage); } } @@ -67044,99 +81698,146 @@ end_of_check: ** allocation errors, an error message held in memory obtained from ** malloc is returned if *pnErr is non-zero. If *pnErr==0 then NULL is ** returned. If a memory allocation error occurs, NULL is returned. +** +** If the first entry in aRoot[] is 0, that indicates that the list of +** root pages is incomplete. This is a "partial integrity-check". This +** happens when performing an integrity check on a single table. The +** zero is skipped, of course. But in addition, the freelist checks +** and the checks to make sure every page is referenced are also skipped, +** since obviously it is not possible to know which pages are covered by +** the unverified btrees. Except, if aRoot[1] is 1, then the freelist +** checks are still performed. */ -SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( +SQLITE_PRIVATE int sqlite3BtreeIntegrityCheck( + sqlite3 *db, /* Database connection that is running the check */ Btree *p, /* The btree to be checked */ - int *aRoot, /* An array of root pages numbers for individual trees */ + Pgno *aRoot, /* An array of root pages numbers for individual trees */ + Mem *aCnt, /* Memory cells to write counts for each tree to */ int nRoot, /* Number of entries in aRoot[] */ int mxErr, /* Stop reporting errors after this many */ - int *pnErr /* Write number of errors seen to this variable */ + int *pnErr, /* OUT: Write number of errors seen to this variable */ + char **pzOut /* OUT: Write the error message string here */ ){ Pgno i; IntegrityCk sCheck; BtShared *pBt = p->pBt; - int savedDbFlags = pBt->db->flags; + u64 savedDbFlags = pBt->db->flags; char zErr[100]; + int bPartial = 0; /* True if not checking all btrees */ + int bCkFreelist = 1; /* True to scan the freelist */ VVA_ONLY( int nRef ); + assert( nRoot>0 ); + assert( aCnt!=0 ); + + /* aRoot[0]==0 means this is a partial check */ + if( aRoot[0]==0 ){ + assert( nRoot>1 ); + bPartial = 1; + if( aRoot[1]!=1 ) bCkFreelist = 0; + } + sqlite3BtreeEnter(p); assert( p->inTrans>TRANS_NONE && pBt->inTransaction>TRANS_NONE ); VVA_ONLY( nRef = sqlite3PagerRefcount(pBt->pPager) ); assert( nRef>=0 ); + memset(&sCheck, 0, sizeof(sCheck)); + sCheck.db = db; sCheck.pBt = pBt; sCheck.pPager = pBt->pPager; - sCheck.nPage = btreePagecount(sCheck.pBt); + sCheck.nCkPage = btreePagecount(sCheck.pBt); sCheck.mxErr = mxErr; - sCheck.nErr = 0; - sCheck.mallocFailed = 0; - sCheck.zPfx = 0; - sCheck.v1 = 0; - sCheck.v2 = 0; - sCheck.aPgRef = 0; - sCheck.heap = 0; sqlite3StrAccumInit(&sCheck.errMsg, 0, zErr, sizeof(zErr), SQLITE_MAX_LENGTH); sCheck.errMsg.printfFlags = SQLITE_PRINTF_INTERNAL; - if( sCheck.nPage==0 ){ + if( sCheck.nCkPage==0 ){ goto integrity_ck_cleanup; } - sCheck.aPgRef = sqlite3MallocZero((sCheck.nPage / 8)+ 1); + sCheck.aPgRef = sqlite3MallocZero((sCheck.nCkPage / 8)+ 1); if( !sCheck.aPgRef ){ - sCheck.mallocFailed = 1; + checkOom(&sCheck); goto integrity_ck_cleanup; } sCheck.heap = (u32*)sqlite3PageMalloc( pBt->pageSize ); if( sCheck.heap==0 ){ - sCheck.mallocFailed = 1; + checkOom(&sCheck); goto integrity_ck_cleanup; } i = PENDING_BYTE_PAGE(pBt); - if( i<=sCheck.nPage ) setPageReferenced(&sCheck, i); + if( i<=sCheck.nCkPage ) setPageReferenced(&sCheck, i); /* Check the integrity of the freelist */ - sCheck.zPfx = "Main freelist: "; - checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), - get4byte(&pBt->pPage1->aData[36])); - sCheck.zPfx = 0; + if( bCkFreelist ){ + sCheck.zPfx = "Freelist: "; + checkList(&sCheck, 1, get4byte(&pBt->pPage1->aData[32]), + get4byte(&pBt->pPage1->aData[36])); + sCheck.zPfx = 0; + } /* Check all the tables. */ - testcase( pBt->db->flags & SQLITE_CellSizeCk ); - pBt->db->flags &= ~SQLITE_CellSizeCk; - for(i=0; (int)iautoVacuum && aRoot[i]>1 ){ - checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + if( !bPartial ){ + if( pBt->autoVacuum ){ + Pgno mx = 0; + Pgno mxInHdr; + for(i=0; (int)ipPage1->aData[52]); + if( mx!=mxInHdr ){ + checkAppendMsg(&sCheck, + "max rootpage (%u) disagrees with header (%u)", + mx, mxInHdr + ); + } + }else if( get4byte(&pBt->pPage1->aData[64])!=0 ){ + checkAppendMsg(&sCheck, + "incremental_vacuum enabled with a max rootpage of zero" + ); } + } #endif - checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + testcase( pBt->db->flags & SQLITE_CellSizeCk ); + pBt->db->flags &= ~(u64)SQLITE_CellSizeCk; + for(i=0; (int)iautoVacuum && aRoot[i]>1 && !bPartial ){ + checkPtrmap(&sCheck, aRoot[i], PTRMAP_ROOTPAGE, 0); + } +#endif + sCheck.v0 = aRoot[i]; + checkTreePage(&sCheck, aRoot[i], ¬Used, LARGEST_INT64); + } + sqlite3MemSetArrayInt64(aCnt, i, sCheck.nRow); } pBt->db->flags = savedDbFlags; /* Make sure every page in the file is referenced */ - for(i=1; i<=sCheck.nPage && sCheck.mxErr; i++){ + if( !bPartial ){ + for(i=1; i<=sCheck.nCkPage && sCheck.mxErr; i++){ #ifdef SQLITE_OMIT_AUTOVACUUM - if( getPageReferenced(&sCheck, i)==0 ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); - } + if( getPageReferenced(&sCheck, i)==0 ){ + checkAppendMsg(&sCheck, "Page %u: never used", i); + } #else - /* If the database supports auto-vacuum, make sure no tables contain - ** references to pointer-map pages. - */ - if( getPageReferenced(&sCheck, i)==0 && - (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Page %d is never used", i); - } - if( getPageReferenced(&sCheck, i)!=0 && - (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ - checkAppendMsg(&sCheck, "Pointer map page %d is referenced", i); - } + /* If the database supports auto-vacuum, make sure no tables contain + ** references to pointer-map pages. + */ + if( getPageReferenced(&sCheck, i)==0 && + (PTRMAP_PAGENO(pBt, i)!=i || !pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, "Page %u: never used", i); + } + if( getPageReferenced(&sCheck, i)!=0 && + (PTRMAP_PAGENO(pBt, i)==i && pBt->autoVacuum) ){ + checkAppendMsg(&sCheck, "Page %u: pointer map referenced", i); + } #endif + } } /* Clean up and report errors. @@ -67144,16 +81845,17 @@ SQLITE_PRIVATE char *sqlite3BtreeIntegrityCheck( integrity_ck_cleanup: sqlite3PageFree(sCheck.heap); sqlite3_free(sCheck.aPgRef); - if( sCheck.mallocFailed ){ - sqlite3StrAccumReset(&sCheck.errMsg); - sCheck.nErr++; - } *pnErr = sCheck.nErr; - if( sCheck.nErr==0 ) sqlite3StrAccumReset(&sCheck.errMsg); + if( sCheck.nErr==0 ){ + sqlite3_str_reset(&sCheck.errMsg); + *pzOut = 0; + }else{ + *pzOut = sqlite3StrAccumFinish(&sCheck.errMsg); + } /* Make sure this analysis did not leave any unref() pages. */ assert( nRef==sqlite3PagerRefcount(pBt->pPager) ); sqlite3BtreeLeave(p); - return sqlite3StrAccumFinish(&sCheck.errMsg); + return sCheck.rc; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ @@ -67183,18 +81885,19 @@ SQLITE_PRIVATE const char *sqlite3BtreeGetJournalname(Btree *p){ } /* -** Return non-zero if a transaction is active. +** Return one of SQLITE_TXN_NONE, SQLITE_TXN_READ, or SQLITE_TXN_WRITE +** to describe the current transaction state of Btree p. */ -SQLITE_PRIVATE int sqlite3BtreeIsInTrans(Btree *p){ +SQLITE_PRIVATE int sqlite3BtreeTxnState(Btree *p){ assert( p==0 || sqlite3_mutex_held(p->db->mutex) ); - return (p && (p->inTrans==TRANS_WRITE)); + return p ? p->inTrans : 0; } #ifndef SQLITE_OMIT_WAL /* ** Run a checkpoint on the Btree passed as the first argument. ** -** Return SQLITE_LOCKED if this or any other connection has an open +** Return SQLITE_LOCKED if this or any other connection has an open ** transaction on the shared-cache the argument Btree is connected to. ** ** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. @@ -67207,7 +81910,7 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int * if( pBt->inTransaction!=TRANS_NONE ){ rc = SQLITE_LOCKED; }else{ - rc = sqlite3PagerCheckpoint(pBt->pPager, eMode, pnLog, pnCkpt); + rc = sqlite3PagerCheckpoint(pBt->pPager, p->db, eMode, pnLog, pnCkpt); } sqlite3BtreeLeave(p); } @@ -67216,14 +81919,8 @@ SQLITE_PRIVATE int sqlite3BtreeCheckpoint(Btree *p, int eMode, int *pnLog, int * #endif /* -** Return non-zero if a read (or write) transaction is active. +** Return true if there is currently a backup running on Btree p. */ -SQLITE_PRIVATE int sqlite3BtreeIsInReadTrans(Btree *p){ - assert( p ); - assert( sqlite3_mutex_held(p->db->mutex) ); - return p->inTrans!=TRANS_NONE; -} - SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ assert( p ); assert( sqlite3_mutex_held(p->db->mutex) ); @@ -67233,20 +81930,20 @@ SQLITE_PRIVATE int sqlite3BtreeIsInBackup(Btree *p){ /* ** This function returns a pointer to a blob of memory associated with ** a single shared-btree. The memory is used by client code for its own -** purposes (for example, to store a high-level schema associated with +** purposes (for example, to store a high-level schema associated with ** the shared-btree). The btree layer manages reference counting issues. ** ** The first time this is called on a shared-btree, nBytes bytes of memory -** are allocated, zeroed, and returned to the caller. For each subsequent +** are allocated, zeroed, and returned to the caller. For each subsequent ** call the nBytes parameter is ignored and a pointer to the same blob -** of memory returned. +** of memory returned. ** ** If the nBytes parameter is 0 and the blob of memory has not yet been ** allocated, a null pointer is returned. If the blob has already been ** allocated, it is returned as normal. ** -** Just before the shared-btree is closed, the function passed as the -** xFree argument when the memory allocation was made is invoked on the +** Just before the shared-btree is closed, the function passed as the +** xFree argument when the memory allocation was made is invoked on the ** blob of allocated memory. The xFree function should not call sqlite3_free() ** on the memory, the btree layer does that. */ @@ -67262,15 +81959,15 @@ SQLITE_PRIVATE void *sqlite3BtreeSchema(Btree *p, int nBytes, void(*xFree)(void } /* -** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared -** btree as the argument handle holds an exclusive lock on the -** sqlite_master table. Otherwise SQLITE_OK. +** Return SQLITE_LOCKED_SHAREDCACHE if another user of the same shared +** btree as the argument handle holds an exclusive lock on the +** sqlite_schema table. Otherwise SQLITE_OK. */ SQLITE_PRIVATE int sqlite3BtreeSchemaLocked(Btree *p){ int rc; assert( sqlite3_mutex_held(p->db->mutex) ); sqlite3BtreeEnter(p); - rc = querySharedCacheTableLock(p, MASTER_ROOT, READ_LOCK); + rc = querySharedCacheTableLock(p, SCHEMA_ROOT, READ_LOCK); assert( rc==SQLITE_OK || rc==SQLITE_LOCKED_SHAREDCACHE ); sqlite3BtreeLeave(p); return rc; @@ -67304,11 +82001,11 @@ SQLITE_PRIVATE int sqlite3BtreeLockTable(Btree *p, int iTab, u8 isWriteLock){ #ifndef SQLITE_OMIT_INCRBLOB /* -** Argument pCsr must be a cursor opened for writing on an -** INTKEY table currently pointing at a valid table entry. +** Argument pCsr must be a cursor opened for writing on an +** INTKEY table currently pointing at a valid table entry. ** This function modifies the data stored as part of that entry. ** -** Only the data content may only be modified, it is not possible to +** Only the data content may only be modified, it is not possible to ** change the length of the data stored. If this function is called with ** parameters that attempt to write past the end of the existing data, ** no modifications are made and SQLITE_CORRUPT is returned. @@ -67339,7 +82036,7 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void VVA_ONLY(rc =) saveAllCursors(pCsr->pBt, pCsr->pgnoRoot, pCsr); assert( rc==SQLITE_OK ); - /* Check some assumptions: + /* Check some assumptions: ** (a) the cursor is open for writing, ** (b) there is a read/write transaction open, ** (c) the connection holds a write-lock on the table (if required), @@ -67353,12 +82050,12 @@ SQLITE_PRIVATE int sqlite3BtreePutData(BtCursor *pCsr, u32 offset, u32 amt, void && pCsr->pBt->inTransaction==TRANS_WRITE ); assert( hasSharedCacheTableLock(pCsr->pBtree, pCsr->pgnoRoot, 0, 2) ); assert( !hasReadConflicts(pCsr->pBtree, pCsr->pgnoRoot) ); - assert( pCsr->apPage[pCsr->iPage]->intKey ); + assert( pCsr->pPage->intKey ); return accessPayload(pCsr, offset, amt, (unsigned char *)z, 1); } -/* +/* ** Mark this cursor as an incremental blob cursor. */ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ @@ -67368,14 +82065,14 @@ SQLITE_PRIVATE void sqlite3BtreeIncrblobCursor(BtCursor *pCur){ #endif /* -** Set both the "read version" (single byte at byte offset 18) and +** Set both the "read version" (single byte at byte offset 18) and ** "write version" (single byte at byte offset 19) fields in the database ** header to iVersion. */ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ BtShared *pBt = pBtree->pBt; int rc; /* Return code */ - + assert( iVersion==1 || iVersion==2 ); /* If setting the version fields to 1, do not automatically open the @@ -67384,11 +82081,11 @@ SQLITE_PRIVATE int sqlite3BtreeSetVersion(Btree *pBtree, int iVersion){ pBt->btsFlags &= ~BTS_NO_WAL; if( iVersion==1 ) pBt->btsFlags |= BTS_NO_WAL; - rc = sqlite3BtreeBeginTrans(pBtree, 0); + rc = sqlite3BtreeBeginTrans(pBtree, 0, 0); if( rc==SQLITE_OK ){ u8 *aData = pBt->pPage1->aData; if( aData[18]!=(u8)iVersion || aData[19]!=(u8)iVersion ){ - rc = sqlite3BtreeBeginTrans(pBtree, 2); + rc = sqlite3BtreeBeginTrans(pBtree, 2, 0); if( rc==SQLITE_OK ){ rc = sqlite3PagerWrite(pBt->pPage1->pDbPage); if( rc==SQLITE_OK ){ @@ -67423,6 +82120,17 @@ SQLITE_PRIVATE int sqlite3BtreeIsReadonly(Btree *p){ */ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); } +/* +** If no transaction is active and the database is not a temp-db, clear +** the in-memory pager cache. +*/ +SQLITE_PRIVATE void sqlite3BtreeClearCache(Btree *p){ + BtShared *pBt = p->pBt; + if( pBt->inTransaction==TRANS_NONE ){ + sqlite3PagerClearCache(pBt->pPager); + } +} + #if !defined(SQLITE_OMIT_SHARED_CACHE) /* ** Return true if the Btree passed as the only argument is sharable. @@ -67430,6 +82138,16 @@ SQLITE_PRIVATE int sqlite3HeaderSizeBtree(void){ return ROUND8(sizeof(MemPage)); SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ return p->sharable; } + +/* +** Return the number of connections to the BtShared object accessed by +** the Btree handle passed as the only argument. For private caches +** this is always 1. For shared caches it may be 1 or greater. +*/ +SQLITE_PRIVATE int sqlite3BtreeConnectionCount(Btree *p){ + testcase( p->sharable ); + return p->pBt->nRef; +} #endif /************** End of btree.c ***********************************************/ @@ -67445,7 +82163,7 @@ SQLITE_PRIVATE int sqlite3BtreeSharable(Btree *p){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** This file contains the implementation of the sqlite3_backup_XXX() +** This file contains the implementation of the sqlite3_backup_XXX() ** API functions and the related features. */ /* #include "sqliteInt.h" */ @@ -67482,15 +82200,15 @@ struct sqlite3_backup { ** Once it has been created using backup_init(), a single sqlite3_backup ** structure may be accessed via two groups of thread-safe entry points: ** -** * Via the sqlite3_backup_XXX() API function backup_step() and +** * Via the sqlite3_backup_XXX() API function backup_step() and ** backup_finish(). Both these functions obtain the source database -** handle mutex and the mutex associated with the source BtShared +** handle mutex and the mutex associated with the source BtShared ** structure, in that order. ** ** * Via the BackupUpdate() and BackupRestart() functions, which are ** invoked by the pager layer to report various state changes in ** the page cache associated with the source database. The mutex -** associated with the source database BtShared structure will always +** associated with the source database BtShared structure will always ** be held when either of these functions are invoked. ** ** The other sqlite3_backup_XXX() API functions, backup_remaining() and @@ -67511,30 +82229,23 @@ struct sqlite3_backup { ** in connection handle pDb. If such a database cannot be found, return ** a NULL pointer and write an error message to pErrorDb. ** -** If the "temp" database is requested, it may need to be opened by this -** function. If an error occurs while doing so, return 0 and write an +** If the "temp" database is requested, it may need to be opened by this +** function. If an error occurs while doing so, return 0 and write an ** error message to pErrorDb. */ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ int i = sqlite3FindDbName(pDb, zDb); if( i==1 ){ - Parse *pParse; + Parse sParse; int rc = 0; - pParse = sqlite3StackAllocZero(pErrorDb, sizeof(*pParse)); - if( pParse==0 ){ - sqlite3ErrorWithMsg(pErrorDb, SQLITE_NOMEM, "out of memory"); - rc = SQLITE_NOMEM_BKPT; - }else{ - pParse->db = pDb; - if( sqlite3OpenTempDatabase(pParse) ){ - sqlite3ErrorWithMsg(pErrorDb, pParse->rc, "%s", pParse->zErrMsg); - rc = SQLITE_ERROR; - } - sqlite3DbFree(pErrorDb, pParse->zErrMsg); - sqlite3ParserReset(pParse); - sqlite3StackFree(pErrorDb, pParse); + sqlite3ParseObjectInit(&sParse,pDb); + if( sqlite3OpenTempDatabase(&sParse) ){ + sqlite3ErrorWithMsg(pErrorDb, sParse.rc, "%s", sParse.zErrMsg); + rc = SQLITE_ERROR; } + sqlite3DbFree(pErrorDb, sParse.zErrMsg); + sqlite3ParseObjectReset(&sParse); if( rc ){ return 0; } @@ -67554,18 +82265,18 @@ static Btree *findBtree(sqlite3 *pErrorDb, sqlite3 *pDb, const char *zDb){ */ static int setDestPgsz(sqlite3_backup *p){ int rc; - rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),-1,0); + rc = sqlite3BtreeSetPageSize(p->pDest,sqlite3BtreeGetPageSize(p->pSrc),0,0); return rc; } /* ** Check that there is no open read-transaction on the b-tree passed as the ** second argument. If there is not, return SQLITE_OK. Otherwise, if there -** is an open read-transaction, return SQLITE_ERROR and leave an error +** is an open read-transaction, return SQLITE_ERROR and leave an error ** message in database handle db. */ static int checkReadTransaction(sqlite3 *db, Btree *p){ - if( sqlite3BtreeIsInReadTrans(p) ){ + if( sqlite3BtreeTxnState(p)!=SQLITE_TXN_NONE ){ sqlite3ErrorWithMsg(db, SQLITE_ERROR, "destination database is in use"); return SQLITE_ERROR; } @@ -67580,7 +82291,7 @@ static int checkReadTransaction(sqlite3 *db, Btree *p){ ** If an error occurs, NULL is returned and an error code and error message ** stored in database handle pDestDb. */ -SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( +SQLITE_API sqlite3_backup *sqlite3_backup_init( sqlite3* pDestDb, /* Database to write to */ const char *zDestDb, /* Name of database within pDestDb */ sqlite3* pSrcDb, /* Database connection to read from */ @@ -67631,14 +82342,13 @@ SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( p->iNext = 1; p->isAttached = 0; - if( 0==p->pSrc || 0==p->pDest - || setDestPgsz(p)==SQLITE_NOMEM - || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK + if( 0==p->pSrc || 0==p->pDest + || checkReadTransaction(pDestDb, p->pDest)!=SQLITE_OK ){ /* One (or both) of the named databases did not exist or an OOM ** error was hit. Or there is a transaction open on the destination - ** database. The error has already been written into the pDestDb - ** handle. All that is left to do here is free the sqlite3_backup + ** database. The error has already been written into the pDestDb + ** handle. All that is left to do here is free the sqlite3_backup ** structure. */ sqlite3_free(p); p = 0; @@ -67654,7 +82364,7 @@ SQLITE_API sqlite3_backup *SQLITE_STDCALL sqlite3_backup_init( } /* -** Argument rc is an SQLite error code. Return true if this error is +** Argument rc is an SQLite error code. Return true if this error is ** considered fatal if encountered during a backup operation. All errors ** are considered fatal except for SQLITE_BUSY and SQLITE_LOCKED. */ @@ -67663,8 +82373,8 @@ static int isFatalError(int rc){ } /* -** Parameter zSrcData points to a buffer containing the data for -** page iSrcPg from the source database. Copy this data into the +** Parameter zSrcData points to a buffer containing the data for +** page iSrcPg from the source database. Copy this data into the ** destination database. */ static int backupOnePage( @@ -67678,13 +82388,6 @@ static int backupOnePage( int nDestPgsz = sqlite3BtreeGetPageSize(p->pDest); const int nCopy = MIN(nSrcPgsz, nDestPgsz); const i64 iEnd = (i64)iSrcPg*(i64)nSrcPgsz; -#ifdef SQLITE_HAS_CODEC - /* Use BtreeGetReserveNoMutex() for the source b-tree, as although it is - ** guaranteed that the shared-mutex is held by this thread, handle - ** p->pSrc may not actually be the owner. */ - int nSrcReserve = sqlite3BtreeGetReserveNoMutex(p->pSrc); - int nDestReserve = sqlite3BtreeGetOptimalReserve(p->pDest); -#endif int rc = SQLITE_OK; i64 iOff; @@ -67693,35 +82396,9 @@ static int backupOnePage( assert( !isFatalError(p->rc) ); assert( iSrcPg!=PENDING_BYTE_PAGE(p->pSrc->pBt) ); assert( zSrcData ); + assert( nSrcPgsz==nDestPgsz || sqlite3PagerIsMemdb(pDestPager)==0 ); - /* Catch the case where the destination is an in-memory database and the - ** page sizes of the source and destination differ. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerIsMemdb(pDestPager) ){ - rc = SQLITE_READONLY; - } - -#ifdef SQLITE_HAS_CODEC - /* Backup is not possible if the page size of the destination is changing - ** and a codec is in use. - */ - if( nSrcPgsz!=nDestPgsz && sqlite3PagerGetCodec(pDestPager)!=0 ){ - rc = SQLITE_READONLY; - } - - /* Backup is not possible if the number of bytes of reserve space differ - ** between source and destination. If there is a difference, try to - ** fix the destination to agree with the source. If that is not possible, - ** then the backup cannot proceed. - */ - if( nSrcReserve!=nDestReserve ){ - u32 newPgsz = nSrcPgsz; - rc = sqlite3PagerSetPagesize(pDestPager, &newPgsz, nSrcReserve); - if( rc==SQLITE_OK && newPgsz!=nSrcPgsz ) rc = SQLITE_READONLY; - } -#endif - - /* This loop runs once for each destination page spanned by the source + /* This loop runs once for each destination page spanned by the source ** page. For each iteration, variable iOff is set to the byte offset ** of the destination page. */ @@ -67740,7 +82417,7 @@ static int backupOnePage( ** Then clear the Btree layer MemPage.isInit flag. Both this module ** and the pager code use this trick (clearing the first byte ** of the page 'extra' space to invalidate the Btree layers - ** cached parse of the page). MemPage.isInit is marked + ** cached parse of the page). MemPage.isInit is marked ** "MUST BE FIRST" for this purpose. */ memcpy(zOut, zIn, nCopy); @@ -67760,7 +82437,7 @@ static int backupOnePage( ** exactly iSize bytes. If pFile is not larger than iSize bytes, then ** this function is a no-op. ** -** Return SQLITE_OK if everything is successful, or an SQLite error +** Return SQLITE_OK if everything is successful, or an SQLite error ** code if an error occurs. */ static int backupTruncateFile(sqlite3_file *pFile, i64 iSize){ @@ -67788,7 +82465,7 @@ static void attachBackupObject(sqlite3_backup *p){ /* ** Copy nPage pages from the source b-tree to the destination. */ -SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage){ int rc; int destMode; /* Destination journal mode */ int pgszSrc = 0; /* Source page size */ @@ -67820,32 +82497,45 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ rc = SQLITE_OK; } - /* Lock the destination database, if it is not locked already. */ - if( SQLITE_OK==rc && p->bDestLocked==0 - && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2)) - ){ - p->bDestLocked = 1; - sqlite3BtreeGetMeta(p->pDest, BTREE_SCHEMA_VERSION, &p->iDestSchema); - } - /* If there is no open read-transaction on the source database, open ** one now. If a transaction is opened here, then it will be closed ** before this function exits. */ - if( rc==SQLITE_OK && 0==sqlite3BtreeIsInReadTrans(p->pSrc) ){ - rc = sqlite3BtreeBeginTrans(p->pSrc, 0); + if( rc==SQLITE_OK && SQLITE_TXN_NONE==sqlite3BtreeTxnState(p->pSrc) ){ + rc = sqlite3BtreeBeginTrans(p->pSrc, 0, 0); bCloseTrans = 1; } + /* If the destination database has not yet been locked (i.e. if this + ** is the first call to backup_step() for the current backup operation), + ** try to set its page size to the same as the source database. This + ** is especially important on ZipVFS systems, as in that case it is + ** not possible to create a database file that uses one page size by + ** writing to it with another. */ + if( p->bDestLocked==0 && rc==SQLITE_OK && setDestPgsz(p)==SQLITE_NOMEM ){ + rc = SQLITE_NOMEM; + } + + /* Lock the destination database, if it is not locked already. */ + if( SQLITE_OK==rc && p->bDestLocked==0 + && SQLITE_OK==(rc = sqlite3BtreeBeginTrans(p->pDest, 2, + (int*)&p->iDestSchema)) + ){ + p->bDestLocked = 1; + } + /* Do not allow backup if the destination database is in WAL mode ** and the page sizes are different between source and destination */ pgszSrc = sqlite3BtreeGetPageSize(p->pSrc); pgszDest = sqlite3BtreeGetPageSize(p->pDest); destMode = sqlite3PagerGetJournalMode(sqlite3BtreePager(p->pDest)); - if( SQLITE_OK==rc && destMode==PAGER_JOURNALMODE_WAL && pgszSrc!=pgszDest ){ + if( SQLITE_OK==rc + && (destMode==PAGER_JOURNALMODE_WAL || sqlite3PagerIsMemdb(pDestPager)) + && pgszSrc!=pgszDest + ){ rc = SQLITE_READONLY; } - + /* Now that there is a read-lock on the source database, query the ** source pager for the number of pages in the database. */ @@ -67872,7 +82562,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ attachBackupObject(p); } } - + /* Update the schema version field in the destination database. This ** is to make sure that the schema-version really does change in ** the case where the source and destination databases have the @@ -67898,12 +82588,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ int nDestTruncate; /* Set nDestTruncate to the final number of pages in the destination ** database. The complication here is that the destination page - ** size may be different to the source page size. + ** size may be different to the source page size. ** - ** If the source page size is smaller than the destination page size, + ** If the source page size is smaller than the destination page size, ** round up. In this case the call to sqlite3OsTruncate() below will ** fix the size of the file. However it is important to call - ** sqlite3PagerTruncateImage() here so that any pages in the + ** sqlite3PagerTruncateImage() here so that any pages in the ** destination file that lie beyond the nDestTruncate page mark are ** journalled by PagerCommitPhaseOne() before they are destroyed ** by the file truncation. @@ -67927,7 +82617,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ ** ** * The destination may need to be truncated, and ** - ** * Data stored on the pages immediately following the + ** * Data stored on the pages immediately following the ** pending-byte page in the source database may need to be ** copied into the destination database. */ @@ -67939,7 +82629,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ i64 iEnd; assert( pFile ); - assert( nDestTruncate==0 + assert( nDestTruncate==0 || (i64)nDestTruncate*(i64)pgszDest >= iSize || ( nDestTruncate==(int)(PENDING_BYTE_PAGE(p->pDest->pBt)-1) && iSize>=PENDING_BYTE && iSize<=PENDING_BYTE+pgszDest @@ -67949,7 +82639,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ ** database has been stored in the journal for pDestPager and the ** journal synced to disk. So at this point we may safely modify ** the database file in any way, knowing that if a power failure - ** occurs, the original database will be reconstructed from the + ** occurs, the original database will be reconstructed from the ** journal file. */ sqlite3PagerPagecount(pDestPager, &nDstPage); for(iPg=nDestTruncate; rc==SQLITE_OK && iPg<=(Pgno)nDstPage; iPg++){ @@ -67969,8 +82659,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ /* Write the extra pages and truncate the database file as required */ iEnd = MIN(PENDING_BYTE + pgszDest, iSize); for( - iOff=PENDING_BYTE+pgszSrc; - rc==SQLITE_OK && iOffpDest, 0)) @@ -68003,7 +82693,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ } } } - + /* If bCloseTrans is true, then this function opened a read transaction ** on the source database. Close the read transaction here. There is ** no need to check the return values of the btree methods here, as @@ -68015,7 +82705,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ TESTONLY( rc2 |= ) sqlite3BtreeCommitPhaseTwo(p->pSrc, 0); assert( rc2==SQLITE_OK ); } - + if( rc==SQLITE_IOERR_NOMEM ){ rc = SQLITE_NOMEM_BKPT; } @@ -68032,7 +82722,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_step(sqlite3_backup *p, int nPage){ /* ** Release all resources associated with an sqlite3_backup* handle. */ -SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p){ +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p){ sqlite3_backup **pp; /* Ptr to head of pagers backup list */ sqlite3 *pSrcDb; /* Source database connection */ int rc; /* Value to return */ @@ -68052,8 +82742,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p){ } if( p->isAttached ){ pp = sqlite3PagerBackupPtr(sqlite3BtreePager(p->pSrc)); + assert( pp!=0 ); while( *pp!=p ){ pp = &(*pp)->pNext; + assert( pp!=0 ); } *pp = p->pNext; } @@ -68084,7 +82776,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_finish(sqlite3_backup *p){ ** Return the number of pages still to be backed up as of the most recent ** call to sqlite3_backup_step(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p){ +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p){ #ifdef SQLITE_ENABLE_API_ARMOR if( p==0 ){ (void)SQLITE_MISUSE_BKPT; @@ -68095,10 +82787,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_remaining(sqlite3_backup *p){ } /* -** Return the total number of pages in the source database as of the most +** Return the total number of pages in the source database as of the most ** recent call to sqlite3_backup_step(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p){ +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p){ #ifdef SQLITE_ENABLE_API_ARMOR if( p==0 ){ (void)SQLITE_MISUSE_BKPT; @@ -68110,7 +82802,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_backup_pagecount(sqlite3_backup *p){ /* ** This function is called after the contents of page iPage of the -** source database have been modified. If page iPage has already been +** source database have been modified. If page iPage has already been ** copied into the destination database, then the data written to the ** destination is now invalidated. The destination copy of iPage needs ** to be updated with the new data before the backup operation is @@ -68153,7 +82845,7 @@ SQLITE_PRIVATE void sqlite3BackupUpdate(sqlite3_backup *pBackup, Pgno iPage, con ** Restart the backup process. This is called when the pager layer ** detects that the database has been modified by an external database ** connection. In this case there is no way of knowing which of the -** pages that have been copied into the destination database are still +** pages that have been copied into the destination database are still ** valid and which are not, so the entire process needs to be restarted. ** ** It is assumed that the mutex associated with the BtShared object @@ -68173,8 +82865,8 @@ SQLITE_PRIVATE void sqlite3BackupRestart(sqlite3_backup *pBackup){ ** Copy the complete content of pBtFrom into pBtTo. A transaction ** must be active for both files. ** -** The size of file pTo may be reduced by this operation. If anything -** goes wrong, the transaction on pTo is rolled back. If successful, the +** The size of file pTo may be reduced by this operation. If anything +** goes wrong, the transaction on pTo is rolled back. If successful, the ** transaction is committed before returning. */ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ @@ -68184,7 +82876,7 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ sqlite3BtreeEnter(pTo); sqlite3BtreeEnter(pFrom); - assert( sqlite3BtreeIsInTrans(pTo) ); + assert( sqlite3BtreeTxnState(pTo)==SQLITE_TXN_WRITE ); pFd = sqlite3PagerFile(sqlite3BtreePager(pTo)); if( pFd->pMethods ){ i64 nByte = sqlite3BtreeGetPageSize(pFrom)*(i64)sqlite3BtreeLastPage(pFrom); @@ -68204,19 +82896,15 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ b.pDest = pTo; b.iNext = 1; -#ifdef SQLITE_HAS_CODEC - sqlite3PagerAlignReserve(sqlite3BtreePager(pTo), sqlite3BtreePager(pFrom)); -#endif - /* 0x7FFFFFFF is the hard limit for the number of pages in a database ** file. By passing this as the number of pages to copy to - ** sqlite3_backup_step(), we can guarantee that the copy finishes + ** sqlite3_backup_step(), we can guarantee that the copy finishes ** within a single call (unless an error occurs). The assert() statement - ** checks this assumption - (p->rc) should be set to either SQLITE_DONE - ** or an error code. - */ + ** checks this assumption - (p->rc) should be set to either SQLITE_DONE + ** or an error code. */ sqlite3_backup_step(&b, 0x7FFFFFFF); assert( b.rc!=SQLITE_OK ); + rc = sqlite3_backup_finish(&b); if( rc==SQLITE_OK ){ pTo->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; @@ -68224,7 +82912,7 @@ SQLITE_PRIVATE int sqlite3BtreeCopyFile(Btree *pTo, Btree *pFrom){ sqlite3PagerClearCache(sqlite3BtreePager(b.pDest)); } - assert( sqlite3BtreeIsInTrans(pTo)==0 ); + assert( sqlite3BtreeTxnState(pTo)!=SQLITE_TXN_WRITE ); copy_finished: sqlite3BtreeLeave(pFrom); sqlite3BtreeLeave(pTo); @@ -68254,6 +82942,11 @@ copy_finished: /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* True if X is a power of two. 0 is considered a power of two here. +** In other words, return true if X has at most one bit set. +*/ +#define ISPOWEROF2(X) (((X)&((X)-1))==0) + #ifdef SQLITE_DEBUG /* ** Check invariants on a Mem object. @@ -68262,8 +82955,8 @@ copy_finished: ** this: assert( sqlite3VdbeCheckMemInvariants(pMem) ); */ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ - /* If MEM_Dyn is set then Mem.xDel!=0. - ** Mem.xDel is might not be initialized if MEM_Dyn is clear. + /* If MEM_Dyn is set then Mem.xDel!=0. + ** Mem.xDel might not be initialized if MEM_Dyn is clear. */ assert( (p->flags & MEM_Dyn)==0 || p->xDel!=0 ); @@ -68273,12 +82966,42 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ** That saves a few cycles in inner loops. */ assert( (p->flags & MEM_Dyn)==0 || p->szMalloc==0 ); - /* Cannot be both MEM_Int and MEM_Real at the same time */ - assert( (p->flags & (MEM_Int|MEM_Real))!=(MEM_Int|MEM_Real) ); + /* Cannot have more than one of MEM_Int, MEM_Real, or MEM_IntReal */ + assert( ISPOWEROF2(p->flags & (MEM_Int|MEM_Real|MEM_IntReal)) ); + + if( p->flags & MEM_Null ){ + /* Cannot be both MEM_Null and some other type */ + assert( (p->flags & (MEM_Int|MEM_Real|MEM_Str|MEM_Blob|MEM_Agg))==0 ); + + /* If MEM_Null is set, then either the value is a pure NULL (the usual + ** case) or it is a pointer set using sqlite3_bind_pointer() or + ** sqlite3_result_pointer(). If a pointer, then MEM_Term must also be + ** set. + */ + if( (p->flags & (MEM_Term|MEM_Subtype))==(MEM_Term|MEM_Subtype) ){ + /* This is a pointer type. There may be a flag to indicate what to + ** do with the pointer. */ + assert( ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + + ((p->flags&MEM_Static)!=0 ? 1 : 0) <= 1 ); + + /* No other bits set */ + assert( (p->flags & ~(MEM_Null|MEM_Term|MEM_Subtype|MEM_FromBind + |MEM_Dyn|MEM_Ephem|MEM_Static))==0 ); + }else{ + /* A pure NULL might have other flags, such as MEM_Static, MEM_Dyn, + ** MEM_Ephem, MEM_Cleared, or MEM_Subtype */ + } + }else{ + /* The MEM_Cleared bit is only allowed on NULLs */ + assert( (p->flags & MEM_Cleared)==0 ); + } /* The szMalloc field holds the correct memory allocation size */ assert( p->szMalloc==0 - || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc) ); + || (p->flags==MEM_Undefined + && p->szMalloc<=sqlite3DbMallocSize(p->db,p->zMalloc)) + || p->szMalloc==sqlite3DbMallocSize(p->db,p->zMalloc)); /* If p holds a string or blob, the Mem.z must point to exactly ** one of the following: @@ -68289,7 +83012,7 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ ** (4) A static string or blob */ if( (p->flags & (MEM_Str|MEM_Blob)) && p->n>0 ){ - assert( + assert( ((p->szMalloc>0 && p->z==p->zMalloc)? 1 : 0) + ((p->flags&MEM_Dyn)!=0 ? 1 : 0) + ((p->flags&MEM_Ephem)!=0 ? 1 : 0) + @@ -68300,6 +83023,93 @@ SQLITE_PRIVATE int sqlite3VdbeCheckMemInvariants(Mem *p){ } #endif +/* +** Render a Mem object which is one of MEM_Int, MEM_Real, or MEM_IntReal +** into a buffer. +*/ +static void vdbeMemRenderNum(int sz, char *zBuf, Mem *p){ + StrAccum acc; + assert( p->flags & (MEM_Int|MEM_Real|MEM_IntReal) ); + assert( sz>22 ); + if( p->flags & MEM_Int ){ +#if GCC_VERSION>=7000000 + /* Work-around for GCC bug + ** https://gcc.gnu.org/bugzilla/show_bug.cgi?id=96270 */ + i64 x; + assert( (p->flags&MEM_Int)*2==sizeof(x) ); + memcpy(&x, (char*)&p->u, (p->flags&MEM_Int)*2); + p->n = sqlite3Int64ToText(x, zBuf); +#else + p->n = sqlite3Int64ToText(p->u.i, zBuf); +#endif + }else{ + sqlite3StrAccumInit(&acc, 0, zBuf, sz, 0); + sqlite3_str_appendf(&acc, "%!.15g", + (p->flags & MEM_IntReal)!=0 ? (double)p->u.i : p->u.r); + assert( acc.zText==zBuf && acc.mxAlloc<=0 ); + zBuf[acc.nChar] = 0; /* Fast version of sqlite3StrAccumFinish(&acc) */ + p->n = acc.nChar; + } +} + +#ifdef SQLITE_DEBUG +/* +** Validity checks on pMem. pMem holds a string. +** +** (1) Check that string value of pMem agrees with its integer or real value. +** (2) Check that the string is correctly zero terminated +** +** A single int or real value always converts to the same strings. But +** many different strings can be converted into the same int or real. +** If a table contains a numeric value and an index is based on the +** corresponding string value, then it is important that the string be +** derived from the numeric value, not the other way around, to ensure +** that the index and table are consistent. See ticket +** https://www.sqlite.org/src/info/343634942dd54ab (2018-01-31) for +** an example. +** +** This routine looks at pMem to verify that if it has both a numeric +** representation and a string representation then the string rep has +** been derived from the numeric and not the other way around. It returns +** true if everything is ok and false if there is a problem. +** +** This routine is for use inside of assert() statements only. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemValidStrRep(Mem *p){ + Mem tmp; + char zBuf[100]; + char *z; + int i, j, incr; + if( (p->flags & MEM_Str)==0 ) return 1; + if( p->db && p->db->mallocFailed ) return 1; + if( p->flags & MEM_Term ){ + /* Insure that the string is properly zero-terminated. Pay particular + ** attention to the case where p->n is odd */ + if( p->szMalloc>0 && p->z==p->zMalloc ){ + assert( p->enc==SQLITE_UTF8 || p->szMalloc >= ((p->n+1)&~1)+2 ); + assert( p->enc!=SQLITE_UTF8 || p->szMalloc >= p->n+1 ); + } + assert( p->z[p->n]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[(p->n+1)&~1]==0 ); + assert( p->enc==SQLITE_UTF8 || p->z[((p->n+1)&~1)+1]==0 ); + } + if( (p->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ) return 1; + memcpy(&tmp, p, sizeof(tmp)); + vdbeMemRenderNum(sizeof(zBuf), zBuf, &tmp); + z = p->z; + i = j = 0; + incr = 1; + if( p->enc!=SQLITE_UTF8 ){ + incr = 2; + if( p->enc==SQLITE_UTF16BE ) z++; + } + while( zBuf[j] ){ + if( zBuf[j++]!=z[i] ) return 0; + i += incr; + } + return 1; +} +#endif /* SQLITE_DEBUG */ /* ** If pMem is an object with a valid string representation, this routine @@ -68318,10 +83128,15 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ #ifndef SQLITE_OMIT_UTF16 int rc; #endif - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( pMem!=0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( desiredEnc==SQLITE_UTF8 || desiredEnc==SQLITE_UTF16LE || desiredEnc==SQLITE_UTF16BE ); - if( !(pMem->flags&MEM_Str) || pMem->enc==desiredEnc ){ + if( !(pMem->flags&MEM_Str) ){ + pMem->enc = desiredEnc; + return SQLITE_OK; + } + if( pMem->enc==desiredEnc ){ return SQLITE_OK; } assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); @@ -68341,8 +83156,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ } /* -** Make sure pMem->z points to a writable allocation of at least -** min(n,32) bytes. +** Make sure pMem->z points to a writable allocation of at least n bytes. ** ** If the bPreserve argument is true, then copy of the content of ** pMem->z into the new allocation. pMem must be either a string or @@ -68351,7 +83165,7 @@ SQLITE_PRIVATE int sqlite3VdbeChangeEncoding(Mem *pMem, int desiredEnc){ */ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPreserve){ assert( sqlite3VdbeCheckMemInvariants(pMem) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); testcase( pMem->db==0 ); /* If the bPreserve flag is set to true, then the memory cell must already @@ -68360,27 +83174,33 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre testcase( bPreserve && pMem->z==0 ); assert( pMem->szMalloc==0 - || pMem->szMalloc==sqlite3DbMallocSize(pMem->db, pMem->zMalloc) ); - if( pMem->szMallocszMalloc>0 && pMem->z==pMem->zMalloc ){ + || (pMem->flags==MEM_Undefined + && pMem->szMalloc<=sqlite3DbMallocSize(pMem->db,pMem->zMalloc)) + || pMem->szMalloc==sqlite3DbMallocSize(pMem->db,pMem->zMalloc)); + if( pMem->szMalloc>0 && bPreserve && pMem->z==pMem->zMalloc ){ + if( pMem->db ){ pMem->z = pMem->zMalloc = sqlite3DbReallocOrFree(pMem->db, pMem->z, n); - bPreserve = 0; }else{ - if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); - pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); - } - if( pMem->zMalloc==0 ){ - sqlite3VdbeMemSetNull(pMem); - pMem->z = 0; - pMem->szMalloc = 0; - return SQLITE_NOMEM_BKPT; - }else{ - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + pMem->zMalloc = sqlite3Realloc(pMem->z, n); + if( pMem->zMalloc==0 ) sqlite3_free(pMem->z); + pMem->z = pMem->zMalloc; } + bPreserve = 0; + }else{ + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, n); + } + if( pMem->zMalloc==0 ){ + sqlite3VdbeMemSetNull(pMem); + pMem->z = 0; + pMem->szMalloc = 0; + return SQLITE_NOMEM_BKPT; + }else{ + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); } - if( bPreserve && pMem->z && pMem->z!=pMem->zMalloc ){ + if( bPreserve && pMem->z ){ + assert( pMem->z!=pMem->zMalloc ); memcpy(pMem->zMalloc, pMem->z, pMem->n); } if( (pMem->flags&MEM_Dyn)!=0 ){ @@ -68400,21 +83220,75 @@ SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3VdbeMemGrow(Mem *pMem, int n, int bPre ** ** Any prior string or blob content in the pMem object may be discarded. ** The pMem->xDel destructor is called, if it exists. Though MEM_Str -** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, and MEM_Null -** values are preserved. +** and MEM_Blob values may be discarded, MEM_Int, MEM_Real, MEM_IntReal, +** and MEM_Null values are preserved. ** ** Return SQLITE_OK on success or an error code (probably SQLITE_NOMEM) ** if unable to complete the resizing. */ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ - assert( szNew>0 ); + assert( CORRUPT_DB || szNew>0 ); assert( (pMem->flags & MEM_Dyn)==0 || pMem->szMalloc==0 ); if( pMem->szMallocflags & MEM_Dyn)==0 ); pMem->z = pMem->zMalloc; - pMem->flags &= (MEM_Null|MEM_Int|MEM_Real); + pMem->flags &= (MEM_Null|MEM_Int|MEM_Real|MEM_IntReal); + return SQLITE_OK; +} + +/* +** If pMem is already a string, detect if it is a zero-terminated +** string, or make it into one if possible, and mark it as such. +** +** This is an optimization. Correct operation continues even if +** this routine is a no-op. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemZeroTerminateIfAble(Mem *pMem){ + if( (pMem->flags & (MEM_Str|MEM_Term|MEM_Ephem|MEM_Static))!=MEM_Str ){ + /* pMem must be a string, and it cannot be an ephemeral or static string */ + return; + } + if( pMem->enc!=SQLITE_UTF8 ) return; + if( NEVER(pMem->z==0) ) return; + if( pMem->flags & MEM_Dyn ){ + if( pMem->xDel==sqlite3_free + && sqlite3_msize(pMem->z) >= (u64)(pMem->n+1) + ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } + if( pMem->xDel==sqlite3RCStrUnref ){ + /* Blindly assume that all RCStr objects are zero-terminated */ + pMem->flags |= MEM_Term; + return; + } + }else if( pMem->szMalloc >= pMem->n+1 ){ + pMem->z[pMem->n] = 0; + pMem->flags |= MEM_Term; + return; + } +} + +/* +** It is already known that pMem contains an unterminated string. +** Add the zero terminator. +** +** Three bytes of zero are added. In this way, there is guaranteed +** to be a double-zero byte at an even byte boundary in order to +** terminate a UTF16 string, even if the initial size of the buffer +** is an odd number of bytes. +*/ +static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ + if( sqlite3VdbeMemGrow(pMem, pMem->n+3, 1) ){ + return SQLITE_NOMEM_BKPT; + } + pMem->z[pMem->n] = 0; + pMem->z[pMem->n+1] = 0; + pMem->z[pMem->n+2] = 0; + pMem->flags |= MEM_Term; return SQLITE_OK; } @@ -68425,18 +83299,15 @@ SQLITE_PRIVATE int sqlite3VdbeMemClearAndResize(Mem *pMem, int szNew){ ** Return SQLITE_OK on success or SQLITE_NOMEM if malloc fails. */ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ - int f; + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags&MEM_RowSet)==0 ); - ExpandBlob(pMem); - f = pMem->flags; - if( (f&(MEM_Str|MEM_Blob)) && (pMem->szMalloc==0 || pMem->z!=pMem->zMalloc) ){ - if( sqlite3VdbeMemGrow(pMem, pMem->n + 2, 1) ){ - return SQLITE_NOMEM_BKPT; + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + if( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ){ + if( ExpandBlob(pMem) ) return SQLITE_NOMEM; + if( pMem->szMalloc==0 || pMem->z!=pMem->zMalloc ){ + int rc = vdbeMemAddTerminator(pMem); + if( rc ) return rc; } - pMem->z[pMem->n] = 0; - pMem->z[pMem->n+1] = 0; - pMem->flags |= MEM_Term; } pMem->flags &= ~MEM_Ephem; #ifdef SQLITE_DEBUG @@ -68452,47 +83323,38 @@ SQLITE_PRIVATE int sqlite3VdbeMemMakeWriteable(Mem *pMem){ */ #ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE int sqlite3VdbeMemExpandBlob(Mem *pMem){ - if( pMem->flags & MEM_Zero ){ - int nByte; - assert( pMem->flags&MEM_Blob ); - assert( (pMem->flags&MEM_RowSet)==0 ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); + int nByte; + assert( pMem!=0 ); + assert( pMem->flags & MEM_Zero ); + assert( (pMem->flags&MEM_Blob)!=0 || MemNullNochng(pMem) ); + testcase( sqlite3_value_nochange(pMem) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - /* Set nByte to the number of bytes required to store the expanded blob. */ - nByte = pMem->n + pMem->u.nZero; - if( nByte<=0 ){ - nByte = 1; - } - if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ - return SQLITE_NOMEM_BKPT; - } - - memset(&pMem->z[pMem->n], 0, pMem->u.nZero); - pMem->n += pMem->u.nZero; - pMem->flags &= ~(MEM_Zero|MEM_Term); + /* Set nByte to the number of bytes required to store the expanded blob. */ + nByte = pMem->n + pMem->u.nZero; + if( nByte<=0 ){ + if( (pMem->flags & MEM_Blob)==0 ) return SQLITE_OK; + nByte = 1; } + if( sqlite3VdbeMemGrow(pMem, nByte, 1) ){ + return SQLITE_NOMEM_BKPT; + } + assert( pMem->z!=0 ); + assert( sqlite3DbMallocSize(pMem->db,pMem->z) >= nByte ); + + memset(&pMem->z[pMem->n], 0, pMem->u.nZero); + pMem->n += pMem->u.nZero; + pMem->flags &= ~(MEM_Zero|MEM_Term); return SQLITE_OK; } #endif -/* -** It is already known that pMem contains an unterminated string. -** Add the zero terminator. -*/ -static SQLITE_NOINLINE int vdbeMemAddTerminator(Mem *pMem){ - if( sqlite3VdbeMemGrow(pMem, pMem->n+2, 1) ){ - return SQLITE_NOMEM_BKPT; - } - pMem->z[pMem->n] = 0; - pMem->z[pMem->n+1] = 0; - pMem->flags |= MEM_Term; - return SQLITE_OK; -} - /* ** Make sure the given Mem is \u0000 terminated. */ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); testcase( (pMem->flags & (MEM_Term|MEM_Str))==(MEM_Term|MEM_Str) ); testcase( (pMem->flags & (MEM_Term|MEM_Str))==0 ); @@ -68504,12 +83366,12 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ } /* -** Add MEM_Str to the set of representations for the given Mem. Numbers -** are converted using sqlite3_snprintf(). Converting a BLOB to a string -** is a no-op. +** Add MEM_Str to the set of representations for the given Mem. This +** routine is only called if pMem is a number of some kind, not a NULL +** or a BLOB. ** -** Existing representations MEM_Int and MEM_Real are invalidated if -** bForce is true but are retained if bForce is false. +** Existing representations MEM_Int, MEM_Real, or MEM_IntReal are invalidated +** if bForce is true but are retained if bForce is false. ** ** A MEM_Null value will never be passed to this function. This function is ** used for converting values to text for returning to the user (i.e. via @@ -68518,37 +83380,28 @@ SQLITE_PRIVATE int sqlite3VdbeMemNulTerminate(Mem *pMem){ ** user and the latter is an internal programming error. */ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ - int fg = pMem->flags; const int nByte = 32; + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( !(fg&MEM_Zero) ); - assert( !(fg&(MEM_Str|MEM_Blob)) ); - assert( fg&(MEM_Int|MEM_Real) ); - assert( (pMem->flags&MEM_RowSet)==0 ); + assert( !(pMem->flags&MEM_Zero) ); + assert( !(pMem->flags&(MEM_Str|MEM_Blob)) ); + assert( pMem->flags&(MEM_Int|MEM_Real|MEM_IntReal) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( sqlite3VdbeMemClearAndResize(pMem, nByte) ){ + pMem->enc = 0; return SQLITE_NOMEM_BKPT; } - /* For a Real or Integer, use sqlite3_snprintf() to produce the UTF-8 - ** string representation of the value. Then, if the required encoding - ** is UTF-16le or UTF-16be do a translation. - ** - ** FIX ME: It would be better if sqlite3_snprintf() could do UTF-16. - */ - if( fg & MEM_Int ){ - sqlite3_snprintf(nByte, pMem->z, "%lld", pMem->u.i); - }else{ - assert( fg & MEM_Real ); - sqlite3_snprintf(nByte, pMem->z, "%!.15g", pMem->u.r); - } - pMem->n = sqlite3Strlen30(pMem->z); + vdbeMemRenderNum(nByte, pMem->z, pMem); + assert( pMem->z!=0 ); + assert( pMem->n==(int)sqlite3Strlen30NN(pMem->z) ); pMem->enc = SQLITE_UTF8; pMem->flags |= MEM_Str|MEM_Term; - if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real); + if( bForce ) pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); sqlite3VdbeChangeEncoding(pMem, enc); return SQLITE_OK; } @@ -68562,28 +83415,56 @@ SQLITE_PRIVATE int sqlite3VdbeMemStringify(Mem *pMem, u8 enc, u8 bForce){ ** otherwise. */ SQLITE_PRIVATE int sqlite3VdbeMemFinalize(Mem *pMem, FuncDef *pFunc){ - int rc = SQLITE_OK; - if( ALWAYS(pFunc && pFunc->xFinalize) ){ - sqlite3_context ctx; - Mem t; - assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); - assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - memset(&ctx, 0, sizeof(ctx)); - memset(&t, 0, sizeof(t)); - t.flags = MEM_Null; - t.db = pMem->db; - ctx.pOut = &t; - ctx.pMem = pMem; - ctx.pFunc = pFunc; - pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ - assert( (pMem->flags & MEM_Dyn)==0 ); - if( pMem->szMalloc>0 ) sqlite3DbFree(pMem->db, pMem->zMalloc); - memcpy(pMem, &t, sizeof(t)); - rc = ctx.isError; - } - return rc; + sqlite3_context ctx; + Mem t; + assert( pFunc!=0 ); + assert( pMem!=0 ); + assert( pMem->db!=0 ); + assert( pFunc->xFinalize!=0 ); + assert( (pMem->flags & MEM_Null)!=0 || pFunc==pMem->u.pDef ); + assert( sqlite3_mutex_held(pMem->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + memset(&t, 0, sizeof(t)); + t.flags = MEM_Null; + t.db = pMem->db; + ctx.pOut = &t; + ctx.pMem = pMem; + ctx.pFunc = pFunc; + ctx.enc = ENC(t.db); + pFunc->xFinalize(&ctx); /* IMP: R-24505-23230 */ + assert( (pMem->flags & MEM_Dyn)==0 ); + if( pMem->szMalloc>0 ) sqlite3DbFreeNN(pMem->db, pMem->zMalloc); + memcpy(pMem, &t, sizeof(t)); + return ctx.isError; } +/* +** Memory cell pAccum contains the context of an aggregate function. +** This routine calls the xValue method for that function and stores +** the results in memory cell pMem. +** +** SQLITE_ERROR is returned if xValue() reports an error. SQLITE_OK +** otherwise. +*/ +#ifndef SQLITE_OMIT_WINDOWFUNC +SQLITE_PRIVATE int sqlite3VdbeMemAggValue(Mem *pAccum, Mem *pOut, FuncDef *pFunc){ + sqlite3_context ctx; + assert( pFunc!=0 ); + assert( pFunc->xValue!=0 ); + assert( (pAccum->flags & MEM_Null)!=0 || pFunc==pAccum->u.pDef ); + assert( pAccum->db!=0 ); + assert( sqlite3_mutex_held(pAccum->db->mutex) ); + memset(&ctx, 0, sizeof(ctx)); + sqlite3VdbeMemSetNull(pOut); + ctx.pOut = pOut; + ctx.pMem = pAccum; + ctx.pFunc = pFunc; + ctx.enc = ENC(pAccum->db); + pFunc->xValue(&ctx); + return ctx.isError; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** If the memory cell contains a value that must be freed by ** invoking the external callback in Mem.xDel, then this routine @@ -68602,15 +83483,8 @@ static SQLITE_NOINLINE void vdbeMemClearExternAndSetNull(Mem *p){ testcase( p->flags & MEM_Dyn ); } if( p->flags&MEM_Dyn ){ - assert( (p->flags&MEM_RowSet)==0 ); assert( p->xDel!=SQLITE_DYNAMIC && p->xDel!=0 ); p->xDel((void *)p->z); - }else if( p->flags&MEM_RowSet ){ - sqlite3RowSetClear(p->u.pRowSet); - }else if( p->flags&MEM_Frame ){ - VdbeFrame *pFrame = p->u.pFrame; - pFrame->pParent = pFrame->v->pDelFrame; - pFrame->v->pDelFrame = pFrame; } p->flags = MEM_Null; } @@ -68628,7 +83502,7 @@ static SQLITE_NOINLINE void vdbeMemClear(Mem *p){ vdbeMemClearExternAndSetNull(p); } if( p->szMalloc ){ - sqlite3DbFree(p->db, p->zMalloc); + sqlite3DbFreeNN(p->db, p->zMalloc); p->szMalloc = 0; } p->z = 0; @@ -68651,34 +83525,12 @@ SQLITE_PRIVATE void sqlite3VdbeMemRelease(Mem *p){ } } -/* -** Convert a 64-bit IEEE double into a 64-bit signed integer. -** If the double is out of range of a 64-bit signed integer then -** return the closest available 64-bit signed integer. +/* Like sqlite3VdbeMemRelease() but faster for cases where we +** know in advance that the Mem is not MEM_Dyn or MEM_Agg. */ -static i64 doubleToInt64(double r){ -#ifdef SQLITE_OMIT_FLOATING_POINT - /* When floating-point is omitted, double and int64 are the same thing */ - return r; -#else - /* - ** Many compilers we encounter do not define constants for the - ** minimum and maximum 64-bit integers, or they define them - ** inconsistently. And many do not understand the "LL" notation. - ** So we define our own static constants here using nothing - ** larger than a 32-bit integer constant. - */ - static const i64 maxInt = LARGEST_INT64; - static const i64 minInt = SMALLEST_INT64; - - if( r<=(double)minInt ){ - return minInt; - }else if( r>=(double)maxInt ){ - return maxInt; - }else{ - return (i64)r; - } -#endif +SQLITE_PRIVATE void sqlite3VdbeMemReleaseMalloc(Mem *p){ + assert( !VdbeMemDynamic(p) ); + if( p->szMalloc ) vdbeMemClear(p); } /* @@ -68692,20 +83544,24 @@ static i64 doubleToInt64(double r){ ** ** If pMem represents a string value, its encoding might be changed. */ -SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ +static SQLITE_NOINLINE i64 memIntValue(const Mem *pMem){ + i64 value = 0; + sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); + return value; +} +SQLITE_PRIVATE i64 sqlite3VdbeIntValue(const Mem *pMem){ int flags; + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); flags = pMem->flags; - if( flags & MEM_Int ){ + if( flags & (MEM_Int|MEM_IntReal) ){ + testcase( flags & MEM_IntReal ); return pMem->u.i; }else if( flags & MEM_Real ){ - return doubleToInt64(pMem->u.r); - }else if( flags & (MEM_Str|MEM_Blob) ){ - i64 value = 0; - assert( pMem->z || pMem->n==0 ); - sqlite3Atoi64(pMem->z, &value, pMem->n, pMem->enc); - return value; + return sqlite3RealToI64(pMem->u.r); + }else if( (flags & (MEM_Str|MEM_Blob))!=0 && pMem->z!=0 ){ + return memIntValue(pMem); }else{ return 0; } @@ -68717,18 +83573,23 @@ SQLITE_PRIVATE i64 sqlite3VdbeIntValue(Mem *pMem){ ** value. If it is a string or blob, try to convert it to a double. ** If it is a NULL, return 0.0. */ +static SQLITE_NOINLINE double memRealValue(Mem *pMem){ + /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ + double val = (double)0; + sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); + return val; +} SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); if( pMem->flags & MEM_Real ){ return pMem->u.r; - }else if( pMem->flags & MEM_Int ){ + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_IntReal ); return (double)pMem->u.i; }else if( pMem->flags & (MEM_Str|MEM_Blob) ){ - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - double val = (double)0; - sqlite3AtoF(pMem->z, &val, pMem->n, pMem->enc); - return val; + return memRealValue(pMem); }else{ /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ return (double)0; @@ -68736,31 +83597,46 @@ SQLITE_PRIVATE double sqlite3VdbeRealValue(Mem *pMem){ } /* -** The MEM structure is already a MEM_Real. Try to also make it a -** MEM_Int if we can. +** Return 1 if pMem represents true, and return 0 if pMem represents false. +** Return the value ifNull if pMem is NULL. +*/ +SQLITE_PRIVATE int sqlite3VdbeBooleanValue(Mem *pMem, int ifNull){ + testcase( pMem->flags & MEM_IntReal ); + if( pMem->flags & (MEM_Int|MEM_IntReal) ) return pMem->u.i!=0; + if( pMem->flags & MEM_Null ) return ifNull; + return sqlite3VdbeRealValue(pMem)!=0.0; +} + +/* +** The MEM structure is already a MEM_Real or MEM_IntReal. Try to +** make it a MEM_Int if we can. */ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ - i64 ix; - assert( pMem->flags & MEM_Real ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( pMem!=0 ); + assert( pMem->flags & (MEM_Real|MEM_IntReal) ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); - ix = doubleToInt64(pMem->u.r); - - /* Only mark the value as an integer if - ** - ** (1) the round-trip conversion real->int->real is a no-op, and - ** (2) The integer is neither the largest nor the smallest - ** possible integer (ticket #3922) - ** - ** The second and third terms in the following conditional enforces - ** the second condition under the assumption that addition overflow causes - ** values to wrap around. - */ - if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + if( pMem->flags & MEM_IntReal ){ MemSetTypeFlag(pMem, MEM_Int); + }else{ + i64 ix = sqlite3RealToI64(pMem->u.r); + + /* Only mark the value as an integer if + ** + ** (1) the round-trip conversion real->int->real is a no-op, and + ** (2) The integer is neither the largest nor the smallest + ** possible integer (ticket #3922) + ** + ** The second and third terms in the following conditional enforces + ** the second condition under the assumption that addition overflow causes + ** values to wrap around. + */ + if( pMem->u.r==ix && ix>SMALLEST_INT64 && ixu.i = ix; + MemSetTypeFlag(pMem, MEM_Int); + } } } @@ -68768,8 +83644,9 @@ SQLITE_PRIVATE void sqlite3VdbeIntegerAffinity(Mem *pMem){ ** Convert pMem to type integer. Invalidate any prior representations. */ SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); pMem->u.i = sqlite3VdbeIntValue(pMem); @@ -68782,6 +83659,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemIntegerify(Mem *pMem){ ** Invalidate any prior representations. */ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); assert( EIGHT_BYTE_ALIGNMENT(pMem) ); @@ -68790,8 +83668,34 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ return SQLITE_OK; } +/* Compare a floating point value to an integer. Return true if the two +** values are the same within the precision of the floating point value. +** +** This function assumes that i was obtained by assignment from r1. +** +** For some versions of GCC on 32-bit machines, if you do the more obvious +** comparison of "r1==(double)i" you sometimes get an answer of false even +** though the r1 and (double)i values are bit-for-bit the same. +*/ +SQLITE_PRIVATE int sqlite3RealSameAsInt(double r1, sqlite3_int64 i){ + double r2 = (double)i; + return r1==0.0 + || (memcmp(&r1, &r2, sizeof(r1))==0 + && i >= -2251799813685248LL && i < 2251799813685248LL); +} + +/* Convert a floating point value to its closest integer. Do so in +** a way that avoids 'outside the range of representable values' warnings +** from UBSAN. +*/ +SQLITE_PRIVATE i64 sqlite3RealToI64(double r){ + if( r<-9223372036854774784.0 ) return SMALLEST_INT64; + if( r>+9223372036854774784.0 ) return LARGEST_INT64; + return (i64)r; +} + /* -** Convert pMem so that it has types MEM_Real or MEM_Int or both. +** Convert pMem so that it has type MEM_Real or MEM_Int. ** Invalidate any prior representations. ** ** Every effort is made to force the conversion, even if the input @@ -68799,19 +83703,28 @@ SQLITE_PRIVATE int sqlite3VdbeMemRealify(Mem *pMem){ ** as much of the string as we can and ignore the rest. */ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ - if( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))==0 ){ + assert( pMem!=0 ); + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + testcase( pMem->flags & MEM_Null ); + if( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))==0 ){ + int rc; + sqlite3_int64 ix; assert( (pMem->flags & (MEM_Blob|MEM_Str))!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - if( 0==sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc) ){ + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( ((rc==0 || rc==1) && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1) + || sqlite3RealSameAsInt(pMem->u.r, (ix = sqlite3RealToI64(pMem->u.r))) + ){ + pMem->u.i = ix; MemSetTypeFlag(pMem, MEM_Int); }else{ - pMem->u.r = sqlite3VdbeRealValue(pMem); MemSetTypeFlag(pMem, MEM_Real); - sqlite3VdbeIntegerAffinity(pMem); } } - assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_Null))!=0 ); - pMem->flags &= ~(MEM_Str|MEM_Blob); + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null))!=0 ); + pMem->flags &= ~(MEM_Str|MEM_Blob|MEM_Zero); return SQLITE_OK; } @@ -68822,14 +83735,14 @@ SQLITE_PRIVATE int sqlite3VdbeMemNumerify(Mem *pMem){ ** affinity even if that results in loss of data. This routine is ** used (for example) to implement the SQL "cast()" operator. */ -SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ - if( pMem->flags & MEM_Null ) return; +SQLITE_PRIVATE int sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ + if( pMem->flags & MEM_Null ) return SQLITE_OK; switch( aff ){ case SQLITE_AFF_BLOB: { /* Really a cast to BLOB */ if( (pMem->flags & MEM_Blob)==0 ){ sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - MemSetTypeFlag(pMem, MEM_Blob); + if( pMem->flags & MEM_Str ) MemSetTypeFlag(pMem, MEM_Blob); }else{ pMem->flags &= ~(MEM_TypeMask&~MEM_Blob); } @@ -68848,15 +83761,20 @@ SQLITE_PRIVATE void sqlite3VdbeMemCast(Mem *pMem, u8 aff, u8 encoding){ break; } default: { + int rc; assert( aff==SQLITE_AFF_TEXT ); assert( MEM_Str==(MEM_Blob>>3) ); pMem->flags |= (pMem->flags&MEM_Blob)>>3; sqlite3ValueApplyAffinity(pMem, SQLITE_AFF_TEXT, encoding); assert( pMem->flags & MEM_Str || pMem->db->mallocFailed ); - pMem->flags &= ~(MEM_Int|MEM_Real|MEM_Blob|MEM_Zero); - break; + pMem->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal|MEM_Blob|MEM_Zero); + if( encoding!=SQLITE_UTF8 ) pMem->n &= ~1; + rc = sqlite3VdbeChangeEncoding(pMem, encoding); + if( rc ) return rc; + sqlite3VdbeMemZeroTerminateIfAble(pMem); } } + return SQLITE_OK; } /* @@ -68892,13 +83810,14 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetNull(Mem *pMem){ } } SQLITE_PRIVATE void sqlite3ValueSetNull(sqlite3_value *p){ - sqlite3VdbeMemSetNull((Mem*)p); + sqlite3VdbeMemSetNull((Mem*)p); } /* ** Delete any previous value and set the value to be a BLOB of length ** n containing all zeros. */ +#ifndef SQLITE_OMIT_INCRBLOB SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ sqlite3VdbeMemRelease(pMem); pMem->flags = MEM_Blob|MEM_Zero; @@ -68908,6 +83827,21 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ pMem->enc = SQLITE_UTF8; pMem->z = 0; } +#else +SQLITE_PRIVATE int sqlite3VdbeMemSetZeroBlob(Mem *pMem, int n){ + int nByte = n>0?n:1; + if( sqlite3VdbeMemGrow(pMem, nByte, 0) ){ + return SQLITE_NOMEM_BKPT; + } + assert( pMem->z!=0 ); + assert( sqlite3DbMallocSize(pMem->db, pMem->z)>=nByte ); + memset(pMem->z, 0, nByte); + pMem->n = n>0?n:0; + pMem->flags = MEM_Blob; + pMem->enc = SQLITE_UTF8; + return SQLITE_OK; +} +#endif /* ** The pMem is known to contain content that needs to be destroyed prior @@ -68933,6 +83867,35 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetInt64(Mem *pMem, i64 val){ } } +/* +** Set the iIdx'th entry of array aMem[] to contain integer value val. +*/ +SQLITE_PRIVATE void sqlite3MemSetArrayInt64(sqlite3_value *aMem, int iIdx, i64 val){ + sqlite3VdbeMemSetInt64(&aMem[iIdx], val); +} + +/* A no-op destructor */ +SQLITE_PRIVATE void sqlite3NoopDestructor(void *p){ UNUSED_PARAMETER(p); } + +/* +** Set the value stored in *pMem should already be a NULL. +** Also store a pointer to go with it. +*/ +SQLITE_PRIVATE void sqlite3VdbeMemSetPointer( + Mem *pMem, + void *pPtr, + const char *zPType, + void (*xDestructor)(void*) +){ + assert( pMem->flags==MEM_Null ); + vdbeMemClear(pMem); + pMem->u.zPType = zPType ? zPType : ""; + pMem->z = pPtr; + pMem->flags = MEM_Null|MEM_Dyn|MEM_Subtype|MEM_Term; + pMem->eSubtype = 'p'; + pMem->xDel = xDestructor ? xDestructor : sqlite3NoopDestructor; +} + #ifndef SQLITE_OMIT_FLOATING_POINT /* ** Delete any previous value and set the value stored in *pMem to val, @@ -68947,26 +83910,36 @@ SQLITE_PRIVATE void sqlite3VdbeMemSetDouble(Mem *pMem, double val){ } #endif +#ifdef SQLITE_DEBUG +/* +** Return true if the Mem holds a RowSet object. This routine is intended +** for use inside of assert() statements. +*/ +SQLITE_PRIVATE int sqlite3VdbeMemIsRowSet(const Mem *pMem){ + return (pMem->flags&(MEM_Blob|MEM_Dyn))==(MEM_Blob|MEM_Dyn) + && pMem->xDel==sqlite3RowSetDelete; +} +#endif + /* ** Delete any previous value and set the value of pMem to be an ** empty boolean index. +** +** Return SQLITE_OK on success and SQLITE_NOMEM if a memory allocation +** error occurs. */ -SQLITE_PRIVATE void sqlite3VdbeMemSetRowSet(Mem *pMem){ +SQLITE_PRIVATE int sqlite3VdbeMemSetRowSet(Mem *pMem){ sqlite3 *db = pMem->db; + RowSet *p; assert( db!=0 ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = sqlite3DbMallocRawNN(db, 64); - if( db->mallocFailed ){ - pMem->flags = MEM_Null; - pMem->szMalloc = 0; - }else{ - assert( pMem->zMalloc ); - pMem->szMalloc = sqlite3DbMallocSize(db, pMem->zMalloc); - pMem->u.pRowSet = sqlite3RowSetInit(db, pMem->zMalloc, pMem->szMalloc); - assert( pMem->u.pRowSet!=0 ); - pMem->flags = MEM_RowSet; - } + p = sqlite3RowSetInit(db); + if( p==0 ) return SQLITE_NOMEM; + pMem->z = (char*)p; + pMem->flags = MEM_Blob|MEM_Dyn; + pMem->xDel = sqlite3RowSetDelete; + return SQLITE_OK; } /* @@ -68982,7 +83955,7 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ } return n>p->db->aLimit[SQLITE_LIMIT_LENGTH]; } - return 0; + return 0; } #ifdef SQLITE_DEBUG @@ -68991,15 +83964,31 @@ SQLITE_PRIVATE int sqlite3VdbeMemTooBig(Mem *p){ ** its link to a shallow copy and by marking any current shallow ** copies of this cell as invalid. ** -** This is used for testing and debugging only - to make sure shallow -** copies are not misused. +** This is used for testing and debugging only - to help ensure that shallow +** copies (created by OP_SCopy) are not misused. */ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ int i; Mem *pX; - for(i=0, pX=pVdbe->aMem; inMem; i++, pX++){ + for(i=1, pX=pVdbe->aMem+1; inMem; i++, pX++){ if( pX->pScopyFrom==pMem ){ - pX->flags |= MEM_Undefined; + u16 mFlags; + if( pVdbe->db->flags & SQLITE_VdbeTrace ){ + sqlite3DebugPrintf("Invalidate R[%d] due to change in R[%d]\n", + (int)(pX - pVdbe->aMem), (int)(pMem - pVdbe->aMem)); + } + /* If pX is marked as a shallow copy of pMem, then try to verify that + ** no significant changes have been made to pX since the OP_SCopy. + ** A significant change would indicated a missed call to this + ** function for pX. Minor changes, such as adding or removing a + ** dual type, are allowed, as long as the underlying value is the + ** same. */ + mFlags = pMem->flags & pX->flags & pX->mScopyFlags; + assert( (mFlags&(MEM_Int|MEM_IntReal))==0 || pMem->u.i==pX->u.i ); + + /* pMem is the register that is changing. But also mark pX as + ** undefined so that we can quickly detect the shallow-copy error */ + pX->flags = MEM_Undefined; pX->pScopyFrom = 0; } } @@ -69007,7 +83996,6 @@ SQLITE_PRIVATE void sqlite3VdbeMemAboutToChange(Vdbe *pVdbe, Mem *pMem){ } #endif /* SQLITE_DEBUG */ - /* ** Make an shallow copy of pFrom into pTo. Prior contents of ** pTo are freed. The pFrom->z field is not duplicated. If @@ -69020,7 +84008,7 @@ static SQLITE_NOINLINE void vdbeClrCopy(Mem *pTo, const Mem *pFrom, int eType){ sqlite3VdbeMemShallowCopy(pTo, pFrom, eType); } SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int srcType){ - assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); assert( pTo->db==pFrom->db ); if( VdbeMemDynamic(pTo) ){ vdbeClrCopy(pTo,pFrom,srcType); return; } memcpy(pTo, pFrom, MEMCELLSIZE); @@ -69038,7 +84026,7 @@ SQLITE_PRIVATE void sqlite3VdbeMemShallowCopy(Mem *pTo, const Mem *pFrom, int sr SQLITE_PRIVATE int sqlite3VdbeMemCopy(Mem *pTo, const Mem *pFrom){ int rc = SQLITE_OK; - assert( (pFrom->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pFrom) ); if( VdbeMemDynamic(pTo) ) vdbeMemClearExternAndSetNull(pTo); memcpy(pTo, pFrom, MEMCELLSIZE); pTo->flags &= ~MEM_Dyn; @@ -69073,8 +84061,8 @@ SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ ** Change the value of a Mem to be a string or a BLOB. ** ** The memory management strategy depends on the value of the xDel -** parameter. If the value passed is SQLITE_TRANSIENT, then the -** string is copied into a (possibly existing) buffer managed by the +** parameter. If the value passed is SQLITE_TRANSIENT, then the +** string is copied into a (possibly existing) buffer managed by the ** Mem structure. Otherwise, any existing buffer is freed and the ** pointer copied. ** @@ -69083,20 +84071,29 @@ SQLITE_PRIVATE void sqlite3VdbeMemMove(Mem *pTo, Mem *pFrom){ ** stored without allocating memory, then it is. If a memory allocation ** is required to store the string, then value of pMem is unchanged. In ** either case, SQLITE_TOOBIG is returned. +** +** The "enc" parameter is the text encoding for the string, or zero +** to store a blob. +** +** If n is negative, then the string consists of all bytes up to but +** excluding the first zero character. The n parameter must be +** non-negative for blobs. */ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( Mem *pMem, /* Memory cell to set to string value */ const char *z, /* String pointer */ - int n, /* Bytes in string, or negative */ + i64 n, /* Bytes in string, or negative */ u8 enc, /* Encoding of z. 0 for BLOBs */ void (*xDel)(void*) /* Destructor function */ ){ - int nByte = n; /* New value for pMem->n */ + i64 nByte = n; /* New value for pMem->n */ int iLimit; /* Maximum allowed string or blob size */ - u16 flags = 0; /* New value for pMem->flags */ + u16 flags; /* New value for pMem->flags */ + assert( pMem!=0 ); assert( pMem->db==0 || sqlite3_mutex_held(pMem->db->mutex) ); - assert( (pMem->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + assert( enc!=0 || n>=0 ); /* If z is a NULL pointer, set pMem to contain an SQL NULL. */ if( !z ){ @@ -69109,16 +84106,30 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( }else{ iLimit = SQLITE_MAX_LENGTH; } - flags = (enc==0?MEM_Blob:MEM_Str); if( nByte<0 ){ assert( enc!=0 ); if( enc==SQLITE_UTF8 ){ - nByte = sqlite3Strlen30(z); - if( nByte>iLimit ) nByte = iLimit+1; + nByte = strlen(z); }else{ for(nByte=0; nByte<=iLimit && (z[nByte] | z[nByte+1]); nByte+=2){} } - flags |= MEM_Term; + flags= MEM_Str|MEM_Term; + }else if( enc==0 ){ + flags = MEM_Blob; + enc = SQLITE_UTF8; + }else{ + flags = MEM_Str; + } + if( nByte>iLimit ){ + if( xDel && xDel!=SQLITE_TRANSIENT ){ + if( xDel==SQLITE_DYNAMIC ){ + sqlite3DbFree(pMem->db, (void*)z); + }else{ + xDel((void*)z); + } + } + sqlite3VdbeMemSetNull(pMem); + return sqlite3ErrorToParser(pMem->db, SQLITE_TOOBIG); } /* The following block sets the new values of Mem.z and Mem.xDel. It @@ -69126,54 +84137,48 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** management (one of MEM_Dyn or MEM_Static). */ if( xDel==SQLITE_TRANSIENT ){ - int nAlloc = nByte; + i64 nAlloc = nByte; if( flags&MEM_Term ){ nAlloc += (enc==SQLITE_UTF8?1:2); } - if( nByte>iLimit ){ - return SQLITE_TOOBIG; - } testcase( nAlloc==0 ); testcase( nAlloc==31 ); testcase( nAlloc==32 ); - if( sqlite3VdbeMemClearAndResize(pMem, MAX(nAlloc,32)) ){ + if( sqlite3VdbeMemClearAndResize(pMem, (int)MAX(nAlloc,32)) ){ return SQLITE_NOMEM_BKPT; } memcpy(pMem->z, z, nAlloc); - }else if( xDel==SQLITE_DYNAMIC ){ - sqlite3VdbeMemRelease(pMem); - pMem->zMalloc = pMem->z = (char *)z; - pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); }else{ sqlite3VdbeMemRelease(pMem); pMem->z = (char *)z; - pMem->xDel = xDel; - flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + if( xDel==SQLITE_DYNAMIC ){ + pMem->zMalloc = pMem->z; + pMem->szMalloc = sqlite3DbMallocSize(pMem->db, pMem->zMalloc); + }else{ + pMem->xDel = xDel; + flags |= ((xDel==SQLITE_STATIC)?MEM_Static:MEM_Dyn); + } } - pMem->n = nByte; + pMem->n = (int)(nByte & 0x7fffffff); pMem->flags = flags; - pMem->enc = (enc==0 ? SQLITE_UTF8 : enc); + pMem->enc = enc; #ifndef SQLITE_OMIT_UTF16 - if( pMem->enc!=SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ + if( enc>SQLITE_UTF8 && sqlite3VdbeMemHandleBom(pMem) ){ return SQLITE_NOMEM_BKPT; } #endif - if( nByte>iLimit ){ - return SQLITE_TOOBIG; - } return SQLITE_OK; } /* ** Move data out of a btree key or data field and into a Mem structure. -** The data or key is taken from the entry that pCur is currently pointing +** The data is payload from the entry that pCur is currently pointing ** to. offset and amt determine what portion of the data or key to retrieve. -** key is true to get the key or false to get data. The result is written -** into the pMem element. +** The result is written into the pMem element. ** ** The pMem object must have been initialized. This routine will use ** pMem->zMalloc to hold the content from the btree, if possible. New @@ -69184,25 +84189,22 @@ SQLITE_PRIVATE int sqlite3VdbeMemSetStr( ** If this routine fails for any reason (malloc returns NULL or unable ** to read from the disk) then the pMem is left in an inconsistent state. */ -static SQLITE_NOINLINE int vdbeMemFromBtreeResize( +SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ u32 offset, /* Offset from the start of data to return bytes from. */ u32 amt, /* Number of bytes to return. */ - int key, /* If true, retrieve from the btree key, not data. */ Mem *pMem /* OUT: Return data in this Mem structure. */ ){ int rc; pMem->flags = MEM_Null; - if( SQLITE_OK==(rc = sqlite3VdbeMemClearAndResize(pMem, amt+2)) ){ - if( key ){ - rc = sqlite3BtreeKey(pCur, offset, amt, pMem->z); - }else{ - rc = sqlite3BtreeData(pCur, offset, amt, pMem->z); - } + if( sqlite3BtreeMaxRecordSize(pCur)z); if( rc==SQLITE_OK ){ - pMem->z[amt] = 0; - pMem->z[amt+1] = 0; - pMem->flags = MEM_Blob|MEM_Term; + pMem->z[amt] = 0; /* Overrun area used when reading malformed records */ + pMem->flags = MEM_Blob; pMem->n = (int)amt; }else{ sqlite3VdbeMemRelease(pMem); @@ -69210,36 +84212,28 @@ static SQLITE_NOINLINE int vdbeMemFromBtreeResize( } return rc; } -SQLITE_PRIVATE int sqlite3VdbeMemFromBtree( +SQLITE_PRIVATE int sqlite3VdbeMemFromBtreeZeroOffset( BtCursor *pCur, /* Cursor pointing at record to retrieve. */ - u32 offset, /* Offset from the start of data to return bytes from. */ u32 amt, /* Number of bytes to return. */ - int key, /* If true, retrieve from the btree key, not data. */ Mem *pMem /* OUT: Return data in this Mem structure. */ ){ - char *zData; /* Data from the btree layer */ u32 available = 0; /* Number of bytes available on the local btree page */ int rc = SQLITE_OK; /* Return code */ assert( sqlite3BtreeCursorIsValid(pCur) ); assert( !VdbeMemDynamic(pMem) ); - /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() + /* Note: the calls to BtreeKeyFetch() and DataFetch() below assert() ** that both the BtShared and database handle mutexes are held. */ - assert( (pMem->flags & MEM_RowSet)==0 ); - if( key ){ - zData = (char *)sqlite3BtreeKeyFetch(pCur, &available); - }else{ - zData = (char *)sqlite3BtreeDataFetch(pCur, &available); - } - assert( zData!=0 ); + assert( !sqlite3VdbeMemIsRowSet(pMem) ); + pMem->z = (char *)sqlite3BtreePayloadFetch(pCur, &available); + assert( pMem->z!=0 ); - if( offset+amt<=available ){ - pMem->z = &zData[offset]; + if( amt<=available ){ pMem->flags = MEM_Blob|MEM_Ephem; pMem->n = (int)amt; }else{ - rc = vdbeMemFromBtreeResize(pCur, offset, amt, key, pMem); + rc = sqlite3VdbeMemFromBtree(pCur, 0, amt, pMem); } return rc; @@ -69254,13 +84248,11 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert( pVal!=0 ); assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); assert( (pVal->flags & (MEM_Null))==0 ); if( pVal->flags & (MEM_Blob|MEM_Str) ){ + if( ExpandBlob(pVal) ) return 0; pVal->flags |= MEM_Str; - if( pVal->flags & MEM_Zero ){ - sqlite3VdbeMemExpandBlob(pVal); - } if( pVal->enc != (enc & ~SQLITE_UTF16_ALIGNED) ){ sqlite3VdbeChangeEncoding(pVal, enc & ~SQLITE_UTF16_ALIGNED); } @@ -69278,6 +84270,7 @@ static SQLITE_NOINLINE const void *valueToText(sqlite3_value* pVal, u8 enc){ assert(pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) || pVal->db==0 || pVal->db->mallocFailed ); if( pVal->enc==(enc & ~SQLITE_UTF16_ALIGNED) ){ + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; }else{ return 0; @@ -69298,8 +84291,9 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ if( !pVal ) return 0; assert( pVal->db==0 || sqlite3_mutex_held(pVal->db->mutex) ); assert( (enc&3)==(enc&~SQLITE_UTF16_ALIGNED) ); - assert( (pVal->flags & MEM_RowSet)==0 ); + assert( !sqlite3VdbeMemIsRowSet(pVal) ); if( (pVal->flags&(MEM_Str|MEM_Term))==(MEM_Str|MEM_Term) && pVal->enc==enc ){ + assert( sqlite3VdbeMemValidStrRep(pVal) ); return pVal->z; } if( pVal->flags&MEM_Null ){ @@ -69308,6 +84302,24 @@ SQLITE_PRIVATE const void *sqlite3ValueText(sqlite3_value* pVal, u8 enc){ return valueToText(pVal, enc); } +/* Return true if sqlit3_value object pVal is a string or blob value +** that uses the destructor specified in the second argument. +** +** TODO: Maybe someday promote this interface into a published API so +** that third-party extensions can get access to it? +*/ +SQLITE_PRIVATE int sqlite3ValueIsOfClass(const sqlite3_value *pVal, void(*xFree)(void*)){ + if( ALWAYS(pVal!=0) + && ALWAYS((pVal->flags & (MEM_Str|MEM_Blob))!=0) + && (pVal->flags & MEM_Dyn)!=0 + && pVal->xDel==xFree + ){ + return 1; + }else{ + return 0; + } +} + /* ** Create a new sqlite3_value object. */ @@ -69321,7 +84333,7 @@ SQLITE_PRIVATE sqlite3_value *sqlite3ValueNew(sqlite3 *db){ } /* -** Context object passed by sqlite3Stat4ProbeSetValue() through to +** Context object passed by sqlite3Stat4ProbeSetValue() through to ** valueNew(). See comments above valueNew() for details. */ struct ValueNewStat4Ctx { @@ -69336,14 +84348,14 @@ struct ValueNewStat4Ctx { ** the second argument to this function is NULL, the object is allocated ** by calling sqlite3ValueNew(). ** -** Otherwise, if the second argument is non-zero, then this function is +** Otherwise, if the second argument is non-zero, then this function is ** being called indirectly by sqlite3Stat4ProbeSetValue(). If it has not -** already been allocated, allocate the UnpackedRecord structure that +** already been allocated, allocate the UnpackedRecord structure that ** that function will return to its caller here. Then return a pointer to ** an sqlite3_value within the UnpackedRecord.a[] array. */ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( p ){ UnpackedRecord *pRec = p->ppRec[0]; @@ -69352,13 +84364,13 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ int nByte; /* Bytes of space to allocate */ int i; /* Counter variable */ int nCol = pIdx->nColumn; /* Number of index columns including rowid */ - + nByte = sizeof(Mem) * nCol + ROUND8(sizeof(UnpackedRecord)); pRec = (UnpackedRecord*)sqlite3DbMallocZero(db, nByte); if( pRec ){ pRec->pKeyInfo = sqlite3KeyInfoOfIndex(p->pParse, pIdx); if( pRec->pKeyInfo ){ - assert( pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField==nCol ); + assert( pRec->pKeyInfo->nAllField==nCol ); assert( pRec->pKeyInfo->enc==ENC(db) ); pRec->aMem = (Mem *)((u8*)pRec + ROUND8(sizeof(UnpackedRecord))); for(i=0; iaMem[i].db = db; } }else{ - sqlite3DbFree(db, pRec); + sqlite3DbFreeNN(db, pRec); pRec = 0; } } if( pRec==0 ) return 0; p->ppRec[0] = pRec; } - + pRec->nField = p->iVal+1; + sqlite3VdbeMemSetNull(&pRec->aMem[p->iVal]); return &pRec->aMem[p->iVal]; } #else UNUSED_PARAMETER(p); -#endif /* defined(SQLITE_ENABLE_STAT3_OR_STAT4) */ +#endif /* defined(SQLITE_ENABLE_STAT4) */ return sqlite3ValueNew(db); } @@ -69392,21 +84405,21 @@ static sqlite3_value *valueNew(sqlite3 *db, struct ValueNewStat4Ctx *p){ ** * the SQLITE_FUNC_NEEDCOLL function flag is not set, ** ** then this routine attempts to invoke the SQL function. Assuming no -** error occurs, output parameter (*ppVal) is set to point to a value +** error occurs, output parameter (*ppVal) is set to point to a value ** object containing the result before returning SQLITE_OK. ** ** Affinity aff is applied to the result of the function before returning. -** If the result is a text value, the sqlite3_value object uses encoding +** If the result is a text value, the sqlite3_value object uses encoding ** enc. ** ** If the conditions above are not met, this function returns SQLITE_OK ** and sets (*ppVal) to NULL. Or, if an error occurs, (*ppVal) is set to ** NULL and an SQLite error code returned. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 static int valueFromFunction( sqlite3 *db, /* The database connection */ - Expr *p, /* The expression to evaluate */ + const Expr *p, /* The expression to evaluate */ u8 enc, /* Encoding to use */ u8 aff, /* Affinity to use */ sqlite3_value **ppVal, /* Write the new value here */ @@ -69423,12 +84436,17 @@ static int valueFromFunction( assert( pCtx!=0 ); assert( (p->flags & EP_TokenOnly)==0 ); + assert( ExprUseXList(p) ); pList = p->x.pList; if( pList ) nVal = pList->nExpr; + assert( !ExprHasProperty(p, EP_IntValue) ); pFunc = sqlite3FindFunction(db, p->u.zToken, nVal, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pFunc==0 ) return SQLITE_OK; +#endif assert( pFunc ); - if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 - || (pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + if( (pFunc->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || (pFunc->funcFlags & (SQLITE_FUNC_NEEDCOLL|SQLITE_FUNC_RUNONLY))!=0 ){ return SQLITE_OK; } @@ -69440,7 +84458,8 @@ static int valueFromFunction( goto value_from_function_out; } for(i=0; ia[i].pExpr, enc, aff, &apVal[i]); + rc = sqlite3Stat4ValueFromExpr(pCtx->pParse, pList->a[i].pExpr, aff, + &apVal[i]); if( apVal[i]==0 || rc!=SQLITE_OK ) goto value_from_function_out; } } @@ -69451,10 +84470,10 @@ static int valueFromFunction( goto value_from_function_out; } - assert( pCtx->pParse->rc==SQLITE_OK ); memset(&ctx, 0, sizeof(ctx)); ctx.pOut = pVal; ctx.pFunc = pFunc; + ctx.enc = ENC(db); pFunc->xSFunc(&ctx, nVal, apVal); if( ctx.isError ){ rc = ctx.isError; @@ -69463,22 +84482,22 @@ static int valueFromFunction( sqlite3ValueApplyAffinity(pVal, aff, SQLITE_UTF8); assert( rc==SQLITE_OK ); rc = sqlite3VdbeChangeEncoding(pVal, enc); - if( rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal) ){ + if( NEVER(rc==SQLITE_OK && sqlite3VdbeMemTooBig(pVal)) ){ rc = SQLITE_TOOBIG; pCtx->pParse->nErr++; } } - pCtx->pParse->rc = rc; value_from_function_out: if( rc!=SQLITE_OK ){ pVal = 0; + pCtx->pParse->rc = rc; } if( apVal ){ for(i=0; iop)==TK_UPLUS || op==TK_SPAN ) pExpr = pExpr->pLeft; - if( NEVER(op==TK_REGISTER) ) op = pExpr->op2; + if( op==TK_REGISTER ) op = pExpr->op2; /* Compressed expressions only appear when parsing the DEFAULT clause ** on a table column definition, and hence only when pCtx==0. This @@ -69527,25 +84543,40 @@ static int valueFromExpr( assert( (pExpr->flags & EP_TokenOnly)==0 || pCtx==0 ); if( op==TK_CAST ){ - u8 aff = sqlite3AffinityType(pExpr->u.zToken,0); + u8 aff; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + aff = sqlite3AffinityType(pExpr->u.zToken,0); rc = valueFromExpr(db, pExpr->pLeft, enc, aff, ppVal, pCtx); testcase( rc!=SQLITE_OK ); if( *ppVal ){ - sqlite3VdbeMemCast(*ppVal, aff, SQLITE_UTF8); - sqlite3ValueApplyAffinity(*ppVal, affinity, SQLITE_UTF8); +#ifdef SQLITE_ENABLE_STAT4 + rc = ExpandBlob(*ppVal); +#else + /* zero-blobs only come from functions, not literal values. And + ** functions are only processed under STAT4 */ + assert( (ppVal[0][0].flags & MEM_Zero)==0 ); +#endif + sqlite3VdbeMemCast(*ppVal, aff, enc); + sqlite3ValueApplyAffinity(*ppVal, affinity, enc); } return rc; } /* Handle negative integers in a single step. This is needed in the - ** case when the value is -9223372036854775808. - */ - if( op==TK_UMINUS - && (pExpr->pLeft->op==TK_INTEGER || pExpr->pLeft->op==TK_FLOAT) ){ - pExpr = pExpr->pLeft; - op = pExpr->op; - negInt = -1; - zNeg = "-"; + ** case when the value is -9223372036854775808. Except - do not do this + ** for hexadecimal literals. */ + if( op==TK_UMINUS ){ + Expr *pLeft = pExpr->pLeft; + if( (pLeft->op==TK_INTEGER || pLeft->op==TK_FLOAT) ){ + if( ExprHasProperty(pLeft, EP_IntValue) + || pLeft->u.zToken[0]!='0' || (pLeft->u.zToken[1] & ~0x20)!='X' + ){ + pExpr = pLeft; + op = pExpr->op; + negInt = -1; + zNeg = "-"; + } + } } if( op==TK_STRING || op==TK_FLOAT || op==TK_INTEGER ){ @@ -69554,29 +84585,52 @@ static int valueFromExpr( if( ExprHasProperty(pExpr, EP_IntValue) ){ sqlite3VdbeMemSetInt64(pVal, (i64)pExpr->u.iValue*negInt); }else{ - zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); - if( zVal==0 ) goto no_mem; - sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + i64 iVal; + if( op==TK_INTEGER && 0==sqlite3DecOrHexToI64(pExpr->u.zToken, &iVal) ){ + sqlite3VdbeMemSetInt64(pVal, iVal*negInt); + }else{ + zVal = sqlite3MPrintf(db, "%s%s", zNeg, pExpr->u.zToken); + if( zVal==0 ) goto no_mem; + sqlite3ValueSetStr(pVal, -1, zVal, SQLITE_UTF8, SQLITE_DYNAMIC); + } } - if( (op==TK_INTEGER || op==TK_FLOAT ) && affinity==SQLITE_AFF_BLOB ){ - sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + if( affinity==SQLITE_AFF_BLOB ){ + if( op==TK_FLOAT ){ + assert( pVal && pVal->z && pVal->flags==(MEM_Str|MEM_Term) ); + sqlite3AtoF(pVal->z, &pVal->u.r, pVal->n, SQLITE_UTF8); + pVal->flags = MEM_Real; + }else if( op==TK_INTEGER ){ + /* This case is required by -9223372036854775808 and other strings + ** that look like integers but cannot be handled by the + ** sqlite3DecOrHexToI64() call above. */ + sqlite3ValueApplyAffinity(pVal, SQLITE_AFF_NUMERIC, SQLITE_UTF8); + } }else{ sqlite3ValueApplyAffinity(pVal, affinity, SQLITE_UTF8); } - if( pVal->flags & (MEM_Int|MEM_Real) ) pVal->flags &= ~MEM_Str; + assert( (pVal->flags & MEM_IntReal)==0 ); + if( pVal->flags & (MEM_Int|MEM_IntReal|MEM_Real) ){ + testcase( pVal->flags & MEM_Int ); + testcase( pVal->flags & MEM_Real ); + pVal->flags &= ~MEM_Str; + } if( enc!=SQLITE_UTF8 ){ rc = sqlite3VdbeChangeEncoding(pVal, enc); } }else if( op==TK_UMINUS ) { /* This branch happens for multiple negative signs. Ex: -(-5) */ - if( SQLITE_OK==sqlite3ValueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal) + if( SQLITE_OK==valueFromExpr(db,pExpr->pLeft,enc,affinity,&pVal,pCtx) && pVal!=0 ){ sqlite3VdbeMemNumerify(pVal); if( pVal->flags & MEM_Real ){ pVal->u.r = -pVal->u.r; }else if( pVal->u.i==SMALLEST_INT64 ){ +#ifndef SQLITE_OMIT_FLOATING_POINT pVal->u.r = -(double)SMALLEST_INT64; +#else + pVal->u.r = LARGEST_INT64; +#endif MemSetTypeFlag(pVal, MEM_Real); }else{ pVal->u.i = -pVal->u.i; @@ -69586,10 +84640,12 @@ static int valueFromExpr( }else if( op==TK_NULL ){ pVal = valueNew(db, pCtx); if( pVal==0 ) goto no_mem; + sqlite3VdbeMemSetNull(pVal); } #ifndef SQLITE_OMIT_BLOB_LITERAL else if( op==TK_BLOB ){ int nVal; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); assert( pExpr->u.zToken[0]=='x' || pExpr->u.zToken[0]=='X' ); assert( pExpr->u.zToken[1]=='\'' ); pVal = valueNew(db, pCtx); @@ -69601,21 +84657,32 @@ static int valueFromExpr( 0, SQLITE_DYNAMIC); } #endif - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 else if( op==TK_FUNCTION && pCtx!=0 ){ rc = valueFromFunction(db, pExpr, enc, affinity, &pVal, pCtx); } #endif + else if( op==TK_TRUEFALSE ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pVal = valueNew(db, pCtx); + if( pVal ){ + pVal->flags = MEM_Int; + pVal->u.i = pExpr->u.zToken[4]==0; + sqlite3ValueApplyAffinity(pVal, affinity, enc); + } + } *ppVal = pVal; return rc; no_mem: - sqlite3OomFault(db); +#ifdef SQLITE_ENABLE_STAT4 + if( pCtx==0 || NEVER(pCtx->pParse->nErr==0) ) +#endif + sqlite3OomFault(db); sqlite3DbFree(db, zVal); assert( *ppVal==0 ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pCtx==0 ) sqlite3ValueFree(pVal); #else assert( pCtx==0 ); sqlite3ValueFree(pVal); @@ -69635,64 +84702,15 @@ no_mem: */ SQLITE_PRIVATE int sqlite3ValueFromExpr( sqlite3 *db, /* The database connection */ - Expr *pExpr, /* The expression to evaluate */ + const Expr *pExpr, /* The expression to evaluate */ u8 enc, /* Encoding to use */ u8 affinity, /* Affinity to use */ sqlite3_value **ppVal /* Write the new value here */ ){ - return valueFromExpr(db, pExpr, enc, affinity, ppVal, 0); -} - -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* -** The implementation of the sqlite_record() function. This function accepts -** a single argument of any type. The return value is a formatted database -** record (a blob) containing the argument value. -** -** This is used to convert the value stored in the 'sample' column of the -** sqlite_stat3 table to the record format SQLite uses internally. -*/ -static void recordFunc( - sqlite3_context *context, - int argc, - sqlite3_value **argv -){ - const int file_format = 1; - u32 iSerial; /* Serial type */ - int nSerial; /* Bytes of space for iSerial as varint */ - u32 nVal; /* Bytes of space required for argv[0] */ - int nRet; - sqlite3 *db; - u8 *aRet; - - UNUSED_PARAMETER( argc ); - iSerial = sqlite3VdbeSerialType(argv[0], file_format, &nVal); - nSerial = sqlite3VarintLen(iSerial); - db = sqlite3_context_db_handle(context); - - nRet = 1 + nSerial + nVal; - aRet = sqlite3DbMallocRawNN(db, nRet); - if( aRet==0 ){ - sqlite3_result_error_nomem(context); - }else{ - aRet[0] = nSerial+1; - putVarint32(&aRet[1], iSerial); - sqlite3VdbeSerialPut(&aRet[1+nSerial], argv[0], iSerial); - sqlite3_result_blob(context, aRet, nRet, SQLITE_TRANSIENT); - sqlite3DbFree(db, aRet); - } -} - -/* -** Register built-in functions used to help read ANALYZE data. -*/ -SQLITE_PRIVATE void sqlite3AnalyzeFunctions(void){ - static FuncDef aAnalyzeTableFuncs[] = { - FUNCTION(sqlite_record, 1, 0, 0, recordFunc), - }; - sqlite3InsertBuiltinFuncs(aAnalyzeTableFuncs, ArraySize(aAnalyzeTableFuncs)); + return pExpr ? valueFromExpr(db, pExpr, enc, affinity, ppVal, 0) : 0; } +#ifdef SQLITE_ENABLE_STAT4 /* ** Attempt to extract a value from pExpr and use it to construct *ppVal. ** @@ -69725,14 +84743,13 @@ static int stat4ValueFromExpr( /* Skip over any TK_COLLATE nodes */ pExpr = sqlite3ExprSkipCollate(pExpr); + assert( pExpr==0 || pExpr->op!=TK_REGISTER || pExpr->op2!=TK_VARIABLE ); if( !pExpr ){ pVal = valueNew(db, pAlloc); if( pVal ){ sqlite3VdbeMemSetNull((Mem*)pVal); } - }else if( pExpr->op==TK_VARIABLE - || NEVER(pExpr->op==TK_REGISTER && pExpr->op2==TK_VARIABLE) - ){ + }else if( pExpr->op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ Vdbe *v; int iBindVar = pExpr->iColumn; sqlite3VdbeSetVarmask(pParse->pVdbe, iBindVar); @@ -69740,9 +84757,7 @@ static int stat4ValueFromExpr( pVal = valueNew(db, pAlloc); if( pVal ){ rc = sqlite3VdbeMemCopy((Mem*)pVal, &v->aVar[iBindVar-1]); - if( rc==SQLITE_OK ){ - sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); - } + sqlite3ValueApplyAffinity(pVal, affinity, ENC(db)); pVal->db = pParse->db; } } @@ -69756,25 +84771,29 @@ static int stat4ValueFromExpr( } /* -** This function is used to allocate and populate UnpackedRecord -** structures intended to be compared against sample index keys stored +** This function is used to allocate and populate UnpackedRecord +** structures intended to be compared against sample index keys stored ** in the sqlite_stat4 table. ** -** A single call to this function attempts to populates field iVal (leftmost -** is 0 etc.) of the unpacked record with a value extracted from expression -** pExpr. Extraction of values is possible if: +** A single call to this function populates zero or more fields of the +** record starting with field iVal (fields are numbered from left to +** right starting with 0). A single field is populated if: ** ** * (pExpr==0). In this case the value is assumed to be an SQL NULL, ** ** * The expression is a bound variable, and this is a reprepare, or ** -** * The sqlite3ValueFromExpr() function is able to extract a value +** * The sqlite3ValueFromExpr() function is able to extract a value ** from the expression (i.e. the expression is a literal value). ** -** If a value can be extracted, the affinity passed as the 5th argument -** is applied to it before it is copied into the UnpackedRecord. Output -** parameter *pbOk is set to true if a value is extracted, or false -** otherwise. +** Or, if pExpr is a TK_VECTOR, one field is populated for each of the +** vector components that match either of the two latter criteria listed +** above. +** +** Before any value is appended to the record, the affinity of the +** corresponding column within index pIdx is applied to it. Before +** this function returns, output parameter *pnExtract is set to the +** number of values appended to the record. ** ** When this function is called, *ppRec must either point to an object ** allocated by an earlier call to this function, or must be NULL. If it @@ -69790,30 +84809,41 @@ SQLITE_PRIVATE int sqlite3Stat4ProbeSetValue( Index *pIdx, /* Index being probed */ UnpackedRecord **ppRec, /* IN/OUT: Probe record */ Expr *pExpr, /* The expression to extract a value from */ - u8 affinity, /* Affinity to use */ + int nElem, /* Maximum number of values to append */ int iVal, /* Array element to populate */ - int *pbOk /* OUT: True if value was extracted */ + int *pnExtract /* OUT: Values appended to the record */ ){ - int rc; - sqlite3_value *pVal = 0; - struct ValueNewStat4Ctx alloc; + int rc = SQLITE_OK; + int nExtract = 0; - alloc.pParse = pParse; - alloc.pIdx = pIdx; - alloc.ppRec = ppRec; - alloc.iVal = iVal; + if( pExpr==0 || pExpr->op!=TK_SELECT ){ + int i; + struct ValueNewStat4Ctx alloc; - rc = stat4ValueFromExpr(pParse, pExpr, affinity, &alloc, &pVal); - assert( pVal==0 || pVal->db==pParse->db ); - *pbOk = (pVal!=0); + alloc.pParse = pParse; + alloc.pIdx = pIdx; + alloc.ppRec = ppRec; + + for(i=0; idb, pIdx, iVal+i); + alloc.iVal = iVal+i; + rc = stat4ValueFromExpr(pParse, pElem, aff, &alloc, &pVal); + if( !pVal ) break; + nExtract++; + } + } + + *pnExtract = nExtract; return rc; } /* ** Attempt to extract a value from expression pExpr using the methods -** as described for sqlite3Stat4ProbeSetValue() above. +** as described for sqlite3Stat4ProbeSetValue() above. ** -** If successful, set *ppVal to point to a new value object and return +** If successful, set *ppVal to point to a new value object and return ** SQLITE_OK. If no value can be extracted, but no other error occurs ** (e.g. OOM), return SQLITE_OK and set *ppVal to NULL. Or, if an error ** does occur, return an SQLite error code. The final value of *ppVal @@ -69833,7 +84863,7 @@ SQLITE_PRIVATE int sqlite3Stat4ValueFromExpr( ** the column value into *ppVal. If *ppVal is initially NULL then a new ** sqlite3_value object is allocated. ** -** If *ppVal is initially NULL then the caller is responsible for +** If *ppVal is initially NULL then the caller is responsible for ** ensuring that the value written into *ppVal is eventually freed. */ SQLITE_PRIVATE int sqlite3Stat4Column( @@ -69843,18 +84873,18 @@ SQLITE_PRIVATE int sqlite3Stat4Column( int iCol, /* Column to extract */ sqlite3_value **ppVal /* OUT: Extracted value */ ){ - u32 t; /* a column type code */ - int nHdr; /* Size of the header in the record */ - int iHdr; /* Next unread header byte */ - int iField; /* Next unread data byte */ - int szField; /* Size of the current data field */ + u32 t = 0; /* a column type code */ + u32 nHdr; /* Size of the header in the record */ + u32 iHdr; /* Next unread header byte */ + i64 iField; /* Next unread data byte */ + u32 szField = 0; /* Size of the current data field */ int i; /* Column index */ u8 *a = (u8*)pRec; /* Typecast byte array */ Mem *pMem = *ppVal; /* Write result into this Mem object */ assert( iCol>0 ); iHdr = getVarint32(a, nHdr); - if( nHdr>nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; + if( nHdr>(u32)nRec || iHdr>=nHdr ) return SQLITE_CORRUPT_BKPT; iField = nHdr; for(i=0; i<=iCol; i++){ iHdr += getVarint32(&a[iHdr], t); @@ -69884,14 +84914,14 @@ SQLITE_PRIVATE int sqlite3Stat4Column( SQLITE_PRIVATE void sqlite3Stat4ProbeFree(UnpackedRecord *pRec){ if( pRec ){ int i; - int nCol = pRec->pKeyInfo->nField+pRec->pKeyInfo->nXField; + int nCol = pRec->pKeyInfo->nAllField; Mem *aMem = pRec->aMem; sqlite3 *db = aMem[0].db; for(i=0; ipKeyInfo); - sqlite3DbFree(db, pRec); + sqlite3DbFreeNN(db, pRec); } } #endif /* ifdef SQLITE_ENABLE_STAT4 */ @@ -69915,7 +84945,7 @@ SQLITE_PRIVATE void sqlite3ValueSetStr( SQLITE_PRIVATE void sqlite3ValueFree(sqlite3_value *v){ if( !v ) return; sqlite3VdbeMemRelease((Mem *)v); - sqlite3DbFree(((Mem*)v)->db, v); + sqlite3DbFreeNN(((Mem*)v)->db, v); } /* @@ -69932,6 +84962,9 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ if( (p->flags & MEM_Str)!=0 && pVal->enc==enc ){ return p->n; } + if( (p->flags & MEM_Str)!=0 && enc!=SQLITE_UTF8 && pVal->enc!=SQLITE_UTF8 ){ + return p->n; + } if( (p->flags & MEM_Blob)!=0 ){ if( p->flags & MEM_Zero ){ return p->n + p->u.nZero; @@ -69957,35 +84990,49 @@ SQLITE_PRIVATE int sqlite3ValueBytes(sqlite3_value *pVal, u8 enc){ ** ************************************************************************* ** This file contains code used for creating, destroying, and populating -** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) +** a VDBE (or an "sqlite3_stmt" as it is known to the outside world.) */ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* Forward references */ +static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef); +static void vdbeFreeOpArray(sqlite3 *, Op *, int); + /* ** Create a new virtual database engine. */ SQLITE_PRIVATE Vdbe *sqlite3VdbeCreate(Parse *pParse){ sqlite3 *db = pParse->db; Vdbe *p; - p = sqlite3DbMallocZero(db, sizeof(Vdbe) ); + p = sqlite3DbMallocRawNN(db, sizeof(Vdbe) ); if( p==0 ) return 0; + memset(&p->aOp, 0, sizeof(Vdbe)-offsetof(Vdbe,aOp)); p->db = db; if( db->pVdbe ){ - db->pVdbe->pPrev = p; + db->pVdbe->ppVPrev = &p->pVNext; } - p->pNext = db->pVdbe; - p->pPrev = 0; + p->pVNext = db->pVdbe; + p->ppVPrev = &db->pVdbe; db->pVdbe = p; - p->magic = VDBE_MAGIC_INIT; + assert( p->eVdbeState==VDBE_INIT_STATE ); p->pParse = pParse; + pParse->pVdbe = p; assert( pParse->aLabel==0 ); assert( pParse->nLabel==0 ); - assert( pParse->nOpAlloc==0 ); + assert( p->nOpAlloc==0 ); assert( pParse->szOpAlloc==0 ); + sqlite3VdbeAddOp2(p, OP_Init, 0, 1); return p; } +/* +** Return the Parse object that owns a Vdbe object. +*/ +SQLITE_PRIVATE Parse *sqlite3VdbeParser(Vdbe *p){ + return p->pParse; +} + /* ** Change the error string stored in Vdbe.zErrMsg */ @@ -70000,55 +85047,98 @@ SQLITE_PRIVATE void sqlite3VdbeError(Vdbe *p, const char *zFormat, ...){ /* ** Remember the SQL string for a prepared statement. */ -SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, int isPrepareV2){ - assert( isPrepareV2==1 || isPrepareV2==0 ); +SQLITE_PRIVATE void sqlite3VdbeSetSql(Vdbe *p, const char *z, int n, u8 prepFlags){ if( p==0 ) return; -#if defined(SQLITE_OMIT_TRACE) && !defined(SQLITE_ENABLE_SQLLOG) - if( !isPrepareV2 ) return; -#endif + p->prepFlags = prepFlags; + if( (prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ + p->expmask = 0; + } assert( p->zSql==0 ); p->zSql = sqlite3DbStrNDup(p->db, z, n); - p->isPrepareV2 = (u8)isPrepareV2; } +#ifdef SQLITE_ENABLE_NORMALIZE /* -** Return the SQL associated with a prepared statement +** Add a new element to the Vdbe->pDblStr list. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_sql(sqlite3_stmt *pStmt){ - Vdbe *p = (Vdbe *)pStmt; - return p ? p->zSql : 0; +SQLITE_PRIVATE void sqlite3VdbeAddDblquoteStr(sqlite3 *db, Vdbe *p, const char *z){ + if( p ){ + int n = sqlite3Strlen30(z); + DblquoteStr *pStr = sqlite3DbMallocRawNN(db, + sizeof(*pStr)+n+1-sizeof(pStr->z)); + if( pStr ){ + pStr->pNextStr = p->pDblStr; + p->pDblStr = pStr; + memcpy(pStr->z, z, n+1); + } + } } +#endif + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** zId of length nId is a double-quoted identifier. Check to see if +** that identifier is really used as a string literal. +*/ +SQLITE_PRIVATE int sqlite3VdbeUsesDoubleQuotedString( + Vdbe *pVdbe, /* The prepared statement */ + const char *zId /* The double-quoted identifier, already dequoted */ +){ + DblquoteStr *pStr; + assert( zId!=0 ); + if( pVdbe->pDblStr==0 ) return 0; + for(pStr=pVdbe->pDblStr; pStr; pStr=pStr->pNextStr){ + if( strcmp(zId, pStr->z)==0 ) return 1; + } + return 0; +} +#endif /* -** Swap all content between two VDBE structures. +** Swap byte-code between two VDBE structures. +** +** This happens after pB was previously run and returned +** SQLITE_SCHEMA. The statement was then reprepared in pA. +** This routine transfers the new bytecode in pA over to pB +** so that pB can be run again. The old pB byte code is +** moved back to pA so that it will be cleaned up when pA is +** finalized. */ SQLITE_PRIVATE void sqlite3VdbeSwap(Vdbe *pA, Vdbe *pB){ - Vdbe tmp, *pTmp; + Vdbe tmp, *pTmp, **ppTmp; char *zTmp; assert( pA->db==pB->db ); tmp = *pA; *pA = *pB; *pB = tmp; - pTmp = pA->pNext; - pA->pNext = pB->pNext; - pB->pNext = pTmp; - pTmp = pA->pPrev; - pA->pPrev = pB->pPrev; - pB->pPrev = pTmp; + pTmp = pA->pVNext; + pA->pVNext = pB->pVNext; + pB->pVNext = pTmp; + ppTmp = pA->ppVPrev; + pA->ppVPrev = pB->ppVPrev; + pB->ppVPrev = ppTmp; zTmp = pA->zSql; pA->zSql = pB->zSql; pB->zSql = zTmp; - pB->isPrepareV2 = pA->isPrepareV2; +#ifdef SQLITE_ENABLE_NORMALIZE + zTmp = pA->zNormSql; + pA->zNormSql = pB->zNormSql; + pB->zNormSql = zTmp; +#endif + pB->expmask = pA->expmask; + pB->prepFlags = pA->prepFlags; + memcpy(pB->aCounter, pA->aCounter, sizeof(pB->aCounter)); + pB->aCounter[SQLITE_STMTSTATUS_REPREPARE]++; } /* -** Resize the Vdbe.aOp array so that it is at least nOp elements larger +** Resize the Vdbe.aOp array so that it is at least nOp elements larger ** than its current size. nOp is guaranteed to be less than or equal ** to 1024/sizeof(Op). ** ** If an out-of-memory error occurs while resizing the array, return -** SQLITE_NOMEM. In this case Vdbe.aOp and Parse.nOpAlloc remain -** unchanged (this is so that any opcodes already allocated can be +** SQLITE_NOMEM. In this case Vdbe.aOp and Vdbe.nOpAlloc remain +** unchanged (this is so that any opcodes already allocated can be ** correctly deallocated along with the rest of the Vdbe). */ static int growOpArray(Vdbe *v, int nOp){ @@ -70056,25 +85146,33 @@ static int growOpArray(Vdbe *v, int nOp){ Parse *p = v->pParse; /* The SQLITE_TEST_REALLOC_STRESS compile-time option is designed to force - ** more frequent reallocs and hence provide more opportunities for + ** more frequent reallocs and hence provide more opportunities for ** simulated OOM faults. SQLITE_TEST_REALLOC_STRESS is generally used ** during testing only. With SQLITE_TEST_REALLOC_STRESS grow the op array ** by the minimum* amount required until the size reaches 512. Normal ** operation (without SQLITE_TEST_REALLOC_STRESS) is to double the current ** size of the op array or add 1KB of space, whichever is smaller. */ #ifdef SQLITE_TEST_REALLOC_STRESS - int nNew = (p->nOpAlloc>=512 ? p->nOpAlloc*2 : p->nOpAlloc+nOp); + sqlite3_int64 nNew = (v->nOpAlloc>=512 ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)v->nOpAlloc+nOp); #else - int nNew = (p->nOpAlloc ? p->nOpAlloc*2 : (int)(1024/sizeof(Op))); + sqlite3_int64 nNew = (v->nOpAlloc ? 2*(sqlite3_int64)v->nOpAlloc + : (sqlite3_int64)(1024/sizeof(Op))); UNUSED_PARAMETER(nOp); #endif - assert( nOp<=(1024/sizeof(Op)) ); - assert( nNew>=(p->nOpAlloc+nOp) ); + /* Ensure that the size of a VDBE does not grow too large */ + if( nNew > p->db->aLimit[SQLITE_LIMIT_VDBE_OP] ){ + sqlite3OomFault(p->db); + return SQLITE_NOMEM; + } + + assert( nOp<=(int)(1024/sizeof(Op)) ); + assert( nNew>=(v->nOpAlloc+nOp) ); pNew = sqlite3DbRealloc(p->db, v->aOp, nNew*sizeof(Op)); if( pNew ){ p->szOpAlloc = sqlite3DbMallocSize(p->db, pNew); - p->nOpAlloc = p->szOpAlloc/sizeof(Op); + v->nOpAlloc = p->szOpAlloc/sizeof(Op); v->aOp = pNew; } return (pNew ? SQLITE_OK : SQLITE_NOMEM_BKPT); @@ -70083,14 +85181,53 @@ static int growOpArray(Vdbe *v, int nOp){ #ifdef SQLITE_DEBUG /* This routine is just a convenient place to set a breakpoint that will ** fire after each opcode is inserted and displayed using -** "PRAGMA vdbe_addoptrace=on". +** "PRAGMA vdbe_addoptrace=on". Parameters "pc" (program counter) and +** pOp are available to make the breakpoint conditional. +** +** Other useful labels for breakpoints include: +** test_trace_breakpoint(pc,pOp) +** sqlite3CorruptError(lineno) +** sqlite3MisuseError(lineno) +** sqlite3CantopenError(lineno) */ -static void test_addop_breakpoint(void){ - static int n = 0; +static void test_addop_breakpoint(int pc, Op *pOp){ + static u64 n = 0; + (void)pc; + (void)pOp; n++; + if( n==LARGEST_UINT64 ) abort(); /* so that n is used, preventing a warning */ } #endif +/* +** Slow paths for sqlite3VdbeAddOp3() and sqlite3VdbeAddOp4Int() for the +** unusual case when we need to increase the size of the Vdbe.aOp[] array +** before adding the new opcode. +*/ +static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ + assert( p->nOpAlloc<=p->nOp ); + if( growOpArray(p, 1) ) return 1; + assert( p->nOpAlloc>p->nOp ); + return sqlite3VdbeAddOp3(p, op, p1, p2, p3); +} +static SQLITE_NOINLINE int addOp4IntSlow( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); + if( p->db->mallocFailed==0 ){ + VdbeOp *pOp = &p->aOp[addr]; + pOp->p4type = P4_INT32; + pOp->p4.i = p4; + } + return addr; +} + + /* ** Add a new instruction to the list of instructions current in the ** VDBE. Return the address of the new instruction. @@ -70101,64 +85238,8 @@ static void test_addop_breakpoint(void){ ** ** op The opcode for this instruction ** -** p1, p2, p3 Operands -** -** Use the sqlite3VdbeResolveLabel() function to fix an address and -** the sqlite3VdbeChangeP4() function to change the value of the P4 -** operand. +** p1, p2, p3, p4 Operands */ -static SQLITE_NOINLINE int growOp3(Vdbe *p, int op, int p1, int p2, int p3){ - assert( p->pParse->nOpAlloc<=p->nOp ); - if( growOpArray(p, 1) ) return 1; - assert( p->pParse->nOpAlloc>p->nOp ); - return sqlite3VdbeAddOp3(p, op, p1, p2, p3); -} -SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ - int i; - VdbeOp *pOp; - - i = p->nOp; - assert( p->magic==VDBE_MAGIC_INIT ); - assert( op>=0 && op<0xff ); - if( p->pParse->nOpAlloc<=i ){ - return growOp3(p, op, p1, p2, p3); - } - p->nOp++; - pOp = &p->aOp[i]; - pOp->opcode = (u8)op; - pOp->p5 = 0; - pOp->p1 = p1; - pOp->p2 = p2; - pOp->p3 = p3; - pOp->p4.p = 0; - pOp->p4type = P4_NOTUSED; -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - pOp->zComment = 0; -#endif -#ifdef SQLITE_DEBUG - if( p->db->flags & SQLITE_VdbeAddopTrace ){ - int jj, kk; - Parse *pParse = p->pParse; - for(jj=kk=0; jjaColCache + jj; - if( x->iLevel>pParse->iCacheLevel || x->iReg==0 ) continue; - printf(" r[%d]={%d:%d}", x->iReg, x->iTable, x->iColumn); - kk++; - } - if( kk ) printf("\n"); - sqlite3VdbePrintOp(0, i, &p->aOp[i]); - test_addop_breakpoint(); - } -#endif -#ifdef VDBE_PROFILE - pOp->cycles = 0; - pOp->cnt = 0; -#endif -#ifdef SQLITE_VDBE_COVERAGE - pOp->iSrcLine = 0; -#endif - return i; -} SQLITE_PRIVATE int sqlite3VdbeAddOp0(Vdbe *p, int op){ return sqlite3VdbeAddOp3(p, op, 0, 0, 0); } @@ -70168,6 +85249,100 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp1(Vdbe *p, int op, int p1){ SQLITE_PRIVATE int sqlite3VdbeAddOp2(Vdbe *p, int op, int p1, int p2){ return sqlite3VdbeAddOp3(p, op, p1, p2, 0); } +SQLITE_PRIVATE int sqlite3VdbeAddOp3(Vdbe *p, int op, int p1, int p2, int p3){ + int i; + VdbeOp *pOp; + + i = p->nOp; + assert( p->eVdbeState==VDBE_INIT_STATE ); + assert( op>=0 && op<0xff ); + if( p->nOpAlloc<=i ){ + return growOp3(p, op, p1, p2, p3); + } + assert( p->aOp!=0 ); + p->nOp++; + pOp = &p->aOp[i]; + assert( pOp!=0 ); + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.p = 0; + pOp->p4type = P4_NOTUSED; + + /* Replicate this logic in sqlite3VdbeAddOp4Int() + ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOp->zComment = 0; +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + test_addop_breakpoint(i, &p->aOp[i]); + } +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; +#endif + /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ** Replicate in sqlite3VdbeAddOp4Int() */ + + return i; +} +SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( + Vdbe *p, /* Add the opcode to this VM */ + int op, /* The new opcode */ + int p1, /* The P1 operand */ + int p2, /* The P2 operand */ + int p3, /* The P3 operand */ + int p4 /* The P4 operand as an integer */ +){ + int i; + VdbeOp *pOp; + + i = p->nOp; + if( p->nOpAlloc<=i ){ + return addOp4IntSlow(p, op, p1, p2, p3, p4); + } + p->nOp++; + pOp = &p->aOp[i]; + assert( pOp!=0 ); + pOp->opcode = (u8)op; + pOp->p5 = 0; + pOp->p1 = p1; + pOp->p2 = p2; + pOp->p3 = p3; + pOp->p4.i = p4; + pOp->p4type = P4_INT32; + + /* Replicate this logic in sqlite3VdbeAddOp3() + ** vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv */ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + pOp->zComment = 0; +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + pOp->nExec = 0; + pOp->nCycle = 0; +#endif +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + sqlite3VdbePrintOp(0, i, &p->aOp[i]); + test_addop_breakpoint(i, &p->aOp[i]); + } +#endif +#ifdef SQLITE_VDBE_COVERAGE + pOp->iSrcLine = 0; +#endif + /* ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + ** Replicate in sqlite3VdbeAddOp3() */ + + return i; +} /* Generate code for an unconditional jump to instruction iDest */ @@ -70189,6 +85364,9 @@ SQLITE_PRIVATE int sqlite3VdbeLoadString(Vdbe *p, int iDest, const char *zStr){ ** "s" character in zTypes[], the register is a string if the argument is ** not NULL, or OP_Null if the value is a null pointer. For each "i" character ** in zTypes[], the register is initialized to an integer. +** +** If the input string does not end with "X" then an OP_ResultRow instruction +** is generated for the values inserted. */ SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, ...){ va_list ap; @@ -70198,12 +85376,15 @@ SQLITE_PRIVATE void sqlite3VdbeMultiLoad(Vdbe *p, int iDest, const char *zTypes, for(i=0; (c = zTypes[i])!=0; i++){ if( c=='s' ){ const char *z = va_arg(ap, const char*); - sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest++, 0, z, 0); + sqlite3VdbeAddOp4(p, z==0 ? OP_Null : OP_String8, 0, iDest+i, 0, z, 0); + }else if( c=='i' ){ + sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest+i); }else{ - assert( c=='i' ); - sqlite3VdbeAddOp2(p, OP_Integer, va_arg(ap, int), iDest++); + goto skip_op_resultrow; } } + sqlite3VdbeAddOp2(p, OP_ResultRow, iDest, i); +skip_op_resultrow: va_end(ap); } @@ -70224,6 +85405,50 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4( return addr; } +/* +** Add an OP_Function or OP_PureFunc opcode. +** +** The eCallCtx argument is information (typically taken from Expr.op2) +** that describes the calling context of the function. 0 means a general +** function call. NC_IsCheck means called by a check constraint, +** NC_IdxExpr means called as part of an index expression. NC_PartIdx +** means in the WHERE clause of a partial index. NC_GenCol means called +** while computing a generated column value. 0 is the usual case. +*/ +SQLITE_PRIVATE int sqlite3VdbeAddFunctionCall( + Parse *pParse, /* Parsing context */ + int p1, /* Constant argument mask */ + int p2, /* First argument register */ + int p3, /* Register into which results are written */ + int nArg, /* Number of argument */ + const FuncDef *pFunc, /* The function to be invoked */ + int eCallCtx /* Calling context */ +){ + Vdbe *v = pParse->pVdbe; + int nByte; + int addr; + sqlite3_context *pCtx; + assert( v ); + nByte = sizeof(*pCtx) + (nArg-1)*sizeof(sqlite3_value*); + pCtx = sqlite3DbMallocRawNN(pParse->db, nByte); + if( pCtx==0 ){ + assert( pParse->db->mallocFailed ); + freeEphemeralFunction(pParse->db, (FuncDef*)pFunc); + return 0; + } + pCtx->pOut = 0; + pCtx->pFunc = (FuncDef*)pFunc; + pCtx->pVdbe = 0; + pCtx->isError = 0; + pCtx->argc = nArg; + pCtx->iOp = sqlite3VdbeCurrentAddr(v); + addr = sqlite3VdbeAddOp4(v, eCallCtx ? OP_PureFunc : OP_Function, + p1, p2, p3, (char*)pCtx, P4_FUNCCTX); + sqlite3VdbeChangeP5(v, eCallCtx & NC_SelfRef); + sqlite3MayAbort(pParse); + return addr; +} + /* ** Add an opcode that includes the p4 value with a P4_INT64 or ** P4_REAL type. @@ -70242,6 +85467,72 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( return sqlite3VdbeAddOp4(p, op, p1, p2, p3, p4copy, p4type); } +#ifndef SQLITE_OMIT_EXPLAIN +/* +** Return the address of the current EXPLAIN QUERY PLAN baseline. +** 0 means "none". +*/ +SQLITE_PRIVATE int sqlite3VdbeExplainParent(Parse *pParse){ + VdbeOp *pOp; + if( pParse->addrExplain==0 ) return 0; + pOp = sqlite3VdbeGetOp(pParse->pVdbe, pParse->addrExplain); + return pOp->p2; +} + +/* +** Set a debugger breakpoint on the following routine in order to +** monitor the EXPLAIN QUERY PLAN code generation. +*/ +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3ExplainBreakpoint(const char *z1, const char *z2){ + (void)z1; + (void)z2; +} +#endif + +/* +** Add a new OP_Explain opcode. +** +** If the bPush flag is true, then make this opcode the parent for +** subsequent Explains until sqlite3VdbeExplainPop() is called. +*/ +SQLITE_PRIVATE int sqlite3VdbeExplain(Parse *pParse, u8 bPush, const char *zFmt, ...){ + int addr = 0; +#if !defined(SQLITE_DEBUG) + /* Always include the OP_Explain opcodes if SQLITE_DEBUG is defined. + ** But omit them (for performance) during production builds */ + if( pParse->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) +#endif + { + char *zMsg; + Vdbe *v; + va_list ap; + int iThis; + va_start(ap, zFmt); + zMsg = sqlite3VMPrintf(pParse->db, zFmt, ap); + va_end(ap); + v = pParse->pVdbe; + iThis = v->nOp; + addr = sqlite3VdbeAddOp4(v, OP_Explain, iThis, pParse->addrExplain, 0, + zMsg, P4_DYNAMIC); + sqlite3ExplainBreakpoint(bPush?"PUSH":"", sqlite3VdbeGetLastOp(v)->p4.z); + if( bPush){ + pParse->addrExplain = iThis; + } + sqlite3VdbeScanStatus(v, iThis, -1, -1, 0, 0); + } + return addr; +} + +/* +** Pop the EXPLAIN QUERY PLAN stack one level. +*/ +SQLITE_PRIVATE void sqlite3VdbeExplainPop(Parse *pParse){ + sqlite3ExplainBreakpoint("POP", 0); + pParse->addrExplain = sqlite3VdbeExplainParent(pParse); +} +#endif /* SQLITE_OMIT_EXPLAIN */ + /* ** Add an OP_ParseSchema opcode. This routine is broken out from ** sqlite3VdbeAddOp4() since it needs to also needs to mark all btrees @@ -70250,26 +85541,12 @@ SQLITE_PRIVATE int sqlite3VdbeAddOp4Dup8( ** The zWhere string must have been obtained from sqlite3_malloc(). ** This routine will take ownership of the allocated memory. */ -SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere){ +SQLITE_PRIVATE void sqlite3VdbeAddParseSchemaOp(Vdbe *p, int iDb, char *zWhere, u16 p5){ int j; sqlite3VdbeAddOp4(p, OP_ParseSchema, iDb, 0, 0, zWhere, P4_DYNAMIC); + sqlite3VdbeChangeP5(p, p5); for(j=0; jdb->nDb; j++) sqlite3VdbeUsesBtree(p, j); -} - -/* -** Add an opcode that includes the p4 value as an integer. -*/ -SQLITE_PRIVATE int sqlite3VdbeAddOp4Int( - Vdbe *p, /* Add the opcode to this VM */ - int op, /* The new opcode */ - int p1, /* The P1 operand */ - int p2, /* The P2 operand */ - int p3, /* The P3 operand */ - int p4 /* The P4 operand as an integer */ -){ - int addr = sqlite3VdbeAddOp3(p, op, p1, p2, p3); - sqlite3VdbeChangeP4(p, addr, SQLITE_INT_TO_PTR(p4), P4_INT32); - return addr; + sqlite3MayAbort(p->pParse); } /* Insert the end of a co-routine @@ -70298,21 +85575,22 @@ SQLITE_PRIVATE void sqlite3VdbeEndCoroutine(Vdbe *v, int regYield){ ** The VDBE knows that a P2 value is a label because labels are ** always negative and P2 values are suppose to be non-negative. ** Hence, a negative P2 value is a label that has yet to be resolved. +** (Later:) This is only true for opcodes that have the OPFLG_JUMP +** property. ** -** Zero is returned if a malloc() fails. +** Variable usage notes: +** +** Parse.aLabel[x] Stores the address that the x-th label resolves +** into. For testing (SQLITE_DEBUG), unresolved +** labels stores -1, but that is not required. +** Parse.nLabelAlloc Number of slots allocated to Parse.aLabel[] +** Parse.nLabel The *negative* of the number of labels that have +** been issued. The negative is stored because +** that gives a performance improvement over storing +** the equivalent positive value. */ -SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ - Parse *p = v->pParse; - int i = p->nLabel++; - assert( v->magic==VDBE_MAGIC_INIT ); - if( (i & (i-1))==0 ){ - p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, - (i*2+1)*sizeof(p->aLabel[0])); - } - if( p->aLabel ){ - p->aLabel[i] = -1; - } - return ADDR(i); +SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Parse *pParse){ + return --pParse->nLabel; } /* @@ -70320,49 +85598,80 @@ SQLITE_PRIVATE int sqlite3VdbeMakeLabel(Vdbe *v){ ** be inserted. The parameter "x" must have been obtained from ** a prior call to sqlite3VdbeMakeLabel(). */ +static SQLITE_NOINLINE void resizeResolveLabel(Parse *p, Vdbe *v, int j){ + int nNewSize = 10 - p->nLabel; + p->aLabel = sqlite3DbReallocOrFree(p->db, p->aLabel, + nNewSize*sizeof(p->aLabel[0])); + if( p->aLabel==0 ){ + p->nLabelAlloc = 0; + }else{ +#ifdef SQLITE_DEBUG + int i; + for(i=p->nLabelAlloc; iaLabel[i] = -1; +#endif + if( nNewSize>=100 && (nNewSize/100)>(p->nLabelAlloc/100) ){ + sqlite3ProgressCheck(p); + } + p->nLabelAlloc = nNewSize; + p->aLabel[j] = v->nOp; + } +} SQLITE_PRIVATE void sqlite3VdbeResolveLabel(Vdbe *v, int x){ Parse *p = v->pParse; int j = ADDR(x); - assert( v->magic==VDBE_MAGIC_INIT ); - assert( jnLabel ); + assert( v->eVdbeState==VDBE_INIT_STATE ); + assert( j<-p->nLabel ); assert( j>=0 ); - if( p->aLabel ){ +#ifdef SQLITE_DEBUG + if( p->db->flags & SQLITE_VdbeAddopTrace ){ + printf("RESOLVE LABEL %d to %d\n", x, v->nOp); + } +#endif + if( p->nLabelAlloc + p->nLabel < 0 ){ + resizeResolveLabel(p,v,j); + }else{ + assert( p->aLabel[j]==(-1) ); /* Labels may only be resolved once */ p->aLabel[j] = v->nOp; } - p->iFixedOp = v->nOp - 1; } /* ** Mark the VDBE as one that can only be run one time. */ SQLITE_PRIVATE void sqlite3VdbeRunOnlyOnce(Vdbe *p){ - p->runOnlyOnce = 1; + sqlite3VdbeAddOp2(p, OP_Expire, 1, 1); } /* -** Mark the VDBE as one that can only be run multiple times. +** Mark the VDBE as one that can be run multiple times. */ SQLITE_PRIVATE void sqlite3VdbeReusable(Vdbe *p){ - p->runOnlyOnce = 0; + int i; + for(i=1; ALWAYS(inOp); i++){ + if( ALWAYS(p->aOp[i].opcode==OP_Expire) ){ + p->aOp[1].opcode = OP_Noop; + break; + } + } } #ifdef SQLITE_DEBUG /* sqlite3AssertMayAbort() logic */ /* ** The following type and function are used to iterate through all opcodes -** in a Vdbe main program and each of the sub-programs (triggers) it may +** in a Vdbe main program and each of the sub-programs (triggers) it may ** invoke directly or indirectly. It should be used as follows: ** ** Op *pOp; ** VdbeOpIter sIter; ** ** memset(&sIter, 0, sizeof(sIter)); -** sIter.v = v; // v is of type Vdbe* +** sIter.v = v; // v is of type Vdbe* ** while( (pOp = opIterNext(&sIter)) ){ ** // Do something with pOp ** } ** sqlite3DbFree(v->db, sIter.apSub); -** +** */ typedef struct VdbeOpIter VdbeOpIter; struct VdbeOpIter { @@ -70395,7 +85704,7 @@ static Op *opIterNext(VdbeOpIter *p){ p->iSub++; p->iAddr = 0; } - + if( pRet->p4type==P4_SUBPROGRAM ){ int nByte = (p->nSub+1)*sizeof(SubProgram*); int j; @@ -70426,9 +85735,11 @@ static Op *opIterNext(VdbeOpIter *p){ ** * OP_HaltIfNull with P1=SQLITE_CONSTRAINT and P2=OE_Abort. ** * OP_Destroy ** * OP_VUpdate +** * OP_VCreate ** * OP_VRename ** * OP_FkCounter with P2==0 (immediate foreign key constraint) -** * OP_CreateTable and OP_InitCoroutine (for CREATE TABLE AS SELECT ...) +** * OP_CreateBtree/BTREE_INTKEY and OP_InitCoroutine +** (for CREATE TABLE AS SELECT ...) ** ** Then check that the value of Parse.mayAbort is true if an ** ABORT may be thrown, or false otherwise. Return true if it does @@ -70441,22 +85752,37 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ int hasAbort = 0; int hasFkCounter = 0; int hasCreateTable = 0; + int hasCreateIndex = 0; int hasInitCoroutine = 0; Op *pOp; VdbeOpIter sIter; + + if( v==0 ) return 0; memset(&sIter, 0, sizeof(sIter)); sIter.v = v; while( (pOp = opIterNext(&sIter))!=0 ){ int opcode = pOp->opcode; - if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename - || ((opcode==OP_Halt || opcode==OP_HaltIfNull) - && ((pOp->p1&0xff)==SQLITE_CONSTRAINT && pOp->p2==OE_Abort)) + if( opcode==OP_Destroy || opcode==OP_VUpdate || opcode==OP_VRename + || opcode==OP_VDestroy + || opcode==OP_VCreate + || opcode==OP_ParseSchema + || opcode==OP_Function || opcode==OP_PureFunc + || ((opcode==OP_Halt || opcode==OP_HaltIfNull) + && ((pOp->p1)!=SQLITE_OK && pOp->p2==OE_Abort)) ){ hasAbort = 1; break; } - if( opcode==OP_CreateTable ) hasCreateTable = 1; + if( opcode==OP_CreateBtree && pOp->p3==BTREE_INTKEY ) hasCreateTable = 1; + if( mayAbort ){ + /* hasCreateIndex may also be set for some DELETE statements that use + ** OP_Clear. So this routine may end up returning true in the case + ** where a "DELETE FROM tbl" has a statement-journal but does not + ** require one. This is not so bad - it is an inefficiency, not a bug. */ + if( opcode==OP_CreateBtree && pOp->p3==BTREE_BLOBKEY ) hasCreateIndex = 1; + if( opcode==OP_Clear ) hasCreateIndex = 1; + } if( opcode==OP_InitCoroutine ) hasInitCoroutine = 1; #ifndef SQLITE_OMIT_FOREIGN_KEY if( opcode==OP_FkCounter && pOp->p1==0 && pOp->p2==1 ){ @@ -70472,10 +85798,37 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** true for this case to prevent the assert() in the callers frame ** from failing. */ return ( v->db->mallocFailed || hasAbort==mayAbort || hasFkCounter - || (hasCreateTable && hasInitCoroutine) ); + || (hasCreateTable && hasInitCoroutine) || hasCreateIndex + ); } #endif /* SQLITE_DEBUG - the sqlite3AssertMayAbort() function */ +#ifdef SQLITE_DEBUG +/* +** Increment the nWrite counter in the VDBE if the cursor is not an +** ephemeral cursor, or if the cursor argument is NULL. +*/ +SQLITE_PRIVATE void sqlite3VdbeIncrWriteCounter(Vdbe *p, VdbeCursor *pC){ + if( pC==0 + || (pC->eCurType!=CURTYPE_SORTER + && pC->eCurType!=CURTYPE_PSEUDO + && !pC->isEphemeral) + ){ + p->nWrite++; + } +} +#endif + +#ifdef SQLITE_DEBUG +/* +** Assert if an Abort at this point in time might result in a corrupt +** database. +*/ +SQLITE_PRIVATE void sqlite3VdbeAssertAbortable(Vdbe *p){ + assert( p->nWrite==0 || p->usesStmtJournal ); +} +#endif + /* ** This routine is called after all opcodes have been inserted. It loops ** through all the opcodes and fixes up some details. @@ -70489,7 +85842,7 @@ SQLITE_PRIVATE int sqlite3VdbeAssertMayAbort(Vdbe *v, int mayAbort){ ** (3) Update the Vdbe.readOnly and Vdbe.bIsReader flags to accurately ** indicate what the prepared statement actually does. ** -** (4) Initialize the p4.xAdvance pointer on opcodes that use it. +** (4) (discontinued) ** ** (5) Reclaim the memory allocated for storing labels. ** @@ -70502,11 +85855,13 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ Op *pOp; Parse *pParse = p->pParse; int *aLabel = pParse->aLabel; + + assert( pParse->db->mallocFailed==0 ); /* tag-20230419-1 */ p->readOnly = 1; p->bIsReader = 0; pOp = &p->aOp[p->nOp-1]; - while(1){ - + assert( p->aOp[0].opcode==OP_Init ); + while( 1 /* Loop terminates when it reaches the OP_Init opcode */ ){ /* Only JUMP opcodes and the short list of special opcodes in the switch ** below need to be considered. The mkopcodeh.tcl generator script groups ** all these opcodes together near the front of the opcode list. Skip @@ -70519,7 +85874,7 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ switch( pOp->opcode ){ case OP_Transaction: { if( pOp->p2!=0 ) p->readOnly = 0; - /* fall thru */ + /* no break */ deliberate_fall_through } case OP_AutoCommit: case OP_Savepoint: { @@ -70535,6 +85890,10 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ p->bIsReader = 1; break; } + case OP_Init: { + assert( pOp->p2>=0 ); + goto resolve_p2_values_loop_exit; + } #ifndef SQLITE_OMIT_VIRTUALTABLE case OP_VUpdate: { if( pOp->p2>nMaxArgs ) nMaxArgs = pOp->p2; @@ -70546,43 +85905,143 @@ static void resolveP2Values(Vdbe *p, int *pMaxFuncArgs){ assert( pOp[-1].opcode==OP_Integer ); n = pOp[-1].p1; if( n>nMaxArgs ) nMaxArgs = n; - break; + /* Fall through into the default case */ + /* no break */ deliberate_fall_through } #endif - case OP_Next: - case OP_NextIfOpen: - case OP_SorterNext: { - pOp->p4.xAdvance = sqlite3BtreeNext; - pOp->p4type = P4_ADVANCE; - break; - } - case OP_Prev: - case OP_PrevIfOpen: { - pOp->p4.xAdvance = sqlite3BtreePrevious; - pOp->p4type = P4_ADVANCE; + default: { + if( pOp->p2<0 ){ + /* The mkopcodeh.tcl script has so arranged things that the only + ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to + ** have non-negative values for P2. */ + assert( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ); + assert( ADDR(pOp->p2)<-pParse->nLabel ); + assert( aLabel!=0 ); /* True because of tag-20230419-1 */ + pOp->p2 = aLabel[ADDR(pOp->p2)]; + } + + /* OPFLG_JUMP opcodes never have P2==0, though OPFLG_JUMP0 opcodes + ** might */ + assert( pOp->p2>0 + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP0)!=0 ); + + /* Jumps never go off the end of the bytecode array */ + assert( pOp->p2nOp + || (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)==0 ); break; } } - if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 && pOp->p2<0 ){ - assert( ADDR(pOp->p2)nLabel ); - pOp->p2 = aLabel[ADDR(pOp->p2)]; - } + /* The mkopcodeh.tcl script has so arranged things that the only + ** non-jump opcodes less than SQLITE_MX_JUMP_CODE are guaranteed to + ** have non-negative values for P2. */ + assert( (sqlite3OpcodeProperty[pOp->opcode]&OPFLG_JUMP)==0 || pOp->p2>=0); } - if( pOp==p->aOp ) break; + assert( pOp>p->aOp ); pOp--; } - sqlite3DbFree(p->db, pParse->aLabel); - pParse->aLabel = 0; +resolve_p2_values_loop_exit: + if( aLabel ){ + sqlite3DbNNFreeNN(p->db, pParse->aLabel); + pParse->aLabel = 0; + } pParse->nLabel = 0; *pMaxFuncArgs = nMaxArgs; assert( p->bIsReader!=0 || DbMaskAllZero(p->btreeMask) ); } +#ifdef SQLITE_DEBUG +/* +** Check to see if a subroutine contains a jump to a location outside of +** the subroutine. If a jump outside the subroutine is detected, add code +** that will cause the program to halt with an error message. +** +** The subroutine consists of opcodes between iFirst and iLast. Jumps to +** locations within the subroutine are acceptable. iRetReg is a register +** that contains the return address. Jumps to outside the range of iFirst +** through iLast are also acceptable as long as the jump destination is +** an OP_Return to iReturnAddr. +** +** A jump to an unresolved label means that the jump destination will be +** beyond the current address. That is normally a jump to an early +** termination and is consider acceptable. +** +** This routine only runs during debug builds. The purpose is (of course) +** to detect invalid escapes out of a subroutine. The OP_Halt opcode +** is generated rather than an assert() or other error, so that ".eqp full" +** will still work to show the original bytecode, to aid in debugging. +*/ +SQLITE_PRIVATE void sqlite3VdbeNoJumpsOutsideSubrtn( + Vdbe *v, /* The byte-code program under construction */ + int iFirst, /* First opcode of the subroutine */ + int iLast, /* Last opcode of the subroutine */ + int iRetReg /* Subroutine return address register */ +){ + VdbeOp *pOp; + Parse *pParse; + int i; + sqlite3_str *pErr = 0; + assert( v!=0 ); + pParse = v->pParse; + assert( pParse!=0 ); + if( pParse->nErr ) return; + assert( iLast>=iFirst ); + assert( iLastnOp ); + pOp = &v->aOp[iFirst]; + for(i=iFirst; i<=iLast; i++, pOp++){ + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_JUMP)!=0 ){ + int iDest = pOp->p2; /* Jump destination */ + if( iDest==0 ) continue; + if( pOp->opcode==OP_Gosub ) continue; + if( pOp->p3==20230325 && pOp->opcode==OP_NotNull ){ + /* This is a deliberately taken illegal branch. tag-20230325-2 */ + continue; + } + if( iDest<0 ){ + int j = ADDR(iDest); + assert( j>=0 ); + if( j>=-pParse->nLabel || pParse->aLabel[j]<0 ){ + continue; + } + iDest = pParse->aLabel[j]; + } + if( iDestiLast ){ + int j = iDest; + for(; jnOp; j++){ + VdbeOp *pX = &v->aOp[j]; + if( pX->opcode==OP_Return ){ + if( pX->p1==iRetReg ) break; + continue; + } + if( pX->opcode==OP_Noop ) continue; + if( pX->opcode==OP_Explain ) continue; + if( pErr==0 ){ + pErr = sqlite3_str_new(0); + }else{ + sqlite3_str_appendchar(pErr, 1, '\n'); + } + sqlite3_str_appendf(pErr, + "Opcode at %d jumps to %d which is outside the " + "subroutine at %d..%d", + i, iDest, iFirst, iLast); + break; + } + } + } + } + if( pErr ){ + char *zErr = sqlite3_str_finish(pErr); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_INTERNAL, OE_Abort, 0, zErr, 0); + sqlite3_free(zErr); + sqlite3MayAbort(pParse); + } +} +#endif /* SQLITE_DEBUG */ + /* ** Return the address of the next instruction to be inserted. */ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ - assert( p->magic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); return p->nOp; } @@ -70596,19 +86055,46 @@ SQLITE_PRIVATE int sqlite3VdbeCurrentAddr(Vdbe *p){ */ #if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) SQLITE_PRIVATE void sqlite3VdbeVerifyNoMallocRequired(Vdbe *p, int N){ - assert( p->nOp + N <= p->pParse->nOpAlloc ); + assert( p->nOp + N <= p->nOpAlloc ); +} +#endif + +/* +** Verify that the VM passed as the only argument does not contain +** an OP_ResultRow opcode. Fail an assert() if it does. This is used +** by code in pragma.c to ensure that the implementation of certain +** pragmas comports with the flags specified in the mkpragmatab.tcl +** script. +*/ +#if defined(SQLITE_DEBUG) && !defined(SQLITE_TEST_REALLOC_STRESS) +SQLITE_PRIVATE void sqlite3VdbeVerifyNoResultRow(Vdbe *p){ + int i; + for(i=0; inOp; i++){ + assert( p->aOp[i].opcode!=OP_ResultRow ); + } +} +#endif + +/* +** Generate code (a single OP_Abortable opcode) that will +** verify that the VDBE program can safely call Abort in the current +** context. +*/ +#if defined(SQLITE_DEBUG) +SQLITE_PRIVATE void sqlite3VdbeVerifyAbortable(Vdbe *p, int onError){ + if( onError==OE_Abort ) sqlite3VdbeAddOp0(p, OP_Abortable); } #endif /* ** This function returns a pointer to the array of opcodes associated with ** the Vdbe passed as the first argument. It is the callers responsibility -** to arrange for the returned array to be eventually freed using the +** to arrange for the returned array to be eventually freed using the ** vdbeFreeOpArray() function. ** ** Before returning, *pnOp is set to the number of entries in the returned -** array. Also, *pnMaxArg is set to the larger of its current value and -** the number of entries in the Vdbe.apArg[] array required to execute the +** array. Also, *pnMaxArg is set to the larger of its current value and +** the number of entries in the Vdbe.apArg[] array required to execute the ** returned program. */ SQLITE_PRIVATE VdbeOp *sqlite3VdbeTakeOpArray(Vdbe *p, int *pnOp, int *pnMaxArg){ @@ -70640,8 +86126,8 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( int i; VdbeOp *pOut, *pFirst; assert( nOp>0 ); - assert( p->magic==VDBE_MAGIC_INIT ); - if( p->nOp + nOp > p->pParse->nOpAlloc && growOpArray(p, nOp) ){ + assert( p->eVdbeState==VDBE_INIT_STATE ); + if( p->nOp + nOp > p->nOpAlloc && growOpArray(p, nOp) ){ return 0; } pFirst = pOut = &p->aOp[p->nOp]; @@ -70682,45 +86168,125 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeAddOpList( SQLITE_PRIVATE void sqlite3VdbeScanStatus( Vdbe *p, /* VM to add scanstatus() to */ int addrExplain, /* Address of OP_Explain (or 0) */ - int addrLoop, /* Address of loop counter */ + int addrLoop, /* Address of loop counter */ int addrVisit, /* Address of rows visited counter */ LogEst nEst, /* Estimated number of output rows */ const char *zName /* Name of table or index being scanned */ ){ - int nByte = (p->nScan+1) * sizeof(ScanStatus); - ScanStatus *aNew; - aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); - if( aNew ){ - ScanStatus *pNew = &aNew[p->nScan++]; - pNew->addrExplain = addrExplain; - pNew->addrLoop = addrLoop; - pNew->addrVisit = addrVisit; - pNew->nEst = nEst; - pNew->zName = sqlite3DbStrDup(p->db, zName); - p->aScan = aNew; + if( IS_STMT_SCANSTATUS(p->db) ){ + sqlite3_int64 nByte = (p->nScan+1) * sizeof(ScanStatus); + ScanStatus *aNew; + aNew = (ScanStatus*)sqlite3DbRealloc(p->db, p->aScan, nByte); + if( aNew ){ + ScanStatus *pNew = &aNew[p->nScan++]; + memset(pNew, 0, sizeof(ScanStatus)); + pNew->addrExplain = addrExplain; + pNew->addrLoop = addrLoop; + pNew->addrVisit = addrVisit; + pNew->nEst = nEst; + pNew->zName = sqlite3DbStrDup(p->db, zName); + p->aScan = aNew; + } } } -#endif + +/* +** Add the range of instructions from addrStart to addrEnd (inclusive) to +** the set of those corresponding to the sqlite3_stmt_scanstatus() counters +** associated with the OP_Explain instruction at addrExplain. The +** sum of the sqlite3Hwtime() values for each of these instructions +** will be returned for SQLITE_SCANSTAT_NCYCLE requests. +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatusRange( + Vdbe *p, + int addrExplain, + int addrStart, + int addrEnd +){ + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrEnd<0 ) addrEnd = sqlite3VdbeCurrentAddr(p)-1; + for(ii=0; iiaAddrRange); ii+=2){ + if( pScan->aAddrRange[ii]==0 ){ + pScan->aAddrRange[ii] = addrStart; + pScan->aAddrRange[ii+1] = addrEnd; + break; + } + } + } + } +} + +/* +** Set the addresses for the SQLITE_SCANSTAT_NLOOP and SQLITE_SCANSTAT_NROW +** counters for the query element associated with the OP_Explain at +** addrExplain. +*/ +SQLITE_PRIVATE void sqlite3VdbeScanStatusCounters( + Vdbe *p, + int addrExplain, + int addrLoop, + int addrVisit +){ + if( IS_STMT_SCANSTATUS(p->db) ){ + ScanStatus *pScan = 0; + int ii; + for(ii=p->nScan-1; ii>=0; ii--){ + pScan = &p->aScan[ii]; + if( pScan->addrExplain==addrExplain ) break; + pScan = 0; + } + if( pScan ){ + if( addrLoop>0 ) pScan->addrLoop = addrLoop; + if( addrVisit>0 ) pScan->addrVisit = addrVisit; + } + } +} +#endif /* defined(SQLITE_ENABLE_STMT_SCANSTATUS) */ /* ** Change the value of the opcode, or P1, P2, P3, or P5 operands ** for a specific instruction. */ -SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, u32 addr, u8 iNewOpcode){ +SQLITE_PRIVATE void sqlite3VdbeChangeOpcode(Vdbe *p, int addr, u8 iNewOpcode){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->opcode = iNewOpcode; } -SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP1(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p1 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP2(Vdbe *p, int addr, int val){ + assert( addr>=0 || p->db->mallocFailed ); sqlite3VdbeGetOp(p,addr)->p2 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, u32 addr, int val){ +SQLITE_PRIVATE void sqlite3VdbeChangeP3(Vdbe *p, int addr, int val){ + assert( addr>=0 ); sqlite3VdbeGetOp(p,addr)->p3 = val; } -SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ - if( !p->db->mallocFailed ) p->aOp[p->nOp-1].p5 = p5; +SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u16 p5){ + assert( p->nOp>0 || p->db->mallocFailed ); + if( p->nOp>0 ) p->aOp[p->nOp-1].p5 = p5; +} + +/* +** If the previous opcode is an OP_Column that delivers results +** into register iDest, then add the OPFLAG_TYPEOFARG flag to that +** opcode. +*/ +SQLITE_PRIVATE void sqlite3VdbeTypeofColumn(Vdbe *p, int iDest){ + VdbeOp *pOp = sqlite3VdbeGetLastOp(p); + if( pOp->p3==iDest && pOp->opcode==OP_Column ){ + pOp->p5 |= OPFLAG_TYPEOFARG; + } } /* @@ -70728,38 +86294,73 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP5(Vdbe *p, u8 p5){ ** the address of the next instruction to be coded. */ SQLITE_PRIVATE void sqlite3VdbeJumpHere(Vdbe *p, int addr){ - p->pParse->iFixedOp = p->nOp - 1; sqlite3VdbeChangeP2(p, addr, p->nOp); } +/* +** Change the P2 operand of the jump instruction at addr so that +** the jump lands on the next opcode. Or if the jump instruction was +** the previous opcode (and is thus a no-op) then simply back up +** the next instruction counter by one slot so that the jump is +** overwritten by the next inserted opcode. +** +** This routine is an optimization of sqlite3VdbeJumpHere() that +** strives to omit useless byte-code like this: +** +** 7 Once 0 8 0 +** 8 ... +*/ +SQLITE_PRIVATE void sqlite3VdbeJumpHereOrPopInst(Vdbe *p, int addr){ + if( addr==p->nOp-1 ){ + assert( p->aOp[addr].opcode==OP_Once + || p->aOp[addr].opcode==OP_If + || p->aOp[addr].opcode==OP_FkIfZero ); + assert( p->aOp[addr].p4type==0 ); +#ifdef SQLITE_VDBE_COVERAGE + sqlite3VdbeGetLastOp(p)->iSrcLine = 0; /* Erase VdbeCoverage() macros */ +#endif + p->nOp--; + }else{ + sqlite3VdbeChangeP2(p, addr, p->nOp); + } +} + /* ** If the input FuncDef structure is ephemeral, then free it. If -** the FuncDef is not ephermal, then do nothing. +** the FuncDef is not ephemeral, then do nothing. */ static void freeEphemeralFunction(sqlite3 *db, FuncDef *pDef){ - if( ALWAYS(pDef) && (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ - sqlite3DbFree(db, pDef); + assert( db!=0 ); + if( (pDef->funcFlags & SQLITE_FUNC_EPHEM)!=0 ){ + sqlite3DbNNFreeNN(db, pDef); } } -static void vdbeFreeOpArray(sqlite3 *, Op *, int); - /* ** Delete a P4 value if necessary. */ +static SQLITE_NOINLINE void freeP4Mem(sqlite3 *db, Mem *p){ + if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); + sqlite3DbNNFreeNN(db, p); +} +static SQLITE_NOINLINE void freeP4FuncCtx(sqlite3 *db, sqlite3_context *p){ + assert( db!=0 ); + freeEphemeralFunction(db, p->pFunc); + sqlite3DbNNFreeNN(db, p); +} static void freeP4(sqlite3 *db, int p4type, void *p4){ assert( db ); switch( p4type ){ case P4_FUNCCTX: { - freeEphemeralFunction(db, ((sqlite3_context*)p4)->pFunc); - /* Fall through into the next case */ + freeP4FuncCtx(db, (sqlite3_context*)p4); + break; } case P4_REAL: case P4_INT64: case P4_DYNAMIC: case P4_INTARRAY: { - sqlite3DbFree(db, p4); + if( p4 ) sqlite3DbNNFreeNN(db, p4); break; } case P4_KEYINFO: { @@ -70772,10 +86373,6 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ break; } #endif - case P4_MPRINTF: { - if( db->pnBytesFreed==0 ) sqlite3_free(p4); - break; - } case P4_FUNCDEF: { freeEphemeralFunction(db, (FuncDef*)p4); break; @@ -70784,9 +86381,7 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ){ sqlite3ValueFree((sqlite3_value*)p4); }else{ - Mem *p = (Mem*)p4; - if( p->szMalloc ) sqlite3DbFree(db, p->zMalloc); - sqlite3DbFree(db, p); + freeP4Mem(db, (Mem*)p4); } break; } @@ -70794,25 +86389,39 @@ static void freeP4(sqlite3 *db, int p4type, void *p4){ if( db->pnBytesFreed==0 ) sqlite3VtabUnlock((VTable *)p4); break; } + case P4_TABLEREF: { + if( db->pnBytesFreed==0 ) sqlite3DeleteTable(db, (Table*)p4); + break; + } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = (SubrtnSig*)p4; + sqlite3DbFree(db, pSig->zAff); + sqlite3DbFree(db, pSig); + break; + } } } /* ** Free the space allocated for aOp and any p4 values allocated for the -** opcodes contained within. If aOp is not NULL it is assumed to contain -** nOp entries. +** opcodes contained within. If aOp is not NULL it is assumed to contain +** nOp entries. */ static void vdbeFreeOpArray(sqlite3 *db, Op *aOp, int nOp){ + assert( nOp>=0 ); + assert( db!=0 ); if( aOp ){ - Op *pOp; - for(pOp=aOp; pOp<&aOp[nOp]; pOp++){ - if( pOp->p4type ) freeP4(db, pOp->p4type, pOp->p4.p); + Op *pOp = &aOp[nOp-1]; + while(1){ /* Exit via break */ + if( pOp->p4type <= P4_FREE_IF_LE ) freeP4(db, pOp->p4type, pOp->p4.p); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS sqlite3DbFree(db, pOp->zComment); -#endif +#endif + if( pOp==aOp ) break; + pOp--; } + sqlite3DbNNFreeNN(db, aOp); } - sqlite3DbFree(db, aOp); } /* @@ -70825,6 +86434,13 @@ SQLITE_PRIVATE void sqlite3VdbeLinkSubProgram(Vdbe *pVdbe, SubProgram *p){ pVdbe->pProgram = p; } +/* +** Return true if the given Vdbe has any SubPrograms. +*/ +SQLITE_PRIVATE int sqlite3VdbeHasSubProgram(Vdbe *pVdbe){ + return pVdbe->pProgram!=0; +} + /* ** Change the opcode at addr into OP_Noop */ @@ -70845,13 +86461,47 @@ SQLITE_PRIVATE int sqlite3VdbeChangeToNoop(Vdbe *p, int addr){ ** then remove it. Return true if and only if an opcode was removed. */ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ - if( (p->nOp-1)>(p->pParse->iFixedOp) && p->aOp[p->nOp-1].opcode==op ){ + if( p->nOp>0 && p->aOp[p->nOp-1].opcode==op ){ return sqlite3VdbeChangeToNoop(p, p->nOp-1); }else{ return 0; } } +#ifdef SQLITE_DEBUG +/* +** Generate an OP_ReleaseReg opcode to indicate that a range of +** registers, except any identified by mask, are no longer in use. +*/ +SQLITE_PRIVATE void sqlite3VdbeReleaseRegisters( + Parse *pParse, /* Parsing context */ + int iFirst, /* Index of first register to be released */ + int N, /* Number of registers to release */ + u32 mask, /* Mask of registers to NOT release */ + int bUndefine /* If true, mark registers as undefined */ +){ + if( N==0 || OptimizationDisabled(pParse->db, SQLITE_ReleaseReg) ) return; + assert( pParse->pVdbe ); + assert( iFirst>=1 ); + assert( iFirst+N-1<=pParse->nMem ); + if( N<=31 && mask!=0 ){ + while( N>0 && (mask&1)!=0 ){ + mask >>= 1; + iFirst++; + N--; + } + while( N>0 && N<=32 && (mask & MASKBIT32(N-1))!=0 ){ + mask &= ~MASKBIT32(N-1); + N--; + } + } + if( N>0 ){ + sqlite3VdbeAddOp3(pParse->pVdbe, OP_ReleaseReg, iFirst, N, *(int*)&mask); + if( bUndefine ) sqlite3VdbeChangeP5(pParse->pVdbe, 1); + } +} +#endif /* SQLITE_DEBUG */ + /* ** Change the value of the P4 operand for a specific instruction. ** This routine is useful when a large program is loaded from a @@ -70862,7 +86512,7 @@ SQLITE_PRIVATE int sqlite3VdbeDeletePriorOpcode(Vdbe *p, u8 op){ ** the string is made into memory obtained from sqlite3_malloc(). ** A value of n==0 means copy bytes of zP4 up to and including the ** first null byte. If n>0 then copy n+1 bytes of zP4. -** +** ** Other values of n (P4_STATIC, P4_COLLSEQ etc.) indicate that zP4 points ** to a string or structure that is guaranteed to exist for the lifetime of ** the Vdbe. In these cases we can just copy the pointer. @@ -70876,7 +86526,7 @@ static void SQLITE_NOINLINE vdbeChangeP4Full( int n ){ if( pOp->p4type ){ - freeP4(p->db, pOp->p4type, pOp->p4.p); + assert( pOp->p4type > P4_FREE_IF_LE ); pOp->p4type = 0; pOp->p4.p = 0; } @@ -70893,7 +86543,7 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int sqlite3 *db; assert( p!=0 ); db = p->db; - assert( p->magic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( p->aOp!=0 || db->mallocFailed ); if( db->mallocFailed ){ if( n!=P4_VTAB ) freeP4(db, n, (void*)*(char**)&zP4); @@ -70922,16 +86572,42 @@ SQLITE_PRIVATE void sqlite3VdbeChangeP4(Vdbe *p, int addr, const char *zP4, int } } +/* +** Change the P4 operand of the most recently coded instruction +** to the value defined by the arguments. This is a high-speed +** version of sqlite3VdbeChangeP4(). +** +** The P4 operand must not have been previously defined. And the new +** P4 must not be P4_INT32. Use sqlite3VdbeChangeP4() in either of +** those cases. +*/ +SQLITE_PRIVATE void sqlite3VdbeAppendP4(Vdbe *p, void *pP4, int n){ + VdbeOp *pOp; + assert( n!=P4_INT32 && n!=P4_VTAB ); + assert( n<=0 ); + if( p->db->mallocFailed ){ + freeP4(p->db, n, pP4); + }else{ + assert( pP4!=0 || n==P4_DYNAMIC ); + assert( p->nOp>0 ); + pOp = &p->aOp[p->nOp-1]; + assert( pOp->p4type==P4_NOTUSED ); + pOp->p4type = n; + pOp->p4.p = pP4; + } +} + /* ** Set the P4 on the most recently added opcode to the KeyInfo for the ** index given. */ SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){ Vdbe *v = pParse->pVdbe; + KeyInfo *pKeyInfo; assert( v!=0 ); assert( pIdx!=0 ); - sqlite3VdbeChangeP4(v, -1, (char*)sqlite3KeyInfoOfIndex(pParse, pIdx), - P4_KEYINFO); + pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pIdx); + if( pKeyInfo ) sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); } #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS @@ -70943,7 +86619,7 @@ SQLITE_PRIVATE void sqlite3VdbeSetP4KeyInfo(Parse *pParse, Index *pIdx){ */ static void vdbeVComment(Vdbe *p, const char *zFormat, va_list ap){ assert( p->nOp>0 || p->aOp==0 ); - assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->db->mallocFailed ); + assert( p->aOp==0 || p->aOp[p->nOp-1].zComment==0 || p->pParse->nErr>0 ); if( p->nOp ){ assert( p->aOp ); sqlite3DbFree(p->db, p->aOp[p->nOp-1].zComment); @@ -70974,19 +86650,19 @@ SQLITE_PRIVATE void sqlite3VdbeNoopComment(Vdbe *p, const char *zFormat, ...){ ** Set the value if the iSrcLine field for the previously coded instruction. */ SQLITE_PRIVATE void sqlite3VdbeSetLineNumber(Vdbe *v, int iLine){ - sqlite3VdbeGetOp(v,-1)->iSrcLine = iLine; + sqlite3VdbeGetLastOp(v)->iSrcLine = iLine; } #endif /* SQLITE_VDBE_COVERAGE */ /* -** Return the opcode for a given address. If the address is -1, then -** return the most recently inserted opcode. +** Return the opcode for a given address. The address must be non-negative. +** See sqlite3VdbeGetLastOp() to get the most recently added opcode. ** ** If a memory allocation error has occurred prior to the calling of this ** routine, then a pointer to a dummy VdbeOp will be returned. That opcode ** is readable but not writable, though it is cast to a writable value. ** The return of a dummy opcode allows the call to continue functioning -** after an OOM fault without having to check to see if the return from +** after an OOM fault without having to check to see if the return from ** this routine is a valid pointer. But because the dummy.opcode is 0, ** dummy will never be written to. This is verified by code inspection and ** by running with Valgrind. @@ -70995,10 +86671,7 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ /* C89 specifies that the constant "dummy" will be initialized to all ** zeros, which is correct. MSVC generates a warning, nevertheless. */ static VdbeOp dummy; /* Ignore the MSVC warning about no initializer */ - assert( p->magic==VDBE_MAGIC_INIT ); - if( addr<0 ){ - addr = p->nOp - 1; - } + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( (addr>=0 && addrnOp) || p->db->mallocFailed ); if( p->db->mallocFailed ){ return (VdbeOp*)&dummy; @@ -71007,6 +86680,12 @@ SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetOp(Vdbe *p, int addr){ } } +/* Return the most recently added opcode +*/ +SQLITE_PRIVATE VdbeOp *sqlite3VdbeGetLastOp(Vdbe *p){ + return sqlite3VdbeGetOp(p, p->nOp - 1); +} + #if defined(SQLITE_ENABLE_EXPLAIN_COMMENTS) /* ** Return an integer value for one of the parameters to the opcode pOp @@ -71033,69 +86712,90 @@ static int translateP(char c, const Op *pOp){ ** "PX@PY+1" -> "r[X..X+Y]" or "r[x]" if y is 0 ** "PY..PY" -> "r[X..Y]" or "r[x]" if y<=x */ -static int displayComment( +SQLITE_PRIVATE char *sqlite3VdbeDisplayComment( + sqlite3 *db, /* Optional - Oom error reporting only */ const Op *pOp, /* The opcode to be commented */ - const char *zP4, /* Previously obtained value for P4 */ - char *zTemp, /* Write result here */ - int nTemp /* Space available in zTemp[] */ + const char *zP4 /* Previously obtained value for P4 */ ){ const char *zOpName; const char *zSynopsis; int nOpName; - int ii, jj; + int ii; + char zAlt[50]; + StrAccum x; + + sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); zOpName = sqlite3OpcodeName(pOp->opcode); nOpName = sqlite3Strlen30(zOpName); if( zOpName[nOpName+1] ){ int seenCom = 0; char c; - zSynopsis = zOpName += nOpName + 1; - for(ii=jj=0; jjzComment); - seenCom = 1; + if( pOp->zComment && pOp->zComment[0] ){ + sqlite3_str_appendall(&x, pOp->zComment); + seenCom = 1; + break; + } }else{ int v1 = translateP(c, pOp); int v2; - sqlite3_snprintf(nTemp-jj, zTemp+jj, "%d", v1); if( strncmp(zSynopsis+ii+1, "@P", 2)==0 ){ ii += 3; - jj += sqlite3Strlen30(zTemp+jj); v2 = translateP(zSynopsis[ii], pOp); if( strncmp(zSynopsis+ii+1,"+1",2)==0 ){ ii += 2; v2++; } - if( v2>1 ){ - sqlite3_snprintf(nTemp-jj, zTemp+jj, "..%d", v1+v2-1); + if( v2<2 ){ + sqlite3_str_appendf(&x, "%d", v1); + }else{ + sqlite3_str_appendf(&x, "%d..%d", v1, v1+v2-1); + } + }else if( strncmp(zSynopsis+ii+1, "@NP", 3)==0 ){ + sqlite3_context *pCtx = pOp->p4.pCtx; + if( pOp->p4type!=P4_FUNCCTX || pCtx->argc==1 ){ + sqlite3_str_appendf(&x, "%d", v1); + }else if( pCtx->argc>1 ){ + sqlite3_str_appendf(&x, "%d..%d", v1, v1+pCtx->argc-1); + }else if( x.accError==0 ){ + assert( x.nChar>2 ); + x.nChar -= 2; + ii++; + } + ii += 3; + }else{ + sqlite3_str_appendf(&x, "%d", v1); + if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ + ii += 4; } - }else if( strncmp(zSynopsis+ii+1, "..P3", 4)==0 && pOp->p3==0 ){ - ii += 4; } } - jj += sqlite3Strlen30(zTemp+jj); }else{ - zTemp[jj++] = c; + sqlite3_str_appendchar(&x, 1, c); } } - if( !seenCom && jjzComment ){ - sqlite3_snprintf(nTemp-jj, zTemp+jj, "; %s", pOp->zComment); - jj += sqlite3Strlen30(zTemp+jj); + if( !seenCom && pOp->zComment ){ + sqlite3_str_appendf(&x, "; %s", pOp->zComment); } - if( jjzComment ){ - sqlite3_snprintf(nTemp, zTemp, "%s", pOp->zComment); - jj = sqlite3Strlen30(zTemp); - }else{ - zTemp[0] = 0; - jj = 0; + sqlite3_str_appendall(&x, pOp->zComment); } - return jj; + if( (x.accError & SQLITE_NOMEM)!=0 && db!=0 ){ + sqlite3OomFault(db); + } + return sqlite3StrAccumFinish(&x); } -#endif /* SQLITE_DEBUG */ +#endif /* SQLITE_ENABLE_EXPLAIN_COMMENTS */ #if VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) /* @@ -71106,23 +86806,24 @@ static void displayP4Expr(StrAccum *p, Expr *pExpr){ const char *zOp = 0; switch( pExpr->op ){ case TK_STRING: - sqlite3XPrintf(p, "%Q", pExpr->u.zToken); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3_str_appendf(p, "%Q", pExpr->u.zToken); break; case TK_INTEGER: - sqlite3XPrintf(p, "%d", pExpr->u.iValue); + sqlite3_str_appendf(p, "%d", pExpr->u.iValue); break; case TK_NULL: - sqlite3XPrintf(p, "NULL"); + sqlite3_str_appendf(p, "NULL"); break; case TK_REGISTER: { - sqlite3XPrintf(p, "r[%d]", pExpr->iTable); + sqlite3_str_appendf(p, "r[%d]", pExpr->iTable); break; } case TK_COLUMN: { if( pExpr->iColumn<0 ){ - sqlite3XPrintf(p, "rowid"); + sqlite3_str_appendf(p, "rowid"); }else{ - sqlite3XPrintf(p, "c%d", (int)pExpr->iColumn); + sqlite3_str_appendf(p, "c%d", (int)pExpr->iColumn); } break; } @@ -71154,18 +86855,18 @@ static void displayP4Expr(StrAccum *p, Expr *pExpr){ case TK_NOTNULL: zOp = "NOTNULL"; break; default: - sqlite3XPrintf(p, "%s", "expr"); + sqlite3_str_appendf(p, "%s", "expr"); break; } if( zOp ){ - sqlite3XPrintf(p, "%s(", zOp); + sqlite3_str_appendf(p, "%s(", zOp); displayP4Expr(p, pExpr->pLeft); if( pExpr->pRight ){ - sqlite3StrAccumAppend(p, ",", 1); + sqlite3_str_append(p, ",", 1); displayP4Expr(p, pExpr->pRight); } - sqlite3StrAccumAppend(p, ")", 1); + sqlite3_str_append(p, ")", 1); } } #endif /* VDBE_DISPLAY_P4 && defined(SQLITE_ENABLE_CURSOR_HINTS) */ @@ -71176,24 +86877,27 @@ static void displayP4Expr(StrAccum *p, Expr *pExpr){ ** Compute a string that describes the P4 parameter for an opcode. ** Use zTemp for any required temporary buffer space. */ -static char *displayP4(Op *pOp, char *zTemp, int nTemp){ - char *zP4 = zTemp; +SQLITE_PRIVATE char *sqlite3VdbeDisplayP4(sqlite3 *db, Op *pOp){ + char *zP4 = 0; StrAccum x; - assert( nTemp>=20 ); - sqlite3StrAccumInit(&x, 0, zTemp, nTemp, 0); + + sqlite3StrAccumInit(&x, 0, 0, 0, SQLITE_MAX_LENGTH); switch( pOp->p4type ){ case P4_KEYINFO: { int j; KeyInfo *pKeyInfo = pOp->p4.pKeyInfo; - assert( pKeyInfo->aSortOrder!=0 ); - sqlite3XPrintf(&x, "k(%d", pKeyInfo->nField); - for(j=0; jnField; j++){ + assert( pKeyInfo->aSortFlags!=0 ); + sqlite3_str_appendf(&x, "k(%d", pKeyInfo->nKeyField); + for(j=0; jnKeyField; j++){ CollSeq *pColl = pKeyInfo->aColl[j]; const char *zColl = pColl ? pColl->zName : ""; if( strcmp(zColl, "BINARY")==0 ) zColl = "B"; - sqlite3XPrintf(&x, ",%s%s", pKeyInfo->aSortOrder[j] ? "-" : "", zColl); + sqlite3_str_appendf(&x, ",%s%s%s", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_DESC) ? "-" : "", + (pKeyInfo->aSortFlags[j] & KEYINFO_ORDER_BIGNULL)? "N." : "", + zColl); } - sqlite3StrAccumAppend(&x, ")", 1); + sqlite3_str_append(&x, ")", 1); break; } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -71203,42 +86907,43 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ } #endif case P4_COLLSEQ: { + static const char *const encnames[] = {"?", "8", "16LE", "16BE"}; CollSeq *pColl = pOp->p4.pColl; - sqlite3XPrintf(&x, "(%.20s)", pColl->zName); + assert( pColl->enc<4 ); + sqlite3_str_appendf(&x, "%.18s-%s", pColl->zName, + encnames[pColl->enc]); break; } case P4_FUNCDEF: { FuncDef *pDef = pOp->p4.pFunc; - sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); + sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } -#ifdef SQLITE_DEBUG case P4_FUNCCTX: { FuncDef *pDef = pOp->p4.pCtx->pFunc; - sqlite3XPrintf(&x, "%s(%d)", pDef->zName, pDef->nArg); + sqlite3_str_appendf(&x, "%s(%d)", pDef->zName, pDef->nArg); break; } -#endif case P4_INT64: { - sqlite3XPrintf(&x, "%lld", *pOp->p4.pI64); + sqlite3_str_appendf(&x, "%lld", *pOp->p4.pI64); break; } case P4_INT32: { - sqlite3XPrintf(&x, "%d", pOp->p4.i); + sqlite3_str_appendf(&x, "%d", pOp->p4.i); break; } case P4_REAL: { - sqlite3XPrintf(&x, "%.16g", *pOp->p4.pReal); + sqlite3_str_appendf(&x, "%.16g", *pOp->p4.pReal); break; } case P4_MEM: { Mem *pMem = pOp->p4.pMem; if( pMem->flags & MEM_Str ){ zP4 = pMem->z; - }else if( pMem->flags & MEM_Int ){ - sqlite3XPrintf(&x, "%lld", pMem->u.i); + }else if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + sqlite3_str_appendf(&x, "%lld", pMem->u.i); }else if( pMem->flags & MEM_Real ){ - sqlite3XPrintf(&x, "%.16g", pMem->u.r); + sqlite3_str_appendf(&x, "%.16g", pMem->u.r); }else if( pMem->flags & MEM_Null ){ zP4 = "NULL"; }else{ @@ -71250,45 +86955,43 @@ static char *displayP4(Op *pOp, char *zTemp, int nTemp){ #ifndef SQLITE_OMIT_VIRTUALTABLE case P4_VTAB: { sqlite3_vtab *pVtab = pOp->p4.pVtab->pVtab; - sqlite3XPrintf(&x, "vtab:%p", pVtab); + sqlite3_str_appendf(&x, "vtab:%p", pVtab); break; } #endif case P4_INTARRAY: { - int i; - int *ai = pOp->p4.ai; - int n = ai[0]; /* The first element of an INTARRAY is always the + u32 i; + u32 *ai = pOp->p4.ai; + u32 n = ai[0]; /* The first element of an INTARRAY is always the ** count of the number of elements to follow */ - for(i=1; ip4.pTab->zName); + zP4 = pOp->p4.pTab->zName; + break; + } + case P4_SUBRTNSIG: { + SubrtnSig *pSig = pOp->p4.pSubrtnSig; + sqlite3_str_appendf(&x, "subrtnsig:%d,%s", pSig->selId, pSig->zAff); break; } default: { zP4 = pOp->p4.z; - if( zP4==0 ){ - zP4 = zTemp; - zTemp[0] = 0; - } } } - sqlite3StrAccumFinish(&x); - assert( zP4!=0 ); - return zP4; + if( zP4 ) sqlite3_str_appendall(&x, zP4); + if( (x.accError & SQLITE_NOMEM)!=0 ){ + sqlite3OomFault(db); + } + return sqlite3StrAccumFinish(&x); } #endif /* VDBE_DISPLAY_P4 */ @@ -71319,13 +87022,13 @@ SQLITE_PRIVATE void sqlite3VdbeUsesBtree(Vdbe *p, int i){ ** ** If SQLite is not threadsafe but does support shared-cache mode, then ** sqlite3BtreeEnter() is invoked to set the BtShared.db variables -** of all of BtShared structures accessible via the database handle +** of all of BtShared structures accessible via the database handle ** associated with the VM. ** ** If SQLite is not threadsafe and does not support shared-cache mode, this ** function is a no-op. ** -** The p->btreeMask field is a bitmask of all btrees that the prepared +** The p->btreeMask field is a bitmask of all btrees that the prepared ** statement p will ever use. Let N be the number of bits in p->btreeMask ** corresponding to btrees that use shared cache. Then the runtime of ** this routine is N*N. But as N is rarely more than 1, this should not @@ -71376,31 +87079,71 @@ SQLITE_PRIVATE void sqlite3VdbeLeave(Vdbe *p){ /* ** Print a single opcode. This routine is used for debugging only. */ -SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, Op *pOp){ +SQLITE_PRIVATE void sqlite3VdbePrintOp(FILE *pOut, int pc, VdbeOp *pOp){ char *zP4; - char zPtr[50]; - char zCom[100]; + char *zCom; + sqlite3 dummyDb; static const char *zFormat1 = "%4d %-13s %4d %4d %4d %-13s %.2X %s\n"; if( pOut==0 ) pOut = stdout; - zP4 = displayP4(pOp, zPtr, sizeof(zPtr)); + sqlite3BeginBenignMalloc(); + dummyDb.mallocFailed = 1; + zP4 = sqlite3VdbeDisplayP4(&dummyDb, pOp); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - displayComment(pOp, zP4, zCom, sizeof(zCom)); + zCom = sqlite3VdbeDisplayComment(0, pOp, zP4); #else - zCom[0] = 0; + zCom = 0; #endif /* NB: The sqlite3OpcodeName() function is implemented by code created ** by the mkopcodeh.awk and mkopcodec.awk scripts which extract the ** information from the vdbe.c source text */ - fprintf(pOut, zFormat1, pc, - sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, zP4, pOp->p5, - zCom + fprintf(pOut, zFormat1, pc, + sqlite3OpcodeName(pOp->opcode), pOp->p1, pOp->p2, pOp->p3, + zP4 ? zP4 : "", pOp->p5, + zCom ? zCom : "" ); fflush(pOut); + sqlite3_free(zP4); + sqlite3_free(zCom); + sqlite3EndBenignMalloc(); } #endif /* -** Release an array of N Mem elements +** Initialize an array of N Mem element. +** +** This is a high-runner, so only those fields that really do need to +** be initialized are set. The Mem structure is organized so that +** the fields that get initialized are nearby and hopefully on the same +** cache line. +** +** Mem.flags = flags +** Mem.db = db +** Mem.szMalloc = 0 +** +** All other fields of Mem can safely remain uninitialized for now. They +** will be initialized before use. +*/ +static void initMemArray(Mem *p, int N, sqlite3 *db, u16 flags){ + if( N>0 ){ + do{ + p->flags = flags; + p->db = db; + p->szMalloc = 0; +#ifdef SQLITE_DEBUG + p->pScopyFrom = 0; +#endif + p++; + }while( (--N)>0 ); + } +} + +/* +** Release auxiliary memory held in an array of N Mem elements. +** +** After this routine returns, all Mem elements in the array will still +** be valid. Those Mem elements that were not holding auxiliary resources +** will be unchanged. Mem elements which had something freed will be +** set to MEM_Undefined. */ static void releaseMemArray(Mem *p, int N){ if( p && N ){ @@ -71417,33 +87160,181 @@ static void releaseMemArray(Mem *p, int N){ assert( sqlite3VdbeCheckMemInvariants(p) ); /* This block is really an inlined version of sqlite3VdbeMemRelease() - ** that takes advantage of the fact that the memory cell value is + ** that takes advantage of the fact that the memory cell value is ** being set to NULL after releasing any dynamic resources. ** - ** The justification for duplicating code is that according to - ** callgrind, this causes a certain test case to hit the CPU 4.7 - ** percent less (x86 linux, gcc version 4.1.2, -O6) than if + ** The justification for duplicating code is that according to + ** callgrind, this causes a certain test case to hit the CPU 4.7 + ** percent less (x86 linux, gcc version 4.1.2, -O6) than if ** sqlite3MemRelease() were called from here. With -O2, this jumps - ** to 6.6 percent. The test case is inserting 1000 rows into a table - ** with no indexes using a single prepared INSERT statement, bind() + ** to 6.6 percent. The test case is inserting 1000 rows into a table + ** with no indexes using a single prepared INSERT statement, bind() ** and reset(). Inserts are grouped into a transaction. */ testcase( p->flags & MEM_Agg ); testcase( p->flags & MEM_Dyn ); - testcase( p->flags & MEM_Frame ); - testcase( p->flags & MEM_RowSet ); - if( p->flags&(MEM_Agg|MEM_Dyn|MEM_Frame|MEM_RowSet) ){ + if( p->flags&(MEM_Agg|MEM_Dyn) ){ + testcase( (p->flags & MEM_Dyn)!=0 && p->xDel==sqlite3VdbeFrameMemDel ); sqlite3VdbeMemRelease(p); + p->flags = MEM_Undefined; }else if( p->szMalloc ){ - sqlite3DbFree(db, p->zMalloc); + sqlite3DbNNFreeNN(db, p->zMalloc); p->szMalloc = 0; + p->flags = MEM_Undefined; } - - p->flags = MEM_Undefined; +#ifdef SQLITE_DEBUG + else{ + p->flags = MEM_Undefined; + } +#endif }while( (++p)iFrameMagic!=SQLITE_FRAME_MAGIC ) return 0; + return 1; +} +#endif + + +/* +** This is a destructor on a Mem object (which is really an sqlite3_value) +** that deletes the Frame object that is attached to it as a blob. +** +** This routine does not delete the Frame right away. It merely adds the +** frame to a list of frames to be deleted when the Vdbe halts. +*/ +SQLITE_PRIVATE void sqlite3VdbeFrameMemDel(void *pArg){ + VdbeFrame *pFrame = (VdbeFrame*)pArg; + assert( sqlite3VdbeFrameIsValid(pFrame) ); + pFrame->pParent = pFrame->v->pDelFrame; + pFrame->v->pDelFrame = pFrame; +} + +#if defined(SQLITE_ENABLE_BYTECODE_VTAB) || !defined(SQLITE_OMIT_EXPLAIN) +/* +** Locate the next opcode to be displayed in EXPLAIN or EXPLAIN +** QUERY PLAN output. +** +** Return SQLITE_ROW on success. Return SQLITE_DONE if there are no +** more opcodes to be displayed. +*/ +SQLITE_PRIVATE int sqlite3VdbeNextOpcode( + Vdbe *p, /* The statement being explained */ + Mem *pSub, /* Storage for keeping track of subprogram nesting */ + int eMode, /* 0: normal. 1: EQP. 2: TablesUsed */ + int *piPc, /* IN/OUT: Current rowid. Overwritten with next rowid */ + int *piAddr, /* OUT: Write index into (*paOp)[] here */ + Op **paOp /* OUT: Write the opcode array here */ +){ + int nRow; /* Stop when row count reaches this */ + int nSub = 0; /* Number of sub-vdbes seen so far */ + SubProgram **apSub = 0; /* Array of sub-vdbes */ + int i; /* Next instruction address */ + int rc = SQLITE_OK; /* Result code */ + Op *aOp = 0; /* Opcode array */ + int iPc; /* Rowid. Copy of value in *piPc */ + + /* When the number of output rows reaches nRow, that means the + ** listing has finished and sqlite3_step() should return SQLITE_DONE. + ** nRow is the sum of the number of rows in the main program, plus + ** the sum of the number of rows in all trigger subprograms encountered + ** so far. The nRow value will increase as new trigger subprograms are + ** encountered, but p->pc will eventually catch up to nRow. + */ + nRow = p->nOp; + if( pSub!=0 ){ + if( pSub->flags&MEM_Blob ){ + /* pSub is initiallly NULL. It is initialized to a BLOB by + ** the P4_SUBPROGRAM processing logic below */ + nSub = pSub->n/sizeof(Vdbe*); + apSub = (SubProgram **)pSub->z; + } + for(i=0; inOp; + } + } + iPc = *piPc; + while(1){ /* Loop exits via break */ + i = iPc++; + if( i>=nRow ){ + p->rc = SQLITE_OK; + rc = SQLITE_DONE; + break; + } + if( inOp ){ + /* The rowid is small enough that we are still in the + ** main program. */ + aOp = p->aOp; + }else{ + /* We are currently listing subprograms. Figure out which one and + ** pick up the appropriate opcode. */ + int j; + i -= p->nOp; + assert( apSub!=0 ); + assert( nSub>0 ); + for(j=0; i>=apSub[j]->nOp; j++){ + i -= apSub[j]->nOp; + assert( inOp || j+1aOp; + } + + /* When an OP_Program opcode is encounter (the only opcode that has + ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms + ** kept in p->aMem[9].z to hold the new program - assuming this subprogram + ** has not already been seen. + */ + if( pSub!=0 && aOp[i].p4type==P4_SUBPROGRAM ){ + int nByte = (nSub+1)*sizeof(SubProgram*); + int j; + for(j=0; jrc = sqlite3VdbeMemGrow(pSub, nByte, nSub!=0); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + break; + } + apSub = (SubProgram **)pSub->z; + apSub[nSub++] = aOp[i].p4.pProgram; + MemSetTypeFlag(pSub, MEM_Blob); + pSub->n = nSub*sizeof(SubProgram*); + nRow += aOp[i].p4.pProgram->nOp; + } + } + if( eMode==0 ) break; +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + if( eMode==2 ){ + Op *pOp = aOp + i; + if( pOp->opcode==OP_OpenRead ) break; + if( pOp->opcode==OP_OpenWrite && (pOp->p5 & OPFLAG_P2ISREG)==0 ) break; + if( pOp->opcode==OP_ReopenIdx ) break; + }else +#endif + { + assert( eMode==1 ); + if( aOp[i].opcode==OP_Explain ) break; + if( aOp[i].opcode==OP_Init && iPc>1 ) break; + } + } + *piPc = iPc; + *piAddr = i; + *paOp = aOp; + return rc; +} +#endif /* SQLITE_ENABLE_BYTECODE_VTAB || !SQLITE_OMIT_EXPLAIN */ + + /* ** Delete a VdbeFrame object and its contents. VdbeFrame objects are ** allocated by the OP_Program opcode in sqlite3VdbeExec(). @@ -71452,8 +87343,9 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ int i; Mem *aMem = VdbeFrameMem(p); VdbeCursor **apCsr = (VdbeCursor **)&aMem[p->nChildMem]; + assert( sqlite3VdbeFrameIsValid(p) ); for(i=0; inChildCsr; i++){ - sqlite3VdbeFreeCursor(p->v, apCsr[i]); + if( apCsr[i] ) sqlite3VdbeFreeCursorNN(p->v, apCsr[i]); } releaseMemArray(aMem, p->nChildMem); sqlite3VdbeDeleteAuxData(p->v->db, &p->pAuxData, -1, 0); @@ -71472,6 +87364,9 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ ** p->explain==2, only OP_Explain instructions are listed and these ** are shown in a different format. p->explain==2 is used to implement ** EXPLAIN QUERY PLAN. +** 2018-04-24: In p->explain==2 mode, the OP_Init opcodes of triggers +** are also shown, so that the boundaries between the main program and +** each trigger are clear. ** ** When p->explain==1, first the main program is listed, then each of ** the trigger subprograms are listed one by one. @@ -71479,17 +87374,17 @@ SQLITE_PRIVATE void sqlite3VdbeFrameDelete(VdbeFrame *p){ SQLITE_PRIVATE int sqlite3VdbeList( Vdbe *p /* The VDBE */ ){ - int nRow; /* Stop when row count reaches this */ - int nSub = 0; /* Number of sub-vdbes seen so far */ - SubProgram **apSub = 0; /* Array of sub-vdbes */ Mem *pSub = 0; /* Memory cell hold array of subprogs */ sqlite3 *db = p->db; /* The database connection */ int i; /* Loop counter */ int rc = SQLITE_OK; /* Return code */ Mem *pMem = &p->aMem[1]; /* First Mem of result set */ + int bListSubprogs = (p->explain==1 || (db->flags & SQLITE_TriggerEQP)!=0); + Op *aOp; /* Array of opcodes */ + Op *pOp; /* Current opcode */ assert( p->explain ); - assert( p->magic==VDBE_MAGIC_RUN ); + assert( p->eVdbeState==VDBE_RUN_STATE ); assert( p->rc==SQLITE_OK || p->rc==SQLITE_BUSY || p->rc==SQLITE_NOMEM ); /* Even though this opcode does not use dynamic strings for @@ -71497,155 +87392,71 @@ SQLITE_PRIVATE int sqlite3VdbeList( ** sqlite3_column_text16(), causing a translation to UTF-16 encoding. */ releaseMemArray(pMem, 8); - p->pResultSet = 0; - if( p->rc==SQLITE_NOMEM_BKPT ){ + if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ sqlite3OomFault(db); return SQLITE_ERROR; } - /* When the number of output rows reaches nRow, that means the - ** listing has finished and sqlite3_step() should return SQLITE_DONE. - ** nRow is the sum of the number of rows in the main program, plus - ** the sum of the number of rows in all trigger subprograms encountered - ** so far. The nRow value will increase as new trigger subprograms are - ** encountered, but p->pc will eventually catch up to nRow. - */ - nRow = p->nOp; - if( p->explain==1 ){ + if( bListSubprogs ){ /* The first 8 memory cells are used for the result set. So we will ** commandeer the 9th cell to use as storage for an array of pointers ** to trigger subprograms. The VDBE is guaranteed to have at least 9 ** cells. */ assert( p->nMem>9 ); pSub = &p->aMem[9]; - if( pSub->flags&MEM_Blob ){ - /* On the first call to sqlite3_step(), pSub will hold a NULL. It is - ** initialized to a BLOB by the P4_SUBPROGRAM processing logic below */ - nSub = pSub->n/sizeof(Vdbe*); - apSub = (SubProgram **)pSub->z; - } - for(i=0; inOp; - } + }else{ + pSub = 0; } - do{ - i = p->pc++; - }while( iexplain==2 && p->aOp[i].opcode!=OP_Explain ); - if( i>=nRow ){ - p->rc = SQLITE_OK; - rc = SQLITE_DONE; - }else if( db->u1.isInterrupted ){ - p->rc = SQLITE_INTERRUPT; - rc = SQLITE_ERROR; - sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); - }else{ - char *zP4; - Op *pOp; - if( inOp ){ - /* The output line number is small enough that we are still in the - ** main program. */ - pOp = &p->aOp[i]; + /* Figure out which opcode is next to display */ + rc = sqlite3VdbeNextOpcode(p, pSub, p->explain==2, &p->pc, &i, &aOp); + + if( rc==SQLITE_OK ){ + pOp = aOp + i; + if( AtomicLoad(&db->u1.isInterrupted) ){ + p->rc = SQLITE_INTERRUPT; + rc = SQLITE_ERROR; + sqlite3VdbeError(p, sqlite3ErrStr(p->rc)); }else{ - /* We are currently listing subprograms. Figure out which one and - ** pick up the appropriate opcode. */ - int j; - i -= p->nOp; - for(j=0; i>=apSub[j]->nOp; j++){ - i -= apSub[j]->nOp; - } - pOp = &apSub[j]->aOp[i]; - } - if( p->explain==1 ){ - pMem->flags = MEM_Int; - pMem->u.i = i; /* Program counter */ - pMem++; - - pMem->flags = MEM_Static|MEM_Str|MEM_Term; - pMem->z = (char*)sqlite3OpcodeName(pOp->opcode); /* Opcode */ - assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - pMem++; - - /* When an OP_Program opcode is encounter (the only opcode that has - ** a P4_SUBPROGRAM argument), expand the size of the array of subprograms - ** kept in p->aMem[9].z to hold the new program - assuming this subprogram - ** has not already been seen. - */ - if( pOp->p4type==P4_SUBPROGRAM ){ - int nByte = (nSub+1)*sizeof(SubProgram*); - int j; - for(j=0; jp4.pProgram ) break; - } - if( j==nSub && SQLITE_OK==sqlite3VdbeMemGrow(pSub, nByte, nSub!=0) ){ - apSub = (SubProgram **)pSub->z; - apSub[nSub++] = pOp->p4.pProgram; - pSub->flags |= MEM_Blob; - pSub->n = nSub*sizeof(SubProgram*); - } - } - } - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p1; /* P1 */ - pMem++; - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p2; /* P2 */ - pMem++; - - pMem->flags = MEM_Int; - pMem->u.i = pOp->p3; /* P3 */ - pMem++; - - if( sqlite3VdbeMemClearAndResize(pMem, 100) ){ /* P4 */ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Str|MEM_Term; - zP4 = displayP4(pOp, pMem->z, pMem->szMalloc); - if( zP4!=pMem->z ){ - sqlite3VdbeMemSetStr(pMem, zP4, -1, SQLITE_UTF8, 0); - }else{ - assert( pMem->z!=0 ); - pMem->n = sqlite3Strlen30(pMem->z); - pMem->enc = SQLITE_UTF8; - } - pMem++; - - if( p->explain==1 ){ - if( sqlite3VdbeMemClearAndResize(pMem, 4) ){ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Str|MEM_Term; - pMem->n = 2; - sqlite3_snprintf(3, pMem->z, "%.2x", pOp->p5); /* P5 */ - pMem->enc = SQLITE_UTF8; - pMem++; - + char *zP4 = sqlite3VdbeDisplayP4(db, pOp); + if( p->explain==2 ){ + sqlite3VdbeMemSetInt64(pMem, pOp->p1); + sqlite3VdbeMemSetInt64(pMem+1, pOp->p2); + sqlite3VdbeMemSetInt64(pMem+2, pOp->p3); + sqlite3VdbeMemSetStr(pMem+3, zP4, -1, SQLITE_UTF8, sqlite3_free); + assert( p->nResColumn==4 ); + }else{ + sqlite3VdbeMemSetInt64(pMem+0, i); + sqlite3VdbeMemSetStr(pMem+1, (char*)sqlite3OpcodeName(pOp->opcode), + -1, SQLITE_UTF8, SQLITE_STATIC); + sqlite3VdbeMemSetInt64(pMem+2, pOp->p1); + sqlite3VdbeMemSetInt64(pMem+3, pOp->p2); + sqlite3VdbeMemSetInt64(pMem+4, pOp->p3); + /* pMem+5 for p4 is done last */ + sqlite3VdbeMemSetInt64(pMem+6, pOp->p5); #ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS - if( sqlite3VdbeMemClearAndResize(pMem, 500) ){ - assert( p->db->mallocFailed ); - return SQLITE_ERROR; - } - pMem->flags = MEM_Str|MEM_Term; - pMem->n = displayComment(pOp, zP4, pMem->z, 500); - pMem->enc = SQLITE_UTF8; + { + char *zCom = sqlite3VdbeDisplayComment(db, pOp, zP4); + sqlite3VdbeMemSetStr(pMem+7, zCom, -1, SQLITE_UTF8, sqlite3_free); + } #else - pMem->flags = MEM_Null; /* Comment */ + sqlite3VdbeMemSetNull(pMem+7); #endif + sqlite3VdbeMemSetStr(pMem+5, zP4, -1, SQLITE_UTF8, sqlite3_free); + assert( p->nResColumn==8 ); + } + p->pResultRow = pMem; + if( db->mallocFailed ){ + p->rc = SQLITE_NOMEM; + rc = SQLITE_ERROR; + }else{ + p->rc = SQLITE_OK; + rc = SQLITE_ROW; + } } - - p->nResColumn = 8 - 4*(p->explain-1); - p->pResultSet = &p->aMem[1]; - p->rc = SQLITE_OK; - rc = SQLITE_ROW; } return rc; } @@ -71705,9 +87516,9 @@ SQLITE_PRIVATE void sqlite3VdbeIOTraceSql(Vdbe *p){ ** of a ReusableSpace object by the allocSpace() routine below. */ struct ReusableSpace { - u8 *pSpace; /* Available memory */ - int nFree; /* Bytes of available memory */ - int nNeeded; /* Total bytes that could not be allocated */ + u8 *pSpace; /* Available memory */ + sqlite3_int64 nFree; /* Bytes of available memory */ + sqlite3_int64 nNeeded; /* Total bytes that could not be allocated */ }; /* Try to allocate nByte bytes of 8-byte aligned bulk memory for pBuf @@ -71727,11 +87538,11 @@ struct ReusableSpace { static void *allocSpace( struct ReusableSpace *p, /* Bulk memory available for allocation */ void *pBuf, /* Pointer to a prior allocation */ - int nByte /* Bytes of memory needed */ + sqlite3_int64 nByte /* Bytes of memory needed. */ ){ assert( EIGHT_BYTE_ALIGNMENT(p->pSpace) ); if( pBuf==0 ){ - nByte = ROUND8(nByte); + nByte = ROUND8P(nByte); if( nByte <= p->nFree ){ p->nFree -= nByte; pBuf = &p->pSpace[p->nFree]; @@ -71748,18 +87559,19 @@ static void *allocSpace( ** running it. */ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#if defined(SQLITE_DEBUG) int i; #endif assert( p!=0 ); - assert( p->magic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE + || p->eVdbeState==VDBE_READY_STATE + || p->eVdbeState==VDBE_HALT_STATE ); /* There should be at least one opcode. */ assert( p->nOp>0 ); - /* Set the magic to VDBE_MAGIC_RUN sooner rather than later. */ - p->magic = VDBE_MAGIC_RUN; + p->eVdbeState = VDBE_READY_STATE; #ifdef SQLITE_DEBUG for(i=0; inMem; i++){ @@ -71776,8 +87588,8 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ p->nFkConstraint = 0; #ifdef VDBE_PROFILE for(i=0; inOp; i++){ - p->aOp[i].cnt = 0; - p->aOp[i].cycles = 0; + p->aOp[i].nExec = 0; + p->aOp[i].nCycle = 0; } #endif } @@ -71787,11 +87599,11 @@ SQLITE_PRIVATE void sqlite3VdbeRewind(Vdbe *p){ ** creating the virtual machine. This involves things such ** as allocating registers and initializing the program counter. ** After the VDBE has be prepped, it can be executed by one or more -** calls to sqlite3VdbeExec(). +** calls to sqlite3VdbeExec(). ** ** This function may be called exactly once on each virtual machine. ** After this routine is called the VM has been "packaged" and is ready -** to run. After this routine is called, further calls to +** to run. After this routine is called, further calls to ** sqlite3VdbeAddOp() functions are prohibited. This routine disconnects ** the Vdbe from the Parse object that helped generate it so that the ** the Vdbe becomes an independent entity and the Parse object can be @@ -71809,24 +87621,23 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( int nMem; /* Number of VM memory registers */ int nCursor; /* Number of cursors required */ int nArg; /* Number of arguments in subprograms */ - int nOnce; /* Number of OP_Once instructions */ int n; /* Loop counter */ struct ReusableSpace x; /* Reusable bulk memory */ assert( p!=0 ); assert( p->nOp>0 ); assert( pParse!=0 ); - assert( p->magic==VDBE_MAGIC_INIT ); + assert( p->eVdbeState==VDBE_INIT_STATE ); assert( pParse==p->pParse ); + p->pVList = pParse->pVList; + pParse->pVList = 0; db = p->db; assert( db->mallocFailed==0 ); nVar = pParse->nVar; nMem = pParse->nMem; nCursor = pParse->nTab; nArg = pParse->nMaxArg; - nOnce = pParse->nOnce; - if( nOnce==0 ) nOnce = 1; /* Ensure at least one byte in p->aOnceFlag[] */ - + /* Each cursor uses a memory cell. The first cursor (cursor 0) can ** use aMem[0] which is not otherwise used by the VDBE program. Allocate ** space at the end of aMem[] for cursors 1 and greater. @@ -71839,95 +87650,95 @@ SQLITE_PRIVATE void sqlite3VdbeMakeReady( ** opcode array. This extra memory will be reallocated for other elements ** of the prepared statement. */ - n = ROUND8(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ + n = ROUND8P(sizeof(Op)*p->nOp); /* Bytes of opcode memory used */ x.pSpace = &((u8*)p->aOp)[n]; /* Unused opcode memory */ assert( EIGHT_BYTE_ALIGNMENT(x.pSpace) ); x.nFree = ROUNDDOWN8(pParse->szOpAlloc - n); /* Bytes of unused memory */ assert( x.nFree>=0 ); - if( x.nFree>0 ){ - memset(x.pSpace, 0, x.nFree); - assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); - } + assert( EIGHT_BYTE_ALIGNMENT(&x.pSpace[x.nFree]) ); resolveP2Values(p, &nArg); p->usesStmtJournal = (u8)(pParse->isMultiWrite && pParse->mayAbort); - if( pParse->explain && nMem<10 ){ - nMem = 10; + if( pParse->explain ){ + if( nMem<10 ) nMem = 10; + p->explain = pParse->explain; + p->nResColumn = 12 - 4*p->explain; } p->expired = 0; /* Memory for registers, parameters, cursor, etc, is allocated in one or two - ** passes. On the first pass, we try to reuse unused memory at the + ** passes. On the first pass, we try to reuse unused memory at the ** end of the opcode array. If we are unable to satisfy all memory ** requirements by reusing the opcode array tail, then the second - ** pass will fill in the remainder using a fresh memory allocation. + ** pass will fill in the remainder using a fresh memory allocation. ** ** This two-pass approach that reuses as much memory as possible from ** the leftover memory at the end of the opcode array. This can significantly ** reduce the amount of memory held by a prepared statement. */ - do { - x.nNeeded = 0; - p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); - p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); - p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); - p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); - p->aOnceFlag = allocSpace(&x, p->aOnceFlag, nOnce); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = allocSpace(&x, p->anExec, p->nOp*sizeof(i64)); -#endif - if( x.nNeeded==0 ) break; - x.pSpace = p->pFree = sqlite3DbMallocZero(db, x.nNeeded); + x.nNeeded = 0; + p->aMem = allocSpace(&x, 0, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, 0, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, 0, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, 0, nCursor*sizeof(VdbeCursor*)); + if( x.nNeeded ){ + x.pSpace = p->pFree = sqlite3DbMallocRawNN(db, x.nNeeded); x.nFree = x.nNeeded; - }while( !db->mallocFailed ); + if( !db->mallocFailed ){ + p->aMem = allocSpace(&x, p->aMem, nMem*sizeof(Mem)); + p->aVar = allocSpace(&x, p->aVar, nVar*sizeof(Mem)); + p->apArg = allocSpace(&x, p->apArg, nArg*sizeof(Mem*)); + p->apCsr = allocSpace(&x, p->apCsr, nCursor*sizeof(VdbeCursor*)); + } + } - p->nCursor = nCursor; - p->nOnceFlag = nOnce; - if( p->aVar ){ + if( db->mallocFailed ){ + p->nVar = 0; + p->nCursor = 0; + p->nMem = 0; + }else{ + p->nCursor = nCursor; p->nVar = (ynVar)nVar; - for(n=0; naVar[n].flags = MEM_Null; - p->aVar[n].db = db; - } - } - p->nzVar = pParse->nzVar; - p->azVar = pParse->azVar; - pParse->nzVar = 0; - pParse->azVar = 0; - if( p->aMem ){ + initMemArray(p->aVar, nVar, db, MEM_Null); p->nMem = nMem; - for(n=0; naMem[n].flags = MEM_Undefined; - p->aMem[n].db = db; - } + initMemArray(p->aMem, nMem, db, MEM_Undefined); + memset(p->apCsr, 0, nCursor*sizeof(VdbeCursor*)); } - p->explain = pParse->explain; sqlite3VdbeRewind(p); } /* -** Close a VDBE cursor and release all the resources that cursor +** Close a VDBE cursor and release all the resources that cursor ** happens to hold. */ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ - if( pCx==0 ){ + if( pCx ) sqlite3VdbeFreeCursorNN(p,pCx); +} +static SQLITE_NOINLINE void freeCursorWithCache(Vdbe *p, VdbeCursor *pCx){ + VdbeTxtBlbCache *pCache = pCx->pCache; + assert( pCx->colCache ); + pCx->colCache = 0; + pCx->pCache = 0; + if( pCache->pCValue ){ + sqlite3RCStrUnref(pCache->pCValue); + pCache->pCValue = 0; + } + sqlite3DbFree(p->db, pCache); + sqlite3VdbeFreeCursorNN(p, pCx); +} +SQLITE_PRIVATE void sqlite3VdbeFreeCursorNN(Vdbe *p, VdbeCursor *pCx){ + if( pCx->colCache ){ + freeCursorWithCache(p, pCx); return; } - assert( pCx->pBt==0 || pCx->eCurType==CURTYPE_BTREE ); switch( pCx->eCurType ){ case CURTYPE_SORTER: { sqlite3VdbeSorterClose(p->db, pCx); break; } case CURTYPE_BTREE: { - if( pCx->pBt ){ - sqlite3BtreeClose(pCx->pBt); - /* The pCx->pCursor will be close automatically, if it exists, by - ** the call above. */ - }else{ - assert( pCx->uc.pCursor!=0 ); - sqlite3BtreeCloseCursor(pCx->uc.pCursor); - } + assert( pCx->uc.pCursor!=0 ); + sqlite3BtreeCloseCursor(pCx->uc.pCursor); break; } #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -71947,14 +87758,12 @@ SQLITE_PRIVATE void sqlite3VdbeFreeCursor(Vdbe *p, VdbeCursor *pCx){ ** Close all cursors in the current frame. */ static void closeCursorsInFrame(Vdbe *p){ - if( p->apCsr ){ - int i; - for(i=0; inCursor; i++){ - VdbeCursor *pC = p->apCsr[i]; - if( pC ){ - sqlite3VdbeFreeCursor(p, pC); - p->apCsr[i] = 0; - } + int i; + for(i=0; inCursor; i++){ + VdbeCursor *pC = p->apCsr[i]; + if( pC ){ + sqlite3VdbeFreeCursorNN(p, pC); + p->apCsr[i] = 0; } } } @@ -71967,11 +87776,6 @@ static void closeCursorsInFrame(Vdbe *p){ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ Vdbe *v = pFrame->v; closeCursorsInFrame(v); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - v->anExec = pFrame->anExec; -#endif - v->aOnceFlag = pFrame->aOnceFlag; - v->nOnceFlag = pFrame->nOnceFlag; v->aOp = pFrame->aOp; v->nOp = pFrame->nOp; v->aMem = pFrame->aMem; @@ -71990,7 +87794,7 @@ SQLITE_PRIVATE int sqlite3VdbeFrameRestore(VdbeFrame *pFrame){ /* ** Close all cursors. ** -** Also release any dynamic memory held by the VM in the Vdbe.aMem memory +** Also release any dynamic memory held by the VM in the Vdbe.aMem memory ** cell array. This is necessary as the memory cell array may contain ** pointers to VdbeFrame objects, which may in turn contain pointers to ** open cursors. @@ -72005,9 +87809,7 @@ static void closeAllCursors(Vdbe *p){ } assert( p->nFrame==0 ); closeCursorsInFrame(p); - if( p->aMem ){ - releaseMemArray(p->aMem, p->nMem); - } + releaseMemArray(p->aMem, p->nMem); while( p->pDelFrame ){ VdbeFrame *pDel = p->pDelFrame; p->pDelFrame = pDel->pParent; @@ -72019,27 +87821,6 @@ static void closeAllCursors(Vdbe *p){ assert( p->pAuxData==0 ); } -/* -** Clean up the VM after a single run. -*/ -static void Cleanup(Vdbe *p){ - sqlite3 *db = p->db; - -#ifdef SQLITE_DEBUG - /* Execute assert() statements to ensure that the Vdbe.apCsr[] and - ** Vdbe.aMem[] arrays have already been cleaned up. */ - int i; - if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); - if( p->aMem ){ - for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); - } -#endif - - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - p->pResultSet = 0; -} - /* ** Set the number of result columns that will be returned by this SQL ** statement. This is now set at compile time, rather than during @@ -72047,21 +87828,18 @@ static void Cleanup(Vdbe *p){ ** be called on an SQL statement before sqlite3_step(). */ SQLITE_PRIVATE void sqlite3VdbeSetNumCols(Vdbe *p, int nResColumn){ - Mem *pColName; int n; sqlite3 *db = p->db; - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); - sqlite3DbFree(db, p->aColName); - n = nResColumn*COLNAME_N; - p->nResColumn = (u16)nResColumn; - p->aColName = pColName = (Mem*)sqlite3DbMallocZero(db, sizeof(Mem)*n ); - if( p->aColName==0 ) return; - while( n-- > 0 ){ - pColName->flags = MEM_Null; - pColName->db = p->db; - pColName++; + if( p->nResAlloc ){ + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); + sqlite3DbFree(db, p->aColName); } + n = nResColumn*COLNAME_N; + p->nResColumn = p->nResAlloc = (u16)nResColumn; + p->aColName = (Mem*)sqlite3DbMallocRawNN(db, sizeof(Mem)*n ); + if( p->aColName==0 ) return; + initMemArray(p->aColName, n, db, MEM_Null); } /* @@ -72083,14 +87861,14 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( ){ int rc; Mem *pColName; - assert( idxnResColumn ); + assert( idxnResAlloc ); assert( vardb->mallocFailed ){ assert( !zName || xDel!=SQLITE_DYNAMIC ); return SQLITE_NOMEM_BKPT; } assert( p->aColName!=0 ); - pColName = &(p->aColName[idx+var*p->nResColumn]); + pColName = &(p->aColName[idx+var*p->nResAlloc]); rc = sqlite3VdbeMemSetStr(pColName, zName, -1, SQLITE_UTF8, xDel); assert( rc!=0 || !zName || (pColName->flags&MEM_Term)!=0 ); return rc; @@ -72100,43 +87878,43 @@ SQLITE_PRIVATE int sqlite3VdbeSetColName( ** A read or write transaction may or may not be active on database handle ** db. If a transaction is active, commit it. If there is a ** write-transaction spanning more than one database file, this routine -** takes care of the master journal trickery. +** takes care of the super-journal trickery. */ static int vdbeCommit(sqlite3 *db, Vdbe *p){ int i; int nTrans = 0; /* Number of databases with an active write-transaction ** that are candidates for a two-phase commit using a - ** master-journal */ + ** super-journal */ int rc = SQLITE_OK; int needXcommit = 0; #ifdef SQLITE_OMIT_VIRTUALTABLE - /* With this option, sqlite3VtabSync() is defined to be simply - ** SQLITE_OK so p is not used. + /* With this option, sqlite3VtabSync() is defined to be simply + ** SQLITE_OK so p is not used. */ UNUSED_PARAMETER(p); #endif /* Before doing anything else, call the xSync() callback for any ** virtual module tables written in this transaction. This has to - ** be done before determining whether a master journal file is + ** be done before determining whether a super-journal file is ** required, as an xSync() callback may add an attached database ** to the transaction. */ rc = sqlite3VtabSync(db, p); /* This loop determines (a) if the commit hook should be invoked and - ** (b) how many database files have open write transactions, not - ** including the temp database. (b) is important because if more than - ** one database file has an open write transaction, a master journal + ** (b) how many database files have open write transactions, not + ** including the temp database. (b) is important because if more than + ** one database file has an open write transaction, a super-journal ** file is required for an atomic commit. - */ - for(i=0; rc==SQLITE_OK && inDb; i++){ + */ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( sqlite3BtreeIsInTrans(pBt) ){ - /* Whether or not a database might need a master journal depends upon + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ + /* Whether or not a database might need a super-journal depends upon ** its journal mode (among other things). This matrix determines which - ** journal modes use a master journal and which do not */ + ** journal modes use a super-journal and which do not */ static const u8 aMJNeeded[] = { /* DELETE */ 1, /* PERSIST */ 1, @@ -72151,7 +87929,8 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ pPager = sqlite3BtreePager(pBt); if( db->aDb[i].safety_level!=PAGER_SYNCHRONOUS_OFF && aMJNeeded[sqlite3PagerGetJournalMode(pPager)] - ){ + && sqlite3PagerIsMemdb(pPager)==0 + ){ assert( i!=1 ); nTrans++; } @@ -72173,11 +87952,11 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ /* The simple case - no more than one database file (not counting the ** TEMP database) has a transaction active. There is no need for the - ** master-journal. + ** super-journal. ** ** If the return value of sqlite3BtreeGetFilename() is a zero length - ** string, it means the main database is :memory: or a temp file. In - ** that case we do not support atomic multi-file commits, so use the + ** string, it means the main database is :memory: or a temp file. In + ** that case we do not support atomic multi-file commits, so use the ** simple case then too. */ if( 0==sqlite3Strlen30(sqlite3BtreeGetFilename(db->aDb[0].pBt)) @@ -72190,7 +87969,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } } - /* Do the commit only if all databases successfully complete phase 1. + /* Do the commit only if all databases successfully complete phase 1. ** If one of the BtreeCommitPhaseOne() calls fails, this indicates an ** IO error while deleting or truncating a journal file. It is unlikely, ** but could happen. In this case abandon processing and return the error. @@ -72207,124 +87986,125 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ } /* The complex case - There is a multi-file write-transaction active. - ** This requires a master journal file to ensure the transaction is + ** This requires a super-journal file to ensure the transaction is ** committed atomically. */ #ifndef SQLITE_OMIT_DISKIO else{ sqlite3_vfs *pVfs = db->pVfs; - char *zMaster = 0; /* File-name for the master journal */ + char *zSuper = 0; /* File-name for the super-journal */ char const *zMainFile = sqlite3BtreeGetFilename(db->aDb[0].pBt); - sqlite3_file *pMaster = 0; + sqlite3_file *pSuperJrnl = 0; i64 offset = 0; int res; int retryCount = 0; int nMainFile; - /* Select a master journal file name */ + /* Select a super-journal file name */ nMainFile = sqlite3Strlen30(zMainFile); - zMaster = sqlite3MPrintf(db, "%s-mjXXXXXX9XXz", zMainFile); - if( zMaster==0 ) return SQLITE_NOMEM_BKPT; + zSuper = sqlite3MPrintf(db, "%.4c%s%.16c", 0,zMainFile,0); + if( zSuper==0 ) return SQLITE_NOMEM_BKPT; + zSuper += 4; do { u32 iRandom; if( retryCount ){ if( retryCount>100 ){ - sqlite3_log(SQLITE_FULL, "MJ delete: %s", zMaster); - sqlite3OsDelete(pVfs, zMaster, 0); + sqlite3_log(SQLITE_FULL, "MJ delete: %s", zSuper); + sqlite3OsDelete(pVfs, zSuper, 0); break; }else if( retryCount==1 ){ - sqlite3_log(SQLITE_FULL, "MJ collide: %s", zMaster); + sqlite3_log(SQLITE_FULL, "MJ collide: %s", zSuper); } } retryCount++; sqlite3_randomness(sizeof(iRandom), &iRandom); - sqlite3_snprintf(13, &zMaster[nMainFile], "-mj%06X9%02X", + sqlite3_snprintf(13, &zSuper[nMainFile], "-mj%06X9%02X", (iRandom>>8)&0xffffff, iRandom&0xff); - /* The antipenultimate character of the master journal name must + /* The antipenultimate character of the super-journal name must ** be "9" to avoid name collisions when using 8+3 filenames. */ - assert( zMaster[sqlite3Strlen30(zMaster)-3]=='9' ); - sqlite3FileSuffix3(zMainFile, zMaster); - rc = sqlite3OsAccess(pVfs, zMaster, SQLITE_ACCESS_EXISTS, &res); + assert( zSuper[sqlite3Strlen30(zSuper)-3]=='9' ); + sqlite3FileSuffix3(zMainFile, zSuper); + rc = sqlite3OsAccess(pVfs, zSuper, SQLITE_ACCESS_EXISTS, &res); }while( rc==SQLITE_OK && res ); if( rc==SQLITE_OK ){ - /* Open the master journal. */ - rc = sqlite3OsOpenMalloc(pVfs, zMaster, &pMaster, + /* Open the super-journal. */ + rc = sqlite3OsOpenMalloc(pVfs, zSuper, &pSuperJrnl, SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE| - SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_MASTER_JOURNAL, 0 + SQLITE_OPEN_EXCLUSIVE|SQLITE_OPEN_SUPER_JOURNAL, 0 ); } if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zSuper-4); return rc; } - + /* Write the name of each database file in the transaction into the new - ** master journal file. If an error occurs at this point close - ** and delete the master journal file. All the individual journal files - ** still have 'null' as the master journal pointer, so they will roll + ** super-journal file. If an error occurs at this point close + ** and delete the super-journal file. All the individual journal files + ** still have 'null' as the super-journal pointer, so they will roll ** back independently if a failure occurs. */ for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( sqlite3BtreeIsInTrans(pBt) ){ + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ char const *zFile = sqlite3BtreeGetJournalname(pBt); if( zFile==0 ){ continue; /* Ignore TEMP and :memory: databases */ } assert( zFile[0]!=0 ); - rc = sqlite3OsWrite(pMaster, zFile, sqlite3Strlen30(zFile)+1, offset); + rc = sqlite3OsWrite(pSuperJrnl, zFile, sqlite3Strlen30(zFile)+1,offset); offset += sqlite3Strlen30(zFile)+1; if( rc!=SQLITE_OK ){ - sqlite3OsCloseFree(pMaster); - sqlite3OsDelete(pVfs, zMaster, 0); - sqlite3DbFree(db, zMaster); + sqlite3OsCloseFree(pSuperJrnl); + sqlite3OsDelete(pVfs, zSuper, 0); + sqlite3DbFree(db, zSuper-4); return rc; } } } - /* Sync the master journal file. If the IOCAP_SEQUENTIAL device + /* Sync the super-journal file. If the IOCAP_SEQUENTIAL device ** flag is set this is not required. */ - if( 0==(sqlite3OsDeviceCharacteristics(pMaster)&SQLITE_IOCAP_SEQUENTIAL) - && SQLITE_OK!=(rc = sqlite3OsSync(pMaster, SQLITE_SYNC_NORMAL)) + if( 0==(sqlite3OsDeviceCharacteristics(pSuperJrnl)&SQLITE_IOCAP_SEQUENTIAL) + && SQLITE_OK!=(rc = sqlite3OsSync(pSuperJrnl, SQLITE_SYNC_NORMAL)) ){ - sqlite3OsCloseFree(pMaster); - sqlite3OsDelete(pVfs, zMaster, 0); - sqlite3DbFree(db, zMaster); + sqlite3OsCloseFree(pSuperJrnl); + sqlite3OsDelete(pVfs, zSuper, 0); + sqlite3DbFree(db, zSuper-4); return rc; } /* Sync all the db files involved in the transaction. The same call - ** sets the master journal pointer in each individual journal. If - ** an error occurs here, do not delete the master journal file. + ** sets the super-journal pointer in each individual journal. If + ** an error occurs here, do not delete the super-journal file. ** ** If the error occurs during the first call to ** sqlite3BtreeCommitPhaseOne(), then there is a chance that the - ** master journal file will be orphaned. But we cannot delete it, - ** in case the master journal file name was written into the journal + ** super-journal file will be orphaned. But we cannot delete it, + ** in case the super-journal file name was written into the journal ** file before the failure occurred. */ - for(i=0; rc==SQLITE_OK && inDb; i++){ + for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ - rc = sqlite3BtreeCommitPhaseOne(pBt, zMaster); + rc = sqlite3BtreeCommitPhaseOne(pBt, zSuper); } } - sqlite3OsCloseFree(pMaster); + sqlite3OsCloseFree(pSuperJrnl); assert( rc!=SQLITE_BUSY ); if( rc!=SQLITE_OK ){ - sqlite3DbFree(db, zMaster); + sqlite3DbFree(db, zSuper-4); return rc; } - /* Delete the master journal file. This commits the transaction. After + /* Delete the super-journal file. This commits the transaction. After ** doing this the directory is synced again before any individual ** transaction files are deleted. */ - rc = sqlite3OsDelete(pVfs, zMaster, 1); - sqlite3DbFree(db, zMaster); - zMaster = 0; + rc = sqlite3OsDelete(pVfs, zSuper, 1); + sqlite3DbFree(db, zSuper-4); + zSuper = 0; if( rc ){ return rc; } @@ -72338,7 +88118,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ */ disable_simulated_io_errors(); sqlite3BeginBenignMalloc(); - for(i=0; inDb; i++){ + for(i=0; inDb; i++){ Btree *pBt = db->aDb[i].pBt; if( pBt ){ sqlite3BtreeCommitPhaseTwo(pBt, 1); @@ -72354,7 +88134,7 @@ static int vdbeCommit(sqlite3 *db, Vdbe *p){ return rc; } -/* +/* ** This routine checks that the sqlite3.nVdbeActive count variable ** matches the number of vdbe's in the list sqlite3.pVdbe that are ** currently active. An assertion fails if the two counts do not match. @@ -72376,7 +88156,7 @@ static void checkActiveVdbeCnt(sqlite3 *db){ if( p->readOnly==0 ) nWrite++; if( p->bIsReader ) nRead++; } - p = p->pNext; + p = p->pVNext; } assert( cnt==db->nVdbeActive ); assert( nWrite==db->nVdbeWrite ); @@ -72390,87 +88170,87 @@ static void checkActiveVdbeCnt(sqlite3 *db){ ** If the Vdbe passed as the first argument opened a statement-transaction, ** close it now. Argument eOp must be either SAVEPOINT_ROLLBACK or ** SAVEPOINT_RELEASE. If it is SAVEPOINT_ROLLBACK, then the statement -** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the +** transaction is rolled back. If eOp is SAVEPOINT_RELEASE, then the ** statement transaction is committed. ** -** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. +** If an IO error occurs, an SQLITE_IOERR_XXX error code is returned. ** Otherwise SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ +static SQLITE_NOINLINE int vdbeCloseStatement(Vdbe *p, int eOp){ sqlite3 *const db = p->db; int rc = SQLITE_OK; + int i; + const int iSavepoint = p->iStatement-1; - /* If p->iStatement is greater than zero, then this Vdbe opened a - ** statement transaction that should be closed here. The only exception - ** is that an IO error may have occurred, causing an emergency rollback. - ** In this case (db->nStatement==0), and there is nothing to do. - */ - if( db->nStatement && p->iStatement ){ - int i; - const int iSavepoint = p->iStatement-1; + assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); + assert( db->nStatement>0 ); + assert( p->iStatement==(db->nStatement+db->nSavepoint) ); - assert( eOp==SAVEPOINT_ROLLBACK || eOp==SAVEPOINT_RELEASE); - assert( db->nStatement>0 ); - assert( p->iStatement==(db->nStatement+db->nSavepoint) ); - - for(i=0; inDb; i++){ - int rc2 = SQLITE_OK; - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - if( eOp==SAVEPOINT_ROLLBACK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); - } - if( rc2==SQLITE_OK ){ - rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); - } - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - db->nStatement--; - p->iStatement = 0; - - if( rc==SQLITE_OK ){ + for(i=0; inDb; i++){ + int rc2 = SQLITE_OK; + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ if( eOp==SAVEPOINT_ROLLBACK ){ - rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_ROLLBACK, iSavepoint); + } + if( rc2==SQLITE_OK ){ + rc2 = sqlite3BtreeSavepoint(pBt, SAVEPOINT_RELEASE, iSavepoint); } if( rc==SQLITE_OK ){ - rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + rc = rc2; } } + } + db->nStatement--; + p->iStatement = 0; - /* If the statement transaction is being rolled back, also restore the - ** database handles deferred constraint counter to the value it had when - ** the statement transaction was opened. */ + if( rc==SQLITE_OK ){ if( eOp==SAVEPOINT_ROLLBACK ){ - db->nDeferredCons = p->nStmtDefCons; - db->nDeferredImmCons = p->nStmtDefImmCons; + rc = sqlite3VtabSavepoint(db, SAVEPOINT_ROLLBACK, iSavepoint); } + if( rc==SQLITE_OK ){ + rc = sqlite3VtabSavepoint(db, SAVEPOINT_RELEASE, iSavepoint); + } + } + + /* If the statement transaction is being rolled back, also restore the + ** database handles deferred constraint counter to the value it had when + ** the statement transaction was opened. */ + if( eOp==SAVEPOINT_ROLLBACK ){ + db->nDeferredCons = p->nStmtDefCons; + db->nDeferredImmCons = p->nStmtDefImmCons; } return rc; } +SQLITE_PRIVATE int sqlite3VdbeCloseStatement(Vdbe *p, int eOp){ + if( p->db->nStatement && p->iStatement ){ + return vdbeCloseStatement(p, eOp); + } + return SQLITE_OK; +} + /* -** This function is called when a transaction opened by the database -** handle associated with the VM passed as an argument is about to be +** This function is called when a transaction opened by the database +** handle associated with the VM passed as an argument is about to be ** committed. If there are outstanding deferred foreign key constraint ** violations, return SQLITE_ERROR. Otherwise, SQLITE_OK. ** -** If there are outstanding FK violations and this function returns +** If there are outstanding FK violations and this function returns ** SQLITE_ERROR, set the result of the VM to SQLITE_CONSTRAINT_FOREIGNKEY ** and write an error message to it. Then return SQLITE_ERROR. */ #ifndef SQLITE_OMIT_FOREIGN_KEY SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ sqlite3 *db = p->db; - if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) - || (!deferred && p->nFkConstraint>0) + if( (deferred && (db->nDeferredCons+db->nDeferredImmCons)>0) + || (!deferred && p->nFkConstraint>0) ){ p->rc = SQLITE_CONSTRAINT_FOREIGNKEY; p->errorAction = OE_Abort; sqlite3VdbeError(p, "FOREIGN KEY constraint failed"); - return SQLITE_ERROR; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ) return SQLITE_ERROR; + return SQLITE_CONSTRAINT_FOREIGNKEY; } return SQLITE_OK; } @@ -72481,9 +88261,9 @@ SQLITE_PRIVATE int sqlite3VdbeCheckFk(Vdbe *p, int deferred){ ** has made changes and is in autocommit mode, then commit those ** changes. If a rollback is needed, then do the rollback. ** -** This routine is the only way to move the state of a VM from -** SQLITE_MAGIC_RUN to SQLITE_MAGIC_HALT. It is harmless to -** call this on a VM that is in the SQLITE_MAGIC_HALT state. +** This routine is the only way to move the sqlite3eOpenState of a VM from +** SQLITE_STATE_RUN to SQLITE_STATE_HALT. It is harmless to +** call this on a VM that is in the SQLITE_STATE_HALT state. ** ** Return an error code. If the commit could not complete because of ** lock contention, return SQLITE_BUSY. If SQLITE_BUSY is returned, it @@ -72495,7 +88275,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ /* This function contains the logic that determines if a statement or ** transaction will be committed or rolled back as a result of the - ** execution of this virtual machine. + ** execution of this virtual machine. ** ** If any of the following errors occur: ** @@ -72509,19 +88289,16 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ ** one, or the complete transaction if there is no statement transaction. */ + assert( p->eVdbeState==VDBE_RUN_STATE ); if( db->mallocFailed ){ p->rc = SQLITE_NOMEM_BKPT; } - if( p->aOnceFlag ) memset(p->aOnceFlag, 0, p->nOnceFlag); closeAllCursors(p); - if( p->magic!=VDBE_MAGIC_RUN ){ - return SQLITE_OK; - } checkActiveVdbeCnt(db); /* No commit or rollback needed if the program never started or if the ** SQL statement does not read or write a database file. */ - if( p->pc>=0 && p->bIsReader ){ + if( p->bIsReader ){ int mrc; /* Primary error code from p->rc */ int eStatementOp = 0; int isSpecialError; /* Set to true if a 'special' error */ @@ -72530,20 +88307,26 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3VdbeEnter(p); /* Check for one of the special errors */ - mrc = p->rc & 0xff; - isSpecialError = mrc==SQLITE_NOMEM || mrc==SQLITE_IOERR - || mrc==SQLITE_INTERRUPT || mrc==SQLITE_FULL; + if( p->rc ){ + mrc = p->rc & 0xff; + isSpecialError = mrc==SQLITE_NOMEM + || mrc==SQLITE_IOERR + || mrc==SQLITE_INTERRUPT + || mrc==SQLITE_FULL; + }else{ + mrc = isSpecialError = 0; + } if( isSpecialError ){ - /* If the query was read-only and the error code is SQLITE_INTERRUPT, - ** no rollback is necessary. Otherwise, at least a savepoint - ** transaction must be rolled back to restore the database to a + /* If the query was read-only and the error code is SQLITE_INTERRUPT, + ** no rollback is necessary. Otherwise, at least a savepoint + ** transaction must be rolled back to restore the database to a ** consistent state. ** ** Even if the statement is read-only, it is important to perform - ** a statement or transaction rollback operation. If the error + ** a statement or transaction rollback operation. If the error ** occurred while writing to the journal, sub-journal or database ** file as part of an effort to free up cache space (see function - ** pagerStress() in pager.c), the rollback is required to restore + ** pagerStress() in pager.c), the rollback is required to restore ** the pager to a consistent state. */ if( !p->readOnly || mrc!=SQLITE_INTERRUPT ){ @@ -72562,19 +88345,19 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } /* Check for immediate foreign key violations. */ - if( p->rc==SQLITE_OK ){ - sqlite3VdbeCheckFk(p, 0); + if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ + (void)sqlite3VdbeCheckFk(p, 0); } - - /* If the auto-commit flag is set and this is the only active writer - ** VM, then we do either a commit or rollback of the current transaction. + + /* If the auto-commit flag is set and this is the only active writer + ** VM, then we do either a commit or rollback of the current transaction. ** - ** Note: This block also runs if one of the special errors handled - ** above has occurred. + ** Note: This block also runs if one of the special errors handled + ** above has occurred. */ - if( !sqlite3VtabInSync(db) - && db->autoCommit - && db->nVdbeWrite==(p->readOnly==0) + if( !sqlite3VtabInSync(db) + && db->autoCommit + && db->nVdbeWrite==(p->readOnly==0) ){ if( p->rc==SQLITE_OK || (p->errorAction==OE_Fail && !isSpecialError) ){ rc = sqlite3VdbeCheckFk(p, 1); @@ -72584,10 +88367,13 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ return SQLITE_ERROR; } rc = SQLITE_CONSTRAINT_FOREIGNKEY; - }else{ - /* The auto-commit flag is true, the vdbe program was successful + }else if( db->flags & SQLITE_CorruptRdOnly ){ + rc = SQLITE_CORRUPT; + db->flags &= ~SQLITE_CorruptRdOnly; + }else{ + /* The auto-commit flag is true, the vdbe program was successful ** or hit an 'OR FAIL' constraint and there are no deferred foreign - ** key constraints to hold up the transaction. This means a commit + ** key constraints to hold up the transaction. This means a commit ** is required. */ rc = vdbeCommit(db, p); } @@ -72595,15 +88381,18 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ sqlite3VdbeLeave(p); return SQLITE_BUSY; }else if( rc!=SQLITE_OK ){ + sqlite3SystemError(db, rc); p->rc = rc; sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; }else{ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)SQLITE_DeferFKs; sqlite3CommitInternalChanges(db); } + }else if( p->rc==SQLITE_SCHEMA && db->nVdbeActive>1 ){ + p->nChange = 0; }else{ sqlite3RollbackAll(db, SQLITE_OK); p->nChange = 0; @@ -72621,7 +88410,7 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ p->nChange = 0; } } - + /* If eStatementOp is non-zero, then a statement transaction needs to ** be committed or rolled back. Call sqlite3VdbeCloseStatement() to ** do so. If this operation returns an error, and the current statement @@ -72642,9 +88431,9 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ p->nChange = 0; } } - + /* If this was an INSERT, UPDATE or DELETE and no statement transaction - ** has been rolled back, update the database connection change-counter. + ** has been rolled back, update the database connection change-counter. */ if( p->changeCntOn ){ if( eStatementOp!=SAVEPOINT_ROLLBACK ){ @@ -72660,22 +88449,20 @@ SQLITE_PRIVATE int sqlite3VdbeHalt(Vdbe *p){ } /* We have successfully halted and closed the VM. Record this fact. */ - if( p->pc>=0 ){ - db->nVdbeActive--; - if( !p->readOnly ) db->nVdbeWrite--; - if( p->bIsReader ) db->nVdbeRead--; - assert( db->nVdbeActive>=db->nVdbeRead ); - assert( db->nVdbeRead>=db->nVdbeWrite ); - assert( db->nVdbeWrite>=0 ); - } - p->magic = VDBE_MAGIC_HALT; + db->nVdbeActive--; + if( !p->readOnly ) db->nVdbeWrite--; + if( p->bIsReader ) db->nVdbeRead--; + assert( db->nVdbeActive>=db->nVdbeRead ); + assert( db->nVdbeRead>=db->nVdbeWrite ); + assert( db->nVdbeWrite>=0 ); + p->eVdbeState = VDBE_HALT_STATE; checkActiveVdbeCnt(db); if( db->mallocFailed ){ p->rc = SQLITE_NOMEM_BKPT; } /* If the auto-commit flag is set to true, then any locks that were held - ** by connection db have now been released. Call sqlite3ConnectionUnlocked() + ** by connection db have now been released. Call sqlite3ConnectionUnlocked() ** to invoke any required unlock-notify callbacks. */ if( db->autoCommit ){ @@ -72697,7 +88484,7 @@ SQLITE_PRIVATE void sqlite3VdbeResetStepResult(Vdbe *p){ /* ** Copy the error code and error message belonging to the VDBE passed -** as the first argument to its database handle (so that they will be +** as the first argument to its database handle (so that they will be ** returned by calls to sqlite3_errcode() and sqlite3_errmsg()). ** ** This function does not clear the VDBE error code or message, just @@ -72713,16 +88500,17 @@ SQLITE_PRIVATE int sqlite3VdbeTransferError(Vdbe *p){ sqlite3ValueSetStr(db->pErr, -1, p->zErrMsg, SQLITE_UTF8, SQLITE_TRANSIENT); sqlite3EndBenignMalloc(); db->bBenignMalloc--; - db->errCode = rc; - }else{ - sqlite3Error(db, rc); + }else if( db->pErr ){ + sqlite3ValueSetNull(db->pErr); } + db->errCode = rc; + db->errByteOffset = -1; return rc; } #ifdef SQLITE_ENABLE_SQLLOG /* -** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, +** If an SQLITE_CONFIG_SQLLOG hook is registered and the VM has been run, ** invoke it. */ static void vdbeInvokeSqllog(Vdbe *v){ @@ -72749,10 +88537,14 @@ static void vdbeInvokeSqllog(Vdbe *v){ ** again. ** ** To look at it another way, this routine resets the state of the -** virtual machine from VDBE_MAGIC_RUN or VDBE_MAGIC_HALT back to -** VDBE_MAGIC_INIT. +** virtual machine from VDBE_RUN_STATE or VDBE_HALT_STATE back to +** VDBE_READY_STATE. */ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ +#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) + int i; +#endif + sqlite3 *db; db = p->db; @@ -72760,32 +88552,40 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ ** error, then it might not have been halted properly. So halt ** it now. */ - sqlite3VdbeHalt(p); + if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); - /* If the VDBE has be run even partially, then transfer the error code + /* If the VDBE has been run even partially, then transfer the error code ** and error message from the VDBE into the main database structure. But ** if the VDBE has just been set to run but has not actually executed any ** instructions yet, leave the main database error information unchanged. */ if( p->pc>=0 ){ vdbeInvokeSqllog(p); - sqlite3VdbeTransferError(p); - sqlite3DbFree(db, p->zErrMsg); - p->zErrMsg = 0; - if( p->runOnlyOnce ) p->expired = 1; - }else if( p->rc && p->expired ){ - /* The expired flag was set on the VDBE before the first call - ** to sqlite3_step(). For consistency (since sqlite3_step() was - ** called), set the database error in this case as well. - */ - sqlite3ErrorWithMsg(db, p->rc, p->zErrMsg ? "%s" : 0, p->zErrMsg); + if( db->pErr || p->zErrMsg ){ + sqlite3VdbeTransferError(p); + }else{ + db->errCode = p->rc; + } + } + + /* Reset register contents and reclaim error message memory. + */ +#ifdef SQLITE_DEBUG + /* Execute assert() statements to ensure that the Vdbe.apCsr[] and + ** Vdbe.aMem[] arrays have already been cleaned up. */ + if( p->apCsr ) for(i=0; inCursor; i++) assert( p->apCsr[i]==0 ); + if( p->aMem ){ + for(i=0; inMem; i++) assert( p->aMem[i].flags==MEM_Undefined ); + } +#endif + if( p->zErrMsg ){ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = 0; } - - /* Reclaim all memory used by the VDBE - */ - Cleanup(p); + p->pResultRow = 0; +#ifdef SQLITE_DEBUG + p->nWrite = 0; +#endif /* Save profiling information from this VDBE run. */ @@ -72793,7 +88593,6 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ { FILE *out = fopen("vdbe_profile.out", "a"); if( out ){ - int i; fprintf(out, "---- "); for(i=0; inOp; i++){ fprintf(out, "%02x", p->aOp[i].opcode); @@ -72811,10 +88610,12 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } for(i=0; inOp; i++){ char zHdr[100]; + i64 cnt = p->aOp[i].nExec; + i64 cycles = p->aOp[i].nCycle; sqlite3_snprintf(sizeof(zHdr), zHdr, "%6u %12llu %8llu ", - p->aOp[i].cnt, - p->aOp[i].cycles, - p->aOp[i].cnt>0 ? p->aOp[i].cycles/p->aOp[i].cnt : 0 + cnt, + cycles, + cnt>0 ? cycles/cnt : 0 ); fprintf(out, "%s", zHdr); sqlite3VdbePrintOp(out, i, &p->aOp[i]); @@ -72823,18 +88624,19 @@ SQLITE_PRIVATE int sqlite3VdbeReset(Vdbe *p){ } } #endif - p->iCurrentTime = 0; - p->magic = VDBE_MAGIC_INIT; return p->rc & db->errMask; } - + /* ** Clean up and delete a VDBE after execution. Return an integer which is ** the result code. Write any error message text into *pzErrMsg. */ SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ int rc = SQLITE_OK; - if( p->magic==VDBE_MAGIC_RUN || p->magic==VDBE_MAGIC_HALT ){ + assert( VDBE_RUN_STATE>VDBE_READY_STATE ); + assert( VDBE_HALT_STATE>VDBE_READY_STATE ); + assert( VDBE_INIT_STATEeVdbeState>=VDBE_READY_STATE ){ rc = sqlite3VdbeReset(p); assert( (rc & p->db->errMask)==rc ); } @@ -72848,8 +88650,8 @@ SQLITE_PRIVATE int sqlite3VdbeFinalize(Vdbe *p){ ** the first argument. ** ** Or, if iOp is greater than or equal to zero, then the destructor is -** only invoked for those auxiliary data pointers created by the user -** function invoked by the OP_Function opcode at instruction iOp of +** only invoked for those auxiliary data pointers created by the user +** function invoked by the OP_Function opcode at instruction iOp of ** VM pVdbe, and only then if: ** ** * the associated function parameter is the 32nd or later (counting @@ -72862,16 +88664,18 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, while( *pp ){ AuxData *pAux = *pp; if( (iOp<0) - || (pAux->iOp==iOp && (pAux->iArg>31 || !(mask & MASKBIT32(pAux->iArg)))) + || (pAux->iAuxOp==iOp + && pAux->iAuxArg>=0 + && (pAux->iAuxArg>31 || !(mask & MASKBIT32(pAux->iAuxArg)))) ){ - testcase( pAux->iArg==31 ); - if( pAux->xDelete ){ - pAux->xDelete(pAux->pAux); + testcase( pAux->iAuxArg==31 ); + if( pAux->xDeleteAux ){ + pAux->xDeleteAux(pAux->pAux); } - *pp = pAux->pNext; + *pp = pAux->pNextAux; sqlite3DbFree(db, pAux); }else{ - pp= &pAux->pNext; + pp= &pAux->pNextAux; } } } @@ -72884,28 +88688,44 @@ SQLITE_PRIVATE void sqlite3VdbeDeleteAuxData(sqlite3 *db, AuxData **pp, int iOp, ** VdbeDelete() also unlinks the Vdbe from the list of VMs associated with ** the database connection and frees the object itself. */ -SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ +static void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SubProgram *pSub, *pNext; - int i; + assert( db!=0 ); assert( p->db==0 || p->db==db ); - releaseMemArray(p->aVar, p->nVar); - releaseMemArray(p->aColName, p->nResColumn*COLNAME_N); + if( p->aColName ){ + releaseMemArray(p->aColName, p->nResAlloc*COLNAME_N); + sqlite3DbNNFreeNN(db, p->aColName); + } for(pSub=p->pProgram; pSub; pSub=pNext){ pNext = pSub->pNext; vdbeFreeOpArray(db, pSub->aOp, pSub->nOp); sqlite3DbFree(db, pSub); } - for(i=p->nzVar-1; i>=0; i--) sqlite3DbFree(db, p->azVar[i]); - sqlite3DbFree(db, p->azVar); - vdbeFreeOpArray(db, p->aOp, p->nOp); - sqlite3DbFree(db, p->aColName); - sqlite3DbFree(db, p->zSql); - sqlite3DbFree(db, p->pFree); -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - for(i=0; inScan; i++){ - sqlite3DbFree(db, p->aScan[i].zName); + if( p->eVdbeState!=VDBE_INIT_STATE ){ + releaseMemArray(p->aVar, p->nVar); + if( p->pVList ) sqlite3DbNNFreeNN(db, p->pVList); + if( p->pFree ) sqlite3DbNNFreeNN(db, p->pFree); + } + vdbeFreeOpArray(db, p->aOp, p->nOp); + if( p->zSql ) sqlite3DbNNFreeNN(db, p->zSql); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3DbFree(db, p->zNormSql); + { + DblquoteStr *pThis, *pNxt; + for(pThis=p->pDblStr; pThis; pThis=pNxt){ + pNxt = pThis->pNextStr; + sqlite3DbFree(db, pThis); + } + } +#endif +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + { + int i; + for(i=0; inScan; i++){ + sqlite3DbFree(db, p->aScan[i].zName); + } + sqlite3DbFree(db, p->aScan); } - sqlite3DbFree(db, p->aScan); #endif } @@ -72915,22 +88735,19 @@ SQLITE_PRIVATE void sqlite3VdbeClearObject(sqlite3 *db, Vdbe *p){ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ sqlite3 *db; - if( NEVER(p==0) ) return; + assert( p!=0 ); db = p->db; + assert( db!=0 ); assert( sqlite3_mutex_held(db->mutex) ); sqlite3VdbeClearObject(db, p); - if( p->pPrev ){ - p->pPrev->pNext = p->pNext; - }else{ - assert( db->pVdbe==p ); - db->pVdbe = p->pNext; + if( db->pnBytesFreed==0 ){ + assert( p->ppVPrev!=0 ); + *p->ppVPrev = p->pVNext; + if( p->pVNext ){ + p->pVNext->ppVPrev = p->ppVPrev; + } } - if( p->pNext ){ - p->pNext->pPrev = p->pPrev; - } - p->magic = VDBE_MAGIC_DEAD; - p->db = 0; - sqlite3DbFree(db, p); + sqlite3DbNNFreeNN(db, p); } /* @@ -72938,7 +88755,7 @@ SQLITE_PRIVATE void sqlite3VdbeDelete(Vdbe *p){ ** carried out. Seek the cursor now. If an error occurs, return ** the appropriate error code. */ -static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeFinishMoveto(VdbeCursor *p){ int res, rc; #ifdef SQLITE_TEST extern int sqlite3_search_count; @@ -72946,7 +88763,7 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ assert( p->deferredMoveto ); assert( p->isTable ); assert( p->eCurType==CURTYPE_BTREE ); - rc = sqlite3BtreeMovetoUnpacked(p->uc.pCursor, 0, p->movetoTarget, 0, &res); + rc = sqlite3BtreeTableMoveto(p->uc.pCursor, p->movetoTarget, 0, &res); if( rc ) return rc; if( res!=0 ) return SQLITE_CORRUPT_BKPT; #ifdef SQLITE_TEST @@ -72964,7 +88781,7 @@ static int SQLITE_NOINLINE handleDeferredMoveto(VdbeCursor *p){ ** is supposed to be pointing. If the row was deleted out from under the ** cursor, set the cursor to point to a NULL row. */ -static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ +SQLITE_PRIVATE int SQLITE_NOINLINE sqlite3VdbeHandleMovedCursor(VdbeCursor *p){ int isDifferentRow, rc; assert( p->eCurType==CURTYPE_BTREE ); assert( p->uc.pCursor!=0 ); @@ -72980,41 +88797,9 @@ static int SQLITE_NOINLINE handleMovedCursor(VdbeCursor *p){ ** if need be. Return any I/O error from the restore operation. */ SQLITE_PRIVATE int sqlite3VdbeCursorRestore(VdbeCursor *p){ - assert( p->eCurType==CURTYPE_BTREE ); + assert( p->eCurType==CURTYPE_BTREE || IsNullCursor(p) ); if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ - return handleMovedCursor(p); - } - return SQLITE_OK; -} - -/* -** Make sure the cursor p is ready to read or write the row to which it -** was last positioned. Return an error code if an OOM fault or I/O error -** prevents us from positioning the cursor to its correct position. -** -** If a MoveTo operation is pending on the given cursor, then do that -** MoveTo now. If no move is pending, check to see if the row has been -** deleted out from under the cursor and if it has, mark the row as -** a NULL row. -** -** If the cursor is already pointing to the correct row and that row has -** not been deleted out from under the cursor, then this routine is a no-op. -*/ -SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ - VdbeCursor *p = *pp; - if( p->eCurType==CURTYPE_BTREE ){ - if( p->deferredMoveto ){ - int iMap; - if( p->aAltMap && (iMap = p->aAltMap[1+*piCol])>0 ){ - *pp = p->pAltCursor; - *piCol = iMap - 1; - return SQLITE_OK; - } - return handleDeferredMoveto(p); - } - if( sqlite3BtreeCursorHasMoved(p->uc.pCursor) ){ - return handleMovedCursor(p); - } + return sqlite3VdbeHandleMovedCursor(p); } return SQLITE_OK; } @@ -73025,7 +88810,7 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ ** sqlite3VdbeSerialType() ** sqlite3VdbeSerialTypeLen() ** sqlite3VdbeSerialLen() -** sqlite3VdbeSerialPut() +** sqlite3VdbeSerialPut() <--- in-lined into OP_MakeRecord as of 2022-04-02 ** sqlite3VdbeSerialGet() ** ** encapsulate the code that serializes values for storage in SQLite @@ -73061,8 +88846,17 @@ SQLITE_PRIVATE int sqlite3VdbeCursorMoveto(VdbeCursor **pp, int *piCol){ ** of SQLite will not understand those serial types. */ +#if 0 /* Inlined into the OP_MakeRecord opcode */ /* ** Return the serial-type for the value stored in pMem. +** +** This routine might convert a large MEM_IntReal value into MEM_Real. +** +** 2019-07-11: The primary user of this subroutine was the OP_MakeRecord +** opcode in the byte-code engine. But by moving this routine in-line, we +** can omit some redundant tests and make that opcode a lot faster. So +** this routine is now only used by the STAT3 logic and STAT3 support has +** ended. The code is kept here for historical reference only. */ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ int flags = pMem->flags; @@ -73073,11 +88867,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ *pLen = 0; return 0; } - if( flags&MEM_Int ){ + if( flags&(MEM_Int|MEM_IntReal) ){ /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ # define MAX_6BYTE ((((i64)0x00008000)<<32)-1) i64 i = pMem->u.i; u64 u; + testcase( flags & MEM_Int ); + testcase( flags & MEM_IntReal ); if( i<0 ){ u = ~i; }else{ @@ -73097,6 +88893,15 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ if( u<=2147483647 ){ *pLen = 4; return 4; } if( u<=MAX_6BYTE ){ *pLen = 6; return 5; } *pLen = 8; + if( flags&MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pMem->u.r = (double)pMem->u.i; + pMem->flags &= ~MEM_IntReal; + pMem->flags |= MEM_Real; + return 7; + } return 6; } if( flags&MEM_Real ){ @@ -73112,12 +88917,13 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialType(Mem *pMem, int file_format, u32 *pLen){ *pLen = n; return ((n*2) + 12 + ((flags&MEM_Str)!=0)); } +#endif /* inlined into OP_MakeRecord */ /* ** The sizes for serial types less than 128 */ -static const u8 sqlite3SmallTypeSizes[] = { - /* 0 1 2 3 4 5 6 7 8 9 */ +SQLITE_PRIVATE const u8 sqlite3SmallTypeSizes[128] = { + /* 0 1 2 3 4 5 6 7 8 9 */ /* 0 */ 0, 1, 2, 3, 4, 6, 8, 8, 0, 0, /* 10 */ 0, 0, 0, 0, 1, 1, 2, 2, 3, 3, /* 20 */ 4, 4, 5, 5, 6, 6, 7, 7, 8, 8, @@ -73140,19 +88946,19 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialTypeLen(u32 serial_type){ if( serial_type>=128 ){ return (serial_type-12)/2; }else{ - assert( serial_type<12 + assert( serial_type<12 || sqlite3SmallTypeSizes[serial_type]==(serial_type - 12)/2 ); return sqlite3SmallTypeSizes[serial_type]; } } SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ assert( serial_type<128 ); - return sqlite3SmallTypeSizes[serial_type]; + return sqlite3SmallTypeSizes[serial_type]; } /* -** If we are on an architecture with mixed-endian floating -** points (ex: ARM7) then swap the lower 4 bytes with the +** If we are on an architecture with mixed-endian floating +** points (ex: ARM7) then swap the lower 4 bytes with the ** upper 4 bytes. Return the result. ** ** For most architectures, this is a no-op. @@ -73174,7 +88980,7 @@ SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ ** (2007-08-30) Frank van Vugt has studied this problem closely ** and has send his findings to the SQLite developers. Frank ** writes that some Linux kernels offer floating point hardware -** emulation that uses only 32-bit mantissas instead of a full +** emulation that uses only 32-bit mantissas instead of a full ** 48-bits as required by the IEEE standard. (This is the ** CONFIG_FPE_FASTFPE option.) On such systems, floating point ** byte swapping becomes very complicated. To avoid problems, @@ -73185,7 +88991,7 @@ SQLITE_PRIVATE u8 sqlite3VdbeOneByteSerialTypeLen(u8 serial_type){ ** so we trust him. */ #ifdef SQLITE_MIXED_ENDIAN_64BIT_FLOAT -static u64 floatSwap(u64 in){ +SQLITE_PRIVATE u64 sqlite3FloatSwap(u64 in){ union { u64 r; u32 i[2]; @@ -73198,59 +89004,8 @@ static u64 floatSwap(u64 in){ u.i[1] = t; return u.r; } -# define swapMixedEndianFloat(X) X = floatSwap(X) -#else -# define swapMixedEndianFloat(X) -#endif +#endif /* SQLITE_MIXED_ENDIAN_64BIT_FLOAT */ -/* -** Write the serialized data blob for the value stored in pMem into -** buf. It is assumed that the caller has allocated sufficient space. -** Return the number of bytes written. -** -** nBuf is the amount of space left in buf[]. The caller is responsible -** for allocating enough space to buf[] to hold the entire field, exclusive -** of the pMem->u.nZero bytes for a MEM_Zero value. -** -** Return the number of bytes actually written into buf[]. The number -** of bytes in the zero-filled tail is included in the return value only -** if those bytes were zeroed in buf[]. -*/ -SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ - u32 len; - - /* Integer and Real */ - if( serial_type<=7 && serial_type>0 ){ - u64 v; - u32 i; - if( serial_type==7 ){ - assert( sizeof(v)==sizeof(pMem->u.r) ); - memcpy(&v, &pMem->u.r, sizeof(v)); - swapMixedEndianFloat(v); - }else{ - v = pMem->u.i; - } - len = i = sqlite3SmallTypeSizes[serial_type]; - assert( i>0 ); - do{ - buf[--i] = (u8)(v&0xFF); - v >>= 8; - }while( i ); - return len; - } - - /* String or blob */ - if( serial_type>=12 ){ - assert( pMem->n + ((pMem->flags & MEM_Zero)?pMem->u.nZero:0) - == (int)sqlite3VdbeSerialTypeLen(serial_type) ); - len = pMem->n; - if( len>0 ) memcpy(buf, pMem->z, len); - return len; - } - - /* NULL or constants 0 or 1 */ - return 0; -} /* Input "x" is a sequence of unsigned characters that represent a ** big-endian integer. Return the equivalent native integer @@ -73263,14 +89018,14 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialPut(u8 *buf, Mem *pMem, u32 serial_type){ /* ** Deserialize the data blob pointed to by buf as serial type serial_type -** and store the result in pMem. Return the number of bytes read. +** and store the result in pMem. ** ** This function is implemented as two separate routines for performance. ** The few cases that require local variables are broken out into a separate ** routine so that in most cases the overhead of moving the stack pointer ** is avoided. -*/ -static u32 SQLITE_NOINLINE serialGet( +*/ +static void serialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ @@ -73302,22 +89057,44 @@ static u32 SQLITE_NOINLINE serialGet( assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); swapMixedEndianFloat(x); memcpy(&pMem->u.r, &x, sizeof(x)); - pMem->flags = sqlite3IsNaN(pMem->u.r) ? MEM_Null : MEM_Real; + pMem->flags = IsNaN(x) ? MEM_Null : MEM_Real; } - return 8; } -SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( +static int serialGet7( + const unsigned char *buf, /* Buffer to deserialize from */ + Mem *pMem /* Memory cell to write value into */ +){ + u64 x = FOUR_BYTE_UINT(buf); + u32 y = FOUR_BYTE_UINT(buf+4); + x = (x<<32) + y; + assert( sizeof(x)==8 && sizeof(pMem->u.r)==8 ); + swapMixedEndianFloat(x); + memcpy(&pMem->u.r, &x, sizeof(x)); + if( IsNaN(x) ){ + pMem->flags = MEM_Null; + return 1; + } + pMem->flags = MEM_Real; + return 0; +} +SQLITE_PRIVATE void sqlite3VdbeSerialGet( const unsigned char *buf, /* Buffer to deserialize from */ u32 serial_type, /* Serial type to deserialize */ Mem *pMem /* Memory cell to write value into */ ){ switch( serial_type ){ - case 10: /* Reserved for future use */ + case 10: { /* Internal use only: NULL with virtual table + ** UPDATE no-change flag set */ + pMem->flags = MEM_Null|MEM_Zero; + pMem->n = 0; + pMem->u.nZero = 0; + return; + } case 11: /* Reserved for future use */ case 0: { /* Null */ /* EVIDENCE-OF: R-24078-09375 Value is a NULL. */ pMem->flags = MEM_Null; - break; + return; } case 1: { /* EVIDENCE-OF: R-44885-25196 Value is an 8-bit twos-complement @@ -73325,7 +89102,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = ONE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 1; + return; } case 2: { /* 2-byte signed integer */ /* EVIDENCE-OF: R-49794-35026 Value is a big-endian 16-bit @@ -73333,7 +89110,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 2; + return; } case 3: { /* 3-byte signed integer */ /* EVIDENCE-OF: R-37839-54301 Value is a big-endian 24-bit @@ -73341,19 +89118,19 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = THREE_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 3; + return; } case 4: { /* 4-byte signed integer */ /* EVIDENCE-OF: R-01849-26079 Value is a big-endian 32-bit ** twos-complement integer. */ pMem->u.i = FOUR_BYTE_INT(buf); -#ifdef __HP_cc +#ifdef __HP_cc /* Work around a sign-extension bug in the HP compiler for HP/UX */ if( buf[0]&0x80 ) pMem->u.i |= 0xffffffff80000000LL; #endif pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 4; + return; } case 5: { /* 6-byte signed integer */ /* EVIDENCE-OF: R-50385-09674 Value is a big-endian 48-bit @@ -73361,13 +89138,14 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->u.i = FOUR_BYTE_UINT(buf+2) + (((i64)1)<<32)*TWO_BYTE_INT(buf); pMem->flags = MEM_Int; testcase( pMem->u.i<0 ); - return 6; + return; } case 6: /* 8-byte signed integer */ case 7: { /* IEEE floating point */ /* These use local variables, so do them in a separate routine ** to avoid having to move the frame pointer in the common case */ - return serialGet(buf,serial_type,pMem); + serialGet(buf,serial_type,pMem); + return; } case 8: /* Integer 0 */ case 9: { /* Integer 1 */ @@ -73375,7 +89153,7 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( /* EVIDENCE-OF: R-18143-12121 Value is the integer 1. */ pMem->u.i = serial_type-8; pMem->flags = MEM_Int; - return 0; + return; } default: { /* EVIDENCE-OF: R-14606-31564 Value is a BLOB that is (N-12)/2 bytes in @@ -73386,10 +89164,10 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( pMem->z = (char *)buf; pMem->n = (serial_type-12)/2; pMem->flags = aFlag[serial_type&1]; - return pMem->n; + return; } } - return 0; + return; } /* ** This routine is used to allocate sufficient space for an UnpackedRecord @@ -73399,49 +89177,32 @@ SQLITE_PRIVATE u32 sqlite3VdbeSerialGet( ** The space is either allocated using sqlite3DbMallocRaw() or from within ** the unaligned buffer passed via the second and third arguments (presumably ** stack space). If the former, then *ppFree is set to a pointer that should -** be eventually freed by the caller using sqlite3DbFree(). Or, if the +** be eventually freed by the caller using sqlite3DbFree(). Or, if the ** allocation comes from the pSpace/szSpace buffer, *ppFree is set to NULL ** before returning. ** ** If an OOM error occurs, NULL is returned. */ SQLITE_PRIVATE UnpackedRecord *sqlite3VdbeAllocUnpackedRecord( - KeyInfo *pKeyInfo, /* Description of the record */ - char *pSpace, /* Unaligned space available */ - int szSpace, /* Size of pSpace[] in bytes */ - char **ppFree /* OUT: Caller should free this pointer */ + KeyInfo *pKeyInfo /* Description of the record */ ){ UnpackedRecord *p; /* Unpacked record to return */ - int nOff; /* Increment pSpace by nOff to align it */ int nByte; /* Number of bytes required for *p */ - - /* We want to shift the pointer pSpace up such that it is 8-byte aligned. - ** Thus, we need to calculate a value, nOff, between 0 and 7, to shift - ** it by. If pSpace is already 8-byte aligned, nOff should be zero. - */ - nOff = (8 - (SQLITE_PTR_TO_INT(pSpace) & 7)) & 7; - nByte = ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nField+1); - if( nByte>szSpace+nOff ){ - p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); - *ppFree = (char *)p; - if( !p ) return 0; - }else{ - p = (UnpackedRecord*)&pSpace[nOff]; - *ppFree = 0; - } - - p->aMem = (Mem*)&((char*)p)[ROUND8(sizeof(UnpackedRecord))]; - assert( pKeyInfo->aSortOrder!=0 ); + nByte = ROUND8P(sizeof(UnpackedRecord)) + sizeof(Mem)*(pKeyInfo->nKeyField+1); + p = (UnpackedRecord *)sqlite3DbMallocRaw(pKeyInfo->db, nByte); + if( !p ) return 0; + p->aMem = (Mem*)&((char*)p)[ROUND8P(sizeof(UnpackedRecord))]; + assert( pKeyInfo->aSortFlags!=0 ); p->pKeyInfo = pKeyInfo; - p->nField = pKeyInfo->nField + 1; + p->nField = pKeyInfo->nKeyField + 1; return p; } /* -** Given the nKey-byte encoding of a record in pKey[], populate the +** Given the nKey-byte encoding of a record in pKey[], populate the ** UnpackedRecord structure indicated by the fourth argument with the ** contents of the decoded record. -*/ +*/ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( KeyInfo *pKeyInfo, /* Information about the record format */ int nKey, /* Size of the binary record */ @@ -73449,7 +89210,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( UnpackedRecord *p /* Populate this structure before returning. */ ){ const unsigned char *aKey = (const unsigned char *)pKey; - int d; + u32 d; u32 idx; /* Offset in aKey[] to read from */ u16 u; /* Unsigned loop counter */ u32 szHdr; @@ -73460,7 +89221,7 @@ SQLITE_PRIVATE void sqlite3VdbeRecordUnpack( idx = getVarint32(aKey, szHdr); d = szHdr; u = 0; - while( idxflags = 0; // sqlite3VdbeSerialGet() will set this for us */ pMem->szMalloc = 0; pMem->z = 0; - d += sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + sqlite3VdbeSerialGet(&aKey[d], serial_type, pMem); + d += sqlite3VdbeSerialTypeLen(serial_type); pMem++; if( (++u)>=p->nField ) break; } - assert( u<=pKeyInfo->nField + 1 ); + if( d>(u32)nKey && u ){ + assert( CORRUPT_DB ); + /* In a corrupt record entry, the last pMem might have been set up using + ** uninitialized memory. Overwrite its value with NULL, to prevent + ** warnings from MSAN. */ + sqlite3VdbeMemSetNull(pMem-1); + } + assert( u<=pKeyInfo->nKeyField + 1 ); p->nField = u; } -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** This function compares two index or table record keys in the same way ** as the sqlite3VdbeRecordCompare() routine. Unlike VdbeRecordCompare(), @@ -73512,19 +89281,19 @@ static int vdbeRecordCompareDebug( /* Compilers may complain that mem1.u.i is potentially uninitialized. ** We could initialize it, as shown here, to silence those complaints. - ** But in fact, mem1.u.i will never actually be used uninitialized, and doing + ** But in fact, mem1.u.i will never actually be used uninitialized, and doing ** the unnecessary initialization has a measurable negative performance ** impact, since this routine is a very high runner. And so, we choose ** to ignore the compiler warnings and leave this variable uninitialized. */ /* mem1.u.i = 0; // not needed, here to silence compiler warning */ - + idx1 = getVarint32(aKey1, szHdr1); if( szHdr1>98307 ) return SQLITE_CORRUPT; d1 = szHdr1; - assert( pKeyInfo->nField+pKeyInfo->nXField>=pPKey2->nField || CORRUPT_DB ); - assert( pKeyInfo->aSortOrder!=0 ); - assert( pKeyInfo->nField>0 ); + assert( pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); + assert( pKeyInfo->aSortFlags!=0 ); + assert( pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); do{ u32 serial_type1; @@ -73538,22 +89307,38 @@ static int vdbeRecordCompareDebug( ** Use that approximation to avoid the more expensive call to ** sqlite3VdbeSerialTypeLen() in the common case. */ - if( d1+serial_type1+2>(u32)nKey1 - && d1+sqlite3VdbeSerialTypeLen(serial_type1)>(u32)nKey1 + if( d1+(u64)serial_type1+2>(u64)nKey1 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)>(u64)nKey1 ){ + if( serial_type1>=1 + && serial_type1<=7 + && d1+(u64)sqlite3VdbeSerialTypeLen(serial_type1)<=(u64)nKey1+8 + && CORRUPT_DB + ){ + return 1; /* corrupt record not detected by + ** sqlite3VdbeRecordCompareWithSkip(). Return true + ** to avoid firing the assert() */ + } break; } /* Extract the values to be compared. */ - d1 += sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + sqlite3VdbeSerialGet(&aKey1[d1], serial_type1, &mem1); + d1 += sqlite3VdbeSerialTypeLen(serial_type1); /* Do the comparison */ - rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], pKeyInfo->aColl[i]); + rc = sqlite3MemCompare(&mem1, &pPKey2->aMem[i], + pKeyInfo->nAllField>i ? pKeyInfo->aColl[i] : 0); if( rc!=0 ){ assert( mem1.szMalloc==0 ); /* See comment below */ - if( pKeyInfo->aSortOrder[i] ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((mem1.flags & MEM_Null) || (pPKey2->aMem[i].flags & MEM_Null)) + ){ + rc = -rc; + } + if( pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC ){ rc = -rc; /* Invert the result for DESC sort order. */ } goto debugCompareEnd; @@ -73582,20 +89367,20 @@ debugCompareEnd: } #endif -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG /* ** Count the number of fields (a.k.a. columns) in the record given by ** pKey,nKey. The verify that this count is less than or equal to the -** limit given by pKeyInfo->nField + pKeyInfo->nXField. +** limit given by pKeyInfo->nAllField. ** ** If this constraint is not satisfied, it means that the high-speed ** vdbeRecordCompareInt() and vdbeRecordCompareString() routines will ** not work correctly. If this assert() ever fires, it probably means -** that the KeyInfo.nField or KeyInfo.nXField values were computed +** that the KeyInfo.nKeyField or KeyInfo.nAllField values were computed ** incorrectly. */ static void vdbeAssertFieldCountWithinLimits( - int nKey, const void *pKey, /* The record to verify */ + int nKey, const void *pKey, /* The record to verify */ const KeyInfo *pKeyInfo /* Compare size with this KeyInfo */ ){ int nField = 0; @@ -73612,7 +89397,7 @@ static void vdbeAssertFieldCountWithinLimits( idx += getVarint32(aKey+idx, notUsed); nField++; } - assert( nField <= pKeyInfo->nField+pKeyInfo->nXField ); + assert( nField <= pKeyInfo->nAllField ); } #else # define vdbeAssertFieldCountWithinLimits(A,B,C) @@ -73621,7 +89406,7 @@ static void vdbeAssertFieldCountWithinLimits( /* ** Both *pMem1 and *pMem2 contain string values. Compare the two values ** using the collation sequence pColl. As usual, return a negative , zero -** or positive value if *pMem1 is less than, equal to or greater than +** or positive value if *pMem1 is less than, equal to or greater than ** *pMem2, respectively. Similar in spirit to "rc = (*pMem1) - (*pMem2);". */ static int vdbeCompareMemString( @@ -73637,7 +89422,6 @@ static int vdbeCompareMemString( }else{ int rc; const void *v1, *v2; - int n1, n2; Mem c1; Mem c2; sqlite3VdbeMemInit(&c1, pMem1->db, MEM_Null); @@ -73645,54 +89429,95 @@ static int vdbeCompareMemString( sqlite3VdbeMemShallowCopy(&c1, pMem1, MEM_Ephem); sqlite3VdbeMemShallowCopy(&c2, pMem2, MEM_Ephem); v1 = sqlite3ValueText((sqlite3_value*)&c1, pColl->enc); - n1 = v1==0 ? 0 : c1.n; v2 = sqlite3ValueText((sqlite3_value*)&c2, pColl->enc); - n2 = v2==0 ? 0 : c2.n; - rc = pColl->xCmp(pColl->pUser, n1, v1, n2, v2); - if( (v1==0 || v2==0) && prcErr ) *prcErr = SQLITE_NOMEM_BKPT; - sqlite3VdbeMemRelease(&c1); - sqlite3VdbeMemRelease(&c2); + if( (v1==0 || v2==0) ){ + if( prcErr ) *prcErr = SQLITE_NOMEM_BKPT; + rc = 0; + }else{ + rc = pColl->xCmp(pColl->pUser, c1.n, v1, c2.n, v2); + } + sqlite3VdbeMemReleaseMalloc(&c1); + sqlite3VdbeMemReleaseMalloc(&c2); return rc; } } +/* +** The input pBlob is guaranteed to be a Blob that is not marked +** with MEM_Zero. Return true if it could be a zero-blob. +*/ +static int isAllZero(const char *z, int n){ + int i; + for(i=0; iz, pB2->z, pB1->n>pB2->n ? pB2->n : pB1->n); +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3BlobCompare(const Mem *pB1, const Mem *pB2){ + int c; + int n1 = pB1->n; + int n2 = pB2->n; + + /* It is possible to have a Blob value that has some non-zero content + ** followed by zero content. But that only comes up for Blobs formed + ** by the OP_MakeRecord opcode, and such Blobs never get passed into + ** sqlite3MemCompare(). */ + assert( (pB1->flags & MEM_Zero)==0 || n1==0 ); + assert( (pB2->flags & MEM_Zero)==0 || n2==0 ); + + if( (pB1->flags|pB2->flags) & MEM_Zero ){ + if( pB1->flags & pB2->flags & MEM_Zero ){ + return pB1->u.nZero - pB2->u.nZero; + }else if( pB1->flags & MEM_Zero ){ + if( !isAllZero(pB2->z, pB2->n) ) return -1; + return pB1->u.nZero - n2; + }else{ + if( !isAllZero(pB1->z, pB1->n) ) return +1; + return n1 - pB2->u.nZero; + } + } + c = memcmp(pB1->z, pB2->z, n1>n2 ? n2 : n1); if( c ) return c; - return pB1->n - pB2->n; + return n1 - n2; } +/* The following two functions are used only within testcase() to prove +** test coverage. These functions do no exist for production builds. +** We must use separate SQLITE_NOINLINE functions here, since otherwise +** optimizer code movement causes gcov to become very confused. +*/ +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) +static int SQLITE_NOINLINE doubleLt(double a, double b){ return a8 ){ - LONGDOUBLE_TYPE x = (LONGDOUBLE_TYPE)i; - if( xr ) return +1; - return 0; +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64 i, double r){ + if( sqlite3IsNaN(r) ){ + /* SQLite considers NaN to be a NULL. And all integer values are greater + ** than NULL */ + return 1; }else{ i64 y; - double s; if( r<-9223372036854775808.0 ) return +1; - if( r>9223372036854775807.0 ) return -1; + if( r>=9223372036854775808.0 ) return -1; y = (i64)r; if( iy ){ - if( y==SMALLEST_INT64 && r>0.0 ) return -1; - return +1; - } - s = (double)i; - if( sr ) return +1; - return 0; + if( i>y ) return +1; + testcase( doubleLt(((double)i),r) ); + testcase( doubleLt(r,((double)i)) ); + testcase( doubleEq(r,((double)i)) ); + return (((double)i)r); } } @@ -73712,8 +89537,8 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C f1 = pMem1->flags; f2 = pMem2->flags; combined_flags = f1|f2; - assert( (combined_flags & MEM_RowSet)==0 ); - + assert( !sqlite3VdbeMemIsRowSet(pMem1) && !sqlite3VdbeMemIsRowSet(pMem2) ); + /* If one value is NULL, it is less than the other. If both values ** are NULL, return 0. */ @@ -73723,8 +89548,13 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C /* At least one of the two values is a number */ - if( combined_flags&(MEM_Int|MEM_Real) ){ - if( (f1 & f2 & MEM_Int)!=0 ){ + if( combined_flags&(MEM_Int|MEM_Real|MEM_IntReal) ){ + testcase( combined_flags & MEM_Int ); + testcase( combined_flags & MEM_Real ); + testcase( combined_flags & MEM_IntReal ); + if( (f1 & f2 & (MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & f2 & MEM_Int ); + testcase( f1 & f2 & MEM_IntReal ); if( pMem1->u.i < pMem2->u.i ) return -1; if( pMem1->u.i > pMem2->u.i ) return +1; return 0; @@ -73734,15 +89564,23 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C if( pMem1->u.r > pMem2->u.r ) return +1; return 0; } - if( (f1&MEM_Int)!=0 ){ + if( (f1&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f1 & MEM_Int ); + testcase( f1 & MEM_IntReal ); if( (f2&MEM_Real)!=0 ){ return sqlite3IntFloatCompare(pMem1->u.i, pMem2->u.r); + }else if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + if( pMem1->u.i < pMem2->u.i ) return -1; + if( pMem1->u.i > pMem2->u.i ) return +1; + return 0; }else{ return -1; } } if( (f1&MEM_Real)!=0 ){ - if( (f2&MEM_Int)!=0 ){ + if( (f2&(MEM_Int|MEM_IntReal))!=0 ){ + testcase( f2 & MEM_Int ); + testcase( f2 & MEM_IntReal ); return -sqlite3IntFloatCompare(pMem2->u.i, pMem1->u.r); }else{ return -1; @@ -73763,7 +89601,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C } assert( pMem1->enc==pMem2->enc || pMem1->db->mallocFailed ); - assert( pMem1->enc==SQLITE_UTF8 || + assert( pMem1->enc==SQLITE_UTF8 || pMem1->enc==SQLITE_UTF16LE || pMem1->enc==SQLITE_UTF16BE ); /* The collation sequence must be defined at this point, even if @@ -73778,7 +89616,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C /* If a NULL pointer was passed as the collate function, fall through ** to the blob case and use memcmp(). */ } - + /* Both values must be blobs. Compare using memcmp(). */ return sqlite3BlobCompare(pMem1, pMem2); } @@ -73786,7 +89624,7 @@ SQLITE_PRIVATE int sqlite3MemCompare(const Mem *pMem1, const Mem *pMem2, const C /* ** The first argument passed to this function is a serial-type that -** corresponds to an integer - all values between 1 and 9 inclusive +** corresponds to an integer - all values between 1 and 9 inclusive ** except 7. The second points to a buffer containing an integer value ** serialized according to serial_type. This function deserializes ** and returns the value. @@ -73828,7 +89666,7 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ /* ** This function compares the two table rows or index records ** specified by {nKey1, pKey1} and pPKey2. It returns a negative, zero -** or positive integer if key1 is less than, equal to or +** or positive integer if key1 is less than, equal to or ** greater than key2. The {nKey1, pKey1} key must be a blob ** created by the OP_MakeRecord opcode of the VDBE. The pPKey2 ** key must be a parsed key such as obtained from @@ -73837,12 +89675,12 @@ static i64 vdbeRecordDecodeInt(u32 serial_type, const u8 *aKey){ ** If argument bSkip is non-zero, it is assumed that the caller has already ** determined that the first fields of the keys are equal. ** -** Key1 and Key2 do not have to contain the same number of fields. If all -** fields that appear in both keys are equal, then pPKey2->default_rc is +** Key1 and Key2 do not have to contain the same number of fields. If all +** fields that appear in both keys are equal, then pPKey2->default_rc is ** returned. ** -** If database corruption is discovered, set pPKey2->errCode to -** SQLITE_CORRUPT and return 0. If an OOM error is encountered, +** If database corruption is discovered, set pPKey2->errCode to +** SQLITE_CORRUPT and return 0. If an OOM error is encountered, ** pPKey2->errCode is set to SQLITE_NOMEM and, if it is not NULL, the ** malloc-failed flag set on database handle (pPKey2->pKeyInfo->db). */ @@ -73857,7 +89695,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( u32 idx1; /* Offset of first type in header */ int rc = 0; /* Return value */ Mem *pRhs = pPKey2->aMem; /* Next field of pPKey2 to compare */ - KeyInfo *pKeyInfo = pPKey2->pKeyInfo; + KeyInfo *pKeyInfo; const unsigned char *aKey1 = (const unsigned char *)pKey1; Mem mem1; @@ -73865,41 +89703,51 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( ** two elements in the keys are equal. Fix the various stack variables so ** that this routine begins comparing at the second field. */ if( bSkip ){ - u32 s1; - idx1 = 1 + getVarint32(&aKey1[1], s1); + u32 s1 = aKey1[1]; + if( s1<0x80 ){ + idx1 = 2; + }else{ + idx1 = 1 + sqlite3GetVarint32(&aKey1[1], &s1); + } szHdr1 = aKey1[0]; d1 = szHdr1 + sqlite3VdbeSerialTypeLen(s1); i = 1; pRhs++; }else{ - idx1 = getVarint32(aKey1, szHdr1); - d1 = szHdr1; - if( d1>(unsigned)nKey1 ){ - pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; - return 0; /* Corruption */ + if( (szHdr1 = aKey1[0])<0x80 ){ + idx1 = 1; + }else{ + idx1 = sqlite3GetVarint32(aKey1, &szHdr1); } + d1 = szHdr1; i = 0; } + if( d1>(unsigned)nKey1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corruption */ + } VVA_ONLY( mem1.szMalloc = 0; ) /* Only needed by assert() statements */ - assert( pPKey2->pKeyInfo->nField+pPKey2->pKeyInfo->nXField>=pPKey2->nField + assert( pPKey2->pKeyInfo->nAllField>=pPKey2->nField || CORRUPT_DB ); - assert( pPKey2->pKeyInfo->aSortOrder!=0 ); - assert( pPKey2->pKeyInfo->nField>0 ); + assert( pPKey2->pKeyInfo->aSortFlags!=0 ); + assert( pPKey2->pKeyInfo->nKeyField>0 ); assert( idx1<=szHdr1 || CORRUPT_DB ); - do{ + while( 1 /*exit-by-break*/ ){ u32 serial_type; /* RHS is an integer */ - if( pRhs->flags & MEM_Int ){ + if( pRhs->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pRhs->flags & MEM_Int ); + testcase( pRhs->flags & MEM_IntReal ); serial_type = aKey1[idx1]; testcase( serial_type==12 ); if( serial_type>=10 ){ - rc = +1; + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else if( serial_type==7 ){ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); + serialGet7(&aKey1[d1], &mem1); rc = -sqlite3IntFloatCompare(pRhs->u.i, mem1.u.r); }else{ i64 lhs = vdbeRecordDecodeInt(serial_type, &aKey1[d1]); @@ -73917,21 +89765,25 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( serial_type = aKey1[idx1]; if( serial_type>=10 ){ /* Serial types 12 or greater are strings and blobs (greater than - ** numbers). Types 10 and 11 are currently "reserved for future + ** numbers). Types 10 and 11 are currently "reserved for future ** use", so it doesn't really matter what the results of comparing - ** them to numberic values are. */ - rc = +1; + ** them to numeric values are. */ + rc = serial_type==10 ? -1 : +1; }else if( serial_type==0 ){ rc = -1; }else{ - sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); if( serial_type==7 ){ - if( mem1.u.ru.r ){ + if( serialGet7(&aKey1[d1], &mem1) ){ + rc = -1; /* mem1 is a NaN */ + }else if( mem1.u.ru.r ){ rc = -1; }else if( mem1.u.r>pRhs->u.r ){ rc = +1; + }else{ + assert( rc==0 ); } }else{ + sqlite3VdbeSerialGet(&aKey1[d1], serial_type, &mem1); rc = sqlite3IntFloatCompare(mem1.u.i, pRhs->u.r); } } @@ -73939,7 +89791,7 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is a string */ else if( pRhs->flags & MEM_Str ){ - getVarint32(&aKey1[idx1], serial_type); + getVarint32NR(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 ){ rc = -1; @@ -73949,7 +89801,9 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( mem1.n = (serial_type - 12) / 2; testcase( (d1+mem1.n)==(unsigned)nKey1 ); testcase( (d1+mem1.n+1)==(unsigned)nKey1 ); - if( (d1+mem1.n) > (unsigned)nKey1 ){ + if( (d1+mem1.n) > (unsigned)nKey1 + || (pKeyInfo = pPKey2->pKeyInfo)->nAllField<=i + ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ }else if( pKeyInfo->aColl[i] ){ @@ -73963,14 +89817,15 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( }else{ int nCmp = MIN(mem1.n, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); - if( rc==0 ) rc = mem1.n - pRhs->n; + if( rc==0 ) rc = mem1.n - pRhs->n; } } } /* RHS is a blob */ else if( pRhs->flags & MEM_Blob ){ - getVarint32(&aKey1[idx1], serial_type); + assert( (pRhs->flags & MEM_Zero)==0 || pRhs->n==0 ); + getVarint32NR(&aKey1[idx1], serial_type); testcase( serial_type==12 ); if( serial_type<12 || (serial_type & 0x01) ){ rc = -1; @@ -73981,6 +89836,12 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( if( (d1+nStr) > (unsigned)nKey1 ){ pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ + }else if( pRhs->flags & MEM_Zero ){ + if( !isAllZero((const char*)&aKey1[d1],nStr) ){ + rc = 1; + }else{ + rc = nStr - pRhs->u.nZero; + } }else{ int nCmp = MIN(nStr, pRhs->n); rc = memcmp(&aKey1[d1], pRhs->z, nCmp); @@ -73992,12 +89853,25 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* RHS is null */ else{ serial_type = aKey1[idx1]; - rc = (serial_type!=0); + if( serial_type==0 + || serial_type==10 + || (serial_type==7 && serialGet7(&aKey1[d1], &mem1)!=0) + ){ + assert( rc==0 ); + }else{ + rc = 1; + } } if( rc!=0 ){ - if( pKeyInfo->aSortOrder[i] ){ - rc = -rc; + int sortFlags = pPKey2->pKeyInfo->aSortFlags[i]; + if( sortFlags ){ + if( (sortFlags & KEYINFO_ORDER_BIGNULL)==0 + || ((sortFlags & KEYINFO_ORDER_DESC) + !=(serial_type==0 || (pRhs->flags&MEM_Null))) + ){ + rc = -rc; + } } assert( vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, rc) ); assert( mem1.szMalloc==0 ); /* See comment below */ @@ -74005,10 +89879,16 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( } i++; + if( i==pPKey2->nField ) break; pRhs++; d1 += sqlite3VdbeSerialTypeLen(serial_type); + if( d1>(unsigned)nKey1 ) break; idx1 += sqlite3VarintLen(serial_type); - }while( idx1<(unsigned)szHdr1 && inField && d1<=(unsigned)nKey1 ); + if( idx1>=(unsigned)szHdr1 ){ + pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; + return 0; /* Corrupt index */ + } + } /* No memory allocation is ever used on mem1. Prove this using ** the following assert(). If the assert() fails, it indicates a @@ -74018,9 +89898,9 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompareWithSkip( /* rc==0 here means that one or both of the keys ran out of fields and ** all the fields up to that point were equal. Return the default_rc ** value. */ - assert( CORRUPT_DB - || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) - || pKeyInfo->db->mallocFailed + assert( CORRUPT_DB + || vdbeRecordCompareDebug(nKey1, pKey1, pPKey2, pPKey2->default_rc) + || pPKey2->pKeyInfo->db->mallocFailed ); pPKey2->eqSeen = 1; return pPKey2->default_rc; @@ -74034,8 +89914,8 @@ SQLITE_PRIVATE int sqlite3VdbeRecordCompare( /* -** This function is an optimized version of sqlite3VdbeRecordCompare() -** that (a) the first field of pPKey2 is an integer, and (b) the +** This function is an optimized version of sqlite3VdbeRecordCompare() +** that (a) the first field of pPKey2 is an integer, and (b) the ** size-of-header varint at the start of (pKey1/nKey1) fits in a single ** byte (i.e. is less than 128). ** @@ -74051,7 +89931,7 @@ static int vdbeRecordCompareInt( int res; u32 y; u64 x; - i64 v = pPKey2->aMem[0].u.i; + i64 v; i64 lhs; vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); @@ -74090,7 +89970,7 @@ static int vdbeRecordCompareInt( testcase( lhs<0 ); break; } - case 8: + case 8: lhs = 0; break; case 9: @@ -74098,11 +89978,11 @@ static int vdbeRecordCompareInt( break; /* This case could be removed without changing the results of running - ** this code. Including it causes gcc to generate a faster switch + ** this code. Including it causes gcc to generate a faster switch ** statement (since the range of switch targets now starts at zero and ** is contiguous) but does not cause any duplicate code to be generated - ** (as gcc is clever enough to combine the two like cases). Other - ** compilers might be similar. */ + ** (as gcc is clever enough to combine the two like cases). Other + ** compilers might be similar. */ case 0: case 7: return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); @@ -74110,12 +89990,14 @@ static int vdbeRecordCompareInt( return sqlite3VdbeRecordCompare(nKey1, pKey1, pPKey2); } + assert( pPKey2->u.i == pPKey2->aMem[0].u.i ); + v = pPKey2->u.i; if( v>lhs ){ res = pPKey2->r1; }else if( vr2; }else if( pPKey2->nField>1 ){ - /* The first fields of the two keys are equal. Compare the trailing + /* The first fields of the two keys are equal. Compare the trailing ** fields. */ res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); }else{ @@ -74130,9 +90012,9 @@ static int vdbeRecordCompareInt( } /* -** This function is an optimized version of sqlite3VdbeRecordCompare() +** This function is an optimized version of sqlite3VdbeRecordCompare() ** that (a) the first field of pPKey2 is a string, that (b) the first field -** uses the collation sequence BINARY and (c) that the size-of-header varint +** uses the collation sequence BINARY and (c) that the size-of-header varint ** at the start of (pKey1/nKey1) fits in a single byte. */ static int vdbeRecordCompareString( @@ -74144,11 +90026,20 @@ static int vdbeRecordCompareString( int res; assert( pPKey2->aMem[0].flags & MEM_Str ); + assert( pPKey2->aMem[0].n == pPKey2->n ); + assert( pPKey2->aMem[0].z == pPKey2->u.z ); vdbeAssertFieldCountWithinLimits(nKey1, pKey1, pPKey2->pKeyInfo); - getVarint32(&aKey1[1], serial_type); + serial_type = (signed char)(aKey1[1]); + +vrcs_restart: if( serial_type<12 ){ + if( serial_type<0 ){ + sqlite3GetVarint32(&aKey1[1], (u32*)&serial_type); + if( serial_type>=12 ) goto vrcs_restart; + assert( CORRUPT_DB ); + } res = pPKey2->r1; /* (pKey1/nKey1) is a number or a null */ - }else if( !(serial_type & 0x01) ){ + }else if( !(serial_type & 0x01) ){ res = pPKey2->r2; /* (pKey1/nKey1) is a blob */ }else{ int nCmp; @@ -74160,11 +90051,15 @@ static int vdbeRecordCompareString( pPKey2->errCode = (u8)SQLITE_CORRUPT_BKPT; return 0; /* Corruption */ } - nCmp = MIN( pPKey2->aMem[0].n, nStr ); - res = memcmp(&aKey1[szHdr], pPKey2->aMem[0].z, nCmp); + nCmp = MIN( pPKey2->n, nStr ); + res = memcmp(&aKey1[szHdr], pPKey2->u.z, nCmp); - if( res==0 ){ - res = nStr - pPKey2->aMem[0].n; + if( res>0 ){ + res = pPKey2->r2; + }else if( res<0 ){ + res = pPKey2->r1; + }else{ + res = nStr - pPKey2->n; if( res==0 ){ if( pPKey2->nField>1 ){ res = sqlite3VdbeRecordCompareWithSkip(nKey1, pKey1, pPKey2, 1); @@ -74177,10 +90072,6 @@ static int vdbeRecordCompareString( }else{ res = pPKey2->r1; } - }else if( res>0 ){ - res = pPKey2->r2; - }else{ - res = pPKey2->r1; } } @@ -74200,7 +90091,7 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ /* varintRecordCompareInt() and varintRecordCompareString() both assume ** that the size-of-header varint that occurs at the start of each record ** fits in a single byte (i.e. is 127 or less). varintRecordCompareInt() - ** also assumes that it is safe to overread a buffer by at least the + ** also assumes that it is safe to overread a buffer by at least the ** maximum possible legal header size plus 8 bytes. Because there is ** guaranteed to be at least 74 (but not 136) bytes of padding following each ** buffer passed to varintRecordCompareInt() this makes it convenient to @@ -74210,9 +90101,12 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ ** The easiest way to enforce this limit is to consider only records with ** 13 fields or less. If the first field is an integer, the maximum legal ** header size is (12*5 + 1 + 1) bytes. */ - if( (p->pKeyInfo->nField + p->pKeyInfo->nXField)<=13 ){ + if( p->pKeyInfo->nAllField<=13 ){ int flags = p->aMem[0].flags; - if( p->pKeyInfo->aSortOrder[0] ){ + if( p->pKeyInfo->aSortFlags[0] ){ + if( p->pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL ){ + return sqlite3VdbeRecordCompare; + } p->r1 = 1; p->r2 = -1; }else{ @@ -74220,13 +90114,18 @@ SQLITE_PRIVATE RecordCompare sqlite3VdbeFindCompare(UnpackedRecord *p){ p->r2 = 1; } if( (flags & MEM_Int) ){ + p->u.i = p->aMem[0].u.i; return vdbeRecordCompareInt; } testcase( flags & MEM_Real ); testcase( flags & MEM_Null ); testcase( flags & MEM_Blob ); - if( (flags & (MEM_Real|MEM_Null|MEM_Blob))==0 && p->pKeyInfo->aColl[0]==0 ){ + if( (flags & (MEM_Real|MEM_IntReal|MEM_Null|MEM_Blob))==0 + && p->pKeyInfo->aColl[0]==0 + ){ assert( flags & MEM_Str ); + p->u.z = p->aMem[0].z; + p->n = p->aMem[0].n; return vdbeRecordCompareString; } } @@ -74253,31 +90152,32 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ /* Get the size of the index entry. Only indices entries of less ** than 2GiB are support - anything large must be database corruption. ** Any corruption is detected in sqlite3BtreeParseCellPtr(), though, so - ** this code can safely assume that nCellKey is 32-bits + ** this code can safely assume that nCellKey is 32-bits */ assert( sqlite3BtreeCursorIsValid(pCur) ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); - assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + nCellKey = sqlite3BtreePayloadSize(pCur); assert( (nCellKey & SQLITE_MAX_U32)==(u64)nCellKey ); /* Read in the complete content of the index entry */ sqlite3VdbeMemInit(&m, db, 0); - rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); if( rc ){ return rc; } /* The index entry must begin with a header size */ - (void)getVarint32((u8*)m.z, szHdr); + getVarint32NR((u8*)m.z, szHdr); testcase( szHdr==3 ); - testcase( szHdr==m.n ); - if( unlikely(szHdr<3 || (int)szHdr>m.n) ){ + testcase( szHdr==(u32)m.n ); + testcase( szHdr>0x7fffffff ); + assert( m.n>=0 ); + if( unlikely(szHdr<3 || szHdr>(unsigned)m.n) ){ goto idx_rowid_corruption; } /* The last field of the index should be an integer - the ROWID. ** Verify that the last entry really is an integer. */ - (void)getVarint32((u8*)&m.z[szHdr-1], typeRowid); + getVarint32NR((u8*)&m.z[szHdr-1], typeRowid); testcase( typeRowid==1 ); testcase( typeRowid==2 ); testcase( typeRowid==3 ); @@ -74298,14 +90198,14 @@ SQLITE_PRIVATE int sqlite3VdbeIdxRowid(sqlite3 *db, BtCursor *pCur, i64 *rowid){ /* Fetch the integer off the end of the index record */ sqlite3VdbeSerialGet((u8*)&m.z[m.n-lenRowid], typeRowid, &v); *rowid = v.u.i; - sqlite3VdbeMemRelease(&m); + sqlite3VdbeMemReleaseMalloc(&m); return SQLITE_OK; /* Jump here if database corruption is detected after m has been ** allocated. Free the m object and return SQLITE_CORRUPT. */ idx_rowid_corruption: testcase( m.szMalloc!=0 ); - sqlite3VdbeMemRelease(&m); + sqlite3VdbeMemReleaseMalloc(&m); return SQLITE_CORRUPT_BKPT; } @@ -74317,7 +90217,7 @@ idx_rowid_corruption: ** ** pUnpacked is either created without a rowid or is truncated so that it ** omits the rowid at the end. The rowid at the end of the index entry -** is ignored as well. Hence, this routine only compares the prefixes +** is ignored as well. Hence, this routine only compares the prefixes ** of the keys prior to the final rowid, not the entire key. */ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( @@ -74334,8 +90234,7 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( assert( pC->eCurType==CURTYPE_BTREE ); pCur = pC->uc.pCursor; assert( sqlite3BtreeCursorIsValid(pCur) ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCur, &nCellKey); - assert( rc==SQLITE_OK ); /* pCur is always valid so KeySize cannot fail */ + nCellKey = sqlite3BtreePayloadSize(pCur); /* nCellKey will always be between 0 and 0xffffffff because of the way ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ if( nCellKey<=0 || nCellKey>0x7fffffff ){ @@ -74343,20 +90242,20 @@ SQLITE_PRIVATE int sqlite3VdbeIdxKeyCompare( return SQLITE_CORRUPT_BKPT; } sqlite3VdbeMemInit(&m, db, 0); - rc = sqlite3VdbeMemFromBtree(pCur, 0, (u32)nCellKey, 1, &m); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); if( rc ){ return rc; } - *res = sqlite3VdbeRecordCompare(m.n, m.z, pUnpacked); - sqlite3VdbeMemRelease(&m); + *res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, pUnpacked, 0); + sqlite3VdbeMemReleaseMalloc(&m); return SQLITE_OK; } /* ** This routine sets the value to be returned by subsequent calls to -** sqlite3_changes() on the database handle 'db'. +** sqlite3_changes() on the database handle 'db'. */ -SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *db, int nChange){ +SQLITE_PRIVATE void sqlite3VdbeSetChanges(sqlite3 *db, i64 nChange){ assert( sqlite3_mutex_held(db->mutex) ); db->nChange = nChange; db->nTotalChange += nChange; @@ -74379,11 +90278,19 @@ SQLITE_PRIVATE void sqlite3VdbeCountChanges(Vdbe *v){ ** programs obsolete. Removing user-defined functions or collating ** sequences, or changing an authorization function are the types of ** things that make prepared statements obsolete. +** +** If iCode is 1, then expiration is advisory. The statement should +** be reprepared before being restarted, but if it is already running +** it is allowed to run to completion. +** +** Internally, this function just sets the Vdbe.expired flag on all +** prepared statements. The flag is set to 1 for an immediate expiration +** and set to 2 for an advisory expiration. */ -SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db){ +SQLITE_PRIVATE void sqlite3ExpirePreparedStatements(sqlite3 *db, int iCode){ Vdbe *p; - for(p = db->pVdbe; p; p=p->pNext){ - p->expired = 1; + for(p = db->pVdbe; p; p=p->pVNext){ + p->expired = iCode+1; } } @@ -74394,9 +90301,16 @@ SQLITE_PRIVATE sqlite3 *sqlite3VdbeDb(Vdbe *v){ return v->db; } +/* +** Return the SQLITE_PREPARE flags for a Vdbe. +*/ +SQLITE_PRIVATE u8 sqlite3VdbePrepareFlags(Vdbe *v){ + return v->prepFlags; +} + /* ** Return a pointer to an sqlite3_value structure containing the value bound -** parameter iVar of VM v. Except, if the value is an SQL NULL, return +** parameter iVar of VM v. Except, if the value is an SQL NULL, return ** 0 instead. Unless it is NULL, apply affinity aff (one of the SQLITE_AFF_* ** constants) to the value before returning it. ** @@ -74406,6 +90320,8 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff assert( iVar>0 ); if( v ){ Mem *pMem = &v->aVar[iVar-1]; + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); if( 0==(pMem->flags & MEM_Null) ){ sqlite3_value *pRet = sqlite3ValueNew(v->db); if( pRet ){ @@ -74425,13 +90341,63 @@ SQLITE_PRIVATE sqlite3_value *sqlite3VdbeGetBoundValue(Vdbe *v, int iVar, u8 aff */ SQLITE_PRIVATE void sqlite3VdbeSetVarmask(Vdbe *v, int iVar){ assert( iVar>0 ); - if( iVar>32 ){ - v->expmask = 0xffffffff; + assert( (v->db->flags & SQLITE_EnableQPSG)==0 + || (v->db->mDbFlags & DBFLAG_InternalFunc)!=0 ); + if( iVar>=32 ){ + v->expmask |= 0x80000000; }else{ v->expmask |= ((u32)1 << (iVar-1)); } } +/* +** Cause a function to throw an error if it was call from OP_PureFunc +** rather than OP_Function. +** +** OP_PureFunc means that the function must be deterministic, and should +** throw an error if it is given inputs that would make it non-deterministic. +** This routine is invoked by date/time functions that use non-deterministic +** features such as 'now'. +*/ +SQLITE_PRIVATE int sqlite3NotPureFunc(sqlite3_context *pCtx){ + const VdbeOp *pOp; +#ifdef SQLITE_ENABLE_STAT4 + if( pCtx->pVdbe==0 ) return 1; +#endif + pOp = pCtx->pVdbe->aOp + pCtx->iOp; + if( pOp->opcode==OP_PureFunc ){ + const char *zContext; + char *zMsg; + if( pOp->p5 & NC_IsCheck ){ + zContext = "a CHECK constraint"; + }else if( pOp->p5 & NC_GenCol ){ + zContext = "a generated column"; + }else{ + zContext = "an index"; + } + zMsg = sqlite3_mprintf("non-deterministic use of %s() in %s", + pCtx->pFunc->zName, zContext); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); + return 0; + } + return 1; +} + +#if defined(SQLITE_ENABLE_CURSOR_HINTS) && defined(SQLITE_DEBUG) +/* +** This Walker callback is used to help verify that calls to +** sqlite3BtreeCursorHint() with opcode BTREE_HINT_RANGE have +** byte-code register values correctly initialized. +*/ +SQLITE_PRIVATE int sqlite3CursorRangeHintExprCheck(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_REGISTER ){ + assert( (pWalker->u.aMem[pExpr->iTable].flags & MEM_Undefined)==0 ); + } + return WRC_Continue; +} +#endif /* SQLITE_ENABLE_CURSOR_HINTS && SQLITE_DEBUG */ + #ifndef SQLITE_OMIT_VIRTUALTABLE /* ** Transfer error message text from an sqlite3_vtab.zErrMsg (text stored @@ -74452,21 +90418,22 @@ SQLITE_PRIVATE void sqlite3VtabImportErrmsg(Vdbe *p, sqlite3_vtab *pVtab){ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* -** If the second argument is not NULL, release any allocations associated +** If the second argument is not NULL, release any allocations associated ** with the memory cells in the p->aMem[] array. Also free the UnpackedRecord ** structure itself, using sqlite3DbFree(). ** ** This function is used to free UnpackedRecord structures allocated by ** the vdbeUnpackRecord() function found in vdbeapi.c. */ -static void vdbeFreeUnpacked(sqlite3 *db, UnpackedRecord *p){ +static void vdbeFreeUnpacked(sqlite3 *db, int nField, UnpackedRecord *p){ + assert( db!=0 ); if( p ){ int i; - for(i=0; inField; i++){ + for(i=0; iaMem[i]; - if( pMem->zMalloc ) sqlite3VdbeMemRelease(pMem); + if( pMem->zMalloc ) sqlite3VdbeMemReleaseMalloc(pMem); } - sqlite3DbFree(db, p); + sqlite3DbNNFreeNN(db, p); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -74485,24 +90452,42 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( const char *zDb, /* Database name */ Table *pTab, /* Modified table */ i64 iKey1, /* Initial key value */ - int iReg /* Register for new.* record */ + int iReg, /* Register for new.* record */ + int iBlobWrite ){ sqlite3 *db = v->db; i64 iKey2; PreUpdate preupdate; const char *zTbl = pTab->zName; static const u8 fakeSortOrder = 0; +#ifdef SQLITE_DEBUG + int nRealCol; + if( pTab->tabFlags & TF_WithoutRowid ){ + nRealCol = sqlite3PrimaryKeyIndex(pTab)->nColumn; + }else if( pTab->tabFlags & TF_HasVirtual ){ + nRealCol = pTab->nNVCol; + }else{ + nRealCol = pTab->nCol; + } +#endif assert( db->pPreUpdate==0 ); memset(&preupdate, 0, sizeof(PreUpdate)); - if( op==SQLITE_UPDATE ){ - iKey2 = v->aMem[iReg].u.i; + if( HasRowid(pTab)==0 ){ + iKey1 = iKey2 = 0; + preupdate.pPk = sqlite3PrimaryKeyIndex(pTab); }else{ - iKey2 = iKey1; + if( op==SQLITE_UPDATE ){ + iKey2 = v->aMem[iReg].u.i; + }else{ + iKey2 = iKey1; + } } - assert( pCsr->nField==pTab->nCol - || (pCsr->nField==pTab->nCol+1 && op==SQLITE_DELETE && iReg==-1) + assert( pCsr!=0 ); + assert( pCsr->eCurType==CURTYPE_BTREE ); + assert( pCsr->nField==nRealCol + || (pCsr->nField==nRealCol+1 && op==SQLITE_DELETE && iReg==-1) ); preupdate.v = v; @@ -74511,24 +90496,32 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( preupdate.iNewReg = iReg; preupdate.keyinfo.db = db; preupdate.keyinfo.enc = ENC(db); - preupdate.keyinfo.nField = pTab->nCol; - preupdate.keyinfo.aSortOrder = (u8*)&fakeSortOrder; + preupdate.keyinfo.nKeyField = pTab->nCol; + preupdate.keyinfo.aSortFlags = (u8*)&fakeSortOrder; preupdate.iKey1 = iKey1; preupdate.iKey2 = iKey2; - preupdate.iPKey = pTab->iPKey; + preupdate.pTab = pTab; + preupdate.iBlobWrite = iBlobWrite; db->pPreUpdate = &preupdate; db->xPreUpdateCallback(db->pPreUpdateArg, db, op, zDb, zTbl, iKey1, iKey2); db->pPreUpdate = 0; sqlite3DbFree(db, preupdate.aRecord); - vdbeFreeUnpacked(db, preupdate.pUnpacked); - vdbeFreeUnpacked(db, preupdate.pNewUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pUnpacked); + vdbeFreeUnpacked(db, preupdate.keyinfo.nKeyField+1, preupdate.pNewUnpacked); if( preupdate.aNew ){ int i; for(i=0; inField; i++){ sqlite3VdbeMemRelease(&preupdate.aNew[i]); } - sqlite3DbFree(db, preupdate.aNew); + sqlite3DbNNFreeNN(db, preupdate.aNew); + } + if( preupdate.apDflt ){ + int i; + for(i=0; inCol; i++){ + sqlite3ValueFree(preupdate.apDflt[i]); + } + sqlite3DbFree(db, preupdate.apDflt); } } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -74552,6 +90545,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( */ /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* #include "opcodes.h" */ #ifndef SQLITE_OMIT_DEPRECATED /* @@ -74562,7 +90556,7 @@ SQLITE_PRIVATE void sqlite3VdbePreUpdateHook( ** collating sequences are registered or if an authorizer function is ** added or changed. */ -SQLITE_API int SQLITE_STDCALL sqlite3_expired(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_expired(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; return p==0 || p->expired; } @@ -74597,12 +90591,21 @@ static int vdbeSafetyNotNull(Vdbe *p){ */ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ sqlite3_int64 iNow; + sqlite3_int64 iElapse; assert( p->startTime>0 ); - assert( db->xProfile!=0 ); + assert( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 ); assert( db->init.busy==0 ); assert( p->zSql!=0 ); sqlite3OsCurrentTimeInt64(db->pVfs, &iNow); - db->xProfile(db->pProfileArg, p->zSql, (iNow - p->startTime)*1000000); + iElapse = (iNow - p->startTime)*1000000; +#ifndef SQLITE_OMIT_DEPRECATED + if( db->xProfile ){ + db->xProfile(db->pProfileArg, p->zSql, iElapse); + } +#endif + if( db->mTrace & SQLITE_TRACE_PROFILE ){ + db->trace.xV2(SQLITE_TRACE_PROFILE, db->pTraceArg, p, (void*)&iElapse); + } p->startTime = 0; } /* @@ -74624,7 +90627,7 @@ static SQLITE_NOINLINE void invokeProfileCallback(sqlite3 *db, Vdbe *p){ ** This routine sets the error code and string returned by ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ /* IMPLEMENTATION-OF: R-57228-12904 Invoking sqlite3_finalize() on a NULL @@ -74636,7 +90639,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt){ if( vdbeSafety(v) ) return SQLITE_MISUSE_BKPT; sqlite3_mutex_enter(db->mutex); checkProfileCallback(db, v); - rc = sqlite3VdbeFinalize(v); + assert( v->eVdbeState>=VDBE_READY_STATE ); + rc = sqlite3VdbeReset(v); + sqlite3VdbeDelete(v); rc = sqlite3ApiExit(db, rc); sqlite3LeaveMutexAndCloseZombie(db); } @@ -74651,7 +90656,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_finalize(sqlite3_stmt *pStmt){ ** This routine sets the error code and string returned by ** sqlite3_errcode(), sqlite3_errmsg() and sqlite3_errmsg16(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt){ int rc; if( pStmt==0 ){ rc = SQLITE_OK; @@ -74672,19 +90677,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_reset(sqlite3_stmt *pStmt){ /* ** Set all the parameters in the compiled SQL statement to NULL. */ -SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt *pStmt){ int i; int rc = SQLITE_OK; Vdbe *p = (Vdbe*)pStmt; #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = ((Vdbe*)pStmt)->db->mutex; + sqlite3_mutex *mutex; +#endif +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif +#if SQLITE_THREADSAFE + mutex = p->db->mutex; #endif sqlite3_mutex_enter(mutex); for(i=0; inVar; i++){ sqlite3VdbeMemRelease(&p->aVar[i]); p->aVar[i].flags = MEM_Null; } - if( p->isPrepareV2 && p->expmask ){ + assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || p->expmask==0 ); + if( p->expmask ){ p->expired = 1; } sqlite3_mutex_leave(mutex); @@ -74696,10 +90710,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_clear_bindings(sqlite3_stmt *pStmt){ ** The following routines extract information from a Mem or sqlite3_value ** structure. */ -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value *pVal){ +SQLITE_API const void *sqlite3_value_blob(sqlite3_value *pVal){ Mem *p = (Mem*)pVal; if( p->flags & (MEM_Blob|MEM_Str) ){ - if( sqlite3VdbeMemExpandBlob(p)!=SQLITE_OK ){ + if( ExpandBlob(p)!=SQLITE_OK ){ assert( p->flags==MEM_Null && p->z==0 ); return 0; } @@ -74709,36 +90723,49 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_value_blob(sqlite3_value *pVal){ return sqlite3_value_text(pVal); } } -SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes(sqlite3_value *pVal){ +SQLITE_API int sqlite3_value_bytes(sqlite3_value *pVal){ return sqlite3ValueBytes(pVal, SQLITE_UTF8); } -SQLITE_API int SQLITE_STDCALL sqlite3_value_bytes16(sqlite3_value *pVal){ +SQLITE_API int sqlite3_value_bytes16(sqlite3_value *pVal){ return sqlite3ValueBytes(pVal, SQLITE_UTF16NATIVE); } -SQLITE_API double SQLITE_STDCALL sqlite3_value_double(sqlite3_value *pVal){ +SQLITE_API double sqlite3_value_double(sqlite3_value *pVal){ return sqlite3VdbeRealValue((Mem*)pVal); } -SQLITE_API int SQLITE_STDCALL sqlite3_value_int(sqlite3_value *pVal){ +SQLITE_API int sqlite3_value_int(sqlite3_value *pVal){ return (int)sqlite3VdbeIntValue((Mem*)pVal); } -SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_value_int64(sqlite3_value *pVal){ +SQLITE_API sqlite_int64 sqlite3_value_int64(sqlite3_value *pVal){ return sqlite3VdbeIntValue((Mem*)pVal); } -SQLITE_API unsigned int SQLITE_STDCALL sqlite3_value_subtype(sqlite3_value *pVal){ +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value *pVal){ Mem *pMem = (Mem*)pVal; return ((pMem->flags & MEM_Subtype) ? pMem->eSubtype : 0); } -SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_value_text(sqlite3_value *pVal){ +SQLITE_API void *sqlite3_value_pointer(sqlite3_value *pVal, const char *zPType){ + Mem *p = (Mem*)pVal; + if( (p->flags&(MEM_TypeMask|MEM_Term|MEM_Subtype)) == + (MEM_Null|MEM_Term|MEM_Subtype) + && zPType!=0 + && p->eSubtype=='p' + && strcmp(p->u.zPType, zPType)==0 + ){ + return (void*)p->z; + }else{ + return 0; + } +} +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value *pVal){ return (const unsigned char *)sqlite3ValueText(pVal, SQLITE_UTF8); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16(sqlite3_value* pVal){ +SQLITE_API const void *sqlite3_value_text16(sqlite3_value* pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16NATIVE); } -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16be(sqlite3_value *pVal){ +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16BE); } -SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value *pVal){ +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value *pVal){ return sqlite3ValueText(pVal, SQLITE_UTF16LE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -74746,47 +90773,107 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_value_text16le(sqlite3_value *pVal ** fundamental datatypes: 64-bit signed integer 64-bit IEEE floating ** point number string BLOB NULL */ -SQLITE_API int SQLITE_STDCALL sqlite3_value_type(sqlite3_value* pVal){ +SQLITE_API int sqlite3_value_type(sqlite3_value* pVal){ static const u8 aType[] = { - SQLITE_BLOB, /* 0x00 */ - SQLITE_NULL, /* 0x01 */ - SQLITE_TEXT, /* 0x02 */ - SQLITE_NULL, /* 0x03 */ - SQLITE_INTEGER, /* 0x04 */ - SQLITE_NULL, /* 0x05 */ - SQLITE_INTEGER, /* 0x06 */ - SQLITE_NULL, /* 0x07 */ - SQLITE_FLOAT, /* 0x08 */ - SQLITE_NULL, /* 0x09 */ - SQLITE_FLOAT, /* 0x0a */ - SQLITE_NULL, /* 0x0b */ - SQLITE_INTEGER, /* 0x0c */ - SQLITE_NULL, /* 0x0d */ - SQLITE_INTEGER, /* 0x0e */ - SQLITE_NULL, /* 0x0f */ - SQLITE_BLOB, /* 0x10 */ - SQLITE_NULL, /* 0x11 */ - SQLITE_TEXT, /* 0x12 */ - SQLITE_NULL, /* 0x13 */ - SQLITE_INTEGER, /* 0x14 */ - SQLITE_NULL, /* 0x15 */ - SQLITE_INTEGER, /* 0x16 */ - SQLITE_NULL, /* 0x17 */ - SQLITE_FLOAT, /* 0x18 */ - SQLITE_NULL, /* 0x19 */ - SQLITE_FLOAT, /* 0x1a */ - SQLITE_NULL, /* 0x1b */ - SQLITE_INTEGER, /* 0x1c */ - SQLITE_NULL, /* 0x1d */ - SQLITE_INTEGER, /* 0x1e */ - SQLITE_NULL, /* 0x1f */ + SQLITE_BLOB, /* 0x00 (not possible) */ + SQLITE_NULL, /* 0x01 NULL */ + SQLITE_TEXT, /* 0x02 TEXT */ + SQLITE_NULL, /* 0x03 (not possible) */ + SQLITE_INTEGER, /* 0x04 INTEGER */ + SQLITE_NULL, /* 0x05 (not possible) */ + SQLITE_INTEGER, /* 0x06 INTEGER + TEXT */ + SQLITE_NULL, /* 0x07 (not possible) */ + SQLITE_FLOAT, /* 0x08 FLOAT */ + SQLITE_NULL, /* 0x09 (not possible) */ + SQLITE_FLOAT, /* 0x0a FLOAT + TEXT */ + SQLITE_NULL, /* 0x0b (not possible) */ + SQLITE_INTEGER, /* 0x0c (not possible) */ + SQLITE_NULL, /* 0x0d (not possible) */ + SQLITE_INTEGER, /* 0x0e (not possible) */ + SQLITE_NULL, /* 0x0f (not possible) */ + SQLITE_BLOB, /* 0x10 BLOB */ + SQLITE_NULL, /* 0x11 (not possible) */ + SQLITE_TEXT, /* 0x12 (not possible) */ + SQLITE_NULL, /* 0x13 (not possible) */ + SQLITE_INTEGER, /* 0x14 INTEGER + BLOB */ + SQLITE_NULL, /* 0x15 (not possible) */ + SQLITE_INTEGER, /* 0x16 (not possible) */ + SQLITE_NULL, /* 0x17 (not possible) */ + SQLITE_FLOAT, /* 0x18 FLOAT + BLOB */ + SQLITE_NULL, /* 0x19 (not possible) */ + SQLITE_FLOAT, /* 0x1a (not possible) */ + SQLITE_NULL, /* 0x1b (not possible) */ + SQLITE_INTEGER, /* 0x1c (not possible) */ + SQLITE_NULL, /* 0x1d (not possible) */ + SQLITE_INTEGER, /* 0x1e (not possible) */ + SQLITE_NULL, /* 0x1f (not possible) */ + SQLITE_FLOAT, /* 0x20 INTREAL */ + SQLITE_NULL, /* 0x21 (not possible) */ + SQLITE_FLOAT, /* 0x22 INTREAL + TEXT */ + SQLITE_NULL, /* 0x23 (not possible) */ + SQLITE_FLOAT, /* 0x24 (not possible) */ + SQLITE_NULL, /* 0x25 (not possible) */ + SQLITE_FLOAT, /* 0x26 (not possible) */ + SQLITE_NULL, /* 0x27 (not possible) */ + SQLITE_FLOAT, /* 0x28 (not possible) */ + SQLITE_NULL, /* 0x29 (not possible) */ + SQLITE_FLOAT, /* 0x2a (not possible) */ + SQLITE_NULL, /* 0x2b (not possible) */ + SQLITE_FLOAT, /* 0x2c (not possible) */ + SQLITE_NULL, /* 0x2d (not possible) */ + SQLITE_FLOAT, /* 0x2e (not possible) */ + SQLITE_NULL, /* 0x2f (not possible) */ + SQLITE_BLOB, /* 0x30 (not possible) */ + SQLITE_NULL, /* 0x31 (not possible) */ + SQLITE_TEXT, /* 0x32 (not possible) */ + SQLITE_NULL, /* 0x33 (not possible) */ + SQLITE_FLOAT, /* 0x34 (not possible) */ + SQLITE_NULL, /* 0x35 (not possible) */ + SQLITE_FLOAT, /* 0x36 (not possible) */ + SQLITE_NULL, /* 0x37 (not possible) */ + SQLITE_FLOAT, /* 0x38 (not possible) */ + SQLITE_NULL, /* 0x39 (not possible) */ + SQLITE_FLOAT, /* 0x3a (not possible) */ + SQLITE_NULL, /* 0x3b (not possible) */ + SQLITE_FLOAT, /* 0x3c (not possible) */ + SQLITE_NULL, /* 0x3d (not possible) */ + SQLITE_FLOAT, /* 0x3e (not possible) */ + SQLITE_NULL, /* 0x3f (not possible) */ }; +#ifdef SQLITE_DEBUG + { + int eType = SQLITE_BLOB; + if( pVal->flags & MEM_Null ){ + eType = SQLITE_NULL; + }else if( pVal->flags & (MEM_Real|MEM_IntReal) ){ + eType = SQLITE_FLOAT; + }else if( pVal->flags & MEM_Int ){ + eType = SQLITE_INTEGER; + }else if( pVal->flags & MEM_Str ){ + eType = SQLITE_TEXT; + } + assert( eType == aType[pVal->flags&MEM_AffMask] ); + } +#endif return aType[pVal->flags&MEM_AffMask]; } +SQLITE_API int sqlite3_value_encoding(sqlite3_value *pVal){ + return pVal->enc; +} + +/* Return true if a parameter to xUpdate represents an unchanged column */ +SQLITE_API int sqlite3_value_nochange(sqlite3_value *pVal){ + return (pVal->flags&(MEM_Null|MEM_Zero))==(MEM_Null|MEM_Zero); +} + +/* Return true if a parameter value originated from an sqlite3_bind() */ +SQLITE_API int sqlite3_value_frombind(sqlite3_value *pVal){ + return (pVal->flags&MEM_FromBind)!=0; +} /* Make a copy of an sqlite3_value object */ -SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value *pOrig){ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value *pOrig){ sqlite3_value *pNew; if( pOrig==0 ) return 0; pNew = sqlite3_malloc( sizeof(*pNew) ); @@ -74802,6 +90889,9 @@ SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value * sqlite3ValueFree(pNew); pNew = 0; } + }else if( pNew->flags & MEM_Null ){ + /* Do not duplicate pointer values */ + pNew->flags &= ~(MEM_Term|MEM_Subtype); } return pNew; } @@ -74809,21 +90899,21 @@ SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_value_dup(const sqlite3_value * /* Destroy an sqlite3_value object previously obtained from ** sqlite3_value_dup(). */ -SQLITE_API void SQLITE_STDCALL sqlite3_value_free(sqlite3_value *pOld){ +SQLITE_API void sqlite3_value_free(sqlite3_value *pOld){ sqlite3ValueFree(pOld); } - + /**************************** sqlite3_result_ ******************************* ** The following routines are used by user-defined functions to specify ** the function result. ** ** The setStrOrError() function calls sqlite3VdbeMemSetStr() to store the -** result as a string or blob but if the string or blob is too large, it -** then sets the error code to SQLITE_TOOBIG +** result as a string or blob. Appropriate errors are set if the string/blob +** is too big or if an OOM occurs. ** ** The invokeValueDestructor(P,X) routine invokes destructor function X() -** on value P is not going to be used and need to be destroyed. +** on value P if P is not going to be used and need to be destroyed. */ static void setResultStrOrError( sqlite3_context *pCtx, /* Function context */ @@ -74832,14 +90922,28 @@ static void setResultStrOrError( u8 enc, /* Encoding of z. 0 for BLOBs */ void (*xDel)(void*) /* Destructor function */ ){ - if( sqlite3VdbeMemSetStr(pCtx->pOut, z, n, enc, xDel)==SQLITE_TOOBIG ){ + Mem *pOut = pCtx->pOut; + int rc = sqlite3VdbeMemSetStr(pOut, z, n, enc, xDel); + if( rc ){ + if( rc==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(pCtx); + }else{ + /* The only errors possible from sqlite3VdbeMemSetStr are + ** SQLITE_TOOBIG and SQLITE_NOMEM */ + assert( rc==SQLITE_NOMEM ); + sqlite3_result_error_nomem(pCtx); + } + return; + } + sqlite3VdbeChangeEncoding(pOut, pCtx->enc); + if( sqlite3VdbeMemTooBig(pOut) ){ sqlite3_result_error_toobig(pCtx); } } static int invokeValueDestructor( const void *p, /* Value to destroy */ void (*xDel)(void*), /* The destructor */ - sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if no NULL */ + sqlite3_context *pCtx /* Set a SQLITE_TOOBIG error if not NULL */ ){ assert( xDel!=SQLITE_DYNAMIC ); if( xDel==0 ){ @@ -74849,172 +90953,302 @@ static int invokeValueDestructor( }else{ xDel((void*)p); } - if( pCtx ) sqlite3_result_error_toobig(pCtx); +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx!=0 ){ + sqlite3_result_error_toobig(pCtx); + } +#else + assert( pCtx!=0 ); + sqlite3_result_error_toobig(pCtx); +#endif return SQLITE_TOOBIG; } -SQLITE_API void SQLITE_STDCALL sqlite3_result_blob( - sqlite3_context *pCtx, - const void *z, - int n, +SQLITE_API void sqlite3_result_blob( + sqlite3_context *pCtx, + const void *z, + int n, void (*xDel)(void *) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 || n<0 ){ + invokeValueDestructor(z, xDel, pCtx); + return; + } +#endif assert( n>=0 ); assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, 0, xDel); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_blob64( - sqlite3_context *pCtx, - const void *z, +SQLITE_API void sqlite3_result_blob64( + sqlite3_context *pCtx, + const void *z, sqlite3_uint64 n, void (*xDel)(void *) ){ - assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, 0, xDel); } } -SQLITE_API void SQLITE_STDCALL sqlite3_result_double(sqlite3_context *pCtx, double rVal){ +SQLITE_API void sqlite3_result_double(sqlite3_context *pCtx, double rVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetDouble(pCtx->pOut, rVal); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ +SQLITE_API void sqlite3_result_error(sqlite3_context *pCtx, const char *z, int n){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; - pCtx->fErrorOrAux = 1; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF8, SQLITE_TRANSIENT); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API void SQLITE_STDCALL sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ +SQLITE_API void sqlite3_result_error16(sqlite3_context *pCtx, const void *z, int n){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_ERROR; - pCtx->fErrorOrAux = 1; sqlite3VdbeMemSetStr(pCtx->pOut, z, n, SQLITE_UTF16NATIVE, SQLITE_TRANSIENT); } #endif -SQLITE_API void SQLITE_STDCALL sqlite3_result_int(sqlite3_context *pCtx, int iVal){ +SQLITE_API void sqlite3_result_int(sqlite3_context *pCtx, int iVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, (i64)iVal); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ +SQLITE_API void sqlite3_result_int64(sqlite3_context *pCtx, i64 iVal){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetInt64(pCtx->pOut, iVal); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_null(sqlite3_context *pCtx){ +SQLITE_API void sqlite3_result_null(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ - Mem *pOut = pCtx->pOut; +SQLITE_API void sqlite3_result_pointer( + sqlite3_context *pCtx, + void *pPtr, + const char *zPType, + void (*xDestructor)(void*) +){ + Mem *pOut; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(pPtr, xDestructor, 0); + return; + } +#endif + pOut = pCtx->pOut; + assert( sqlite3_mutex_held(pOut->db->mutex) ); + sqlite3VdbeMemRelease(pOut); + pOut->flags = MEM_Null; + sqlite3VdbeMemSetPointer(pOut, pPtr, zPType, xDestructor); +} +SQLITE_API void sqlite3_result_subtype(sqlite3_context *pCtx, unsigned int eSubtype){ + Mem *pOut; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif +#if defined(SQLITE_STRICT_SUBTYPE) && SQLITE_STRICT_SUBTYPE+0!=0 + if( pCtx->pFunc!=0 + && (pCtx->pFunc->funcFlags & SQLITE_RESULT_SUBTYPE)==0 + ){ + char zErr[200]; + sqlite3_snprintf(sizeof(zErr), zErr, + "misuse of sqlite3_result_subtype() by %s()", + pCtx->pFunc->zName); + sqlite3_result_error(pCtx, zErr, -1); + return; + } +#endif /* SQLITE_STRICT_SUBTYPE */ + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); pOut->eSubtype = eSubtype & 0xff; pOut->flags |= MEM_Subtype; } -SQLITE_API void SQLITE_STDCALL sqlite3_result_text( - sqlite3_context *pCtx, - const char *z, +SQLITE_API void sqlite3_result_text( + sqlite3_context *pCtx, + const char *z, int n, void (*xDel)(void *) ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); setResultStrOrError(pCtx, z, n, SQLITE_UTF8, xDel); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_text64( - sqlite3_context *pCtx, - const char *z, +SQLITE_API void sqlite3_result_text64( + sqlite3_context *pCtx, + const char *z, sqlite3_uint64 n, void (*xDel)(void *), unsigned char enc ){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ){ + invokeValueDestructor(z, xDel, 0); + return; + } +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); assert( xDel!=SQLITE_DYNAMIC ); - if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + if( enc!=SQLITE_UTF8 ){ + if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; + n &= ~(u64)1; + } if( n>0x7fffffff ){ (void)invokeValueDestructor(z, xDel, pCtx); }else{ setResultStrOrError(pCtx, z, (int)n, enc, xDel); + sqlite3VdbeMemZeroTerminateIfAble(pCtx->pOut); } } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API void SQLITE_STDCALL sqlite3_result_text16( - sqlite3_context *pCtx, - const void *z, - int n, +SQLITE_API void sqlite3_result_text16( + sqlite3_context *pCtx, + const void *z, + int n, void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16NATIVE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16NATIVE, xDel); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_text16be( - sqlite3_context *pCtx, - const void *z, - int n, +SQLITE_API void sqlite3_result_text16be( + sqlite3_context *pCtx, + const void *z, + int n, void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16BE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16BE, xDel); } -SQLITE_API void SQLITE_STDCALL sqlite3_result_text16le( - sqlite3_context *pCtx, - const void *z, - int n, +SQLITE_API void sqlite3_result_text16le( + sqlite3_context *pCtx, + const void *z, + int n, void (*xDel)(void *) ){ assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - setResultStrOrError(pCtx, z, n, SQLITE_UTF16LE, xDel); + setResultStrOrError(pCtx, z, n & ~(u64)1, SQLITE_UTF16LE, xDel); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API void SQLITE_STDCALL sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ +SQLITE_API void sqlite3_result_value(sqlite3_context *pCtx, sqlite3_value *pValue){ + Mem *pOut; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; + if( pValue==0 ){ + sqlite3_result_null(pCtx); + return; + } +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - sqlite3VdbeMemCopy(pCtx->pOut, pValue); + sqlite3VdbeMemCopy(pOut, pValue); + sqlite3VdbeChangeEncoding(pOut, pCtx->enc); + if( sqlite3VdbeMemTooBig(pOut) ){ + sqlite3_result_error_toobig(pCtx); + } } -SQLITE_API void SQLITE_STDCALL sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ - assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - sqlite3VdbeMemSetZeroBlob(pCtx->pOut, n); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context *pCtx, int n){ + sqlite3_result_zeroblob64(pCtx, n>0 ? n : 0); } -SQLITE_API int SQLITE_STDCALL sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ - Mem *pOut = pCtx->pOut; +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context *pCtx, u64 n){ + Mem *pOut; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return SQLITE_MISUSE_BKPT; +#endif + pOut = pCtx->pOut; assert( sqlite3_mutex_held(pOut->db->mutex) ); if( n>(u64)pOut->db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(pCtx); return SQLITE_TOOBIG; } +#ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); return SQLITE_OK; +#else + return sqlite3VdbeMemSetZeroBlob(pCtx->pOut, (int)n); +#endif } -SQLITE_API void SQLITE_STDCALL sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ - pCtx->isError = errCode; - pCtx->fErrorOrAux = 1; +SQLITE_API void sqlite3_result_error_code(sqlite3_context *pCtx, int errCode){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif + pCtx->isError = errCode ? errCode : -1; #ifdef SQLITE_DEBUG if( pCtx->pVdbe ) pCtx->pVdbe->rcApp = errCode; #endif if( pCtx->pOut->flags & MEM_Null ){ - sqlite3VdbeMemSetStr(pCtx->pOut, sqlite3ErrStr(errCode), -1, - SQLITE_UTF8, SQLITE_STATIC); + setResultStrOrError(pCtx, sqlite3ErrStr(errCode), -1, SQLITE_UTF8, + SQLITE_STATIC); } } /* Force an SQLITE_TOOBIG error. */ -SQLITE_API void SQLITE_STDCALL sqlite3_result_error_toobig(sqlite3_context *pCtx){ +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); pCtx->isError = SQLITE_TOOBIG; - pCtx->fErrorOrAux = 1; - sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, + sqlite3VdbeMemSetStr(pCtx->pOut, "string or blob too big", -1, SQLITE_UTF8, SQLITE_STATIC); } /* An SQLITE_NOMEM error. */ -SQLITE_API void SQLITE_STDCALL sqlite3_result_error_nomem(sqlite3_context *pCtx){ +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context *pCtx){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); sqlite3VdbeMemSetNull(pCtx->pOut); pCtx->isError = SQLITE_NOMEM_BKPT; - pCtx->fErrorOrAux = 1; sqlite3OomFault(pCtx->pOut->db); } +#ifndef SQLITE_UNTESTABLE +/* Force the INT64 value currently stored as the result to be +** a MEM_IntReal value. See the SQLITE_TESTCTRL_RESULT_INTREAL +** test-control. +*/ +SQLITE_PRIVATE void sqlite3ResultIntReal(sqlite3_context *pCtx){ + assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); + if( pCtx->pOut->flags & MEM_Int ){ + pCtx->pOut->flags &= ~MEM_Int; + pCtx->pOut->flags |= MEM_IntReal; + } +} +#endif + + /* -** This function is called after a transaction has been committed. It +** This function is called after a transaction has been committed. It ** invokes callbacks registered with sqlite3_wal_hook() as required. */ static int doWalCallbacks(sqlite3 *db){ @@ -75028,8 +91262,8 @@ static int doWalCallbacks(sqlite3 *db){ sqlite3BtreeEnter(pBt); nEntry = sqlite3PagerWalCallback(sqlite3BtreePager(pBt)); sqlite3BtreeLeave(pBt); - if( db->xWalCallback && nEntry>0 && rc==SQLITE_OK ){ - rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zName, nEntry); + if( nEntry>0 && db->xWalCallback && rc==SQLITE_OK ){ + rc = db->xWalCallback(db->pWalArg, db, db->aDb[i].zDbSName, nEntry); } } } @@ -75043,7 +91277,7 @@ static int doWalCallbacks(sqlite3 *db){ ** statement is completely executed or an error occurs. ** ** This routine implements the bulk of the logic behind the sqlite_step() -** API. The only thing omitted is the automatic recompile if a +** API. The only thing omitted is the automatic recompile if a ** schema change has occurred. That detail is handled by the ** outer sqlite3_step() wrapper procedure. */ @@ -75052,72 +91286,83 @@ static int sqlite3Step(Vdbe *p){ int rc; assert(p); - if( p->magic!=VDBE_MAGIC_RUN ){ - /* We used to require that sqlite3_reset() be called before retrying - ** sqlite3_step() after any error or after SQLITE_DONE. But beginning - ** with version 3.7.0, we changed this so that sqlite3_reset() would - ** be called automatically instead of throwing the SQLITE_MISUSE error. - ** This "automatic-reset" change is not technically an incompatibility, - ** since any application that receives an SQLITE_MISUSE is broken by - ** definition. - ** - ** Nevertheless, some published applications that were originally written - ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE - ** returns, and those were broken by the automatic-reset change. As a - ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the - ** legacy behavior of returning SQLITE_MISUSE for cases where the - ** previous sqlite3_step() returned something other than a SQLITE_LOCKED - ** or SQLITE_BUSY error. - */ -#ifdef SQLITE_OMIT_AUTORESET - if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ - sqlite3_reset((sqlite3_stmt*)p); - }else{ - return SQLITE_MISUSE_BKPT; - } -#else - sqlite3_reset((sqlite3_stmt*)p); -#endif - } - - /* Check that malloc() has not failed. If it has, return early. */ db = p->db; - if( db->mallocFailed ){ - p->rc = SQLITE_NOMEM; - return SQLITE_NOMEM_BKPT; - } + if( p->eVdbeState!=VDBE_RUN_STATE ){ + restart_step: + if( p->eVdbeState==VDBE_READY_STATE ){ + if( p->expired ){ + p->rc = SQLITE_SCHEMA; + rc = SQLITE_ERROR; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ + /* If this statement was prepared using saved SQL and an + ** error has occurred, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same + ** value. + */ + rc = sqlite3VdbeTransferError(p); + } + goto end_of_step; + } - if( p->pc<=0 && p->expired ){ - p->rc = SQLITE_SCHEMA; - rc = SQLITE_ERROR; - goto end_of_step; - } - if( p->pc<0 ){ - /* If there are no other statements currently running, then - ** reset the interrupt flag. This prevents a call to sqlite3_interrupt - ** from interrupting a statement that has not yet started. - */ - if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; - } + /* If there are no other statements currently running, then + ** reset the interrupt flag. This prevents a call to sqlite3_interrupt + ** from interrupting a statement that has not yet started. + */ + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } - assert( db->nVdbeWrite>0 || db->autoCommit==0 - || (db->nDeferredCons==0 && db->nDeferredImmCons==0) - ); + assert( db->nVdbeWrite>0 || db->autoCommit==0 + || (db->nDeferredCons==0 && db->nDeferredImmCons==0) + ); #ifndef SQLITE_OMIT_TRACE - if( db->xProfile && !db->init.busy && p->zSql ){ - sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); - }else{ - assert( p->startTime==0 ); - } + if( (db->mTrace & (SQLITE_TRACE_PROFILE|SQLITE_TRACE_XPROFILE))!=0 + && !db->init.busy && p->zSql ){ + sqlite3OsCurrentTimeInt64(db->pVfs, &p->startTime); + }else{ + assert( p->startTime==0 ); + } #endif - db->nVdbeActive++; - if( p->readOnly==0 ) db->nVdbeWrite++; - if( p->bIsReader ) db->nVdbeRead++; - p->pc = 0; + db->nVdbeActive++; + if( p->readOnly==0 ) db->nVdbeWrite++; + if( p->bIsReader ) db->nVdbeRead++; + p->pc = 0; + p->eVdbeState = VDBE_RUN_STATE; + }else + + if( ALWAYS(p->eVdbeState==VDBE_HALT_STATE) ){ + /* We used to require that sqlite3_reset() be called before retrying + ** sqlite3_step() after any error or after SQLITE_DONE. But beginning + ** with version 3.7.0, we changed this so that sqlite3_reset() would + ** be called automatically instead of throwing the SQLITE_MISUSE error. + ** This "automatic-reset" change is not technically an incompatibility, + ** since any application that receives an SQLITE_MISUSE is broken by + ** definition. + ** + ** Nevertheless, some published applications that were originally written + ** for version 3.6.23 or earlier do in fact depend on SQLITE_MISUSE + ** returns, and those were broken by the automatic-reset change. As a + ** a work-around, the SQLITE_OMIT_AUTORESET compile-time restores the + ** legacy behavior of returning SQLITE_MISUSE for cases where the + ** previous sqlite3_step() returned something other than a SQLITE_LOCKED + ** or SQLITE_BUSY error. + */ +#ifdef SQLITE_OMIT_AUTORESET + if( (rc = p->rc&0xff)==SQLITE_BUSY || rc==SQLITE_LOCKED ){ + sqlite3_reset((sqlite3_stmt*)p); + }else{ + return SQLITE_MISUSE_BKPT; + } +#else + sqlite3_reset((sqlite3_stmt*)p); +#endif + assert( p->eVdbeState==VDBE_READY_STATE ); + goto restart_step; + } } + #ifdef SQLITE_DEBUG p->rcApp = SQLITE_OK; #endif @@ -75132,42 +91377,44 @@ static int sqlite3Step(Vdbe *p){ db->nVdbeExec--; } -#ifndef SQLITE_OMIT_TRACE - /* If the statement completed successfully, invoke the profile callback */ - if( rc!=SQLITE_ROW ) checkProfileCallback(db, p); -#endif - - if( rc==SQLITE_DONE ){ + if( rc==SQLITE_ROW ){ assert( p->rc==SQLITE_OK ); - p->rc = doWalCallbacks(db); - if( p->rc!=SQLITE_OK ){ - rc = SQLITE_ERROR; + assert( db->mallocFailed==0 ); + db->errCode = SQLITE_ROW; + return SQLITE_ROW; + }else{ +#ifndef SQLITE_OMIT_TRACE + /* If the statement completed successfully, invoke the profile callback */ + checkProfileCallback(db, p); +#endif + p->pResultRow = 0; + if( rc==SQLITE_DONE && db->autoCommit ){ + assert( p->rc==SQLITE_OK ); + p->rc = doWalCallbacks(db); + if( p->rc!=SQLITE_OK ){ + rc = SQLITE_ERROR; + } + }else if( rc!=SQLITE_DONE && (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ){ + /* If this statement was prepared using saved SQL and an + ** error has occurred, then return the error code in p->rc to the + ** caller. Set the error code in the database handle to the same value. + */ + rc = sqlite3VdbeTransferError(p); } } db->errCode = rc; if( SQLITE_NOMEM==sqlite3ApiExit(p->db, p->rc) ){ p->rc = SQLITE_NOMEM_BKPT; + if( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 ) rc = p->rc; } end_of_step: - /* At this point local variable rc holds the value that should be - ** returned if this statement was compiled using the legacy - ** sqlite3_prepare() interface. According to the docs, this can only - ** be one of the values in the first assert() below. Variable p->rc - ** contains the value that would be returned if sqlite3_finalize() - ** were called on statement p. - */ - assert( rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR + /* There are only a limited number of result codes allowed from the + ** statements prepared using the legacy sqlite3_prepare() interface */ + assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 + || rc==SQLITE_ROW || rc==SQLITE_DONE || rc==SQLITE_ERROR || (rc&0xff)==SQLITE_BUSY || rc==SQLITE_MISUSE ); - assert( (p->rc!=SQLITE_ROW && p->rc!=SQLITE_DONE) || p->rc==p->rcApp ); - if( p->isPrepareV2 && rc!=SQLITE_ROW && rc!=SQLITE_DONE ){ - /* If this statement was prepared using sqlite3_prepare_v2(), and an - ** error has occurred, then return the error code in p->rc to the - ** caller. Set the error code in the database handle to the same value. - */ - rc = sqlite3VdbeTransferError(p); - } return (rc&db->errMask); } @@ -75176,9 +91423,8 @@ end_of_step: ** sqlite3Step() to do most of the work. If a schema error occurs, ** call sqlite3Reprepare() and try again. */ -SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_step(sqlite3_stmt *pStmt){ int rc = SQLITE_OK; /* Result from sqlite3Step() */ - int rc2 = SQLITE_OK; /* Result from sqlite3Reprepare() */ Vdbe *v = (Vdbe*)pStmt; /* the prepared statement */ int cnt = 0; /* Counter to prevent infinite loop of reprepares */ sqlite3 *db; /* The database connection */ @@ -75188,36 +91434,40 @@ SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt *pStmt){ } db = v->db; sqlite3_mutex_enter(db->mutex); - v->doingRerun = 0; while( (rc = sqlite3Step(v))==SQLITE_SCHEMA && cnt++ < SQLITE_MAX_SCHEMA_RETRY ){ int savedPc = v->pc; - rc2 = rc = sqlite3Reprepare(v); - if( rc!=SQLITE_OK) break; + rc = sqlite3Reprepare(v); + if( rc!=SQLITE_OK ){ + /* This case occurs after failing to recompile an sql statement. + ** The error message from the SQL compiler has already been loaded + ** into the database handle. This block copies the error message + ** from the database handle into the statement and sets the statement + ** program counter to 0 to ensure that when the statement is + ** finalized or reset the parser error message is available via + ** sqlite3_errmsg() and sqlite3_errcode(). + */ + const char *zErr = (const char *)sqlite3_value_text(db->pErr); + sqlite3DbFree(db, v->zErrMsg); + if( !db->mallocFailed ){ + v->zErrMsg = sqlite3DbStrDup(db, zErr); + v->rc = rc = sqlite3ApiExit(db, rc); + } else { + v->zErrMsg = 0; + v->rc = rc = SQLITE_NOMEM_BKPT; + } + break; + } sqlite3_reset(pStmt); - if( savedPc>=0 ) v->doingRerun = 1; + if( savedPc>=0 ){ + /* Setting minWriteFileFormat to 254 is a signal to the OP_Init and + ** OP_Trace opcodes to *not* perform SQLITE_TRACE_STMT because it has + ** already been done once on a prior invocation that failed due to + ** SQLITE_SCHEMA. tag-20220401a */ + v->minWriteFileFormat = 254; + } assert( v->expired==0 ); } - if( rc2!=SQLITE_OK ){ - /* This case occurs after failing to recompile an sql statement. - ** The error message from the SQL compiler has already been loaded - ** into the database handle. This block copies the error message - ** from the database handle into the statement and sets the statement - ** program counter to 0 to ensure that when the statement is - ** finalized or reset the parser error message is available via - ** sqlite3_errmsg() and sqlite3_errcode(). - */ - const char *zErr = (const char *)sqlite3_value_text(db->pErr); - sqlite3DbFree(db, v->zErrMsg); - if( !db->mallocFailed ){ - v->zErrMsg = sqlite3DbStrDup(db, zErr); - v->rc = rc2; - } else { - v->zErrMsg = 0; - v->rc = rc = SQLITE_NOMEM_BKPT; - } - } - rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; } @@ -75227,7 +91477,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_step(sqlite3_stmt *pStmt){ ** Extract the user data from a sqlite3_context structure and return a ** pointer to it. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context *p){ +SQLITE_API void *sqlite3_user_data(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#endif assert( p && p->pFunc ); return p->pFunc->pUserData; } @@ -75242,11 +91495,120 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_user_data(sqlite3_context *p){ ** sqlite3_create_function16() routines that originally registered the ** application defined function. */ -SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context *p){ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else assert( p && p->pOut ); +#endif return p->pOut->db; } +/* +** If this routine is invoked from within an xColumn method of a virtual +** table, then it returns true if and only if the the call is during an +** UPDATE operation and the value of the column will not be modified +** by the UPDATE. +** +** If this routine is called from any context other than within the +** xColumn method of a virtual table, then the return value is meaningless +** and arbitrary. +** +** Virtual table implements might use this routine to optimize their +** performance by substituting a NULL result, or some other light-weight +** value, as a signal to the xUpdate routine that the column is unchanged. +*/ +SQLITE_API int sqlite3_vtab_nochange(sqlite3_context *p){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return 0; +#else + assert( p ); +#endif + return sqlite3_value_nochange(p->pOut); +} + +/* +** The destructor function for a ValueList object. This needs to be +** a separate function, unknowable to the application, to ensure that +** calls to sqlite3_vtab_in_first()/sqlite3_vtab_in_next() that are not +** preceded by activation of IN processing via sqlite3_vtab_int() do not +** try to access a fake ValueList object inserted by a hostile extension. +*/ +SQLITE_PRIVATE void sqlite3VdbeValueListFree(void *pToDelete){ + sqlite3_free(pToDelete); +} + +/* +** Implementation of sqlite3_vtab_in_first() (if bNext==0) and +** sqlite3_vtab_in_next() (if bNext!=0). +*/ +static int valueFromValueList( + sqlite3_value *pVal, /* Pointer to the ValueList object */ + sqlite3_value **ppOut, /* Store the next value from the list here */ + int bNext /* 1 for _next(). 0 for _first() */ +){ + int rc; + ValueList *pRhs; + + *ppOut = 0; + if( pVal==0 ) return SQLITE_MISUSE_BKPT; + if( (pVal->flags & MEM_Dyn)==0 || pVal->xDel!=sqlite3VdbeValueListFree ){ + return SQLITE_ERROR; + }else{ + assert( (pVal->flags&(MEM_TypeMask|MEM_Term|MEM_Subtype)) == + (MEM_Null|MEM_Term|MEM_Subtype) ); + assert( pVal->eSubtype=='p' ); + assert( pVal->u.zPType!=0 && strcmp(pVal->u.zPType,"ValueList")==0 ); + pRhs = (ValueList*)pVal->z; + } + if( bNext ){ + rc = sqlite3BtreeNext(pRhs->pCsr, 0); + }else{ + int dummy = 0; + rc = sqlite3BtreeFirst(pRhs->pCsr, &dummy); + assert( rc==SQLITE_OK || sqlite3BtreeEof(pRhs->pCsr) ); + if( sqlite3BtreeEof(pRhs->pCsr) ) rc = SQLITE_DONE; + } + if( rc==SQLITE_OK ){ + u32 sz; /* Size of current row in bytes */ + Mem sMem; /* Raw content of current row */ + memset(&sMem, 0, sizeof(sMem)); + sz = sqlite3BtreePayloadSize(pRhs->pCsr); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pRhs->pCsr,(int)sz,&sMem); + if( rc==SQLITE_OK ){ + u8 *zBuf = (u8*)sMem.z; + u32 iSerial; + sqlite3_value *pOut = pRhs->pOut; + int iOff = 1 + getVarint32(&zBuf[1], iSerial); + sqlite3VdbeSerialGet(&zBuf[iOff], iSerial, pOut); + pOut->enc = ENC(pOut->db); + if( (pOut->flags & MEM_Ephem)!=0 && sqlite3VdbeMemMakeWriteable(pOut) ){ + rc = SQLITE_NOMEM; + }else{ + *ppOut = pOut; + } + } + sqlite3VdbeMemRelease(&sMem); + } + return rc; +} + +/* +** Set the iterator value pVal to point to the first value in the set. +** Set (*ppOut) to point to this value before returning. +*/ +SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut){ + return valueFromValueList(pVal, ppOut, 0); +} + +/* +** Set the iterator value pVal to point to the next value in the set. +** Set (*ppOut) to point to this value before returning. +*/ +SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut){ + return valueFromValueList(pVal, ppOut, 1); +} + /* ** Return the current time for a statement. If the current time ** is requested more than once within the same run of a single prepared @@ -75256,7 +91618,7 @@ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_context_db_handle(sqlite3_context *p) */ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ int rc; -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifndef SQLITE_ENABLE_STAT4 sqlite3_int64 *piTime = &p->pVdbe->iCurrentTime; assert( p->pVdbe!=0 ); #else @@ -75270,28 +91632,6 @@ SQLITE_PRIVATE sqlite3_int64 sqlite3StmtCurrentTime(sqlite3_context *p){ return *piTime; } -/* -** The following is the implementation of an SQL function that always -** fails with an error message stating that the function is used in the -** wrong context. The sqlite3_overload_function() API might construct -** SQL function that use this routine so that the functions will exist -** for name resolution but are actually overloaded by the xFindFunction -** method of virtual tables. -*/ -SQLITE_PRIVATE void sqlite3InvalidFunction( - sqlite3_context *context, /* The function calling context */ - int NotUsed, /* Number of arguments to the function */ - sqlite3_value **NotUsed2 /* Value of each argument */ -){ - const char *zName = context->pFunc->zName; - char *zErr; - UNUSED_PARAMETER2(NotUsed, NotUsed2); - zErr = sqlite3_mprintf( - "unable to use function %s in the requested context", zName); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); -} - /* ** Create a new aggregate context for p and return a pointer to ** its pMem->z element. @@ -75318,7 +91658,7 @@ static SQLITE_NOINLINE void *createAggContext(sqlite3_context *p, int nByte){ ** context is allocated on the first call. Subsequent calls return the ** same context that was returned on prior calls. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context *p, int nByte){ +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context *p, int nByte){ assert( p && p->pFunc && p->pFunc->xFinalize ); assert( sqlite3_mutex_held(p->pOut->db->mutex) ); testcase( nByte<0 ); @@ -75332,65 +91672,83 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_aggregate_context(sqlite3_context *p, in /* ** Return the auxiliary data pointer, if any, for the iArg'th argument to ** the user-function defined by pCtx. +** +** The left-most argument is 0. +** +** Undocumented behavior: If iArg is negative then access a cache of +** auxiliary data pointers that is available to all functions within a +** single prepared statement. The iArg values must match. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context *pCtx, int iArg){ AuxData *pAuxData; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return 0; +#endif assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); -#if SQLITE_ENABLE_STAT3_OR_STAT4 +#if SQLITE_ENABLE_STAT4 if( pCtx->pVdbe==0 ) return 0; #else assert( pCtx->pVdbe!=0 ); #endif - for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ - if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; + for(pAuxData=pCtx->pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){ + if( pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){ + return pAuxData->pAux; + } } - - return (pAuxData ? pAuxData->pAux : 0); + return 0; } /* ** Set the auxiliary data pointer and delete function, for the iArg'th ** argument to the user-function defined by pCtx. Any previous value is ** deleted by calling the delete function specified when it was set. +** +** The left-most argument is 0. +** +** Undocumented behavior: If iArg is negative then make the data available +** to all functions within the current prepared statement using iArg as an +** access code. */ -SQLITE_API void SQLITE_STDCALL sqlite3_set_auxdata( - sqlite3_context *pCtx, - int iArg, - void *pAux, +SQLITE_API void sqlite3_set_auxdata( + sqlite3_context *pCtx, + int iArg, + void *pAux, void (*xDelete)(void*) ){ AuxData *pAuxData; - Vdbe *pVdbe = pCtx->pVdbe; + Vdbe *pVdbe; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pCtx==0 ) return; +#endif + pVdbe= pCtx->pVdbe; assert( sqlite3_mutex_held(pCtx->pOut->db->mutex) ); - if( iArg<0 ) goto failed; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( pVdbe==0 ) goto failed; #else assert( pVdbe!=0 ); #endif - for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNext){ - if( pAuxData->iOp==pCtx->iOp && pAuxData->iArg==iArg ) break; + for(pAuxData=pVdbe->pAuxData; pAuxData; pAuxData=pAuxData->pNextAux){ + if( pAuxData->iAuxArg==iArg && (pAuxData->iAuxOp==pCtx->iOp || iArg<0) ){ + break; + } } if( pAuxData==0 ){ pAuxData = sqlite3DbMallocZero(pVdbe->db, sizeof(AuxData)); if( !pAuxData ) goto failed; - pAuxData->iOp = pCtx->iOp; - pAuxData->iArg = iArg; - pAuxData->pNext = pVdbe->pAuxData; + pAuxData->iAuxOp = pCtx->iOp; + pAuxData->iAuxArg = iArg; + pAuxData->pNextAux = pVdbe->pAuxData; pVdbe->pAuxData = pAuxData; - if( pCtx->fErrorOrAux==0 ){ - pCtx->isError = 0; - pCtx->fErrorOrAux = 1; - } - }else if( pAuxData->xDelete ){ - pAuxData->xDelete(pAuxData->pAux); + if( pCtx->isError==0 ) pCtx->isError = -1; + }else if( pAuxData->xDeleteAux ){ + pAuxData->xDeleteAux(pAuxData->pAux); } pAuxData->pAux = pAux; - pAuxData->xDelete = xDelete; + pAuxData->xDeleteAux = xDelete; return; failed: @@ -75401,7 +91759,7 @@ failed: #ifndef SQLITE_OMIT_DEPRECATED /* -** Return the number of times the Step function of an aggregate has been +** Return the number of times the Step function of an aggregate has been ** called. ** ** This function is deprecated. Do not use it for new code. It is @@ -75409,7 +91767,7 @@ failed: ** implementations should keep their own counts within their aggregate ** context. */ -SQLITE_API int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context *p){ +SQLITE_API int sqlite3_aggregate_count(sqlite3_context *p){ assert( p && p->pMem && p->pFunc && p->pFunc->xFinalize ); return p->pMem->n; } @@ -75418,18 +91776,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_aggregate_count(sqlite3_context *p){ /* ** Return the number of columns in the result set for the statement pStmt. */ -SQLITE_API int SQLITE_STDCALL sqlite3_column_count(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - return pVm ? pVm->nResColumn : 0; + if( pVm==0 ) return 0; + return pVm->nResColumn; } /* ** Return the number of values available from the current row of the ** currently executing statement pStmt. */ -SQLITE_API int SQLITE_STDCALL sqlite3_data_count(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt){ Vdbe *pVm = (Vdbe *)pStmt; - if( pVm==0 || pVm->pResultSet==0 ) return 0; + if( pVm==0 || pVm->pResultRow==0 ) return 0; return pVm->nResColumn; } @@ -75446,25 +91805,25 @@ static const Mem *columnNullValue(void){ ** these assert()s from failing, when building with SQLITE_DEBUG defined ** using gcc, we force nullMem to be 8-byte aligned using the magical ** __attribute__((aligned(8))) macro. */ - static const Mem nullMem + static const Mem nullMem #if defined(SQLITE_DEBUG) && defined(__GNUC__) - __attribute__((aligned(8))) + __attribute__((aligned(8))) #endif = { /* .u = */ {0}, + /* .z = */ (char*)0, + /* .n = */ (int)0, /* .flags = */ (u16)MEM_Null, /* .enc = */ (u8)0, /* .eSubtype = */ (u8)0, - /* .n = */ (int)0, - /* .z = */ (char*)0, - /* .zMalloc = */ (char*)0, + /* .db = */ (sqlite3*)0, /* .szMalloc = */ (int)0, /* .uTemp = */ (u32)0, - /* .db = */ (sqlite3*)0, + /* .zMalloc = */ (char*)0, /* .xDel = */ (void(*)(void*))0, #ifdef SQLITE_DEBUG /* .pScopyFrom = */ (Mem*)0, - /* .pFiller = */ (void*)0, + /* .mScopyFlags= */ 0, #endif }; return &nullMem; @@ -75481,23 +91840,22 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ Mem *pOut; pVm = (Vdbe *)pStmt; - if( pVm && pVm->pResultSet!=0 && inResColumn && i>=0 ){ - sqlite3_mutex_enter(pVm->db->mutex); - pOut = &pVm->pResultSet[i]; + if( pVm==0 ) return (Mem*)columnNullValue(); + assert( pVm->db ); + sqlite3_mutex_enter(pVm->db->mutex); + if( pVm->pResultRow!=0 && inResColumn && i>=0 ){ + pOut = &pVm->pResultRow[i]; }else{ - if( pVm && ALWAYS(pVm->db) ){ - sqlite3_mutex_enter(pVm->db->mutex); - sqlite3Error(pVm->db, SQLITE_RANGE); - } + sqlite3Error(pVm->db, SQLITE_RANGE); pOut = (Mem*)columnNullValue(); } return pOut; } /* -** This function is called after invoking an sqlite3_value_XXX function on a +** This function is called after invoking an sqlite3_value_XXX function on a ** column value (i.e. a value returned by evaluating an SQL expression in the -** select list of a SELECT statement) that may cause a malloc() failure. If +** select list of a SELECT statement) that may cause a malloc() failure. If ** malloc() has failed, the threads mallocFailed flag is cleared and the result ** code of statement pStmt set to SQLITE_NOMEM. ** @@ -75510,7 +91868,7 @@ static Mem *columnMem(sqlite3_stmt *pStmt, int i){ ** sqlite3_column_real() ** sqlite3_column_bytes() ** sqlite3_column_bytes16() -** sqiite3_column_blob() +** sqlite3_column_blob() */ static void columnMallocFailure(sqlite3_stmt *pStmt) { @@ -75521,6 +91879,8 @@ static void columnMallocFailure(sqlite3_stmt *pStmt) */ Vdbe *p = (Vdbe *)pStmt; if( p ){ + assert( p->db!=0 ); + assert( sqlite3_mutex_held(p->db->mutex) ); p->rc = sqlite3ApiExit(p->db, p->rc); sqlite3_mutex_leave(p->db->mutex); } @@ -75530,47 +91890,47 @@ static void columnMallocFailure(sqlite3_stmt *pStmt) ** The following routines are used to access elements of the current row ** in the result set. */ -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt *pStmt, int i){ const void *val; val = sqlite3_value_blob( columnMem(pStmt,i) ); /* Even though there is no encoding conversion, value_blob() might - ** need to call malloc() to expand the result of a zeroblob() - ** expression. + ** need to call malloc() to expand the result of a zeroblob() + ** expression. */ columnMallocFailure(pStmt); return val; } -SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_bytes( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API int SQLITE_STDCALL sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_bytes16( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API double SQLITE_STDCALL sqlite3_column_double(sqlite3_stmt *pStmt, int i){ +SQLITE_API double sqlite3_column_double(sqlite3_stmt *pStmt, int i){ double val = sqlite3_value_double( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API int SQLITE_STDCALL sqlite3_column_int(sqlite3_stmt *pStmt, int i){ +SQLITE_API int sqlite3_column_int(sqlite3_stmt *pStmt, int i){ int val = sqlite3_value_int( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ +SQLITE_API sqlite_int64 sqlite3_column_int64(sqlite3_stmt *pStmt, int i){ sqlite_int64 val = sqlite3_value_int64( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API const unsigned char *SQLITE_STDCALL sqlite3_column_text(sqlite3_stmt *pStmt, int i){ +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt *pStmt, int i){ const unsigned char *val = sqlite3_value_text( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } -SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt *pStmt, int i){ +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt *pStmt, int i){ Mem *pOut = columnMem(pStmt, i); if( pOut->flags&MEM_Static ){ pOut->flags &= ~MEM_Static; @@ -75580,18 +91940,44 @@ SQLITE_API sqlite3_value *SQLITE_STDCALL sqlite3_column_value(sqlite3_stmt *pStm return (sqlite3_value *)pOut; } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt *pStmt, int i){ const void *val = sqlite3_value_text16( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return val; } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt *pStmt, int i){ +SQLITE_API int sqlite3_column_type(sqlite3_stmt *pStmt, int i){ int iType = sqlite3_value_type( columnMem(pStmt,i) ); columnMallocFailure(pStmt); return iType; } +/* +** Column names appropriate for EXPLAIN or EXPLAIN QUERY PLAN. +*/ +static const char * const azExplainColNames8[] = { + "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", /* EXPLAIN */ + "id", "parent", "notused", "detail" /* EQP */ +}; +static const u16 azExplainColNames16data[] = { + /* 0 */ 'a', 'd', 'd', 'r', 0, + /* 5 */ 'o', 'p', 'c', 'o', 'd', 'e', 0, + /* 12 */ 'p', '1', 0, + /* 15 */ 'p', '2', 0, + /* 18 */ 'p', '3', 0, + /* 21 */ 'p', '4', 0, + /* 24 */ 'p', '5', 0, + /* 27 */ 'c', 'o', 'm', 'm', 'e', 'n', 't', 0, + /* 35 */ 'i', 'd', 0, + /* 38 */ 'p', 'a', 'r', 'e', 'n', 't', 0, + /* 45 */ 'n', 'o', 't', 'u', 's', 'e', 'd', 0, + /* 53 */ 'd', 'e', 't', 'a', 'i', 'l', 0 +}; +static const u8 iExplainColNames16[] = { + 0, 5, 12, 15, 18, 21, 24, 27, + 35, 38, 45, 53 +}; + /* ** Convert the N-th element of pStmt->pColName[] into a string using ** xFunc() then return that string. If N is out of range, return 0. @@ -75609,10 +91995,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_column_type(sqlite3_stmt *pStmt, int i){ ** or a constant) then useTypes 2, 3, and 4 return NULL. */ static const void *columnName( - sqlite3_stmt *pStmt, - int N, - const void *(*xFunc)(Mem*), - int useType + sqlite3_stmt *pStmt, /* The statement */ + int N, /* Which column to get the name for */ + int useUtf16, /* True to return the name as UTF16 */ + int useType /* What type of name */ ){ const void *ret; Vdbe *p; @@ -75624,25 +92010,48 @@ static const void *columnName( return 0; } #endif + if( N<0 ) return 0; ret = 0; p = (Vdbe *)pStmt; db = p->db; assert( db!=0 ); - n = sqlite3_column_count(pStmt); - if( N=0 ){ + sqlite3_mutex_enter(db->mutex); + + if( p->explain ){ + if( useType>0 ) goto columnName_end; + n = p->explain==1 ? 8 : 4; + if( N>=n ) goto columnName_end; + if( useUtf16 ){ + int i = iExplainColNames16[N + 8*p->explain - 8]; + ret = (void*)&azExplainColNames16data[i]; + }else{ + ret = (void*)azExplainColNames8[N + 8*p->explain - 8]; + } + goto columnName_end; + } + n = p->nResColumn; + if( NmallocFailed; N += useType*n; - sqlite3_mutex_enter(db->mutex); - assert( db->mallocFailed==0 ); - ret = xFunc(&p->aColName[N]); - /* A malloc may have failed inside of the xFunc() call. If this +#ifndef SQLITE_OMIT_UTF16 + if( useUtf16 ){ + ret = sqlite3_value_text16((sqlite3_value*)&p->aColName[N]); + }else +#endif + { + ret = sqlite3_value_text((sqlite3_value*)&p->aColName[N]); + } + /* A malloc may have failed inside of the _text() call. If this ** is the case, clear the mallocFailed flag and return NULL. */ - if( db->mallocFailed ){ + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + if( db->mallocFailed > prior_mallocFailed ){ sqlite3OomClear(db); ret = 0; } - sqlite3_mutex_leave(db->mutex); } +columnName_end: + sqlite3_mutex_leave(db->mutex); return ret; } @@ -75650,14 +92059,12 @@ static const void *columnName( ** Return the name of the Nth column of the result set returned by SQL ** statement pStmt. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_column_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_NAME); +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_NAME); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_NAME); +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_NAME); } #endif @@ -75675,14 +92082,12 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_name16(sqlite3_stmt *pStmt, ** Return the column declaration type (if applicable) of the 'i'th column ** of the result set of SQL statement pStmt. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DECLTYPE); +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_DECLTYPE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DECLTYPE); +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_DECLTYPE); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_OMIT_DECLTYPE */ @@ -75693,14 +92098,12 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_decltype16(sqlite3_stmt *pS ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_DATABASE); +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_DATABASE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_DATABASE); +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_DATABASE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -75709,14 +92112,12 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_database_name16(sqlite3_stm ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_TABLE); +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_TABLE); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_TABLE); +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_TABLE); } #endif /* SQLITE_OMIT_UTF16 */ @@ -75725,70 +92126,77 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_column_table_name16(sqlite3_stmt * ** NULL is returned if the result column is an expression or constant or ** anything else which is not an unambiguous reference to a database column. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text, COLNAME_COLUMN); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 0, COLNAME_COLUMN); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API const void *SQLITE_STDCALL sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ - return columnName( - pStmt, N, (const void*(*)(Mem*))sqlite3_value_text16, COLNAME_COLUMN); +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt *pStmt, int N){ + return columnName(pStmt, N, 1, COLNAME_COLUMN); } #endif /* SQLITE_OMIT_UTF16 */ #endif /* SQLITE_ENABLE_COLUMN_METADATA */ /******************************* sqlite3_bind_ *************************** -** +** ** Routines used to attach values to wildcards in a compiled SQL statement. */ /* -** Unbind the value bound to variable i in virtual machine p. This is the +** Unbind the value bound to variable i in virtual machine p. This is the ** the same as binding a NULL value to the column. If the "i" parameter is -** out of range, then SQLITE_RANGE is returned. Othewise SQLITE_OK. +** out of range, then SQLITE_RANGE is returned. Otherwise SQLITE_OK. ** ** A successful evaluation of this routine acquires the mutex on p. ** the mutex is released if any kind of error occurs. ** ** The error code stored in database p->db is overwritten with the return ** value in any case. +** +** (tag-20240917-01) If vdbeUnbind(p,(u32)(i-1)) returns SQLITE_OK, +** that means all of the the following will be true: +** +** p!=0 +** p->pVar!=0 +** i>0 +** i<=p->nVar +** +** An assert() is normally added after vdbeUnbind() to help static analyzers +** realize this. */ -static int vdbeUnbind(Vdbe *p, int i){ +static int vdbeUnbind(Vdbe *p, unsigned int i){ Mem *pVar; if( vdbeSafetyNotNull(p) ){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(p->db->mutex); - if( p->magic!=VDBE_MAGIC_RUN || p->pc>=0 ){ - sqlite3Error(p->db, SQLITE_MISUSE); + if( p->eVdbeState!=VDBE_READY_STATE ){ + sqlite3Error(p->db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(p->db->mutex); - sqlite3_log(SQLITE_MISUSE, + sqlite3_log(SQLITE_MISUSE, "bind on a busy prepared statement: [%s]", p->zSql); return SQLITE_MISUSE_BKPT; } - if( i<1 || i>p->nVar ){ + if( i>=(unsigned int)p->nVar ){ sqlite3Error(p->db, SQLITE_RANGE); sqlite3_mutex_leave(p->db->mutex); return SQLITE_RANGE; } - i--; pVar = &p->aVar[i]; sqlite3VdbeMemRelease(pVar); pVar->flags = MEM_Null; - sqlite3Error(p->db, SQLITE_OK); + p->db->errCode = SQLITE_OK; - /* If the bit corresponding to this variable in Vdbe.expmask is set, then + /* If the bit corresponding to this variable in Vdbe.expmask is set, then ** binding a new value to this variable invalidates the current query plan. ** - ** IMPLEMENTATION-OF: R-48440-37595 If the specific value bound to host + ** IMPLEMENTATION-OF: R-57496-20354 If the specific value bound to a host ** parameter in the WHERE clause might influence the choice of query plan ** for a statement, then the statement will be automatically recompiled, ** as if there had been a schema change, on the first sqlite3_step() call ** following any change to the bindings of that parameter. */ - if( p->isPrepareV2 && - ((i<32 && p->expmask & ((u32)1 << i)) || p->expmask==0xffffffff) - ){ + assert( (p->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || p->expmask==0 ); + if( p->expmask!=0 && (p->expmask & (i>=31 ? 0x80000000 : (u32)1<expired = 1; } return SQLITE_OK; @@ -75801,7 +92209,7 @@ static int bindText( sqlite3_stmt *pStmt, /* The statement to bind against */ int i, /* Index of the parameter to bind */ const void *zData, /* Pointer to the data to be bound */ - int nData, /* Number of bytes of data to be bound */ + i64 nData, /* Number of bytes of data to be bound */ void (*xDel)(void*), /* Destructor for the data */ u8 encoding /* Encoding for the data */ ){ @@ -75809,16 +92217,19 @@ static int bindText( Mem *pVar; int rc; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ if( zData!=0 ){ pVar = &p->aVar[i-1]; rc = sqlite3VdbeMemSetStr(pVar, zData, nData, encoding, xDel); if( rc==SQLITE_OK && encoding!=0 ){ rc = sqlite3VdbeChangeEncoding(pVar, ENC(p->db)); } - sqlite3Error(p->db, rc); - rc = sqlite3ApiExit(p->db, rc); + if( rc ){ + sqlite3Error(p->db, rc); + rc = sqlite3ApiExit(p->db, rc); + } } sqlite3_mutex_leave(p->db->mutex); }else if( xDel!=SQLITE_STATIC && xDel!=SQLITE_TRANSIENT ){ @@ -75831,11 +92242,11 @@ static int bindText( /* ** Bind a blob value to an SQL statement variable. */ -SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob( - sqlite3_stmt *pStmt, - int i, - const void *zData, - int nData, +SQLITE_API int sqlite3_bind_blob( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int nData, void (*xDel)(void*) ){ #ifdef SQLITE_ENABLE_API_ARMOR @@ -75843,89 +92254,106 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob( #endif return bindText(pStmt, i, zData, nData, xDel, 0); } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_blob64( - sqlite3_stmt *pStmt, - int i, - const void *zData, - sqlite3_uint64 nData, +SQLITE_API int sqlite3_bind_blob64( + sqlite3_stmt *pStmt, + int i, + const void *zData, + sqlite3_uint64 nData, void (*xDel)(void*) ){ assert( xDel!=SQLITE_DYNAMIC ); - if( nData>0x7fffffff ){ - return invokeValueDestructor(zData, xDel, 0); - }else{ - return bindText(pStmt, i, zData, (int)nData, xDel, 0); - } + return bindText(pStmt, i, zData, nData, xDel, 0); } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ +SQLITE_API int sqlite3_bind_double(sqlite3_stmt *pStmt, int i, double rValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3VdbeMemSetDouble(&p->aVar[i-1], rValue); sqlite3_mutex_leave(p->db->mutex); } return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ +SQLITE_API int sqlite3_bind_int(sqlite3_stmt *p, int i, int iValue){ return sqlite3_bind_int64(p, i, (i64)iValue); } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt *pStmt, int i, sqlite_int64 iValue){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3VdbeMemSetInt64(&p->aVar[i-1], iValue); sqlite3_mutex_leave(p->db->mutex); } return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ +SQLITE_API int sqlite3_bind_null(sqlite3_stmt *pStmt, int i){ int rc; Vdbe *p = (Vdbe*)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ sqlite3_mutex_leave(p->db->mutex); } return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_text( - sqlite3_stmt *pStmt, - int i, - const char *zData, - int nData, +SQLITE_API int sqlite3_bind_pointer( + sqlite3_stmt *pStmt, + int i, + void *pPtr, + const char *zPTtype, + void (*xDestructor)(void*) +){ + int rc; + Vdbe *p = (Vdbe*)pStmt; + rc = vdbeUnbind(p, (u32)(i-1)); + if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ + sqlite3VdbeMemSetPointer(&p->aVar[i-1], pPtr, zPTtype, xDestructor); + sqlite3_mutex_leave(p->db->mutex); + }else if( xDestructor ){ + xDestructor(pPtr); + } + return rc; +} +SQLITE_API int sqlite3_bind_text( + sqlite3_stmt *pStmt, + int i, + const char *zData, + int nData, void (*xDel)(void*) ){ return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF8); } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_text64( - sqlite3_stmt *pStmt, - int i, - const char *zData, - sqlite3_uint64 nData, +SQLITE_API int sqlite3_bind_text64( + sqlite3_stmt *pStmt, + int i, + const char *zData, + sqlite3_uint64 nData, void (*xDel)(void*), unsigned char enc ){ assert( xDel!=SQLITE_DYNAMIC ); - if( nData>0x7fffffff ){ - return invokeValueDestructor(zData, xDel, 0); - }else{ + if( enc!=SQLITE_UTF8 ){ if( enc==SQLITE_UTF16 ) enc = SQLITE_UTF16NATIVE; - return bindText(pStmt, i, zData, (int)nData, xDel, enc); + nData &= ~(u16)1; } + return bindText(pStmt, i, zData, nData, xDel, enc); } #ifndef SQLITE_OMIT_UTF16 -SQLITE_API int SQLITE_STDCALL sqlite3_bind_text16( - sqlite3_stmt *pStmt, - int i, - const void *zData, - int nData, +SQLITE_API int sqlite3_bind_text16( + sqlite3_stmt *pStmt, + int i, + const void *zData, + int n, void (*xDel)(void*) ){ - return bindText(pStmt, i, zData, nData, xDel, SQLITE_UTF16NATIVE); + return bindText(pStmt, i, zData, n & ~(u64)1, xDel, SQLITE_UTF16NATIVE); } #endif /* SQLITE_OMIT_UTF16 */ -SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ +SQLITE_API int sqlite3_bind_value(sqlite3_stmt *pStmt, int i, const sqlite3_value *pValue){ int rc; switch( sqlite3_value_type((sqlite3_value*)pValue) ){ case SQLITE_INTEGER: { @@ -75933,7 +92361,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt *pStmt, int i, con break; } case SQLITE_FLOAT: { - rc = sqlite3_bind_double(pStmt, i, pValue->u.r); + assert( pValue->flags & (MEM_Real|MEM_IntReal) ); + rc = sqlite3_bind_double(pStmt, i, + (pValue->flags & MEM_Real) ? pValue->u.r : (double)pValue->u.i + ); break; } case SQLITE_BLOB: { @@ -75956,19 +92387,27 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_value(sqlite3_stmt *pStmt, int i, con } return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt *pStmt, int i, int n){ int rc; Vdbe *p = (Vdbe *)pStmt; - rc = vdbeUnbind(p, i); + rc = vdbeUnbind(p, (u32)(i-1)); if( rc==SQLITE_OK ){ + assert( p!=0 && p->aVar!=0 && i>0 && i<=p->nVar ); /* tag-20240917-01 */ +#ifndef SQLITE_OMIT_INCRBLOB sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); +#else + rc = sqlite3VdbeMemSetZeroBlob(&p->aVar[i-1], n); +#endif sqlite3_mutex_leave(p->db->mutex); } return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ +SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i, sqlite3_uint64 n){ int rc; Vdbe *p = (Vdbe *)pStmt; +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(p->db->mutex); if( n>(u64)p->db->aLimit[SQLITE_LIMIT_LENGTH] ){ rc = SQLITE_TOOBIG; @@ -75983,9 +92422,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_zeroblob64(sqlite3_stmt *pStmt, int i /* ** Return the number of wildcards that can be potentially bound to. -** This routine is added to support DBD::SQLite. +** This routine is added to support DBD::SQLite. */ -SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; return p ? p->nVar : 0; } @@ -75996,12 +92435,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_count(sqlite3_stmt *pStmt){ ** ** The result is always UTF-8. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt *pStmt, int i){ Vdbe *p = (Vdbe*)pStmt; - if( p==0 || i<1 || i>p->nzVar ){ - return 0; - } - return p->azVar[i-1]; + if( p==0 ) return 0; + return sqlite3VListNumToName(p->pVList, i); } /* @@ -76010,21 +92447,10 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_bind_parameter_name(sqlite3_stmt * ** return 0. */ SQLITE_PRIVATE int sqlite3VdbeParameterIndex(Vdbe *p, const char *zName, int nName){ - int i; - if( p==0 ){ - return 0; - } - if( zName ){ - for(i=0; inzVar; i++){ - const char *z = p->azVar[i]; - if( z && strncmp(z,zName,nName)==0 && z[nName]==0 ){ - return i+1; - } - } - } - return 0; + if( p==0 || zName==0 ) return 0; + return sqlite3VListNameToNum(p->pVList, zName, nName); } -SQLITE_API int SQLITE_STDCALL sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt *pStmt, const char *zName){ return sqlite3VdbeParameterIndex((Vdbe*)pStmt, zName, sqlite3Strlen30(zName)); } @@ -76058,16 +92484,18 @@ SQLITE_PRIVATE int sqlite3TransferBindings(sqlite3_stmt *pFromStmt, sqlite3_stmt ** an SQLITE_ERROR is returned. Nothing else can go wrong, so otherwise ** SQLITE_OK is returned. */ -SQLITE_API int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ +SQLITE_API int sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, sqlite3_stmt *pToStmt){ Vdbe *pFrom = (Vdbe*)pFromStmt; Vdbe *pTo = (Vdbe*)pToStmt; if( pFrom->nVar!=pTo->nVar ){ return SQLITE_ERROR; } - if( pTo->isPrepareV2 && pTo->expmask ){ + assert( (pTo->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || pTo->expmask==0 ); + if( pTo->expmask ){ pTo->expired = 1; } - if( pFrom->isPrepareV2 && pFrom->expmask ){ + assert( (pFrom->prepFlags & SQLITE_PREPARE_SAVESQL)!=0 || pFrom->expmask==0 ); + if( pFrom->expmask ){ pFrom->expired = 1; } return sqlite3TransferBindings(pFromStmt, pToStmt); @@ -76080,7 +92508,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_transfer_bindings(sqlite3_stmt *pFromStmt, ** the first argument to the sqlite3_prepare() that was used to create ** the statement in the first place. */ -SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt *pStmt){ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->db : 0; } @@ -76088,16 +92516,60 @@ SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3_db_handle(sqlite3_stmt *pStmt){ ** Return true if the prepared statement is guaranteed to not modify the ** database. */ -SQLITE_API int SQLITE_STDCALL sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt){ return pStmt ? ((Vdbe*)pStmt)->readOnly : 1; } +/* +** Return 1 if the statement is an EXPLAIN and return 2 if the +** statement is an EXPLAIN QUERY PLAN +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt){ + return pStmt ? ((Vdbe*)pStmt)->explain : 0; +} + +/* +** Set the explain mode for a statement. +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode){ + Vdbe *v = (Vdbe*)pStmt; + int rc; +#ifdef SQLITE_ENABLE_API_ARMOR + if( pStmt==0 ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(v->db->mutex); + if( ((int)v->explain)==eMode ){ + rc = SQLITE_OK; + }else if( eMode<0 || eMode>2 ){ + rc = SQLITE_ERROR; + }else if( (v->prepFlags & SQLITE_PREPARE_SAVESQL)==0 ){ + rc = SQLITE_ERROR; + }else if( v->eVdbeState!=VDBE_READY_STATE ){ + rc = SQLITE_BUSY; + }else if( v->nMem>=10 && (eMode!=2 || v->haveEqpOps) ){ + /* No reprepare necessary */ + v->explain = eMode; + rc = SQLITE_OK; + }else{ + v->explain = eMode; + rc = sqlite3Reprepare(v); + v->haveEqpOps = eMode==2; + } + if( v->explain ){ + v->nResColumn = 12 - 4*v->explain; + }else{ + v->nResColumn = v->nResAlloc; + } + sqlite3_mutex_leave(v->db->mutex); + return rc; +} + /* ** Return true if the prepared statement is in need of being reset. */ -SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt *pStmt){ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt *pStmt){ Vdbe *v = (Vdbe*)pStmt; - return v!=0 && v->pc>=0 && v->magic==VDBE_MAGIC_RUN; + return v!=0 && v->eVdbeState==VDBE_RUN_STATE; } /* @@ -76106,7 +92578,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_busy(sqlite3_stmt *pStmt){ ** prepared statement for the database connection. Return NULL if there ** are no more. */ -SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt){ sqlite3_stmt *pNext; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(pDb) ){ @@ -76118,7 +92590,7 @@ SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_ if( pStmt==0 ){ pNext = (sqlite3_stmt*)pDb->pVdbe; }else{ - pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pNext; + pNext = (sqlite3_stmt*)((Vdbe*)pStmt)->pVNext; } sqlite3_mutex_leave(pDb->mutex); return pNext; @@ -76127,20 +92599,84 @@ SQLITE_API sqlite3_stmt *SQLITE_STDCALL sqlite3_next_stmt(sqlite3 *pDb, sqlite3_ /* ** Return the value of a status counter for a prepared statement */ -SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ Vdbe *pVdbe = (Vdbe*)pStmt; u32 v; #ifdef SQLITE_ENABLE_API_ARMOR - if( !pStmt ){ + if( !pStmt + || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) + ){ (void)SQLITE_MISUSE_BKPT; return 0; } #endif - v = pVdbe->aCounter[op]; - if( resetFlag ) pVdbe->aCounter[op] = 0; + if( op==SQLITE_STMTSTATUS_MEMUSED ){ + sqlite3 *db = pVdbe->db; + sqlite3_mutex_enter(db->mutex); + v = 0; + db->pnBytesFreed = (int*)&v; + assert( db->lookaside.pEnd==db->lookaside.pTrueEnd ); + db->lookaside.pEnd = db->lookaside.pStart; + sqlite3VdbeDelete(pVdbe); + db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; + sqlite3_mutex_leave(db->mutex); + }else{ + v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + } return (int)v; } +/* +** Return the SQL associated with a prepared statement +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + return p ? p->zSql : 0; +} + +/* +** Return the SQL associated with a prepared statement with +** bound parameters expanded. Space to hold the returned string is +** obtained from sqlite3_malloc(). The caller is responsible for +** freeing the returned string by passing it to sqlite3_free(). +** +** The SQLITE_TRACE_SIZE_LIMIT puts an upper bound on the size of +** expanded bound parameters. +*/ +SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt){ +#ifdef SQLITE_OMIT_TRACE + return 0; +#else + char *z = 0; + const char *zSql = sqlite3_sql(pStmt); + if( zSql ){ + Vdbe *p = (Vdbe *)pStmt; + sqlite3_mutex_enter(p->db->mutex); + z = sqlite3VdbeExpandSql(p, zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return z; +#endif +} + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Return the normalized SQL associated with a prepared statement. +*/ +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt){ + Vdbe *p = (Vdbe *)pStmt; + if( p==0 ) return 0; + if( p->zNormSql==0 && ALWAYS(p->zSql!=0) ){ + sqlite3_mutex_enter(p->db->mutex); + p->zNormSql = sqlite3Normalize(p, p->zSql); + sqlite3_mutex_leave(p->db->mutex); + } + return p->zNormSql; +} +#endif /* SQLITE_ENABLE_NORMALIZE */ + #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* ** Allocate and populate an UnpackedRecord structure based on the serialized @@ -76148,16 +92684,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, i ** if successful, or a NULL pointer if an OOM error is encountered. */ static UnpackedRecord *vdbeUnpackRecord( - KeyInfo *pKeyInfo, - int nKey, + KeyInfo *pKeyInfo, + int nKey, const void *pKey ){ - char *dummy; /* Dummy argument for AllocUnpackedRecord() */ UnpackedRecord *pRet; /* Return value */ - pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo, 0, 0, &dummy); + pRet = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( pRet ){ - memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nField+1)); + memset(pRet->aMem, 0, sizeof(Mem)*(pKeyInfo->nKeyField+1)); sqlite3VdbeRecordUnpack(pKeyInfo, nKey, pKey, pRet); } return pRet; @@ -76167,16 +92702,26 @@ static UnpackedRecord *vdbeUnpackRecord( ** This function is called from within a pre-update callback to retrieve ** a field of the row currently being updated or deleted. */ -SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; +SQLITE_API int sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ + PreUpdate *p; + Mem *pMem; int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 || ppValue==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + p = db->pPreUpdate; /* Test that this call is being made from within an SQLITE_DELETE or ** SQLITE_UPDATE pre-update callback, and that iIdx is within range. */ if( !p || p->op==SQLITE_INSERT ){ rc = SQLITE_MISUSE_BKPT; goto preupdate_old_out; } + if( p->pPk ){ + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_old_out; @@ -76187,11 +92732,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlit u32 nRec; u8 *aRec; - rc = sqlite3BtreeDataSize(p->pCsr->uc.pCursor, &nRec); - if( rc!=SQLITE_OK ) goto preupdate_old_out; + assert( p->pCsr->eCurType==CURTYPE_BTREE ); + nRec = sqlite3BtreePayloadSize(p->pCsr->uc.pCursor); aRec = sqlite3DbMallocRaw(db, nRec); if( !aRec ) goto preupdate_old_out; - rc = sqlite3BtreeData(p->pCsr->uc.pCursor, 0, nRec, aRec); + rc = sqlite3BtreePayload(p->pCsr->uc.pCursor, 0, nRec, aRec); if( rc==SQLITE_OK ){ p->pUnpacked = vdbeUnpackRecord(&p->keyinfo, nRec, aRec); if( !p->pUnpacked ) rc = SQLITE_NOMEM; @@ -76203,12 +92748,39 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlit p->aRecord = aRec; } - if( iIdx>=p->pUnpacked->nField ){ - *ppValue = (sqlite3_value *)columnNullValue(); - }else{ - *ppValue = &p->pUnpacked->aMem[iIdx]; - if( iIdx==p->iPKey ){ - sqlite3VdbeMemSetInt64(*ppValue, p->iKey1); + pMem = *ppValue = &p->pUnpacked->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey1); + }else if( iIdx>=p->pUnpacked->nField ){ + /* This occurs when the table has been extended using ALTER TABLE + ** ADD COLUMN. The value to return is the default value of the column. */ + Column *pCol = &p->pTab->aCol[iIdx]; + if( pCol->iDflt>0 ){ + if( p->apDflt==0 ){ + int nByte = sizeof(sqlite3_value*)*p->pTab->nCol; + p->apDflt = (sqlite3_value**)sqlite3DbMallocZero(db, nByte); + if( p->apDflt==0 ) goto preupdate_old_out; + } + if( p->apDflt[iIdx]==0 ){ + sqlite3_value *pVal = 0; + Expr *pDflt; + assert( p->pTab!=0 && IsOrdinaryTable(p->pTab) ); + pDflt = p->pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; + rc = sqlite3ValueFromExpr(db, pDflt, ENC(db), pCol->affinity, &pVal); + if( rc==SQLITE_OK && pVal==0 ){ + rc = SQLITE_CORRUPT_BKPT; + } + p->apDflt[iIdx] = pVal; + } + *ppValue = p->apDflt[iIdx]; + }else{ + *ppValue = (sqlite3_value *)columnNullValue(); + } + }else if( p->pTab->aCol[iIdx].affinity==SQLITE_AFF_REAL ){ + if( pMem->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_IntReal ); + sqlite3VdbeMemRealify(pMem); } } @@ -76223,9 +92795,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_old(sqlite3 *db, int iIdx, sqlit ** This function is called from within a pre-update callback to retrieve ** the number of columns in the row being updated, deleted or inserted. */ -SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_count(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; - return (p ? p->keyinfo.nField : 0); +SQLITE_API int sqlite3_preupdate_count(sqlite3 *db){ + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif + return (p ? p->keyinfo.nKeyField : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ @@ -76235,32 +92812,62 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_count(sqlite3 *db){ ** only. It returns zero if the change that caused the callback was made ** immediately by a user SQL statement. Or, if the change was made by a ** trigger program, it returns the number of trigger programs currently -** on the stack (1 for a top-level trigger, 2 for a trigger fired by a +** on the stack (1 for a top-level trigger, 2 for a trigger fired by a ** top-level trigger etc.). ** ** For the purposes of the previous paragraph, a foreign key CASCADE, SET NULL ** or SET DEFAULT action is considered a trigger. */ -SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_depth(sqlite3 *db){ - PreUpdate *p = db->pPreUpdate; +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *db){ + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif return (p ? p->v->nFrame : 0); } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +/* +** This function is designed to be called from within a pre-update callback +** only. +*/ +SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *db){ + PreUpdate *p; +#ifdef SQLITE_ENABLE_API_ARMOR + p = db!=0 ? db->pPreUpdate : 0; +#else + p = db->pPreUpdate; +#endif + return (p ? p->iBlobWrite : -1); +} +#endif + #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* ** This function is called from within a pre-update callback to retrieve ** a field of the row currently being updated or inserted. */ -SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ - PreUpdate *p = db->pPreUpdate; +SQLITE_API int sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlite3_value **ppValue){ + PreUpdate *p; int rc = SQLITE_OK; Mem *pMem; +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 || ppValue==0 ){ + return SQLITE_MISUSE_BKPT; + } +#endif + p = db->pPreUpdate; if( !p || p->op==SQLITE_DELETE ){ rc = SQLITE_MISUSE_BKPT; goto preupdate_new_out; } + if( p->pPk && p->op!=SQLITE_UPDATE ){ + iIdx = sqlite3TableColumnToIndex(p->pPk, iIdx); + } if( iIdx>=p->pCsr->nField || iIdx<0 ){ rc = SQLITE_RANGE; goto preupdate_new_out; @@ -76272,7 +92879,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlit UnpackedRecord *pUnpack = p->pNewUnpacked; if( !pUnpack ){ Mem *pData = &p->v->aMem[p->iNewReg]; - rc = sqlite3VdbeMemExpandBlob(pData); + rc = ExpandBlob(pData); if( rc!=SQLITE_OK ) goto preupdate_new_out; pUnpack = vdbeUnpackRecord(&p->keyinfo, pData->n, pData->z); if( !pUnpack ){ @@ -76281,13 +92888,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlit } p->pNewUnpacked = pUnpack; } - if( iIdx>=pUnpack->nField ){ + pMem = &pUnpack->aMem[iIdx]; + if( iIdx==p->pTab->iPKey ){ + sqlite3VdbeMemSetInt64(pMem, p->iKey2); + }else if( iIdx>=pUnpack->nField ){ pMem = (sqlite3_value *)columnNullValue(); - }else{ - pMem = &pUnpack->aMem[iIdx]; - if( iIdx==p->iPKey ){ - sqlite3VdbeMemSetInt64(pMem, p->iKey2); - } } }else{ /* For an UPDATE, memory cell (p->iNewReg+1+iIdx) contains the required @@ -76306,7 +92911,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlit assert( iIdx>=0 && iIdxpCsr->nField ); pMem = &p->aNew[iIdx]; if( pMem->flags==0 ){ - if( iIdx==p->iPKey ){ + if( iIdx==p->pTab->iPKey ){ sqlite3VdbeMemSetInt64(pMem, p->iKey2); }else{ rc = sqlite3VdbeMemCopy(pMem, &p->v->aMem[p->iNewReg+1+iIdx]); @@ -76326,23 +92931,79 @@ SQLITE_API int SQLITE_STDCALL sqlite3_preupdate_new(sqlite3 *db, int iIdx, sqlit /* ** Return status data for a single loop within query pStmt. */ -SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( +SQLITE_API int sqlite3_stmt_scanstatus_v2( sqlite3_stmt *pStmt, /* Prepared statement being queried */ - int idx, /* Index of loop to report on */ + int iScan, /* Index of loop to report on */ int iScanStatusOp, /* Which metric to return */ + int flags, void *pOut /* OUT: Write the answer here */ ){ Vdbe *p = (Vdbe*)pStmt; - ScanStatus *pScan; - if( idx<0 || idx>=p->nScan ) return 1; + VdbeOp *aOp; + int nOp; + ScanStatus *pScan = 0; + int idx; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( p==0 || pOut==0 + || iScanStatusOpSQLITE_SCANSTAT_NCYCLE ){ + return 1; + } +#endif + aOp = p->aOp; + nOp = p->nOp; + if( p->pFrame ){ + VdbeFrame *pFrame; + for(pFrame=p->pFrame; pFrame->pParent; pFrame=pFrame->pParent); + aOp = pFrame->aOp; + nOp = pFrame->nOp; + } + + if( iScan<0 ){ + int ii; + if( iScanStatusOp==SQLITE_SCANSTAT_NCYCLE ){ + i64 res = 0; + for(ii=0; iinScan; idx++){ + pScan = &p->aScan[idx]; + if( pScan->zName ){ + iScan--; + if( iScan<0 ) break; + } + } + } + if( idx>=p->nScan ) return 1; + assert( pScan==0 || pScan==&p->aScan[idx] ); pScan = &p->aScan[idx]; + switch( iScanStatusOp ){ case SQLITE_SCANSTAT_NLOOP: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrLoop]; + if( pScan->addrLoop>0 ){ + *(sqlite3_int64*)pOut = aOp[pScan->addrLoop].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_NVISIT: { - *(sqlite3_int64*)pOut = p->anExec[pScan->addrVisit]; + if( pScan->addrVisit>0 ){ + *(sqlite3_int64*)pOut = aOp[pScan->addrVisit].nExec; + }else{ + *(sqlite3_int64*)pOut = -1; + } break; } case SQLITE_SCANSTAT_EST: { @@ -76361,7 +93022,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( } case SQLITE_SCANSTAT_EXPLAIN: { if( pScan->addrExplain ){ - *(const char**)pOut = p->aOp[ pScan->addrExplain ].p4.z; + *(const char**)pOut = aOp[ pScan->addrExplain ].p4.z; }else{ *(const char**)pOut = 0; } @@ -76369,12 +93030,51 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( } case SQLITE_SCANSTAT_SELECTID: { if( pScan->addrExplain ){ - *(int*)pOut = p->aOp[ pScan->addrExplain ].p1; + *(int*)pOut = aOp[ pScan->addrExplain ].p1; }else{ *(int*)pOut = -1; } break; } + case SQLITE_SCANSTAT_PARENTID: { + if( pScan->addrExplain ){ + *(int*)pOut = aOp[ pScan->addrExplain ].p2; + }else{ + *(int*)pOut = -1; + } + break; + } + case SQLITE_SCANSTAT_NCYCLE: { + i64 res = 0; + if( pScan->aAddrRange[0]==0 ){ + res = -1; + }else{ + int ii; + for(ii=0; iiaAddrRange); ii+=2){ + int iIns = pScan->aAddrRange[ii]; + int iEnd = pScan->aAddrRange[ii+1]; + if( iIns==0 ) break; + if( iIns>0 ){ + while( iIns<=iEnd ){ + res += aOp[iIns].nCycle; + iIns++; + } + }else{ + int iOp; + for(iOp=0; iOpp1!=iEnd ) continue; + if( (sqlite3OpcodeProperty[pOp->opcode] & OPFLG_NCYCLE)==0 ){ + continue; + } + res += aOp[iOp].nCycle; + } + } + } + } + *(i64*)pOut = res; + break; + } default: { return 1; } @@ -76382,12 +93082,29 @@ SQLITE_API int SQLITE_STDCALL sqlite3_stmt_scanstatus( return 0; } +/* +** Return status data for a single loop within query pStmt. +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement being queried */ + int iScan, /* Index of loop to report on */ + int iScanStatusOp, /* Which metric to return */ + void *pOut /* OUT: Write the answer here */ +){ + return sqlite3_stmt_scanstatus_v2(pStmt, iScan, iScanStatusOp, 0, pOut); +} + /* ** Zero all counters associated with the sqlite3_stmt_scanstatus() data. */ -SQLITE_API void SQLITE_STDCALL sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt *pStmt){ Vdbe *p = (Vdbe*)pStmt; - memset(p->anExec, 0, p->nOp * sizeof(i64)); + int ii; + for(ii=0; p!=0 && iinOp; ii++){ + Op *pOp = &p->aOp[ii]; + pOp->nExec = 0; + pOp->nCycle = 0; + } } #endif /* SQLITE_ENABLE_STMT_SCANSTATUS */ @@ -76443,8 +93160,8 @@ static int findNextHostParameter(const char *zSql, int *pnToken){ /* ** This function returns a pointer to a nul-terminated string in memory ** obtained from sqlite3DbMalloc(). If sqlite3.nVdbeExec is 1, then the -** string contains a copy of zRawSql but with host parameters expanded to -** their current bindings. Or, if sqlite3.nVdbeExec is greater than 1, +** string contains a copy of zRawSql but with host parameters expanded to +** their current bindings. Or, if sqlite3.nVdbeExec is greater than 1, ** then the returned string holds a copy of zRawSql with "-- " prepended ** to each line of text. ** @@ -76476,26 +93193,27 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( int i; /* Loop counter */ Mem *pVar; /* Value of a host parameter */ StrAccum out; /* Accumulate the output here */ - char zBase[100]; /* Initial working space */ +#ifndef SQLITE_OMIT_UTF16 + Mem utf8; /* Used to convert UTF16 into UTF8 for display */ +#endif db = p->db; - sqlite3StrAccumInit(&out, db, zBase, sizeof(zBase), - db->aLimit[SQLITE_LIMIT_LENGTH]); + sqlite3StrAccumInit(&out, 0, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); if( db->nVdbeExec>1 ){ while( *zRawSql ){ const char *zStart = zRawSql; while( *(zRawSql++)!='\n' && *zRawSql ); - sqlite3StrAccumAppend(&out, "-- ", 3); + sqlite3_str_append(&out, "-- ", 3); assert( (zRawSql - zStart) > 0 ); - sqlite3StrAccumAppend(&out, zStart, (int)(zRawSql-zStart)); + sqlite3_str_append(&out, zStart, (int)(zRawSql-zStart)); } }else if( p->nVar==0 ){ - sqlite3StrAccumAppend(&out, zRawSql, sqlite3Strlen30(zRawSql)); + sqlite3_str_append(&out, zRawSql, sqlite3Strlen30(zRawSql)); }else{ while( zRawSql[0] ){ n = findNextHostParameter(zRawSql, &nToken); assert( n>0 ); - sqlite3StrAccumAppend(&out, zRawSql, n); + sqlite3_str_append(&out, zRawSql, n); zRawSql += n; assert( zRawSql[0] || nToken==0 ); if( nToken==0 ) break; @@ -76517,25 +93235,27 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( assert( idx>0 ); } zRawSql += nToken; - nextIndex = idx + 1; + nextIndex = MAX(idx + 1, nextIndex); assert( idx>0 && idx<=p->nVar ); pVar = &p->aVar[idx-1]; if( pVar->flags & MEM_Null ){ - sqlite3StrAccumAppend(&out, "NULL", 4); - }else if( pVar->flags & MEM_Int ){ - sqlite3XPrintf(&out, "%lld", pVar->u.i); + sqlite3_str_append(&out, "NULL", 4); + }else if( pVar->flags & (MEM_Int|MEM_IntReal) ){ + sqlite3_str_appendf(&out, "%lld", pVar->u.i); }else if( pVar->flags & MEM_Real ){ - sqlite3XPrintf(&out, "%!.15g", pVar->u.r); + sqlite3_str_appendf(&out, "%!.15g", pVar->u.r); }else if( pVar->flags & MEM_Str ){ int nOut; /* Number of bytes of the string text to include in output */ #ifndef SQLITE_OMIT_UTF16 u8 enc = ENC(db); - Mem utf8; if( enc!=SQLITE_UTF8 ){ memset(&utf8, 0, sizeof(utf8)); utf8.db = db; sqlite3VdbeMemSetStr(&utf8, pVar->z, pVar->n, enc, SQLITE_STATIC); - sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8); + if( SQLITE_NOMEM==sqlite3VdbeChangeEncoding(&utf8, SQLITE_UTF8) ){ + out.accError = SQLITE_NOMEM; + out.nAlloc = 0; + } pVar = &utf8; } #endif @@ -76545,38 +93265,39 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( nOut = SQLITE_TRACE_SIZE_LIMIT; while( nOutn && (pVar->z[nOut]&0xc0)==0x80 ){ nOut++; } } -#endif - sqlite3XPrintf(&out, "'%.*q'", nOut, pVar->z); +#endif + sqlite3_str_appendf(&out, "'%.*q'", nOut, pVar->z); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOutn ){ - sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); + sqlite3_str_appendf(&out, "/*+%d bytes*/", pVar->n-nOut); } #endif #ifndef SQLITE_OMIT_UTF16 if( enc!=SQLITE_UTF8 ) sqlite3VdbeMemRelease(&utf8); #endif }else if( pVar->flags & MEM_Zero ){ - sqlite3XPrintf(&out, "zeroblob(%d)", pVar->u.nZero); + sqlite3_str_appendf(&out, "zeroblob(%d)", pVar->u.nZero); }else{ int nOut; /* Number of bytes of the blob to include in output */ assert( pVar->flags & MEM_Blob ); - sqlite3StrAccumAppend(&out, "x'", 2); + sqlite3_str_append(&out, "x'", 2); nOut = pVar->n; #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOut>SQLITE_TRACE_SIZE_LIMIT ) nOut = SQLITE_TRACE_SIZE_LIMIT; #endif for(i=0; iz[i]&0xff); + sqlite3_str_appendf(&out, "%02x", pVar->z[i]&0xff); } - sqlite3StrAccumAppend(&out, "'", 1); + sqlite3_str_append(&out, "'", 1); #ifdef SQLITE_TRACE_SIZE_LIMIT if( nOutn ){ - sqlite3XPrintf(&out, "/*+%d bytes*/", pVar->n-nOut); + sqlite3_str_appendf(&out, "/*+%d bytes*/", pVar->n-nOut); } #endif } } } + if( out.accError ) sqlite3_str_reset(&out); return sqlite3StrAccumFinish(&out); } @@ -76607,6 +93328,104 @@ SQLITE_PRIVATE char *sqlite3VdbeExpandSql( /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ +/* +** High-resolution hardware timer used for debugging and testing only. +*/ +#if defined(VDBE_PROFILE) \ + || defined(SQLITE_PERFORMANCE_TRACE) \ + || defined(SQLITE_ENABLE_STMT_SCANSTATUS) +/************** Include hwtime.h in the middle of vdbe.c *********************/ +/************** Begin file hwtime.h ******************************************/ +/* +** 2008 May 27 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains inline asm code for retrieving "high-performance" +** counters for x86 and x86_64 class CPUs. +*/ +#ifndef SQLITE_HWTIME_H +#define SQLITE_HWTIME_H + +/* +** The following routine only works on Pentium-class (or newer) processors. +** It uses the RDTSC opcode to read the cycle count value out of the +** processor and returns that value. This can be used for high-res +** profiling. +*/ +#if !defined(__STRICT_ANSI__) && \ + (defined(__GNUC__) || defined(_MSC_VER)) && \ + (defined(i386) || defined(__i386__) || defined(_M_IX86)) + + #if defined(__GNUC__) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + + #elif defined(_MSC_VER) + + __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ + __asm { + rdtsc + ret ; return value at EDX:EAX + } + } + + #endif + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__x86_64__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned int lo, hi; + __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); + return (sqlite_uint64)hi << 32 | lo; + } + +#elif !defined(__STRICT_ANSI__) && (defined(__GNUC__) && defined(__ppc__)) + + __inline__ sqlite_uint64 sqlite3Hwtime(void){ + unsigned long long retval; + unsigned long junk; + __asm__ __volatile__ ("\n\ + 1: mftbu %1\n\ + mftb %L0\n\ + mftbu %0\n\ + cmpw %0,%1\n\ + bne 1b" + : "=r" (retval), "=r" (junk)); + return retval; + } + +#else + + /* + ** asm() is needed for hardware timing support. Without asm(), + ** disable the sqlite3Hwtime() routine. + ** + ** sqlite3Hwtime() is only used for some obscure debugging + ** and analysis configurations, not in any deliverable, so this + ** should not be a great loss. + */ +SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } + +#endif + +#endif /* !defined(SQLITE_HWTIME_H) */ + +/************** End of hwtime.h **********************************************/ +/************** Continuing where we left off in vdbe.c ***********************/ +#endif + /* ** Invoke this macro on memory cells just prior to changing the ** value of the cell. This macro verifies that shallow copies are @@ -76697,54 +93516,114 @@ SQLITE_API int sqlite3_found_count = 0; ** Test a register to see if it exceeds the current maximum blob size. ** If it does, record the new maximum blob size. */ -#if defined(SQLITE_TEST) && !defined(SQLITE_OMIT_BUILTIN_TEST) +#if defined(SQLITE_TEST) && !defined(SQLITE_UNTESTABLE) # define UPDATE_MAX_BLOBSIZE(P) updateMaxBlobsize(P) #else # define UPDATE_MAX_BLOBSIZE(P) #endif +#ifdef SQLITE_DEBUG +/* This routine provides a convenient place to set a breakpoint during +** tracing with PRAGMA vdbe_trace=on. The breakpoint fires right after +** each opcode is printed. Variables "pc" (program counter) and pOp are +** available to add conditionals to the breakpoint. GDB example: +** +** break test_trace_breakpoint if pc=22 +** +** Other useful labels for breakpoints include: +** test_addop_breakpoint(pc,pOp) +** sqlite3CorruptError(lineno) +** sqlite3MisuseError(lineno) +** sqlite3CantopenError(lineno) +*/ +static void test_trace_breakpoint(int pc, Op *pOp, Vdbe *v){ + static u64 n = 0; + (void)pc; + (void)pOp; + (void)v; + n++; + if( n==LARGEST_UINT64 ) abort(); /* So that n is used, preventing a warning */ +} +#endif + /* ** Invoke the VDBE coverage callback, if that callback is defined. This ** feature is used for test suite validation only and does not appear an ** production builds. ** -** M is an integer, 2 or 3, that indices how many different ways the -** branch can go. It is usually 2. "I" is the direction the branch -** goes. 0 means falls through. 1 means branch is taken. 2 means the -** second alternative branch is taken. +** M is the type of branch. I is the direction taken for this instance of +** the branch. +** +** M: 2 - two-way branch (I=0: fall-thru 1: jump ) +** 3 - two-way + NULL (I=0: fall-thru 1: jump 2: NULL ) +** 4 - OP_Jump (I=0: jump p1 1: jump p2 2: jump p3) +** +** In other words, if M is 2, then I is either 0 (for fall-through) or +** 1 (for when the branch is taken). If M is 3, the I is 0 for an +** ordinary fall-through, I is 1 if the branch was taken, and I is 2 +** if the result of comparison is NULL. For M=3, I=2 the jump may or +** may not be taken, depending on the SQLITE_JUMPIFNULL flags in p5. +** When M is 4, that means that an OP_Jump is being run. I is 0, 1, or 2 +** depending on if the operands are less than, equal, or greater than. ** ** iSrcLine is the source code line (from the __LINE__ macro) that -** generated the VDBE instruction. This instrumentation assumes that all -** source code is in a single file (the amalgamation). Special values 1 -** and 2 for the iSrcLine parameter mean that this particular branch is -** always taken or never taken, respectively. +** generated the VDBE instruction combined with flag bits. The source +** code line number is in the lower 24 bits of iSrcLine and the upper +** 8 bytes are flags. The lower three bits of the flags indicate +** values for I that should never occur. For example, if the branch is +** always taken, the flags should be 0x05 since the fall-through and +** alternate branch are never taken. If a branch is never taken then +** flags should be 0x06 since only the fall-through approach is allowed. +** +** Bit 0x08 of the flags indicates an OP_Jump opcode that is only +** interested in equal or not-equal. In other words, I==0 and I==2 +** should be treated as equivalent +** +** Since only a line number is retained, not the filename, this macro +** only works for amalgamation builds. But that is ok, since these macros +** should be no-ops except for special builds used to measure test coverage. */ #if !defined(SQLITE_VDBE_COVERAGE) # define VdbeBranchTaken(I,M) #else # define VdbeBranchTaken(I,M) vdbeTakeBranch(pOp->iSrcLine,I,M) - static void vdbeTakeBranch(int iSrcLine, u8 I, u8 M){ - if( iSrcLine<=2 && ALWAYS(iSrcLine>0) ){ - M = iSrcLine; - /* Assert the truth of VdbeCoverageAlwaysTaken() and - ** VdbeCoverageNeverTaken() */ - assert( (M & I)==I ); - }else{ - if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ - sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, - iSrcLine,I,M); + static void vdbeTakeBranch(u32 iSrcLine, u8 I, u8 M){ + u8 mNever; + assert( I<=2 ); /* 0: fall through, 1: taken, 2: alternate taken */ + assert( M<=4 ); /* 2: two-way branch, 3: three-way branch, 4: OP_Jump */ + assert( I> 24; + assert( (I & mNever)==0 ); + if( sqlite3GlobalConfig.xVdbeBranch==0 ) return; /*NO_TEST*/ + /* Invoke the branch coverage callback with three arguments: + ** iSrcLine - the line number of the VdbeCoverage() macro, with + ** flags removed. + ** I - Mask of bits 0x07 indicating which cases are are + ** fulfilled by this instance of the jump. 0x01 means + ** fall-thru, 0x02 means taken, 0x04 means NULL. Any + ** impossible cases (ex: if the comparison is never NULL) + ** are filled in automatically so that the coverage + ** measurement logic does not flag those impossible cases + ** as missed coverage. + ** M - Type of jump. Same as M argument above + */ + I |= mNever; + if( M==2 ) I |= 0x04; + if( M==4 ){ + I |= 0x08; + if( (mNever&0x08)!=0 && (I&0x05)!=0) I |= 0x05; /*NO_TEST*/ } + sqlite3GlobalConfig.xVdbeBranch(sqlite3GlobalConfig.pVdbeBranchArg, + iSrcLine&0xffffff, I, M); } #endif -/* -** Convert the given register into a string if it isn't one -** already. Return non-zero if a malloc() fails. -*/ -#define Stringify(P, enc) \ - if(((P)->flags&(MEM_Str|MEM_Blob))==0 && sqlite3VdbeMemStringify(P,enc,0)) \ - { goto no_mem; } - /* ** An ephemeral string value (signified by the MEM_Ephem flag) contains ** a pointer to a dynamically allocated string where some other entity @@ -76771,11 +93650,10 @@ static VdbeCursor *allocateCursor( Vdbe *p, /* The virtual machine */ int iCur, /* Index of the new VdbeCursor */ int nField, /* Number of fields in the table or index */ - int iDb, /* Database the cursor belongs to, or -1 */ u8 eCurType /* Type of the new cursor */ ){ /* Find the memory cell that will be used to store the blob of memory - ** required for this VdbeCursor structure. It is convenient to use a + ** required for this VdbeCursor structure. It is convenient to use a ** vdbe memory cell to manage the memory allocation required for a ** VdbeCursor structure for the following reasons: ** @@ -76796,31 +93674,64 @@ static VdbeCursor *allocateCursor( int nByte; VdbeCursor *pCx = 0; - nByte = - ROUND8(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + + nByte = + ROUND8P(sizeof(VdbeCursor)) + 2*sizeof(u32)*nField + (eCurType==CURTYPE_BTREE?sqlite3BtreeCursorSize():0); assert( iCur>=0 && iCurnCursor ); if( p->apCsr[iCur] ){ /*OPTIMIZATION-IF-FALSE*/ - sqlite3VdbeFreeCursor(p, p->apCsr[iCur]); + sqlite3VdbeFreeCursorNN(p, p->apCsr[iCur]); p->apCsr[iCur] = 0; } - if( SQLITE_OK==sqlite3VdbeMemClearAndResize(pMem, nByte) ){ - p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->z; - memset(pCx, 0, sizeof(VdbeCursor)); - pCx->eCurType = eCurType; - pCx->iDb = iDb; - pCx->nField = nField; - pCx->aOffset = &pCx->aType[nField]; - if( eCurType==CURTYPE_BTREE ){ - pCx->uc.pCursor = (BtCursor*) - &pMem->z[ROUND8(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; - sqlite3BtreeCursorZero(pCx->uc.pCursor); + + /* There used to be a call to sqlite3VdbeMemClearAndResize() to make sure + ** the pMem used to hold space for the cursor has enough storage available + ** in pMem->zMalloc. But for the special case of the aMem[] entries used + ** to hold cursors, it is faster to in-line the logic. */ + assert( pMem->flags==MEM_Undefined ); + assert( (pMem->flags & MEM_Dyn)==0 ); + assert( pMem->szMalloc==0 || pMem->z==pMem->zMalloc ); + if( pMem->szMallocszMalloc>0 ){ + sqlite3DbFreeNN(pMem->db, pMem->zMalloc); } + pMem->z = pMem->zMalloc = sqlite3DbMallocRaw(pMem->db, nByte); + if( pMem->zMalloc==0 ){ + pMem->szMalloc = 0; + return 0; + } + pMem->szMalloc = nByte; + } + + p->apCsr[iCur] = pCx = (VdbeCursor*)pMem->zMalloc; + memset(pCx, 0, offsetof(VdbeCursor,pAltCursor)); + pCx->eCurType = eCurType; + pCx->nField = nField; + pCx->aOffset = &pCx->aType[nField]; + if( eCurType==CURTYPE_BTREE ){ + pCx->uc.pCursor = (BtCursor*) + &pMem->z[ROUND8P(sizeof(VdbeCursor))+2*sizeof(u32)*nField]; + sqlite3BtreeCursorZero(pCx->uc.pCursor); } return pCx; } +/* +** The string in pRec is known to look like an integer and to have a +** floating point value of rValue. Return true and set *piValue to the +** integer value if the string is in range to be an integer. Otherwise, +** return false. +*/ +static int alsoAnInt(Mem *pRec, double rValue, i64 *piValue){ + i64 iValue; + iValue = sqlite3RealToI64(rValue); + if( sqlite3RealSameAsInt(rValue,iValue) ){ + *piValue = iValue; + return 1; + } + return 0==sqlite3Atoi64(pRec->z, piValue, pRec->n, pRec->enc); +} + /* ** Try to convert a value into a numeric representation if we can ** do so without loss of information. In other words, if the string @@ -76838,18 +93749,23 @@ static VdbeCursor *allocateCursor( */ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ double rValue; - i64 iValue; u8 enc = pRec->enc; - assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real))==MEM_Str ); - if( sqlite3AtoF(pRec->z, &rValue, pRec->n, enc)==0 ) return; - if( 0==sqlite3Atoi64(pRec->z, &iValue, pRec->n, enc) ){ - pRec->u.i = iValue; + int rc; + assert( (pRec->flags & (MEM_Str|MEM_Int|MEM_Real|MEM_IntReal))==MEM_Str ); + rc = sqlite3AtoF(pRec->z, &rValue, pRec->n, enc); + if( rc<=0 ) return; + if( rc==1 && alsoAnInt(pRec, rValue, &pRec->u.i) ){ pRec->flags |= MEM_Int; }else{ pRec->u.r = rValue; pRec->flags |= MEM_Real; if( bTryForInt ) sqlite3VdbeIntegerAffinity(pRec); } + /* TEXT->NUMERIC is many->one. Hence, it is important to invalidate the + ** string representation after computing a numeric equivalent, because the + ** string representation might not be the canonical representation for the + ** numeric value. Ticket [343634942dd54ab57b7024] 2018-01-31. */ + pRec->flags &= ~MEM_Str; } /* @@ -76858,16 +93774,21 @@ static void applyNumericAffinity(Mem *pRec, int bTryForInt){ ** SQLITE_AFF_INTEGER: ** SQLITE_AFF_REAL: ** SQLITE_AFF_NUMERIC: -** Try to convert pRec to an integer representation or a +** Try to convert pRec to an integer representation or a ** floating-point representation if an integer representation ** is not possible. Note that the integer representation is ** always preferred, even if the affinity is REAL, because ** an integer representation is more space efficient on disk. ** +** SQLITE_AFF_FLEXNUM: +** If the value is text, then try to convert it into a number of +** some kind (integer or real) but do not make any other changes. +** ** SQLITE_AFF_TEXT: ** Convert pRec to a text representation. ** ** SQLITE_AFF_BLOB: +** SQLITE_AFF_NONE: ** No-op. pRec is unchanged. */ static void applyAffinity( @@ -76877,26 +93798,29 @@ static void applyAffinity( ){ if( affinity>=SQLITE_AFF_NUMERIC ){ assert( affinity==SQLITE_AFF_INTEGER || affinity==SQLITE_AFF_REAL - || affinity==SQLITE_AFF_NUMERIC ); + || affinity==SQLITE_AFF_NUMERIC || affinity==SQLITE_AFF_FLEXNUM ); if( (pRec->flags & MEM_Int)==0 ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags & MEM_Real)==0 ){ + if( (pRec->flags & (MEM_Real|MEM_IntReal))==0 ){ if( pRec->flags & MEM_Str ) applyNumericAffinity(pRec,1); - }else{ + }else if( affinity<=SQLITE_AFF_REAL ){ sqlite3VdbeIntegerAffinity(pRec); } } }else if( affinity==SQLITE_AFF_TEXT ){ /* Only attempt the conversion to TEXT if there is an integer or real ** representation (blob and NULL do not get converted) but no string - ** representation. It would be harmless to repeat the conversion if + ** representation. It would be harmless to repeat the conversion if ** there is already a string rep, but it is pointless to waste those ** CPU cycles. */ if( 0==(pRec->flags&MEM_Str) ){ /*OPTIMIZATION-IF-FALSE*/ - if( (pRec->flags&(MEM_Real|MEM_Int)) ){ + if( (pRec->flags&(MEM_Real|MEM_Int|MEM_IntReal)) ){ + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_Real ); + testcase( pRec->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pRec, enc, 1); } } - pRec->flags &= ~(MEM_Real|MEM_Int); + pRec->flags &= ~(MEM_Real|MEM_Int|MEM_IntReal); } } @@ -76906,7 +93830,7 @@ static void applyAffinity( ** is appropriate. But only do the conversion if it is possible without ** loss of information and return the revised type of the argument. */ -SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value *pVal){ +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value *pVal){ int eType = sqlite3_value_type(pVal); if( eType==SQLITE_TEXT ){ Mem *pMem = (Mem*)pVal; @@ -76917,12 +93841,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_value_numeric_type(sqlite3_value *pVal){ } /* -** Exported version of applyAffinity(). This one works on sqlite3_value*, +** Exported version of applyAffinity(). This one works on sqlite3_value*, ** not the internal Mem* type. */ SQLITE_PRIVATE void sqlite3ValueApplyAffinity( - sqlite3_value *pVal, - u8 affinity, + sqlite3_value *pVal, + u8 affinity, u8 enc ){ applyAffinity((Mem *)pVal, affinity, enc); @@ -76935,12 +93859,24 @@ SQLITE_PRIVATE void sqlite3ValueApplyAffinity( ** accordingly. */ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ - assert( (pMem->flags & (MEM_Int|MEM_Real))==0 ); + int rc; + sqlite3_int64 ix; + assert( (pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal))==0 ); assert( (pMem->flags & (MEM_Str|MEM_Blob))!=0 ); - if( sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc)==0 ){ - return 0; + if( ExpandBlob(pMem) ){ + pMem->u.i = 0; + return MEM_Int; } - if( sqlite3Atoi64(pMem->z, &pMem->u.i, pMem->n, pMem->enc)==SQLITE_OK ){ + rc = sqlite3AtoF(pMem->z, &pMem->u.r, pMem->n, pMem->enc); + if( rc<=0 ){ + if( rc==0 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)<=1 ){ + pMem->u.i = ix; + return MEM_Int; + }else{ + return MEM_Real; + } + }else if( rc==1 && sqlite3Atoi64(pMem->z, &ix, pMem->n, pMem->enc)==0 ){ + pMem->u.i = ix; return MEM_Int; } return MEM_Real; @@ -76948,18 +93884,24 @@ static u16 SQLITE_NOINLINE computeNumericType(Mem *pMem){ /* ** Return the numeric type for pMem, either MEM_Int or MEM_Real or both or -** none. +** none. ** ** Unlike applyNumericAffinity(), this routine does not modify pMem->flags. ** But it does set pMem->u.r and pMem->u.i appropriately. */ static u16 numericType(Mem *pMem){ - if( pMem->flags & (MEM_Int|MEM_Real) ){ - return pMem->flags & (MEM_Int|MEM_Real); - } - if( pMem->flags & (MEM_Str|MEM_Blob) ){ - return computeNumericType(pMem); + assert( (pMem->flags & MEM_Null)==0 + || pMem->db==0 || pMem->db->mallocFailed ); + if( pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null) ){ + testcase( pMem->flags & MEM_Int ); + testcase( pMem->flags & MEM_Real ); + testcase( pMem->flags & MEM_IntReal ); + return pMem->flags & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Null); } + assert( pMem->flags & (MEM_Str|MEM_Blob) ); + testcase( pMem->flags & MEM_Str ); + testcase( pMem->flags & MEM_Blob ); + return computeNumericType(pMem); return 0; } @@ -76968,12 +93910,9 @@ static u16 numericType(Mem *pMem){ ** Write a nice string representation of the contents of cell pMem ** into buffer zBuf, length nBuf. */ -SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ - char *zCsr = zBuf; +SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, StrAccum *pStr){ int f = pMem->flags; - static const char *const encnames[] = {"(X)", "(8)", "(16LE)", "(16BE)"}; - if( f&MEM_Blob ){ int i; char c; @@ -76989,59 +93928,43 @@ SQLITE_PRIVATE void sqlite3VdbeMemPrettyPrint(Mem *pMem, char *zBuf){ }else{ c = 's'; } - - sqlite3_snprintf(100, zCsr, "%c", c); - zCsr += sqlite3Strlen30(zCsr); - sqlite3_snprintf(100, zCsr, "%d[", pMem->n); - zCsr += sqlite3Strlen30(zCsr); - for(i=0; i<16 && in; i++){ - sqlite3_snprintf(100, zCsr, "%02X", ((int)pMem->z[i] & 0xFF)); - zCsr += sqlite3Strlen30(zCsr); + sqlite3_str_appendf(pStr, "%cx[", c); + for(i=0; i<25 && in; i++){ + sqlite3_str_appendf(pStr, "%02X", ((int)pMem->z[i] & 0xFF)); } - for(i=0; i<16 && in; i++){ + sqlite3_str_appendf(pStr, "|"); + for(i=0; i<25 && in; i++){ char z = pMem->z[i]; - if( z<32 || z>126 ) *zCsr++ = '.'; - else *zCsr++ = z; + sqlite3_str_appendchar(pStr, 1, (z<32||z>126)?'.':z); } - - sqlite3_snprintf(100, zCsr, "]%s", encnames[pMem->enc]); - zCsr += sqlite3Strlen30(zCsr); + sqlite3_str_appendf(pStr,"]"); if( f & MEM_Zero ){ - sqlite3_snprintf(100, zCsr,"+%dz",pMem->u.nZero); - zCsr += sqlite3Strlen30(zCsr); + sqlite3_str_appendf(pStr, "+%dz",pMem->u.nZero); } - *zCsr = '\0'; }else if( f & MEM_Str ){ - int j, k; - zBuf[0] = ' '; + int j; + u8 c; if( f & MEM_Dyn ){ - zBuf[1] = 'z'; + c = 'z'; assert( (f & (MEM_Static|MEM_Ephem))==0 ); }else if( f & MEM_Static ){ - zBuf[1] = 't'; + c = 't'; assert( (f & (MEM_Dyn|MEM_Ephem))==0 ); }else if( f & MEM_Ephem ){ - zBuf[1] = 'e'; + c = 'e'; assert( (f & (MEM_Static|MEM_Dyn))==0 ); }else{ - zBuf[1] = 's'; + c = 's'; } - k = 2; - sqlite3_snprintf(100, &zBuf[k], "%d", pMem->n); - k += sqlite3Strlen30(&zBuf[k]); - zBuf[k++] = '['; - for(j=0; j<15 && jn; j++){ - u8 c = pMem->z[j]; - if( c>=0x20 && c<0x7f ){ - zBuf[k++] = c; - }else{ - zBuf[k++] = '.'; - } + sqlite3_str_appendf(pStr, " %c%d[", c, pMem->n); + for(j=0; j<25 && jn; j++){ + c = pMem->z[j]; + sqlite3_str_appendchar(pStr, 1, (c>=0x20&&c<=0x7f) ? c : '.'); + } + sqlite3_str_appendf(pStr, "]%s", encnames[pMem->enc]); + if( f & MEM_Term ){ + sqlite3_str_appendf(pStr, "(0-term)"); } - zBuf[k++] = ']'; - sqlite3_snprintf(100,&zBuf[k], encnames[pMem->enc]); - k += sqlite3Strlen30(&zBuf[k]); - zBuf[k++] = 0; } } #endif @@ -77054,144 +93977,69 @@ static void memTracePrint(Mem *p){ if( p->flags & MEM_Undefined ){ printf(" undefined"); }else if( p->flags & MEM_Null ){ - printf(" NULL"); + printf(p->flags & MEM_Zero ? " NULL-nochng" : " NULL"); }else if( (p->flags & (MEM_Int|MEM_Str))==(MEM_Int|MEM_Str) ){ printf(" si:%lld", p->u.i); + }else if( (p->flags & (MEM_IntReal))!=0 ){ + printf(" ir:%lld", p->u.i); }else if( p->flags & MEM_Int ){ printf(" i:%lld", p->u.i); #ifndef SQLITE_OMIT_FLOATING_POINT }else if( p->flags & MEM_Real ){ - printf(" r:%g", p->u.r); + printf(" r:%.17g", p->u.r); #endif - }else if( p->flags & MEM_RowSet ){ + }else if( sqlite3VdbeMemIsRowSet(p) ){ printf(" (rowset)"); }else{ - char zBuf[200]; - sqlite3VdbeMemPrettyPrint(p, zBuf); - printf(" %s", zBuf); + StrAccum acc; + char zBuf[1000]; + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3VdbeMemPrettyPrint(p, &acc); + printf(" %s", sqlite3StrAccumFinish(&acc)); } if( p->flags & MEM_Subtype ) printf(" subtype=0x%02x", p->eSubtype); } static void registerTrace(int iReg, Mem *p){ - printf("REG[%d] = ", iReg); + printf("R[%d] = ", iReg); memTracePrint(p); + if( p->pScopyFrom ){ + printf(" <== R[%d]", (int)(p->pScopyFrom - &p[-iReg])); + } printf("\n"); + sqlite3VdbeCheckMemInvariants(p); +} +/**/ void sqlite3PrintMem(Mem *pMem){ + memTracePrint(pMem); + printf("\n"); + fflush(stdout); } #endif +#ifdef SQLITE_DEBUG +/* +** Show the values of all registers in the virtual machine. Used for +** interactive debugging. +*/ +SQLITE_PRIVATE void sqlite3VdbeRegisterDump(Vdbe *v){ + int i; + for(i=1; inMem; i++) registerTrace(i, v->aMem+i); +} +#endif /* SQLITE_DEBUG */ + + #ifdef SQLITE_DEBUG # define REGISTER_TRACE(R,M) if(db->flags&SQLITE_VdbeTrace)registerTrace(R,M) #else # define REGISTER_TRACE(R,M) #endif - -#ifdef VDBE_PROFILE - -/* -** hwtime.h contains inline assembler code for implementing -** high-performance timing routines. -*/ -/************** Include hwtime.h in the middle of vdbe.c *********************/ -/************** Begin file hwtime.h ******************************************/ -/* -** 2008 May 27 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This file contains inline asm code for retrieving "high-performance" -** counters for x86 class CPUs. -*/ -#ifndef _HWTIME_H_ -#define _HWTIME_H_ - -/* -** The following routine only works on pentium-class (or newer) processors. -** It uses the RDTSC opcode to read the cycle count value out of the -** processor and returns that value. This can be used for high-res -** profiling. -*/ -#if (defined(__GNUC__) || defined(_MSC_VER)) && \ - (defined(i386) || defined(__i386__) || defined(_M_IX86)) - - #if defined(__GNUC__) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned int lo, hi; - __asm__ __volatile__ ("rdtsc" : "=a" (lo), "=d" (hi)); - return (sqlite_uint64)hi << 32 | lo; - } - - #elif defined(_MSC_VER) - - __declspec(naked) __inline sqlite_uint64 __cdecl sqlite3Hwtime(void){ - __asm { - rdtsc - ret ; return value at EDX:EAX - } - } - - #endif - -#elif (defined(__GNUC__) && defined(__x86_64__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long val; - __asm__ __volatile__ ("rdtsc" : "=A" (val)); - return val; - } - -#elif (defined(__GNUC__) && defined(__ppc__)) - - __inline__ sqlite_uint64 sqlite3Hwtime(void){ - unsigned long long retval; - unsigned long junk; - __asm__ __volatile__ ("\n\ - 1: mftbu %1\n\ - mftb %L0\n\ - mftbu %0\n\ - cmpw %0,%1\n\ - bne 1b" - : "=r" (retval), "=r" (junk)); - return retval; - } - -#else - - #error Need implementation of sqlite3Hwtime() for your platform. - - /* - ** To compile without implementing sqlite3Hwtime() for your platform, - ** you can remove the above #error and use the following - ** stub function. You will lose timing support for many - ** of the debugging and testing utilities, but it should at - ** least compile and run. - */ -SQLITE_PRIVATE sqlite_uint64 sqlite3Hwtime(void){ return ((sqlite_uint64)0); } - -#endif - -#endif /* !defined(_HWTIME_H_) */ - -/************** End of hwtime.h **********************************************/ -/************** Continuing where we left off in vdbe.c ***********************/ - -#endif - #ifndef NDEBUG /* ** This function is only called from within an assert() expression. It ** checks that the sqlite3.nTransaction variable is correctly set to -** the number of non-transaction savepoints currently in the +** the number of non-transaction savepoints currently in the ** linked list starting at sqlite3.pSavepoint. -** +** ** Usage: ** ** assert( checkSavepointCount(db) ); @@ -77228,66 +94076,194 @@ static Mem *out2Prerelease(Vdbe *p, VdbeOp *pOp){ } } +/* +** Compute a bloom filter hash using pOp->p4.i registers from aMem[] beginning +** with pOp->p3. Return the hash. +*/ +static u64 filterHash(const Mem *aMem, const Op *pOp){ + int i, mx; + u64 h = 0; + + assert( pOp->p4type==P4_INT32 ); + for(i=pOp->p3, mx=i+pOp->p4.i; iflags & (MEM_Int|MEM_IntReal) ){ + h += p->u.i; + }else if( p->flags & MEM_Real ){ + h += sqlite3VdbeIntValue(p); + }else if( p->flags & (MEM_Str|MEM_Blob) ){ + /* All strings have the same hash and all blobs have the same hash, + ** though, at least, those hashes are different from each other and + ** from NULL. */ + h += 4093 + (p->flags & (MEM_Str|MEM_Blob)); + } + } + return h; +} + + +/* +** For OP_Column, factor out the case where content is loaded from +** overflow pages, so that the code to implement this case is separate +** the common case where all content fits on the page. Factoring out +** the code reduces register pressure and helps the common case +** to run faster. +*/ +static SQLITE_NOINLINE int vdbeColumnFromOverflow( + VdbeCursor *pC, /* The BTree cursor from which we are reading */ + int iCol, /* The column to read */ + int t, /* The serial-type code for the column value */ + i64 iOffset, /* Offset to the start of the content value */ + u32 cacheStatus, /* Current Vdbe.cacheCtr value */ + u32 colCacheCtr, /* Current value of the column cache counter */ + Mem *pDest /* Store the value into this register. */ +){ + int rc; + sqlite3 *db = pDest->db; + int encoding = pDest->enc; + int len = sqlite3VdbeSerialTypeLen(t); + assert( pC->eCurType==CURTYPE_BTREE ); + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) return SQLITE_TOOBIG; + if( len > 4000 && pC->pKeyInfo==0 ){ + /* Cache large column values that are on overflow pages using + ** an RCStr (reference counted string) so that if they are reloaded, + ** that do not have to be copied a second time. The overhead of + ** creating and managing the cache is such that this is only + ** profitable for larger TEXT and BLOB values. + ** + ** Only do this on table-btrees so that writes to index-btrees do not + ** need to clear the cache. This buys performance in the common case + ** in exchange for generality. + */ + VdbeTxtBlbCache *pCache; + char *pBuf; + if( pC->colCache==0 ){ + pC->pCache = sqlite3DbMallocZero(db, sizeof(VdbeTxtBlbCache) ); + if( pC->pCache==0 ) return SQLITE_NOMEM; + pC->colCache = 1; + } + pCache = pC->pCache; + if( pCache->pCValue==0 + || pCache->iCol!=iCol + || pCache->cacheStatus!=cacheStatus + || pCache->colCacheCtr!=colCacheCtr + || pCache->iOffset!=sqlite3BtreeOffset(pC->uc.pCursor) + ){ + if( pCache->pCValue ) sqlite3RCStrUnref(pCache->pCValue); + pBuf = pCache->pCValue = sqlite3RCStrNew( len+3 ); + if( pBuf==0 ) return SQLITE_NOMEM; + rc = sqlite3BtreePayload(pC->uc.pCursor, iOffset, len, pBuf); + if( rc ) return rc; + pBuf[len] = 0; + pBuf[len+1] = 0; + pBuf[len+2] = 0; + pCache->iCol = iCol; + pCache->cacheStatus = cacheStatus; + pCache->colCacheCtr = colCacheCtr; + pCache->iOffset = sqlite3BtreeOffset(pC->uc.pCursor); + }else{ + pBuf = pCache->pCValue; + } + assert( t>=12 ); + sqlite3RCStrRef(pBuf); + if( t&1 ){ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, encoding, + sqlite3RCStrUnref); + pDest->flags |= MEM_Term; + }else{ + rc = sqlite3VdbeMemSetStr(pDest, pBuf, len, 0, + sqlite3RCStrUnref); + } + }else{ + rc = sqlite3VdbeMemFromBtree(pC->uc.pCursor, iOffset, len, pDest); + if( rc ) return rc; + sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); + if( (t&1)!=0 && encoding==SQLITE_UTF8 ){ + pDest->z[len] = 0; + pDest->flags |= MEM_Term; + } + } + pDest->flags &= ~MEM_Ephem; + return rc; +} + + +/* +** Return the symbolic name for the data type of a pMem +*/ +static const char *vdbeMemTypeName(Mem *pMem){ + static const char *azTypes[] = { + /* SQLITE_INTEGER */ "INT", + /* SQLITE_FLOAT */ "REAL", + /* SQLITE_TEXT */ "TEXT", + /* SQLITE_BLOB */ "BLOB", + /* SQLITE_NULL */ "NULL" + }; + return azTypes[sqlite3_value_type(pMem)-1]; +} /* ** Execute as much of a VDBE program as we can. -** This is the core of sqlite3_step(). +** This is the core of sqlite3_step(). */ SQLITE_PRIVATE int sqlite3VdbeExec( Vdbe *p /* The VDBE */ ){ Op *aOp = p->aOp; /* Copy of p->aOp */ Op *pOp = aOp; /* Current operation */ -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) - Op *pOrigOp; /* Value of pOp at the top of the loop */ -#endif #ifdef SQLITE_DEBUG + Op *pOrigOp; /* Value of pOp at the top of the loop */ int nExtraDelete = 0; /* Verifies FORDELETE and AUXDELETE flags */ + u8 iCompareIsInit = 0; /* iCompare is initialized */ #endif int rc = SQLITE_OK; /* Value to return */ sqlite3 *db = p->db; /* The database */ u8 resetSchemaOnFault = 0; /* Reset schema after an error if positive */ u8 encoding = ENC(db); /* The database encoding */ - int iCompare = 0; /* Result of last OP_Compare operation */ - unsigned nVmStep = 0; /* Number of virtual machine steps */ + int iCompare = 0; /* Result of last comparison */ + u64 nVmStep = 0; /* Number of virtual machine steps */ #ifndef SQLITE_OMIT_PROGRESS_CALLBACK - unsigned nProgressLimit = 0;/* Invoke xProgress() when nVmStep reaches this */ + u64 nProgressLimit; /* Invoke xProgress() when nVmStep reaches this */ #endif Mem *aMem = p->aMem; /* Copy of p->aMem */ Mem *pIn1 = 0; /* 1st input operand */ Mem *pIn2 = 0; /* 2nd input operand */ Mem *pIn3 = 0; /* 3rd input operand */ Mem *pOut = 0; /* Output operand */ - int *aPermute = 0; /* Permutation of columns for OP_Compare */ - i64 lastRowid = db->lastRowid; /* Saved value of the last insert ROWID */ -#ifdef VDBE_PROFILE - u64 start; /* CPU clock count at start of opcode */ + u32 colCacheCtr = 0; /* Column cache counter */ +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) || defined(VDBE_PROFILE) + u64 *pnCycle = 0; + int bStmtScanStatus = IS_STMT_SCANSTATUS(db)!=0; #endif /*** INSERT STACK UNION HERE ***/ - assert( p->magic==VDBE_MAGIC_RUN ); /* sqlite3_step() verifies this */ - sqlite3VdbeEnter(p); + assert( p->eVdbeState==VDBE_RUN_STATE ); /* sqlite3_step() verifies this */ + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeEnter(p); + } +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + if( db->xProgress ){ + u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; + assert( 0 < db->nProgressOps ); + nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); + }else{ + nProgressLimit = LARGEST_UINT64; + } +#endif if( p->rc==SQLITE_NOMEM ){ /* This happens if a malloc() inside a call to sqlite3_column_text() or ** sqlite3_column_text16() failed. */ goto no_mem; } assert( p->rc==SQLITE_OK || (p->rc&0xff)==SQLITE_BUSY ); - assert( p->bIsReader || p->readOnly!=0 ); + testcase( p->rc!=SQLITE_OK ); p->rc = SQLITE_OK; + assert( p->bIsReader || p->readOnly!=0 ); p->iCurrentTime = 0; assert( p->explain==0 ); - p->pResultSet = 0; db->busyHandler.nBusy = 0; - if( db->u1.isInterrupted ) goto abort_due_to_interrupt; + if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; sqlite3VdbeIOTraceSql(p); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - if( db->xProgress ){ - u32 iPrior = p->aCounter[SQLITE_STMTSTATUS_VM_STEP]; - assert( 0 < db->nProgressOps ); - nProgressLimit = db->nProgressOps - (iPrior % db->nProgressOps); - } -#endif #ifdef SQLITE_DEBUG sqlite3BeginBenignMalloc(); if( p->pc==0 @@ -77321,12 +94297,18 @@ SQLITE_PRIVATE int sqlite3VdbeExec( assert( rc==SQLITE_OK ); assert( pOp>=aOp && pOp<&aOp[p->nOp]); -#ifdef VDBE_PROFILE - start = sqlite3Hwtime(); -#endif nVmStep++; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - if( p->anExec ) p->anExec[(int)(pOp-aOp)]++; + +#if defined(VDBE_PROFILE) + pOp->nExec++; + pnCycle = &pOp->nCycle; + if( sqlite3NProfileCnt==0 ) *pnCycle -= sqlite3Hwtime(); +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( bStmtScanStatus ){ + pOp->nExec++; + pnCycle = &pOp->nCycle; + *pnCycle -= sqlite3Hwtime(); + } #endif /* Only allow tracing if SQLITE_DEBUG is defined. @@ -77334,9 +94316,10 @@ SQLITE_PRIVATE int sqlite3VdbeExec( #ifdef SQLITE_DEBUG if( db->flags & SQLITE_VdbeTrace ){ sqlite3VdbePrintOp(stdout, (int)(pOp - aOp), pOp); + test_trace_breakpoint((int)(pOp - aOp),pOp,p); } #endif - + /* Check to see if we need to simulate an interrupt. This only happens ** if we have a special test build. @@ -77387,10 +94370,10 @@ SQLITE_PRIVATE int sqlite3VdbeExec( } } #endif -#if defined(SQLITE_DEBUG) || defined(VDBE_PROFILE) +#ifdef SQLITE_DEBUG pOrigOp = pOp; #endif - + switch( pOp->opcode ){ /***************************************************************************** @@ -77431,7 +94414,7 @@ SQLITE_PRIVATE int sqlite3VdbeExec( /* Opcode: Goto * P2 * * * ** ** An unconditional jump to address P2. -** The next instruction executed will be +** The next instruction executed will be ** the one at index P2 from the beginning of ** the program. ** @@ -77441,13 +94424,27 @@ SQLITE_PRIVATE int sqlite3VdbeExec( ** to the current line should be indented for EXPLAIN output. */ case OP_Goto: { /* jump */ + +#ifdef SQLITE_DEBUG + /* In debugging mode, when the p5 flags is set on an OP_Goto, that + ** means we should really jump back to the preceding OP_ReleaseReg + ** instruction. */ + if( pOp->p5 ){ + assert( pOp->p2 < (int)(pOp - aOp) ); + assert( pOp->p2 > 1 ); + pOp = &aOp[pOp->p2 - 2]; + assert( pOp[1].opcode==OP_ReleaseReg ); + goto check_for_interrupt; + } +#endif + jump_to_p2_and_check_for_interrupt: pOp = &aOp[pOp->p2 - 1]; /* Opcodes that are used as the bottom of a loop (OP_Next, OP_Prev, - ** OP_VNext, OP_RowSetNext, or OP_SorterNext) all jump here upon + ** OP_VNext, or OP_SorterNext) all jump here upon ** completion. Check to see if sqlite3_interrupt() has been called - ** or if the progress callback needs to be invoked. + ** or if the progress callback needs to be invoked. ** ** This code uses unstructured "goto" statements and does not look clean. ** But that is not due to sloppy coding habits. The code is written this @@ -77455,7 +94452,7 @@ jump_to_p2_and_check_for_interrupt: ** checks on every opcode. This helps sqlite3_step() to run about 1.5% ** faster according to "valgrind --tool=cachegrind" */ check_for_interrupt: - if( db->u1.isInterrupted ) goto abort_due_to_interrupt; + if( AtomicLoad(&db->u1.isInterrupted) ) goto abort_due_to_interrupt; #ifndef SQLITE_OMIT_PROGRESS_CALLBACK /* Call the progress callback if it is configured and the required number ** of VDBE ops have been executed (either since this invocation of @@ -77463,16 +94460,17 @@ check_for_interrupt: ** If the progress callback returns non-zero, exit the virtual machine with ** a return code SQLITE_ABORT. */ - if( db->xProgress!=0 && nVmStep>=nProgressLimit ){ + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ assert( db->nProgressOps!=0 ); - nProgressLimit = nVmStep + db->nProgressOps - (nVmStep%db->nProgressOps); + nProgressLimit += db->nProgressOps; if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = LARGEST_UINT64; rc = SQLITE_INTERRUPT; goto abort_due_to_error; } } #endif - + break; } @@ -77489,24 +94487,39 @@ case OP_Gosub: { /* jump */ pIn1->flags = MEM_Int; pIn1->u.i = (int)(pOp-aOp); REGISTER_TRACE(pOp->p1, pIn1); - - /* Most jump operations do a goto to this spot in order to update - ** the pOp pointer. */ -jump_to_p2: - pOp = &aOp[pOp->p2 - 1]; - break; + goto jump_to_p2_and_check_for_interrupt; } -/* Opcode: Return P1 * * * * +/* Opcode: Return P1 P2 P3 * * ** -** Jump to the next instruction after the address in register P1. After -** the jump, register P1 becomes undefined. +** Jump to the address stored in register P1. If P1 is a return address +** register, then this accomplishes a return from a subroutine. +** +** If P3 is 1, then the jump is only taken if register P1 holds an integer +** values, otherwise execution falls through to the next opcode, and the +** OP_Return becomes a no-op. If P3 is 0, then register P1 must hold an +** integer or else an assert() is raised. P3 should be set to 1 when +** this opcode is used in combination with OP_BeginSubrtn, and set to 0 +** otherwise. +** +** The value in register P1 is unchanged by this opcode. +** +** P2 is not used by the byte-code engine. However, if P2 is positive +** and also less than the current address, then the "EXPLAIN" output +** formatter in the CLI will indent all opcodes from the P2 opcode up +** to be not including the current Return. P2 should be the first opcode +** in the subroutine from which this opcode is returning. Thus the P2 +** value is a byte-code indentation hint. See tag-20220407a in +** wherecode.c and shell.c. */ case OP_Return: { /* in1 */ pIn1 = &aMem[pOp->p1]; - assert( pIn1->flags==MEM_Int ); - pOp = &aOp[pIn1->u.i]; - pIn1->flags = MEM_Undefined; + if( pIn1->flags & MEM_Int ){ + if( pOp->p3 ){ VdbeBranchTaken(1, 2); } + pOp = &aOp[pIn1->u.i]; + }else if( ALWAYS(pOp->p3) ){ + VdbeBranchTaken(0, 2); + } break; } @@ -77521,7 +94534,7 @@ case OP_Return: { /* in1 */ ** ** See also: EndCoroutine */ -case OP_InitCoroutine: { /* jump */ +case OP_InitCoroutine: { /* jump0 */ assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); assert( pOp->p2>=0 && pOp->p2nOp ); assert( pOp->p3>=0 && pOp->p3nOp ); @@ -77529,7 +94542,14 @@ case OP_InitCoroutine: { /* jump */ assert( !VdbeMemDynamic(pOut) ); pOut->u.i = pOp->p3 - 1; pOut->flags = MEM_Int; - if( pOp->p2 ) goto jump_to_p2; + if( pOp->p2==0 ) break; + + /* Most jump operations do a goto to this spot in order to update + ** the pOp pointer. */ +jump_to_p2: + assert( pOp->p2>0 ); /* There are never any jumps to instruction 0 */ + assert( pOp->p2nOp ); /* Jumps must be in range */ + pOp = &aOp[pOp->p2 - 1]; break; } @@ -77537,7 +94557,9 @@ case OP_InitCoroutine: { /* jump */ ** ** The instruction at the address in register P1 is a Yield. ** Jump to the P2 parameter of that Yield. -** After the jump, register P1 becomes undefined. +** After the jump, the value register P1 is left with a value +** such that subsequent OP_Yields go back to the this same +** OP_EndCoroutine instruction. ** ** See also: InitCoroutine */ @@ -77549,8 +94571,8 @@ case OP_EndCoroutine: { /* in1 */ pCaller = &aOp[pIn1->u.i]; assert( pCaller->opcode==OP_Yield ); assert( pCaller->p2>=0 && pCaller->p2nOp ); + pIn1->u.i = (int)(pOp - p->aOp) - 1; pOp = &aOp[pCaller->p2 - 1]; - pIn1->flags = MEM_Undefined; break; } @@ -77567,7 +94589,7 @@ case OP_EndCoroutine: { /* in1 */ ** ** See also: InitCoroutine */ -case OP_Yield: { /* in1, jump */ +case OP_Yield: { /* in1, jump0 */ int pcDest; pIn1 = &aMem[pOp->p1]; assert( VdbeMemDynamic(pIn1)==0 ); @@ -77580,7 +94602,7 @@ case OP_Yield: { /* in1, jump */ } /* Opcode: HaltIfNull P1 P2 P3 P4 P5 -** Synopsis: if r[P3]=null halt +** Synopsis: if r[P3]=null halt ** ** Check the value in register P3. If it is NULL then Halt using ** parameter P1, P2, and P4 as if this were a Halt instruction. If the @@ -77589,11 +94611,15 @@ case OP_Yield: { /* in1, jump */ */ case OP_HaltIfNull: { /* in3 */ pIn3 = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } +#endif if( (pIn3->flags & MEM_Null)==0 ) break; /* Fall through into OP_Halt */ + /* no break */ deliberate_fall_through } -/* Opcode: Halt P1 P2 * P4 P5 +/* Opcode: Halt P1 P2 P3 P4 P5 ** ** Exit immediately. All open cursors, etc are closed ** automatically. @@ -77604,20 +94630,24 @@ case OP_HaltIfNull: { /* in3 */ ** whether or not to rollback the current transaction. Do not rollback ** if P2==OE_Fail. Do the rollback if P2==OE_Rollback. If P2==OE_Abort, ** then back out all changes that have occurred during this execution of the -** VDBE, but do not rollback the transaction. +** VDBE, but do not rollback the transaction. ** -** If P4 is not null then it is an error message string. +** If P3 is not zero and P4 is NULL, then P3 is a register that holds the +** text of an error message. ** -** P5 is a value between 0 and 4, inclusive, that modifies the P4 string. +** If P3 is zero and P4 is not null then the error message string is held +** in P4. ** -** 0: (no change) -** 1: NOT NULL contraint failed: P4 +** P5 is a value between 1 and 4, inclusive, then the P4 error message +** string is modified as follows: +** +** 1: NOT NULL constraint failed: P4 ** 2: UNIQUE constraint failed: P4 ** 3: CHECK constraint failed: P4 ** 4: FOREIGN KEY constraint failed: P4 ** -** If P5 is not zero and P4 is NULL, then everything after the ":" is -** omitted. +** If P3 is zero and P5 is not zero and P4 is NULL, then everything after +** the ":" is omitted. ** ** There is an implied "Halt 0 0 0" instruction inserted at the very end of ** every program. So a jump past the last instruction of the program @@ -77627,17 +94657,27 @@ case OP_Halt: { VdbeFrame *pFrame; int pcx; - pcx = (int)(pOp - aOp); - if( pOp->p1==SQLITE_OK && p->pFrame ){ +#ifdef SQLITE_DEBUG + if( pOp->p2==OE_Abort ){ sqlite3VdbeAssertAbortable(p); } +#endif + assert( pOp->p4type==P4_NOTUSED + || pOp->p4type==P4_STATIC + || pOp->p4type==P4_DYNAMIC ); + + /* A deliberately coded "OP_Halt SQLITE_INTERNAL * * * *" opcode indicates + ** something is wrong with the code generator. Raise an assertion in order + ** to bring this to the attention of fuzzers and other testing tools. */ + assert( pOp->p1!=SQLITE_INTERNAL ); + + if( p->pFrame && pOp->p1==SQLITE_OK ){ /* Halt the sub-program. Return control to the parent frame. */ pFrame = p->pFrame; p->pFrame = pFrame->pParent; p->nFrame--; sqlite3VdbeSetChanges(db, p->nChange); pcx = sqlite3VdbeFrameRestore(pFrame); - lastRowid = db->lastRowid; if( pOp->p2==OE_Ignore ){ - /* Instruction pcx is the OP_Program that invoked the sub-program + /* Instruction pcx is the OP_Program that invoked the sub-program ** currently being halted. If the p2 instruction of this OP_Halt ** instruction is set to OE_Ignore, then the sub-program is throwing ** an IGNORE exception. In this case jump to the address specified @@ -77651,10 +94691,14 @@ case OP_Halt: { } p->rc = pOp->p1; p->errorAction = (u8)pOp->p2; - p->pc = pcx; - assert( pOp->p5>=0 && pOp->p5<=4 ); + assert( pOp->p5<=4 ); if( p->rc ){ - if( pOp->p5 ){ + if( pOp->p3>0 && pOp->p4type==P4_NOTUSED ){ + const char *zErr; + assert( pOp->p3<=(p->nMem + 1 - p->nCursor) ); + zErr = sqlite3ValueText(&aMem[pOp->p3], SQLITE_UTF8); + sqlite3VdbeError(p, "%s", zErr); + }else if( pOp->p5 ){ static const char * const azType[] = { "NOT NULL", "UNIQUE", "CHECK", "FOREIGN KEY" }; testcase( pOp->p5==1 ); @@ -77668,6 +94712,7 @@ case OP_Halt: { }else{ sqlite3VdbeError(p, "%s", pOp->p4.z); } + pcx = (int)(pOp - aOp); sqlite3_log(pOp->p1, "abort at %d in [%s]: %s", pcx, p->zSql, p->zErrMsg); } rc = sqlite3VdbeHalt(p); @@ -77725,7 +94770,7 @@ case OP_Real: { /* same as TK_FLOAT, out2 */ /* Opcode: String8 * P2 * P4 * ** Synopsis: r[P2]='P4' ** -** P4 points to a nul terminated UTF-8 string. This opcode is transformed +** P4 points to a nul terminated UTF-8 string. This opcode is transformed ** into a String opcode before it is executed for the first time. During ** this transformation, the length of string P4 is computed and stored ** as the P1 parameter. @@ -77733,13 +94778,13 @@ case OP_Real: { /* same as TK_FLOAT, out2 */ case OP_String8: { /* same as TK_STRING, out2 */ assert( pOp->p4.z!=0 ); pOut = out2Prerelease(p, pOp); - pOp->opcode = OP_String; pOp->p1 = sqlite3Strlen30(pOp->p4.z); #ifndef SQLITE_OMIT_UTF16 if( encoding!=SQLITE_UTF8 ){ rc = sqlite3VdbeMemSetStr(pOut, pOp->p4.z, -1, SQLITE_UTF8, SQLITE_STATIC); assert( rc==SQLITE_OK || rc==SQLITE_TOOBIG ); + if( rc ) goto too_big; if( SQLITE_OK!=sqlite3VdbeChangeEncoding(pOut, encoding) ) goto no_mem; assert( pOut->szMalloc>0 && pOut->zMalloc==pOut->z ); assert( VdbeMemDynamic(pOut)==0 ); @@ -77752,15 +94797,16 @@ case OP_String8: { /* same as TK_STRING, out2 */ pOp->p4.z = pOut->z; pOp->p1 = pOut->n; } - testcase( rc==SQLITE_TOOBIG ); #endif if( pOp->p1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; } + pOp->opcode = OP_String; assert( rc==SQLITE_OK ); /* Fall through to the next case, OP_String */ + /* no break */ deliberate_fall_through } - + /* Opcode: String P1 P2 P3 P4 P5 ** Synopsis: r[P2]='P4' (len=P1) ** @@ -77792,8 +94838,30 @@ case OP_String: { /* out2 */ break; } +/* Opcode: BeginSubrtn * P2 * * * +** Synopsis: r[P2]=NULL +** +** Mark the beginning of a subroutine that can be entered in-line +** or that can be called using OP_Gosub. The subroutine should +** be terminated by an OP_Return instruction that has a P1 operand that +** is the same as the P2 operand to this opcode and that has P3 set to 1. +** If the subroutine is entered in-line, then the OP_Return will simply +** fall through. But if the subroutine is entered using OP_Gosub, then +** the OP_Return will jump back to the first instruction after the OP_Gosub. +** +** This routine works by loading a NULL into the P2 register. When the +** return address register contains a NULL, the OP_Return instruction is +** a no-op that simply falls through to the next instruction (assuming that +** the OP_Return opcode has a P3 value of 1). Thus if the subroutine is +** entered in-line, then the OP_Return will cause in-line execution to +** continue. But if the subroutine is entered via OP_Gosub, then the +** OP_Return will cause a return to the address following the OP_Gosub. +** +** This opcode is identical to OP_Null. It has a different name +** only to make the byte code easier to read and verify. +*/ /* Opcode: Null P1 P2 P3 * * -** Synopsis: r[P2..P3]=NULL +** Synopsis: r[P2..P3]=NULL ** ** Write a NULL into registers P2. If P3 greater than P2, then also write ** NULL into register P3 and every register in between P2 and P3. If P3 @@ -77804,6 +94872,7 @@ case OP_String: { /* out2 */ ** NULL values will not compare equal even if SQLITE_NULLEQ is set on ** OP_Ne or OP_Eq. */ +case OP_BeginSubrtn: case OP_Null: { /* out2 */ int cnt; u16 nullFlag; @@ -77811,18 +94880,23 @@ case OP_Null: { /* out2 */ cnt = pOp->p3-pOp->p2; assert( pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->flags = nullFlag = pOp->p1 ? (MEM_Null|MEM_Cleared) : MEM_Null; + pOut->n = 0; +#ifdef SQLITE_DEBUG + pOut->uTemp = 0; +#endif while( cnt>0 ){ pOut++; memAboutToChange(p, pOut); sqlite3VdbeMemSetNull(pOut); pOut->flags = nullFlag; + pOut->n = 0; cnt--; } break; } /* Opcode: SoftNull P1 * * * * -** Synopsis: r[P1]=NULL +** Synopsis: r[P1]=NULL ** ** Set register P1 to have the value NULL as seen by the OP_MakeRecord ** instruction, but do not free any string or blob memory associated with @@ -77832,7 +94906,7 @@ case OP_Null: { /* out2 */ case OP_SoftNull: { assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); pOut = &aMem[pOp->p1]; - pOut->flags = (pOut->flags|MEM_Null)&~MEM_Undefined; + pOut->flags = (pOut->flags&~(MEM_Undefined|MEM_AffMask))|MEM_Null; break; } @@ -77840,42 +94914,47 @@ case OP_SoftNull: { ** Synopsis: r[P2]=P4 (len=P1) ** ** P4 points to a blob of data P1 bytes long. Store this -** blob in register P2. +** blob in register P2. If P4 is a NULL pointer, then construct +** a zero-filled blob that is P1 bytes long in P2. */ case OP_Blob: { /* out2 */ assert( pOp->p1 <= SQLITE_MAX_LENGTH ); pOut = out2Prerelease(p, pOp); - sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + if( pOp->p4.z==0 ){ + sqlite3VdbeMemSetZeroBlob(pOut, pOp->p1); + if( sqlite3VdbeMemExpandBlob(pOut) ) goto no_mem; + }else{ + sqlite3VdbeMemSetStr(pOut, pOp->p4.z, pOp->p1, 0, 0); + } pOut->enc = encoding; UPDATE_MAX_BLOBSIZE(pOut); break; } -/* Opcode: Variable P1 P2 * P4 * -** Synopsis: r[P2]=parameter(P1,P4) +/* Opcode: Variable P1 P2 * * * +** Synopsis: r[P2]=parameter(P1) ** ** Transfer the values of bound parameter P1 into register P2 -** -** If the parameter is named, then its name appears in P4. -** The P4 value is used by sqlite3_bind_parameter_name(). */ case OP_Variable: { /* out2 */ Mem *pVar; /* Value being transferred */ assert( pOp->p1>0 && pOp->p1<=p->nVar ); - assert( pOp->p4.z==0 || pOp->p4.z==p->azVar[pOp->p1-1] ); pVar = &p->aVar[pOp->p1 - 1]; if( sqlite3VdbeMemTooBig(pVar) ){ goto too_big; } - pOut = out2Prerelease(p, pOp); - sqlite3VdbeMemShallowCopy(pOut, pVar, MEM_Static); + pOut = &aMem[pOp->p2]; + if( VdbeMemDynamic(pOut) ) sqlite3VdbeMemSetNull(pOut); + memcpy(pOut, pVar, MEMCELLSIZE); + pOut->flags &= ~(MEM_Dyn|MEM_Ephem); + pOut->flags |= MEM_Static|MEM_FromBind; UPDATE_MAX_BLOBSIZE(pOut); break; } /* Opcode: Move P1 P2 P3 * * -** Synopsis: r[P2@P3]=r[P1@P3] +** Synopsis: r[P2@P3]=r[P1@P3] ** ** Move the P3 values in register P1..P1+P3-1 over into ** registers P2..P2+P3-1. Registers P1..P1+P3-1 are @@ -77903,8 +94982,13 @@ case OP_Move: { memAboutToChange(p, pOut); sqlite3VdbeMemMove(pOut, pIn1); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom>=&aMem[p1] && pOut->pScopyFrompScopyFrom += pOp->p2 - p1; + pIn1->pScopyFrom = 0; + { int i; + for(i=1; inMem; i++){ + if( aMem[i].pScopyFrom==pIn1 ){ + aMem[i].pScopyFrom = pOut; + } + } } #endif Deephemeralize(pOut); @@ -77915,11 +94999,16 @@ case OP_Move: { break; } -/* Opcode: Copy P1 P2 P3 * * +/* Opcode: Copy P1 P2 P3 * P5 ** Synopsis: r[P2@P3+1]=r[P1@P3+1] ** ** Make a copy of registers P1..P1+P3 into registers P2..P2+P3. ** +** If the 0x0002 bit of P5 is set then also clear the MEM_Subtype flag in the +** destination. The 0x0001 bit of P5 indicates that this Copy opcode cannot +** be merged. The 0x0001 bit is used by the query planner and does not +** come into play during query execution. +** ** This instruction makes a deep copy of the value. A duplicate ** is made of any string or blob constant. See also OP_SCopy. */ @@ -77931,8 +95020,12 @@ case OP_Copy: { pOut = &aMem[pOp->p2]; assert( pOut!=pIn1 ); while( 1 ){ + memAboutToChange(p, pOut); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); Deephemeralize(pOut); + if( (pOut->flags & MEM_Subtype)!=0 && (pOp->p5 & 0x0002)!=0 ){ + pOut->flags &= ~MEM_Subtype; + } #ifdef SQLITE_DEBUG pOut->pScopyFrom = 0; #endif @@ -77963,7 +95056,8 @@ case OP_SCopy: { /* out2 */ assert( pOut!=pIn1 ); sqlite3VdbeMemShallowCopy(pOut, pIn1, MEM_Ephem); #ifdef SQLITE_DEBUG - if( pOut->pScopyFrom==0 ) pOut->pScopyFrom = pIn1; + pOut->pScopyFrom = pIn1; + pOut->mScopyFlags = pIn1->flags; #endif break; } @@ -77984,8 +95078,26 @@ case OP_IntCopy: { /* out2 */ break; } +/* Opcode: FkCheck * * * * * +** +** Halt with an SQLITE_CONSTRAINT error if there are any unresolved +** foreign key constraint violations. If there are no foreign key +** constraint violations, this is a no-op. +** +** FK constraint violations are also checked when the prepared statement +** exits. This opcode is used to raise foreign key constraint errors prior +** to returning results such as a row change count or the result of a +** RETURNING clause. +*/ +case OP_FkCheck: { + if( (rc = sqlite3VdbeCheckFk(p,0))!=SQLITE_OK ){ + goto abort_due_to_error; + } + break; +} + /* Opcode: ResultRow P1 P2 * * * -** Synopsis: output=r[P1@P2] +** Synopsis: output=r[P1@P2] ** ** The registers P1 through P1+P2-1 contain a single row of ** results. This opcode causes the sqlite3_step() call to terminate @@ -77994,72 +95106,32 @@ case OP_IntCopy: { /* out2 */ ** the result row. */ case OP_ResultRow: { - Mem *pMem; - int i; assert( p->nResColumn==pOp->p2 ); - assert( pOp->p1>0 ); + assert( pOp->p1>0 || CORRUPT_DB ); assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); -#ifndef SQLITE_OMIT_PROGRESS_CALLBACK - /* Run the progress counter just before returning. - */ - if( db->xProgress!=0 - && nVmStep>=nProgressLimit - && db->xProgress(db->pProgressArg)!=0 - ){ - rc = SQLITE_INTERRUPT; - goto abort_due_to_error; + p->cacheCtr = (p->cacheCtr + 2)|1; + p->pResultRow = &aMem[pOp->p1]; +#ifdef SQLITE_DEBUG + { + Mem *pMem = p->pResultRow; + int i; + for(i=0; ip2; i++){ + assert( memIsValid(&pMem[i]) ); + REGISTER_TRACE(pOp->p1+i, &pMem[i]); + /* The registers in the result will not be used again when the + ** prepared statement restarts. This is because sqlite3_column() + ** APIs might have caused type conversions of made other changes to + ** the register values. Therefore, we can go ahead and break any + ** OP_SCopy dependencies. */ + pMem[i].pScopyFrom = 0; + } } #endif - - /* If this statement has violated immediate foreign key constraints, do - ** not return the number of rows modified. And do not RELEASE the statement - ** transaction. It needs to be rolled back. */ - if( SQLITE_OK!=(rc = sqlite3VdbeCheckFk(p, 0)) ){ - assert( db->flags&SQLITE_CountRows ); - assert( p->usesStmtJournal ); - goto abort_due_to_error; - } - - /* If the SQLITE_CountRows flag is set in sqlite3.flags mask, then - ** DML statements invoke this opcode to return the number of rows - ** modified to the user. This is the only way that a VM that - ** opens a statement transaction may invoke this opcode. - ** - ** In case this is such a statement, close any statement transaction - ** opened by this VM before returning control to the user. This is to - ** ensure that statement-transactions are always nested, not overlapping. - ** If the open statement-transaction is not closed here, then the user - ** may step another VM that opens its own statement transaction. This - ** may lead to overlapping statement transactions. - ** - ** The statement transaction is never a top-level transaction. Hence - ** the RELEASE call below can never fail. - */ - assert( p->iStatement==0 || db->flags&SQLITE_CountRows ); - rc = sqlite3VdbeCloseStatement(p, SAVEPOINT_RELEASE); - assert( rc==SQLITE_OK ); - - /* Invalidate all ephemeral cursor row caches */ - p->cacheCtr = (p->cacheCtr + 2)|1; - - /* Make sure the results of the current row are \000 terminated - ** and have an assigned type. The results are de-ephemeralized as - ** a side effect. - */ - pMem = p->pResultSet = &aMem[pOp->p1]; - for(i=0; ip2; i++){ - assert( memIsValid(&pMem[i]) ); - Deephemeralize(&pMem[i]); - assert( (pMem[i].flags & MEM_Ephem)==0 - || (pMem[i].flags & (MEM_Str|MEM_Blob))==0 ); - sqlite3VdbeMemNulTerminate(&pMem[i]); - REGISTER_TRACE(pOp->p1+i, &pMem[i]); - } if( db->mallocFailed ) goto no_mem; - - /* Return SQLITE_ROW - */ + if( db->mTrace & SQLITE_TRACE_ROW ){ + db->trace.xV2(SQLITE_TRACE_ROW, db->pTraceArg, p, 0); + } p->pc = (int)(pOp - aOp) + 1; rc = SQLITE_ROW; goto vdbe_return; @@ -78079,19 +95151,37 @@ case OP_ResultRow: { ** to avoid a memcpy(). */ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ - i64 nByte; + i64 nByte; /* Total size of the output string or blob */ + u16 flags1; /* Initial flags for P1 */ + u16 flags2; /* Initial flags for P2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; pOut = &aMem[pOp->p3]; + testcase( pOut==pIn2 ); assert( pIn1!=pOut ); - if( (pIn1->flags | pIn2->flags) & MEM_Null ){ + flags1 = pIn1->flags; + testcase( flags1 & MEM_Null ); + testcase( pIn2->flags & MEM_Null ); + if( (flags1 | pIn2->flags) & MEM_Null ){ sqlite3VdbeMemSetNull(pOut); break; } - if( ExpandBlob(pIn1) || ExpandBlob(pIn2) ) goto no_mem; - Stringify(pIn1, encoding); - Stringify(pIn2, encoding); + if( (flags1 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn1,encoding,0) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + }else if( (flags1 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn1) ) goto no_mem; + flags1 = pIn1->flags & ~MEM_Str; + } + flags2 = pIn2->flags; + if( (flags2 & (MEM_Str|MEM_Blob))==0 ){ + if( sqlite3VdbeMemStringify(pIn2,encoding,0) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + }else if( (flags2 & MEM_Zero)!=0 ){ + if( sqlite3VdbeMemExpandBlob(pIn2) ) goto no_mem; + flags2 = pIn2->flags & ~MEM_Str; + } nByte = pIn1->n + pIn2->n; if( nByte>db->aLimit[SQLITE_LIMIT_LENGTH] ){ goto too_big; @@ -78102,8 +95192,13 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ MemSetTypeFlag(pOut, MEM_Str); if( pOut!=pIn2 ){ memcpy(pOut->z, pIn2->z, pIn2->n); + assert( (pIn2->flags & MEM_Dyn) == (flags2 & MEM_Dyn) ); + pIn2->flags = flags2; } memcpy(&pOut->z[pIn2->n], pIn1->z, pIn1->n); + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; + if( encoding>SQLITE_UTF8 ) nByte &= ~1; pOut->z[nByte]=0; pOut->z[nByte+1] = 0; pOut->flags |= MEM_Term; @@ -78114,14 +95209,14 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ } /* Opcode: Add P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]+r[P2] +** Synopsis: r[P3]=r[P1]+r[P2] ** ** Add the value in register P1 to the value in register P2 ** and store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: Multiply P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]*r[P2] +** Synopsis: r[P3]=r[P1]*r[P2] ** ** ** Multiply the value in register P1 by the value in register P2 @@ -78129,25 +95224,25 @@ case OP_Concat: { /* same as TK_CONCAT, in1, in2, out3 */ ** If either input is NULL, the result is NULL. */ /* Opcode: Subtract P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]-r[P1] +** Synopsis: r[P3]=r[P2]-r[P1] ** ** Subtract the value in register P1 from the value in register P2 ** and store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: Divide P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]/r[P1] +** Synopsis: r[P3]=r[P2]/r[P1] ** ** Divide the value in register P1 by the value in register P2 -** and store the result in register P3 (P3=P2/P1). If the value in -** register P1 is zero, then the result is NULL. If either input is +** and store the result in register P3 (P3=P2/P1). If the value in +** register P1 is zero, then the result is NULL. If either input is ** NULL, the result is NULL. */ /* Opcode: Remainder P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]%r[P1] +** Synopsis: r[P3]=r[P2]%r[P1] ** -** Compute the remainder after integer register P2 is divided by -** register P1 and store the result in register P3. +** Compute the remainder after integer register P2 is divided by +** register P1 and store the result in register P3. ** If the value in register P1 is zero the result is NULL. ** If either operand is NULL, the result is NULL. */ @@ -78156,8 +95251,6 @@ case OP_Subtract: /* same as TK_MINUS, in1, in2, out3 */ case OP_Multiply: /* same as TK_STAR, in1, in2, out3 */ case OP_Divide: /* same as TK_SLASH, in1, in2, out3 */ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ - char bIntint; /* Started out as two integer operands */ - u16 flags; /* Combined MEM_* flags from both inputs */ u16 type1; /* Numeric type of left operand */ u16 type2; /* Numeric type of right operand */ i64 iA; /* Integer value of left operand */ @@ -78166,16 +95259,14 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ double rB; /* Real value of right operand */ pIn1 = &aMem[pOp->p1]; - type1 = numericType(pIn1); + type1 = pIn1->flags; pIn2 = &aMem[pOp->p2]; - type2 = numericType(pIn2); + type2 = pIn2->flags; pOut = &aMem[pOp->p3]; - flags = pIn1->flags | pIn2->flags; - if( (flags & MEM_Null)!=0 ) goto arithmetic_result_is_null; if( (type1 & type2 & MEM_Int)!=0 ){ +int_math: iA = pIn1->u.i; iB = pIn2->u.i; - bIntint = 1; switch( pOp->opcode ){ case OP_Add: if( sqlite3AddInt64(&iB,iA) ) goto fp_math; break; case OP_Subtract: if( sqlite3SubInt64(&iB,iA) ) goto fp_math; break; @@ -78195,8 +95286,12 @@ case OP_Remainder: { /* same as TK_REM, in1, in2, out3 */ } pOut->u.i = iB; MemSetTypeFlag(pOut, MEM_Int); + }else if( ((type1 | type2) & MEM_Null)!=0 ){ + goto arithmetic_result_is_null; }else{ - bIntint = 0; + type1 = numericType(pIn1); + type2 = numericType(pIn2); + if( (type1 & type2 & MEM_Int)!=0 ) goto int_math; fp_math: rA = sqlite3VdbeRealValue(pIn1); rB = sqlite3VdbeRealValue(pIn2); @@ -78211,8 +95306,8 @@ fp_math: break; } default: { - iA = (i64)rA; - iB = (i64)rB; + iA = sqlite3VdbeIntValue(pIn1); + iB = sqlite3VdbeIntValue(pIn2); if( iA==0 ) goto arithmetic_result_is_null; if( iA==-1 ) iA = 1; rB = (double)(iB % iA); @@ -78228,9 +95323,6 @@ fp_math: } pOut->u.r = rB; MemSetTypeFlag(pOut, MEM_Real); - if( ((type1|type2)&MEM_Real)==0 && !bIntint ){ - sqlite3VdbeIntegerAffinity(pOut); - } #endif } break; @@ -78242,7 +95334,7 @@ arithmetic_result_is_null: /* Opcode: CollSeq P1 * * P4 ** -** P4 is a pointer to a CollSeq struct. If the next call to a user function +** P4 is a pointer to a CollSeq object. If the next call to a user function ** or aggregate calls sqlite3GetFuncCollSeq(), this collation sequence will ** be returned. This is used by the built-in min(), max() and nullif() ** functions. @@ -78263,135 +95355,22 @@ case OP_CollSeq: { break; } -/* Opcode: Function0 P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) -** -** Invoke a user function (P4 is a pointer to a FuncDef object that -** defines the function) with P5 arguments taken from register P2 and -** successors. The result of the function is stored in register P3. -** Register P3 must not be one of the function inputs. -** -** P1 is a 32-bit bitmask indicating whether or not each argument to the -** function was determined to be constant at compile time. If the first -** argument was constant then bit 0 of P1 is set. This is used to determine -** whether meta data associated with a user function argument using the -** sqlite3_set_auxdata() API may be safely retained until the next -** invocation of this opcode. -** -** See also: Function, AggStep, AggFinal -*/ -/* Opcode: Function P1 P2 P3 P4 P5 -** Synopsis: r[P3]=func(r[P2@P5]) -** -** Invoke a user function (P4 is a pointer to an sqlite3_context object that -** contains a pointer to the function to be run) with P5 arguments taken -** from register P2 and successors. The result of the function is stored -** in register P3. Register P3 must not be one of the function inputs. -** -** P1 is a 32-bit bitmask indicating whether or not each argument to the -** function was determined to be constant at compile time. If the first -** argument was constant then bit 0 of P1 is set. This is used to determine -** whether meta data associated with a user function argument using the -** sqlite3_set_auxdata() API may be safely retained until the next -** invocation of this opcode. -** -** SQL functions are initially coded as OP_Function0 with P4 pointing -** to a FuncDef object. But on first evaluation, the P4 operand is -** automatically converted into an sqlite3_context object and the operation -** changed to this OP_Function opcode. In this way, the initialization of -** the sqlite3_context object occurs only once, rather than once for each -** evaluation of the function. -** -** See also: Function0, AggStep, AggFinal -*/ -case OP_Function0: { - int n; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCDEF ); - n = pOp->p5; - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); - assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); - if( pCtx==0 ) goto no_mem; - pCtx->pOut = 0; - pCtx->pFunc = pOp->p4.pFunc; - pCtx->iOp = (int)(pOp - aOp); - pCtx->pVdbe = p; - pCtx->argc = n; - pOp->p4type = P4_FUNCCTX; - pOp->p4.pCtx = pCtx; - pOp->opcode = OP_Function; - /* Fall through into OP_Function */ -} -case OP_Function: { - int i; - sqlite3_context *pCtx; - - assert( pOp->p4type==P4_FUNCCTX ); - pCtx = pOp->p4.pCtx; - - /* If this function is inside of a trigger, the register array in aMem[] - ** might change from one evaluation to the next. The next block of code - ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ - pOut = &aMem[pOp->p3]; - if( pCtx->pOut != pOut ){ - pCtx->pOut = pOut; - for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; - } - - memAboutToChange(p, pCtx->pOut); -#ifdef SQLITE_DEBUG - for(i=0; iargc; i++){ - assert( memIsValid(pCtx->argv[i]) ); - REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); - } -#endif - MemSetTypeFlag(pCtx->pOut, MEM_Null); - pCtx->fErrorOrAux = 0; - db->lastRowid = lastRowid; - (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ - lastRowid = db->lastRowid; /* Remember rowid changes made by xSFunc */ - - /* If the function returned an error, throw an exception */ - if( pCtx->fErrorOrAux ){ - if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); - rc = pCtx->isError; - } - sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); - if( rc ) goto abort_due_to_error; - } - - /* Copy the result of the function into register P3 */ - if( pOut->flags & (MEM_Str|MEM_Blob) ){ - sqlite3VdbeChangeEncoding(pCtx->pOut, encoding); - if( sqlite3VdbeMemTooBig(pCtx->pOut) ) goto too_big; - } - - REGISTER_TRACE(pOp->p3, pCtx->pOut); - UPDATE_MAX_BLOBSIZE(pCtx->pOut); - break; -} - /* Opcode: BitAnd P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]&r[P2] +** Synopsis: r[P3]=r[P1]&r[P2] ** ** Take the bit-wise AND of the values in register P1 and P2 and ** store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: BitOr P1 P2 P3 * * -** Synopsis: r[P3]=r[P1]|r[P2] +** Synopsis: r[P3]=r[P1]|r[P2] ** ** Take the bit-wise OR of the values in register P1 and P2 and ** store the result in register P3. ** If either input is NULL, the result is NULL. */ /* Opcode: ShiftLeft P1 P2 P3 * * -** Synopsis: r[P3]=r[P2]<>r[P1] +** Synopsis: r[P3]=r[P2]>>r[P1] ** ** Shift the integer value in register P2 to the right by the ** number of bits specified by the integer in register P1. @@ -78459,8 +95438,8 @@ case OP_ShiftRight: { /* same as TK_RSHIFT, in1, in2, out3 */ } /* Opcode: AddImm P1 P2 * * * -** Synopsis: r[P1]=r[P1]+P2 -** +** Synopsis: r[P1]=r[P1]+P2 +** ** Add the constant P2 to the value in register P1. ** The result is always an integer. ** @@ -78470,23 +95449,23 @@ case OP_AddImm: { /* in1 */ pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); sqlite3VdbeMemIntegerify(pIn1); - pIn1->u.i += pOp->p2; + *(u64*)&pIn1->u.i += (u64)pOp->p2; break; } /* Opcode: MustBeInt P1 P2 * * * -** +** ** Force the value in register P1 to be an integer. If the value ** in P1 is not an integer and cannot be converted into an integer ** without data loss, then jump immediately to P2, or if P2==0 ** raise an SQLITE_MISMATCH exception. */ -case OP_MustBeInt: { /* jump, in1 */ +case OP_MustBeInt: { /* jump0, in1 */ pIn1 = &aMem[pOp->p1]; if( (pIn1->flags & MEM_Int)==0 ){ applyAffinity(pIn1, SQLITE_AFF_NUMERIC, encoding); - VdbeBranchTaken((pIn1->flags&MEM_Int)==0, 2); if( (pIn1->flags & MEM_Int)==0 ){ + VdbeBranchTaken(1, 2); if( pOp->p2==0 ){ rc = SQLITE_MISMATCH; goto abort_due_to_error; @@ -78495,6 +95474,7 @@ case OP_MustBeInt: { /* jump, in1 */ } } } + VdbeBranchTaken(0, 2); MemSetTypeFlag(pIn1, MEM_Int); break; } @@ -78511,25 +95491,28 @@ case OP_MustBeInt: { /* jump, in1 */ */ case OP_RealAffinity: { /* in1 */ pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Int ){ + if( pIn1->flags & (MEM_Int|MEM_IntReal) ){ + testcase( pIn1->flags & MEM_Int ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemRealify(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); } break; } #endif -#ifndef SQLITE_OMIT_CAST +#if !defined(SQLITE_OMIT_CAST) || !defined(SQLITE_OMIT_ANALYZE) /* Opcode: Cast P1 P2 * * * ** Synopsis: affinity(r[P1]) ** ** Force the value in register P1 to be the type defined by P2. -** +** **
              -**
            • TEXT -**
            • BLOB -**
            • NUMERIC -**
            • INTEGER -**
            • REAL +**
            • P2=='A' → BLOB +**
            • P2=='B' → TEXT +**
            • P2=='C' → NUMERIC +**
            • P2=='D' → INTEGER +**
            • P2=='E' → REAL **
            ** ** A NULL value is not changed by this routine. It remains NULL. @@ -78544,32 +95527,74 @@ case OP_Cast: { /* in1 */ pIn1 = &aMem[pOp->p1]; memAboutToChange(p, pIn1); rc = ExpandBlob(pIn1); - sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); - UPDATE_MAX_BLOBSIZE(pIn1); if( rc ) goto abort_due_to_error; + rc = sqlite3VdbeMemCast(pIn1, pOp->p2, encoding); + if( rc ) goto abort_due_to_error; + UPDATE_MAX_BLOBSIZE(pIn1); + REGISTER_TRACE(pOp->p1, pIn1); break; } #endif /* SQLITE_OMIT_CAST */ -/* Opcode: Lt P1 P2 P3 P4 P5 -** Synopsis: if r[P1]r[P3] goto P2 +** Synopsis: IF r[P3]>r[P1] ** ** This works just like the Lt opcode except that the jump is taken if ** the content of register P3 is greater than the content of ** register P1. See the Lt opcode for additional information. */ /* Opcode: Ge P1 P2 P3 P4 P5 -** Synopsis: if r[P1]>=r[P3] goto P2 +** Synopsis: IF r[P3]>=r[P1] ** ** This works just like the Lt opcode except that the jump is taken if ** the content of register P3 is greater than or equal to the content of @@ -78639,7 +95634,7 @@ case OP_Lt: /* same as TK_LT, jump, in1, in3 */ case OP_Le: /* same as TK_LE, jump, in1, in3 */ case OP_Gt: /* same as TK_GT, jump, in1, in3 */ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ - int res; /* Result of the comparison of pIn1 against pIn3 */ + int res, res2; /* Result of the comparison of pIn1 against pIn3 */ char affinity; /* Affinity to use for comparison */ u16 flags1; /* Copy of initial value of pIn1->flags */ u16 flags3; /* Copy of initial value of pIn3->flags */ @@ -78648,6 +95643,33 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ pIn3 = &aMem[pOp->p3]; flags1 = pIn1->flags; flags3 = pIn3->flags; + if( (flags1 & flags3 & MEM_Int)!=0 ){ + /* Common case of comparison of two integers */ + if( pIn3->u.i > pIn1->u.i ){ + if( sqlite3aGTb[pOp->opcode] ){ + VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); + goto jump_to_p2; + } + iCompare = +1; + VVA_ONLY( iCompareIsInit = 1; ) + }else if( pIn3->u.i < pIn1->u.i ){ + if( sqlite3aLTb[pOp->opcode] ){ + VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); + goto jump_to_p2; + } + iCompare = -1; + VVA_ONLY( iCompareIsInit = 1; ) + }else{ + if( sqlite3aEQb[pOp->opcode] ){ + VdbeBranchTaken(1, (pOp->p5 & SQLITE_NULLEQ)?2:3); + goto jump_to_p2; + } + iCompare = 0; + VVA_ONLY( iCompareIsInit = 1; ) + } + VdbeBranchTaken(0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + break; + } if( (flags1 | flags3)&MEM_Null ){ /* One or both operands are NULL */ if( pOp->p5 & SQLITE_NULLEQ ){ @@ -78655,112 +95677,140 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ ** OP_Eq or OP_Ne) then take the jump or not depending on whether ** or not both operands are null. */ - assert( pOp->opcode==OP_Eq || pOp->opcode==OP_Ne ); assert( (flags1 & MEM_Cleared)==0 ); - assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 ); - if( (flags1&MEM_Null)!=0 - && (flags3&MEM_Null)!=0 + assert( (pOp->p5 & SQLITE_JUMPIFNULL)==0 || CORRUPT_DB ); + testcase( (pOp->p5 & SQLITE_JUMPIFNULL)!=0 ); + if( (flags1&flags3&MEM_Null)!=0 && (flags3&MEM_Cleared)==0 ){ - res = 0; /* Results are equal */ + res = 0; /* Operands are equal */ }else{ - res = 1; /* Results are not equal */ + res = ((flags3 & MEM_Null) ? -1 : +1); /* Operands are not equal */ } }else{ /* SQLITE_NULLEQ is clear and at least one operand is NULL, ** then the result is always NULL. ** The jump is taken if the SQLITE_JUMPIFNULL bit is set. */ - if( pOp->p5 & SQLITE_STOREP2 ){ - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); - MemSetTypeFlag(pOut, MEM_Null); - REGISTER_TRACE(pOp->p2, pOut); - }else{ - VdbeBranchTaken(2,3); - if( pOp->p5 & SQLITE_JUMPIFNULL ){ - goto jump_to_p2; - } + VdbeBranchTaken(2,3); + if( pOp->p5 & SQLITE_JUMPIFNULL ){ + goto jump_to_p2; } + iCompare = 1; /* Operands are not equal */ + VVA_ONLY( iCompareIsInit = 1; ) break; } }else{ - /* Neither operand is NULL. Do a comparison. */ + /* Neither operand is NULL and we couldn't do the special high-speed + ** integer comparison case. So do a general-case comparison. */ affinity = pOp->p5 & SQLITE_AFF_MASK; if( affinity>=SQLITE_AFF_NUMERIC ){ if( (flags1 | flags3)&MEM_Str ){ - if( (flags1 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags1 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn1,0); + assert( flags3==pIn3->flags || CORRUPT_DB ); + flags3 = pIn3->flags; } - if( (flags3 & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + if( (flags3 & (MEM_Int|MEM_IntReal|MEM_Real|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3,0); } } - }else if( affinity==SQLITE_AFF_TEXT ){ - if( (flags1 & MEM_Str)==0 && (flags1 & (MEM_Int|MEM_Real))!=0 ){ + }else if( affinity==SQLITE_AFF_TEXT && ((flags1 | flags3) & MEM_Str)!=0 ){ + if( (flags1 & MEM_Str)!=0 ){ + pIn1->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags1&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn1->flags & MEM_Int ); testcase( pIn1->flags & MEM_Real ); + testcase( pIn1->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn1, encoding, 1); testcase( (flags1&MEM_Dyn) != (pIn1->flags&MEM_Dyn) ); flags1 = (pIn1->flags & ~MEM_TypeMask) | (flags1 & MEM_TypeMask); + if( NEVER(pIn1==pIn3) ) flags3 = flags1 | MEM_Str; } - if( (flags3 & MEM_Str)==0 && (flags3 & (MEM_Int|MEM_Real))!=0 ){ + if( (flags3 & MEM_Str)!=0 ){ + pIn3->flags &= ~(MEM_Int|MEM_Real|MEM_IntReal); + }else if( (flags3&(MEM_Int|MEM_Real|MEM_IntReal))!=0 ){ testcase( pIn3->flags & MEM_Int ); testcase( pIn3->flags & MEM_Real ); + testcase( pIn3->flags & MEM_IntReal ); sqlite3VdbeMemStringify(pIn3, encoding, 1); testcase( (flags3&MEM_Dyn) != (pIn3->flags&MEM_Dyn) ); flags3 = (pIn3->flags & ~MEM_TypeMask) | (flags3 & MEM_TypeMask); } } assert( pOp->p4type==P4_COLLSEQ || pOp->p4.pColl==0 ); - if( flags1 & MEM_Zero ){ - sqlite3VdbeMemExpandBlob(pIn1); - flags1 &= ~MEM_Zero; - } - if( flags3 & MEM_Zero ){ - sqlite3VdbeMemExpandBlob(pIn3); - flags3 &= ~MEM_Zero; - } res = sqlite3MemCompare(pIn3, pIn1, pOp->p4.pColl); } - switch( pOp->opcode ){ - case OP_Eq: res = res==0; break; - case OP_Ne: res = res!=0; break; - case OP_Lt: res = res<0; break; - case OP_Le: res = res<=0; break; - case OP_Gt: res = res>0; break; - default: res = res>=0; break; + + /* At this point, res is negative, zero, or positive if reg[P1] is + ** less than, equal to, or greater than reg[P3], respectively. Compute + ** the answer to this operator in res2, depending on what the comparison + ** operator actually is. The next block of code depends on the fact + ** that the 6 comparison operators are consecutive integers in this + ** order: NE, EQ, GT, LE, LT, GE */ + assert( OP_Eq==OP_Ne+1 ); assert( OP_Gt==OP_Ne+2 ); assert( OP_Le==OP_Ne+3 ); + assert( OP_Lt==OP_Ne+4 ); assert( OP_Ge==OP_Ne+5 ); + if( res<0 ){ + res2 = sqlite3aLTb[pOp->opcode]; + }else if( res==0 ){ + res2 = sqlite3aEQb[pOp->opcode]; + }else{ + res2 = sqlite3aGTb[pOp->opcode]; } + iCompare = res; + VVA_ONLY( iCompareIsInit = 1; ) /* Undo any changes made by applyAffinity() to the input registers. */ - assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); - pIn1->flags = flags1; assert( (pIn3->flags & MEM_Dyn) == (flags3 & MEM_Dyn) ); pIn3->flags = flags3; + assert( (pIn1->flags & MEM_Dyn) == (flags1 & MEM_Dyn) ); + pIn1->flags = flags1; - if( pOp->p5 & SQLITE_STOREP2 ){ - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); - MemSetTypeFlag(pOut, MEM_Int); - pOut->u.i = res; - REGISTER_TRACE(pOp->p2, pOut); - }else{ - VdbeBranchTaken(res!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); - if( res ){ - goto jump_to_p2; - } + VdbeBranchTaken(res2!=0, (pOp->p5 & SQLITE_NULLEQ)?2:3); + if( res2 ){ + goto jump_to_p2; } break; } +/* Opcode: ElseEq * P2 * * * +** +** This opcode must follow an OP_Lt or OP_Gt comparison operator. There +** can be zero or more OP_ReleaseReg opcodes intervening, but no other +** opcodes are allowed to occur between this instruction and the previous +** OP_Lt or OP_Gt. +** +** If the result of an OP_Eq comparison on the same two operands as +** the prior OP_Lt or OP_Gt would have been true, then jump to P2. If +** the result of an OP_Eq comparison on the two previous operands +** would have been false or NULL, then fall through. +*/ +case OP_ElseEq: { /* same as TK_ESCAPE, jump */ + +#ifdef SQLITE_DEBUG + /* Verify the preconditions of this opcode - that it follows an OP_Lt or + ** OP_Gt with zero or more intervening OP_ReleaseReg opcodes */ + int iAddr; + for(iAddr = (int)(pOp - aOp) - 1; ALWAYS(iAddr>=0); iAddr--){ + if( aOp[iAddr].opcode==OP_ReleaseReg ) continue; + assert( aOp[iAddr].opcode==OP_Lt || aOp[iAddr].opcode==OP_Gt ); + break; + } +#endif /* SQLITE_DEBUG */ + assert( iCompareIsInit ); + VdbeBranchTaken(iCompare==0, 2); + if( iCompare==0 ) goto jump_to_p2; + break; +} + + /* Opcode: Permutation * * * P4 * ** -** Set the permutation used by the OP_Compare operator to be the array -** of integers in P4. +** Set the permutation used by the OP_Compare operator in the next +** instruction. The permutation is stored in the P4 operand. ** -** The permutation is only valid until the next OP_Compare that has -** the OPFLAG_PERMUTE bit set in P5. Typically the OP_Permutation should -** occur immediately prior to the OP_Compare. +** The permutation is only valid for the next opcode which must be +** an OP_Compare that has the OPFLAG_PERMUTE bit set in P5. ** ** The first integer in the P4 integer array is the length of the array ** and does not become part of the permutation. @@ -78768,7 +95818,8 @@ case OP_Ge: { /* same as TK_GE, jump, in1, in3 */ case OP_Permutation: { assert( pOp->p4type==P4_INTARRAY ); assert( pOp->p4.ai ); - aPermute = pOp->p4.ai + 1; + assert( pOp[1].opcode==OP_Compare ); + assert( pOp[1].p5 & OPFLAG_PERMUTE ); break; } @@ -78791,6 +95842,8 @@ case OP_Permutation: { ** The comparison is a sort comparison, so NULLs compare equal, ** NULLs are less than numbers, numbers are less than strings, ** and strings are less than blobs. +** +** This opcode must be immediately followed by an OP_Jump opcode. */ case OP_Compare: { int n; @@ -78798,21 +95851,30 @@ case OP_Compare: { int p1; int p2; const KeyInfo *pKeyInfo; - int idx; + u32 idx; CollSeq *pColl; /* Collating sequence to use on this term */ int bRev; /* True for DESCENDING sort order */ + u32 *aPermute; /* The permutation */ - if( (pOp->p5 & OPFLAG_PERMUTE)==0 ) aPermute = 0; + if( (pOp->p5 & OPFLAG_PERMUTE)==0 ){ + aPermute = 0; + }else{ + assert( pOp>aOp ); + assert( pOp[-1].opcode==OP_Permutation ); + assert( pOp[-1].p4type==P4_INTARRAY ); + aPermute = pOp[-1].p4.ai + 1; + assert( aPermute!=0 ); + } n = pOp->p3; pKeyInfo = pOp->p4.pKeyInfo; assert( n>0 ); assert( pKeyInfo!=0 ); p1 = pOp->p1; p2 = pOp->p2; -#if SQLITE_DEBUG +#ifdef SQLITE_DEBUG if( aPermute ){ int k, mx = 0; - for(k=0; kmx ) mx = aPermute[k]; + for(k=0; k(u32)mx ) mx = aPermute[k]; assert( p1>0 && p1+mx<=(p->nMem+1 - p->nCursor)+1 ); assert( p2>0 && p2+mx<=(p->nMem+1 - p->nCursor)+1 ); }else{ @@ -78821,37 +95883,47 @@ case OP_Compare: { } #endif /* SQLITE_DEBUG */ for(i=0; inField ); + assert( inKeyField ); pColl = pKeyInfo->aColl[i]; - bRev = pKeyInfo->aSortOrder[i]; + bRev = (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_DESC); iCompare = sqlite3MemCompare(&aMem[p1+idx], &aMem[p2+idx], pColl); + VVA_ONLY( iCompareIsInit = 1; ) if( iCompare ){ + if( (pKeyInfo->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) + && ((aMem[p1+idx].flags & MEM_Null) || (aMem[p2+idx].flags & MEM_Null)) + ){ + iCompare = -iCompare; + } if( bRev ) iCompare = -iCompare; break; } } - aPermute = 0; + assert( pOp[1].opcode==OP_Jump ); break; } /* Opcode: Jump P1 P2 P3 * * ** ** Jump to the instruction at address P1, P2, or P3 depending on whether -** in the most recent OP_Compare instruction the P1 vector was less than +** in the most recent OP_Compare instruction the P1 vector was less than, ** equal to, or greater than the P2 vector, respectively. +** +** This opcode must immediately follow an OP_Compare opcode. */ case OP_Jump: { /* jump */ + assert( pOp>aOp && pOp[-1].opcode==OP_Compare ); + assert( iCompareIsInit ); if( iCompare<0 ){ - VdbeBranchTaken(0,3); pOp = &aOp[pOp->p1 - 1]; + VdbeBranchTaken(0,4); pOp = &aOp[pOp->p1 - 1]; }else if( iCompare==0 ){ - VdbeBranchTaken(1,3); pOp = &aOp[pOp->p2 - 1]; + VdbeBranchTaken(1,4); pOp = &aOp[pOp->p2 - 1]; }else{ - VdbeBranchTaken(2,3); pOp = &aOp[pOp->p3 - 1]; + VdbeBranchTaken(2,4); pOp = &aOp[pOp->p3 - 1]; } break; } @@ -78881,18 +95953,8 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ int v1; /* Left operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ int v2; /* Right operand: 0==FALSE, 1==TRUE, 2==UNKNOWN or NULL */ - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - v1 = 2; - }else{ - v1 = sqlite3VdbeIntValue(pIn1)!=0; - } - pIn2 = &aMem[pOp->p2]; - if( pIn2->flags & MEM_Null ){ - v2 = 2; - }else{ - v2 = sqlite3VdbeIntValue(pIn2)!=0; - } + v1 = sqlite3VdbeBooleanValue(&aMem[pOp->p1], 2); + v2 = sqlite3VdbeBooleanValue(&aMem[pOp->p2], 2); if( pOp->opcode==OP_And ){ static const unsigned char and_logic[] = { 0, 0, 0, 0, 1, 2, 0, 2, 2 }; v1 = and_logic[v1*3+v2]; @@ -78910,26 +95972,55 @@ case OP_Or: { /* same as TK_OR, in1, in2, out3 */ break; } +/* Opcode: IsTrue P1 P2 P3 P4 * +** Synopsis: r[P2] = coalesce(r[P1]==TRUE,P3) ^ P4 +** +** This opcode implements the IS TRUE, IS FALSE, IS NOT TRUE, and +** IS NOT FALSE operators. +** +** Interpret the value in register P1 as a boolean value. Store that +** boolean (a 0 or 1) in register P2. Or if the value in register P1 is +** NULL, then the P3 is stored in register P2. Invert the answer if P4 +** is 1. +** +** The logic is summarized like this: +** +**
              +**
            • If P3==0 and P4==0 then r[P2] := r[P1] IS TRUE +**
            • If P3==1 and P4==1 then r[P2] := r[P1] IS FALSE +**
            • If P3==0 and P4==1 then r[P2] := r[P1] IS NOT TRUE +**
            • If P3==1 and P4==0 then r[P2] := r[P1] IS NOT FALSE +**
            +*/ +case OP_IsTrue: { /* in1, out2 */ + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p4.i==0 || pOp->p4.i==1 ); + assert( pOp->p3==0 || pOp->p3==1 ); + sqlite3VdbeMemSetInt64(&aMem[pOp->p2], + sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3) ^ pOp->p4.i); + break; +} + /* Opcode: Not P1 P2 * * * ** Synopsis: r[P2]= !r[P1] ** ** Interpret the value in register P1 as a boolean value. Store the -** boolean complement in register P2. If the value in register P1 is +** boolean complement in register P2. If the value in register P1 is ** NULL, then a NULL is stored in P2. */ case OP_Not: { /* same as TK_NOT, in1, out2 */ pIn1 = &aMem[pOp->p1]; pOut = &aMem[pOp->p2]; - sqlite3VdbeMemSetNull(pOut); if( (pIn1->flags & MEM_Null)==0 ){ - pOut->flags = MEM_Int; - pOut->u.i = !sqlite3VdbeIntValue(pIn1); + sqlite3VdbeMemSetInt64(pOut, !sqlite3VdbeBooleanValue(pIn1,0)); + }else{ + sqlite3VdbeMemSetNull(pOut); } break; } /* Opcode: BitNot P1 P2 * * * -** Synopsis: r[P1]= ~r[P1] +** Synopsis: r[P2]= ~r[P1] ** ** Interpret the content of register P1 as an integer. Store the ** ones-complement of the P1 value into register P2. If P1 holds @@ -78948,23 +96039,39 @@ case OP_BitNot: { /* same as TK_BITNOT, in1, out2 */ /* Opcode: Once P1 P2 * * * ** -** Check the "once" flag number P1. If it is set, jump to instruction P2. -** Otherwise, set the flag and fall through to the next instruction. -** In other words, this opcode causes all following opcodes up through P2 -** (but not including P2) to run just once and to be skipped on subsequent -** times through the loop. +** Fall through to the next instruction the first time this opcode is +** encountered on each invocation of the byte-code program. Jump to P2 +** on the second and all subsequent encounters during the same invocation. ** -** All "once" flags are initially cleared whenever a prepared statement -** first begins to run. +** Top-level programs determine first invocation by comparing the P1 +** operand against the P1 operand on the OP_Init opcode at the beginning +** of the program. If the P1 values differ, then fall through and make +** the P1 of this opcode equal to the P1 of OP_Init. If P1 values are +** the same then take the jump. +** +** For subprograms, there is a bitmask in the VdbeFrame that determines +** whether or not the jump should be taken. The bitmask is necessary +** because the self-altering code trick does not work for recursive +** triggers. */ case OP_Once: { /* jump */ - assert( pOp->p1nOnceFlag ); - VdbeBranchTaken(p->aOnceFlag[pOp->p1]!=0, 2); - if( p->aOnceFlag[pOp->p1] ){ - goto jump_to_p2; + u32 iAddr; /* Address of this instruction */ + assert( p->aOp[0].opcode==OP_Init ); + if( p->pFrame ){ + iAddr = (int)(pOp - p->aOp); + if( (p->pFrame->aOnce[iAddr/8] & (1<<(iAddr & 7)))!=0 ){ + VdbeBranchTaken(1, 2); + goto jump_to_p2; + } + p->pFrame->aOnce[iAddr/8] |= 1<<(iAddr & 7); }else{ - p->aOnceFlag[pOp->p1] = 1; + if( p->aOp[0].p1==pOp->p1 ){ + VdbeBranchTaken(1, 2); + goto jump_to_p2; + } } + VdbeBranchTaken(0, 2); + pOp->p1 = p->aOp[0].p1; break; } @@ -78974,35 +96081,30 @@ case OP_Once: { /* jump */ ** is considered true if it is numeric and non-zero. If the value ** in P1 is NULL then take the jump if and only if P3 is non-zero. */ +case OP_If: { /* jump, in1 */ + int c; + c = sqlite3VdbeBooleanValue(&aMem[pOp->p1], pOp->p3); + VdbeBranchTaken(c!=0, 2); + if( c ) goto jump_to_p2; + break; +} + /* Opcode: IfNot P1 P2 P3 * * ** ** Jump to P2 if the value in register P1 is False. The value ** is considered false if it has a numeric value of zero. If the value ** in P1 is NULL then take the jump if and only if P3 is non-zero. */ -case OP_If: /* jump, in1 */ case OP_IfNot: { /* jump, in1 */ int c; - pIn1 = &aMem[pOp->p1]; - if( pIn1->flags & MEM_Null ){ - c = pOp->p3; - }else{ -#ifdef SQLITE_OMIT_FLOATING_POINT - c = sqlite3VdbeIntValue(pIn1)!=0; -#else - c = sqlite3VdbeRealValue(pIn1)!=0.0; -#endif - if( pOp->opcode==OP_IfNot ) c = !c; - } + c = !sqlite3VdbeBooleanValue(&aMem[pOp->p1], !pOp->p3); VdbeBranchTaken(c!=0, 2); - if( c ){ - goto jump_to_p2; - } + if( c ) goto jump_to_p2; break; } /* Opcode: IsNull P1 P2 * * * -** Synopsis: if r[P1]==NULL goto P2 +** Synopsis: if r[P1]==NULL goto P2 ** ** Jump to P2 if the value in register P1 is NULL. */ @@ -79015,10 +96117,121 @@ case OP_IsNull: { /* same as TK_ISNULL, jump, in1 */ break; } +/* Opcode: IsType P1 P2 P3 P4 P5 +** Synopsis: if typeof(P1.P3) in P5 goto P2 +** +** Jump to P2 if the type of a column in a btree is one of the types specified +** by the P5 bitmask. +** +** P1 is normally a cursor on a btree for which the row decode cache is +** valid through at least column P3. In other words, there should have been +** a prior OP_Column for column P3 or greater. If the cursor is not valid, +** then this opcode might give spurious results. +** The the btree row has fewer than P3 columns, then use P4 as the +** datatype. +** +** If P1 is -1, then P3 is a register number and the datatype is taken +** from the value in that register. +** +** P5 is a bitmask of data types. SQLITE_INTEGER is the least significant +** (0x01) bit. SQLITE_FLOAT is the 0x02 bit. SQLITE_TEXT is 0x04. +** SQLITE_BLOB is 0x08. SQLITE_NULL is 0x10. +** +** WARNING: This opcode does not reliably distinguish between NULL and REAL +** when P1>=0. If the database contains a NaN value, this opcode will think +** that the datatype is REAL when it should be NULL. When P1<0 and the value +** is already stored in register P3, then this opcode does reliably +** distinguish between NULL and REAL. The problem only arises then P1>=0. +** +** Take the jump to address P2 if and only if the datatype of the +** value determined by P1 and P3 corresponds to one of the bits in the +** P5 bitmask. +** +*/ +case OP_IsType: { /* jump */ + VdbeCursor *pC; + u16 typeMask; + u32 serialType; + + assert( pOp->p1>=(-1) && pOp->p1nCursor ); + assert( pOp->p1>=0 || (pOp->p3>=0 && pOp->p3<=(p->nMem+1 - p->nCursor)) ); + if( pOp->p1>=0 ){ + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=0 ); + if( pOp->p3nHdrParsed ){ + serialType = pC->aType[pOp->p3]; + if( serialType>=12 ){ + if( serialType&1 ){ + typeMask = 0x04; /* SQLITE_TEXT */ + }else{ + typeMask = 0x08; /* SQLITE_BLOB */ + } + }else{ + static const unsigned char aMask[] = { + 0x10, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x2, + 0x01, 0x01, 0x10, 0x10 + }; + testcase( serialType==0 ); + testcase( serialType==1 ); + testcase( serialType==2 ); + testcase( serialType==3 ); + testcase( serialType==4 ); + testcase( serialType==5 ); + testcase( serialType==6 ); + testcase( serialType==7 ); + testcase( serialType==8 ); + testcase( serialType==9 ); + testcase( serialType==10 ); + testcase( serialType==11 ); + typeMask = aMask[serialType]; + } + }else{ + typeMask = 1 << (pOp->p4.i - 1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + }else{ + assert( memIsValid(&aMem[pOp->p3]) ); + typeMask = 1 << (sqlite3_value_type((sqlite3_value*)&aMem[pOp->p3])-1); + testcase( typeMask==0x01 ); + testcase( typeMask==0x02 ); + testcase( typeMask==0x04 ); + testcase( typeMask==0x08 ); + testcase( typeMask==0x10 ); + } + VdbeBranchTaken( (typeMask & pOp->p5)!=0, 2); + if( typeMask & pOp->p5 ){ + goto jump_to_p2; + } + break; +} + +/* Opcode: ZeroOrNull P1 P2 P3 * * +** Synopsis: r[P2] = 0 OR NULL +** +** If both registers P1 and P3 are NOT NULL, then store a zero in +** register P2. If either registers P1 or P3 are NULL then put +** a NULL in register P2. +*/ +case OP_ZeroOrNull: { /* in1, in2, out2, in3 */ + if( (aMem[pOp->p1].flags & MEM_Null)!=0 + || (aMem[pOp->p3].flags & MEM_Null)!=0 + ){ + sqlite3VdbeMemSetNull(aMem + pOp->p2); + }else{ + sqlite3VdbeMemSetInt64(aMem + pOp->p2, 0); + } + break; +} + /* Opcode: NotNull P1 P2 * * * ** Synopsis: if r[P1]!=NULL goto P2 ** -** Jump to P2 if the value in register P1 is not NULL. +** Jump to P2 if the value in register P1 is not NULL. */ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ pIn1 = &aMem[pOp->p1]; @@ -79029,36 +96242,91 @@ case OP_NotNull: { /* same as TK_NOTNULL, jump, in1 */ break; } +/* Opcode: IfNullRow P1 P2 P3 * * +** Synopsis: if P1.nullRow then r[P3]=NULL, goto P2 +** +** Check the cursor P1 to see if it is currently pointing at a NULL row. +** If it is, then set register P3 to NULL and jump immediately to P2. +** If P1 is not on a NULL row, then fall through without making any +** changes. +** +** If P1 is not an open cursor, then this opcode is a no-op. +*/ +case OP_IfNullRow: { /* jump */ + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + if( pC && pC->nullRow ){ + sqlite3VdbeMemSetNull(aMem + pOp->p3); + goto jump_to_p2; + } + break; +} + +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC +/* Opcode: Offset P1 P2 P3 * * +** Synopsis: r[P3] = sqlite_offset(P1) +** +** Store in register r[P3] the byte offset into the database file that is the +** start of the payload for the record at which that cursor P1 is currently +** pointing. +** +** P2 is the column number for the argument to the sqlite_offset() function. +** This opcode does not use P2 itself, but the P2 value is used by the +** code generator. The P1, P2, and P3 operands to this opcode are the +** same as for OP_Column. +** +** This opcode is only available if SQLite is compiled with the +** -DSQLITE_ENABLE_OFFSET_SQL_FUNC option. +*/ +case OP_Offset: { /* out3 */ + VdbeCursor *pC; /* The VDBE cursor */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + pOut = &p->aMem[pOp->p3]; + if( pC==0 || pC->eCurType!=CURTYPE_BTREE ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + if( pC->deferredMoveto ){ + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + } + if( sqlite3BtreeEof(pC->uc.pCursor) ){ + sqlite3VdbeMemSetNull(pOut); + }else{ + sqlite3VdbeMemSetInt64(pOut, sqlite3BtreeOffset(pC->uc.pCursor)); + } + } + break; +} +#endif /* SQLITE_ENABLE_OFFSET_SQL_FUNC */ + /* Opcode: Column P1 P2 P3 P4 P5 -** Synopsis: r[P3]=PX +** Synopsis: r[P3]=PX cursor P1 column P2 ** ** Interpret the data that cursor P1 points to as a structure built using ** the MakeRecord instruction. (See the MakeRecord opcode for additional ** information about the format of the data.) Extract the P2-th column -** from this record. If there are less that (P2+1) +** from this record. If there are less than (P2+1) ** values in the record, extract a NULL. ** ** The value extracted is stored in register P3. ** -** If the column contains fewer than P2 fields, then extract a NULL. Or, +** If the record contains fewer than P2 fields, then extract a NULL. Or, ** if the P4 argument is a P4_MEM use the value of the P4 argument as ** the result. ** -** If the OPFLAG_CLEARCACHE bit is set on P5 and P1 is a pseudo-table cursor, -** then the cache of the cursor is reset prior to extracting the column. -** The first OP_Column against a pseudo-table after the value of the content -** register has changed should have this bit set. -** -** If the OPFLAG_LENGTHARG and OPFLAG_TYPEOFARG bits are set on P5 when -** the result is guaranteed to only be used as the argument of a length() -** or typeof() function, respectively. The loading of large blobs can be -** skipped for length() and all content loading can be skipped for typeof(). +** If the OPFLAG_LENGTHARG bit is set in P5 then the result is guaranteed +** to only be used by the length() function or the equivalent. The content +** of large blobs is not loaded, thus saving CPU cycles. If the +** OPFLAG_TYPEOFARG bit is set then the result will only be used by the +** typeof() function or the IS NULL or IS NOT NULL operators or the +** equivalent. In this case, all content loading can be omitted. */ -case OP_Column: { - i64 payloadSize64; /* Number of bytes in the record */ - int p2; /* column number to retrieve */ +case OP_Column: { /* ncycle */ + u32 p2; /* column number to retrieve */ VdbeCursor *pC; /* The VDBE cursor */ - BtCursor *pCrsr; /* The BTree cursor */ + BtCursor *pCrsr; /* The B-Tree cursor corresponding to pC */ u32 *aOffset; /* aOffset[i] is offset to start of data for i-th column */ int len; /* The length of the serialized data for the column */ int i; /* Loop counter */ @@ -79067,79 +96335,75 @@ case OP_Column: { const u8 *zData; /* Part of the record being decoded */ const u8 *zHdr; /* Next unparsed byte of the header */ const u8 *zEndHdr; /* Pointer to first byte after the header */ - u32 offset; /* Offset into the data */ u64 offset64; /* 64-bit offset */ - u32 avail; /* Number of bytes of available data */ u32 t; /* A type code from the record header */ Mem *pReg; /* PseudoTable input register */ - pC = p->apCsr[pOp->p1]; - p2 = pOp->p2; - - /* If the cursor cache is stale, bring it up-to-date */ - rc = sqlite3VdbeCursorMoveto(&pC, &p2); - - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pDest = &aMem[pOp->p3]; - memAboutToChange(p, pDest); assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + pC = p->apCsr[pOp->p1]; + p2 = (u32)pOp->p2; + +op_column_restart: assert( pC!=0 ); - assert( p2nField ); + assert( p2<(u32)pC->nField + || (pC->eCurType==CURTYPE_PSEUDO && pC->seekResult==0) ); aOffset = pC->aOffset; + assert( aOffset==pC->aType+pC->nField ); assert( pC->eCurType!=CURTYPE_VTAB ); assert( pC->eCurType!=CURTYPE_PSEUDO || pC->nullRow ); assert( pC->eCurType!=CURTYPE_SORTER ); - pCrsr = pC->uc.pCursor; - if( rc ) goto abort_due_to_error; - if( pC->cacheStatus!=p->cacheCtr ){ + if( pC->cacheStatus!=p->cacheCtr ){ /*OPTIMIZATION-IF-FALSE*/ if( pC->nullRow ){ - if( pC->eCurType==CURTYPE_PSEUDO ){ - assert( pC->uc.pseudoTableReg>0 ); - pReg = &aMem[pC->uc.pseudoTableReg]; + if( pC->eCurType==CURTYPE_PSEUDO && pC->seekResult>0 ){ + /* For the special case of as pseudo-cursor, the seekResult field + ** identifies the register that holds the record */ + pReg = &aMem[pC->seekResult]; assert( pReg->flags & MEM_Blob ); assert( memIsValid(pReg) ); - pC->payloadSize = pC->szRow = avail = pReg->n; + pC->payloadSize = pC->szRow = pReg->n; pC->aRow = (u8*)pReg->z; }else{ + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); sqlite3VdbeMemSetNull(pDest); goto op_column_out; } }else{ + pCrsr = pC->uc.pCursor; + if( pC->deferredMoveto ){ + u32 iMap; + assert( !pC->isEphemeral ); + if( pC->ub.aAltMap && (iMap = pC->ub.aAltMap[1+p2])>0 ){ + pC = pC->pAltCursor; + p2 = iMap - 1; + goto op_column_restart; + } + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + }else if( sqlite3BtreeCursorHasMoved(pCrsr) ){ + rc = sqlite3VdbeHandleMovedCursor(pC); + if( rc ) goto abort_due_to_error; + goto op_column_restart; + } assert( pC->eCurType==CURTYPE_BTREE ); assert( pCrsr ); - if( pC->isTable==0 ){ - assert( sqlite3BtreeCursorIsValid(pCrsr) ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &payloadSize64); - assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ - /* sqlite3BtreeParseCellPtr() uses getVarint32() to extract the - ** payload size, so it is impossible for payloadSize64 to be - ** larger than 32 bits. */ - assert( (payloadSize64 & SQLITE_MAX_U32)==(u64)payloadSize64 ); - pC->aRow = sqlite3BtreeKeyFetch(pCrsr, &avail); - pC->payloadSize = (u32)payloadSize64; - }else{ - assert( sqlite3BtreeCursorIsValid(pCrsr) ); - VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &pC->payloadSize); - assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ - pC->aRow = sqlite3BtreeDataFetch(pCrsr, &avail); - } - assert( avail<=65536 ); /* Maximum page size is 64KiB */ - if( pC->payloadSize <= (u32)avail ){ - pC->szRow = pC->payloadSize; - }else if( pC->payloadSize > (u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - }else{ - pC->szRow = avail; - } + assert( sqlite3BtreeCursorIsValid(pCrsr) ); + pC->payloadSize = sqlite3BtreePayloadSize(pCrsr); + pC->aRow = sqlite3BtreePayloadFetch(pCrsr, &pC->szRow); + assert( pC->szRow<=pC->payloadSize ); + assert( pC->szRow<=65536 ); /* Maximum page size is 64KiB */ } pC->cacheStatus = p->cacheCtr; - pC->iHdrOffset = getVarint32(pC->aRow, offset); + if( (aOffset[0] = pC->aRow[0])<0x80 ){ + pC->iHdrOffset = 1; + }else{ + pC->iHdrOffset = sqlite3GetVarint32(pC->aRow, aOffset); + } pC->nHdrParsed = 0; - aOffset[0] = offset; - - if( availszRowaRow does not have to hold the entire row, but it does at least ** need to cover the header of the record. If pC->aRow does not contain ** the complete header, then set it to zero, forcing the header to be @@ -79156,18 +96420,32 @@ case OP_Column: { ** 3-byte type for each of the maximum of 32768 columns plus three ** extra bytes for the header length itself. 32768*3 + 3 = 98307. */ - if( offset > 98307 || offset > pC->payloadSize ){ - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; + if( aOffset[0] > 98307 || aOffset[0] > pC->payloadSize ){ + goto op_column_corrupt; } + }else{ + /* This is an optimization. By skipping over the first few tests + ** (ex: pC->nHdrParsed<=p2) in the next section, we achieve a + ** measurable performance gain. + ** + ** This branch is taken even if aOffset[0]==0. Such a record is never + ** generated by SQLite, and could be considered corruption, but we + ** accept it for historical reasons. When aOffset[0]==0, the code this + ** branch jumps to reads past the end of the record, but never more + ** than a few bytes. Even if the record occurs at the end of the page + ** content area, the "page header" comes after the page content and so + ** this overread is harmless. Similar overreads can occur for a corrupt + ** database file. + */ + zData = pC->aRow; + assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ + testcase( aOffset[0]==0 ); + goto op_column_read_header; } - - /* The following goto is an optimization. It can be omitted and - ** everything will still work. But OP_Column is measurably faster - ** by skipping the subsequent conditional, which is always true. - */ - assert( pC->nHdrParsed<=p2 ); /* Conditional skipped */ - goto op_column_read_header; + }else if( sqlite3BtreeCursorHasMoved(pC->uc.pCursor) ){ + rc = sqlite3VdbeHandleMovedCursor(pC); + if( rc ) goto abort_due_to_error; + goto op_column_restart; } /* Make sure at least the first p2+1 entries of the header have been @@ -79175,40 +96453,38 @@ case OP_Column: { */ if( pC->nHdrParsed<=p2 ){ /* If there is more header available for parsing in the record, try - ** to extract additional fields up through the p2+1-th field + ** to extract additional fields up through the p2+1-th field */ - op_column_read_header: if( pC->iHdrOffsetaRow==0 ){ memset(&sMem, 0, sizeof(sMem)); - rc = sqlite3VdbeMemFromBtree(pCrsr, 0, aOffset[0], !pC->isTable, &sMem); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pC->uc.pCursor,aOffset[0],&sMem); if( rc!=SQLITE_OK ) goto abort_due_to_error; zData = (u8*)sMem.z; }else{ zData = pC->aRow; } - + /* Fill in pC->aType[i] and aOffset[i] values through the p2-th field. */ + op_column_read_header: i = pC->nHdrParsed; offset64 = aOffset[i]; zHdr = zData + pC->iHdrOffset; zEndHdr = zData + aOffset[0]; - assert( i<=p2 && zHdr=zEndHdr ); do{ - if( (t = zHdr[0])<0x80 ){ + if( (pC->aType[i] = t = zHdr[0])<0x80 ){ zHdr++; offset64 += sqlite3VdbeOneByteSerialTypeLen(t); }else{ zHdr += sqlite3GetVarint32(zHdr, &t); + pC->aType[i] = t; offset64 += sqlite3VdbeSerialTypeLen(t); } - pC->aType[i++] = t; - aOffset[i] = (u32)(offset64 & 0xffffffff); - }while( i<=p2 && zHdrnHdrParsed = i; - pC->iHdrOffset = (u32)(zHdr - zData); - + aOffset[++i] = (u32)(offset64 & 0xffffffff); + }while( (u32)i<=p2 && zHdr=zEndHdr && (zHdr>zEndHdr || offset64!=pC->payloadSize)) || (offset64 > pC->payloadSize) ){ - if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); - rc = SQLITE_CORRUPT_BKPT; - goto abort_due_to_error; + if( aOffset[0]==0 ){ + i = 0; + zHdr = zEndHdr; + }else{ + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + goto op_column_corrupt; + } } - if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); + pC->nHdrParsed = i; + pC->iHdrOffset = (u32)(zHdr - zData); + if( pC->aRow==0 ) sqlite3VdbeMemRelease(&sMem); }else{ t = 0; } @@ -79232,6 +96514,8 @@ case OP_Column: { ** columns. So the result will be either the default value or a NULL. */ if( pC->nHdrParsed<=p2 ){ + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); if( pOp->p4type==P4_MEM ){ sqlite3VdbeMemShallowCopy(pDest, pOp->p4.pMem, MEM_Static); }else{ @@ -79249,10 +96533,13 @@ case OP_Column: { */ assert( p2nHdrParsed ); assert( rc==SQLITE_OK ); + pDest = &aMem[pOp->p3]; + memAboutToChange(p, pDest); assert( sqlite3VdbeCheckMemInvariants(pDest) ); - if( VdbeMemDynamic(pDest) ) sqlite3VdbeMemSetNull(pDest); + if( VdbeMemDynamic(pDest) ){ + sqlite3VdbeMemSetNull(pDest); + } assert( t==pC->aType[p2] ); - pDest->enc = encoding; if( pC->szRow>=aOffset[p2+1] ){ /* This is the common case where the desired content fits on the original ** page - where the content is not on an overflow page */ @@ -79266,7 +96553,9 @@ case OP_Column: { */ static const u16 aFlag[] = { MEM_Blob, MEM_Str|MEM_Term }; pDest->n = len = (t-12)/2; + pDest->enc = encoding; if( pDest->szMalloc < len+2 ){ + if( len>db->aLimit[SQLITE_LIMIT_LENGTH] ) goto too_big; pDest->flags = MEM_Null; if( sqlite3VdbeMemGrow(pDest, len+2, 0) ) goto no_mem; }else{ @@ -79278,25 +96567,39 @@ case OP_Column: { pDest->flags = aFlag[t&1]; } }else{ + u8 p5; + pDest->enc = encoding; + assert( pDest->db==db ); /* This branch happens only when content is on overflow pages */ - if( ((pOp->p5 & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG))!=0 - && ((t>=12 && (t&1)==0) || (pOp->p5 & OPFLAG_TYPEOFARG)!=0)) - || (len = sqlite3VdbeSerialTypeLen(t))==0 + if( ((p5 = (pOp->p5 & OPFLAG_BYTELENARG))!=0 + && (p5==OPFLAG_TYPEOFARG + || (t>=12 && ((t&1)==0 || p5==OPFLAG_BYTELENARG)) + ) + ) + || sqlite3VdbeSerialTypeLen(t)==0 ){ /* Content is irrelevant for ** 1. the typeof() function, ** 2. the length(X) function if X is a blob, and ** 3. if the content length is zero. ** So we might as well use bogus content rather than reading - ** content from disk. */ - static u8 aZero[8]; /* This is the bogus content */ - sqlite3VdbeSerialGet(aZero, t, pDest); + ** content from disk. + ** + ** Although sqlite3VdbeSerialGet() may read at most 8 bytes from the + ** buffer passed to it, debugging function VdbeMemPrettyPrint() may + ** read more. Use the global constant sqlite3CtypeMap[] as the array, + ** as that array is 256 bytes long (plenty for VdbeMemPrettyPrint()) + ** and it begins with a bunch of zeros. + */ + sqlite3VdbeSerialGet((u8*)sqlite3CtypeMap, t, pDest); }else{ - rc = sqlite3VdbeMemFromBtree(pCrsr, aOffset[p2], len, !pC->isTable, - pDest); - if( rc!=SQLITE_OK ) goto abort_due_to_error; - sqlite3VdbeSerialGet((const u8*)pDest->z, t, pDest); - pDest->flags &= ~MEM_Ephem; + rc = vdbeColumnFromOverflow(pC, p2, t, aOffset[p2], + p->cacheCtr, colCacheCtr, pDest); + if( rc ){ + if( rc==SQLITE_NOMEM ) goto no_mem; + if( rc==SQLITE_TOOBIG ) goto too_big; + goto abort_due_to_error; + } } } @@ -79304,6 +96607,119 @@ op_column_out: UPDATE_MAX_BLOBSIZE(pDest); REGISTER_TRACE(pOp->p3, pDest); break; + +op_column_corrupt: + if( aOp[0].p3>0 ){ + pOp = &aOp[aOp[0].p3-1]; + break; + }else{ + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } +} + +/* Opcode: TypeCheck P1 P2 P3 P4 * +** Synopsis: typecheck(r[P1@P2]) +** +** Apply affinities to the range of P2 registers beginning with P1. +** Take the affinities from the Table object in P4. If any value +** cannot be coerced into the correct type, then raise an error. +** +** This opcode is similar to OP_Affinity except that this opcode +** forces the register type to the Table column type. This is used +** to implement "strict affinity". +** +** GENERATED ALWAYS AS ... STATIC columns are only checked if P3 +** is zero. When P3 is non-zero, no type checking occurs for +** static generated columns. Virtual columns are computed at query time +** and so they are never checked. +** +** Preconditions: +** +**
              +**
            • P2 should be the number of non-virtual columns in the +** table of P4. +**
            • Table P4 should be a STRICT table. +**
            +** +** If any precondition is false, an assertion fault occurs. +*/ +case OP_TypeCheck: { + Table *pTab; + Column *aCol; + int i; + + assert( pOp->p4type==P4_TABLE ); + pTab = pOp->p4.pTab; + assert( pTab->tabFlags & TF_Strict ); + assert( pTab->nNVCol==pOp->p2 ); + aCol = pTab->aCol; + pIn1 = &aMem[pOp->p1]; + for(i=0; inCol; i++){ + if( aCol[i].colFlags & COLFLAG_GENERATED ){ + if( aCol[i].colFlags & COLFLAG_VIRTUAL ) continue; + if( pOp->p3 ){ pIn1++; continue; } + } + assert( pIn1 < &aMem[pOp->p1+pOp->p2] ); + applyAffinity(pIn1, aCol[i].affinity, encoding); + if( (pIn1->flags & MEM_Null)==0 ){ + switch( aCol[i].eCType ){ + case COLTYPE_BLOB: { + if( (pIn1->flags & MEM_Blob)==0 ) goto vdbe_type_error; + break; + } + case COLTYPE_INTEGER: + case COLTYPE_INT: { + if( (pIn1->flags & MEM_Int)==0 ) goto vdbe_type_error; + break; + } + case COLTYPE_TEXT: { + if( (pIn1->flags & MEM_Str)==0 ) goto vdbe_type_error; + break; + } + case COLTYPE_REAL: { + testcase( (pIn1->flags & (MEM_Real|MEM_IntReal))==MEM_Real ); + assert( (pIn1->flags & MEM_IntReal)==0 ); + if( pIn1->flags & MEM_Int ){ + /* When applying REAL affinity, if the result is still an MEM_Int + ** that will fit in 6 bytes, then change the type to MEM_IntReal + ** so that we keep the high-resolution integer value but know that + ** the type really wants to be REAL. */ + testcase( pIn1->u.i==140737488355328LL ); + testcase( pIn1->u.i==140737488355327LL ); + testcase( pIn1->u.i==-140737488355328LL ); + testcase( pIn1->u.i==-140737488355329LL ); + if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL){ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + }else{ + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~MEM_Int; + } + }else if( (pIn1->flags & (MEM_Real|MEM_IntReal))==0 ){ + goto vdbe_type_error; + } + break; + } + default: { + /* COLTYPE_ANY. Accept anything. */ + break; + } + } + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + pIn1++; + } + assert( pIn1 == &aMem[pOp->p1+pOp->p2] ); + break; + +vdbe_type_error: + sqlite3VdbeError(p, "cannot store %s value in %s column %s.%s", + vdbeMemTypeName(pIn1), sqlite3StdType[aCol[i].eCType-1], + pTab->zName, aCol[i].zCnName); + rc = SQLITE_CONSTRAINT_DATATYPE; + goto abort_due_to_error; } /* Opcode: Affinity P1 P2 * P4 * @@ -79311,22 +96727,43 @@ op_column_out: ** ** Apply affinities to a range of P2 registers starting with P1. ** -** P4 is a string that is P2 characters long. The nth character of the -** string indicates the column affinity that should be used for the nth +** P4 is a string that is P2 characters long. The N-th character of the +** string indicates the column affinity that should be used for the N-th ** memory cell in the range. */ case OP_Affinity: { const char *zAffinity; /* The affinity to be applied */ - char cAff; /* A single character of affinity */ zAffinity = pOp->p4.z; assert( zAffinity!=0 ); + assert( pOp->p2>0 ); assert( zAffinity[pOp->p2]==0 ); pIn1 = &aMem[pOp->p1]; - while( (cAff = *(zAffinity++))!=0 ){ + while( 1 /*exit-by-break*/ ){ assert( pIn1 <= &p->aMem[(p->nMem+1 - p->nCursor)] ); - assert( memIsValid(pIn1) ); - applyAffinity(pIn1, cAff, encoding); + assert( zAffinity[0]==SQLITE_AFF_NONE || memIsValid(pIn1) ); + applyAffinity(pIn1, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pIn1->flags & MEM_Int)!=0 ){ + /* When applying REAL affinity, if the result is still an MEM_Int + ** that will fit in 6 bytes, then change the type to MEM_IntReal + ** so that we keep the high-resolution integer value but know that + ** the type really wants to be REAL. */ + testcase( pIn1->u.i==140737488355328LL ); + testcase( pIn1->u.i==140737488355327LL ); + testcase( pIn1->u.i==-140737488355328LL ); + testcase( pIn1->u.i==-140737488355329LL ); + if( pIn1->u.i<=140737488355327LL && pIn1->u.i>=-140737488355328LL ){ + pIn1->flags |= MEM_IntReal; + pIn1->flags &= ~MEM_Int; + }else{ + pIn1->u.r = (double)pIn1->u.i; + pIn1->flags |= MEM_Real; + pIn1->flags &= ~(MEM_Int|MEM_Str); + } + } + REGISTER_TRACE((int)(pIn1-aMem), pIn1); + zAffinity++; + if( zAffinity[0]==0 ) break; pIn1++; } break; @@ -79339,17 +96776,27 @@ case OP_Affinity: { ** use as a data record in a database table or as a key ** in an index. The OP_Column opcode can decode the record later. ** -** P4 may be a string that is P2 characters long. The nth character of the -** string indicates the column affinity that should be used for the nth +** P4 may be a string that is P2 characters long. The N-th character of the +** string indicates the column affinity that should be used for the N-th ** field of the index key. ** ** The mapping from character to affinity is given by the SQLITE_AFF_ ** macros defined in sqliteInt.h. ** ** If P4 is NULL then all index fields have the affinity BLOB. +** +** The meaning of P5 depends on whether or not the SQLITE_ENABLE_NULL_TRIM +** compile-time option is enabled: +** +** * If SQLITE_ENABLE_NULL_TRIM is enabled, then the P5 is the index +** of the right-most table that can be null-trimmed. +** +** * If SQLITE_ENABLE_NULL_TRIM is omitted, then P5 has the value +** OPFLAG_NOCHNG_MAGIC if the OP_MakeRecord opcode is allowed to +** accept no-change records with serial_type 10. This value is +** only used inside an assert() and does not affect the end result. */ case OP_MakeRecord: { - u8 *zNewRecord; /* A buffer to hold the data for the new record */ Mem *pRec; /* The new record */ u64 nData; /* Number of bytes of data space */ int nHdr; /* Number of bytes of header space */ @@ -79361,22 +96808,21 @@ case OP_MakeRecord: { Mem *pLast; /* Last field of the record */ int nField; /* Number of fields in the record */ char *zAffinity; /* The affinity string for the record */ - int file_format; /* File format to use for encoding */ - int i; /* Space used in zNewRecord[] header */ - int j; /* Space used in zNewRecord[] content */ u32 len; /* Length of a field */ + u8 *zHdr; /* Where to write next byte of the header */ + u8 *zPayload; /* Where to write next byte of the payload */ /* Assuming the record contains N fields, the record format looks ** like this: ** ** ------------------------------------------------------------------------ - ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | + ** | hdr-size | type 0 | type 1 | ... | type N-1 | data0 | ... | data N-1 | ** ------------------------------------------------------------------------ ** ** Data(0) is taken from register P1. Data(1) comes from register P1+1 ** and so forth. ** - ** Each type field is a varint representing the serial type of the + ** Each type field is a varint representing the serial type of the ** corresponding data element (see sqlite3VdbeSerialType()). The ** hdr-size field is also a varint which is the offset from the beginning ** of the record to data0. @@ -79390,7 +96836,6 @@ case OP_MakeRecord: { pData0 = &aMem[nField]; nField = pOp->p2; pLast = &pData0[nField-1]; - file_format = p->minWriteFileFormat; /* Identify the output register */ assert( pOp->p3p1 || pOp->p3>=pOp->p1+pOp->p2 ); @@ -79403,30 +96848,149 @@ case OP_MakeRecord: { if( zAffinity ){ pRec = pData0; do{ - applyAffinity(pRec++, *(zAffinity++), encoding); + applyAffinity(pRec, zAffinity[0], encoding); + if( zAffinity[0]==SQLITE_AFF_REAL && (pRec->flags & MEM_Int) ){ + pRec->flags |= MEM_IntReal; + pRec->flags &= ~(MEM_Int); + } + REGISTER_TRACE((int)(pRec-aMem), pRec); + zAffinity++; + pRec++; assert( zAffinity[0]==0 || pRec<=pLast ); }while( zAffinity[0] ); } +#ifdef SQLITE_ENABLE_NULL_TRIM + /* NULLs can be safely trimmed from the end of the record, as long as + ** as the schema format is 2 or more and none of the omitted columns + ** have a non-NULL default value. Also, the record must be left with + ** at least one field. If P5>0 then it will be one more than the + ** index of the right-most column with a non-NULL default value */ + if( pOp->p5 ){ + while( (pLast->flags & MEM_Null)!=0 && nField>pOp->p5 ){ + pLast--; + nField--; + } + } +#endif + /* Loop through the elements that will make up the record to figure - ** out how much space is required for the new record. + ** out how much space is required for the new record. After this loop, + ** the Mem.uTemp field of each term should hold the serial-type that will + ** be used for that term in the generated record: + ** + ** Mem.uTemp value type + ** --------------- --------------- + ** 0 NULL + ** 1 1-byte signed integer + ** 2 2-byte signed integer + ** 3 3-byte signed integer + ** 4 4-byte signed integer + ** 5 6-byte signed integer + ** 6 8-byte signed integer + ** 7 IEEE float + ** 8 Integer constant 0 + ** 9 Integer constant 1 + ** 10,11 reserved for expansion + ** N>=12 and even BLOB + ** N>=13 and odd text + ** + ** The following additional values are computed: + ** nHdr Number of bytes needed for the record header + ** nData Number of bytes of data space needed for the record + ** nZero Zero bytes at the end of the record */ pRec = pLast; do{ assert( memIsValid(pRec) ); - pRec->uTemp = serial_type = sqlite3VdbeSerialType(pRec, file_format, &len); - if( pRec->flags & MEM_Zero ){ - if( nData ){ - if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + if( pRec->flags & MEM_Null ){ + if( pRec->flags & MEM_Zero ){ + /* Values with MEM_Null and MEM_Zero are created by xColumn virtual + ** table methods that never invoke sqlite3_result_xxxxx() while + ** computing an unchanging column value in an UPDATE statement. + ** Give such values a special internal-use-only serial-type of 10 + ** so that they can be passed through to xUpdate and have + ** a true sqlite3_value_nochange(). */ +#ifndef SQLITE_ENABLE_NULL_TRIM + assert( pOp->p5==OPFLAG_NOCHNG_MAGIC || CORRUPT_DB ); +#endif + pRec->uTemp = 10; }else{ - nZero += pRec->u.nZero; - len -= pRec->u.nZero; + pRec->uTemp = 0; } + nHdr++; + }else if( pRec->flags & (MEM_Int|MEM_IntReal) ){ + /* Figure out whether to use 1, 2, 4, 6 or 8 bytes. */ + i64 i = pRec->u.i; + u64 uu; + testcase( pRec->flags & MEM_Int ); + testcase( pRec->flags & MEM_IntReal ); + if( i<0 ){ + uu = ~i; + }else{ + uu = i; + } + nHdr++; + testcase( uu==127 ); testcase( uu==128 ); + testcase( uu==32767 ); testcase( uu==32768 ); + testcase( uu==8388607 ); testcase( uu==8388608 ); + testcase( uu==2147483647 ); testcase( uu==2147483648LL ); + testcase( uu==140737488355327LL ); testcase( uu==140737488355328LL ); + if( uu<=127 ){ + if( (i&1)==i && p->minWriteFileFormat>=4 ){ + pRec->uTemp = 8+(u32)uu; + }else{ + nData++; + pRec->uTemp = 1; + } + }else if( uu<=32767 ){ + nData += 2; + pRec->uTemp = 2; + }else if( uu<=8388607 ){ + nData += 3; + pRec->uTemp = 3; + }else if( uu<=2147483647 ){ + nData += 4; + pRec->uTemp = 4; + }else if( uu<=140737488355327LL ){ + nData += 6; + pRec->uTemp = 5; + }else{ + nData += 8; + if( pRec->flags & MEM_IntReal ){ + /* If the value is IntReal and is going to take up 8 bytes to store + ** as an integer, then we might as well make it an 8-byte floating + ** point value */ + pRec->u.r = (double)pRec->u.i; + pRec->flags &= ~MEM_IntReal; + pRec->flags |= MEM_Real; + pRec->uTemp = 7; + }else{ + pRec->uTemp = 6; + } + } + }else if( pRec->flags & MEM_Real ){ + nHdr++; + nData += 8; + pRec->uTemp = 7; + }else{ + assert( db->mallocFailed || pRec->flags&(MEM_Str|MEM_Blob) ); + assert( pRec->n>=0 ); + len = (u32)pRec->n; + serial_type = (len*2) + 12 + ((pRec->flags & MEM_Str)!=0); + if( pRec->flags & MEM_Zero ){ + serial_type += pRec->u.nZero*2; + if( nData ){ + if( sqlite3VdbeMemExpandBlob(pRec) ) goto no_mem; + len += pRec->u.nZero; + }else{ + nZero += pRec->u.nZero; + } + } + nData += len; + nHdr += sqlite3VarintLen(serial_type); + pRec->uTemp = serial_type; } - nData += len; - testcase( serial_type==127 ); - testcase( serial_type==128 ); - nHdr += serial_type<=127 ? 1 : sqlite3VarintLen(serial_type); if( pRec==pData0 ) break; pRec--; }while(1); @@ -79447,57 +97011,118 @@ case OP_MakeRecord: { if( nVarintdb->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } - /* Make sure the output register has a buffer large enough to store + /* Make sure the output register has a buffer large enough to store ** the new record. The output register (pOp->p3) is not allowed to ** be one of the input registers (because the following call to ** sqlite3VdbeMemClearAndResize() could clobber the value before it is used). */ - if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ - goto no_mem; + if( nByte+nZero<=pOut->szMalloc ){ + /* The output register is already large enough to hold the record. + ** No error checks or buffer enlargement is required */ + pOut->z = pOut->zMalloc; + }else{ + /* Need to make sure that the output is not too big and then enlarge + ** the output register to hold the full result */ + if( nByte+nZero>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; + } + if( sqlite3VdbeMemClearAndResize(pOut, (int)nByte) ){ + goto no_mem; + } } - zNewRecord = (u8 *)pOut->z; - - /* Write the record */ - i = putVarint32(zNewRecord, nHdr); - j = nHdr; - assert( pData0<=pLast ); - pRec = pData0; - do{ - serial_type = pRec->uTemp; - /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more - ** additional varints, one per column. */ - i += putVarint32(&zNewRecord[i], serial_type); /* serial type */ - /* EVIDENCE-OF: R-64536-51728 The values for each column in the record - ** immediately follow the header. */ - j += sqlite3VdbeSerialPut(&zNewRecord[j], pRec, serial_type); /* content */ - }while( (++pRec)<=pLast ); - assert( i==nHdr ); - assert( j==nByte ); - - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pOut->n = (int)nByte; pOut->flags = MEM_Blob; if( nZero ){ pOut->u.nZero = nZero; pOut->flags |= MEM_Zero; } - pOut->enc = SQLITE_UTF8; /* In case the blob is ever converted to text */ - REGISTER_TRACE(pOp->p3, pOut); UPDATE_MAX_BLOBSIZE(pOut); + zHdr = (u8 *)pOut->z; + zPayload = zHdr + nHdr; + + /* Write the record */ + if( nHdr<0x80 ){ + *(zHdr++) = nHdr; + }else{ + zHdr += sqlite3PutVarint(zHdr,nHdr); + } + assert( pData0<=pLast ); + pRec = pData0; + while( 1 /*exit-by-break*/ ){ + serial_type = pRec->uTemp; + /* EVIDENCE-OF: R-06529-47362 Following the size varint are one or more + ** additional varints, one per column. + ** EVIDENCE-OF: R-64536-51728 The values for each column in the record + ** immediately follow the header. */ + if( serial_type<=7 ){ + *(zHdr++) = serial_type; + if( serial_type==0 ){ + /* NULL value. No change in zPayload */ + }else{ + u64 v; + if( serial_type==7 ){ + assert( sizeof(v)==sizeof(pRec->u.r) ); + memcpy(&v, &pRec->u.r, sizeof(v)); + swapMixedEndianFloat(v); + }else{ + v = pRec->u.i; + } + len = sqlite3SmallTypeSizes[serial_type]; + assert( len>=1 && len<=8 && len!=5 && len!=7 ); + switch( len ){ + default: zPayload[7] = (u8)(v&0xff); v >>= 8; + zPayload[6] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 6: zPayload[5] = (u8)(v&0xff); v >>= 8; + zPayload[4] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 4: zPayload[3] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 3: zPayload[2] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 2: zPayload[1] = (u8)(v&0xff); v >>= 8; + /* no break */ deliberate_fall_through + case 1: zPayload[0] = (u8)(v&0xff); + } + zPayload += len; + } + }else if( serial_type<0x80 ){ + *(zHdr++) = serial_type; + if( serial_type>=14 && pRec->n>0 ){ + assert( pRec->z!=0 ); + memcpy(zPayload, pRec->z, pRec->n); + zPayload += pRec->n; + } + }else{ + zHdr += sqlite3PutVarint(zHdr, serial_type); + if( pRec->n ){ + assert( pRec->z!=0 ); + memcpy(zPayload, pRec->z, pRec->n); + zPayload += pRec->n; + } + } + if( pRec==pLast ) break; + pRec++; + } + assert( nHdr==(int)(zHdr - (u8*)pOut->z) ); + assert( nByte==(int)(zPayload - (u8*)pOut->z) ); + + assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); + REGISTER_TRACE(pOp->p3, pOut); break; } -/* Opcode: Count P1 P2 * * * +/* Opcode: Count P1 P2 P3 * * ** Synopsis: r[P2]=count() ** -** Store the number of entries (an integer value) in the table or index -** opened by cursor P1 in register P2 +** Store the number of entries (an integer value) in the table or index +** opened by cursor P1 in register P2. +** +** If P3==0, then an exact count is obtained, which involves visiting +** every btree page of the table. But if P3 is non-zero, an estimate +** is returned based on the current cursor position. */ -#ifndef SQLITE_OMIT_BTREECOUNT case OP_Count: { /* out2 */ i64 nEntry; BtCursor *pCrsr; @@ -79505,20 +97130,24 @@ case OP_Count: { /* out2 */ assert( p->apCsr[pOp->p1]->eCurType==CURTYPE_BTREE ); pCrsr = p->apCsr[pOp->p1]->uc.pCursor; assert( pCrsr ); - nEntry = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3BtreeCount(pCrsr, &nEntry); - if( rc ) goto abort_due_to_error; + if( pOp->p3 ){ + nEntry = sqlite3BtreeRowCountEst(pCrsr); + }else{ + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(db, pCrsr, &nEntry); + if( rc ) goto abort_due_to_error; + } pOut = out2Prerelease(p, pOp); pOut->u.i = nEntry; - break; + goto check_for_interrupt; } -#endif /* Opcode: Savepoint P1 * * P4 * ** ** Open, release or rollback the savepoint named by parameter P4, depending -** on the value of P1. To open a new savepoint, P1==0. To release (commit) an -** existing savepoint, P1==1, or to rollback an existing savepoint P1==2. +** on the value of P1. To open a new savepoint set P1==0 (SAVEPOINT_BEGIN). +** To release (commit) an existing savepoint set P1==1 (SAVEPOINT_RELEASE). +** To rollback an existing savepoint set P1==2 (SAVEPOINT_ROLLBACK). */ case OP_Savepoint: { int p1; /* Value of P1 operand */ @@ -79534,7 +97163,7 @@ case OP_Savepoint: { zName = pOp->p4.z; /* Assert that the p1 parameter is valid. Also that if there is no open - ** transaction, then there cannot be any savepoints. + ** transaction, then there cannot be any savepoints. */ assert( db->pSavepoint==0 || db->autoCommit==0 ); assert( p1==SAVEPOINT_BEGIN||p1==SAVEPOINT_RELEASE||p1==SAVEPOINT_ROLLBACK ); @@ -79544,7 +97173,7 @@ case OP_Savepoint: { if( p1==SAVEPOINT_BEGIN ){ if( db->nVdbeWrite>0 ){ - /* A new savepoint cannot be created if there are active write + /* A new savepoint cannot be created if there are active write ** statements (i.e. open read/write incremental blob handles). */ sqlite3VdbeError(p, "cannot open savepoint - SQL statements in progress"); @@ -79568,7 +97197,7 @@ case OP_Savepoint: { if( pNew ){ pNew->zName = (char *)&pNew[1]; memcpy(pNew->zName, zName, nName+1); - + /* If there is no open transaction, then mark this as a special ** "transaction savepoint". */ if( db->autoCommit ){ @@ -79586,12 +97215,13 @@ case OP_Savepoint: { } } }else{ + assert( p1==SAVEPOINT_RELEASE || p1==SAVEPOINT_ROLLBACK ); iSavepoint = 0; /* Find the named savepoint. If there is no such savepoint, then an ** an error is returned to the user. */ for( - pSavepoint = db->pSavepoint; + pSavepoint = db->pSavepoint; pSavepoint && sqlite3StrICmp(pSavepoint->zName, zName); pSavepoint = pSavepoint->pNext ){ @@ -79601,7 +97231,7 @@ case OP_Savepoint: { sqlite3VdbeError(p, "no such savepoint: %s", zName); rc = SQLITE_ERROR; }else if( db->nVdbeWrite>0 && p1==SAVEPOINT_RELEASE ){ - /* It is not possible to release (commit) a savepoint if there are + /* It is not possible to release (commit) a savepoint if there are ** active write statements. */ sqlite3VdbeError(p, "cannot release savepoint - " @@ -79610,8 +97240,8 @@ case OP_Savepoint: { }else{ /* Determine whether or not this is a transaction savepoint. If so, - ** and this is a RELEASE command, then the current transaction - ** is committed. + ** and this is a RELEASE command, then the current transaction + ** is committed. */ int isTransaction = pSavepoint->pNext==0 && db->isTransactionSavepoint; if( isTransaction && p1==SAVEPOINT_RELEASE ){ @@ -79625,13 +97255,17 @@ case OP_Savepoint: { p->rc = rc = SQLITE_BUSY; goto vdbe_return; } - db->isTransactionSavepoint = 0; rc = p->rc; + if( rc ){ + db->autoCommit = 0; + }else{ + db->isTransactionSavepoint = 0; + } }else{ int isSchemaChange; iSavepoint = db->nSavepoint - iSavepoint - 1; if( p1==SAVEPOINT_ROLLBACK ){ - isSchemaChange = (db->flags & SQLITE_InternChanges)!=0; + isSchemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0; for(ii=0; iinDb; ii++){ rc = sqlite3BtreeTripAllCursors(db->aDb[ii].pBt, SQLITE_ABORT_ROLLBACK, @@ -79639,6 +97273,7 @@ case OP_Savepoint: { if( rc!=SQLITE_OK ) goto abort_due_to_error; } }else{ + assert( p1==SAVEPOINT_RELEASE ); isSchemaChange = 0; } for(ii=0; iinDb; ii++){ @@ -79648,13 +97283,14 @@ case OP_Savepoint: { } } if( isSchemaChange ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); sqlite3ResetAllSchemasOfConnection(db); - db->flags = (db->flags | SQLITE_InternChanges); + db->mDbFlags |= DBFLAG_SchemaChange; } } - - /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all + if( rc ) goto abort_due_to_error; + + /* Regardless of whether this is a RELEASE or ROLLBACK, destroy all ** savepoints nested inside of the savepoint being operated on. */ while( db->pSavepoint!=pSavepoint ){ pTmp = db->pSavepoint; @@ -79663,8 +97299,8 @@ case OP_Savepoint: { db->nSavepoint--; } - /* If it is a RELEASE, then destroy the savepoint being operated on - ** too. If it is a ROLLBACK TO, then set the number of deferred + /* If it is a RELEASE, then destroy the savepoint being operated on + ** too. If it is a ROLLBACK TO, then set the number of deferred ** constraint violations present in the database to the value stored ** when the savepoint was created. */ if( p1==SAVEPOINT_RELEASE ){ @@ -79675,6 +97311,7 @@ case OP_Savepoint: { db->nSavepoint--; } }else{ + assert( p1==SAVEPOINT_ROLLBACK ); db->nDeferredCons = pSavepoint->nDeferredCons; db->nDeferredImmCons = pSavepoint->nDeferredImmCons; } @@ -79686,7 +97323,10 @@ case OP_Savepoint: { } } if( rc ) goto abort_due_to_error; - + if( p->eVdbeState==VDBE_HALT_STATE ){ + rc = SQLITE_DONE; + goto vdbe_return; + } break; } @@ -79717,7 +97357,7 @@ case OP_AutoCommit: { db->autoCommit = 1; }else if( desiredAutoCommit && db->nVdbeWrite>0 ){ /* If this instruction implements a COMMIT and other VMs are writing - ** return an error indicating that the other VMs must complete first. + ** return an error indicating that the other VMs must complete first. */ sqlite3VdbeError(p, "cannot commit transaction - " "SQL statements in progress"); @@ -79734,7 +97374,6 @@ case OP_AutoCommit: { p->rc = rc = SQLITE_BUSY; goto vdbe_return; } - assert( db->nStatement==0 ); sqlite3CloseSavepoints(db); if( p->rc==SQLITE_OK ){ rc = SQLITE_DONE; @@ -79747,20 +97386,21 @@ case OP_AutoCommit: { (!desiredAutoCommit)?"cannot start a transaction within a transaction":( (iRollback)?"cannot rollback - no transaction is active": "cannot commit - no transaction is active")); - + rc = SQLITE_ERROR; goto abort_due_to_error; } - break; + /*NOTREACHED*/ assert(0); } /* Opcode: Transaction P1 P2 P3 P4 P5 ** ** Begin a transaction on database P1 if a transaction is not already ** active. -** If P2 is non-zero, then a write-transaction is started, or if a +** If P2 is non-zero, then a write-transaction is started, or if a ** read-transaction is already active, it is upgraded to a write-transaction. -** If P2 is zero, then a read-transaction is started. +** If P2 is zero, then a read-transaction is started. If P2 is 2 or more +** then an exclusive transaction is started. ** ** P1 is the index of the database file on which the transaction is ** started. Index 0 is the main database file and index 1 is the @@ -79790,39 +97430,50 @@ case OP_AutoCommit: { */ case OP_Transaction: { Btree *pBt; - int iMeta; - int iGen; + Db *pDb; + int iMeta = 0; assert( p->bIsReader ); assert( p->readOnly==0 || pOp->p2==0 ); + assert( pOp->p2>=0 && pOp->p2<=2 ); assert( pOp->p1>=0 && pOp->p1nDb ); assert( DbMaskTest(p->btreeMask, pOp->p1) ); - if( pOp->p2 && (db->flags & SQLITE_QueryOnly)!=0 ){ - rc = SQLITE_READONLY; + assert( rc==SQLITE_OK ); + if( pOp->p2 && (db->flags & (SQLITE_QueryOnly|SQLITE_CorruptRdOnly))!=0 ){ + if( db->flags & SQLITE_QueryOnly ){ + /* Writes prohibited by the "PRAGMA query_only=TRUE" statement */ + rc = SQLITE_READONLY; + }else{ + /* Writes prohibited due to a prior SQLITE_CORRUPT in the current + ** transaction */ + rc = SQLITE_CORRUPT; + } goto abort_due_to_error; } - pBt = db->aDb[pOp->p1].pBt; + pDb = &db->aDb[pOp->p1]; + pBt = pDb->pBt; if( pBt ){ - rc = sqlite3BtreeBeginTrans(pBt, pOp->p2); + rc = sqlite3BtreeBeginTrans(pBt, pOp->p2, &iMeta); testcase( rc==SQLITE_BUSY_SNAPSHOT ); testcase( rc==SQLITE_BUSY_RECOVERY ); - if( (rc&0xff)==SQLITE_BUSY ){ - p->pc = (int)(pOp - aOp); - p->rc = rc; - goto vdbe_return; - } if( rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_BUSY ){ + p->pc = (int)(pOp - aOp); + p->rc = rc; + goto vdbe_return; + } goto abort_due_to_error; } - if( pOp->p2 && p->usesStmtJournal - && (db->autoCommit==0 || db->nVdbeRead>1) + if( p->usesStmtJournal + && pOp->p2 + && (db->autoCommit==0 || db->nVdbeRead>1) ){ - assert( sqlite3BtreeIsInTrans(pBt) ); + assert( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ); if( p->iStatement==0 ){ assert( db->nStatement>=0 && db->nSavepoint>=0 ); - db->nStatement++; + db->nStatement++; p->iStatement = db->nSavepoint + db->nStatement; } @@ -79837,23 +97488,20 @@ case OP_Transaction: { p->nStmtDefCons = db->nDeferredCons; p->nStmtDefImmCons = db->nDeferredImmCons; } - - /* Gather the schema version number for checking: - ** IMPLEMENTATION-OF: R-32195-19465 The schema version is used by SQLite - ** each time a query is executed to ensure that the internal cache of the - ** schema used when compiling the SQL query matches the schema of the - ** database against which the compiled query is actually executed. - */ - sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&iMeta); - iGen = db->aDb[pOp->p1].pSchema->iGeneration; - }else{ - iGen = iMeta = 0; } assert( pOp->p5==0 || pOp->p4type==P4_INT32 ); - if( pOp->p5 && (iMeta!=pOp->p3 || iGen!=pOp->p4.i) ){ + if( rc==SQLITE_OK + && pOp->p5 + && (iMeta!=pOp->p3 || pDb->pSchema->iGeneration!=pOp->p4.i) + ){ + /* + ** IMPLEMENTATION-OF: R-03189-51135 As each SQL statement runs, the schema + ** version is checked to ensure that the schema has not changed since the + ** SQL statement was prepared. + */ sqlite3DbFree(db, p->zErrMsg); p->zErrMsg = sqlite3DbStrDup(db, "database schema has changed"); - /* If the schema-cookie from the database file matches the cookie + /* If the schema-cookie from the database file matches the cookie ** stored with the in-memory representation of the schema, do ** not reload the schema from the database file. ** @@ -79863,7 +97511,7 @@ case OP_Transaction: { ** prepared queries. If such a query is out-of-date, we do not want to ** discard the database schema, as the user code implementing the ** v-table would have to be ready for the sqlite3_vtab structure itself - ** to be invalidated whenever sqlite3_step() is called from within + ** to be invalidated whenever sqlite3_step() is called from within ** a v-table method. */ if( db->aDb[pOp->p1].pSchema->schema_cookie!=iMeta ){ @@ -79871,6 +97519,11 @@ case OP_Transaction: { } p->expired = 1; rc = SQLITE_SCHEMA; + + /* Set changeCntOn to 0 to prevent the value returned by sqlite3_changes() + ** from being modified in sqlite3VdbeHalt(). If this statement is + ** reprepared, changeCntOn will be set again. */ + p->changeCntOn = 0; } if( rc ) goto abort_due_to_error; break; @@ -79907,18 +97560,25 @@ case OP_ReadCookie: { /* out2 */ break; } -/* Opcode: SetCookie P1 P2 P3 * * +/* Opcode: SetCookie P1 P2 P3 * P5 ** ** Write the integer value P3 into cookie number P2 of database P1. ** P2==1 is the schema version. P2==2 is the database format. -** P2==3 is the recommended pager cache -** size, and so forth. P1==0 is the main database file and P1==1 is the +** P2==3 is the recommended pager cache +** size, and so forth. P1==0 is the main database file and P1==1 is the ** database file used to store temporary tables. ** ** A transaction must be started before executing this opcode. +** +** If P2 is the SCHEMA_VERSION cookie (cookie number 1) then the internal +** schema version is set to P3-P5. The "PRAGMA schema_version=N" statement +** has P5 set to 1, so that the internal schema version will be different +** from the database schema version, resulting in a schema reset. */ case OP_SetCookie: { Db *pDb; + + sqlite3VdbeIncrWriteCounter(p, 0); assert( pOp->p2p1>=0 && pOp->p1nDb ); assert( DbMaskTest(p->btreeMask, pOp->p1) ); @@ -79930,8 +97590,9 @@ case OP_SetCookie: { rc = sqlite3BtreeUpdateMeta(pDb->pBt, pOp->p2, pOp->p3); if( pOp->p2==BTREE_SCHEMA_VERSION ){ /* When the schema cookie changes, record the new cookie internally */ - pDb->pSchema->schema_cookie = pOp->p3; - db->flags |= SQLITE_InternChanges; + *(u32*)&pDb->pSchema->schema_cookie = *(u32*)&pOp->p3 - pOp->p5; + db->mDbFlags |= DBFLAG_SchemaChange; + sqlite3FkClearTriggerCache(db, pOp->p1); }else if( pOp->p2==BTREE_FILE_FORMAT ){ /* Record changes in the file format */ pDb->pSchema->file_format = pOp->p3; @@ -79939,7 +97600,7 @@ case OP_SetCookie: { if( pOp->p1==1 ){ /* Invalidate all prepared statements whenever the TEMP database ** schema is changed. Ticket #1644 */ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); p->expired = 0; } if( rc ) goto abort_due_to_error; @@ -79950,71 +97611,90 @@ case OP_SetCookie: { ** Synopsis: root=P2 iDb=P3 ** ** Open a read-only cursor for the database table whose root page is -** P2 in a database file. The database file is determined by P3. -** P3==0 means the main database, P3==1 means the database used for +** P2 in a database file. The database file is determined by P3. +** P3==0 means the main database, P3==1 means the database used for ** temporary tables, and P3>1 means used the corresponding attached ** database. Give the new cursor an identifier of P1. The P1 ** values need not be contiguous but all P1 values should be small integers. ** It is an error for P1 to be negative. ** -** If P5!=0 then use the content of register P2 as the root page, not -** the value of P2 itself. -** -** There will be a read lock on the database whenever there is an -** open cursor. If the database was unlocked prior to this instruction -** then a read lock is acquired as part of this instruction. A read -** lock allows other processes to read the database but prohibits -** any other process from modifying the database. The read lock is -** released when all cursors are closed. If this instruction attempts -** to get a read lock but fails, the script terminates with an -** SQLITE_BUSY error code. +** Allowed P5 bits: +**
              +**
            • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
            ** ** The P4 value may be either an integer (P4_INT32) or a pointer to -** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table. +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. ** ** See also: OpenWrite, ReopenIdx */ /* Opcode: ReopenIdx P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** -** The ReopenIdx opcode works exactly like ReadOpen except that it first -** checks to see if the cursor on P1 is already open with a root page -** number of P2 and if it is this opcode becomes a no-op. In other words, +** The ReopenIdx opcode works like OP_OpenRead except that it first +** checks to see if the cursor on P1 is already open on the same +** b-tree and if it is this opcode becomes a no-op. In other words, ** if the cursor is already open, do not reopen it. ** -** The ReopenIdx opcode may only be used with P5==0 and with P4 being -** a P4_KEYINFO object. Furthermore, the P3 value must be the same as -** every other ReopenIdx or OpenRead for the same cursor number. +** The ReopenIdx opcode may only be used with P5==0 or P5==OPFLAG_SEEKEQ +** and with P4 being a P4_KEYINFO object. Furthermore, the P3 value must +** be the same as every other ReopenIdx or OpenRead for the same cursor +** number. ** -** See the OpenRead opcode documentation for additional information. +** Allowed P5 bits: +**
              +**
            • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
            +** +** See also: OP_OpenRead, OP_OpenWrite */ /* Opcode: OpenWrite P1 P2 P3 P4 P5 ** Synopsis: root=P2 iDb=P3 ** ** Open a read/write cursor named P1 on the table or index whose root -** page is P2. Or if P5!=0 use the content of register P2 to find the -** root page. +** page is P2 (or whose root page is held in register P2 if the +** OPFLAG_P2ISREG bit is set in P5 - see below). ** ** The P4 value may be either an integer (P4_INT32) or a pointer to -** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo -** structure, then said structure defines the content and collating -** sequence of the index being opened. Otherwise, if P4 is an integer -** value, it is set to the number of columns in the table, or to the -** largest index of any column of the table that is actually used. +** a KeyInfo structure (P4_KEYINFO). If it is a pointer to a KeyInfo +** object, then table being opened must be an [index b-tree] where the +** KeyInfo object defines the content and collating +** sequence of that index b-tree. Otherwise, if P4 is an integer +** value, then the table being opened must be a [table b-tree] with a +** number of columns no less than the value of P4. ** -** This instruction works just like OpenRead except that it opens the cursor -** in read/write mode. For a given table, there can be one or more read-only -** cursors or a single read/write cursor but not both. +** Allowed P5 bits: +**
              +**
            • 0x02 OPFLAG_SEEKEQ: This cursor will only be used for +** equality lookups (implemented as a pair of opcodes OP_SeekGE/OP_IdxGT +** of OP_SeekLE/OP_IdxLT) +**
            • 0x08 OPFLAG_FORDELETE: This cursor is used only to seek +** and subsequently delete entries in an index btree. This is a +** hint to the storage engine that the storage engine is allowed to +** ignore. The hint is not used by the official SQLite b*tree storage +** engine, but is used by COMDB2. +**
            • 0x10 OPFLAG_P2ISREG: Use the content of register P2 +** as the root page, not the value of P2 itself. +**
            ** -** See also OpenRead. +** This instruction works like OpenRead except that it opens the cursor +** in read/write mode. +** +** See also: OP_OpenRead, OP_ReopenIdx */ -case OP_ReopenIdx: { +case OP_ReopenIdx: { /* ncycle */ int nField; KeyInfo *pKeyInfo; - int p2; + u32 p2; int iDb; int wrFlag; Btree *pX; @@ -80026,11 +97706,13 @@ case OP_ReopenIdx: { pCur = p->apCsr[pOp->p1]; if( pCur && pCur->pgnoRoot==(u32)pOp->p2 ){ assert( pCur->iDb==pOp->p3 ); /* Guaranteed by the code generator */ + assert( pCur->eCurType==CURTYPE_BTREE ); + sqlite3BtreeClearCursor(pCur->uc.pCursor); goto open_cursor_set_hints; } /* If the cursor is not currently open or is open on a different ** index, then fall through into OP_OpenRead to force a reopen */ -case OP_OpenRead: +case OP_OpenRead: /* ncycle */ case OP_OpenWrite: assert( pOp->opcode==OP_OpenWrite || pOp->p5==0 || pOp->p5==OPFLAG_SEEKEQ ); @@ -80038,14 +97720,14 @@ case OP_OpenWrite: assert( pOp->opcode==OP_OpenRead || pOp->opcode==OP_ReopenIdx || p->readOnly==0 ); - if( p->expired ){ + if( p->expired==1 ){ rc = SQLITE_ABORT_ROLLBACK; goto abort_due_to_error; } nField = 0; pKeyInfo = 0; - p2 = pOp->p2; + p2 = (u32)pOp->p2; iDb = pOp->p3; assert( iDb>=0 && iDbnDb ); assert( DbMaskTest(p->btreeMask, iDb) ); @@ -80059,36 +97741,38 @@ case OP_OpenWrite: if( pDb->pSchema->file_format < p->minWriteFileFormat ){ p->minWriteFileFormat = pDb->pSchema->file_format; } + if( pOp->p5 & OPFLAG_P2ISREG ){ + assert( p2>0 ); + assert( p2<=(u32)(p->nMem+1 - p->nCursor) ); + pIn2 = &aMem[p2]; + assert( memIsValid(pIn2) ); + assert( (pIn2->flags & MEM_Int)!=0 ); + sqlite3VdbeMemIntegerify(pIn2); + p2 = (int)pIn2->u.i; + /* The p2 value always comes from a prior OP_CreateBtree opcode and + ** that opcode will always set the p2 value to 2 or more or else fail. + ** If there were a failure, the prepared statement would have halted + ** before reaching this instruction. */ + assert( p2>=2 ); + } }else{ wrFlag = 0; - } - if( pOp->p5 & OPFLAG_P2ISREG ){ - assert( p2>0 ); - assert( p2<=(p->nMem+1 - p->nCursor) ); - pIn2 = &aMem[p2]; - assert( memIsValid(pIn2) ); - assert( (pIn2->flags & MEM_Int)!=0 ); - sqlite3VdbeMemIntegerify(pIn2); - p2 = (int)pIn2->u.i; - /* The p2 value always comes from a prior OP_CreateTable opcode and - ** that opcode will always set the p2 value to 2 or more or else fail. - ** If there were a failure, the prepared statement would have halted - ** before reaching this instruction. */ - assert( p2>=2 ); + assert( (pOp->p5 & OPFLAG_P2ISREG)==0 ); } if( pOp->p4type==P4_KEYINFO ){ pKeyInfo = pOp->p4.pKeyInfo; assert( pKeyInfo->enc==ENC(db) ); assert( pKeyInfo->db==db ); - nField = pKeyInfo->nField+pKeyInfo->nXField; + nField = pKeyInfo->nAllField; }else if( pOp->p4type==P4_INT32 ){ nField = pOp->p4.i; } assert( pOp->p1>=0 ); assert( nField>=0 ); testcase( nField==0 ); /* Table with INTEGER PRIMARY KEY and nothing else */ - pCur = allocateCursor(p, pOp->p1, nField, iDb, CURTYPE_BTREE); + pCur = allocateCursor(p, pOp->p1, nField, CURTYPE_BTREE); if( pCur==0 ) goto no_mem; + pCur->iDb = iDb; pCur->nullRow = 1; pCur->isOrdered = 1; pCur->pgnoRoot = p2; @@ -80100,30 +97784,68 @@ case OP_OpenWrite: /* Set the VdbeCursor.isTable variable. Previous versions of ** SQLite used to check if the root-page flags were sane at this point ** and report database corruption if they were not, but this check has - ** since moved into the btree layer. */ + ** since moved into the btree layer. */ pCur->isTable = pOp->p4type!=P4_KEYINFO; open_cursor_set_hints: assert( OPFLAG_BULKCSR==BTREE_BULKLOAD ); assert( OPFLAG_SEEKEQ==BTREE_SEEK_EQ ); testcase( pOp->p5 & OPFLAG_BULKCSR ); -#ifdef SQLITE_ENABLE_CURSOR_HINTS testcase( pOp->p2 & OPFLAG_SEEKEQ ); -#endif sqlite3BtreeCursorHintFlags(pCur->uc.pCursor, (pOp->p5 & (OPFLAG_BULKCSR|OPFLAG_SEEKEQ))); if( rc ) goto abort_due_to_error; break; } -/* Opcode: OpenEphemeral P1 P2 * P4 P5 +/* Opcode: OpenDup P1 P2 * * * +** +** Open a new cursor P1 that points to the same ephemeral table as +** cursor P2. The P2 cursor must have been opened by a prior OP_OpenEphemeral +** opcode. Only ephemeral cursors may be duplicated. +** +** Duplicate ephemeral cursors are used for self-joins of materialized views. +*/ +case OP_OpenDup: { /* ncycle */ + VdbeCursor *pOrig; /* The original cursor to be duplicated */ + VdbeCursor *pCx; /* The new cursor */ + + pOrig = p->apCsr[pOp->p2]; + assert( pOrig ); + assert( pOrig->isEphemeral ); /* Only ephemeral cursors can be duplicated */ + + pCx = allocateCursor(p, pOp->p1, pOrig->nField, CURTYPE_BTREE); + if( pCx==0 ) goto no_mem; + pCx->nullRow = 1; + pCx->isEphemeral = 1; + pCx->pKeyInfo = pOrig->pKeyInfo; + pCx->isTable = pOrig->isTable; + pCx->pgnoRoot = pOrig->pgnoRoot; + pCx->isOrdered = pOrig->isOrdered; + pCx->ub.pBtx = pOrig->ub.pBtx; + pCx->noReuse = 1; + pOrig->noReuse = 1; + rc = sqlite3BtreeCursor(pCx->ub.pBtx, pCx->pgnoRoot, BTREE_WRCSR, + pCx->pKeyInfo, pCx->uc.pCursor); + /* The sqlite3BtreeCursor() routine can only fail for the first cursor + ** opened for a database. Since there is already an open cursor when this + ** opcode is run, the sqlite3BtreeCursor() cannot fail */ + assert( rc==SQLITE_OK ); + break; +} + + +/* Opcode: OpenEphemeral P1 P2 P3 P4 P5 ** Synopsis: nColumn=P2 ** ** Open a new cursor P1 to a transient table. -** The cursor is always opened read/write even if +** The cursor is always opened read/write even if ** the main database is read-only. The ephemeral ** table is deleted automatically when the cursor is closed. ** +** If the cursor P1 is already opened on an ephemeral table, the table +** is cleared (all content is erased). +** ** P2 is the number of columns in the ephemeral table. ** The cursor points to a BTree table if P4==0 and to a BTree index ** if P4 is not 0. If P4 is not NULL, it points to a KeyInfo structure @@ -80133,6 +97855,10 @@ open_cursor_set_hints: ** in btree.h. These flags control aspects of the operation of ** the btree. The BTREE_OMIT_JOURNAL and BTREE_SINGLE flags are ** added automatically. +** +** If P3 is positive, then reg[P3] is modified slightly so that it +** can be used as zero-length data for OP_Insert. This is an optimization +** that avoids an extra OP_Blob opcode to initialize that register. */ /* Opcode: OpenAutoindex P1 P2 * P4 * ** Synopsis: nColumn=P2 @@ -80142,12 +97868,12 @@ open_cursor_set_hints: ** by this opcode will be used for automatically created transient ** indices in joins. */ -case OP_OpenAutoindex: -case OP_OpenEphemeral: { +case OP_OpenAutoindex: /* ncycle */ +case OP_OpenEphemeral: { /* ncycle */ VdbeCursor *pCx; KeyInfo *pKeyInfo; - static const int vfsFlags = + static const int vfsFlags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | @@ -80155,42 +97881,69 @@ case OP_OpenEphemeral: { SQLITE_OPEN_TRANSIENT_DB; assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_BTREE); - if( pCx==0 ) goto no_mem; - pCx->nullRow = 1; - pCx->isEphemeral = 1; - rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->pBt, - BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, vfsFlags); - if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pCx->pBt, 1); + if( pOp->p3>0 ){ + /* Make register reg[P3] into a value that can be used as the data + ** form sqlite3BtreeInsert() where the length of the data is zero. */ + assert( pOp->p2==0 ); /* Only used when number of columns is zero */ + assert( pOp->opcode==OP_OpenEphemeral ); + assert( aMem[pOp->p3].flags & MEM_Null ); + aMem[pOp->p3].n = 0; + aMem[pOp->p3].z = ""; } - if( rc==SQLITE_OK ){ - /* If a transient index is required, create it by calling - ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before - ** opening it. If a transient table is required, just use the - ** automatically created table with root-page 1 (an BLOB_INTKEY table). - */ - if( (pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ - int pgno; - assert( pOp->p4type==P4_KEYINFO ); - rc = sqlite3BtreeCreateTable(pCx->pBt, &pgno, BTREE_BLOBKEY | pOp->p5); + pCx = p->apCsr[pOp->p1]; + if( pCx && !pCx->noReuse && ALWAYS(pOp->p2<=pCx->nField) ){ + /* If the ephemeral table is already open and has no duplicates from + ** OP_OpenDup, then erase all existing content so that the table is + ** empty again, rather than creating a new table. */ + assert( pCx->isEphemeral ); + pCx->seqCount = 0; + pCx->cacheStatus = CACHE_STALE; + rc = sqlite3BtreeClearTable(pCx->ub.pBtx, pCx->pgnoRoot, 0); + }else{ + pCx = allocateCursor(p, pOp->p1, pOp->p2, CURTYPE_BTREE); + if( pCx==0 ) goto no_mem; + pCx->isEphemeral = 1; + rc = sqlite3BtreeOpen(db->pVfs, 0, db, &pCx->ub.pBtx, + BTREE_OMIT_JOURNAL | BTREE_SINGLE | pOp->p5, + vfsFlags); + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pCx->ub.pBtx, 1, 0); if( rc==SQLITE_OK ){ - assert( pgno==MASTER_ROOT+1 ); - assert( pKeyInfo->db==db ); - assert( pKeyInfo->enc==ENC(db) ); - pCx->pKeyInfo = pKeyInfo; - rc = sqlite3BtreeCursor(pCx->pBt, pgno, BTREE_WRCSR, - pKeyInfo, pCx->uc.pCursor); + /* If a transient index is required, create it by calling + ** sqlite3BtreeCreateTable() with the BTREE_BLOBKEY flag before + ** opening it. If a transient table is required, just use the + ** automatically created table with root-page 1 (an BLOB_INTKEY table). + */ + if( (pCx->pKeyInfo = pKeyInfo = pOp->p4.pKeyInfo)!=0 ){ + assert( pOp->p4type==P4_KEYINFO ); + rc = sqlite3BtreeCreateTable(pCx->ub.pBtx, &pCx->pgnoRoot, + BTREE_BLOBKEY | pOp->p5); + if( rc==SQLITE_OK ){ + assert( pCx->pgnoRoot==SCHEMA_ROOT+1 ); + assert( pKeyInfo->db==db ); + assert( pKeyInfo->enc==ENC(db) ); + rc = sqlite3BtreeCursor(pCx->ub.pBtx, pCx->pgnoRoot, BTREE_WRCSR, + pKeyInfo, pCx->uc.pCursor); + } + pCx->isTable = 0; + }else{ + pCx->pgnoRoot = SCHEMA_ROOT; + rc = sqlite3BtreeCursor(pCx->ub.pBtx, SCHEMA_ROOT, BTREE_WRCSR, + 0, pCx->uc.pCursor); + pCx->isTable = 1; + } + } + pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + if( rc ){ + assert( !sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) ); + sqlite3BtreeClose(pCx->ub.pBtx); + }else{ + assert( sqlite3BtreeClosesWithCursor(pCx->ub.pBtx, pCx->uc.pCursor) ); } - pCx->isTable = 0; - }else{ - rc = sqlite3BtreeCursor(pCx->pBt, MASTER_ROOT, BTREE_WRCSR, - 0, pCx->uc.pCursor); - pCx->isTable = 1; } } if( rc ) goto abort_due_to_error; - pCx->isOrdered = (pOp->p5!=BTREE_UNORDERED); + pCx->nullRow = 1; break; } @@ -80209,7 +97962,7 @@ case OP_SorterOpen: { assert( pOp->p1>=0 ); assert( pOp->p2>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p2, -1, CURTYPE_SORTER); + pCx = allocateCursor(p, pOp->p1, pOp->p2, CURTYPE_SORTER); if( pCx==0 ) goto no_mem; pCx->pKeyInfo = pOp->p4.pKeyInfo; assert( pCx->pKeyInfo->db==db ); @@ -80242,7 +97995,7 @@ case OP_SequenceTest: { ** ** Open a new cursor that points to a fake table that contains a single ** row of data. The content of that one row is the content of memory -** register P2. In other words, cursor P1 becomes an alias for the +** register P2. In other words, cursor P1 becomes an alias for the ** MEM_Blob content contained in register P2. ** ** A pseudo-table created by this opcode is used to hold a single @@ -80251,18 +98004,24 @@ case OP_SequenceTest: { ** is the only cursor opcode that works with a pseudo-table. ** ** P3 is the number of fields in the records that will be stored by -** the pseudo-table. +** the pseudo-table. If P2 is 0 or negative then the pseudo-cursor +** will return NULL for every column. */ case OP_OpenPseudo: { VdbeCursor *pCx; assert( pOp->p1>=0 ); assert( pOp->p3>=0 ); - pCx = allocateCursor(p, pOp->p1, pOp->p3, -1, CURTYPE_PSEUDO); + pCx = allocateCursor(p, pOp->p1, pOp->p3, CURTYPE_PSEUDO); if( pCx==0 ) goto no_mem; pCx->nullRow = 1; - pCx->uc.pseudoTableReg = pOp->p2; + pCx->seekResult = pOp->p2; pCx->isTable = 1; + /* Give this pseudo-cursor a fake BtCursor pointer so that pCx + ** can be safely passed to sqlite3VdbeCursorMoveto(). This avoids a test + ** for pCx->eCurType==CURTYPE_BTREE inside of sqlite3VdbeCursorMoveto() + ** which is a performance optimization */ + pCx->uc.pCursor = sqlite3BtreeFakeValidCursor(); assert( pOp->p5==0 ); break; } @@ -80272,7 +98031,7 @@ case OP_OpenPseudo: { ** Close a cursor previously opened as P1. If P1 is not ** currently open, this instruction is a no-op. */ -case OP_Close: { +case OP_Close: { /* ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); sqlite3VdbeFreeCursor(p, p->apCsr[pOp->p1]); p->apCsr[pOp->p1] = 0; @@ -80302,21 +98061,23 @@ case OP_ColumnsUsed: { /* Opcode: SeekGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as the key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as the key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. ** -** Reposition cursor P1 so that it points to the smallest entry that -** is greater than or equal to the key value. If there are no records +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than or equal to the key value. If there are no records ** greater than or equal to the key and P2 is not zero, then jump to P2. ** ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this -** opcode will always land on a record that equally equals the key, or -** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this -** opcode must be followed by an IdxLE opcode with the same arguments. -** The IdxLE opcode will be skipped if this opcode succeeds, but the -** IdxLE opcode will be used on subsequent loop iterations. +** opcode will either land on a record that exactly matches the key, or +** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, +** this opcode must be followed by an IdxLE opcode with the same arguments. +** The IdxGT opcode will be skipped if this opcode succeeds, but the +** IdxGT opcode will be used on subsequent loop iterations. The +** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this +** is an equality search. ** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is @@ -80327,13 +98088,13 @@ case OP_ColumnsUsed: { /* Opcode: SeekGT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as a key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. ** -** Reposition cursor P1 so that it points to the smallest entry that -** is greater than the key value. If there are no records greater than +** Reposition cursor P1 so that it points to the smallest entry that +** is greater than the key value. If there are no records greater than ** the key and P2 is not zero, then jump to P2. ** ** This opcode leaves the cursor configured to move in forward order, @@ -80342,16 +98103,16 @@ case OP_ColumnsUsed: { ** ** See also: Found, NotFound, SeekLt, SeekGe, SeekLe */ -/* Opcode: SeekLT P1 P2 P3 P4 * +/* Opcode: SeekLT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as a key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. ** -** Reposition cursor P1 so that it points to the largest entry that -** is less than the key value. If there are no records less than +** Reposition cursor P1 so that it points to the largest entry that +** is less than the key value. If there are no records less than ** the key and P2 is not zero, then jump to P2. ** ** This opcode leaves the cursor configured to move in reverse order, @@ -80363,13 +98124,13 @@ case OP_ColumnsUsed: { /* Opcode: SeekLE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), -** use the value in register P3 as a key. If cursor P1 refers -** to an SQL index, then P3 is the first in an array of P4 registers -** that are used as an unpacked index key. +** If cursor P1 refers to an SQL table (B-Tree that uses integer keys), +** use the value in register P3 as a key. If cursor P1 refers +** to an SQL index, then P3 is the first in an array of P4 registers +** that are used as an unpacked index key. ** -** Reposition cursor P1 so that it points to the largest entry that -** is less than or equal to the key value. If there are no records +** Reposition cursor P1 so that it points to the largest entry that +** is less than or equal to the key value. If there are no records ** less than or equal to the key and P2 is not zero, then jump to P2. ** ** This opcode leaves the cursor configured to move in reverse order, @@ -80377,18 +98138,20 @@ case OP_ColumnsUsed: { ** configured to use Prev, not Next. ** ** If the cursor P1 was opened using the OPFLAG_SEEKEQ flag, then this -** opcode will always land on a record that equally equals the key, or -** else jump immediately to P2. When the cursor is OPFLAG_SEEKEQ, this -** opcode must be followed by an IdxGE opcode with the same arguments. +** opcode will either land on a record that exactly matches the key, or +** else it will cause a jump to P2. When the cursor is OPFLAG_SEEKEQ, +** this opcode must be followed by an IdxLE opcode with the same arguments. ** The IdxGE opcode will be skipped if this opcode succeeds, but the -** IdxGE opcode will be used on subsequent loop iterations. +** IdxGE opcode will be used on subsequent loop iterations. The +** OPFLAG_SEEKEQ flags is a hint to the btree layer to say that this +** is an equality search. ** ** See also: Found, NotFound, SeekGt, SeekGe, SeekLt */ -case OP_SeekLT: /* jump, in3 */ -case OP_SeekLE: /* jump, in3 */ -case OP_SeekGE: /* jump, in3 */ -case OP_SeekGT: { /* jump, in3 */ +case OP_SeekLT: /* jump0, in3, group, ncycle */ +case OP_SeekLE: /* jump0, in3, group, ncycle */ +case OP_SeekGE: /* jump0, in3, group, ncycle */ +case OP_SeekGT: { /* jump0, in3, group, ncycle */ int res; /* Comparison result */ int oc; /* Opcode */ VdbeCursor *pC; /* The cursor to seek */ @@ -80414,28 +98177,41 @@ case OP_SeekGT: { /* jump, in3 */ pC->seekOp = pOp->opcode; #endif + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; if( pC->isTable ){ - /* The BTREE_SEEK_EQ flag is only set on index cursors */ - assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 ); + u16 flags3, newType; + /* The OPFLAG_SEEKEQ/BTREE_SEEK_EQ flag is only set on index cursors */ + assert( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ)==0 + || CORRUPT_DB ); /* The input value in P3 might be of any type: integer, real, string, ** blob, or NULL. But it needs to be an integer before we can do ** the seek, so convert it. */ pIn3 = &aMem[pOp->p3]; - if( (pIn3->flags & (MEM_Int|MEM_Real|MEM_Str))==MEM_Str ){ + flags3 = pIn3->flags; + if( (flags3 & (MEM_Int|MEM_Real|MEM_IntReal|MEM_Str))==MEM_Str ){ applyNumericAffinity(pIn3, 0); } - iKey = sqlite3VdbeIntValue(pIn3); + iKey = sqlite3VdbeIntValue(pIn3); /* Get the integer key value */ + newType = pIn3->flags; /* Record the type after applying numeric affinity */ + pIn3->flags = flags3; /* But convert the type back to its original */ /* If the P3 value could not be converted into an integer without ** loss of information, then special processing is required... */ - if( (pIn3->flags & MEM_Int)==0 ){ - if( (pIn3->flags & MEM_Real)==0 ){ - /* If the P3 value cannot be converted into any kind of a number, - ** then the seek is not possible, so jump to P2 */ - VdbeBranchTaken(1,2); goto jump_to_p2; - break; + if( (newType & (MEM_Int|MEM_IntReal))==0 ){ + int c; + if( (newType & MEM_Real)==0 ){ + if( (newType & MEM_Null) || oc>=OP_SeekGE ){ + VdbeBranchTaken(1,2); + goto jump_to_p2; + }else{ + rc = sqlite3BtreeLast(pC->uc.pCursor, &res); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + goto seek_not_found; + } } + c = sqlite3IntFloatCompare(iKey, pIn3->u.r); /* If the approximation iKey is larger than the actual real search ** term, substitute >= for > and < for <=. e.g. if the search term @@ -80444,7 +98220,7 @@ case OP_SeekGT: { /* jump, in3 */ ** (x > 4.9) -> (x >= 5) ** (x <= 4.9) -> (x < 5) */ - if( pIn3->u.r<(double)iKey ){ + if( c>0 ){ assert( OP_SeekGE==(OP_SeekGT-1) ); assert( OP_SeekLT==(OP_SeekLE-1) ); assert( (OP_SeekLE & 0x0001)==(OP_SeekGT & 0x0001) ); @@ -80453,27 +98229,30 @@ case OP_SeekGT: { /* jump, in3 */ /* If the approximation iKey is smaller than the actual real search ** term, substitute <= for < and > for >=. */ - else if( pIn3->u.r>(double)iKey ){ + else if( c<0 ){ assert( OP_SeekLE==(OP_SeekLT+1) ); assert( OP_SeekGT==(OP_SeekGE+1) ); assert( (OP_SeekLT & 0x0001)==(OP_SeekGE & 0x0001) ); if( (oc & 0x0001)==(OP_SeekLT & 0x0001) ) oc++; } - } - rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)iKey, 0, &res); + } + rc = sqlite3BtreeTableMoveto(pC->uc.pCursor, (u64)iKey, 0, &res); pC->movetoTarget = iKey; /* Used by OP_Delete */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } }else{ - /* For a cursor with the BTREE_SEEK_EQ hint, only the OP_SeekGE and - ** OP_SeekLE opcodes are allowed, and these must be immediately followed - ** by an OP_IdxGT or OP_IdxLT opcode, respectively, with the same key. + /* For a cursor with the OPFLAG_SEEKEQ/BTREE_SEEK_EQ hint, only the + ** OP_SeekGE and OP_SeekLE opcodes are allowed, and these must be + ** immediately followed by an OP_IdxGT or OP_IdxLT opcode, respectively, + ** with the same key. */ if( sqlite3BtreeCursorHasHint(pC->uc.pCursor, BTREE_SEEK_EQ) ){ eqOnly = 1; assert( pOp->opcode==OP_SeekGE || pOp->opcode==OP_SeekLE ); assert( pOp[1].opcode==OP_IdxLT || pOp[1].opcode==OP_IdxGT ); + assert( pOp->opcode==OP_SeekGE || pOp[1].opcode==OP_IdxLT ); + assert( pOp->opcode==OP_SeekLE || pOp[1].opcode==OP_IdxGT ); assert( pOp[1].p1==pOp[0].p1 ); assert( pOp[1].p2==pOp[0].p2 ); assert( pOp[1].p3==pOp[0].p3 ); @@ -80501,11 +98280,16 @@ case OP_SeekGT: { /* jump, in3 */ r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; i0 ) REGISTER_TRACE(pOp->p3+i, &r.aMem[i]); + } + } #endif - ExpandBlob(r.aMem); r.eqSeen = 0; - rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, &r, 0, 0, &res); + rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &res); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } @@ -80514,16 +98298,21 @@ case OP_SeekGT: { /* jump, in3 */ goto seek_not_found; } } - pC->deferredMoveto = 0; - pC->cacheStatus = CACHE_STALE; #ifdef SQLITE_TEST sqlite3_search_count++; #endif if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); if( res<0 || (res==0 && oc==OP_SeekGT) ){ res = 0; - rc = sqlite3BtreeNext(pC->uc.pCursor, &res); - if( rc!=SQLITE_OK ) goto abort_due_to_error; + rc = sqlite3BtreeNext(pC->uc.pCursor, 0); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + res = 1; + }else{ + goto abort_due_to_error; + } + } }else{ res = 0; } @@ -80531,8 +98320,15 @@ case OP_SeekGT: { /* jump, in3 */ assert( oc==OP_SeekLT || oc==OP_SeekLE ); if( res>0 || (res==0 && oc==OP_SeekLT) ){ res = 0; - rc = sqlite3BtreePrevious(pC->uc.pCursor, &res); - if( rc!=SQLITE_OK ) goto abort_due_to_error; + rc = sqlite3BtreePrevious(pC->uc.pCursor, 0); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + res = 1; + }else{ + goto abort_due_to_error; + } + } }else{ /* res might be negative because the table is empty. Check to ** see if this is the case. @@ -80551,7 +98347,240 @@ seek_not_found: } break; } - + + +/* Opcode: SeekScan P1 P2 * * P5 +** Synopsis: Scan-ahead up to P1 rows +** +** This opcode is a prefix opcode to OP_SeekGE. In other words, this +** opcode must be immediately followed by OP_SeekGE. This constraint is +** checked by assert() statements. +** +** This opcode uses the P1 through P4 operands of the subsequent +** OP_SeekGE. In the text that follows, the operands of the subsequent +** OP_SeekGE opcode are denoted as SeekOP.P1 through SeekOP.P4. Only +** the P1, P2 and P5 operands of this opcode are also used, and are called +** This.P1, This.P2 and This.P5. +** +** This opcode helps to optimize IN operators on a multi-column index +** where the IN operator is on the later terms of the index by avoiding +** unnecessary seeks on the btree, substituting steps to the next row +** of the b-tree instead. A correct answer is obtained if this opcode +** is omitted or is a no-op. +** +** The SeekGE.P3 and SeekGE.P4 operands identify an unpacked key which +** is the desired entry that we want the cursor SeekGE.P1 to be pointing +** to. Call this SeekGE.P3/P4 row the "target". +** +** If the SeekGE.P1 cursor is not currently pointing to a valid row, +** then this opcode is a no-op and control passes through into the OP_SeekGE. +** +** If the SeekGE.P1 cursor is pointing to a valid row, then that row +** might be the target row, or it might be near and slightly before the +** target row, or it might be after the target row. If the cursor is +** currently before the target row, then this opcode attempts to position +** the cursor on or after the target row by invoking sqlite3BtreeStep() +** on the cursor between 1 and This.P1 times. +** +** The This.P5 parameter is a flag that indicates what to do if the +** cursor ends up pointing at a valid row that is past the target +** row. If This.P5 is false (0) then a jump is made to SeekGE.P2. If +** This.P5 is true (non-zero) then a jump is made to This.P2. The P5==0 +** case occurs when there are no inequality constraints to the right of +** the IN constraint. The jump to SeekGE.P2 ends the loop. The P5!=0 case +** occurs when there are inequality constraints to the right of the IN +** operator. In that case, the This.P2 will point either directly to or +** to setup code prior to the OP_IdxGT or OP_IdxGE opcode that checks for +** loop terminate. +** +** Possible outcomes from this opcode:
              +** +**
            1. If the cursor is initially not pointed to any valid row, then +** fall through into the subsequent OP_SeekGE opcode. +** +**
            2. If the cursor is left pointing to a row that is before the target +** row, even after making as many as This.P1 calls to +** sqlite3BtreeNext(), then also fall through into OP_SeekGE. +** +**
            3. If the cursor is left pointing at the target row, either because it +** was at the target row to begin with or because one or more +** sqlite3BtreeNext() calls moved the cursor to the target row, +** then jump to This.P2.., +** +**
            4. If the cursor started out before the target row and a call to +** to sqlite3BtreeNext() moved the cursor off the end of the index +** (indicating that the target row definitely does not exist in the +** btree) then jump to SeekGE.P2, ending the loop. +** +**
            5. If the cursor ends up on a valid row that is past the target row +** (indicating that the target row does not exist in the btree) then +** jump to SeekOP.P2 if This.P5==0 or to This.P2 if This.P5>0. +**
            +*/ +case OP_SeekScan: { /* ncycle */ + VdbeCursor *pC; + int res; + int nStep; + UnpackedRecord r; + + assert( pOp[1].opcode==OP_SeekGE ); + + /* If pOp->p5 is clear, then pOp->p2 points to the first instruction past the + ** OP_IdxGT that follows the OP_SeekGE. Otherwise, it points to the first + ** opcode past the OP_SeekGE itself. */ + assert( pOp->p2>=(int)(pOp-aOp)+2 ); +#ifdef SQLITE_DEBUG + if( pOp->p5==0 ){ + /* There are no inequality constraints following the IN constraint. */ + assert( pOp[1].p1==aOp[pOp->p2-1].p1 ); + assert( pOp[1].p2==aOp[pOp->p2-1].p2 ); + assert( pOp[1].p3==aOp[pOp->p2-1].p3 ); + assert( aOp[pOp->p2-1].opcode==OP_IdxGT + || aOp[pOp->p2-1].opcode==OP_IdxGE ); + testcase( aOp[pOp->p2-1].opcode==OP_IdxGE ); + }else{ + /* There are inequality constraints. */ + assert( pOp->p2==(int)(pOp-aOp)+2 ); + assert( aOp[pOp->p2-1].opcode==OP_SeekGE ); + } +#endif + + assert( pOp->p1>0 ); + pC = p->apCsr[pOp[1].p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( !pC->isTable ); + if( !sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... cursor not valid - fall through\n"); + } +#endif + break; + } + nStep = pOp->p1; + assert( nStep>=1 ); + r.pKeyInfo = pC->pKeyInfo; + r.nField = (u16)pOp[1].p4.i; + r.default_rc = 0; + r.aMem = &aMem[pOp[1].p3]; +#ifdef SQLITE_DEBUG + { + int i; + for(i=0; i0 && pOp->p5==0 ){ + seekscan_search_fail: + /* Jump to SeekGE.P2, ending the loop */ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... %d steps and then skip\n", pOp->p1 - nStep); + } +#endif + VdbeBranchTaken(1,3); + pOp++; + goto jump_to_p2; + } + if( res>=0 ){ + /* Jump to This.P2, bypassing the OP_SeekGE opcode */ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... %d steps and then success\n", pOp->p1 - nStep); + } +#endif + VdbeBranchTaken(2,3); + goto jump_to_p2; + break; + } + if( nStep<=0 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("... fall through after %d steps\n", pOp->p1); + } +#endif + VdbeBranchTaken(0,3); + break; + } + nStep--; + pC->cacheStatus = CACHE_STALE; + rc = sqlite3BtreeNext(pC->uc.pCursor, 0); + if( rc ){ + if( rc==SQLITE_DONE ){ + rc = SQLITE_OK; + goto seekscan_search_fail; + }else{ + goto abort_due_to_error; + } + } + } + + break; +} + + +/* Opcode: SeekHit P1 P2 P3 * * +** Synopsis: set P2<=seekHit<=P3 +** +** Increase or decrease the seekHit value for cursor P1, if necessary, +** so that it is no less than P2 and no greater than P3. +** +** The seekHit integer represents the maximum of terms in an index for which +** there is known to be at least one match. If the seekHit value is smaller +** than the total number of equality terms in an index lookup, then the +** OP_IfNoHope opcode might run to see if the IN loop can be abandoned +** early, thus saving work. This is part of the IN-early-out optimization. +** +** P1 must be a valid b-tree cursor. +*/ +case OP_SeekHit: { /* ncycle */ + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pOp->p3>=pOp->p2 ); + if( pC->seekHitp2 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("seekHit changes from %d to %d\n", pC->seekHit, pOp->p2); + } +#endif + pC->seekHit = pOp->p2; + }else if( pC->seekHit>pOp->p3 ){ +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("seekHit changes from %d to %d\n", pC->seekHit, pOp->p3); + } +#endif + pC->seekHit = pOp->p3; + } + break; +} + +/* Opcode: IfNotOpen P1 P2 * * * +** Synopsis: if( !csr[P1] ) goto P2 +** +** If cursor P1 is not open or if P1 is set to a NULL row using the +** OP_NullRow opcode, then jump to instruction P2. Otherwise, fall through. +*/ +case OP_IfNotOpen: { /* jump */ + VdbeCursor *pCur; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pCur = p->apCsr[pOp->p1]; + VdbeBranchTaken(pCur==0 || pCur->nullRow, 2); + if( pCur==0 || pCur->nullRow ){ + goto jump_to_p2_and_check_for_interrupt; + } + break; +} /* Opcode: Found P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -80576,9 +98605,9 @@ seek_not_found: ** If P4==0 then register P3 holds a blob constructed by MakeRecord. If ** P4>0 then register P3 is the first of P4 registers that form an unpacked ** record. -** +** ** Cursor P1 is on an index btree. If the record identified by P3 and P4 -** is not the prefix of any entry in P1 then a jump is made to P2. If P1 +** is not the prefix of any entry in P1 then a jump is made to P2. If P1 ** does contain an entry whose prefix matches the P3/P4 record then control ** falls through to the next instruction and P1 is left pointing at the ** matching entry. @@ -80587,7 +98616,38 @@ seek_not_found: ** advanced in either direction. In other words, the Next and Prev ** opcodes do not work after this operation. ** -** See also: Found, NotExists, NoConflict +** See also: Found, NotExists, NoConflict, IfNoHope +*/ +/* Opcode: IfNoHope P1 P2 P3 P4 * +** Synopsis: key=r[P3@P4] +** +** Register P3 is the first of P4 registers that form an unpacked +** record. Cursor P1 is an index btree. P2 is a jump destination. +** In other words, the operands to this opcode are the same as the +** operands to OP_NotFound and OP_IdxGT. +** +** This opcode is an optimization attempt only. If this opcode always +** falls through, the correct answer is still obtained, but extra work +** is performed. +** +** A value of N in the seekHit flag of cursor P1 means that there exists +** a key P3:N that will match some record in the index. We want to know +** if it is possible for a record P3:P4 to match some record in the +** index. If it is not possible, we can skip some work. So if seekHit +** is less than P4, attempt to find out if a match is possible by running +** OP_NotFound. +** +** This opcode is used in IN clause processing for a multi-column key. +** If an IN clause is attached to an element of the key other than the +** left-most element, and if there are no matches on the most recent +** seek over the whole key, then it might be that one of the key element +** to the left is prohibiting a match, and hence there is "no hope" of +** any match regardless of how many IN clause elements are checked. +** In such a case, we abandon the IN clause search early, using this +** opcode. The opcode name comes from the fact that the +** jump is taken if there is "no hope" of achieving a match. +** +** See also: NotFound, SeekHit */ /* Opcode: NoConflict P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] @@ -80595,7 +98655,7 @@ seek_not_found: ** If P4==0 then register P3 holds a blob constructed by MakeRecord. If ** P4>0 then register P3 is the first of P4 registers that form an unpacked ** record. -** +** ** Cursor P1 is on an index btree. If the record identified by P3 and P4 ** contains any NULL value, jump immediately to P2. If all terms of the ** record are not-NULL then a check is done to determine if any row in the @@ -80612,18 +98672,28 @@ seek_not_found: ** ** See also: NotFound, Found, NotExists */ -case OP_NoConflict: /* jump, in3 */ -case OP_NotFound: /* jump, in3 */ -case OP_Found: { /* jump, in3 */ +case OP_IfNoHope: { /* jump, in3, ncycle */ + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + printf("seekHit is %d\n", pC->seekHit); + } +#endif + if( pC->seekHit>=pOp->p4.i ) break; + /* Fall through into OP_NotFound */ + /* no break */ deliberate_fall_through +} +case OP_NoConflict: /* jump, in3, ncycle */ +case OP_NotFound: /* jump, in3, ncycle */ +case OP_Found: { /* jump, in3, ncycle */ int alreadyExists; - int takeJump; int ii; VdbeCursor *pC; - int res; - char *pFree; UnpackedRecord *pIdxKey; UnpackedRecord r; - char aTempRec[ROUND8(sizeof(UnpackedRecord)) + sizeof(Mem)*4 + 7]; #ifdef SQLITE_TEST if( pOp->opcode!=OP_NoConflict ) sqlite3_found_count++; @@ -80636,52 +98706,42 @@ case OP_Found: { /* jump, in3 */ #ifdef SQLITE_DEBUG pC->seekOp = pOp->opcode; #endif - pIn3 = &aMem[pOp->p3]; + r.aMem = &aMem[pOp->p3]; assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); assert( pC->isTable==0 ); - pFree = 0; - if( pOp->p4.i>0 ){ + r.nField = (u16)pOp->p4.i; + if( r.nField>0 ){ + /* Key values in an array of registers */ r.pKeyInfo = pC->pKeyInfo; - r.nField = (u16)pOp->p4.i; - r.aMem = pIn3; + r.default_rc = 0; +#ifdef SQLITE_DEBUG + (void)sqlite3FaultSim(50); /* For use by --counter in TH3 */ for(ii=0; iip3+ii, &r.aMem[ii]); + } #endif - } - pIdxKey = &r; + rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, &r, &pC->seekResult); }else{ - pIdxKey = sqlite3VdbeAllocUnpackedRecord( - pC->pKeyInfo, aTempRec, sizeof(aTempRec), &pFree - ); + /* Composite key generated by OP_MakeRecord */ + assert( r.aMem->flags & MEM_Blob ); + assert( pOp->opcode!=OP_NoConflict ); + rc = ExpandBlob(r.aMem); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc ) goto no_mem; + pIdxKey = sqlite3VdbeAllocUnpackedRecord(pC->pKeyInfo); if( pIdxKey==0 ) goto no_mem; - assert( pIn3->flags & MEM_Blob ); - ExpandBlob(pIn3); - sqlite3VdbeRecordUnpack(pC->pKeyInfo, pIn3->n, pIn3->z, pIdxKey); + sqlite3VdbeRecordUnpack(pC->pKeyInfo, r.aMem->n, r.aMem->z, pIdxKey); + pIdxKey->default_rc = 0; + rc = sqlite3BtreeIndexMoveto(pC->uc.pCursor, pIdxKey, &pC->seekResult); + sqlite3DbFreeNN(db, pIdxKey); } - pIdxKey->default_rc = 0; - takeJump = 0; - if( pOp->opcode==OP_NoConflict ){ - /* For the OP_NoConflict opcode, take the jump if any of the - ** input fields are NULL, since any key with a NULL will not - ** conflict */ - for(ii=0; iinField; ii++){ - if( pIdxKey->aMem[ii].flags & MEM_Null ){ - takeJump = 1; - break; - } - } - } - rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, pIdxKey, 0, 0, &res); - sqlite3DbFree(db, pFree); if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - pC->seekResult = res; - alreadyExists = (res==0); + alreadyExists = (pC->seekResult==0); pC->nullRow = 1-alreadyExists; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; @@ -80689,22 +98749,44 @@ case OP_Found: { /* jump, in3 */ VdbeBranchTaken(alreadyExists!=0,2); if( alreadyExists ) goto jump_to_p2; }else{ - VdbeBranchTaken(takeJump||alreadyExists==0,2); - if( takeJump || !alreadyExists ) goto jump_to_p2; + if( !alreadyExists ){ + VdbeBranchTaken(1,2); + goto jump_to_p2; + } + if( pOp->opcode==OP_NoConflict ){ + /* For the OP_NoConflict opcode, take the jump if any of the + ** input fields are NULL, since any key with a NULL will not + ** conflict */ + for(ii=0; iiopcode==OP_IfNoHope ){ + pC->seekHit = pOp->p4.i; + } } break; } -/* Opcode: NotExists P1 P2 P3 * * +/* Opcode: SeekRowid P1 P2 P3 * * ** Synopsis: intkey=r[P3] ** ** P1 is the index of a cursor open on an SQL table btree (with integer -** keys). P3 is an integer rowid. If P1 does not contain a record with -** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an -** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** keys). If register P3 does not contain an integer or if P1 does not +** contain a record with rowid P3 then jump immediately to P2. +** Or, if P2 is 0, raise an SQLITE_CORRUPT error. If P1 does contain +** a record with rowid P3 then ** leave the cursor pointing at that record and fall through to the next ** instruction. ** +** The OP_NotExists opcode performs the same operation, but with OP_NotExists +** the P3 register must be guaranteed to contain an integer value. With this +** opcode, register P3 might not contain an integer. +** ** The OP_NotFound opcode performs the same operation on index btrees ** (with arbitrary multi-value keys). ** @@ -80712,29 +98794,73 @@ case OP_Found: { /* jump, in3 */ ** in either direction. In other words, the Next and Prev opcodes will ** not work following this opcode. ** -** See also: Found, NotFound, NoConflict +** See also: Found, NotFound, NoConflict, SeekRowid */ -case OP_NotExists: { /* jump, in3 */ +/* Opcode: NotExists P1 P2 P3 * * +** Synopsis: intkey=r[P3] +** +** P1 is the index of a cursor open on an SQL table btree (with integer +** keys). P3 is an integer rowid. If P1 does not contain a record with +** rowid P3 then jump immediately to P2. Or, if P2 is 0, raise an +** SQLITE_CORRUPT error. If P1 does contain a record with rowid P3 then +** leave the cursor pointing at that record and fall through to the next +** instruction. +** +** The OP_SeekRowid opcode performs the same operation but also allows the +** P3 register to contain a non-integer value, in which case the jump is +** always taken. This opcode requires that P3 always contain an integer. +** +** The OP_NotFound opcode performs the same operation on index btrees +** (with arbitrary multi-value keys). +** +** This opcode leaves the cursor in a state where it cannot be advanced +** in either direction. In other words, the Next and Prev opcodes will +** not work following this opcode. +** +** See also: Found, NotFound, NoConflict, SeekRowid +*/ +case OP_SeekRowid: { /* jump0, in3, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; u64 iKey; pIn3 = &aMem[pOp->p3]; - assert( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_Int ); + testcase( pIn3->flags & MEM_IntReal ); + testcase( pIn3->flags & MEM_Real ); + testcase( (pIn3->flags & (MEM_Str|MEM_Int))==MEM_Str ); + if( (pIn3->flags & (MEM_Int|MEM_IntReal))==0 ){ + /* If pIn3->u.i does not contain an integer, compute iKey as the + ** integer value of pIn3. Jump to P2 if pIn3 cannot be converted + ** into an integer without loss of information. Take care to avoid + ** changing the datatype of pIn3, however, as it is used by other + ** parts of the prepared statement. */ + Mem x = pIn3[0]; + applyAffinity(&x, SQLITE_AFF_NUMERIC, encoding); + if( (x.flags & MEM_Int)==0 ) goto jump_to_p2; + iKey = x.u.i; + goto notExistsWithKey; + } + /* Fall through into OP_NotExists */ + /* no break */ deliberate_fall_through +case OP_NotExists: /* jump, in3, ncycle */ + pIn3 = &aMem[pOp->p3]; + assert( (pIn3->flags & MEM_Int)!=0 || pOp->opcode==OP_SeekRowid ); assert( pOp->p1>=0 && pOp->p1nCursor ); + iKey = pIn3->u.i; +notExistsWithKey: pC = p->apCsr[pOp->p1]; assert( pC!=0 ); #ifdef SQLITE_DEBUG - pC->seekOp = 0; + if( pOp->opcode==OP_SeekRowid ) pC->seekOp = OP_SeekRowid; #endif assert( pC->isTable ); assert( pC->eCurType==CURTYPE_BTREE ); pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); res = 0; - iKey = pIn3->u.i; - rc = sqlite3BtreeMovetoUnpacked(pCrsr, 0, iKey, 0, &res); + rc = sqlite3BtreeTableMoveto(pCrsr, iKey, 0, &res); assert( rc==SQLITE_OK || res==0 ); pC->movetoTarget = iKey; /* Used by OP_Delete */ pC->nullRow = 0; @@ -80760,7 +98886,7 @@ case OP_NotExists: { /* jump, in3 */ ** Find the next available sequence number for cursor P1. ** Write the sequence number into register P2. ** The sequence number on the cursor is incremented after this -** instruction. +** instruction. */ case OP_Sequence: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); @@ -80780,9 +98906,9 @@ case OP_Sequence: { /* out2 */ ** table that cursor P1 points to. The new record number is written ** written to register P2. ** -** If P3>0 then P3 is a register in the root frame of this VDBE that holds +** If P3>0 then P3 is a register in the root frame of this VDBE that holds ** the largest previously generated record number. No new record numbers are -** allowed to be less than this value. When this value reaches its maximum, +** allowed to be less than this value. When this value reaches its maximum, ** an SQLITE_FULL error is generated. The P3 register is updated with the ' ** generated record number. This P3 mechanism is used to help implement the ** AUTOINCREMENT feature. @@ -80792,8 +98918,10 @@ case OP_NewRowid: { /* out2 */ VdbeCursor *pC; /* Cursor of table to get the new rowid */ int res; /* Result of an sqlite3BtreeLast() */ int cnt; /* Counter to limit the number of searches */ +#ifndef SQLITE_OMIT_AUTOINCREMENT Mem *pMem; /* Register holding largest rowid for AUTOINCREMENT */ VdbeFrame *pFrame; /* Root frame of VDBE */ +#endif v = 0; res = 0; @@ -80801,6 +98929,7 @@ case OP_NewRowid: { /* out2 */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); + assert( pC->isTable ); assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); { @@ -80838,8 +98967,7 @@ case OP_NewRowid: { /* out2 */ v = 1; /* IMP: R-61914-48074 */ }else{ assert( sqlite3BtreeCursorIsValid(pC->uc.pCursor) ); - rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v); - assert( rc==SQLITE_OK ); /* Cannot fail following BtreeLast() */ + v = sqlite3BtreeIntegerKey(pC->uc.pCursor); if( v>=MAX_ROWID ){ pC->useRandomRowid = 1; }else{ @@ -80869,7 +98997,7 @@ case OP_NewRowid: { /* out2 */ sqlite3VdbeMemIntegerify(pMem); assert( (pMem->flags & MEM_Int)!=0 ); /* mem(P3) holds an integer */ if( pMem->u.i==MAX_ROWID || pC->useRandomRowid ){ - rc = SQLITE_FULL; /* IMP: R-12275-61338 */ + rc = SQLITE_FULL; /* IMP: R-17817-00630 */ goto abort_due_to_error; } if( vu.i+1 ){ @@ -80889,7 +99017,7 @@ case OP_NewRowid: { /* out2 */ do{ sqlite3_randomness(sizeof(v), &v); v &= (MAX_ROWID>>1); v++; /* Ensure that v is greater than zero */ - }while( ((rc = sqlite3BtreeMovetoUnpacked(pC->uc.pCursor, 0, (u64)v, + }while( ((rc = sqlite3BtreeTableMoveto(pC->uc.pCursor, (u64)v, 0, &res))==SQLITE_OK) && (res==0) && (++cnt<100)); @@ -80921,21 +99049,18 @@ case OP_NewRowid: { /* out2 */ ** then rowid is stored for subsequent return by the ** sqlite3_last_insert_rowid() function (otherwise it is unmodified). ** -** If the OPFLAG_USESEEKRESULT flag of P5 is set and if the result of -** the last seek operation (OP_NotExists) was a success, then this -** operation will not attempt to find the appropriate row before doing -** the insert but will instead overwrite the row that the cursor is -** currently pointing to. Presumably, the prior OP_NotExists opcode -** has already positioned the cursor correctly. This is an optimization -** that boosts performance by avoiding redundant seeks. +** If the OPFLAG_USESEEKRESULT flag of P5 is set, the implementation might +** run faster by avoiding an unnecessary seek on cursor P1. However, +** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior +** seeks on the cursor or if the most recent seek used a key equal to P3. ** ** If the OPFLAG_ISUPDATE flag is set, then this opcode is part of an ** UPDATE operation. Otherwise (if the flag is clear) then this opcode ** is part of an INSERT operation. The difference is only important to ** the update hook. ** -** Parameter P4 may point to a Table structure, or may be NULL. If it is -** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked +** Parameter P4 may point to a Table structure, or may be NULL. If it is +** not NULL, then the update-hook (sqlite3.xUpdateCallback) is invoked ** following a successful insert. ** ** (WARNING/TODO: If P1 is a pseudo-cursor and P2 is dynamically @@ -80947,98 +99072,121 @@ case OP_NewRowid: { /* out2 */ ** This instruction only works on tables. The equivalent instruction ** for indices is OP_IdxInsert. */ -/* Opcode: InsertInt P1 P2 P3 P4 P5 -** Synopsis: intkey=P3 data=r[P2] -** -** This works exactly like OP_Insert except that the key is the -** integer value P3, not the value of the integer stored in register P3. -*/ -case OP_Insert: -case OP_InsertInt: { +case OP_Insert: { Mem *pData; /* MEM cell holding data for the record to be inserted */ Mem *pKey; /* MEM cell holding key for the record */ - i64 iKey; /* The integer ROWID or key for the record to be inserted */ VdbeCursor *pC; /* Cursor to table into which insert is written */ - int nZero; /* Number of zero-bytes to append */ int seekResult; /* Result of prior seek or 0 if no USESEEKRESULT flag */ const char *zDb; /* database name - used by the update hook */ Table *pTab; /* Table structure - used by update and pre-update hooks */ - int op; /* Opcode for update hook: SQLITE_UPDATE or SQLITE_INSERT */ + BtreePayload x; /* Payload to be inserted */ - op = 0; pData = &aMem[pOp->p2]; assert( pOp->p1>=0 && pOp->p1nCursor ); assert( memIsValid(pData) ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->deferredMoveto==0 ); assert( pC->uc.pCursor!=0 ); - assert( pC->isTable ); + assert( (pOp->p5 & OPFLAG_ISNOOP) || pC->isTable ); assert( pOp->p4type==P4_TABLE || pOp->p4type>=P4_STATIC ); REGISTER_TRACE(pOp->p2, pData); + sqlite3VdbeIncrWriteCounter(p, pC); - if( pOp->opcode==OP_Insert ){ - pKey = &aMem[pOp->p3]; - assert( pKey->flags & MEM_Int ); - assert( memIsValid(pKey) ); - REGISTER_TRACE(pOp->p3, pKey); - iKey = pKey->u.i; - }else{ - assert( pOp->opcode==OP_InsertInt ); - iKey = pOp->p3; - } + pKey = &aMem[pOp->p3]; + assert( pKey->flags & MEM_Int ); + assert( memIsValid(pKey) ); + REGISTER_TRACE(pOp->p3, pKey); + x.nKey = pKey->u.i; if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ - assert( pC->isTable ); assert( pC->iDb>=0 ); - zDb = db->aDb[pC->iDb].zName; + zDb = db->aDb[pC->iDb].zDbSName; pTab = pOp->p4.pTab; - assert( HasRowid(pTab) ); - op = ((pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT); + assert( (pOp->p5 & OPFLAG_ISNOOP) || HasRowid(pTab) ); }else{ - pTab = 0; /* Not needed. Silence a comiler warning. */ - zDb = 0; /* Not needed. Silence a compiler warning. */ + pTab = 0; + zDb = 0; } #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Invoke the pre-update hook, if any */ - if( db->xPreUpdateCallback - && pOp->p4type==P4_TABLE - && !(pOp->p5 & OPFLAG_ISUPDATE) - ){ - sqlite3VdbePreUpdateHook(p, pC, SQLITE_INSERT, zDb, pTab, iKey, pOp->p2); + if( pTab ){ + if( db->xPreUpdateCallback && !(pOp->p5 & OPFLAG_ISUPDATE) ){ + sqlite3VdbePreUpdateHook(p,pC,SQLITE_INSERT,zDb,pTab,x.nKey,pOp->p2,-1); + } + if( db->xUpdateCallback==0 || pTab->aCol==0 ){ + /* Prevent post-update hook from running in cases when it should not */ + pTab = 0; + } } + if( pOp->p5 & OPFLAG_ISNOOP ) break; #endif - if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = lastRowid = iKey; - if( pData->flags & MEM_Null ){ - pData->z = 0; - pData->n = 0; - }else{ - assert( pData->flags & (MEM_Blob|MEM_Str) ); + assert( (pOp->p5 & OPFLAG_LASTROWID)==0 || (pOp->p5 & OPFLAG_NCHANGE)!=0 ); + if( pOp->p5 & OPFLAG_NCHANGE ){ + p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; } + assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); + x.pData = pData->z; + x.nData = pData->n; seekResult = ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0); if( pData->flags & MEM_Zero ){ - nZero = pData->u.nZero; + x.nZero = pData->u.nZero; }else{ - nZero = 0; + x.nZero = 0; } - rc = sqlite3BtreeInsert(pC->uc.pCursor, 0, iKey, - pData->z, pData->n, nZero, - (pOp->p5 & OPFLAG_APPEND)!=0, seekResult + x.pKey = 0; + assert( BTREE_PREFORMAT==OPFLAG_PREFORMAT ); + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), + seekResult ); pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; + colCacheCtr++; /* Invoke the update-hook if required. */ if( rc ) goto abort_due_to_error; - if( db->xUpdateCallback && op ){ - db->xUpdateCallback(db->pUpdateArg, op, zDb, pTab->zName, iKey); + if( pTab ){ + assert( db->xUpdateCallback!=0 ); + assert( pTab->aCol!=0 ); + db->xUpdateCallback(db->pUpdateArg, + (pOp->p5 & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_INSERT, + zDb, pTab->zName, x.nKey); } break; } +/* Opcode: RowCell P1 P2 P3 * * +** +** P1 and P2 are both open cursors. Both must be opened on the same type +** of table - intkey or index. This opcode is used as part of copying +** the current row from P2 into P1. If the cursors are opened on intkey +** tables, register P3 contains the rowid to use with the new record in +** P1. If they are opened on index tables, P3 is not used. +** +** This opcode must be followed by either an Insert or InsertIdx opcode +** with the OPFLAG_PREFORMAT flag set to complete the insert operation. +*/ +case OP_RowCell: { + VdbeCursor *pDest; /* Cursor to write to */ + VdbeCursor *pSrc; /* Cursor to read from */ + i64 iKey; /* Rowid value to insert with */ + assert( pOp[1].opcode==OP_Insert || pOp[1].opcode==OP_IdxInsert ); + assert( pOp[1].opcode==OP_Insert || pOp->p3==0 ); + assert( pOp[1].opcode==OP_IdxInsert || pOp->p3>0 ); + assert( pOp[1].p5 & OPFLAG_PREFORMAT ); + pDest = p->apCsr[pOp->p1]; + pSrc = p->apCsr[pOp->p2]; + iKey = pOp->p3 ? aMem[pOp->p3].u.i : 0; + rc = sqlite3BtreeTransferRow(pDest->uc.pCursor, pSrc->uc.pCursor, iKey); + if( rc!=SQLITE_OK ) goto abort_due_to_error; + break; +}; + /* Opcode: Delete P1 P2 P3 P4 P5 ** ** Delete the record at which the P1 cursor is currently pointing. @@ -81047,27 +99195,32 @@ case OP_InsertInt: { ** the cursor will be left pointing at either the next or the previous ** record in the table. If it is left pointing at the next record, then ** the next Next instruction will be a no-op. As a result, in this case -** it is ok to delete a record from within a Next loop. If +** it is ok to delete a record from within a Next loop. If ** OPFLAG_SAVEPOSITION bit of P5 is clear, then the cursor will be ** left in an undefined state. ** ** If the OPFLAG_AUXDELETE bit is set on P5, that indicates that this -** delete one of several associated with deleting a table row and all its -** associated index entries. Exactly one of those deletes is the "primary" -** delete. The others are all on OPFLAG_FORDELETE cursors or else are -** marked with the AUXDELETE flag. +** delete is one of several associated with deleting a table row and +** all its associated index entries. Exactly one of those deletes is +** the "primary" delete. The others are all on OPFLAG_FORDELETE +** cursors or else are marked with the AUXDELETE flag. ** -** If the OPFLAG_NCHANGE flag of P2 (NB: P2 not P5) is set, then the row -** change count is incremented (otherwise not). +** If the OPFLAG_NCHANGE (0x01) flag of P2 (NB: P2 not P5) is set, then +** the row change count is incremented (otherwise not). +** +** If the OPFLAG_ISNOOP (0x40) flag of P2 (not P5!) is set, then the +** pre-update-hook for deletes is run, but the btree is otherwise unchanged. +** This happens when the OP_Delete is to be shortly followed by an OP_Insert +** with the same key, causing the btree entry to be overwritten. ** ** P1 must not be pseudo-table. It has to be a real table with ** multiple rows. ** -** If P4 is not NULL then it points to a Table struture. In this case either +** If P4 is not NULL then it points to a Table object. In this case either ** the update or pre-update hook, or both, may be invoked. The P1 cursor must -** have been positioned using OP_NotFound prior to invoking this opcode in -** this case. Specifically, if one is configured, the pre-update hook is -** invoked if P4 is not NULL. The update-hook is invoked if one is configured, +** have been positioned using OP_NotFound prior to invoking this opcode in +** this case. Specifically, if one is configured, the pre-update hook is +** invoked if P4 is not NULL. The update-hook is invoked if one is configured, ** P4 is not NULL, and the OPFLAG_NCHANGE flag is set in P2. ** ** If the OPFLAG_ISUPDATE flag is set in P2, then P3 contains the address @@ -81087,50 +99240,58 @@ case OP_Delete: { assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0 ); assert( pC->deferredMoveto==0 ); + sqlite3VdbeIncrWriteCounter(p, pC); #ifdef SQLITE_DEBUG - if( pOp->p4type==P4_TABLE && HasRowid(pOp->p4.pTab) && pOp->p5==0 ){ + if( pOp->p4type==P4_TABLE + && HasRowid(pOp->p4.pTab) + && pOp->p5==0 + && sqlite3BtreeCursorIsValidNN(pC->uc.pCursor) + ){ /* If p5 is zero, the seek operation that positioned the cursor prior to ** OP_Delete will have also set the pC->movetoTarget field to the rowid of ** the row that is being deleted */ - i64 iKey = 0; - sqlite3BtreeKeySize(pC->uc.pCursor, &iKey); - assert( pC->movetoTarget==iKey ); + i64 iKey = sqlite3BtreeIntegerKey(pC->uc.pCursor); + assert( CORRUPT_DB || pC->movetoTarget==iKey ); } #endif /* If the update-hook or pre-update-hook will be invoked, set zDb to ** the name of the db to pass as to it. Also set local pTab to a copy ** of p4.pTab. Finally, if p5 is true, indicating that this cursor was - ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set + ** last moved with OP_Next or OP_Prev, not Seek or NotFound, set ** VdbeCursor.movetoTarget to the current rowid. */ if( pOp->p4type==P4_TABLE && HAS_UPDATE_HOOK(db) ){ assert( pC->iDb>=0 ); assert( pOp->p4.pTab!=0 ); - zDb = db->aDb[pC->iDb].zName; + zDb = db->aDb[pC->iDb].zDbSName; pTab = pOp->p4.pTab; if( (pOp->p5 & OPFLAG_SAVEPOSITION)!=0 && pC->isTable ){ - sqlite3BtreeKeySize(pC->uc.pCursor, &pC->movetoTarget); + pC->movetoTarget = sqlite3BtreeIntegerKey(pC->uc.pCursor); } }else{ - zDb = 0; /* Not needed. Silence a compiler warning. */ - pTab = 0; /* Not needed. Silence a compiler warning. */ + zDb = 0; + pTab = 0; } #ifdef SQLITE_ENABLE_PREUPDATE_HOOK /* Invoke the pre-update-hook if required. */ - if( db->xPreUpdateCallback && pOp->p4.pTab && HasRowid(pTab) ){ - assert( !(opflags & OPFLAG_ISUPDATE) || (aMem[pOp->p3].flags & MEM_Int) ); + assert( db->xPreUpdateCallback==0 || pTab==pOp->p4.pTab ); + if( db->xPreUpdateCallback && pTab ){ + assert( !(opflags & OPFLAG_ISUPDATE) + || HasRowid(pTab)==0 + || (aMem[pOp->p3].flags & MEM_Int) + ); sqlite3VdbePreUpdateHook(p, pC, - (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, + (opflags & OPFLAG_ISUPDATE) ? SQLITE_UPDATE : SQLITE_DELETE, zDb, pTab, pC->movetoTarget, - pOp->p3 + pOp->p3, -1 ); } if( opflags & OPFLAG_ISNOOP ) break; #endif - - /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ + + /* Only flags that can be set are SAVEPOISTION and AUXDELETE */ assert( (pOp->p5 & ~(OPFLAG_SAVEPOSITION|OPFLAG_AUXDELETE))==0 ); assert( OPFLAG_SAVEPOSITION==BTREE_SAVEPOSITION ); assert( OPFLAG_AUXDELETE==BTREE_AUXDELETE ); @@ -81151,12 +99312,14 @@ case OP_Delete: { rc = sqlite3BtreeDelete(pC->uc.pCursor, pOp->p5); pC->cacheStatus = CACHE_STALE; + colCacheCtr++; + pC->seekResult = 0; if( rc ) goto abort_due_to_error; /* Invoke the update-hook if required. */ if( opflags & OPFLAG_NCHANGE ){ p->nChange++; - if( db->xUpdateCallback && HasRowid(pTab) ){ + if( db->xUpdateCallback && ALWAYS(pTab!=0) && HasRowid(pTab) ){ db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, pC->movetoTarget); assert( pC->iDb>=0 ); @@ -81179,10 +99342,10 @@ case OP_ResetCount: { } /* Opcode: SorterCompare P1 P2 P3 P4 -** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 +** Synopsis: if key(P1)!=trim(r[P3],P4) goto P2 ** ** P1 is a sorter cursor. This instruction compares a prefix of the -** record blob in register P3 against a prefix of the entry that +** record blob in register P3 against a prefix of the entry that ** the sorter cursor currently points to. Only the first P4 fields ** of r[P3] and the sorter record are compared. ** @@ -81217,13 +99380,13 @@ case OP_SorterCompare: { ** Write into register P2 the current sorter data for sorter cursor P1. ** Then clear the column header cache on cursor P3. ** -** This opcode is normally use to move a record out of the sorter and into +** This opcode is normally used to move a record out of the sorter and into ** a register that is the source for a pseudo-table cursor created using ** OpenPseudo. That pseudo-table cursor is the one that is identified by ** parameter P3. Clearing the P3 column cache as part of this opcode saves ** us from having to issue a separate NullRow instruction to clear that cache. */ -case OP_SorterData: { +case OP_SorterData: { /* ncycle */ VdbeCursor *pC; pOut = &aMem[pOp->p2]; @@ -81237,99 +99400,76 @@ case OP_SorterData: { break; } -/* Opcode: RowData P1 P2 * * * +/* Opcode: RowData P1 P2 P3 * * ** Synopsis: r[P2]=data ** -** Write into register P2 the complete row data for cursor P1. -** There is no interpretation of the data. -** It is just copied onto the P2 register exactly as +** Write into register P2 the complete row content for the row at +** which cursor P1 is currently pointing. +** There is no interpretation of the data. +** It is just copied onto the P2 register exactly as ** it is found in the database file. ** +** If cursor P1 is an index, then the content is the key of the row. +** If cursor P2 is a table, then the content extracted is the data. +** ** If the P1 cursor must be pointing to a valid row (not a NULL row) ** of a real table, not a pseudo-table. -*/ -/* Opcode: RowKey P1 P2 * * * -** Synopsis: r[P2]=key ** -** Write into register P2 the complete row key for cursor P1. -** There is no interpretation of the data. -** The key is copied onto the P2 register exactly as -** it is found in the database file. +** If P3!=0 then this opcode is allowed to make an ephemeral pointer +** into the database page. That means that the content of the output +** register will be invalidated as soon as the cursor moves - including +** moves caused by other cursors that "save" the current cursors +** position in order that they can write to the same table. If P3==0 +** then a copy of the data is made into memory. P3!=0 is faster, but +** P3==0 is safer. ** -** If the P1 cursor must be pointing to a valid row (not a NULL row) -** of a real table, not a pseudo-table. +** If P3!=0 then the content of the P2 register is unsuitable for use +** in OP_Result and any OP_Result will invalidate the P2 register content. +** The P2 register content is invalidated by opcodes like OP_Function or +** by any use of another cursor pointing to the same table. */ -case OP_RowKey: case OP_RowData: { VdbeCursor *pC; BtCursor *pCrsr; u32 n; - i64 n64; - pOut = &aMem[pOp->p2]; - memAboutToChange(p, pOut); + pOut = out2Prerelease(p, pOp); - /* Note that RowKey and RowData are really exactly the same instruction */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); assert( isSorter(pC)==0 ); - assert( pC->isTable || pOp->opcode!=OP_RowData ); - assert( pC->isTable==0 || pOp->opcode==OP_RowData ); assert( pC->nullRow==0 ); assert( pC->uc.pCursor!=0 ); pCrsr = pC->uc.pCursor; - /* The OP_RowKey and OP_RowData opcodes always follow OP_NotExists or - ** OP_Rewind/Op_Next with no intervening instructions that might invalidate - ** the cursor. If this where not the case, on of the following assert()s + /* The OP_RowData opcodes always follow OP_NotExists or + ** OP_SeekRowid or OP_Rewind/Op_Next with no intervening instructions + ** that might invalidate the cursor. + ** If this where not the case, on of the following assert()s ** would fail. Should this ever change (because of changes in the code ** generator) then the fix would be to insert a call to ** sqlite3VdbeCursorMoveto(). */ assert( pC->deferredMoveto==0 ); assert( sqlite3BtreeCursorIsValid(pCrsr) ); -#if 0 /* Not required due to the previous to assert() statements */ - rc = sqlite3VdbeCursorMoveto(pC); - if( rc!=SQLITE_OK ) goto abort_due_to_error; -#endif - if( pC->isTable==0 ){ - assert( !pC->isTable ); - VVA_ONLY(rc =) sqlite3BtreeKeySize(pCrsr, &n64); - assert( rc==SQLITE_OK ); /* True because of CursorMoveto() call above */ - if( n64>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } - n = (u32)n64; - }else{ - VVA_ONLY(rc =) sqlite3BtreeDataSize(pCrsr, &n); - assert( rc==SQLITE_OK ); /* DataSize() cannot fail */ - if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ - goto too_big; - } + n = sqlite3BtreePayloadSize(pCrsr); + if( n>(u32)db->aLimit[SQLITE_LIMIT_LENGTH] ){ + goto too_big; } testcase( n==0 ); - if( sqlite3VdbeMemClearAndResize(pOut, MAX(n,32)) ){ - goto no_mem; - } - pOut->n = n; - MemSetTypeFlag(pOut, MEM_Blob); - if( pC->isTable==0 ){ - rc = sqlite3BtreeKey(pCrsr, 0, n, pOut->z); - }else{ - rc = sqlite3BtreeData(pCrsr, 0, n, pOut->z); - } + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCrsr, n, pOut); if( rc ) goto abort_due_to_error; - pOut->enc = SQLITE_UTF8; /* In case the blob is ever cast to text */ + if( !pOp->p3 ) Deephemeralize(pOut); UPDATE_MAX_BLOBSIZE(pOut); REGISTER_TRACE(pOp->p2, pOut); break; } /* Opcode: Rowid P1 P2 * * * -** Synopsis: r[P2]=rowid +** Synopsis: r[P2]=PX rowid of P1 ** ** Store in register P2 an integer which is the key of the table entry that ** P1 is currently point to. @@ -81338,7 +99478,7 @@ case OP_RowData: { ** be a separate OP_VRowid opcode for use with virtual tables, but this ** one opcode now works for both table types. */ -case OP_Rowid: { /* out2 */ +case OP_Rowid: { /* out2, ncycle */ VdbeCursor *pC; i64 v; sqlite3_vtab *pVtab; @@ -81373,8 +99513,7 @@ case OP_Rowid: { /* out2 */ pOut->flags = MEM_Null; break; } - rc = sqlite3BtreeKeySize(pC->uc.pCursor, &v); - assert( rc==SQLITE_OK ); /* Always so because of CursorRestore() above */ + v = sqlite3BtreeIntegerKey(pC->uc.pCursor); } pOut->u.i = v; break; @@ -81385,25 +99524,50 @@ case OP_Rowid: { /* out2 */ ** Move the cursor P1 to a null row. Any OP_Column operations ** that occur while the cursor is on the null row will always ** write a NULL. +** +** If cursor P1 is not previously opened, open it now to a special +** pseudo-cursor that always returns NULL for every column. */ case OP_NullRow: { VdbeCursor *pC; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; - assert( pC!=0 ); + if( pC==0 ){ + /* If the cursor is not already open, create a special kind of + ** pseudo-cursor that always gives null rows. */ + pC = allocateCursor(p, pOp->p1, 1, CURTYPE_PSEUDO); + if( pC==0 ) goto no_mem; + pC->seekResult = 0; + pC->isTable = 1; + pC->noReuse = 1; + pC->uc.pCursor = sqlite3BtreeFakeValidCursor(); + } pC->nullRow = 1; pC->cacheStatus = CACHE_STALE; if( pC->eCurType==CURTYPE_BTREE ){ assert( pC->uc.pCursor!=0 ); sqlite3BtreeClearCursor(pC->uc.pCursor); } +#ifdef SQLITE_DEBUG + if( pC->seekOp==0 ) pC->seekOp = OP_NullRow; +#endif break; } -/* Opcode: Last P1 P2 P3 * * +/* Opcode: SeekEnd P1 * * * * ** -** The next use of the Rowid or Column or Prev instruction for P1 +** Position cursor P1 at the end of the btree for the purpose of +** appending a new entry onto the btree. +** +** It is assumed that the cursor is used only for appending and so +** if the cursor is valid, then the cursor must already be pointing +** at the end of the btree and so no changes are made to +** the cursor. +*/ +/* Opcode: Last P1 P2 * * * +** +** The next use of the Rowid or Column or Prev instruction for P1 ** will refer to the last entry in the database table or index. ** If the table or index is empty and P2>0, then jump immediately to P2. ** If P2 is 0 or if the table or index is not empty, fall through @@ -81413,7 +99577,8 @@ case OP_NullRow: { ** from the end toward the beginning. In other words, the cursor is ** configured to use Prev, not Next. */ -case OP_Last: { /* jump */ +case OP_SeekEnd: /* ncycle */ +case OP_Last: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; @@ -81425,14 +99590,20 @@ case OP_Last: { /* jump */ pCrsr = pC->uc.pCursor; res = 0; assert( pCrsr!=0 ); +#ifdef SQLITE_DEBUG + pC->seekOp = pOp->opcode; +#endif + if( pOp->opcode==OP_SeekEnd ){ + assert( pOp->p2==0 ); + pC->seekResult = -1; + if( sqlite3BtreeCursorIsValidNN(pCrsr) ){ + break; + } + } rc = sqlite3BtreeLast(pCrsr, &res); pC->nullRow = (u8)res; pC->deferredMoveto = 0; pC->cacheStatus = CACHE_STALE; - pC->seekResult = pOp->p3; -#ifdef SQLITE_DEBUG - pC->seekOp = OP_Last; -#endif if( rc ) goto abort_due_to_error; if( pOp->p2>0 ){ VdbeBranchTaken(res!=0,2); @@ -81441,7 +99612,53 @@ case OP_Last: { /* jump */ break; } +/* Opcode: IfSizeBetween P1 P2 P3 P4 * +** +** Let N be the approximate number of rows in the table or index +** with cursor P1 and let X be 10*log2(N) if N is positive or -1 +** if N is zero. +** +** Jump to P2 if X is in between P3 and P4, inclusive. +*/ +case OP_IfSizeBetween: { /* jump */ + VdbeCursor *pC; + BtCursor *pCrsr; + int res; + i64 sz; + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p4type==P4_INT32 ); + assert( pOp->p3>=-1 && pOp->p3<=640*2 ); + assert( pOp->p4.i>=-1 && pOp->p4.i<=640*2 ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + pCrsr = pC->uc.pCursor; + assert( pCrsr ); + rc = sqlite3BtreeFirst(pCrsr, &res); + if( rc ) goto abort_due_to_error; + if( res!=0 ){ + sz = -1; /* -Infinity encoding */ + }else{ + sz = sqlite3BtreeRowCountEst(pCrsr); + assert( sz>0 ); + sz = sqlite3LogEst((u64)sz); + } + res = sz>=pOp->p3 && sz<=pOp->p4.i; + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; +} + + +/* Opcode: SorterSort P1 P2 * * * +** +** After all records have been inserted into the Sorter object +** identified by P1, invoke this opcode to actually do the sorting. +** Jump to P2 if there are no records to be sorted. +** +** This opcode is an alias for OP_Sort and OP_Rewind that is used +** for Sorter objects. +*/ /* Opcode: Sort P1 P2 * * * ** ** This opcode does exactly the same thing as OP_Rewind except that @@ -81454,33 +99671,40 @@ case OP_Last: { /* jump */ ** regression tests can determine whether or not the optimizer is ** correctly optimizing out sorts. */ -case OP_SorterSort: /* jump */ -case OP_Sort: { /* jump */ +case OP_SorterSort: /* jump ncycle */ +case OP_Sort: { /* jump ncycle */ #ifdef SQLITE_TEST sqlite3_sort_count++; sqlite3_search_count--; #endif p->aCounter[SQLITE_STMTSTATUS_SORT]++; /* Fall through into OP_Rewind */ + /* no break */ deliberate_fall_through } /* Opcode: Rewind P1 P2 * * * ** -** The next use of the Rowid or Column or Next instruction for P1 +** The next use of the Rowid or Column or Next instruction for P1 ** will refer to the first entry in the database table or index. ** If the table or index is empty, jump immediately to P2. -** If the table or index is not empty, fall through to the following +** If the table or index is not empty, fall through to the following ** instruction. ** +** If P2 is zero, that is an assertion that the P1 table is never +** empty and hence the jump will never be taken. +** ** This opcode leaves the cursor configured to move in forward order, ** from the beginning toward the end. In other words, the cursor is ** configured to use Next, not Prev. */ -case OP_Rewind: { /* jump */ +case OP_Rewind: { /* jump0, ncycle */ VdbeCursor *pC; BtCursor *pCrsr; int res; assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5==0 ); + assert( pOp->p2>=0 && pOp->p2nOp ); + pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( isSorter(pC)==(pOp->opcode==OP_SorterSort) ); @@ -81500,13 +99724,14 @@ case OP_Rewind: { /* jump */ } if( rc ) goto abort_due_to_error; pC->nullRow = (u8)res; - assert( pOp->p2>0 && pOp->p2nOp ); - VdbeBranchTaken(res!=0,2); - if( res ) goto jump_to_p2; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + } break; } -/* Opcode: Next P1 P2 P3 P4 P5 +/* Opcode: Next P1 P2 P3 * P5 ** ** Advance cursor P1 so that it points to the next key/data pair in its ** table or index. If there are no more key/value pairs then fall through @@ -81525,20 +99750,12 @@ case OP_Rewind: { /* jump */ ** omitted if that index had been unique. P3 is usually 0. P3 is ** always either 0 or 1. ** -** P4 is always of type P4_ADVANCE. The function pointer points to -** sqlite3BtreeNext(). -** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. ** -** See also: Prev, NextIfOpen +** See also: Prev */ -/* Opcode: NextIfOpen P1 P2 P3 P4 P5 -** -** This opcode works just like Next except that if cursor P1 is not -** open it behaves a no-op. -*/ -/* Opcode: Prev P1 P2 P3 P4 P5 +/* Opcode: Prev P1 P2 P3 * P5 ** ** Back up cursor P1 so that it points to the previous key/data pair in its ** table or index. If there is no previous key/value pairs then fall through @@ -81558,132 +99775,168 @@ case OP_Rewind: { /* jump */ ** omitted if that index had been unique. P3 is usually 0. P3 is ** always either 0 or 1. ** -** P4 is always of type P4_ADVANCE. The function pointer points to -** sqlite3BtreePrevious(). -** ** If P5 is positive and the jump is taken, then event counter ** number P5-1 in the prepared statement is incremented. */ -/* Opcode: PrevIfOpen P1 P2 P3 P4 P5 +/* Opcode: SorterNext P1 P2 * * P5 ** -** This opcode works just like Prev except that if cursor P1 is not -** open it behaves a no-op. +** This opcode works just like OP_Next except that P1 must be a +** sorter object for which the OP_SorterSort opcode has been +** invoked. This opcode advances the cursor to the next sorted +** record, or jumps to P2 if there are no more sorted records. */ case OP_SorterNext: { /* jump */ VdbeCursor *pC; - int res; pC = p->apCsr[pOp->p1]; assert( isSorter(pC) ); - res = 0; - rc = sqlite3VdbeSorterNext(db, pC, &res); + rc = sqlite3VdbeSorterNext(db, pC); goto next_tail; -case OP_PrevIfOpen: /* jump */ -case OP_NextIfOpen: /* jump */ - if( p->apCsr[pOp->p1]==0 ) break; - /* Fall through */ -case OP_Prev: /* jump */ -case OP_Next: /* jump */ + +case OP_Prev: /* jump, ncycle */ assert( pOp->p1>=0 && pOp->p1nCursor ); - assert( pOp->p5aCounter) ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); pC = p->apCsr[pOp->p1]; - res = pOp->p3; assert( pC!=0 ); assert( pC->deferredMoveto==0 ); assert( pC->eCurType==CURTYPE_BTREE ); - assert( res==0 || (res==1 && pC->isTable==0) ); - testcase( res==1 ); - assert( pOp->opcode!=OP_Next || pOp->p4.xAdvance==sqlite3BtreeNext ); - assert( pOp->opcode!=OP_Prev || pOp->p4.xAdvance==sqlite3BtreePrevious ); - assert( pOp->opcode!=OP_NextIfOpen || pOp->p4.xAdvance==sqlite3BtreeNext ); - assert( pOp->opcode!=OP_PrevIfOpen || pOp->p4.xAdvance==sqlite3BtreePrevious); + assert( pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE + || pC->seekOp==OP_Last || pC->seekOp==OP_IfNoHope + || pC->seekOp==OP_NullRow); + rc = sqlite3BtreePrevious(pC->uc.pCursor, pOp->p3); + goto next_tail; - /* The Next opcode is only used after SeekGT, SeekGE, and Rewind. - ** The Prev opcode is only used after SeekLT, SeekLE, and Last. */ - assert( pOp->opcode!=OP_Next || pOp->opcode!=OP_NextIfOpen - || pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE - || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found); - assert( pOp->opcode!=OP_Prev || pOp->opcode!=OP_PrevIfOpen - || pC->seekOp==OP_SeekLT || pC->seekOp==OP_SeekLE - || pC->seekOp==OP_Last ); +case OP_Next: /* jump, ncycle */ + assert( pOp->p1>=0 && pOp->p1nCursor ); + assert( pOp->p5==0 + || pOp->p5==SQLITE_STMTSTATUS_FULLSCAN_STEP + || pOp->p5==SQLITE_STMTSTATUS_AUTOINDEX); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->deferredMoveto==0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->seekOp==OP_SeekGT || pC->seekOp==OP_SeekGE + || pC->seekOp==OP_Rewind || pC->seekOp==OP_Found + || pC->seekOp==OP_NullRow|| pC->seekOp==OP_SeekRowid + || pC->seekOp==OP_IfNoHope); + rc = sqlite3BtreeNext(pC->uc.pCursor, pOp->p3); - rc = pOp->p4.xAdvance(pC->uc.pCursor, &res); next_tail: pC->cacheStatus = CACHE_STALE; - VdbeBranchTaken(res==0,2); - if( rc ) goto abort_due_to_error; - if( res==0 ){ + VdbeBranchTaken(rc==SQLITE_OK,2); + if( rc==SQLITE_OK ){ pC->nullRow = 0; p->aCounter[pOp->p5]++; #ifdef SQLITE_TEST sqlite3_search_count++; #endif goto jump_to_p2_and_check_for_interrupt; - }else{ - pC->nullRow = 1; } + if( rc!=SQLITE_DONE ) goto abort_due_to_error; + rc = SQLITE_OK; + pC->nullRow = 1; goto check_for_interrupt; } -/* Opcode: IdxInsert P1 P2 P3 * P5 +/* Opcode: IdxInsert P1 P2 P3 P4 P5 ** Synopsis: key=r[P2] ** ** Register P2 holds an SQL index key made using the ** MakeRecord instructions. This opcode writes that key ** into the index P1. Data for the entry is nil. ** -** P3 is a flag that provides a hint to the b-tree layer that this -** insert is likely to be an append. +** If P4 is not zero, then it is the number of values in the unpacked +** key of reg(P2). In that case, P3 is the index of the first register +** for the unpacked key. The availability of the unpacked key can sometimes +** be an optimization. +** +** If P5 has the OPFLAG_APPEND bit set, that is a hint to the b-tree layer +** that this insert is likely to be an append. ** ** If P5 has the OPFLAG_NCHANGE bit set, then the change counter is ** incremented by this instruction. If the OPFLAG_NCHANGE bit is clear, ** then the change counter is unchanged. ** -** If P5 has the OPFLAG_USESEEKRESULT bit set, then the cursor must have -** just done a seek to the spot where the new entry is to be inserted. -** This flag avoids doing an extra seek. +** If the OPFLAG_USESEEKRESULT flag of P5 is set, the implementation might +** run faster by avoiding an unnecessary seek on cursor P1. However, +** the OPFLAG_USESEEKRESULT flag must only be set if there have been no prior +** seeks on the cursor or if the most recent seek used a key equivalent +** to P2. ** ** This instruction only works for indices. The equivalent instruction ** for tables is OP_Insert. */ -case OP_SorterInsert: /* in2 */ case OP_IdxInsert: { /* in2 */ VdbeCursor *pC; - int nKey; - const char *zKey; + BtreePayload x; assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; + sqlite3VdbeIncrWriteCounter(p, pC); assert( pC!=0 ); - assert( isSorter(pC)==(pOp->opcode==OP_SorterInsert) ); + assert( !isSorter(pC) ); pIn2 = &aMem[pOp->p2]; - assert( pIn2->flags & MEM_Blob ); + assert( (pIn2->flags & MEM_Blob) || (pOp->p5 & OPFLAG_PREFORMAT) ); if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; - assert( pC->eCurType==CURTYPE_BTREE || pOp->opcode==OP_SorterInsert ); + assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->isTable==0 ); rc = ExpandBlob(pIn2); if( rc ) goto abort_due_to_error; - if( pOp->opcode==OP_SorterInsert ){ - rc = sqlite3VdbeSorterWrite(pC, pIn2); - }else{ - nKey = pIn2->n; - zKey = pIn2->z; - rc = sqlite3BtreeInsert(pC->uc.pCursor, zKey, nKey, "", 0, 0, pOp->p3, - ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) - ); - assert( pC->deferredMoveto==0 ); - pC->cacheStatus = CACHE_STALE; - } + x.nKey = pIn2->n; + x.pKey = pIn2->z; + x.aMem = aMem + pOp->p3; + x.nMem = (u16)pOp->p4.i; + rc = sqlite3BtreeInsert(pC->uc.pCursor, &x, + (pOp->p5 & (OPFLAG_APPEND|OPFLAG_SAVEPOSITION|OPFLAG_PREFORMAT)), + ((pOp->p5 & OPFLAG_USESEEKRESULT) ? pC->seekResult : 0) + ); + assert( pC->deferredMoveto==0 ); + pC->cacheStatus = CACHE_STALE; if( rc) goto abort_due_to_error; break; } -/* Opcode: IdxDelete P1 P2 P3 * * +/* Opcode: SorterInsert P1 P2 * * * +** Synopsis: key=r[P2] +** +** Register P2 holds an SQL index key made using the +** MakeRecord instructions. This opcode writes that key +** into the sorter P1. Data for the entry is nil. +*/ +case OP_SorterInsert: { /* in2 */ + VdbeCursor *pC; + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + sqlite3VdbeIncrWriteCounter(p, pC); + assert( pC!=0 ); + assert( isSorter(pC) ); + pIn2 = &aMem[pOp->p2]; + assert( pIn2->flags & MEM_Blob ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc ) goto abort_due_to_error; + rc = sqlite3VdbeSorterWrite(pC, pIn2); + if( rc) goto abort_due_to_error; + break; +} + +/* Opcode: IdxDelete P1 P2 P3 * P5 ** Synopsis: key=r[P2@P3] ** ** The content of P3 registers starting at register P2 form -** an unpacked index key. This opcode removes that entry from the +** an unpacked index key. This opcode removes that entry from the ** index opened by cursor P1. +** +** If P5 is not zero, then raise an SQLITE_CORRUPT_INDEX error +** if no matching index entry is found. This happens when running +** an UPDATE or DELETE statement and the index entry to be updated +** or deleted is not found. For some uses of IdxDelete +** (example: the EXCEPT operator) it does not matter that no matching +** entry is found. For those cases, P5 is zero. Also, do not raise +** this (self-correcting and non-critical) error if in writable_schema mode. */ case OP_IdxDelete: { VdbeCursor *pC; @@ -81697,26 +99950,30 @@ case OP_IdxDelete: { pC = p->apCsr[pOp->p1]; assert( pC!=0 ); assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3VdbeIncrWriteCounter(p, pC); pCrsr = pC->uc.pCursor; assert( pCrsr!=0 ); - assert( pOp->p5==0 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p3; r.default_rc = 0; r.aMem = &aMem[pOp->p2]; - rc = sqlite3BtreeMovetoUnpacked(pCrsr, &r, 0, 0, &res); + rc = sqlite3BtreeIndexMoveto(pCrsr, &r, &res); if( rc ) goto abort_due_to_error; if( res==0 ){ rc = sqlite3BtreeDelete(pCrsr, BTREE_AUXDELETE); if( rc ) goto abort_due_to_error; + }else if( pOp->p5 && !sqlite3WritableSchema(db) ){ + rc = sqlite3ReportError(SQLITE_CORRUPT_INDEX, __LINE__, "index corruption"); + goto abort_due_to_error; } assert( pC->deferredMoveto==0 ); pC->cacheStatus = CACHE_STALE; + pC->seekResult = 0; break; } -/* Opcode: Seek P1 * P3 P4 * -** Synopsis: Move P3 to P1.rowid +/* Opcode: DeferredSeek P1 * P3 P4 * +** Synopsis: Move P3 to P1.rowid if needed ** ** P1 is an open index cursor and P3 is a cursor on the corresponding ** table. This opcode does a deferred seek of the P3 table cursor @@ -81728,8 +99985,8 @@ case OP_IdxDelete: { ** ** P4 may be an array of integers (type P4_INTARRAY) containing ** one entry for each column in the P3 table. If array entry a(i) -** is non-zero, then reading column a(i)-1 from cursor P3 is -** equivalent to performing the deferred seek and then reading column i +** is non-zero, then reading column a(i)-1 from cursor P3 is +** equivalent to performing the deferred seek and then reading column i ** from P1. This information is stored in P3 and used to redirect ** reads against P3 over to P1, thus possibly avoiding the need to ** seek and read cursor P3. @@ -81743,18 +100000,18 @@ case OP_IdxDelete: { ** ** See also: Rowid, MakeRecord. */ -case OP_Seek: -case OP_IdxRowid: { /* out2 */ - VdbeCursor *pC; /* The P1 index cursor */ - VdbeCursor *pTabCur; /* The P2 table cursor (OP_Seek only) */ - i64 rowid; /* Rowid that P1 current points to */ +case OP_DeferredSeek: /* ncycle */ +case OP_IdxRowid: { /* out2, ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ + VdbeCursor *pTabCur; /* The P2 table cursor (OP_DeferredSeek only) */ + i64 rowid; /* Rowid that P1 current points to */ assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); - assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->eCurType==CURTYPE_BTREE || IsNullCursor(pC) ); assert( pC->uc.pCursor!=0 ); - assert( pC->isTable==0 ); + assert( pC->isTable==0 || IsNullCursor(pC) ); assert( pC->deferredMoveto==0 ); assert( !pC->nullRow || pOp->opcode==OP_IdxRowid ); @@ -81762,10 +100019,10 @@ case OP_IdxRowid: { /* out2 */ ** of sqlite3VdbeCursorRestore() and sqlite3VdbeIdxRowid(). */ rc = sqlite3VdbeCursorRestore(pC); - /* sqlite3VbeCursorRestore() can only fail if the record has been deleted - ** out from under the cursor. That will never happens for an IdxRowid - ** or Seek opcode */ - if( NEVER(rc!=SQLITE_OK) ) goto abort_due_to_error; + /* sqlite3VdbeCursorRestore() may fail if the cursor has been disturbed + ** since it was last positioned and an error (e.g. OOM or an IO error) + ** occurs while trying to reposition it. */ + if( rc!=SQLITE_OK ) goto abort_due_to_error; if( !pC->nullRow ){ rowid = 0; /* Not needed. Only used to silence a warning. */ @@ -81773,7 +100030,7 @@ case OP_IdxRowid: { /* out2 */ if( rc!=SQLITE_OK ){ goto abort_due_to_error; } - if( pOp->opcode==OP_Seek ){ + if( pOp->opcode==OP_DeferredSeek ){ assert( pOp->p3>=0 && pOp->p3nCursor ); pTabCur = p->apCsr[pOp->p3]; assert( pTabCur!=0 ); @@ -81783,13 +100040,15 @@ case OP_IdxRowid: { /* out2 */ pTabCur->nullRow = 0; pTabCur->movetoTarget = rowid; pTabCur->deferredMoveto = 1; + pTabCur->cacheStatus = CACHE_STALE; assert( pOp->p4type==P4_INTARRAY || pOp->p4.ai==0 ); - pTabCur->aAltMap = pOp->p4.ai; + assert( !pTabCur->isEphemeral ); + pTabCur->ub.aAltMap = pOp->p4.ai; + assert( !pC->isEphemeral ); pTabCur->pAltCursor = pC; }else{ pOut = out2Prerelease(p, pOp); pOut->u.i = rowid; - pOut->flags = MEM_Int; } }else{ assert( pOp->opcode==OP_IdxRowid ); @@ -81798,32 +100057,50 @@ case OP_IdxRowid: { /* out2 */ break; } -/* Opcode: IdxGE P1 P2 P3 P4 P5 +/* Opcode: FinishSeek P1 * * * * +** +** If cursor P1 was previously moved via OP_DeferredSeek, complete that +** seek operation now, without further delay. If the cursor seek has +** already occurred, this instruction is a no-op. +*/ +case OP_FinishSeek: { /* ncycle */ + VdbeCursor *pC; /* The P1 index cursor */ + + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + if( pC->deferredMoveto ){ + rc = sqlite3VdbeFinishMoveto(pC); + if( rc ) goto abort_due_to_error; + } + break; +} + +/* Opcode: IdxGE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** The P4 register values beginning with P3 form an unpacked index -** key that omits the PRIMARY KEY. Compare this key value against the index -** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY. Compare this key value against the index +** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID ** fields at the end. ** ** If the P1 index entry is greater than or equal to the key value ** then jump to P2. Otherwise fall through to the next instruction. */ -/* Opcode: IdxGT P1 P2 P3 P4 P5 +/* Opcode: IdxGT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** The P4 register values beginning with P3 form an unpacked index -** key that omits the PRIMARY KEY. Compare this key value against the index -** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID +** The P4 register values beginning with P3 form an unpacked index +** key that omits the PRIMARY KEY. Compare this key value against the index +** that P1 is currently pointing to, ignoring the PRIMARY KEY or ROWID ** fields at the end. ** ** If the P1 index entry is greater than the key value ** then jump to P2. Otherwise fall through to the next instruction. */ -/* Opcode: IdxLT P1 P2 P3 P4 P5 +/* Opcode: IdxLT P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** The P4 register values beginning with P3 form an unpacked index +** The P4 register values beginning with P3 form an unpacked index ** key that omits the PRIMARY KEY or ROWID. Compare this key value against ** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or ** ROWID on the P1 index. @@ -81831,10 +100108,10 @@ case OP_IdxRowid: { /* out2 */ ** If the P1 index entry is less than the key value then jump to P2. ** Otherwise fall through to the next instruction. */ -/* Opcode: IdxLE P1 P2 P3 P4 P5 +/* Opcode: IdxLE P1 P2 P3 P4 * ** Synopsis: key=r[P3@P4] ** -** The P4 register values beginning with P3 form an unpacked index +** The P4 register values beginning with P3 form an unpacked index ** key that omits the PRIMARY KEY or ROWID. Compare this key value against ** the index that P1 is currently pointing to, ignoring the PRIMARY KEY or ** ROWID on the P1 index. @@ -81842,10 +100119,10 @@ case OP_IdxRowid: { /* out2 */ ** If the P1 index entry is less than or equal to the key value then jump ** to P2. Otherwise fall through to the next instruction. */ -case OP_IdxLE: /* jump */ -case OP_IdxGT: /* jump */ -case OP_IdxLT: /* jump */ -case OP_IdxGE: { /* jump */ +case OP_IdxLE: /* jump, ncycle */ +case OP_IdxGT: /* jump, ncycle */ +case OP_IdxLT: /* jump, ncycle */ +case OP_IdxGE: { /* jump, ncycle */ VdbeCursor *pC; int res; UnpackedRecord r; @@ -81857,7 +100134,6 @@ case OP_IdxGE: { /* jump */ assert( pC->eCurType==CURTYPE_BTREE ); assert( pC->uc.pCursor!=0); assert( pC->deferredMoveto==0 ); - assert( pOp->p5==0 || pOp->p5==1 ); assert( pOp->p4type==P4_INT32 ); r.pKeyInfo = pC->pKeyInfo; r.nField = (u16)pOp->p4.i; @@ -81870,10 +100146,39 @@ case OP_IdxGE: { /* jump */ } r.aMem = &aMem[pOp->p3]; #ifdef SQLITE_DEBUG - { int i; for(i=0; ip3+i, &aMem[pOp->p3+i]); + } + } #endif - res = 0; /* Not needed. Only used to silence a warning. */ - rc = sqlite3VdbeIdxKeyCompare(db, pC, &r, &res); + + /* Inlined version of sqlite3VdbeIdxKeyCompare() */ + { + i64 nCellKey = 0; + BtCursor *pCur; + Mem m; + + assert( pC->eCurType==CURTYPE_BTREE ); + pCur = pC->uc.pCursor; + assert( sqlite3BtreeCursorIsValid(pCur) ); + nCellKey = sqlite3BtreePayloadSize(pCur); + /* nCellKey will always be between 0 and 0xffffffff because of the way + ** that btreeParseCellPtr() and sqlite3GetVarint32() are implemented */ + if( nCellKey<=0 || nCellKey>0x7fffffff ){ + rc = SQLITE_CORRUPT_BKPT; + goto abort_due_to_error; + } + sqlite3VdbeMemInit(&m, db, 0); + rc = sqlite3VdbeMemFromBtreeZeroOffset(pCur, (u32)nCellKey, &m); + if( rc ) goto abort_due_to_error; + res = sqlite3VdbeRecordCompareWithSkip(m.n, m.z, &r, 0); + sqlite3VdbeMemReleaseMalloc(&m); + } + /* End of inlined sqlite3VdbeIdxKeyCompare() */ + assert( (OP_IdxLE&1)==(OP_IdxLT&1) && (OP_IdxGE&1)==(OP_IdxGT&1) ); if( (pOp->opcode&1)==(OP_IdxLT&1) ){ assert( pOp->opcode==OP_IdxLE || pOp->opcode==OP_IdxLT ); @@ -81883,7 +100188,7 @@ case OP_IdxGE: { /* jump */ res++; } VdbeBranchTaken(res>0,2); - if( rc ) goto abort_due_to_error; + assert( rc==SQLITE_OK ); if( res>0 ) goto jump_to_p2; break; } @@ -81894,17 +100199,24 @@ case OP_IdxGE: { /* jump */ ** file is given by P1. ** ** The table being destroyed is in the main database file if P3==0. If -** P3==1 then the table to be clear is in the auxiliary database file +** P3==1 then the table to be destroyed is in the auxiliary database file ** that is used to store tables create using CREATE TEMPORARY TABLE. ** ** If AUTOVACUUM is enabled then it is possible that another root page ** might be moved into the newly deleted root page in order to keep all ** root pages contiguous at the beginning of the database. The former ** value of the root page that moved - its value before the move occurred - -** is stored in register P2. If no page -** movement was required (because the table being dropped was already -** the last one in the database) then a zero is stored in register P2. -** If AUTOVACUUM is disabled then a zero is stored in register P2. +** is stored in register P2. If no page movement was required (because the +** table being dropped was already the last one in the database) then a +** zero is stored in register P2. If AUTOVACUUM is disabled then a zero +** is stored in register P2. +** +** This opcode throws an error if there are any active reader VMs when +** it is invoked. This is done to avoid the difficulty associated with +** updating existing cursors when a root page is moved in an AUTOVACUUM +** database. This error is thrown even if the database is not an AUTOVACUUM +** db in order to avoid introducing an incompatibility between autovacuum +** and non-autovacuum modes. ** ** See also: Clear */ @@ -81912,6 +100224,7 @@ case OP_Destroy: { /* out2 */ int iMoved; int iDb; + sqlite3VdbeIncrWriteCounter(p, 0); assert( p->readOnly==0 ); assert( pOp->p1>1 ); pOut = out2Prerelease(p, pOp); @@ -81946,27 +100259,25 @@ case OP_Destroy: { /* out2 */ ** in the database file is given by P1. But, unlike Destroy, do not ** remove the table or index from the database file. ** -** The table being clear is in the main database file if P2==0. If -** P2==1 then the table to be clear is in the auxiliary database file +** The table being cleared is in the main database file if P2==0. If +** P2==1 then the table to be cleared is in the auxiliary database file ** that is used to store tables create using CREATE TEMPORARY TABLE. ** -** If the P3 value is non-zero, then the table referred to must be an -** intkey table (an SQL table, not an index). In this case the row change -** count is incremented by the number of rows in the table being cleared. -** If P3 is greater than zero, then the value stored in register P3 is -** also incremented by the number of rows in the table being cleared. +** If the P3 value is non-zero, then the row change count is incremented +** by the number of rows in the table being cleared. If P3 is greater +** than zero, then the value stored in register P3 is also incremented +** by the number of rows in the table being cleared. ** ** See also: Destroy */ case OP_Clear: { - int nChange; - + i64 nChange; + + sqlite3VdbeIncrWriteCounter(p, 0); nChange = 0; assert( p->readOnly==0 ); assert( DbMaskTest(p->btreeMask, pOp->p2) ); - rc = sqlite3BtreeClearTable( - db->aDb[pOp->p2].pBt, pOp->p1, (pOp->p3 ? &nChange : 0) - ); + rc = sqlite3BtreeClearTable(db->aDb[pOp->p2].pBt, (u32)pOp->p1, &nChange); if( pOp->p3 ){ p->nChange += nChange; if( pOp->p3>0 ){ @@ -81989,7 +100300,7 @@ case OP_Clear: { */ case OP_ResetSorter: { VdbeCursor *pC; - + assert( pOp->p1>=0 && pOp->p1nCursor ); pC = p->apCsr[pOp->p1]; assert( pC!=0 ); @@ -82004,71 +100315,105 @@ case OP_ResetSorter: { break; } -/* Opcode: CreateTable P1 P2 * * * -** Synopsis: r[P2]=root iDb=P1 +/* Opcode: CreateBtree P1 P2 P3 * * +** Synopsis: r[P2]=root iDb=P1 flags=P3 ** -** Allocate a new table in the main database file if P1==0 or in the -** auxiliary database file if P1==1 or in an attached database if -** P1>1. Write the root page number of the new table into -** register P2 -** -** The difference between a table and an index is this: A table must -** have a 4-byte integer key and can have arbitrary data. An index -** has an arbitrary key but no data. -** -** See also: CreateIndex +** Allocate a new b-tree in the main database file if P1==0 or in the +** TEMP database file if P1==1 or in an attached database if +** P1>1. The P3 argument must be 1 (BTREE_INTKEY) for a rowid table +** it must be 2 (BTREE_BLOBKEY) for an index or WITHOUT ROWID table. +** The root page number of the new b-tree is stored in register P2. */ -/* Opcode: CreateIndex P1 P2 * * * -** Synopsis: r[P2]=root iDb=P1 -** -** Allocate a new index in the main database file if P1==0 or in the -** auxiliary database file if P1==1 or in an attached database if -** P1>1. Write the root page number of the new table into -** register P2. -** -** See documentation on OP_CreateTable for additional information. -*/ -case OP_CreateIndex: /* out2 */ -case OP_CreateTable: { /* out2 */ - int pgno; - int flags; +case OP_CreateBtree: { /* out2 */ + Pgno pgno; Db *pDb; + sqlite3VdbeIncrWriteCounter(p, 0); pOut = out2Prerelease(p, pOp); pgno = 0; + assert( pOp->p3==BTREE_INTKEY || pOp->p3==BTREE_BLOBKEY ); assert( pOp->p1>=0 && pOp->p1nDb ); assert( DbMaskTest(p->btreeMask, pOp->p1) ); assert( p->readOnly==0 ); pDb = &db->aDb[pOp->p1]; assert( pDb->pBt!=0 ); - if( pOp->opcode==OP_CreateTable ){ - /* flags = BTREE_INTKEY; */ - flags = BTREE_INTKEY; - }else{ - flags = BTREE_BLOBKEY; - } - rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, flags); + rc = sqlite3BtreeCreateTable(pDb->pBt, &pgno, pOp->p3); if( rc ) goto abort_due_to_error; pOut->u.i = pgno; break; } +/* Opcode: SqlExec P1 P2 * P4 * +** +** Run the SQL statement or statements specified in the P4 string. +** +** The P1 parameter is a bitmask of options: +** +** 0x0001 Disable Auth and Trace callbacks while the statements +** in P4 are running. +** +** 0x0002 Set db->nAnalysisLimit to P2 while the statements in +** P4 are running. +** +*/ +case OP_SqlExec: { + char *zErr; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth; +#endif + u8 mTrace; + int savedAnalysisLimit; + + sqlite3VdbeIncrWriteCounter(p, 0); + db->nSqlExec++; + zErr = 0; +#ifndef SQLITE_OMIT_AUTHORIZATION + xAuth = db->xAuth; +#endif + mTrace = db->mTrace; + savedAnalysisLimit = db->nAnalysisLimit; + if( pOp->p1 & 0x0001 ){ +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + db->mTrace = 0; + } + if( pOp->p1 & 0x0002 ){ + db->nAnalysisLimit = pOp->p2; + } + rc = sqlite3_exec(db, pOp->p4.z, 0, 0, &zErr); + db->nSqlExec--; +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + db->mTrace = mTrace; + db->nAnalysisLimit = savedAnalysisLimit; + if( zErr || rc ){ + sqlite3VdbeError(p, "%s", zErr); + sqlite3_free(zErr); + if( rc==SQLITE_NOMEM ) goto no_mem; + goto abort_due_to_error; + } + break; +} + /* Opcode: ParseSchema P1 * * P4 * ** -** Read and parse all entries from the SQLITE_MASTER table of database P1 -** that match the WHERE clause P4. +** Read and parse all entries from the schema table of database P1 +** that match the WHERE clause P4. If P4 is a NULL pointer, then the +** entire schema for P1 is reparsed. ** ** This opcode invokes the parser to create a new virtual machine, ** then runs the new virtual machine. It is thus a re-entrant opcode. */ case OP_ParseSchema: { int iDb; - const char *zMaster; + const char *zSchema; char *zSql; InitData initData; /* Any prepared statement that invokes this opcode will hold mutexes - ** on every btree. This is a prerequisite for invoking + ** on every btree. This is a prerequisite for invoking ** sqlite3InitCallback(). */ #ifdef SQLITE_DEBUG @@ -82079,25 +100424,46 @@ case OP_ParseSchema: { iDb = pOp->p1; assert( iDb>=0 && iDbnDb ); - assert( DbHasProperty(db, iDb, DB_SchemaLoaded) ); - /* Used to be a conditional */ { - zMaster = SCHEMA_TABLE(iDb); + assert( DbHasProperty(db, iDb, DB_SchemaLoaded) + || db->mallocFailed + || (CORRUPT_DB && (db->flags & SQLITE_NoSchemaError)!=0) ); + +#ifndef SQLITE_OMIT_ALTERTABLE + if( pOp->p4.z==0 ){ + sqlite3SchemaClear(db->aDb[iDb].pSchema); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + rc = sqlite3InitOne(db, iDb, &p->zErrMsg, pOp->p5); + db->mDbFlags |= DBFLAG_SchemaChange; + p->expired = 0; + }else +#endif + { + zSchema = LEGACY_SCHEMA_TABLE; initData.db = db; - initData.iDb = pOp->p1; + initData.iDb = iDb; initData.pzErrMsg = &p->zErrMsg; + initData.mInitFlags = 0; + initData.mxPage = sqlite3BtreeLastPage(db->aDb[iDb].pBt); zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM '%q'.%s WHERE %s ORDER BY rowid", - db->aDb[iDb].zName, zMaster, pOp->p4.z); + "SELECT*FROM\"%w\".%s WHERE %s ORDER BY rowid", + db->aDb[iDb].zDbSName, zSchema, pOp->p4.z); if( zSql==0 ){ rc = SQLITE_NOMEM_BKPT; }else{ assert( db->init.busy==0 ); db->init.busy = 1; initData.rc = SQLITE_OK; + initData.nInitRow = 0; assert( !db->mallocFailed ); rc = sqlite3_exec(db, zSql, sqlite3InitCallback, &initData, 0); if( rc==SQLITE_OK ) rc = initData.rc; - sqlite3DbFree(db, zSql); + if( rc==SQLITE_OK && initData.nInitRow==0 ){ + /* The OP_ParseSchema opcode with a non-NULL P4 argument should parse + ** at least one SQL statement. Any less than that indicates that + ** the sqlite_schema table is corrupt. */ + rc = SQLITE_CORRUPT_BKPT; + } + sqlite3DbFreeNN(db, zSql); db->init.busy = 0; } } @@ -82108,7 +100474,7 @@ case OP_ParseSchema: { } goto abort_due_to_error; } - break; + break; } #if !defined(SQLITE_OMIT_ANALYZE) @@ -82122,7 +100488,7 @@ case OP_LoadAnalysis: { assert( pOp->p1>=0 && pOp->p1nDb ); rc = sqlite3AnalysisLoad(db, pOp->p1); if( rc ) goto abort_due_to_error; - break; + break; } #endif /* !defined(SQLITE_OMIT_ANALYZE) */ @@ -82130,11 +100496,12 @@ case OP_LoadAnalysis: { ** ** Remove the internal (in-memory) data structures that describe ** the table named P4 in database P1. This is called after a table -** is dropped from disk (using the Destroy opcode) in order to keep +** is dropped from disk (using the Destroy opcode) in order to keep ** the internal representation of the ** schema consistent with what is on disk. */ case OP_DropTable: { + sqlite3VdbeIncrWriteCounter(p, 0); sqlite3UnlinkAndDeleteTable(db, pOp->p1, pOp->p4.z); break; } @@ -82148,6 +100515,7 @@ case OP_DropTable: { ** schema consistent with what is on disk. */ case OP_DropIndex: { + sqlite3VdbeIncrWriteCounter(p, 0); sqlite3UnlinkAndDeleteIndex(db, pOp->p1, pOp->p4.z); break; } @@ -82156,11 +100524,12 @@ case OP_DropIndex: { ** ** Remove the internal (in-memory) data structures that describe ** the trigger named P4 in database P1. This is called after a trigger -** is dropped from disk (using the Destroy opcode) in order to keep +** is dropped from disk (using the Destroy opcode) in order to keep ** the internal representation of the ** schema consistent with what is on disk. */ case OP_DropTrigger: { + sqlite3VdbeIncrWriteCounter(p, 0); sqlite3UnlinkAndDeleteTrigger(db, pOp->p1, pOp->p4.z); break; } @@ -82170,12 +100539,12 @@ case OP_DropTrigger: { /* Opcode: IntegrityCk P1 P2 P3 P4 P5 ** ** Do an analysis of the currently open database. Store in -** register P1 the text of an error message describing any problems. -** If no problems are found, store a NULL in register P1. +** register (P1+1) the text of an error message describing any problems. +** If no problems are found, store a NULL in register (P1+1). ** -** The register P3 contains the maximum number of allowed errors. -** At most reg(P3) errors will be reported. -** In other words, the analysis stops as soon as reg(P1) errors are +** The register (P1) contains one less than the maximum number of allowed +** errors. At most reg(P1) errors will be reported. +** In other words, the analysis stops as soon as reg(P1) errors are ** seen. Reg(P1) is updated with the number of errors remaining. ** ** The root page numbers of all tables in the database are integers @@ -82188,44 +100557,47 @@ case OP_DropTrigger: { */ case OP_IntegrityCk: { int nRoot; /* Number of tables to check. (Number of root pages.) */ - int *aRoot; /* Array of rootpage numbers for tables to be checked */ + Pgno *aRoot; /* Array of rootpage numbers for tables to be checked */ int nErr; /* Number of errors reported */ char *z; /* Text of the error report */ Mem *pnErr; /* Register keeping track of errors remaining */ assert( p->bIsReader ); + assert( pOp->p4type==P4_INTARRAY ); nRoot = pOp->p2; aRoot = pOp->p4.ai; assert( nRoot>0 ); - assert( aRoot[nRoot]==0 ); - assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); - pnErr = &aMem[pOp->p3]; + assert( aRoot!=0 ); + assert( aRoot[0]==(Pgno)nRoot ); + assert( pOp->p1>0 && (pOp->p1+1)<=(p->nMem+1 - p->nCursor) ); + pnErr = &aMem[pOp->p1]; assert( (pnErr->flags & MEM_Int)!=0 ); assert( (pnErr->flags & (MEM_Str|MEM_Blob))==0 ); - pIn1 = &aMem[pOp->p1]; + pIn1 = &aMem[pOp->p1+1]; assert( pOp->p5nDb ); assert( DbMaskTest(p->btreeMask, pOp->p5) ); - z = sqlite3BtreeIntegrityCheck(db->aDb[pOp->p5].pBt, aRoot, nRoot, - (int)pnErr->u.i, &nErr); - pnErr->u.i -= nErr; + rc = sqlite3BtreeIntegrityCheck(db, db->aDb[pOp->p5].pBt, &aRoot[1], + &aMem[pOp->p3], nRoot, (int)pnErr->u.i+1, &nErr, &z); sqlite3VdbeMemSetNull(pIn1); if( nErr==0 ){ assert( z==0 ); - }else if( z==0 ){ - goto no_mem; + }else if( rc ){ + sqlite3_free(z); + goto abort_due_to_error; }else{ + pnErr->u.i -= nErr-1; sqlite3VdbeMemSetStr(pIn1, z, -1, SQLITE_UTF8, sqlite3_free); } UPDATE_MAX_BLOBSIZE(pIn1); sqlite3VdbeChangeEncoding(pIn1, encoding); - break; + goto check_for_interrupt; } #endif /* SQLITE_OMIT_INTEGRITY_CHECK */ /* Opcode: RowSetAdd P1 P2 * * * -** Synopsis: rowset(P1)=r[P2] +** Synopsis: rowset(P1)=r[P2] ** -** Insert the integer value held by register P2 into a boolean index +** Insert the integer value held by register P2 into a RowSet object ** held in register P1. ** ** An assertion fails if P2 is not an integer. @@ -82234,27 +100606,29 @@ case OP_RowSetAdd: { /* in1, in2 */ pIn1 = &aMem[pOp->p1]; pIn2 = &aMem[pOp->p2]; assert( (pIn2->flags & MEM_Int)!=0 ); - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; } - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn2->u.i); + assert( sqlite3VdbeMemIsRowSet(pIn1) ); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn2->u.i); break; } /* Opcode: RowSetRead P1 P2 P3 * * -** Synopsis: r[P3]=rowset(P1) +** Synopsis: r[P3]=rowset(P1) ** -** Extract the smallest value from boolean index P1 and put that value into -** register P3. Or, if boolean index P1 is initially empty, leave P3 +** Extract the smallest value from the RowSet object in P1 +** and put that value into register P3. +** Or, if RowSet object P1 is initially empty, leave P3 ** unchanged and jump to instruction P2. */ case OP_RowSetRead: { /* jump, in1, out3 */ i64 val; pIn1 = &aMem[pOp->p1]; - if( (pIn1->flags & MEM_RowSet)==0 - || sqlite3RowSetNext(pIn1->u.pRowSet, &val)==0 + assert( (pIn1->flags & MEM_Blob)==0 || sqlite3VdbeMemIsRowSet(pIn1) ); + if( (pIn1->flags & MEM_Blob)==0 + || sqlite3RowSetNext((RowSet*)pIn1->z, &val)==0 ){ /* The boolean index is empty */ sqlite3VdbeMemSetNull(pIn1); @@ -82277,15 +100651,14 @@ case OP_RowSetRead: { /* jump, in1, out3 */ ** integer in P3 into the RowSet and continue on to the ** next opcode. ** -** The RowSet object is optimized for the case where successive sets -** of integers, where each set contains no duplicates. Each set -** of values is identified by a unique P4 value. The first set -** must have P4==0, the final set P4=-1. P4 must be either -1 or -** non-negative. For non-negative values of P4 only the lower 4 -** bits are significant. +** The RowSet object is optimized for the case where sets of integers +** are inserted in distinct phases, which each set contains no duplicates. +** Each set is identified by a unique P4 value. The first set +** must have P4==0, the final set must have P4==-1, and for all other sets +** must have P4>0. ** ** This allows optimizations: (a) when P4==0 there is no need to test -** the rowset object for P3, as it is guaranteed not to contain it, +** the RowSet object for P3, as it is guaranteed not to contain it, ** (b) when P4==-1 there is no need to insert the value, as it will ** never be tested for, and (c) when a value that is part of set X is ** inserted, there is no need to search to see if the same value was @@ -82304,20 +100677,19 @@ case OP_RowSetTest: { /* jump, in1, in3 */ /* If there is anything other than a rowset object in memory cell P1, ** delete it now and initialize P1 with an empty rowset */ - if( (pIn1->flags & MEM_RowSet)==0 ){ - sqlite3VdbeMemSetRowSet(pIn1); - if( (pIn1->flags & MEM_RowSet)==0 ) goto no_mem; + if( (pIn1->flags & MEM_Blob)==0 ){ + if( sqlite3VdbeMemSetRowSet(pIn1) ) goto no_mem; } - + assert( sqlite3VdbeMemIsRowSet(pIn1) ); assert( pOp->p4type==P4_INT32 ); assert( iSet==-1 || iSet>=0 ); if( iSet ){ - exists = sqlite3RowSetTest(pIn1->u.pRowSet, iSet, pIn3->u.i); + exists = sqlite3RowSetTest((RowSet*)pIn1->z, iSet, pIn3->u.i); VdbeBranchTaken(exists!=0,2); if( exists ) goto jump_to_p2; } if( iSet>=0 ){ - sqlite3RowSetInsert(pIn1->u.pRowSet, pIn3->u.i); + sqlite3RowSetInsert((RowSet*)pIn1->z, pIn3->u.i); } break; } @@ -82327,20 +100699,22 @@ case OP_RowSetTest: { /* jump, in1, in3 */ /* Opcode: Program P1 P2 P3 P4 P5 ** -** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). +** Execute the trigger program passed as P4 (type P4_SUBPROGRAM). ** -** P1 contains the address of the memory cell that contains the first memory -** cell in an array of values used as arguments to the sub-program. P2 -** contains the address to jump to if the sub-program throws an IGNORE -** exception using the RAISE() function. Register P3 contains the address -** of a memory cell in this (the parent) VM that is used to allocate the +** P1 contains the address of the memory cell that contains the first memory +** cell in an array of values used as arguments to the sub-program. P2 +** contains the address to jump to if the sub-program throws an IGNORE +** exception using the RAISE() function. P2 might be zero, if there is +** no possibility that an IGNORE exception will be raised. +** Register P3 contains the address +** of a memory cell in this (the parent) VM that is used to allocate the ** memory required by the sub-vdbe at runtime. ** ** P4 is a pointer to the VM containing the trigger program. ** ** If P5 is non-zero, then recursive program invocation is enabled. */ -case OP_Program: { /* jump */ +case OP_Program: { /* jump0 */ int nMem; /* Number of memory registers for sub-program */ int nByte; /* Bytes of runtime space required for sub-program */ Mem *pRt; /* Register to allocate runtime space */ @@ -82353,17 +100727,17 @@ case OP_Program: { /* jump */ pProgram = pOp->p4.pProgram; pRt = &aMem[pOp->p3]; assert( pProgram->nOp>0 ); - - /* If the p5 flag is clear, then recursive invocation of triggers is + + /* If the p5 flag is clear, then recursive invocation of triggers is ** disabled for backwards compatibility (p5 is set if this sub-program ** is really a trigger, not a foreign key action, and the flag set ** and cleared by the "PRAGMA recursive_triggers" command is clear). - ** - ** It is recursive invocation of triggers, at the SQL level, that is - ** disabled. In some cases a single trigger may generate more than one - ** SubProgram (if the trigger may be executed with more than one different + ** + ** It is recursive invocation of triggers, at the SQL level, that is + ** disabled. In some cases a single trigger may generate more than one + ** SubProgram (if the trigger may be executed with more than one different ** ON CONFLICT algorithm). SubProgram structures associated with a - ** single trigger all have the same value for the SubProgram.token + ** single trigger all have the same value for the SubProgram.token ** variable. */ if( pOp->p5 ){ t = pProgram->token; @@ -82379,10 +100753,10 @@ case OP_Program: { /* jump */ /* Register pRt is used to store the memory required to save the state ** of the current program, and the memory required at runtime to execute - ** the trigger program. If this trigger has been fired before, then pRt + ** the trigger program. If this trigger has been fired before, then pRt ** is already allocated. Otherwise, it must be initialized. */ - if( (pRt->flags&MEM_Frame)==0 ){ - /* SubProgram.nMem is set to the number of memory cells used by the + if( (pRt->flags&MEM_Blob)==0 ){ + /* SubProgram.nMem is set to the number of memory cells used by the ** program stored in SubProgram.aOp. As well as these, one memory ** cell is required for each cursor used by the program. Set local ** variable nMem (and later, VdbeFrame.nChildMem) to this value. @@ -82392,15 +100766,17 @@ case OP_Program: { /* jump */ if( pProgram->nCsr==0 ) nMem++; nByte = ROUND8(sizeof(VdbeFrame)) + nMem * sizeof(Mem) - + pProgram->nCsr * sizeof(VdbeCursor *) - + pProgram->nOnce * sizeof(u8); + + pProgram->nCsr * sizeof(VdbeCursor*) + + (pProgram->nOp + 7)/8; pFrame = sqlite3DbMallocZero(db, nByte); if( !pFrame ){ goto no_mem; } sqlite3VdbeMemRelease(pRt); - pRt->flags = MEM_Frame; - pRt->u.pFrame = pFrame; + pRt->flags = MEM_Blob|MEM_Dyn; + pRt->z = (char*)pFrame; + pRt->n = nByte; + pRt->xDel = sqlite3VdbeFrameMemDel; pFrame->v = p; pFrame->nChildMem = nMem; @@ -82413,10 +100789,8 @@ case OP_Program: { /* jump */ pFrame->aOp = p->aOp; pFrame->nOp = p->nOp; pFrame->token = pProgram->token; - pFrame->aOnceFlag = p->aOnceFlag; - pFrame->nOnceFlag = p->nOnceFlag; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - pFrame->anExec = p->anExec; +#ifdef SQLITE_DEBUG + pFrame->iFrameMagic = SQLITE_FRAME_MAGIC; #endif pEnd = &VdbeFrameMem(pFrame)[pFrame->nChildMem]; @@ -82425,8 +100799,9 @@ case OP_Program: { /* jump */ pMem->db = db; } }else{ - pFrame = pRt->u.pFrame; - assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem + pFrame = (VdbeFrame*)pRt->z; + assert( pRt->xDel==sqlite3VdbeFrameMemDel ); + assert( pProgram->nMem+pProgram->nCsr==pFrame->nChildMem || (pProgram->nCsr==0 && pProgram->nMem+1==pFrame->nChildMem) ); assert( pProgram->nCsr==pFrame->nChildCsr ); assert( (int)(pOp - aOp)==pFrame->pc ); @@ -82434,7 +100809,7 @@ case OP_Program: { /* jump */ p->nFrame++; pFrame->pParent = p->pFrame; - pFrame->lastRowid = lastRowid; + pFrame->lastRowid = db->lastRowid; pFrame->nChange = p->nChange; pFrame->nDbChange = p->db->nChange; assert( pFrame->pAuxData==0 ); @@ -82446,25 +100821,31 @@ case OP_Program: { /* jump */ p->nMem = pFrame->nChildMem; p->nCursor = (u16)pFrame->nChildCsr; p->apCsr = (VdbeCursor **)&aMem[p->nMem]; + pFrame->aOnce = (u8*)&p->apCsr[pProgram->nCsr]; + memset(pFrame->aOnce, 0, (pProgram->nOp + 7)/8); p->aOp = aOp = pProgram->aOp; p->nOp = pProgram->nOp; - p->aOnceFlag = (u8 *)&p->apCsr[p->nCursor]; - p->nOnceFlag = pProgram->nOnce; -#ifdef SQLITE_ENABLE_STMT_SCANSTATUS - p->anExec = 0; +#ifdef SQLITE_DEBUG + /* Verify that second and subsequent executions of the same trigger do not + ** try to reuse register values from the first use. */ + { + int i; + for(i=0; inMem; i++){ + aMem[i].pScopyFrom = 0; /* Prevent false-positive AboutToChange() errs */ + MemSetTypeFlag(&aMem[i], MEM_Undefined); /* Fault if this reg is reused */ + } + } #endif pOp = &aOp[-1]; - memset(p->aOnceFlag, 0, p->nOnceFlag); - - break; + goto check_for_interrupt; } /* Opcode: Param P1 P2 * * * ** -** This opcode is only ever present in sub-programs called via the -** OP_Program instruction. Copy a value currently stored in a memory -** cell of the calling (parent) frame to cell P2 in the current frames -** address space. This is used by trigger programs to access the new.* +** This opcode is only ever present in sub-programs called via the +** OP_Program instruction. Copy a value currently stored in a memory +** cell of the calling (parent) frame to cell P2 in the current frames +** address space. This is used by trigger programs to access the new.* ** and old.* values. ** ** The address of the cell in the parent frame is determined by adding @@ -82476,7 +100857,7 @@ case OP_Param: { /* out2 */ Mem *pIn; pOut = out2Prerelease(p, pOp); pFrame = p->pFrame; - pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1]; + pIn = &pFrame->aMem[pOp->p1 + pFrame->aOp[pFrame->pc].p1]; sqlite3VdbeMemShallowCopy(pOut, pIn, MEM_Ephem); break; } @@ -82488,8 +100869,8 @@ case OP_Param: { /* out2 */ ** Synopsis: fkctr[P1]+=P2 ** ** Increment a "constraint counter" by P2 (P2 may be negative or positive). -** If P1 is non-zero, the database constraint counter is incremented -** (deferred foreign key constraints). Otherwise, if P1 is zero, the +** If P1 is non-zero, the database constraint counter is incremented +** (deferred foreign key constraints). Otherwise, if P1 is zero, the ** statement counter is incremented (immediate foreign key constraints). */ case OP_FkCounter: { @@ -82507,7 +100888,7 @@ case OP_FkCounter: { ** Synopsis: if fkctr[P1]==0 goto P2 ** ** This opcode tests if a foreign key constraint-counter is currently zero. -** If so, jump to instruction P2. Otherwise, fall through to the next +** If so, jump to instruction P2. Otherwise, fall through to the next ** instruction. ** ** If P1 is non-zero, then the jump is taken if the database constraint-counter @@ -82533,7 +100914,7 @@ case OP_FkIfZero: { /* jump */ ** ** P1 is a register in the root frame of this VM (the root frame is ** different from the current frame if this instruction is being executed -** within a sub-program). Set the value of register P1 to the maximum of +** within a sub-program). Set the value of register P1 to the maximum of ** its current value and the value in register P2. ** ** This instruction throws an error if the memory cell is not initially @@ -82583,7 +100964,7 @@ case OP_IfPos: { /* jump, in1 */ ** Synopsis: if r[P1]>0 then r[P2]=r[P1]+max(0,r[P3]) else r[P2]=(-1) ** ** This opcode performs a commonly used computation associated with -** LIMIT and OFFSET process. r[P1] holds the limit counter. r[P3] +** LIMIT and OFFSET processing. r[P1] holds the limit counter. r[P3] ** holds the offset counter. The opcode computes the combined value ** of the LIMIT and OFFSET and stores that value in r[P2]. The r[P2] ** value computed is the total number of rows that will need to be @@ -82593,34 +100974,47 @@ case OP_IfPos: { /* jump, in1 */ ** and r[P2] is set to be the value of the LIMIT, r[P1]. ** ** if r[P1] is zero or negative, that means there is no LIMIT -** and r[P2] is set to -1. +** and r[P2] is set to -1. ** ** Otherwise, r[P2] is set to the sum of r[P1] and r[P3]. */ case OP_OffsetLimit: { /* in1, out2, in3 */ + i64 x; pIn1 = &aMem[pOp->p1]; pIn3 = &aMem[pOp->p3]; pOut = out2Prerelease(p, pOp); assert( pIn1->flags & MEM_Int ); assert( pIn3->flags & MEM_Int ); - pOut->u.i = pIn1->u.i<=0 ? -1 : pIn1->u.i+(pIn3->u.i>0?pIn3->u.i:0); + x = pIn1->u.i; + if( x<=0 || sqlite3AddInt64(&x, pIn3->u.i>0?pIn3->u.i:0) ){ + /* If the LIMIT is less than or equal to zero, loop forever. This + ** is documented. But also, if the LIMIT+OFFSET exceeds 2^63 then + ** also loop forever. This is undocumented. In fact, one could argue + ** that the loop should terminate. But assuming 1 billion iterations + ** per second (far exceeding the capabilities of any current hardware) + ** it would take nearly 300 years to actually reach the limit. So + ** looping forever is a reasonable approximation. */ + pOut->u.i = -1; + }else{ + pOut->u.i = x; + } break; } -/* Opcode: IfNotZero P1 P2 P3 * * -** Synopsis: if r[P1]!=0 then r[P1]-=P3, goto P2 +/* Opcode: IfNotZero P1 P2 * * * +** Synopsis: if r[P1]!=0 then r[P1]--, goto P2 ** ** Register P1 must contain an integer. If the content of register P1 is -** initially nonzero, then subtract P3 from the value in register P1 and -** jump to P2. If register P1 is initially zero, leave it unchanged -** and fall through. +** initially greater than zero, then decrement the value in register P1. +** If it is non-zero (negative or positive) and then also jump to P2. +** If register P1 is initially zero, leave it unchanged and fall through. */ case OP_IfNotZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); VdbeBranchTaken(pIn1->u.i<0, 2); if( pIn1->u.i ){ - pIn1->u.i -= pOp->p3; + if( pIn1->u.i>0 ) pIn1->u.i--; goto jump_to_p2; } break; @@ -82629,37 +101023,48 @@ case OP_IfNotZero: { /* jump, in1 */ /* Opcode: DecrJumpZero P1 P2 * * * ** Synopsis: if (--r[P1])==0 goto P2 ** -** Register P1 must hold an integer. Decrement the value in register P1 -** then jump to P2 if the new value is exactly zero. +** Register P1 must hold an integer. Decrement the value in P1 +** and jump to P2 if the new value is exactly zero. */ case OP_DecrJumpZero: { /* jump, in1 */ pIn1 = &aMem[pOp->p1]; assert( pIn1->flags&MEM_Int ); - pIn1->u.i--; + if( pIn1->u.i>SMALLEST_INT64 ) pIn1->u.i--; VdbeBranchTaken(pIn1->u.i==0, 2); if( pIn1->u.i==0 ) goto jump_to_p2; break; } -/* Opcode: AggStep0 * P2 P3 P4 P5 +/* Opcode: AggStep * P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to the FuncDef -** structure that specifies the function. Register P3 is the +** Execute the xStep function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the ** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. */ -/* Opcode: AggStep * P2 P3 P4 P5 +/* Opcode: AggInverse * P2 P3 P4 P5 +** Synopsis: accum=r[P3] inverse(r[P2@P5]) +** +** Execute the xInverse function for an aggregate. +** The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. +** +** The P5 arguments are taken from register P2 and its +** successors. +*/ +/* Opcode: AggStep1 P1 P2 P3 P4 P5 ** Synopsis: accum=r[P3] step(r[P2@P5]) ** -** Execute the step function for an aggregate. The -** function has P5 arguments. P4 is a pointer to an sqlite3_context -** object that is used to run the function. Register P3 is -** as the accumulator. +** Execute the xStep (if P1==0) or xInverse (if P1!=0) function for an +** aggregate. The function has P5 arguments. P4 is a pointer to the +** FuncDef structure that specifies the function. Register P3 is the +** accumulator. ** ** The P5 arguments are taken from register P2 and its ** successors. @@ -82670,41 +101075,74 @@ case OP_DecrJumpZero: { /* jump, in1 */ ** sqlite3_context only happens once, instead of on each call to the ** step function. */ -case OP_AggStep0: { +case OP_AggInverse: +case OP_AggStep: { int n; sqlite3_context *pCtx; + u64 nAlloc; assert( pOp->p4type==P4_FUNCDEF ); n = pOp->p5; assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); assert( n==0 || (pOp->p2>0 && pOp->p2+n<=(p->nMem+1 - p->nCursor)+1) ); assert( pOp->p3p2 || pOp->p3>=pOp->p2+n ); - pCtx = sqlite3DbMallocRawNN(db, sizeof(*pCtx) + (n-1)*sizeof(sqlite3_value*)); + + /* Allocate space for (a) the context object and (n-1) extra pointers + ** to append to the sqlite3_context.argv[1] array, and (b) a memory + ** cell in which to store the accumulation. Be careful that the memory + ** cell is 8-byte aligned, even on platforms where a pointer is 32-bits. + ** + ** Note: We could avoid this by using a regular memory cell from aMem[] for + ** the accumulator, instead of allocating one here. */ + nAlloc = ROUND8P( sizeof(pCtx[0]) + (n-1)*sizeof(sqlite3_value*) ); + pCtx = sqlite3DbMallocRawNN(db, nAlloc + sizeof(Mem)); if( pCtx==0 ) goto no_mem; + pCtx->pOut = (Mem*)((u8*)pCtx + nAlloc); + assert( EIGHT_BYTE_ALIGNMENT(pCtx->pOut) ); + + sqlite3VdbeMemInit(pCtx->pOut, db, MEM_Null); pCtx->pMem = 0; pCtx->pFunc = pOp->p4.pFunc; pCtx->iOp = (int)(pOp - aOp); pCtx->pVdbe = p; + pCtx->skipFlag = 0; + pCtx->isError = 0; + pCtx->enc = encoding; pCtx->argc = n; pOp->p4type = P4_FUNCCTX; pOp->p4.pCtx = pCtx; - pOp->opcode = OP_AggStep; + + /* OP_AggInverse must have P1==1 and OP_AggStep must have P1==0 */ + assert( pOp->p1==(pOp->opcode==OP_AggInverse) ); + + pOp->opcode = OP_AggStep1; /* Fall through into OP_AggStep */ + /* no break */ deliberate_fall_through } -case OP_AggStep: { +case OP_AggStep1: { int i; sqlite3_context *pCtx; Mem *pMem; - Mem t; assert( pOp->p4type==P4_FUNCCTX ); pCtx = pOp->p4.pCtx; pMem = &aMem[pOp->p3]; +#ifdef SQLITE_DEBUG + if( pOp->p1 ){ + /* This is an OP_AggInverse call. Verify that xStep has always + ** been called at least once prior to any xInverse call. */ + assert( pMem->uTemp==0x1122e0e3 ); + }else{ + /* This is an OP_AggStep call. Mark it as such. */ + pMem->uTemp = 0x1122e0e3; + } +#endif + /* If this function is inside of a trigger, the register array in aMem[] ** might change from one evaluation to the next. The next block of code ** checks to see if the register array has changed, and if so it - ** reinitializes the relavant parts of the sqlite3_context object */ + ** reinitializes the relevant parts of the sqlite3_context object */ if( pCtx->pMem != pMem ){ pCtx->pMem = pMem; for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; @@ -82718,57 +101156,88 @@ case OP_AggStep: { #endif pMem->n++; - sqlite3VdbeMemInit(&t, db, MEM_Null); - pCtx->pOut = &t; - pCtx->fErrorOrAux = 0; - pCtx->skipFlag = 0; + assert( pCtx->pOut->flags==MEM_Null ); + assert( pCtx->isError==0 ); + assert( pCtx->skipFlag==0 ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p1 ){ + (pCtx->pFunc->xInverse)(pCtx,pCtx->argc,pCtx->argv); + }else +#endif (pCtx->pFunc->xSFunc)(pCtx,pCtx->argc,pCtx->argv); /* IMP: R-24505-23230 */ - if( pCtx->fErrorOrAux ){ - if( pCtx->isError ){ - sqlite3VdbeError(p, "%s", sqlite3_value_text(&t)); + + if( pCtx->isError ){ + if( pCtx->isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pCtx->pOut)); rc = pCtx->isError; } - sqlite3VdbeMemRelease(&t); + if( pCtx->skipFlag ){ + assert( pOp[-1].opcode==OP_CollSeq ); + i = pOp[-1].p1; + if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); + pCtx->skipFlag = 0; + } + sqlite3VdbeMemRelease(pCtx->pOut); + pCtx->pOut->flags = MEM_Null; + pCtx->isError = 0; if( rc ) goto abort_due_to_error; - }else{ - assert( t.flags==MEM_Null ); - } - if( pCtx->skipFlag ){ - assert( pOp[-1].opcode==OP_CollSeq ); - i = pOp[-1].p1; - if( i ) sqlite3VdbeMemSetInt64(&aMem[i], 1); } + assert( pCtx->pOut->flags==MEM_Null ); + assert( pCtx->skipFlag==0 ); break; } /* Opcode: AggFinal P1 P2 * P4 * ** Synopsis: accum=r[P1] N=P2 ** -** Execute the finalizer function for an aggregate. P1 is -** the memory location that is the accumulator for the aggregate. +** P1 is the memory location that is the accumulator for an aggregate +** or window function. Execute the finalizer function +** for an aggregate and store the result in P1. ** ** P2 is the number of arguments that the step function takes and ** P4 is a pointer to the FuncDef for this function. The P2 ** argument is not used by this opcode. It is only there to disambiguate ** functions that can take varying numbers of arguments. The -** P4 argument is only needed for the degenerate case where +** P4 argument is only needed for the case where ** the step function was not previously called. */ +/* Opcode: AggValue * P2 P3 P4 * +** Synopsis: r[P3]=value N=P2 +** +** Invoke the xValue() function and store the result in register P3. +** +** P2 is the number of arguments that the step function takes and +** P4 is a pointer to the FuncDef for this function. The P2 +** argument is not used by this opcode. It is only there to disambiguate +** functions that can take varying numbers of arguments. The +** P4 argument is only needed for the case where +** the step function was not previously called. +*/ +case OP_AggValue: case OP_AggFinal: { Mem *pMem; assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + assert( pOp->p3==0 || pOp->opcode==OP_AggValue ); pMem = &aMem[pOp->p1]; assert( (pMem->flags & ~(MEM_Null|MEM_Agg))==0 ); - rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pOp->p3 ){ + memAboutToChange(p, &aMem[pOp->p3]); + rc = sqlite3VdbeMemAggValue(pMem, &aMem[pOp->p3], pOp->p4.pFunc); + pMem = &aMem[pOp->p3]; + }else +#endif + { + rc = sqlite3VdbeMemFinalize(pMem, pOp->p4.pFunc); + } + if( rc ){ sqlite3VdbeError(p, "%s", sqlite3_value_text(pMem)); goto abort_due_to_error; } sqlite3VdbeChangeEncoding(pMem, encoding); UPDATE_MAX_BLOBSIZE(pMem); - if( sqlite3VdbeMemTooBig(pMem) ){ - goto too_big; - } + REGISTER_TRACE((int)(pMem-aMem), pMem); break; } @@ -82805,9 +101274,9 @@ case OP_Checkpoint: { } for(i=0, pMem = &aMem[pOp->p3]; i<3; i++, pMem++){ sqlite3VdbeMemSetInt64(pMem, (i64)aRes[i]); - } + } break; -}; +}; #endif #ifndef SQLITE_OMIT_PRAGMA @@ -82833,9 +101302,9 @@ case OP_JournalMode: { /* out2 */ pOut = out2Prerelease(p, pOp); eNew = pOp->p3; - assert( eNew==PAGER_JOURNALMODE_DELETE - || eNew==PAGER_JOURNALMODE_TRUNCATE - || eNew==PAGER_JOURNALMODE_PERSIST + assert( eNew==PAGER_JOURNALMODE_DELETE + || eNew==PAGER_JOURNALMODE_TRUNCATE + || eNew==PAGER_JOURNALMODE_PERSIST || eNew==PAGER_JOURNALMODE_OFF || eNew==PAGER_JOURNALMODE_MEMORY || eNew==PAGER_JOURNALMODE_WAL @@ -82848,13 +101317,14 @@ case OP_JournalMode: { /* out2 */ pPager = sqlite3BtreePager(pBt); eOld = sqlite3PagerGetJournalMode(pPager); if( eNew==PAGER_JOURNALMODE_QUERY ) eNew = eOld; + assert( sqlite3BtreeHoldsMutex(pBt) ); if( !sqlite3PagerOkToChangeJournalMode(pPager) ) eNew = eOld; #ifndef SQLITE_OMIT_WAL zFilename = sqlite3PagerFilename(pPager, 1); /* Do not allow a transition to journal_mode=WAL for a database - ** in temporary storage or if the VFS does not support shared memory + ** in temporary storage or if the VFS does not support shared memory */ if( eNew==PAGER_JOURNALMODE_WAL && (sqlite3Strlen30(zFilename)==0 /* Temp file */ @@ -82874,14 +101344,14 @@ case OP_JournalMode: { /* out2 */ ); goto abort_due_to_error; }else{ - + if( eOld==PAGER_JOURNALMODE_WAL ){ /* If leaving WAL mode, close the log file. If successful, the call - ** to PagerCloseWal() checkpoints and deletes the write-ahead-log - ** file. An EXCLUSIVE lock may still be held on the database file - ** after a successful return. + ** to PagerCloseWal() checkpoints and deletes the write-ahead-log + ** file. An EXCLUSIVE lock may still be held on the database file + ** after a successful return. */ - rc = sqlite3PagerCloseWal(pPager); + rc = sqlite3PagerCloseWal(pPager, db); if( rc==SQLITE_OK ){ sqlite3PagerSetJournalMode(pPager, eNew); } @@ -82890,11 +101360,11 @@ case OP_JournalMode: { /* out2 */ ** as an intermediate */ sqlite3PagerSetJournalMode(pPager, PAGER_JOURNALMODE_OFF); } - + /* Open a transaction on the database file. Regardless of the journal ** mode, this transaction always uses a rollback journal. */ - assert( sqlite3BtreeIsInTrans(pBt)==0 ); + assert( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE ); if( rc==SQLITE_OK ){ rc = sqlite3BtreeSetVersion(pBt, (eNew==PAGER_JOURNALMODE_WAL ? 2 : 1)); } @@ -82916,15 +101386,19 @@ case OP_JournalMode: { /* out2 */ #endif /* SQLITE_OMIT_PRAGMA */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* Opcode: Vacuum * * * * * +/* Opcode: Vacuum P1 P2 * * * ** -** Vacuum the entire database. This opcode will cause other virtual -** machines to be created and run. It may not be called from within -** a transaction. +** Vacuum the entire database P1. P1 is 0 for "main", and 2 or more +** for an attached database. The "temp" database may not be vacuumed. +** +** If P2 is not zero, then it is a register holding a string which is +** the file into which the result of vacuum should be written. When +** P2 is zero, the vacuum overwrites the original database. */ case OP_Vacuum: { assert( p->readOnly==0 ); - rc = sqlite3RunVacuum(&p->zErrMsg, db); + rc = sqlite3RunVacuum(&p->zErrMsg, db, pOp->p1, + pOp->p2 ? &aMem[pOp->p2] : 0); if( rc ) goto abort_due_to_error; break; } @@ -82955,31 +101429,68 @@ case OP_IncrVacuum: { /* jump */ } #endif -/* Opcode: Expire P1 * * * * +/* Opcode: Expire P1 P2 * * * ** ** Cause precompiled statements to expire. When an expired statement ** is executed using sqlite3_step() it will either automatically ** reprepare itself (if it was originally created using sqlite3_prepare_v2()) ** or it will fail with SQLITE_SCHEMA. -** +** ** If P1 is 0, then all SQL statements become expired. If P1 is non-zero, ** then only the currently executing statement is expired. +** +** If P2 is 0, then SQL statements are expired immediately. If P2 is 1, +** then running SQL statements are allowed to continue to run to completion. +** The P2==1 case occurs when a CREATE INDEX or similar schema change happens +** that might help the statement run faster but which does not affect the +** correctness of operation. */ case OP_Expire: { + assert( pOp->p2==0 || pOp->p2==1 ); if( !pOp->p1 ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, pOp->p2); }else{ - p->expired = 1; + p->expired = pOp->p2+1; } break; } +/* Opcode: CursorLock P1 * * * * +** +** Lock the btree to which cursor P1 is pointing so that the btree cannot be +** written by an other cursor. +*/ +case OP_CursorLock: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorPin(pC->uc.pCursor); + break; +} + +/* Opcode: CursorUnlock P1 * * * * +** +** Unlock the btree to which cursor P1 is pointing so that it can be +** written by other cursors. +*/ +case OP_CursorUnlock: { + VdbeCursor *pC; + assert( pOp->p1>=0 && pOp->p1nCursor ); + pC = p->apCsr[pOp->p1]; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + sqlite3BtreeCursorUnpin(pC->uc.pCursor); + break; +} + #ifndef SQLITE_OMIT_SHARED_CACHE /* Opcode: TableLock P1 P2 P3 P4 * ** Synopsis: iDb=P1 root=P2 write=P3 ** ** Obtain a lock on a particular table. This instruction is only used when -** the shared-cache feature is enabled. +** the shared-cache feature is enabled. ** ** P1 is the index of the database in sqlite3.aDb[] of the database ** on which the lock is acquired. A readlock is obtained if P3==0 or @@ -82992,8 +101503,8 @@ case OP_Expire: { */ case OP_TableLock: { u8 isWriteLock = (u8)pOp->p3; - if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommitted) ){ - int p1 = pOp->p1; + if( isWriteLock || 0==(db->flags&SQLITE_ReadUncommit) ){ + int p1 = pOp->p1; assert( p1>=0 && p1nDb ); assert( DbMaskTest(p->btreeMask, p1) ); assert( isWriteLock==0 || isWriteLock==1 ); @@ -83013,7 +101524,7 @@ case OP_TableLock: { #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VBegin * * * P4 * ** -** P4 may be a pointer to an sqlite3_vtab structure. If so, call the +** P4 may be a pointer to an sqlite3_vtab structure. If so, call the ** xBegin method for that table. ** ** Also, whether or not P4 is set, check that this is not being called from @@ -83033,7 +101544,7 @@ case OP_VBegin: { #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VCreate P1 P2 * * * ** -** P2 is a register that holds the name of a virtual table in database +** P2 is a register that holds the name of a virtual table in database ** P1. Call the xCreate method for that table. */ case OP_VCreate: { @@ -83069,6 +101580,7 @@ case OP_VDestroy: { db->nVDestroy++; rc = sqlite3VtabCallDestroy(db, pOp->p1, pOp->p4.z); db->nVDestroy--; + assert( p->errorAction==OE_Abort && p->usesStmtJournal ); if( rc ) goto abort_due_to_error; break; } @@ -83081,7 +101593,7 @@ case OP_VDestroy: { ** P1 is a cursor number. This opcode opens a cursor to the virtual ** table and stores that cursor in P1. */ -case OP_VOpen: { +case OP_VOpen: { /* ncycle */ VdbeCursor *pCur; sqlite3_vtab_cursor *pVCur; sqlite3_vtab *pVtab; @@ -83104,7 +101616,7 @@ case OP_VOpen: { pVCur->pVtab = pVtab; /* Initialize vdbe cursor object */ - pCur = allocateCursor(p, pOp->p1, 0, -1, CURTYPE_VTAB); + pCur = allocateCursor(p, pOp->p1, 0, CURTYPE_VTAB); if( pCur ){ pCur->uc.pVCur = pVCur; pVtab->nRef++; @@ -83117,6 +101629,80 @@ case OP_VOpen: { } #endif /* SQLITE_OMIT_VIRTUALTABLE */ +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VCheck P1 P2 P3 P4 * +** +** P4 is a pointer to a Table object that is a virtual table in schema P1 +** that supports the xIntegrity() method. This opcode runs the xIntegrity() +** method for that virtual table, using P3 as the integer argument. If +** an error is reported back, the table name is prepended to the error +** message and that message is stored in P2. If no errors are seen, +** register P2 is set to NULL. +*/ +case OP_VCheck: { /* out2 */ + Table *pTab; + sqlite3_vtab *pVtab; + const sqlite3_module *pModule; + char *zErr = 0; + + pOut = &aMem[pOp->p2]; + sqlite3VdbeMemSetNull(pOut); /* Innocent until proven guilty */ + assert( pOp->p4type==P4_TABLEREF ); + pTab = pOp->p4.pTab; + assert( pTab!=0 ); + assert( pTab->nTabRef>0 ); + assert( IsVirtual(pTab) ); + if( pTab->u.vtab.p==0 ) break; + pVtab = pTab->u.vtab.p->pVtab; + assert( pVtab!=0 ); + pModule = pVtab->pModule; + assert( pModule!=0 ); + assert( pModule->iVersion>=4 ); + assert( pModule->xIntegrity!=0 ); + sqlite3VtabLock(pTab->u.vtab.p); + assert( pOp->p1>=0 && pOp->p1nDb ); + rc = pModule->xIntegrity(pVtab, db->aDb[pOp->p1].zDbSName, pTab->zName, + pOp->p3, &zErr); + sqlite3VtabUnlock(pTab->u.vtab.p); + if( rc ){ + sqlite3_free(zErr); + goto abort_due_to_error; + } + if( zErr ){ + sqlite3VdbeMemSetStr(pOut, zErr, -1, SQLITE_UTF8, sqlite3_free); + } + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* Opcode: VInitIn P1 P2 P3 * * +** Synopsis: r[P2]=ValueList(P1,P3) +** +** Set register P2 to be a pointer to a ValueList object for cursor P1 +** with cache register P3 and output register P3+1. This ValueList object +** can be used as the first argument to sqlite3_vtab_in_first() and +** sqlite3_vtab_in_next() to extract all of the values stored in the P1 +** cursor. Register P3 is used to hold the values returned by +** sqlite3_vtab_in_first() and sqlite3_vtab_in_next(). +*/ +case OP_VInitIn: { /* out2, ncycle */ + VdbeCursor *pC; /* The cursor containing the RHS values */ + ValueList *pRhs; /* New ValueList object to put in reg[P2] */ + + pC = p->apCsr[pOp->p1]; + pRhs = sqlite3_malloc64( sizeof(*pRhs) ); + if( pRhs==0 ) goto no_mem; + pRhs->pCsr = pC->uc.pCursor; + pRhs->pOut = &aMem[pOp->p3]; + pOut = out2Prerelease(p, pOp); + pOut->flags = MEM_Null; + sqlite3VdbeMemSetPointer(pOut, pRhs, "ValueList", sqlite3VdbeValueListFree); + break; +} +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + #ifndef SQLITE_OMIT_VIRTUALTABLE /* Opcode: VFilter P1 P2 P3 P4 * ** Synopsis: iplan=r[P3] zplan='P4' @@ -83137,7 +101723,7 @@ case OP_VOpen: { ** ** A jump is made to P2 if the result set after filtering would be empty. */ -case OP_VFilter: { /* jump */ +case OP_VFilter: { /* jump, ncycle */ int nArg; int iQuery; const sqlite3_module *pModule; @@ -83155,6 +101741,7 @@ case OP_VFilter: { /* jump */ pCur = p->apCsr[pOp->p1]; assert( memIsValid(pQuery) ); REGISTER_TRACE(pOp->p3, pQuery); + assert( pCur!=0 ); assert( pCur->eCurType==CURTYPE_VTAB ); pVCur = pCur->uc.pVCur; pVtab = pVCur->pVtab; @@ -83166,7 +101753,6 @@ case OP_VFilter: { /* jump */ iQuery = (int)pQuery->u.i; /* Invoke the xFilter method */ - res = 0; apArg = p->apArg; for(i = 0; iapCsr[pOp->p1]; - assert( pCur->eCurType==CURTYPE_VTAB ); + assert( pCur!=0 ); assert( pOp->p3>0 && pOp->p3<=(p->nMem+1 - p->nCursor) ); pDest = &aMem[pOp->p3]; memAboutToChange(p, pDest); @@ -83205,24 +101799,34 @@ case OP_VColumn: { sqlite3VdbeMemSetNull(pDest); break; } + assert( pCur->eCurType==CURTYPE_VTAB ); pVtab = pCur->uc.pVCur->pVtab; pModule = pVtab->pModule; assert( pModule->xColumn ); memset(&sContext, 0, sizeof(sContext)); sContext.pOut = pDest; - MemSetTypeFlag(pDest, MEM_Null); + sContext.enc = encoding; + nullFunc.pUserData = 0; + nullFunc.funcFlags = SQLITE_RESULT_SUBTYPE; + sContext.pFunc = &nullFunc; + assert( pOp->p5==OPFLAG_NOCHNG || pOp->p5==0 ); + if( pOp->p5 & OPFLAG_NOCHNG ){ + sqlite3VdbeMemSetNull(pDest); + pDest->flags = MEM_Null|MEM_Zero; + pDest->u.nZero = 0; + }else{ + MemSetTypeFlag(pDest, MEM_Null); + } rc = pModule->xColumn(pCur->uc.pVCur, &sContext, pOp->p2); sqlite3VtabImportErrmsg(p, pVtab); - if( sContext.isError ){ + if( sContext.isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pDest)); rc = sContext.isError; } sqlite3VdbeChangeEncoding(pDest, encoding); REGISTER_TRACE(pOp->p3, pDest); UPDATE_MAX_BLOBSIZE(pDest); - if( sqlite3VdbeMemTooBig(pDest) ){ - goto too_big; - } if( rc ) goto abort_due_to_error; break; } @@ -83235,14 +101839,14 @@ case OP_VColumn: { ** jump to instruction P2. Or, if the virtual table has reached ** the end of its result set, then fall through to the next instruction. */ -case OP_VNext: { /* jump */ +case OP_VNext: { /* jump, ncycle */ sqlite3_vtab *pVtab; const sqlite3_module *pModule; int res; VdbeCursor *pCur; - res = 0; pCur = p->apCsr[pOp->p1]; + assert( pCur!=0 ); assert( pCur->eCurType==CURTYPE_VTAB ); if( pCur->nullRow ){ break; @@ -83253,7 +101857,7 @@ case OP_VNext: { /* jump */ /* Invoke the xNext() method of the module. There is no way for the ** underlying implementation to return an error if one occurs during - ** xNext(). Instead, if an error occurs, true is returned (indicating that + ** xNext(). Instead, if an error occurs, true is returned (indicating that ** data is available) and the error code returned when xColumn or ** some other method is next invoked on the save virtual table cursor. */ @@ -83280,7 +101884,10 @@ case OP_VNext: { /* jump */ case OP_VRename: { sqlite3_vtab *pVtab; Mem *pName; + int isLegacy; + isLegacy = (db->flags & SQLITE_LegacyAlter); + db->flags |= SQLITE_LegacyAlter; pVtab = pOp->p4.pVtab->pVtab; pName = &aMem[pOp->p1]; assert( pVtab->pModule->xRename ); @@ -83294,6 +101901,7 @@ case OP_VRename: { rc = sqlite3VdbeChangeEncoding(pName, SQLITE_UTF8); if( rc ) goto abort_due_to_error; rc = pVtab->pModule->xRename(pVtab, pName->z); + if( isLegacy==0 ) db->flags &= ~(u64)SQLITE_LegacyAlter; sqlite3VtabImportErrmsg(p, pVtab); p->expired = 0; if( rc ) goto abort_due_to_error; @@ -83307,23 +101915,23 @@ case OP_VRename: { ** ** P4 is a pointer to a virtual table object, an sqlite3_vtab structure. ** This opcode invokes the corresponding xUpdate method. P2 values -** are contiguous memory cells starting at P3 to pass to the xUpdate -** invocation. The value in register (P3+P2-1) corresponds to the +** are contiguous memory cells starting at P3 to pass to the xUpdate +** invocation. The value in register (P3+P2-1) corresponds to the ** p2th element of the argv array passed to xUpdate. ** ** The xUpdate method will do a DELETE or an INSERT or both. ** The argv[0] element (which corresponds to memory cell P3) -** is the rowid of a row to delete. If argv[0] is NULL then no -** deletion occurs. The argv[1] element is the rowid of the new -** row. This can be NULL to have the virtual table select the new -** rowid for itself. The subsequent elements in the array are +** is the rowid of a row to delete. If argv[0] is NULL then no +** deletion occurs. The argv[1] element is the rowid of the new +** row. This can be NULL to have the virtual table select the new +** rowid for itself. The subsequent elements in the array are ** the values of columns in the new row. ** ** If P2==1 then no insert is performed. argv[0] is the rowid of ** a row to delete. ** ** P1 is a boolean flag. If it is set to true and the xUpdate call -** is successful, then the value returned by sqlite3_last_insert_rowid() +** is successful, then the value returned by sqlite3_last_insert_rowid() ** is set to the value of the rowid for the row just inserted. ** ** P5 is the error actions (OE_Replace, OE_Fail, OE_Ignore, etc) to @@ -83334,14 +101942,16 @@ case OP_VUpdate: { const sqlite3_module *pModule; int nArg; int i; - sqlite_int64 rowid; + sqlite_int64 rowid = 0; Mem **apArg; Mem *pX; - assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback + assert( pOp->p2==1 || pOp->p5==OE_Fail || pOp->p5==OE_Rollback || pOp->p5==OE_Abort || pOp->p5==OE_Ignore || pOp->p5==OE_Replace ); assert( p->readOnly==0 ); + if( db->mallocFailed ) goto no_mem; + sqlite3VdbeIncrWriteCounter(p, 0); pVtab = pOp->p4.pVtab->pVtab; if( pVtab==0 || NEVER(pVtab->pModule==0) ){ rc = SQLITE_LOCKED; @@ -83366,7 +101976,7 @@ case OP_VUpdate: { sqlite3VtabImportErrmsg(p, pVtab); if( rc==SQLITE_OK && pOp->p1 ){ assert( nArg>1 && apArg[0] && (apArg[0]->flags&MEM_Null) ); - db->lastRowid = lastRowid = rowid; + db->lastRowid = rowid; } if( (rc&0xff)==SQLITE_CONSTRAINT && pOp->p4.pVtab->bConstraint ){ if( pOp->p5==OE_Ignore ){ @@ -83421,9 +102031,232 @@ case OP_MaxPgcnt: { /* out2 */ } #endif +/* Opcode: Function P1 P2 P3 P4 * +** Synopsis: r[P3]=func(r[P2@NP]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with arguments taken +** from register P2 and successors. The number of arguments is in +** the sqlite3_context object that P4 points to. +** The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** See also: AggStep, AggFinal, PureFunc +*/ +/* Opcode: PureFunc P1 P2 P3 P4 * +** Synopsis: r[P3]=func(r[P2@NP]) +** +** Invoke a user function (P4 is a pointer to an sqlite3_context object that +** contains a pointer to the function to be run) with arguments taken +** from register P2 and successors. The number of arguments is in +** the sqlite3_context object that P4 points to. +** The result of the function is stored +** in register P3. Register P3 must not be one of the function inputs. +** +** P1 is a 32-bit bitmask indicating whether or not each argument to the +** function was determined to be constant at compile time. If the first +** argument was constant then bit 0 of P1 is set. This is used to determine +** whether meta data associated with a user function argument using the +** sqlite3_set_auxdata() API may be safely retained until the next +** invocation of this opcode. +** +** This opcode works exactly like OP_Function. The only difference is in +** its name. This opcode is used in places where the function must be +** purely non-deterministic. Some built-in date/time functions can be +** either deterministic of non-deterministic, depending on their arguments. +** When those function are used in a non-deterministic way, they will check +** to see if they were called using OP_PureFunc instead of OP_Function, and +** if they were, they throw an error. +** +** See also: AggStep, AggFinal, Function +*/ +case OP_PureFunc: /* group */ +case OP_Function: { /* group */ + int i; + sqlite3_context *pCtx; -/* Opcode: Init * P2 * P4 * -** Synopsis: Start at P2 + assert( pOp->p4type==P4_FUNCCTX ); + pCtx = pOp->p4.pCtx; + + /* If this function is inside of a trigger, the register array in aMem[] + ** might change from one evaluation to the next. The next block of code + ** checks to see if the register array has changed, and if so it + ** reinitializes the relevant parts of the sqlite3_context object */ + pOut = &aMem[pOp->p3]; + if( pCtx->pOut != pOut ){ + pCtx->pVdbe = p; + pCtx->pOut = pOut; + pCtx->enc = encoding; + for(i=pCtx->argc-1; i>=0; i--) pCtx->argv[i] = &aMem[pOp->p2+i]; + } + assert( pCtx->pVdbe==p ); + + memAboutToChange(p, pOut); +#ifdef SQLITE_DEBUG + for(i=0; iargc; i++){ + assert( memIsValid(pCtx->argv[i]) ); + REGISTER_TRACE(pOp->p2+i, pCtx->argv[i]); + } +#endif + MemSetTypeFlag(pOut, MEM_Null); + assert( pCtx->isError==0 ); + (*pCtx->pFunc->xSFunc)(pCtx, pCtx->argc, pCtx->argv);/* IMP: R-24505-23230 */ + + /* If the function returned an error, throw an exception */ + if( pCtx->isError ){ + if( pCtx->isError>0 ){ + sqlite3VdbeError(p, "%s", sqlite3_value_text(pOut)); + rc = pCtx->isError; + } + sqlite3VdbeDeleteAuxData(db, &p->pAuxData, pCtx->iOp, pOp->p1); + pCtx->isError = 0; + if( rc ) goto abort_due_to_error; + } + + assert( (pOut->flags&MEM_Str)==0 + || pOut->enc==encoding + || db->mallocFailed ); + assert( !sqlite3VdbeMemTooBig(pOut) ); + + REGISTER_TRACE(pOp->p3, pOut); + UPDATE_MAX_BLOBSIZE(pOut); + break; +} + +/* Opcode: ClrSubtype P1 * * * * +** Synopsis: r[P1].subtype = 0 +** +** Clear the subtype from register P1. +*/ +case OP_ClrSubtype: { /* in1 */ + pIn1 = &aMem[pOp->p1]; + pIn1->flags &= ~MEM_Subtype; + break; +} + +/* Opcode: GetSubtype P1 P2 * * * +** Synopsis: r[P2] = r[P1].subtype +** +** Extract the subtype value from register P1 and write that subtype +** into register P2. If P1 has no subtype, then P1 gets a NULL. +*/ +case OP_GetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Subtype ){ + sqlite3VdbeMemSetInt64(pOut, pIn1->eSubtype); + }else{ + sqlite3VdbeMemSetNull(pOut); + } + break; +} + +/* Opcode: SetSubtype P1 P2 * * * +** Synopsis: r[P2].subtype = r[P1] +** +** Set the subtype value of register P2 to the integer from register P1. +** If P1 is NULL, clear the subtype from p2. +*/ +case OP_SetSubtype: { /* in1 out2 */ + pIn1 = &aMem[pOp->p1]; + pOut = &aMem[pOp->p2]; + if( pIn1->flags & MEM_Null ){ + pOut->flags &= ~MEM_Subtype; + }else{ + assert( pIn1->flags & MEM_Int ); + pOut->flags |= MEM_Subtype; + pOut->eSubtype = (u8)(pIn1->u.i & 0xff); + } + break; +} + +/* Opcode: FilterAdd P1 * P3 P4 * +** Synopsis: filter(P1) += key(P3@P4) +** +** Compute a hash on the P4 registers starting with r[P3] and +** add that hash to the bloom filter contained in r[P1]. +*/ +case OP_FilterAdd: { + u64 h; + + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + pIn1 = &aMem[pOp->p1]; + assert( pIn1->flags & MEM_Blob ); + assert( pIn1->n>0 ); + h = filterHash(aMem, pOp); +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + int ii; + for(ii=pOp->p3; iip3+pOp->p4.i; ii++){ + registerTrace(ii, &aMem[ii]); + } + printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n)); + } +#endif + h %= (pIn1->n*8); + pIn1->z[h/8] |= 1<<(h&7); + break; +} + +/* Opcode: Filter P1 P2 P3 P4 * +** Synopsis: if key(P3@P4) not in filter(P1) goto P2 +** +** Compute a hash on the key contained in the P4 registers starting +** with r[P3]. Check to see if that hash is found in the +** bloom filter hosted by register P1. If it is not present then +** maybe jump to P2. Otherwise fall through. +** +** False negatives are harmless. It is always safe to fall through, +** even if the value is in the bloom filter. A false negative causes +** more CPU cycles to be used, but it should still yield the correct +** answer. However, an incorrect answer may well arise from a +** false positive - if the jump is taken when it should fall through. +*/ +case OP_Filter: { /* jump */ + u64 h; + + assert( pOp->p1>0 && pOp->p1<=(p->nMem+1 - p->nCursor) ); + pIn1 = &aMem[pOp->p1]; + assert( (pIn1->flags & MEM_Blob)!=0 ); + assert( pIn1->n >= 1 ); + h = filterHash(aMem, pOp); +#ifdef SQLITE_DEBUG + if( db->flags&SQLITE_VdbeTrace ){ + int ii; + for(ii=pOp->p3; iip3+pOp->p4.i; ii++){ + registerTrace(ii, &aMem[ii]); + } + printf("hash: %llu modulo %d -> %u\n", h, pIn1->n, (int)(h%pIn1->n)); + } +#endif + h %= (pIn1->n*8); + if( (pIn1->z[h/8] & (1<<(h&7)))==0 ){ + VdbeBranchTaken(1, 2); + p->aCounter[SQLITE_STMTSTATUS_FILTER_HIT]++; + goto jump_to_p2; + }else{ + p->aCounter[SQLITE_STMTSTATUS_FILTER_MISS]++; + VdbeBranchTaken(0, 2); + } + break; +} + +/* Opcode: Trace P1 P2 * P4 * +** +** Write P4 on the statement trace output if statement tracing is +** enabled. +** +** Operand P1 must be 0x7fffffff and P2 must positive. +*/ +/* Opcode: Init P1 P2 P3 P4 * +** Synopsis: Start at P2 ** ** Programs contain a single instance of this opcode as the very first ** opcode. @@ -83433,27 +102266,61 @@ case OP_MaxPgcnt: { /* out2 */ ** Or if P4 is blank, use the string returned by sqlite3_sql(). ** ** If P2 is not zero, jump to instruction P2. +** +** Increment the value of P1 so that OP_Once opcodes will jump the +** first time they are evaluated for this run. +** +** If P3 is not zero, then it is an address to jump to if an SQLITE_CORRUPT +** error is encountered. */ -case OP_Init: { /* jump */ +case OP_Trace: +case OP_Init: { /* jump0 */ + int i; +#ifndef SQLITE_OMIT_TRACE char *zTrace; - char *z; +#endif + + /* If the P4 argument is not NULL, then it must be an SQL comment string. + ** The "--" string is broken up to prevent false-positives with srcck1.c. + ** + ** This assert() provides evidence for: + ** EVIDENCE-OF: R-50676-09860 The callback can compute the same text that + ** would have been returned by the legacy sqlite3_trace() interface by + ** using the X argument when X begins with "--" and invoking + ** sqlite3_expanded_sql(P) otherwise. + */ + assert( pOp->p4.z==0 || strncmp(pOp->p4.z, "-" "- ", 3)==0 ); + + /* OP_Init is always instruction 0 */ + assert( pOp==p->aOp || pOp->opcode==OP_Trace ); #ifndef SQLITE_OMIT_TRACE - if( db->xTrace - && !p->doingRerun + if( (db->mTrace & (SQLITE_TRACE_STMT|SQLITE_TRACE_LEGACY))!=0 + && p->minWriteFileFormat!=254 /* tag-20220401a */ && (zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql))!=0 ){ - z = sqlite3VdbeExpandSql(p, zTrace); - db->xTrace(db->pTraceArg, z); - sqlite3DbFree(db, z); +#ifndef SQLITE_OMIT_DEPRECATED + if( db->mTrace & SQLITE_TRACE_LEGACY ){ + char *z = sqlite3VdbeExpandSql(p, zTrace); + db->trace.xLegacy(db->pTraceArg, z); + sqlite3_free(z); + }else +#endif + if( db->nVdbeExec>1 ){ + char *z = sqlite3MPrintf(db, "-- %s", zTrace); + (void)db->trace.xV2(SQLITE_TRACE_STMT, db->pTraceArg, p, z); + sqlite3DbFree(db, z); + }else{ + (void)db->trace.xV2(SQLITE_TRACE_STMT, db->pTraceArg, p, zTrace); + } } #ifdef SQLITE_USE_FCNTL_TRACE zTrace = (pOp->p4.z ? pOp->p4.z : p->zSql); if( zTrace ){ - int i; - for(i=0; inDb; i++){ - if( DbMaskTest(p->btreeMask, i)==0 ) continue; - sqlite3_file_control(db, db->aDb[i].zName, SQLITE_FCNTL_TRACE, zTrace); + int j; + for(j=0; jnDb; j++){ + if( DbMaskTest(p->btreeMask, j)==0 ) continue; + sqlite3_file_control(db, db->aDb[j].zDbSName, SQLITE_FCNTL_TRACE, zTrace); } } #endif /* SQLITE_USE_FCNTL_TRACE */ @@ -83465,8 +102332,17 @@ case OP_Init: { /* jump */ } #endif /* SQLITE_DEBUG */ #endif /* SQLITE_OMIT_TRACE */ - if( pOp->p2 ) goto jump_to_p2; - break; + assert( pOp->p2>0 ); + if( pOp->p1>=sqlite3GlobalConfig.iOnceResetThreshold ){ + if( pOp->opcode==OP_Trace ) break; + for(i=1; inOp; i++){ + if( p->aOp[i].opcode==OP_Once ) p->aOp[i].p1 = 0; + } + pOp->p1 = 0; + } + pOp->p1++; + p->aCounter[SQLITE_STMTSTATUS_RUN]++; + goto jump_to_p2; } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -83492,19 +102368,100 @@ case OP_CursorHint: { } #endif /* SQLITE_ENABLE_CURSOR_HINTS */ +#ifdef SQLITE_DEBUG +/* Opcode: Abortable * * * * * +** +** Verify that an Abort can happen. Assert if an Abort at this point +** might cause database corruption. This opcode only appears in debugging +** builds. +** +** An Abort is safe if either there have been no writes, or if there is +** an active statement journal. +*/ +case OP_Abortable: { + sqlite3VdbeAssertAbortable(p); + break; +} +#endif + +#ifdef SQLITE_DEBUG +/* Opcode: ReleaseReg P1 P2 P3 * P5 +** Synopsis: release r[P1@P2] mask P3 +** +** Release registers from service. Any content that was in the +** the registers is unreliable after this opcode completes. +** +** The registers released will be the P2 registers starting at P1, +** except if bit ii of P3 set, then do not release register P1+ii. +** In other words, P3 is a mask of registers to preserve. +** +** Releasing a register clears the Mem.pScopyFrom pointer. That means +** that if the content of the released register was set using OP_SCopy, +** a change to the value of the source register for the OP_SCopy will no longer +** generate an assertion fault in sqlite3VdbeMemAboutToChange(). +** +** If P5 is set, then all released registers have their type set +** to MEM_Undefined so that any subsequent attempt to read the released +** register (before it is reinitialized) will generate an assertion fault. +** +** P5 ought to be set on every call to this opcode. +** However, there are places in the code generator will release registers +** before their are used, under the (valid) assumption that the registers +** will not be reallocated for some other purpose before they are used and +** hence are safe to release. +** +** This opcode is only available in testing and debugging builds. It is +** not generated for release builds. The purpose of this opcode is to help +** validate the generated bytecode. This opcode does not actually contribute +** to computing an answer. +*/ +case OP_ReleaseReg: { + Mem *pMem; + int i; + u32 constMask; + assert( pOp->p1>0 ); + assert( pOp->p1+pOp->p2<=(p->nMem+1 - p->nCursor)+1 ); + pMem = &aMem[pOp->p1]; + constMask = pOp->p3; + for(i=0; ip2; i++, pMem++){ + if( i>=32 || (constMask & MASKBIT32(i))==0 ){ + pMem->pScopyFrom = 0; + if( i<32 && pOp->p5 ) MemSetTypeFlag(pMem, MEM_Undefined); + } + } + break; +} +#endif + /* Opcode: Noop * * * * * ** -** Do nothing. This instruction is often useful as a jump -** destination. +** Do nothing. Continue downward to the next opcode. */ -/* -** The magic Explain opcode are only inserted when explain==2 (which -** is to say when the EXPLAIN QUERY PLAN syntax is used.) -** This opcode records information from the optimizer. It is the -** the same as a no-op. This opcodesnever appears in a real VM program. +/* Opcode: Explain P1 P2 P3 P4 * +** +** This is the same as OP_Noop during normal query execution. The +** purpose of this opcode is to hold information about the query +** plan for the purpose of EXPLAIN QUERY PLAN output. +** +** The P4 value is human-readable text that describes the query plan +** element. Something like "SCAN t1" or "SEARCH t2 USING INDEX t2x1". +** +** The P1 value is the ID of the current element and P2 is the parent +** element for the case of nested query plan elements. If P2 is zero +** then this element is a top-level element. +** +** For loop elements, P3 is the estimated code of each invocation of this +** element. +** +** As with all opcodes, the meanings of the parameters for OP_Explain +** are subject to change from one release to the next. Applications +** should not attempt to interpret or use any of the information +** contained in the OP_Explain opcode. The information provided by this +** opcode is intended for testing and debugging use only. */ -default: { /* This is really OP_Noop and OP_Explain */ +default: { /* This is really OP_Noop, OP_Explain */ assert( pOp->opcode==OP_Noop || pOp->opcode==OP_Explain ); + break; } @@ -83516,11 +102473,13 @@ default: { /* This is really OP_Noop and OP_Explain */ *****************************************************************************/ } -#ifdef VDBE_PROFILE - { - u64 endTime = sqlite3Hwtime(); - if( endTime>start ) pOrigOp->cycles += endTime - start; - pOrigOp->cnt++; +#if defined(VDBE_PROFILE) + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; } #endif @@ -83542,6 +102501,12 @@ default: { /* This is really OP_Noop and OP_Explain */ if( opProperty & OPFLG_OUT3 ){ registerTrace(pOrigOp->p3, &aMem[pOrigOp->p3]); } + if( opProperty==0xff ){ + /* Never happens. This code exists to avoid a harmless linkage + ** warning about sqlite3VdbeRegisterDump() being defined but not + ** used. */ + sqlite3VdbeRegisterDump(p); + } } #endif /* SQLITE_DEBUG */ #endif /* NDEBUG */ @@ -83551,18 +102516,37 @@ default: { /* This is really OP_Noop and OP_Explain */ ** an error of some kind. */ abort_due_to_error: - if( db->mallocFailed ) rc = SQLITE_NOMEM_BKPT; + if( db->mallocFailed ){ + rc = SQLITE_NOMEM_BKPT; + }else if( rc==SQLITE_IOERR_CORRUPTFS ){ + rc = SQLITE_CORRUPT_BKPT; + } assert( rc ); +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeTrace ){ + const char *zTrace = p->zSql; + if( zTrace==0 ){ + if( aOp[0].opcode==OP_Trace ){ + zTrace = aOp[0].p4.z; + } + if( zTrace==0 ) zTrace = "???"; + } + printf("ABORT-due-to-error (rc=%d): %s\n", rc, zTrace); + } +#endif if( p->zErrMsg==0 && rc!=SQLITE_IOERR_NOMEM ){ sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); } p->rc = rc; sqlite3SystemError(db, rc); testcase( sqlite3GlobalConfig.xLog!=0 ); - sqlite3_log(rc, "statement aborts at %d: [%s] %s", + sqlite3_log(rc, "statement aborts at %d: [%s] %s", (int)(pOp - aOp), p->zSql, p->zErrMsg); - sqlite3VdbeHalt(p); + if( p->eVdbeState==VDBE_RUN_STATE ) sqlite3VdbeHalt(p); if( rc==SQLITE_IOERR_NOMEM ) sqlite3OomFault(db); + if( rc==SQLITE_CORRUPT && db->autoCommit==0 ){ + db->flags |= SQLITE_CorruptRdOnly; + } rc = SQLITE_ERROR; if( resetSchemaOnFault>0 ){ sqlite3ResetOneSchema(db, resetSchemaOnFault-1); @@ -83572,12 +102556,34 @@ abort_due_to_error: ** release the mutexes on btrees that were acquired at the ** top. */ vdbe_return: - db->lastRowid = lastRowid; - testcase( nVmStep>0 ); +#if defined(VDBE_PROFILE) + if( pnCycle ){ + *pnCycle += sqlite3NProfileCnt ? sqlite3NProfileCnt : sqlite3Hwtime(); + pnCycle = 0; + } +#elif defined(SQLITE_ENABLE_STMT_SCANSTATUS) + if( pnCycle ){ + *pnCycle += sqlite3Hwtime(); + pnCycle = 0; + } +#endif + +#ifndef SQLITE_OMIT_PROGRESS_CALLBACK + while( nVmStep>=nProgressLimit && db->xProgress!=0 ){ + nProgressLimit += db->nProgressOps; + if( db->xProgress(db->pProgressArg) ){ + nProgressLimit = LARGEST_UINT64; + rc = SQLITE_INTERRUPT; + goto abort_due_to_error; + } + } +#endif p->aCounter[SQLITE_STMTSTATUS_VM_STEP] += (int)nVmStep; - sqlite3VdbeLeave(p); - assert( rc!=SQLITE_OK || nExtraDelete==0 - || sqlite3_strlike("DELETE%",p->zSql,0)!=0 + if( DbMaskNonZero(p->lockMask) ){ + sqlite3VdbeLeave(p); + } + assert( rc!=SQLITE_OK || nExtraDelete==0 + || sqlite3_strlike("DELETE%",p->zSql,0)!=0 ); return rc; @@ -83601,10 +102607,8 @@ no_mem: ** flag. */ abort_due_to_interrupt: - assert( db->u1.isInterrupted ); - rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_INTERRUPT; - p->rc = rc; - sqlite3VdbeError(p, "%s", sqlite3ErrStr(rc)); + assert( AtomicLoad(&db->u1.isInterrupted) ); + rc = SQLITE_INTERRUPT; goto abort_due_to_error; } @@ -83636,10 +102640,9 @@ abort_due_to_interrupt: */ typedef struct Incrblob Incrblob; struct Incrblob { - int flags; /* Copy of "flags" passed to sqlite3_blob_open() */ int nByte; /* Size of open blob, in bytes */ int iOffset; /* Byte offset of blob in cursor data */ - int iCol; /* Table column this handle is open on */ + u16 iCol; /* Table column this handle is open on */ BtCursor *pCsr; /* Cursor pointing at blob row */ sqlite3_stmt *pStmt; /* Statement holding cursor open */ sqlite3 *db; /* The associated database */ @@ -83662,7 +102665,7 @@ struct Incrblob { ** sqlite3DbFree(). ** ** If an error does occur, then the b-tree cursor is closed. All subsequent -** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will +** calls to sqlite3_blob_read(), blob_write() or blob_reopen() will ** immediately return SQLITE_ABORT. */ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ @@ -83670,17 +102673,30 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ char *zErr = 0; /* Error message */ Vdbe *v = (Vdbe *)p->pStmt; - /* Set the value of the SQL statements only variable to integer iRow. - ** This is done directly instead of using sqlite3_bind_int64() to avoid - ** triggering asserts related to mutexes. + /* Set the value of register r[1] in the SQL statement to integer iRow. + ** This is done directly as a performance optimization */ - assert( v->aVar[0].flags&MEM_Int ); - v->aVar[0].u.i = iRow; + sqlite3VdbeMemSetInt64(&v->aMem[1], iRow); - rc = sqlite3_step(p->pStmt); + /* If the statement has been run before (and is paused at the OP_ResultRow) + ** then back it up to the point where it does the OP_NotExists. This could + ** have been down with an extra OP_Goto, but simply setting the program + ** counter is faster. */ + if( v->pc>4 ){ + v->pc = 4; + assert( v->aOp[v->pc].opcode==OP_NotExists ); + rc = sqlite3VdbeExec(v); + }else{ + rc = sqlite3_step(p->pStmt); + } if( rc==SQLITE_ROW ){ VdbeCursor *pC = v->apCsr[0]; - u32 type = pC->aType[p->iCol]; + u32 type; + assert( pC!=0 ); + assert( pC->eCurType==CURTYPE_BTREE ); + type = pC->nHdrParsed>p->iCol ? pC->aType[p->iCol] : 0; + testcase( pC->nHdrParsed==p->iCol ); + testcase( pC->nHdrParsed==p->iCol+1 ); if( type<12 ){ zErr = sqlite3MPrintf(p->db, "cannot open value of type %s", type==0?"null": type==7?"real": "integer" @@ -83719,13 +102735,13 @@ static int blobSeekToRow(Incrblob *p, sqlite3_int64 iRow, char **pzErr){ /* ** Open a blob handle. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( +SQLITE_API int sqlite3_blob_open( sqlite3* db, /* The database connection */ const char *zDb, /* The attached database containing the blob */ const char *zTable, /* The table containing the blob */ const char *zColumn, /* The column containing the blob */ sqlite_int64 iRow, /* The row containing the glob */ - int flags, /* True -> read/write access, false -> read-only */ + int wrFlag, /* True -> read/write access, false -> read-only */ sqlite3_blob **ppBlob /* Handle for accessing the blob returned here */ ){ int nAttempt = 0; @@ -83733,8 +102749,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( int rc = SQLITE_OK; char *zErr = 0; Table *pTab; - Parse *pParse = 0; Incrblob *pBlob = 0; + Parse sParse; #ifdef SQLITE_ENABLE_API_ARMOR if( ppBlob==0 ){ @@ -83743,57 +102759,58 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( #endif *ppBlob = 0; #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) || zTable==0 ){ + if( !sqlite3SafetyCheckOk(db) || zTable==0 || zColumn==0 ){ return SQLITE_MISUSE_BKPT; } #endif - flags = !!flags; /* flags = (flags ? 1 : 0); */ + wrFlag = !!wrFlag; /* wrFlag = (wrFlag ? 1 : 0); */ sqlite3_mutex_enter(db->mutex); pBlob = (Incrblob *)sqlite3DbMallocZero(db, sizeof(Incrblob)); - if( !pBlob ) goto blob_open_out; - pParse = sqlite3StackAllocRaw(db, sizeof(*pParse)); - if( !pParse ) goto blob_open_out; - - do { - memset(pParse, 0, sizeof(Parse)); - pParse->db = db; + while(1){ + sqlite3ParseObjectInit(&sParse,db); + if( !pBlob ) goto blob_open_out; sqlite3DbFree(db, zErr); zErr = 0; sqlite3BtreeEnterAll(db); - pTab = sqlite3LocateTable(pParse, 0, zTable, zDb); + pTab = sqlite3LocateTable(&sParse, 0, zTable, zDb); if( pTab && IsVirtual(pTab) ){ pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open virtual table: %s", zTable); + sqlite3ErrorMsg(&sParse, "cannot open virtual table: %s", zTable); } if( pTab && !HasRowid(pTab) ){ pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open table without rowid: %s", zTable); + sqlite3ErrorMsg(&sParse, "cannot open table without rowid: %s", zTable); + } + if( pTab && (pTab->tabFlags&TF_HasGenerated)!=0 ){ + pTab = 0; + sqlite3ErrorMsg(&sParse, "cannot open table with generated columns: %s", + zTable); } #ifndef SQLITE_OMIT_VIEW - if( pTab && pTab->pSelect ){ + if( pTab && IsView(pTab) ){ pTab = 0; - sqlite3ErrorMsg(pParse, "cannot open view: %s", zTable); + sqlite3ErrorMsg(&sParse, "cannot open view: %s", zTable); } #endif if( !pTab ){ - if( pParse->zErrMsg ){ + if( sParse.zErrMsg ){ sqlite3DbFree(db, zErr); - zErr = pParse->zErrMsg; - pParse->zErrMsg = 0; + zErr = sParse.zErrMsg; + sParse.zErrMsg = 0; } rc = SQLITE_ERROR; sqlite3BtreeLeaveAll(db); goto blob_open_out; } pBlob->pTab = pTab; - pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zName; + pBlob->zDb = db->aDb[sqlite3SchemaToIndex(db, pTab->pSchema)].zDbSName; /* Now search pTab for the exact column. */ for(iCol=0; iColnCol; iCol++) { - if( sqlite3StrICmp(pTab->aCol[iCol].zName, zColumn)==0 ){ + if( sqlite3StrICmp(pTab->aCol[iCol].zCnName, zColumn)==0 ){ break; } } @@ -83806,20 +102823,20 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( } /* If the value is being opened for writing, check that the - ** column is not indexed, and that it is not part of a foreign key. - ** It is against the rules to open a column to which either of these - ** descriptions applies for writing. */ - if( flags ){ + ** column is not indexed, and that it is not part of a foreign key. + */ + if( wrFlag ){ const char *zFault = 0; Index *pIdx; #ifndef SQLITE_OMIT_FOREIGN_KEY if( db->flags&SQLITE_ForeignKeys ){ /* Check that the column is not part of an FK child key definition. It ** is not necessary to check if it is part of a parent key, as parent - ** key columns must be indexed. The check below will pick up this + ** key columns must be indexed. The check below will pick up this ** case. */ FKey *pFKey; - for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + assert( IsOrdinaryTable(pTab) ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ int j; for(j=0; jnCol; j++){ if( pFKey->aCol[j].iFrom==iCol ){ @@ -83847,11 +102864,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( } } - pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(pParse); + pBlob->pStmt = (sqlite3_stmt *)sqlite3VdbeCreate(&sParse); assert( pBlob->pStmt || db->mallocFailed ); if( pBlob->pStmt ){ - - /* This VDBE program seeks a btree cursor to the identified + + /* This VDBE program seeks a btree cursor to the identified ** db/table/row entry. The reason for using a vdbe program instead ** of writing code to use the b-tree layer directly is that the ** vdbe program will take advantage of the various transaction, @@ -83859,37 +102876,36 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( ** ** After seeking the cursor, the vdbe executes an OP_ResultRow. ** Code external to the Vdbe then "borrows" the b-tree cursor and - ** uses it to implement the blob_read(), blob_write() and + ** uses it to implement the blob_read(), blob_write() and ** blob_bytes() functions. ** ** The sqlite3_blob_close() function finalizes the vdbe program, - ** which closes the b-tree cursor and (possibly) commits the + ** which closes the b-tree cursor and (possibly) commits the ** transaction. */ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList openBlob[] = { {OP_TableLock, 0, 0, 0}, /* 0: Acquire a read or write lock */ {OP_OpenRead, 0, 0, 0}, /* 1: Open a cursor */ - {OP_Variable, 1, 1, 0}, /* 2: Move ?1 into reg[1] */ - {OP_NotExists, 0, 7, 1}, /* 3: Seek the cursor */ - {OP_Column, 0, 0, 1}, /* 4 */ - {OP_ResultRow, 1, 0, 0}, /* 5 */ - {OP_Goto, 0, 2, 0}, /* 6 */ - {OP_Close, 0, 0, 0}, /* 7 */ - {OP_Halt, 0, 0, 0}, /* 8 */ + /* blobSeekToRow() will initialize r[1] to the desired rowid */ + {OP_NotExists, 0, 5, 1}, /* 2: Seek the cursor to rowid=r[1] */ + {OP_Column, 0, 0, 1}, /* 3 */ + {OP_ResultRow, 1, 0, 0}, /* 4 */ + {OP_Halt, 0, 0, 0}, /* 5 */ }; Vdbe *v = (Vdbe *)pBlob->pStmt; int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); VdbeOp *aOp; - sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, flags, + sqlite3VdbeAddOp4Int(v, OP_Transaction, iDb, wrFlag, pTab->pSchema->schema_cookie, pTab->pSchema->iGeneration); - sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP5(v, 1); + assert( sqlite3VdbeCurrentAddr(v)==2 || db->mallocFailed ); aOp = sqlite3VdbeAddOpList(v, ArraySize(openBlob), openBlob, iLn); /* Make sure a mutex is held on the table to be accessed */ - sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeUsesBtree(v, iDb); if( db->mallocFailed==0 ){ assert( aOp!=0 ); @@ -83899,46 +102915,46 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_open( #else aOp[0].p1 = iDb; aOp[0].p2 = pTab->tnum; - aOp[0].p3 = flags; - sqlite3VdbeChangeP4(v, 1, pTab->zName, P4_TRANSIENT); + aOp[0].p3 = wrFlag; + sqlite3VdbeChangeP4(v, 2, pTab->zName, P4_TRANSIENT); } if( db->mallocFailed==0 ){ #endif - /* Remove either the OP_OpenWrite or OpenRead. Set the P2 + /* Remove either the OP_OpenWrite or OpenRead. Set the P2 ** parameter of the other to pTab->tnum. */ - if( flags ) aOp[1].opcode = OP_OpenWrite; + if( wrFlag ) aOp[1].opcode = OP_OpenWrite; aOp[1].p2 = pTab->tnum; - aOp[1].p3 = iDb; + aOp[1].p3 = iDb; /* Configure the number of columns. Configure the cursor to ** think that the table has one more column than it really ** does. An OP_Column to retrieve this imaginary column will ** always return an SQL NULL. This is useful because it means - ** we can invoke OP_Column to fill in the vdbe cursors type + ** we can invoke OP_Column to fill in the vdbe cursors type ** and offset cache without causing any IO. */ aOp[1].p4type = P4_INT32; aOp[1].p4.i = pTab->nCol+1; - aOp[4].p2 = pTab->nCol; + aOp[3].p2 = pTab->nCol; - pParse->nVar = 1; - pParse->nMem = 1; - pParse->nTab = 1; - sqlite3VdbeMakeReady(v, pParse); + sParse.nVar = 0; + sParse.nMem = 1; + sParse.nTab = 1; + sqlite3VdbeMakeReady(v, &sParse); } } - - pBlob->flags = flags; + pBlob->iCol = iCol; pBlob->db = db; sqlite3BtreeLeaveAll(db); if( db->mallocFailed ){ goto blob_open_out; } - sqlite3_bind_int64(pBlob->pStmt, 1, iRow); rc = blobSeekToRow(pBlob, iRow, &zErr); - } while( (++nAttempt)=SQLITE_MAX_SCHEMA_RETRY || rc!=SQLITE_SCHEMA ) break; + sqlite3ParseObjectReset(&sParse); + } blob_open_out: if( rc==SQLITE_OK && db->mallocFailed==0 ){ @@ -83947,10 +102963,9 @@ blob_open_out: if( pBlob && pBlob->pStmt ) sqlite3VdbeFinalize((Vdbe *)pBlob->pStmt); sqlite3DbFree(db, pBlob); } - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); + sqlite3ParseObjectReset(&sParse); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -83960,17 +102975,18 @@ blob_open_out: ** Close a blob handle that was previously created using ** sqlite3_blob_open(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *pBlob){ +SQLITE_API int sqlite3_blob_close(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; int rc; sqlite3 *db; if( p ){ + sqlite3_stmt *pStmt = p->pStmt; db = p->db; sqlite3_mutex_enter(db->mutex); - rc = sqlite3_finalize(p->pStmt); sqlite3DbFree(db, p); sqlite3_mutex_leave(db->mutex); + rc = sqlite3_finalize(pStmt); }else{ rc = SQLITE_OK; } @@ -83981,10 +102997,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_close(sqlite3_blob *pBlob){ ** Perform a read or write operation on a blob */ static int blobReadWrite( - sqlite3_blob *pBlob, - void *z, - int n, - int iOffset, + sqlite3_blob *pBlob, + void *z, + int n, + int iOffset, int (*xCall)(BtCursor*, u32, u32, void*) ){ int rc; @@ -84014,23 +103030,25 @@ static int blobReadWrite( #ifdef SQLITE_ENABLE_PREUPDATE_HOOK if( xCall==sqlite3BtreePutData && db->xPreUpdateCallback ){ - /* If a pre-update hook is registered and this is a write cursor, - ** invoke it here. - ** + /* If a pre-update hook is registered and this is a write cursor, + ** invoke it here. + ** ** TODO: The preupdate-hook is passed SQLITE_DELETE, even though this ** operation should really be an SQLITE_UPDATE. This is probably - ** incorrect, but is convenient because at this point the new.* values - ** are not easily obtainable. And for the sessions module, an - ** SQLITE_UPDATE where the PK columns do not change is handled in the + ** incorrect, but is convenient because at this point the new.* values + ** are not easily obtainable. And for the sessions module, an + ** SQLITE_UPDATE where the PK columns do not change is handled in the ** same way as an SQLITE_DELETE (the SQLITE_DELETE code is actually ** slightly more efficient). Since you cannot write to a PK column ** using the incremental-blob API, this works. For the sessions module ** anyhow. */ sqlite3_int64 iKey; - sqlite3BtreeKeySize(p->pCsr, &iKey); + iKey = sqlite3BtreeIntegerKey(p->pCsr); + assert( v->apCsr[0]!=0 ); + assert( v->apCsr[0]->eCurType==CURTYPE_BTREE ); sqlite3VdbePreUpdateHook( - v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1 + v, v->apCsr[0], SQLITE_DELETE, p->zDb, p->pTab, iKey, -1, p->iCol ); } #endif @@ -84053,14 +103071,14 @@ static int blobReadWrite( /* ** Read data from a blob handle. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ - return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreeData); +SQLITE_API int sqlite3_blob_read(sqlite3_blob *pBlob, void *z, int n, int iOffset){ + return blobReadWrite(pBlob, z, n, iOffset, sqlite3BtreePayloadChecked); } /* ** Write data to a blob handle. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *pBlob, const void *z, int n, int iOffset){ return blobReadWrite(pBlob, (void *)z, n, iOffset, sqlite3BtreePutData); } @@ -84070,7 +103088,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_write(sqlite3_blob *pBlob, const void ** The Incrblob.nByte field is fixed for the lifetime of the Incrblob ** so no mutex is required for access. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *pBlob){ +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *pBlob){ Incrblob *p = (Incrblob *)pBlob; return (p && p->pStmt) ? p->nByte : 0; } @@ -84081,11 +103099,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_bytes(sqlite3_blob *pBlob){ ** ** If an error occurs, or if the specified row does not exist or does not ** contain a blob or text value, then an error code is returned and the -** database handle error code and message set. If this happens, then all -** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) +** database handle error code and message set. If this happens, then all +** subsequent calls to sqlite3_blob_xxx() functions (except blob_close()) ** immediately return SQLITE_ABORT. */ -SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_int64 iRow){ int rc; Incrblob *p = (Incrblob *)pBlob; sqlite3 *db; @@ -84101,9 +103119,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i rc = SQLITE_ABORT; }else{ char *zErr; + ((Vdbe*)p->pStmt)->rc = SQLITE_OK; rc = blobSeekToRow(p, iRow, &zErr); if( rc!=SQLITE_OK ){ - sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : 0), zErr); + sqlite3ErrorWithMsg(db, rc, (zErr ? "%s" : (char*)0), zErr); sqlite3DbFree(db, zErr); } assert( rc!=SQLITE_SCHEMA ); @@ -84176,7 +103195,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i ** is like Close() followed by Init() only ** much faster. ** -** The interfaces above must be called in a particular order. Write() can +** The interfaces above must be called in a particular order. Write() can ** only occur in between Init()/Reset() and Rewind(). Next(), Rowkey(), and ** Compare() can only occur in between Rewind() and Close()/Reset(). i.e. ** @@ -84184,16 +103203,16 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i ** for each record: Write() ** Rewind() ** Rowkey()/Compare() -** Next() +** Next() ** Close() ** ** Algorithm: ** -** Records passed to the sorter via calls to Write() are initially held +** Records passed to the sorter via calls to Write() are initially held ** unsorted in main memory. Assuming the amount of memory used never exceeds ** a threshold, when Rewind() is called the set of records is sorted using ** an in-memory merge sort. In this case, no temporary files are required -** and subsequent calls to Rowkey(), Next() and Compare() read records +** and subsequent calls to Rowkey(), Next() and Compare() read records ** directly from main memory. ** ** If the amount of space used to store records in main memory exceeds the @@ -84203,10 +103222,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i ** of PMAs may be created by merging existing PMAs together - for example ** merging two or more level-0 PMAs together creates a level-1 PMA. ** -** The threshold for the amount of main memory to use before flushing +** The threshold for the amount of main memory to use before flushing ** records to a PMA is roughly the same as the limit configured for the -** page-cache of the main database. Specifically, the threshold is set to -** the value returned by "PRAGMA main.page_size" multipled by +** page-cache of the main database. Specifically, the threshold is set to +** the value returned by "PRAGMA main.page_size" multiplied by ** that returned by "PRAGMA main.cache_size", in bytes. ** ** If the sorter is running in single-threaded mode, then all PMAs generated @@ -84223,28 +103242,28 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i ** than zero, and (b) worker threads have been enabled at runtime by calling ** "PRAGMA threads=N" with some value of N greater than 0. ** -** When Rewind() is called, any data remaining in memory is flushed to a +** When Rewind() is called, any data remaining in memory is flushed to a ** final PMA. So at this point the data is stored in some number of sorted ** PMAs within temporary files on disk. ** ** If there are fewer than SORTER_MAX_MERGE_COUNT PMAs in total and the ** sorter is running in single-threaded mode, then these PMAs are merged -** incrementally as keys are retreived from the sorter by the VDBE. The +** incrementally as keys are retrieved from the sorter by the VDBE. The ** MergeEngine object, described in further detail below, performs this ** merge. ** ** Or, if running in multi-threaded mode, then a background thread is ** launched to merge the existing PMAs. Once the background thread has -** merged T bytes of data into a single sorted PMA, the main thread +** merged T bytes of data into a single sorted PMA, the main thread ** begins reading keys from that PMA while the background thread proceeds ** with merging the next T bytes of data. And so on. ** -** Parameter T is set to half the value of the memory threshold used +** Parameter T is set to half the value of the memory threshold used ** by Write() above to determine when to create a new PMA. ** -** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when -** Rewind() is called, then a hierarchy of incremental-merges is used. -** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on +** If there are more than SORTER_MAX_MERGE_COUNT PMAs in total when +** Rewind() is called, then a hierarchy of incremental-merges is used. +** First, T bytes of data from the first SORTER_MAX_MERGE_COUNT PMAs on ** disk are merged together. Then T bytes of data from the second set, and ** so on, such that no operation ever merges more than SORTER_MAX_MERGE_COUNT ** PMAs at a time. This done is to improve locality. @@ -84259,7 +103278,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_blob_reopen(sqlite3_blob *pBlob, sqlite3_i /* #include "sqliteInt.h" */ /* #include "vdbeInt.h" */ -/* +/* ** If SQLITE_DEBUG_SORTER_THREADS is defined, this module outputs various ** messages to stderr that may be helpful in understanding the performance ** characteristics of the sorter in multi-threaded mode. @@ -84288,7 +103307,7 @@ typedef struct SorterList SorterList; /* In-memory list of records */ typedef struct IncrMerger IncrMerger; /* Read & merge multiple PMAs */ /* -** A container for a temp file handle and the current amount of data +** A container for a temp file handle and the current amount of data ** stored in the file. */ struct SorterFile { @@ -84307,7 +103326,7 @@ struct SorterFile { struct SorterList { SorterRecord *pList; /* Linked list of records */ u8 *aMemory; /* If non-NULL, bulk memory to hold pList */ - int szPMA; /* Size of pList as PMA in bytes */ + i64 szPMA; /* Size of pList as PMA in bytes */ }; /* @@ -84328,17 +103347,17 @@ struct SorterList { ** the MergeEngine.nTree variable. ** ** The final (N/2) elements of aTree[] contain the results of comparing -** pairs of PMA keys together. Element i contains the result of +** pairs of PMA keys together. Element i contains the result of ** comparing aReadr[2*i-N] and aReadr[2*i-N+1]. Whichever key is smaller, the -** aTree element is set to the index of it. +** aTree element is set to the index of it. ** ** For the purposes of this comparison, EOF is considered greater than any ** other key value. If the keys are equal (only possible with two EOF ** values), it doesn't matter which index is stored. ** -** The (N/4) elements of aTree[] that precede the final (N/2) described +** The (N/4) elements of aTree[] that precede the final (N/2) described ** above contains the index of the smallest of each block of 4 PmaReaders -** And so on. So that aTree[1] contains the index of the PmaReader that +** And so on. So that aTree[1] contains the index of the PmaReader that ** currently points to the smallest key value. aTree[0] is unused. ** ** Example: @@ -84354,7 +103373,7 @@ struct SorterList { ** ** aTree[] = { X, 5 0, 5 0, 3, 5, 6 } ** -** The current element is "Apple" (the value of the key indicated by +** The current element is "Apple" (the value of the key indicated by ** PmaReader 5). When the Next() operation is invoked, PmaReader 5 will ** be advanced to the next key in its segment. Say the next key is ** "Eggplant": @@ -84392,11 +103411,11 @@ struct MergeEngine { ** ** Essentially, this structure contains all those fields of the VdbeSorter ** structure for which each thread requires a separate instance. For example, -** each thread requries its own UnpackedRecord object to unpack records in +** each thread requeries its own UnpackedRecord object to unpack records in ** as part of comparison operations. ** -** Before a background thread is launched, variable bDone is set to 0. Then, -** right before it exits, the thread itself sets bDone to 1. This is used for +** Before a background thread is launched, variable bDone is set to 0. Then, +** right before it exits, the thread itself sets bDone to 1. This is used for ** two purposes: ** ** 1. When flushing the contents of memory to a level-0 PMA on disk, to @@ -84416,10 +103435,10 @@ typedef int (*SorterCompare)(SortSubtask*,int*,const void*,int,const void*,int); struct SortSubtask { SQLiteThread *pThread; /* Background thread, if any */ int bDone; /* Set if thread is finished but not joined */ + int nPMA; /* Number of PMAs currently in file */ VdbeSorter *pSorter; /* Sorter that owns this sub-task */ UnpackedRecord *pUnpacked; /* Space to unpack a record */ SorterList list; /* List for thread to write to a PMA */ - int nPMA; /* Number of PMAs currently in file */ SorterCompare xCompare; /* Compare function to use */ SorterFile file; /* Temp file for level-0 PMAs */ SorterFile file2; /* Space for other PMAs */ @@ -84427,7 +103446,7 @@ struct SortSubtask { /* -** Main sorter structure. A single instance of this is allocated for each +** Main sorter structure. A single instance of this is allocated for each ** sorter cursor created by the VDBE. ** ** mxKeysize: @@ -84464,7 +103483,7 @@ struct VdbeSorter { ** PMA, in sorted order. The next key to be read is cached in nKey/aKey. ** aKey might point into aMap or into aBuffer. If neither of those locations ** contain a contiguous representation of the key, then aAlloc is allocated -** and the key is copied into aAlloc and aKey is made to poitn to aAlloc. +** and the key is copied into aAlloc and aKey is made to point to aAlloc. ** ** pFd==0 at EOF. */ @@ -84483,21 +103502,21 @@ struct PmaReader { }; /* -** Normally, a PmaReader object iterates through an existing PMA stored +** Normally, a PmaReader object iterates through an existing PMA stored ** within a temp file. However, if the PmaReader.pIncr variable points to ** an object of the following type, it may be used to iterate/merge through ** multiple PMAs simultaneously. ** -** There are two types of IncrMerger object - single (bUseThread==0) and -** multi-threaded (bUseThread==1). +** There are two types of IncrMerger object - single (bUseThread==0) and +** multi-threaded (bUseThread==1). ** -** A multi-threaded IncrMerger object uses two temporary files - aFile[0] -** and aFile[1]. Neither file is allowed to grow to more than mxSz bytes in -** size. When the IncrMerger is initialized, it reads enough data from -** pMerger to populate aFile[0]. It then sets variables within the -** corresponding PmaReader object to read from that file and kicks off -** a background thread to populate aFile[1] with the next mxSz bytes of -** sorted record data from pMerger. +** A multi-threaded IncrMerger object uses two temporary files - aFile[0] +** and aFile[1]. Neither file is allowed to grow to more than mxSz bytes in +** size. When the IncrMerger is initialized, it reads enough data from +** pMerger to populate aFile[0]. It then sets variables within the +** corresponding PmaReader object to read from that file and kicks off +** a background thread to populate aFile[1] with the next mxSz bytes of +** sorted record data from pMerger. ** ** When the PmaReader reaches the end of aFile[0], it blocks until the ** background thread has finished populating aFile[1]. It then exchanges @@ -84508,7 +103527,7 @@ struct PmaReader { ** ** A single-threaded IncrMerger does not open any temporary files of its ** own. Instead, it has exclusive access to mxSz bytes of space beginning -** at offset iStartOff of file pTask->file2. And instead of using a +** at offset iStartOff of file pTask->file2. And instead of using a ** background thread to prepare data for the PmaReader, with a single ** threaded IncrMerger the allocate part of pTask->file2 is "refilled" with ** keys from pMerger by the calling thread whenever the PmaReader runs out @@ -84620,7 +103639,7 @@ static int vdbePmaReadBlob( assert( p->aBuffer ); - /* If there is no more data to be read from the buffer, read the next + /* If there is no more data to be read from the buffer, read the next ** p->nBuffer bytes of data from the file into it. Or, if there are less ** than p->nBuffer bytes remaining in the PMA, read all remaining data. */ iBuf = p->iReadOff % p->nBuffer; @@ -84641,11 +103660,11 @@ static int vdbePmaReadBlob( assert( rc!=SQLITE_IOERR_SHORT_READ ); if( rc!=SQLITE_OK ) return rc; } - nAvail = p->nBuffer - iBuf; + nAvail = p->nBuffer - iBuf; if( nByte<=nAvail ){ /* The requested data is available in the in-memory buffer. In this - ** case there is no need to make a copy of the data, just return a + ** case there is no need to make a copy of the data, just return a ** pointer into the buffer to the caller. */ *ppOut = &p->aBuffer[iBuf]; p->iReadOff += nByte; @@ -84658,7 +103677,7 @@ static int vdbePmaReadBlob( /* Extend the p->aAlloc[] allocation if required. */ if( p->nAllocnAlloc*2); + sqlite3_int64 nNew = MAX(128, 2*(sqlite3_int64)p->nAlloc); while( nByte>nNew ) nNew = nNew*2; aNew = sqlite3Realloc(p->aAlloc, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; @@ -84677,13 +103696,14 @@ static int vdbePmaReadBlob( while( nRem>0 ){ int rc; /* vdbePmaReadBlob() return code */ int nCopy; /* Number of bytes to copy */ - u8 *aNext; /* Pointer to buffer to copy data from */ + u8 *aNext = 0; /* Pointer to buffer to copy data from */ nCopy = nRem; if( nRem>p->nBuffer ) nCopy = p->nBuffer; rc = vdbePmaReadBlob(p, nCopy, &aNext); if( rc!=SQLITE_OK ) return rc; assert( aNext!=p->aAlloc ); + assert( aNext!=0 ); memcpy(&p->aAlloc[nByte - nRem], aNext, nCopy); nRem -= nCopy; } @@ -84724,7 +103744,7 @@ static int vdbePmaReadVarint(PmaReader *p, u64 *pnOut){ /* ** Attempt to memory map file pFile. If successful, set *pp to point to the -** new mapping and return SQLITE_OK. If the mapping is not attempted +** new mapping and return SQLITE_OK. If the mapping is not attempted ** (because the file is too large or the VFS layer is configured not to use ** mmap), return SQLITE_OK and set *pp to NULL. ** @@ -84745,7 +103765,7 @@ static int vdbeSorterMapFile(SortSubtask *pTask, SorterFile *pFile, u8 **pp){ /* ** Attach PmaReader pReadr to file pFile (if it is not already attached to -** that file) and seek it to offset iOff within the file. Return SQLITE_OK +** that file) and seek it to offset iOff within the file. Return SQLITE_OK ** if successful, or an SQLite error code if an error occurs. */ static int vdbePmaReaderSeek( @@ -84835,11 +103855,11 @@ static int vdbePmaReaderNext(PmaReader *pReadr){ /* ** Initialize PmaReader pReadr to scan through the PMA stored in file pFile -** starting at offset iStart and ending at offset iEof-1. This function -** leaves the PmaReader pointing to the first key in the PMA (or EOF if the +** starting at offset iStart and ending at offset iEof-1. This function +** leaves the PmaReader pointing to the first key in the PMA (or EOF if the ** PMA is empty). ** -** If the pnByte parameter is NULL, then it is assumed that the file +** If the pnByte parameter is NULL, then it is assumed that the file ** contains a single PMA, and that that PMA omits the initial length varint. */ static int vdbePmaReaderInit( @@ -84872,7 +103892,7 @@ static int vdbePmaReaderInit( /* ** A version of vdbeSorterCompare() that assumes that it has already been -** determined that the first field of key1 is equal to the first field of +** determined that the first field of key1 is equal to the first field of ** key2. */ static int vdbeSorterCompareTail( @@ -84890,7 +103910,7 @@ static int vdbeSorterCompareTail( } /* -** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, +** Compare key1 (buffer pKey1, size nKey1 bytes) with key2 (buffer pKey2, ** size nKey2 bytes). Use (pTask->pKeyInfo) for the collation sequences ** used by the comparison. Return the result of the comparison. ** @@ -84936,21 +103956,22 @@ static int vdbeSorterCompareText( int n2; int res; - getVarint32(&p1[1], n1); n1 = (n1 - 13) / 2; - getVarint32(&p2[1], n2); n2 = (n2 - 13) / 2; - res = memcmp(v1, v2, MIN(n1, n2)); + getVarint32NR(&p1[1], n1); + getVarint32NR(&p2[1], n2); + res = memcmp(v1, v2, (MIN(n1, n2) - 13)/2); if( res==0 ){ res = n1 - n2; } if( res==0 ){ - if( pTask->pSorter->pKeyInfo->nField>1 ){ + if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } }else{ - if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); + if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ res = res * -1; } } @@ -84979,47 +104000,47 @@ static int vdbeSorterCompareInt( assert( (s1>0 && s1<7) || s1==8 || s1==9 ); assert( (s2>0 && s2<7) || s2==8 || s2==9 ); - if( s1>7 && s2>7 ){ + if( s1==s2 ){ + /* The two values have the same sign. Compare using memcmp(). */ + static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8, 0, 0, 0 }; + const u8 n = aLen[s1]; + int i; + res = 0; + for(i=0; i7 && s2>7 ){ res = s1 - s2; }else{ - if( s1==s2 ){ - if( (*v1 ^ *v2) & 0x80 ){ - /* The two values have different signs */ - res = (*v1 & 0x80) ? -1 : +1; - }else{ - /* The two values have the same sign. Compare using memcmp(). */ - static const u8 aLen[] = {0, 1, 2, 3, 4, 6, 8 }; - int i; - res = 0; - for(i=0; i7 ){ + res = +1; + }else if( s1>7 ){ + res = -1; }else{ - if( s2>7 ){ - res = +1; - }else if( s1>7 ){ - res = -1; - }else{ - res = s1 - s2; - } - assert( res!=0 ); + res = s1 - s2; + } + assert( res!=0 ); - if( res>0 ){ - if( *v1 & 0x80 ) res = -1; - }else{ - if( *v2 & 0x80 ) res = +1; - } + if( res>0 ){ + if( *v1 & 0x80 ) res = -1; + }else{ + if( *v2 & 0x80 ) res = +1; } } if( res==0 ){ - if( pTask->pSorter->pKeyInfo->nField>1 ){ + if( pTask->pSorter->pKeyInfo->nKeyField>1 ){ res = vdbeSorterCompareTail( pTask, pbKey2Cached, pKey1, nKey1, pKey2, nKey2 ); } - }else if( pTask->pSorter->pKeyInfo->aSortOrder[0] ){ + }else if( pTask->pSorter->pKeyInfo->aSortFlags[0] ){ + assert( !(pTask->pSorter->pKeyInfo->aSortFlags[0]&KEYINFO_ORDER_BIGNULL) ); res = res * -1; } @@ -85029,13 +104050,13 @@ static int vdbeSorterCompareInt( /* ** Initialize the temporary index cursor just opened as a sorter cursor. ** -** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nField) +** Usually, the sorter module uses the value of (pCsr->pKeyInfo->nKeyField) ** to determine the number of fields that should be compared from the ** records being sorted. However, if the value passed as argument nField ** is non-zero and the sorter is able to guarantee a stable sort, nField ** is used instead. This is used when sorting records for a CREATE INDEX ** statement. In this case, keys are always delivered to the sorter in -** order of the primary key, which happens to be make up the final part +** order of the primary key, which happens to be make up the final part ** of the records being sorted. So if the sort is stable, there is never ** any reason to compare PK fields and they can be ignored for a small ** performance boost. @@ -85080,9 +104101,10 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } #endif - assert( pCsr->pKeyInfo && pCsr->pBt==0 ); + assert( pCsr->pKeyInfo ); + assert( !pCsr->isEphemeral ); assert( pCsr->eCurType==CURTYPE_SORTER ); - szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nField-1)*sizeof(CollSeq*); + szKeyInfo = sizeof(KeyInfo) + (pCsr->pKeyInfo->nKeyField-1)*sizeof(CollSeq*); sz = sizeof(VdbeSorter) + nWorker * sizeof(SortSubtask); pSorter = (VdbeSorter*)sqlite3DbMallocZero(db, sz + szKeyInfo); @@ -85090,14 +104112,16 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( if( pSorter==0 ){ rc = SQLITE_NOMEM_BKPT; }else{ + Btree *pBt = db->aDb[0].pBt; pSorter->pKeyInfo = pKeyInfo = (KeyInfo*)((u8*)pSorter + sz); memcpy(pKeyInfo, pCsr->pKeyInfo, szKeyInfo); pKeyInfo->db = 0; if( nField && nWorker==0 ){ - pKeyInfo->nXField += (pKeyInfo->nField - nField); - pKeyInfo->nField = nField; + pKeyInfo->nKeyField = nField; } - pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(db->aDb[0].pBt); + sqlite3BtreeEnter(pBt); + pSorter->pgsz = pgsz = sqlite3BtreeGetPageSize(pBt); + sqlite3BtreeLeave(pBt); pSorter->nTask = nWorker + 1; pSorter->iPrev = (u8)(nWorker - 1); pSorter->bUseThreads = (pSorter->nTask>1); @@ -85123,11 +104147,9 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( mxCache = MIN(mxCache, SQLITE_MAX_PMASZ); pSorter->mxPmaSize = MAX(pSorter->mnPmaSize, (int)mxCache); - /* EVIDENCE-OF: R-26747-61719 When the application provides any amount of - ** scratch memory using SQLITE_CONFIG_SCRATCH, SQLite avoids unnecessary - ** large heap allocations. - */ - if( sqlite3GlobalConfig.pScratch==0 ){ + /* Avoid large memory allocations if the application has requested + ** SQLITE_CONFIG_SMALL_MALLOC. */ + if( sqlite3GlobalConfig.bSmallMalloc==0 ){ assert( pSorter->iMemory==0 ); pSorter->nMemory = pgsz; pSorter->list.aMemory = (u8*)sqlite3Malloc(pgsz); @@ -85135,8 +104157,9 @@ SQLITE_PRIVATE int sqlite3VdbeSorterInit( } } - if( (pKeyInfo->nField+pKeyInfo->nXField)<13 + if( pKeyInfo->nAllField<13 && (pKeyInfo->aColl[0]==0 || pKeyInfo->aColl[0]==db->pDfltColl) + && (pKeyInfo->aSortFlags[0] & KEYINFO_ORDER_BIGNULL)==0 ){ pSorter->typeMask = SORTER_TYPE_INTEGER | SORTER_TYPE_TEXT; } @@ -85159,7 +104182,7 @@ static void vdbeSorterRecordFree(sqlite3 *db, SorterRecord *pRecord){ } /* -** Free all resources owned by the object indicated by argument pTask. All +** Free all resources owned by the object indicated by argument pTask. All ** fields of *pTask are zeroed before returning. */ static void vdbeSortSubtaskCleanup(sqlite3 *db, SortSubtask *pTask){ @@ -85192,8 +104215,9 @@ static void vdbeSorterWorkDebug(SortSubtask *pTask, const char *zEvent){ fprintf(stderr, "%lld:%d %s\n", t, iTask, zEvent); } static void vdbeSorterRewindDebug(const char *zEvent){ - i64 t; - sqlite3OsCurrentTimeInt64(sqlite3_vfs_find(0), &t); + i64 t = 0; + sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + if( ALWAYS(pVfs) ) sqlite3OsCurrentTimeInt64(pVfs, &t); fprintf(stderr, "%lld:X %s\n", t, zEvent); } static void vdbeSorterPopulateDebug( @@ -85258,7 +104282,7 @@ static int vdbeSorterCreateThread( } /* -** Join all outstanding threads launched by SorterWrite() to create +** Join all outstanding threads launched by SorterWrite() to create ** level-0 PMAs. */ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ @@ -85267,10 +104291,10 @@ static int vdbeSorterJoinAll(VdbeSorter *pSorter, int rcin){ /* This function is always called by the main user thread. ** - ** If this function is being called after SorterRewind() has been called, + ** If this function is being called after SorterRewind() has been called, ** it is possible that thread pSorter->aTask[pSorter->nTask-1].pThread ** is currently attempt to join one of the other threads. To avoid a race - ** condition where this thread also attempts to join the same object, join + ** condition where this thread also attempts to join the same object, join ** thread pSorter->aTask[pSorter->nTask-1].pThread first. */ for(i=pSorter->nTask-1; i>=0; i--){ SortSubtask *pTask = &pSorter->aTask[i]; @@ -85407,7 +104431,7 @@ static void vdbeSorterExtendFile(sqlite3 *db, sqlite3_file *pFd, i64 nByte){ sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_CHUNK_SIZE, &chunksize); sqlite3OsFileControlHint(pFd, SQLITE_FCNTL_SIZE_HINT, &nByte); sqlite3OsFetch(pFd, 0, (int)nByte, &p); - sqlite3OsUnfetch(pFd, 0, p); + if( p ) sqlite3OsUnfetch(pFd, 0, p); } } #else @@ -85442,19 +104466,15 @@ static int vdbeSorterOpenTempFile( } /* -** If it has not already been allocated, allocate the UnpackedRecord -** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or +** If it has not already been allocated, allocate the UnpackedRecord +** structure at pTask->pUnpacked. Return SQLITE_OK if successful (or ** if no allocation was required), or SQLITE_NOMEM otherwise. */ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ if( pTask->pUnpacked==0 ){ - char *pFree; - pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord( - pTask->pSorter->pKeyInfo, 0, 0, &pFree - ); - assert( pTask->pUnpacked==(UnpackedRecord*)pFree ); - if( pFree==0 ) return SQLITE_NOMEM_BKPT; - pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nField; + pTask->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pTask->pSorter->pKeyInfo); + if( pTask->pUnpacked==0 ) return SQLITE_NOMEM_BKPT; + pTask->pUnpacked->nField = pTask->pSorter->pKeyInfo->nKeyField; pTask->pUnpacked->errCode = 0; } return SQLITE_OK; @@ -85463,19 +104483,18 @@ static int vdbeSortAllocUnpacked(SortSubtask *pTask){ /* ** Merge the two sorted lists p1 and p2 into a single list. -** Set *ppOut to the head of the new list. */ -static void vdbeSorterMerge( +static SorterRecord *vdbeSorterMerge( SortSubtask *pTask, /* Calling thread context */ SorterRecord *p1, /* First list to merge */ - SorterRecord *p2, /* Second list to merge */ - SorterRecord **ppOut /* OUT: Head of merged list */ + SorterRecord *p2 /* Second list to merge */ ){ SorterRecord *pFinal = 0; SorterRecord **pp = &pFinal; int bCached = 0; - while( p1 && p2 ){ + assert( p1!=0 && p2!=0 ); + for(;;){ int res; res = pTask->xCompare( pTask, &bCached, SRVAL(p1), p1->nVal, SRVAL(p2), p2->nVal @@ -85485,15 +104504,22 @@ static void vdbeSorterMerge( *pp = p1; pp = &p1->u.pNext; p1 = p1->u.pNext; + if( p1==0 ){ + *pp = p2; + break; + } }else{ *pp = p2; pp = &p2->u.pNext; p2 = p2->u.pNext; bCached = 0; + if( p2==0 ){ + *pp = p1; + break; + } } } - *pp = p1 ? p1 : p2; - *ppOut = pFinal; + return pFinal; } /* @@ -85504,32 +104530,28 @@ static SorterCompare vdbeSorterGetCompare(VdbeSorter *p){ if( p->typeMask==SORTER_TYPE_INTEGER ){ return vdbeSorterCompareInt; }else if( p->typeMask==SORTER_TYPE_TEXT ){ - return vdbeSorterCompareText; + return vdbeSorterCompareText; } return vdbeSorterCompare; } /* -** Sort the linked list of records headed at pTask->pList. Return -** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if +** Sort the linked list of records headed at pTask->pList. Return +** SQLITE_OK if successful, or an SQLite error code (i.e. SQLITE_NOMEM) if ** an error occurs. */ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ int i; - SorterRecord **aSlot; SorterRecord *p; int rc; + SorterRecord *aSlot[64]; rc = vdbeSortAllocUnpacked(pTask); if( rc!=SQLITE_OK ) return rc; p = pList->pList; pTask->xCompare = vdbeSorterGetCompare(pTask->pSorter); - - aSlot = (SorterRecord **)sqlite3MallocZero(64 * sizeof(SorterRecord *)); - if( !aSlot ){ - return SQLITE_NOMEM_BKPT; - } + memset(aSlot, 0, sizeof(aSlot)); while( p ){ SorterRecord *pNext; @@ -85546,7 +104568,7 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ p->u.pNext = 0; for(i=0; aSlot[i]; i++){ - vdbeSorterMerge(pTask, p, aSlot[i], &p); + p = vdbeSorterMerge(pTask, p, aSlot[i]); aSlot[i] = 0; } aSlot[i] = p; @@ -85554,14 +104576,14 @@ static int vdbeSorterSort(SortSubtask *pTask, SorterList *pList){ } p = 0; - for(i=0; i<64; i++){ - vdbeSorterMerge(pTask, p, aSlot[i], &p); + for(i=0; ipList = p; - sqlite3_free(aSlot); - assert( pTask->pUnpacked->errCode==SQLITE_OK - || pTask->pUnpacked->errCode==SQLITE_NOMEM + assert( pTask->pUnpacked->errCode==SQLITE_OK + || pTask->pUnpacked->errCode==SQLITE_NOMEM ); return pTask->pUnpacked->errCode; } @@ -85602,8 +104624,8 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ memcpy(&p->aBuffer[p->iBufEnd], &pData[nData-nRem], nCopy); p->iBufEnd += nCopy; if( p->iBufEnd==p->nBuffer ){ - p->eFWErr = sqlite3OsWrite(p->pFd, - &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->eFWErr = sqlite3OsWrite(p->pFd, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); p->iBufStart = p->iBufEnd = 0; @@ -85618,7 +104640,7 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ /* ** Flush any buffered data to disk and clean up the PMA-writer object. ** The results of using the PMA-writer after this call are undefined. -** Return SQLITE_OK if flushing the buffered data succeeds or is not +** Return SQLITE_OK if flushing the buffered data succeeds or is not ** required. Otherwise, return an SQLite error code. ** ** Before returning, set *piEof to the offset immediately following the @@ -85627,8 +104649,8 @@ static void vdbePmaWriteBlob(PmaWriter *p, u8 *pData, int nData){ static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ int rc; if( p->eFWErr==0 && ALWAYS(p->aBuffer) && p->iBufEnd>p->iBufStart ){ - p->eFWErr = sqlite3OsWrite(p->pFd, - &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, + p->eFWErr = sqlite3OsWrite(p->pFd, + &p->aBuffer[p->iBufStart], p->iBufEnd - p->iBufStart, p->iWriteOff + p->iBufStart ); } @@ -85640,11 +104662,11 @@ static int vdbePmaWriterFinish(PmaWriter *p, i64 *piEof){ } /* -** Write value iVal encoded as a varint to the PMA. Return +** Write value iVal encoded as a varint to the PMA. Return ** SQLITE_OK if successful, or an SQLite error code if an error occurs. */ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ - int nByte; + int nByte; u8 aByte[10]; nByte = sqlite3PutVarint(aByte, iVal); vdbePmaWriteBlob(p, aByte, nByte); @@ -85652,7 +104674,7 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ /* ** Write the current contents of in-memory linked-list pList to a level-0 -** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if +** PMA in the temp file belonging to sub-task pTask. Return SQLITE_OK if ** successful, or an SQLite error code otherwise. ** ** The format of a PMA is: @@ -85660,8 +104682,8 @@ static void vdbePmaWriteVarint(PmaWriter *p, u64 iVal){ ** * A varint. This varint contains the total number of bytes of content ** in the PMA (not including the varint itself). ** -** * One or more records packed end-to-end in order of ascending keys. -** Each record consists of a varint followed by a blob of data (the +** * One or more records packed end-to-end in order of ascending keys. +** Each record consists of a varint followed by a blob of data (the ** key). The varint is the number of bytes in the blob of data. */ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ @@ -85670,7 +104692,7 @@ static int vdbeSorterListToPMA(SortSubtask *pTask, SorterList *pList){ PmaWriter writer; /* Object used to write to the file */ #ifdef SQLITE_DEBUG - /* Set iSz to the expected size of file pTask->file after writing the PMA. + /* Set iSz to the expected size of file pTask->file after writing the PMA. ** This is used by an assert() statement at the end of this function. */ i64 iSz = pList->szPMA + sqlite3VarintLen(pList->szPMA) + pTask->file.iEof; #endif @@ -85823,7 +104845,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ SortSubtask *pTask = 0; /* Thread context used to create new PMA */ int nWorker = (pSorter->nTask-1); - /* Set the flag to indicate that at least one PMA has been written. + /* Set the flag to indicate that at least one PMA has been written. ** Or will be, anyhow. */ pSorter->bUsePMA = 1; @@ -85833,7 +104855,7 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ ** the background thread from a sub-tasks previous turn is still running, ** skip it. If the first (pSorter->nTask-1) sub-tasks are all still busy, ** fall back to using the final sub-task. The first (pSorter->nTask-1) - ** sub-tasks are prefered as they use background threads - the final + ** sub-tasks are preferred as they use background threads - the final ** sub-task uses the main thread. */ for(i=0; iiPrev + i + 1) % nWorker; @@ -85850,13 +104872,16 @@ static int vdbeSorterFlushPMA(VdbeSorter *pSorter){ rc = vdbeSorterListToPMA(&pSorter->aTask[nWorker], &pSorter->list); }else{ /* Launch a background thread for this operation */ - u8 *aMem = pTask->list.aMemory; - void *pCtx = (void*)pTask; + u8 *aMem; + void *pCtx; + assert( pTask!=0 ); assert( pTask->pThread==0 && pTask->bDone==0 ); assert( pTask->list.pList==0 ); assert( pTask->list.aMemory==0 || pSorter->list.aMemory!=0 ); + aMem = pTask->list.aMemory; + pCtx = (void*)pTask; pSorter->iPrev = (u8)(pTask - pSorter->aTask); pTask->list = pSorter->list; pSorter->list.pList = 0; @@ -85888,13 +104913,13 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( int rc = SQLITE_OK; /* Return Code */ SorterRecord *pNew; /* New list element */ int bFlush; /* True to flush contents of memory to PMA */ - int nReq; /* Bytes of memory required */ - int nPMA; /* Bytes of PMA space required */ + i64 nReq; /* Bytes of memory required */ + i64 nPMA; /* Bytes of PMA space required */ int t; /* serial type of first record field */ assert( pCsr->eCurType==CURTYPE_SORTER ); pSorter = pCsr->uc.pSorter; - getVarint32((const u8*)&pVal->z[1], t); + getVarint32NR((const u8*)&pVal->z[1], t); if( t>0 && t<10 && t!=7 ){ pSorter->typeMask &= SORTER_TYPE_INTEGER; }else if( t>10 && (t & 0x01) ){ @@ -85911,14 +104936,14 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( ** If using the single large allocation mode (pSorter->aMemory!=0), then ** flush the contents of memory to a new PMA if (a) at least one value is ** already in memory and (b) the new value will not fit in memory. - ** + ** ** Or, if using separate allocations for each record, flush the contents ** of memory to a PMA if either of the following are true: ** - ** * The total memory allocated for the in-memory list is greater + ** * The total memory allocated for the in-memory list is greater ** than (page-size * cache-size), or ** - ** * The total memory allocated for the in-memory list is greater + ** * The total memory allocated for the in-memory list is greater ** than (page-size * 10) and sqlite3HeapNearlyFull() returns true. */ nReq = pVal->n + sizeof(SorterRecord); @@ -85950,15 +104975,19 @@ SQLITE_PRIVATE int sqlite3VdbeSorterWrite( if( nMin>pSorter->nMemory ){ u8 *aNew; - int iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; - int nNew = pSorter->nMemory * 2; + sqlite3_int64 nNew = 2 * (sqlite3_int64)pSorter->nMemory; + int iListOff = -1; + if( pSorter->list.pList ){ + iListOff = (u8*)pSorter->list.pList - pSorter->list.aMemory; + } while( nNew < nMin ) nNew = nNew*2; if( nNew > pSorter->mxPmaSize ) nNew = pSorter->mxPmaSize; if( nNew < nMin ) nNew = nMin; - aNew = sqlite3Realloc(pSorter->list.aMemory, nNew); if( !aNew ) return SQLITE_NOMEM_BKPT; - pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + if( iListOff>=0 ){ + pSorter->list.pList = (SorterRecord*)&aNew[iListOff]; + } pSorter->list.aMemory = aNew; pSorter->nMemory = nNew; } @@ -86053,11 +105082,11 @@ static int vdbeIncrBgPopulate(IncrMerger *pIncr){ ** aFile[0] such that the PmaReader should start rereading it from the ** beginning. ** -** For single-threaded objects, this is accomplished by literally reading -** keys from pIncr->pMerger and repopulating aFile[0]. +** For single-threaded objects, this is accomplished by literally reading +** keys from pIncr->pMerger and repopulating aFile[0]. ** -** For multi-threaded objects, all that is required is to wait until the -** background thread is finished (if it is not already) and then swap +** For multi-threaded objects, all that is required is to wait until the +** background thread is finished (if it is not already) and then swap ** aFile[0] and aFile[1] in place. If the contents of pMerger have not ** been exhausted, this function also launches a new background thread ** to populate the new aFile[1]. @@ -86120,6 +105149,7 @@ static int vdbeIncrMergerNew( vdbeMergeEngineFree(pMerger); rc = SQLITE_NOMEM_BKPT; } + assert( *ppOut!=0 || rc!=SQLITE_OK ); return rc; } @@ -86197,7 +105227,7 @@ static void vdbeMergeEngineCompare( #define INCRINIT_TASK 1 #define INCRINIT_ROOT 2 -/* +/* ** Forward reference required as the vdbeIncrMergeInit() and ** vdbePmaReaderIncrInit() routines are called mutually recursively when ** building a merge tree. @@ -86206,7 +105236,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); /* ** Initialize the MergeEngine object passed as the second argument. Once this -** function returns, the first key of merged data may be read from the +** function returns, the first key of merged data may be read from the ** MergeEngine object in the usual fashion. ** ** If argument eMode is INCRINIT_ROOT, then it is assumed that any IncrMerge @@ -86216,8 +105246,8 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode); ** required is to call vdbePmaReaderNext() on each PmaReader to point it at ** its first key. ** -** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use -** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data +** Otherwise, if eMode is any value other than INCRINIT_ROOT, then use +** vdbePmaReaderIncrMergeInit() to initialize each PmaReader that feeds data ** to pMerger. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. @@ -86229,7 +105259,11 @@ static int vdbeMergeEngineInit( ){ int rc = SQLITE_OK; /* Return code */ int i; /* For looping over PmaReader objects */ - int nTree = pMerger->nTree; + int nTree; /* Number of subtrees to merge */ + + /* Failure to allocate the merge would have been detected prior to + ** invoking this routine */ + assert( pMerger!=0 ); /* eMode is always INCRINIT_NORMAL in single-threaded mode */ assert( SQLITE_MAX_WORKER_THREADS>0 || eMode==INCRINIT_NORMAL ); @@ -86238,6 +105272,7 @@ static int vdbeMergeEngineInit( assert( pMerger->pTask==0 ); pMerger->pTask = pTask; + nTree = pMerger->nTree; for(i=0; i0 && eMode==INCRINIT_ROOT ){ /* PmaReaders should be normally initialized in order, as if they are @@ -86267,19 +105302,19 @@ static int vdbeMergeEngineInit( ** object at (pReadr->pIncr). ** ** If argument eMode is set to INCRINIT_NORMAL, then all PmaReaders -** in the sub-tree headed by pReadr are also initialized. Data is then -** loaded into the buffers belonging to pReadr and it is set to point to +** in the sub-tree headed by pReadr are also initialized. Data is then +** loaded into the buffers belonging to pReadr and it is set to point to ** the first key in its range. ** ** If argument eMode is set to INCRINIT_TASK, then pReadr is guaranteed ** to be a multi-threaded PmaReader and this function is being called in a -** background thread. In this case all PmaReaders in the sub-tree are +** background thread. In this case all PmaReaders in the sub-tree are ** initialized as for INCRINIT_NORMAL and the aFile[1] buffer belonging to ** pReadr is populated. However, pReadr itself is not set up to point ** to its first key. A call to vdbePmaReaderNext() is still required to do -** that. +** that. ** -** The reason this function does not call vdbePmaReaderNext() immediately +** The reason this function does not call vdbePmaReaderNext() immediately ** in the INCRINIT_TASK case is that vdbePmaReaderNext() assumes that it has ** to block on thread (pTask->thread) before accessing aFile[1]. But, since ** this entire function is being run by thread (pTask->thread), that will @@ -86304,7 +105339,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ rc = vdbeMergeEngineInit(pTask, pIncr->pMerger, eMode); - /* Set up the required files for pIncr. A multi-theaded IncrMerge object + /* Set up the required files for pIncr. A multi-threaded IncrMerge object ** requires two temp files to itself, whereas a single-threaded object ** only requires a region of pTask->file2. */ if( rc==SQLITE_OK ){ @@ -86335,12 +105370,12 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ if( rc==SQLITE_OK && pIncr->bUseThread ){ /* Use the current thread to populate aFile[1], even though this ** PmaReader is multi-threaded. If this is an INCRINIT_TASK object, - ** then this function is already running in background thread - ** pIncr->pTask->thread. + ** then this function is already running in background thread + ** pIncr->pTask->thread. ** - ** If this is the INCRINIT_ROOT object, then it is running in the + ** If this is the INCRINIT_ROOT object, then it is running in the ** main VDBE thread. But that is Ok, as that thread cannot return - ** control to the VDBE or proceed with anything useful until the + ** control to the VDBE or proceed with anything useful until the ** first results are ready from this merger object anyway. */ assert( eMode==INCRINIT_ROOT || eMode==INCRINIT_TASK ); @@ -86357,7 +105392,7 @@ static int vdbePmaReaderIncrMergeInit(PmaReader *pReadr, int eMode){ #if SQLITE_MAX_WORKER_THREADS>0 /* -** The main routine for vdbePmaReaderIncrMergeInit() operations run in +** The main routine for vdbePmaReaderIncrMergeInit() operations run in ** background threads. */ static void *vdbePmaReaderBgIncrInit(void *pCtx){ @@ -86375,8 +105410,8 @@ static void *vdbePmaReaderBgIncrInit(void *pCtx){ ** (if pReadr->pIncr==0), then this function is a no-op. Otherwise, it invokes ** the vdbePmaReaderIncrMergeInit() function with the parameters passed to ** this routine to initialize the incremental merge. -** -** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), +** +** If the IncrMerger object is multi-threaded (IncrMerger.bUseThread==1), ** then a background thread is launched to call vdbePmaReaderIncrMergeInit(). ** Or, if the IncrMerger is single threaded, the same function is called ** using the current thread. @@ -86406,7 +105441,7 @@ static int vdbePmaReaderIncrInit(PmaReader *pReadr, int eMode){ ** to NULL and return an SQLite error code. ** ** When this function is called, *piOffset is set to the offset of the -** first PMA to read from pTask->file. Assuming no error occurs, it is +** first PMA to read from pTask->file. Assuming no error occurs, it is ** set to the offset immediately following the last byte of the last ** PMA before returning. If an error does occur, then the final value of ** *piOffset is undefined. @@ -86516,12 +105551,12 @@ static int vdbeSorterAddToTree( /* ** This function is called as part of a SorterRewind() operation on a sorter ** that has already written two or more level-0 PMAs to one or more temp -** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that +** files. It builds a tree of MergeEngine/IncrMerger/PmaReader objects that ** can be used to incrementally merge all PMAs on disk. ** ** If successful, SQLITE_OK is returned and *ppOut set to point to the ** MergeEngine object at the root of the tree before returning. Or, if an -** error occurs, an SQLite error code is returned and the final value +** error occurs, an SQLite error code is returned and the final value ** of *ppOut is undefined. */ static int vdbeSorterMergeTreeBuild( @@ -86533,8 +105568,8 @@ static int vdbeSorterMergeTreeBuild( int iTask; #if SQLITE_MAX_WORKER_THREADS>0 - /* If the sorter uses more than one task, then create the top-level - ** MergeEngine here. This MergeEngine will read data from exactly + /* If the sorter uses more than one task, then create the top-level + ** MergeEngine here. This MergeEngine will read data from exactly ** one PmaReader per sub-task. */ assert( pSorter->bUseThreads || pSorter->nTask==1 ); if( pSorter->nTask>1 ){ @@ -86643,7 +105678,7 @@ static int vdbeSorterSetupMerge(VdbeSorter *pSorter){ } for(iTask=0; rc==SQLITE_OK && iTasknTask; iTask++){ /* Check that: - ** + ** ** a) The incremental merge object is configured to use the ** right task, and ** b) If it is using task (nTask-1), it is configured to run @@ -86706,7 +105741,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ return rc; } - /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() + /* Write the current in-memory list to a PMA. When the VdbeSorterWrite() ** function flushes the contents of memory to disk, it immediately always ** creates a new list consisting of a single key immediately afterwards. ** So the list is never empty at this point. */ @@ -86718,7 +105753,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ vdbeSorterRewindDebug("rewind"); - /* Assuming no errors have occurred, set up a merger structure to + /* Assuming no errors have occurred, set up a merger structure to ** incrementally read and merge all remaining PMAs. */ assert( pSorter->pReader==0 ); if( rc==SQLITE_OK ){ @@ -86731,9 +105766,13 @@ SQLITE_PRIVATE int sqlite3VdbeSorterRewind(const VdbeCursor *pCsr, int *pbEof){ } /* -** Advance to the next element in the sorter. +** Advance to the next element in the sorter. Return value: +** +** SQLITE_OK success +** SQLITE_DONE end of data +** otherwise some kind of error. */ -SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, int *pbEof){ +SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr){ VdbeSorter *pSorter; int rc; /* Return code */ @@ -86747,27 +105786,28 @@ SQLITE_PRIVATE int sqlite3VdbeSorterNext(sqlite3 *db, const VdbeCursor *pCsr, in #if SQLITE_MAX_WORKER_THREADS>0 if( pSorter->bUseThreads ){ rc = vdbePmaReaderNext(pSorter->pReader); - *pbEof = (pSorter->pReader->pFd==0); + if( rc==SQLITE_OK && pSorter->pReader->pFd==0 ) rc = SQLITE_DONE; }else #endif /*if( !pSorter->bUseThreads )*/ { + int res = 0; assert( pSorter->pMerger!=0 ); assert( pSorter->pMerger->pTask==(&pSorter->aTask[0]) ); - rc = vdbeMergeEngineStep(pSorter->pMerger, pbEof); + rc = vdbeMergeEngineStep(pSorter->pMerger, &res); + if( rc==SQLITE_OK && res ) rc = SQLITE_DONE; } }else{ SorterRecord *pFree = pSorter->list.pList; pSorter->list.pList = pFree->u.pNext; pFree->u.pNext = 0; if( pSorter->list.aMemory==0 ) vdbeSorterRecordFree(db, pFree); - *pbEof = !pSorter->list.pList; - rc = SQLITE_OK; + rc = pSorter->list.pList ? SQLITE_OK : SQLITE_DONE; } return rc; } /* -** Return a pointer to a buffer owned by the sorter that contains the +** Return a pointer to a buffer owned by the sorter that contains the ** current key. */ static void *vdbeSorterRowkey( @@ -86847,9 +105887,7 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( r2 = pSorter->pUnpacked; pKeyInfo = pCsr->pKeyInfo; if( r2==0 ){ - char *p; - r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo,0,0,&p); - assert( pSorter->pUnpacked==(UnpackedRecord*)p ); + r2 = pSorter->pUnpacked = sqlite3VdbeAllocUnpackedRecord(pKeyInfo); if( r2==0 ) return SQLITE_NOMEM_BKPT; r2->nField = nKeyCol; } @@ -86869,6 +105907,455 @@ SQLITE_PRIVATE int sqlite3VdbeSorterCompare( } /************** End of vdbesort.c ********************************************/ +/************** Begin file vdbevtab.c ****************************************/ +/* +** 2020-03-23 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file implements virtual-tables for examining the bytecode content +** of a prepared statement. +*/ +/* #include "sqliteInt.h" */ +#if defined(SQLITE_ENABLE_BYTECODE_VTAB) && !defined(SQLITE_OMIT_VIRTUALTABLE) +/* #include "vdbeInt.h" */ + +/* An instance of the bytecode() table-valued function. +*/ +typedef struct bytecodevtab bytecodevtab; +struct bytecodevtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ + int bTablesUsed; /* 2 for tables_used(). 0 for bytecode(). */ +}; + +/* A cursor for scanning through the bytecode +*/ +typedef struct bytecodevtab_cursor bytecodevtab_cursor; +struct bytecodevtab_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3_stmt *pStmt; /* The statement whose bytecode is displayed */ + int iRowid; /* The rowid of the output table */ + int iAddr; /* Address */ + int needFinalize; /* Cursors owns pStmt and must finalize it */ + int showSubprograms; /* Provide a listing of subprograms */ + Op *aOp; /* Operand array */ + char *zP4; /* Rendered P4 value */ + const char *zType; /* tables_used.type */ + const char *zSchema; /* tables_used.schema */ + const char *zName; /* tables_used.name */ + Mem sub; /* Subprograms */ +}; + +/* +** Create a new bytecode() table-valued function. +*/ +static int bytecodevtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + bytecodevtab *pNew; + int rc; + int isTabUsed = pAux!=0; + const char *azSchema[2] = { + /* bytecode() schema */ + "CREATE TABLE x(" + "addr INT," + "opcode TEXT," + "p1 INT," + "p2 INT," + "p3 INT," + "p4 TEXT," + "p5 INT," + "comment TEXT," + "subprog TEXT," + "nexec INT," + "ncycle INT," + "stmt HIDDEN" + ");", + + /* Tables_used() schema */ + "CREATE TABLE x(" + "type TEXT," + "schema TEXT," + "name TEXT," + "wr INT," + "subprog TEXT," + "stmt HIDDEN" + ");" + }; + + (void)argc; + (void)argv; + (void)pzErr; + rc = sqlite3_declare_vtab(db, azSchema[isTabUsed]); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + pNew->bTablesUsed = isTabUsed*2; + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int bytecodevtabDisconnect(sqlite3_vtab *pVtab){ + bytecodevtab *p = (bytecodevtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int bytecodevtabOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + bytecodevtab *pVTab = (bytecodevtab*)p; + bytecodevtab_cursor *pCur; + pCur = sqlite3_malloc( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + sqlite3VdbeMemInit(&pCur->sub, pVTab->db, 1); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* +** Clear all internal content from a bytecodevtab cursor. +*/ +static void bytecodevtabCursorClear(bytecodevtab_cursor *pCur){ + sqlite3_free(pCur->zP4); + pCur->zP4 = 0; + sqlite3VdbeMemRelease(&pCur->sub); + sqlite3VdbeMemSetNull(&pCur->sub); + if( pCur->needFinalize ){ + sqlite3_finalize(pCur->pStmt); + } + pCur->pStmt = 0; + pCur->needFinalize = 0; + pCur->zType = 0; + pCur->zSchema = 0; + pCur->zName = 0; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int bytecodevtabClose(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtabCursorClear(pCur); + sqlite3_free(pCur); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int bytecodevtabNext(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtab *pTab = (bytecodevtab*)cur->pVtab; + int rc; + if( pCur->zP4 ){ + sqlite3_free(pCur->zP4); + pCur->zP4 = 0; + } + if( pCur->zName ){ + pCur->zName = 0; + pCur->zType = 0; + pCur->zSchema = 0; + } + rc = sqlite3VdbeNextOpcode( + (Vdbe*)pCur->pStmt, + pCur->showSubprograms ? &pCur->sub : 0, + pTab->bTablesUsed, + &pCur->iRowid, + &pCur->iAddr, + &pCur->aOp); + if( rc!=SQLITE_OK ){ + sqlite3VdbeMemSetNull(&pCur->sub); + pCur->aOp = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int bytecodevtabEof(sqlite3_vtab_cursor *cur){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + return pCur->aOp==0; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int bytecodevtabColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + bytecodevtab *pVTab = (bytecodevtab*)cur->pVtab; + Op *pOp = pCur->aOp + pCur->iAddr; + if( pVTab->bTablesUsed ){ + if( i==4 ){ + i = 8; + }else{ + if( i<=2 && pCur->zType==0 ){ + Schema *pSchema; + HashElem *k; + int iDb = pOp->p3; + Pgno iRoot = (Pgno)pOp->p2; + sqlite3 *db = pVTab->db; + pSchema = db->aDb[iDb].pSchema; + pCur->zSchema = db->aDb[iDb].zDbSName; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pTab = (Table*)sqliteHashData(k); + if( !IsVirtual(pTab) && pTab->tnum==iRoot ){ + pCur->zName = pTab->zName; + pCur->zType = "table"; + break; + } + } + if( pCur->zName==0 ){ + for(k=sqliteHashFirst(&pSchema->idxHash); k; k=sqliteHashNext(k)){ + Index *pIdx = (Index*)sqliteHashData(k); + if( pIdx->tnum==iRoot ){ + pCur->zName = pIdx->zName; + pCur->zType = "index"; + } + } + } + } + i += 20; + } + } + switch( i ){ + case 0: /* addr */ + sqlite3_result_int(ctx, pCur->iAddr); + break; + case 1: /* opcode */ + sqlite3_result_text(ctx, (char*)sqlite3OpcodeName(pOp->opcode), + -1, SQLITE_STATIC); + break; + case 2: /* p1 */ + sqlite3_result_int(ctx, pOp->p1); + break; + case 3: /* p2 */ + sqlite3_result_int(ctx, pOp->p2); + break; + case 4: /* p3 */ + sqlite3_result_int(ctx, pOp->p3); + break; + case 5: /* p4 */ + case 7: /* comment */ + if( pCur->zP4==0 ){ + pCur->zP4 = sqlite3VdbeDisplayP4(pVTab->db, pOp); + } + if( i==5 ){ + sqlite3_result_text(ctx, pCur->zP4, -1, SQLITE_STATIC); + }else{ +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + char *zCom = sqlite3VdbeDisplayComment(pVTab->db, pOp, pCur->zP4); + sqlite3_result_text(ctx, zCom, -1, sqlite3_free); +#endif + } + break; + case 6: /* p5 */ + sqlite3_result_int(ctx, pOp->p5); + break; + case 8: { /* subprog */ + Op *aOp = pCur->aOp; + assert( aOp[0].opcode==OP_Init ); + assert( aOp[0].p4.z==0 || strncmp(aOp[0].p4.z,"-" "- ",3)==0 ); + if( pCur->iRowid==pCur->iAddr+1 ){ + break; /* Result is NULL for the main program */ + }else if( aOp[0].p4.z!=0 ){ + sqlite3_result_text(ctx, aOp[0].p4.z+3, -1, SQLITE_STATIC); + }else{ + sqlite3_result_text(ctx, "(FK)", 4, SQLITE_STATIC); + } + break; + } + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + case 9: /* nexec */ + sqlite3_result_int64(ctx, pOp->nExec); + break; + case 10: /* ncycle */ + sqlite3_result_int64(ctx, pOp->nCycle); + break; +#else + case 9: /* nexec */ + case 10: /* ncycle */ + sqlite3_result_int(ctx, 0); + break; +#endif + + case 20: /* tables_used.type */ + sqlite3_result_text(ctx, pCur->zType, -1, SQLITE_STATIC); + break; + case 21: /* tables_used.schema */ + sqlite3_result_text(ctx, pCur->zSchema, -1, SQLITE_STATIC); + break; + case 22: /* tables_used.name */ + sqlite3_result_text(ctx, pCur->zName, -1, SQLITE_STATIC); + break; + case 23: /* tables_used.wr */ + sqlite3_result_int(ctx, pOp->opcode==OP_OpenWrite); + break; + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int bytecodevtabRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor*)cur; + *pRowid = pCur->iRowid; + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int bytecodevtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + bytecodevtab_cursor *pCur = (bytecodevtab_cursor *)pVtabCursor; + bytecodevtab *pVTab = (bytecodevtab *)pVtabCursor->pVtab; + int rc = SQLITE_OK; + (void)idxStr; + + bytecodevtabCursorClear(pCur); + pCur->iRowid = 0; + pCur->iAddr = 0; + pCur->showSubprograms = idxNum==0; + assert( argc==1 ); + if( sqlite3_value_type(argv[0])==SQLITE_TEXT ){ + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v2(pVTab->db, zSql, -1, &pCur->pStmt, 0); + pCur->needFinalize = 1; + } + }else{ + pCur->pStmt = (sqlite3_stmt*)sqlite3_value_pointer(argv[0],"stmt-pointer"); + } + if( pCur->pStmt==0 ){ + pVTab->base.zErrMsg = sqlite3_mprintf( + "argument to %s() is not a valid SQL statement", + pVTab->bTablesUsed ? "tables_used" : "bytecode" + ); + rc = SQLITE_ERROR; + }else{ + bytecodevtabNext(pVtabCursor); + } + return rc; +} + +/* +** We must have a single stmt=? constraint that will be passed through +** into the xFilter method. If there is no valid stmt=? constraint, +** then return an SQLITE_CONSTRAINT error. +*/ +static int bytecodevtabBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; + int rc = SQLITE_CONSTRAINT; + struct sqlite3_index_constraint *p; + bytecodevtab *pVTab = (bytecodevtab*)tab; + int iBaseCol = pVTab->bTablesUsed ? 4 : 10; + pIdxInfo->estimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==iBaseCol+1 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + } + if( p->op==SQLITE_INDEX_CONSTRAINT_ISNULL && p->iColumn==iBaseCol ){ + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 1; + } + } + return rc; +} + +/* +** This following structure defines all the methods for the +** virtual table. +*/ +static sqlite3_module bytecodevtabModule = { + /* iVersion */ 0, + /* xCreate */ 0, + /* xConnect */ bytecodevtabConnect, + /* xBestIndex */ bytecodevtabBestIndex, + /* xDisconnect */ bytecodevtabDisconnect, + /* xDestroy */ 0, + /* xOpen */ bytecodevtabOpen, + /* xClose */ bytecodevtabClose, + /* xFilter */ bytecodevtabFilter, + /* xNext */ bytecodevtabNext, + /* xEof */ bytecodevtabEof, + /* xColumn */ bytecodevtabColumn, + /* xRowid */ bytecodevtabRowid, + /* xUpdate */ 0, + /* xBegin */ 0, + /* xSync */ 0, + /* xCommit */ 0, + /* xRollback */ 0, + /* xFindMethod */ 0, + /* xRename */ 0, + /* xSavepoint */ 0, + /* xRelease */ 0, + /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 +}; + + +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3 *db){ + int rc; + rc = sqlite3_create_module(db, "bytecode", &bytecodevtabModule, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3_create_module(db, "tables_used", &bytecodevtabModule, &db); + } + return rc; +} +#elif defined(SQLITE_ENABLE_BYTECODE_VTAB) +SQLITE_PRIVATE int sqlite3VdbeBytecodeVtabInit(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_BYTECODE_VTAB */ + +/************** End of vdbevtab.c ********************************************/ /************** Begin file memjournal.c **************************************/ /* ** 2008 October 7 @@ -86942,7 +106429,6 @@ struct MemJournal { int nChunkSize; /* In-memory chunk-size */ int nSpill; /* Bytes of data before flushing */ - int nSize; /* Bytes of data currently in memory */ FileChunk *pFirst; /* Head of in-memory chunk-list */ FilePoint endpoint; /* Pointer to the end of the file */ FilePoint readpoint; /* Pointer to the end of the last xRead() */ @@ -86968,17 +106454,13 @@ static int memjrnlRead( int iChunkOffset; FileChunk *pChunk; -#ifdef SQLITE_ENABLE_ATOMIC_WRITE if( (iAmt+iOfst)>p->endpoint.iOffset ){ return SQLITE_IOERR_SHORT_READ; } -#endif - - assert( (iAmt+iOfst)<=p->endpoint.iOffset ); assert( p->readpoint.iOffset==0 || p->readpoint.pChunk!=0 ); if( p->readpoint.iOffset!=iOfst || iOfst==0 ){ sqlite3_int64 iOff = 0; - for(pChunk=p->pFirst; + for(pChunk=p->pFirst; ALWAYS(pChunk) && (iOff+p->nChunkSize)<=iOfst; pChunk=pChunk->pNext ){ @@ -87007,14 +106489,13 @@ static int memjrnlRead( /* ** Free the list of FileChunk structures headed at MemJournal.pFirst. */ -static void memjrnlFreeChunks(MemJournal *p){ +static void memjrnlFreeChunks(FileChunk *pFirst){ FileChunk *pIter; FileChunk *pNext; - for(pIter=p->pFirst; pIter; pIter=pNext){ + for(pIter=pFirst; pIter; pIter=pNext){ pNext = pIter->pNext; sqlite3_free(pIter); - } - p->pFirst = 0; + } } /* @@ -87041,7 +106522,7 @@ static int memjrnlCreateFile(MemJournal *p){ } if( rc==SQLITE_OK ){ /* No error has occurred. Free the in-memory buffers. */ - memjrnlFreeChunks(©); + memjrnlFreeChunks(copy.pFirst); } } if( rc!=SQLITE_OK ){ @@ -87056,6 +106537,9 @@ static int memjrnlCreateFile(MemJournal *p){ } +/* Forward reference */ +static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size); + /* ** Write data to the file. */ @@ -87085,22 +106569,21 @@ static int memjrnlWrite( ** access writes are not required. The only exception to this is when ** the in-memory journal is being used by a connection using the ** atomic-write optimization. In this case the first 28 bytes of the - ** journal file may be written as part of committing the transaction. */ - assert( iOfst==p->endpoint.iOffset || iOfst==0 ); -#ifdef SQLITE_ENABLE_ATOMIC_WRITE + ** journal file may be written as part of committing the transaction. */ + assert( iOfst<=p->endpoint.iOffset ); + if( iOfst>0 && iOfst!=p->endpoint.iOffset ){ + memjrnlTruncate(pJfd, iOfst); + } if( iOfst==0 && p->pFirst ){ assert( p->nChunkSize>iAmt ); memcpy((u8*)p->pFirst->zChunk, zBuf, iAmt); - }else -#else - assert( iOfst>0 || p->pFirst==0 ); -#endif - { + }else{ while( nWrite>0 ){ FileChunk *pChunk = p->endpoint.pChunk; int iChunkOffset = (int)(p->endpoint.iOffset%p->nChunkSize); int iSpace = MIN(nWrite, p->nChunkSize - iChunkOffset); + assert( pChunk!=0 || iChunkOffset==0 ); if( iChunkOffset==0 ){ /* New chunk is required to extend the file. */ FileChunk *pNew = sqlite3_malloc(fileChunkSize(p->nChunkSize)); @@ -87115,15 +106598,15 @@ static int memjrnlWrite( assert( !p->pFirst ); p->pFirst = pNew; } - p->endpoint.pChunk = pNew; + pChunk = p->endpoint.pChunk = pNew; } - memcpy((u8*)p->endpoint.pChunk->zChunk + iChunkOffset, zWrite, iSpace); + assert( pChunk!=0 ); + memcpy((u8*)pChunk->zChunk + iChunkOffset, zWrite, iSpace); zWrite += iSpace; nWrite -= iSpace; p->endpoint.iOffset += iSpace; } - p->nSize = iAmt + iOfst; } } @@ -87131,19 +106614,29 @@ static int memjrnlWrite( } /* -** Truncate the file. -** -** If the journal file is already on disk, truncate it there. Or, if it -** is still in main memory but is being truncated to zero bytes in size, -** ignore +** Truncate the in-memory file. */ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ MemJournal *p = (MemJournal *)pJfd; - if( ALWAYS(size==0) ){ - memjrnlFreeChunks(p); - p->nSize = 0; - p->endpoint.pChunk = 0; - p->endpoint.iOffset = 0; + assert( p->endpoint.pChunk==0 || p->endpoint.pChunk->pNext==0 ); + if( sizeendpoint.iOffset ){ + FileChunk *pIter = 0; + if( size==0 ){ + memjrnlFreeChunks(p->pFirst); + p->pFirst = 0; + }else{ + i64 iOff = p->nChunkSize; + for(pIter=p->pFirst; ALWAYS(pIter) && iOffpNext){ + iOff += p->nChunkSize; + } + if( ALWAYS(pIter) ){ + memjrnlFreeChunks(pIter->pNext); + pIter->pNext = 0; + } + } + + p->endpoint.pChunk = pIter; + p->endpoint.iOffset = size; p->readpoint.pChunk = 0; p->readpoint.iOffset = 0; } @@ -87155,15 +106648,15 @@ static int memjrnlTruncate(sqlite3_file *pJfd, sqlite_int64 size){ */ static int memjrnlClose(sqlite3_file *pJfd){ MemJournal *p = (MemJournal *)pJfd; - memjrnlFreeChunks(p); + memjrnlFreeChunks(p->pFirst); return SQLITE_OK; } /* ** Sync the file. ** -** If the real file has been created, call its xSync method. Otherwise, -** syncing an in-memory journal is a no-op. +** If the real file has been created, call its xSync method. Otherwise, +** syncing an in-memory journal is a no-op. */ static int memjrnlSync(sqlite3_file *pJfd, int flags){ UNUSED_PARAMETER2(pJfd, flags); @@ -87204,11 +106697,11 @@ static const struct sqlite3_io_methods MemJournalMethods = { 0 /* xUnfetch */ }; -/* -** Open a journal file. +/* +** Open a journal file. ** -** The behaviour of the journal file depends on the value of parameter -** nSpill. If nSpill is 0, then the journal file is always create and +** The behaviour of the journal file depends on the value of parameter +** nSpill. If nSpill is 0, then the journal file is always create and ** accessed using the underlying VFS. If nSpill is less than zero, then ** all content is always stored in main-memory. Finally, if nSpill is a ** positive value, then the journal file is initially created in-memory @@ -87225,6 +106718,8 @@ SQLITE_PRIVATE int sqlite3JournalOpen( ){ MemJournal *p = (MemJournal*)pJfd; + assert( zName || nSpill<0 || (flags & SQLITE_OPEN_EXCLUSIVE) ); + /* Zero the file-handle object. If nSpill was passed zero, initialize ** it using the sqlite3OsOpen() function of the underlying VFS. In this ** case none of the code in this module is executed as a result of calls @@ -87241,7 +106736,7 @@ SQLITE_PRIVATE int sqlite3JournalOpen( assert( MEMJOURNAL_DFLT_FILECHUNKSIZE==fileChunkSize(p->nChunkSize) ); } - p->pMethod = (const sqlite3_io_methods*)&MemJournalMethods; + pJfd->pMethods = (const sqlite3_io_methods*)&MemJournalMethods; p->nSpill = nSpill; p->flags = flags; p->zJournal = zName; @@ -87256,17 +106751,31 @@ SQLITE_PRIVATE void sqlite3MemJournalOpen(sqlite3_file *pJfd){ sqlite3JournalOpen(0, 0, pJfd, 0, -1); } -#ifdef SQLITE_ENABLE_ATOMIC_WRITE +#if defined(SQLITE_ENABLE_ATOMIC_WRITE) \ + || defined(SQLITE_ENABLE_BATCH_ATOMIC_WRITE) /* -** If the argument p points to a MemJournal structure that is not an +** If the argument p points to a MemJournal structure that is not an ** in-memory-only journal file (i.e. is one that was opened with a +ve -** nSpill parameter), and the underlying file has not yet been created, -** create it now. +** nSpill parameter or as SQLITE_OPEN_MAIN_JOURNAL), and the underlying +** file has not yet been created, create it now. */ -SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *p){ +SQLITE_PRIVATE int sqlite3JournalCreate(sqlite3_file *pJfd){ int rc = SQLITE_OK; - if( p->pMethods==&MemJournalMethods && ((MemJournal*)p)->nSpill>0 ){ - rc = memjrnlCreateFile((MemJournal*)p); + MemJournal *p = (MemJournal*)pJfd; + if( pJfd->pMethods==&MemJournalMethods && ( +#ifdef SQLITE_ENABLE_ATOMIC_WRITE + p->nSpill>0 +#else + /* While this appears to not be possible without ATOMIC_WRITE, the + ** paths are complex, so it seems prudent to leave the test in as + ** a NEVER(), in case our analysis is subtly flawed. */ + NEVER(p->nSpill>0) +#endif +#ifdef SQLITE_ENABLE_BATCH_ATOMIC_WRITE + || (p->flags & SQLITE_OPEN_MAIN_JOURNAL) +#endif + )){ + rc = memjrnlCreateFile(p); } return rc; } @@ -87281,7 +106790,7 @@ SQLITE_PRIVATE int sqlite3JournalIsInMemory(sqlite3_file *p){ return p->pMethods==&MemJournalMethods; } -/* +/* ** Return the number of bytes required to store a JournalFile that uses vfs ** pVfs to create the underlying on-disk files. */ @@ -87310,6 +106819,31 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ /* #include */ +#if !defined(SQLITE_OMIT_WINDOWFUNC) +/* +** Walk all expressions linked into the list of Window objects passed +** as the second argument. +*/ +static int walkWindowList(Walker *pWalker, Window *pList, int bOneOnly){ + Window *pWin; + for(pWin=pList; pWin; pWin=pWin->pNextWin){ + int rc; + rc = sqlite3WalkExprList(pWalker, pWin->pOrderBy); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExprList(pWalker, pWin->pPartition); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pFilter); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pStart); + if( rc ) return WRC_Abort; + rc = sqlite3WalkExpr(pWalker, pWin->pEnd); + if( rc ) return WRC_Abort; + if( bOneOnly ) break; + } + return WRC_Continue; +} +#endif + /* ** Walk an expression tree. Invoke the callback once for each node ** of the expression, while descending. (In other words, the callback @@ -87320,34 +106854,51 @@ SQLITE_PRIVATE int sqlite3JournalSize(sqlite3_vfs *pVfs){ ** ** WRC_Continue Continue descending down the tree. ** -** WRC_Prune Do not descend into child nodes. But allow +** WRC_Prune Do not descend into child nodes, but allow ** the walk to continue with sibling nodes. ** ** WRC_Abort Do no more callbacks. Unwind the stack and -** return the top-level walk call. +** return from the top-level walk call. ** ** The return value from this routine is WRC_Abort to abandon the tree walk ** and WRC_Continue to continue. */ -static SQLITE_NOINLINE int walkExpr(Walker *pWalker, Expr *pExpr){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3WalkExprNN(Walker *pWalker, Expr *pExpr){ int rc; testcase( ExprHasProperty(pExpr, EP_TokenOnly) ); testcase( ExprHasProperty(pExpr, EP_Reduced) ); - rc = pWalker->xExprCallback(pWalker, pExpr); - if( rc==WRC_Continue - && !ExprHasProperty(pExpr,EP_TokenOnly) ){ - if( sqlite3WalkExpr(pWalker, pExpr->pLeft) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, pExpr->pRight) ) return WRC_Abort; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; - }else{ - if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + while(1){ + rc = pWalker->xExprCallback(pWalker, pExpr); + if( rc ) return rc & WRC_Abort; + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + assert( pExpr->x.pList==0 || pExpr->pRight==0 ); + if( pExpr->pLeft && sqlite3WalkExprNN(pWalker, pExpr->pLeft) ){ + return WRC_Abort; + } + if( pExpr->pRight ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); + pExpr = pExpr->pRight; + continue; + }else if( ExprUseXSelect(pExpr) ){ + assert( !ExprHasProperty(pExpr, EP_WinFunc) ); + if( sqlite3WalkSelect(pWalker, pExpr->x.pSelect) ) return WRC_Abort; + }else{ + if( pExpr->x.pList ){ + if( sqlite3WalkExprList(pWalker, pExpr->x.pList) ) return WRC_Abort; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( walkWindowList(pWalker, pExpr->y.pWin, 1) ) return WRC_Abort; + } +#endif + } } + break; } - return rc & WRC_Abort; + return WRC_Continue; } SQLITE_PRIVATE int sqlite3WalkExpr(Walker *pWalker, Expr *pExpr){ - return pExpr ? walkExpr(pWalker,pExpr) : WRC_Continue; + return pExpr ? sqlite3WalkExprNN(pWalker,pExpr) : WRC_Continue; } /* @@ -87365,6 +106916,16 @@ SQLITE_PRIVATE int sqlite3WalkExprList(Walker *pWalker, ExprList *p){ return WRC_Continue; } +/* +** This is a no-op callback for Walker->xSelectCallback2. If this +** callback is set, then the Select->pWinDefn list is traversed. +*/ +SQLITE_PRIVATE void sqlite3WalkWinDefnDummyCallback(Walker *pWalker, Select *p){ + UNUSED_PARAMETER(pWalker); + UNUSED_PARAMETER(p); + /* No-op */ +} + /* ** Walk all expressions associated with SELECT statement p. Do ** not invoke the SELECT callback on p, but do (of course) invoke @@ -87378,7 +106939,22 @@ SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ if( sqlite3WalkExpr(pWalker, p->pHaving) ) return WRC_Abort; if( sqlite3WalkExprList(pWalker, p->pOrderBy) ) return WRC_Abort; if( sqlite3WalkExpr(pWalker, p->pLimit) ) return WRC_Abort; - if( sqlite3WalkExpr(pWalker, p->pOffset) ) return WRC_Abort; +#if !defined(SQLITE_OMIT_WINDOWFUNC) + if( p->pWinDefn ){ + Parse *pParse; + if( pWalker->xSelectCallback2==sqlite3WalkWinDefnDummyCallback + || ((pParse = pWalker->pParse)!=0 && IN_RENAME_OBJECT) +#ifndef SQLITE_OMIT_CTE + || pWalker->xSelectCallback2==sqlite3SelectPopWith +#endif + ){ + /* The following may return WRC_Abort if there are unresolvable + ** symbols (e.g. a table that does not exist) in a window definition. */ + int rc = walkWindowList(pWalker, p->pWinDefn, 0); + return rc; + } + } +#endif return WRC_Continue; } @@ -87386,18 +106962,20 @@ SQLITE_PRIVATE int sqlite3WalkSelectExpr(Walker *pWalker, Select *p){ ** Walk the parse trees associated with all subqueries in the ** FROM clause of SELECT statement p. Do not invoke the select ** callback on p, but do invoke it on each FROM clause subquery -** and on any subqueries further down in the tree. Return +** and on any subqueries further down in the tree. Return ** WRC_Abort or WRC_Continue; */ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ SrcList *pSrc; int i; - struct SrcList_item *pItem; + SrcItem *pItem; pSrc = p->pSrc; if( ALWAYS(pSrc) ){ for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - if( sqlite3WalkSelect(pWalker, pItem->pSelect) ){ + if( pItem->fg.isSubquery + && sqlite3WalkSelect(pWalker, pItem->u4.pSubq->pSelect) + ){ return WRC_Abort; } if( pItem->fg.isTabFunc @@ -87408,17 +106986,18 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ } } return WRC_Continue; -} +} /* ** Call sqlite3WalkExpr() for every expression in Select statement p. ** Invoke sqlite3WalkSelect() for subqueries in the FROM clause and -** on the compound select chain, p->pPrior. +** on the compound select chain, p->pPrior. ** ** If it is not NULL, the xSelectCallback() callback is invoked before ** the walk of the expressions and FROM clause. The xSelectCallback2() -** method, if it is not NULL, is invoked following the walk of the -** expressions and FROM clause. +** method is invoked following the walk of the expressions and FROM clause, +** but only if both xSelectCallback and xSelectCallback2 are both non-NULL +** and if the expressions and FROM clause both return WRC_Continue; ** ** Return WRC_Continue under normal conditions. Return WRC_Abort if ** there is an abort request. @@ -87428,29 +107007,59 @@ SQLITE_PRIVATE int sqlite3WalkSelectFrom(Walker *pWalker, Select *p){ */ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ int rc; - if( p==0 || (pWalker->xSelectCallback==0 && pWalker->xSelectCallback2==0) ){ - return WRC_Continue; - } - rc = WRC_Continue; - pWalker->walkerDepth++; - while( p ){ - if( pWalker->xSelectCallback ){ - rc = pWalker->xSelectCallback(pWalker, p); - if( rc ) break; - } + if( p==0 ) return WRC_Continue; + if( pWalker->xSelectCallback==0 ) return WRC_Continue; + do{ + rc = pWalker->xSelectCallback(pWalker, p); + if( rc ) return rc & WRC_Abort; if( sqlite3WalkSelectExpr(pWalker, p) || sqlite3WalkSelectFrom(pWalker, p) ){ - pWalker->walkerDepth--; return WRC_Abort; } if( pWalker->xSelectCallback2 ){ pWalker->xSelectCallback2(pWalker, p); } p = p->pPrior; - } + }while( p!=0 ); + return WRC_Continue; +} + +/* Increase the walkerDepth when entering a subquery, and +** decrease when leaving the subquery. +*/ +SQLITE_PRIVATE int sqlite3WalkerDepthIncrease(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pSelect); + pWalker->walkerDepth++; + return WRC_Continue; +} +SQLITE_PRIVATE void sqlite3WalkerDepthDecrease(Walker *pWalker, Select *pSelect){ + UNUSED_PARAMETER(pSelect); pWalker->walkerDepth--; - return rc & WRC_Abort; +} + + +/* +** No-op routine for the parse-tree walker. +** +** When this routine is the Walker.xExprCallback then expression trees +** are walked without any actions being taken at each node. Presumably, +** when this routine is used for Walker.xExprCallback then +** Walker.xSelectCallback is set to do something useful for every +** subquery in the parser tree. +*/ +SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; +} + +/* +** No-op routine for the parse-tree walker for SELECT statements. +** subquery in the parser tree. +*/ +SQLITE_PRIVATE int sqlite3SelectWalkNoop(Walker *NotUsed, Select *NotUsed2){ + UNUSED_PARAMETER2(NotUsed, NotUsed2); + return WRC_Continue; } /************** End of walker.c **********************************************/ @@ -87472,8 +107081,11 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** table and column. */ /* #include "sqliteInt.h" */ -/* #include */ -/* #include */ + +/* +** Magic table number to mean the EXCLUDED table in an UPSERT statement. +*/ +#define EXCLUDED_TABLE_NUMBER 2 /* ** Walk the expression tree pExpr and increase the aggregate function @@ -87483,6 +107095,8 @@ SQLITE_PRIVATE int sqlite3WalkSelect(Walker *pWalker, Select *p){ ** ** incrAggFunctionDepth(pExpr,n) is the main routine. incrAggDepth(..) ** is a helper function - a callback for the tree walker. +** +** See also the sqlite3WindowExtraAggFuncDepth() routine in window.c */ static int incrAggDepth(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_AGG_FUNCTION ) pExpr->op2 += pWalker->u.n; @@ -87522,7 +107136,6 @@ static void resolveAlias( ExprList *pEList, /* A result set */ int iCol, /* A column in the result set. 0..pEList->nExpr-1 */ Expr *pExpr, /* Transform this into an alias to the result set */ - const char *zType, /* "GROUP" or "ORDER" or "" */ int nSubquery /* Number of subqueries that the label is moving */ ){ Expr *pOrig; /* The iCol-th column of the result set */ @@ -87532,64 +107145,64 @@ static void resolveAlias( assert( iCol>=0 && iColnExpr ); pOrig = pEList->a[iCol].pExpr; assert( pOrig!=0 ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + if( pExpr->pAggInfo ) return; db = pParse->db; pDup = sqlite3ExprDup(db, pOrig, 0); - if( pDup==0 ) return; - if( zType[0]!='G' ) incrAggFunctionDepth(pDup, nSubquery); - if( pExpr->op==TK_COLLATE ){ - pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); - } - ExprSetProperty(pDup, EP_Alias); - - /* Before calling sqlite3ExprDelete(), set the EP_Static flag. This - ** prevents ExprDelete() from deleting the Expr structure itself, - ** allowing it to be repopulated by the memcpy() on the following line. - ** The pExpr->u.zToken might point into memory that will be freed by the - ** sqlite3DbFree(db, pDup) on the last line of this block, so be sure to - ** make a copy of the token before doing the sqlite3DbFree(). - */ - ExprSetProperty(pExpr, EP_Static); - sqlite3ExprDelete(db, pExpr); - memcpy(pExpr, pDup, sizeof(*pExpr)); - if( !ExprHasProperty(pExpr, EP_IntValue) && pExpr->u.zToken!=0 ){ - assert( (pExpr->flags & (EP_Reduced|EP_TokenOnly))==0 ); - pExpr->u.zToken = sqlite3DbStrDup(db, pExpr->u.zToken); - pExpr->flags |= EP_MemToken; - } - sqlite3DbFree(db, pDup); -} - - -/* -** Return TRUE if the name zCol occurs anywhere in the USING clause. -** -** Return FALSE if the USING clause is NULL or if it does not contain -** zCol. -*/ -static int nameInUsingClause(IdList *pUsing, const char *zCol){ - if( pUsing ){ - int k; - for(k=0; knId; k++){ - if( sqlite3StrICmp(pUsing->a[k].zName, zCol)==0 ) return 1; + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + pDup = 0; + }else{ + Expr temp; + incrAggFunctionDepth(pDup, nSubquery); + if( pExpr->op==TK_COLLATE ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pDup = sqlite3ExprAddCollateString(pParse, pDup, pExpr->u.zToken); } + memcpy(&temp, pDup, sizeof(Expr)); + memcpy(pDup, pExpr, sizeof(Expr)); + memcpy(pExpr, &temp, sizeof(Expr)); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + if( ALWAYS(pExpr->y.pWin!=0) ){ + pExpr->y.pWin->pOwner = pExpr; + } + } + sqlite3ExprDeferredDelete(pParse, pDup); } - return 0; } /* -** Subqueries stores the original database, table and column names for their -** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN". -** Check to see if the zSpan given to this routine matches the zDb, zTab, -** and zCol. If any of zDb, zTab, and zCol are NULL then those fields will -** match anything. +** Subqueries store the original database, table and column names for their +** result sets in ExprList.a[].zSpan, in the form "DATABASE.TABLE.COLUMN", +** and mark the expression-list item by setting ExprList.a[].fg.eEName +** to ENAME_TAB. +** +** Check to see if the zSpan/eEName of the expression-list item passed to this +** routine matches the zDb, zTab, and zCol. If any of zDb, zTab, and zCol are +** NULL then those fields will match anything. Return true if there is a match, +** or false otherwise. +** +** SF_NestedFrom subqueries also store an entry for the implicit rowid (or +** _rowid_, or oid) column by setting ExprList.a[].fg.eEName to ENAME_ROWID, +** and setting zSpan to "DATABASE.TABLE.". This type of pItem +** argument matches if zCol is a rowid alias. If it is not NULL, (*pbRowid) +** is set to 1 if there is this kind of match. */ -SQLITE_PRIVATE int sqlite3MatchSpanName( - const char *zSpan, +SQLITE_PRIVATE int sqlite3MatchEName( + const struct ExprList_item *pItem, const char *zCol, const char *zTab, - const char *zDb + const char *zDb, + int *pbRowid ){ int n; + const char *zSpan; + int eEName = pItem->fg.eEName; + if( eEName!=ENAME_TAB && (eEName!=ENAME_ROWID || NEVER(pbRowid==0)) ){ + return 0; + } + assert( pbRowid==0 || *pbRowid==0 ); + zSpan = pItem->zEName; for(n=0; ALWAYS(zSpan[n]) && zSpan[n]!='.'; n++){} if( zDb && (sqlite3StrNICmp(zSpan, zDb, n)!=0 || zDb[n]!=0) ){ return 0; @@ -87600,15 +107213,110 @@ SQLITE_PRIVATE int sqlite3MatchSpanName( return 0; } zSpan += n+1; - if( zCol && sqlite3StrICmp(zSpan, zCol)!=0 ){ - return 0; + if( zCol ){ + if( eEName==ENAME_TAB && sqlite3StrICmp(zSpan, zCol)!=0 ) return 0; + if( eEName==ENAME_ROWID && sqlite3IsRowid(zCol)==0 ) return 0; } + if( eEName==ENAME_ROWID ) *pbRowid = 1; return 1; } +/* +** Return TRUE if the double-quoted string mis-feature should be supported. +*/ +static int areDoubleQuotedStringsEnabled(sqlite3 *db, NameContext *pTopNC){ + if( db->init.busy ) return 1; /* Always support for legacy schemas */ + if( pTopNC->ncFlags & NC_IsDDL ){ + /* Currently parsing a DDL statement */ + if( sqlite3WritableSchema(db) && (db->flags & SQLITE_DqsDML)!=0 ){ + return 1; + } + return (db->flags & SQLITE_DqsDDL)!=0; + }else{ + /* Currently parsing a DML statement */ + return (db->flags & SQLITE_DqsDML)!=0; + } +} + +/* +** The argument is guaranteed to be a non-NULL Expr node of type TK_COLUMN. +** return the appropriate colUsed mask. +*/ +SQLITE_PRIVATE Bitmask sqlite3ExprColUsed(Expr *pExpr){ + int n; + Table *pExTab; + + n = pExpr->iColumn; + assert( ExprUseYTab(pExpr) ); + pExTab = pExpr->y.pTab; + assert( pExTab!=0 ); + assert( n < pExTab->nCol ); + if( (pExTab->tabFlags & TF_HasGenerated)!=0 + && (pExTab->aCol[n].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pExTab->nCol==BMS-1 ); + testcase( pExTab->nCol==BMS ); + return pExTab->nCol>=BMS ? ALLBITS : MASKBIT(pExTab->nCol)-1; + }else{ + testcase( n==BMS-1 ); + testcase( n==BMS ); + if( n>=BMS ) n = BMS-1; + return ((Bitmask)1)<db, TK_COLUMN, 0, 0); + if( pNew ){ + pNew->iTable = pMatch->iCursor; + pNew->iColumn = iColumn; + pNew->y.pTab = pMatch->pSTab; + assert( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ); + ExprSetProperty(pNew, EP_CanBeNull); + *ppList = sqlite3ExprListAppend(pParse, *ppList, pNew); + } +} + +/* +** Return TRUE (non-zero) if zTab is a valid name for the schema table pTab. +*/ +static SQLITE_NOINLINE int isValidSchemaTableName( + const char *zTab, /* Name as it appears in the SQL */ + Table *pTab, /* The schema table we are trying to match */ + const char *zDb /* non-NULL if a database qualifier is present */ +){ + const char *zLegacy; + assert( pTab!=0 ); + assert( pTab->tnum==1 ); + if( sqlite3StrNICmp(zTab, "sqlite_", 7)!=0 ) return 0; + zLegacy = pTab->zName; + if( strcmp(zLegacy+7, &LEGACY_TEMP_SCHEMA_TABLE[7])==0 ){ + if( sqlite3StrICmp(zTab+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ + return 1; + } + if( zDb==0 ) return 0; + if( sqlite3StrICmp(zTab+7, &LEGACY_SCHEMA_TABLE[7])==0 ) return 1; + if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; + }else{ + if( sqlite3StrICmp(zTab+7, &PREFERRED_SCHEMA_TABLE[7])==0 ) return 1; + } + return 0; +} + /* ** Given the name of a column of the form X.Y.Z or Y.Z or just Z, look up -** that name in the set of source tables in pSrcList and make the pExpr +** that name in the set of source tables in pSrcList and make the pExpr ** expression node refer back to that source column. The following changes ** are made to pExpr: ** @@ -87616,7 +107324,7 @@ SQLITE_PRIVATE int sqlite3MatchSpanName( ** (even if X is implied). ** pExpr->iTable Set to the cursor number for the table obtained ** from pSrcList. -** pExpr->pTab Points to the Table structure of X.Y (even if +** pExpr->y.pTab Points to the Table structure of X.Y (even if ** X and/or Y are implied.) ** pExpr->iColumn Set to the column number within the table. ** pExpr->op Set to TK_COLUMN. @@ -87637,30 +107345,32 @@ static int lookupName( Parse *pParse, /* The parsing context */ const char *zDb, /* Name of the database containing table, or NULL */ const char *zTab, /* Name of table containing column, or NULL */ - const char *zCol, /* Name of the column. */ + const Expr *pRight, /* Name of the column. */ NameContext *pNC, /* The name context used to resolve the name */ Expr *pExpr /* Make this EXPR node point to the selected column */ ){ int i, j; /* Loop counters */ int cnt = 0; /* Number of matching column names */ - int cntTab = 0; /* Number of matching table names */ + int cntTab = 0; /* Number of potential "rowid" matches */ int nSubquery = 0; /* How many levels of subquery */ sqlite3 *db = pParse->db; /* The database connection */ - struct SrcList_item *pItem; /* Use for looping over pSrcList items */ - struct SrcList_item *pMatch = 0; /* The matching pSrcList item */ + SrcItem *pItem; /* Use for looping over pSrcList items */ + SrcItem *pMatch = 0; /* The matching pSrcList item */ NameContext *pTopNC = pNC; /* First namecontext in the list */ Schema *pSchema = 0; /* Schema of the expression */ - int isTrigger = 0; /* True if resolved to a trigger column */ - Table *pTab = 0; /* Table hold the row */ + int eNewExprOp = TK_COLUMN; /* New value for pExpr->op on success */ + Table *pTab = 0; /* Table holding the row */ Column *pCol; /* A column of pTab */ + ExprList *pFJMatch = 0; /* Matches for FULL JOIN .. USING */ + const char *zCol = pRight->u.zToken; assert( pNC ); /* the name context cannot be NULL. */ assert( zCol ); /* The Z in X.Y.Z cannot be NULL */ + assert( zDb==0 || zTab!=0 ); assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); /* Initialize the node to no-match */ pExpr->iTable = -1; - pExpr->pTab = 0; ExprSetVVAProperty(pExpr, EP_NoReduce); /* Translate the schema name in zDb into a pointer to the corresponding @@ -87678,105 +107388,250 @@ static int lookupName( zDb = 0; }else{ for(i=0; inDb; i++){ - assert( db->aDb[i].zName ); - if( sqlite3StrICmp(db->aDb[i].zName,zDb)==0 ){ + assert( db->aDb[i].zDbSName ); + if( sqlite3StrICmp(db->aDb[i].zDbSName,zDb)==0 ){ pSchema = db->aDb[i].pSchema; break; } } + if( i==db->nDb && sqlite3StrICmp("main", zDb)==0 ){ + /* This branch is taken when the main database has been renamed + ** using SQLITE_DBCONFIG_MAINDBNAME. */ + pSchema = db->aDb[0].pSchema; + zDb = db->aDb[0].zDbSName; + } } } /* Start at the inner-most context and move outward until a match is found */ - while( pNC && cnt==0 ){ + assert( pNC && cnt==0 ); + do{ ExprList *pEList; SrcList *pSrcList = pNC->pSrcList; if( pSrcList ){ for(i=0, pItem=pSrcList->a; inSrc; i++, pItem++){ - pTab = pItem->pTab; + u8 hCol; + pTab = pItem->pSTab; assert( pTab!=0 && pTab->zName!=0 ); - assert( pTab->nCol>0 ); - if( pItem->pSelect && (pItem->pSelect->selFlags & SF_NestedFrom)!=0 ){ + assert( pTab->nCol>0 || pParse->nErr ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem)); + if( pItem->fg.isNestedFrom ){ + /* In this case, pItem is a subquery that has been formed from a + ** parenthesized subset of the FROM clause terms. Example: + ** .... FROM t1 LEFT JOIN (t2 RIGHT JOIN t3 USING(x)) USING(y) ... + ** \_________________________/ + ** This pItem -------------^ + */ int hit = 0; - pEList = pItem->pSelect->pEList; + Select *pSel; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + pSel = pItem->u4.pSubq->pSelect; + assert( pSel!=0 ); + pEList = pSel->pEList; + assert( pEList!=0 ); + assert( pEList->nExpr==pTab->nCol ); for(j=0; jnExpr; j++){ - if( sqlite3MatchSpanName(pEList->a[j].zSpan, zCol, zTab, zDb) ){ - cnt++; - cntTab = 2; - pMatch = pItem; - pExpr->iColumn = j; - hit = 1; + int bRowid = 0; /* True if possible rowid match */ + if( !sqlite3MatchEName(&pEList->a[j], zCol, zTab, zDb, &bRowid) ){ + continue; } + if( bRowid==0 ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + } + } + cnt++; + hit = 1; + }else if( cnt>0 ){ + /* This is a potential rowid match, but there has already been + ** a real match found. So this can be ignored. */ + continue; + } + cntTab++; + pMatch = pItem; + pExpr->iColumn = j; + pEList->a[j].fg.bUsed = 1; + + /* rowid cannot be part of a USING clause - assert() this. */ + assert( bRowid==0 || pEList->a[j].fg.bUsingTerm==0 ); + if( pEList->a[j].fg.bUsingTerm ) break; } if( hit || zTab==0 ) continue; } - if( zDb && pTab->pSchema!=pSchema ){ - continue; - } + assert( zDb==0 || zTab!=0 ); if( zTab ){ - const char *zTabName = pItem->zAlias ? pItem->zAlias : pTab->zName; - assert( zTabName!=0 ); - if( sqlite3StrICmp(zTabName, zTab)!=0 ){ - continue; + if( zDb ){ + if( pTab->pSchema!=pSchema ) continue; + if( pSchema==0 && strcmp(zDb,"*")!=0 ) continue; + } + if( pItem->zAlias!=0 ){ + if( sqlite3StrICmp(zTab, pItem->zAlias)!=0 ){ + continue; + } + }else if( sqlite3StrICmp(zTab, pTab->zName)!=0 ){ + if( pTab->tnum!=1 ) continue; + if( !isValidSchemaTableName(zTab, pTab, zDb) ) continue; + } + assert( ExprUseYTab(pExpr) ); + if( IN_RENAME_OBJECT && pItem->zAlias ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)&pExpr->y.pTab); } } - if( 0==(cntTab++) ){ - pMatch = pItem; - } + hCol = sqlite3StrIHash(zCol); for(j=0, pCol=pTab->aCol; jnCol; j++, pCol++){ - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ - /* If there has been exactly one prior match and this match - ** is for the right-hand table of a NATURAL JOIN or is in a - ** USING clause, then skip this match. - */ - if( cnt==1 ){ - if( pItem->fg.jointype & JT_NATURAL ) continue; - if( nameInUsingClause(pItem->pUsing, zCol) ) continue; + if( pCol->hName==hCol + && sqlite3StrICmp(pCol->zCnName, zCol)==0 + ){ + if( cnt>0 ){ + if( pItem->fg.isUsing==0 + || sqlite3IdListIndex(pItem->u3.pUsing, zCol)<0 + ){ + /* Two or more tables have the same column name which is + ** not joined by USING. This is an error. Signal as much + ** by clearing pFJMatch and letting cnt go above 1. */ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else + if( (pItem->fg.jointype & JT_RIGHT)==0 ){ + /* An INNER or LEFT JOIN. Use the left-most table */ + continue; + }else + if( (pItem->fg.jointype & JT_LEFT)==0 ){ + /* A RIGHT JOIN. Use the right-most table */ + cnt = 0; + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + }else{ + /* For a FULL JOIN, we must construct a coalesce() func */ + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + } } cnt++; pMatch = pItem; /* Substitute the rowid (column -1) for the INTEGER PRIMARY KEY */ pExpr->iColumn = j==pTab->iPKey ? -1 : (i16)j; + if( pItem->fg.isNestedFrom ){ + sqlite3SrcItemColumnUsed(pItem, j); + } break; } } + if( 0==cnt && VisibleRowid(pTab) ){ + /* pTab is a potential ROWID match. Keep track of it and match + ** the ROWID later if that seems appropriate. (Search for "cntTab" + ** to find related code.) Only allow a ROWID match if there is + ** a single ROWID match candidate. + */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + /* In SQLITE_ALLOW_ROWID_IN_VIEW mode, allow a ROWID match + ** if there is a single VIEW candidate or if there is a single + ** non-VIEW candidate plus multiple VIEW candidates. In other + ** words non-VIEW candidate terms take precedence over VIEWs. + */ + if( cntTab==0 + || (cntTab==1 + && pMatch!=0 + && ALWAYS(pMatch->pSTab!=0) + && (pMatch->pSTab->tabFlags & TF_Ephemeral)!=0 + && (pTab->tabFlags & TF_Ephemeral)==0) + ){ + cntTab = 1; + pMatch = pItem; + }else{ + cntTab++; + } +#else + /* The (much more common) non-SQLITE_ALLOW_ROWID_IN_VIEW case is + ** simpler since we require exactly one candidate, which will + ** always be a non-VIEW + */ + cntTab++; + pMatch = pItem; +#endif + } } if( pMatch ){ pExpr->iTable = pMatch->iCursor; - pExpr->pTab = pMatch->pTab; - /* RIGHT JOIN not (yet) supported */ - assert( (pMatch->fg.jointype & JT_RIGHT)==0 ); - if( (pMatch->fg.jointype & JT_LEFT)!=0 ){ + assert( ExprUseYTab(pExpr) ); + pExpr->y.pTab = pMatch->pSTab; + if( (pMatch->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 ){ ExprSetProperty(pExpr, EP_CanBeNull); } - pSchema = pExpr->pTab->pSchema; + pSchema = pExpr->y.pTab->pSchema; } } /* if( pSrcList ) */ -#ifndef SQLITE_OMIT_TRIGGER - /* If we have not already resolved the name, then maybe - ** it is a new.* or old.* trigger argument reference +#if !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) + /* If we have not already resolved the name, then maybe + ** it is a new.* or old.* trigger argument reference. Or + ** maybe it is an excluded.* from an upsert. Or maybe it is + ** a reference in the RETURNING clause to a table being modified. */ - if( zDb==0 && zTab!=0 && cntTab==0 && pParse->pTriggerTab!=0 ){ - int op = pParse->eTriggerOp; - assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); - if( op!=TK_DELETE && sqlite3StrICmp("new",zTab) == 0 ){ - pExpr->iTable = 1; - pTab = pParse->pTriggerTab; - }else if( op!=TK_INSERT && sqlite3StrICmp("old",zTab)==0 ){ - pExpr->iTable = 0; - pTab = pParse->pTriggerTab; - }else{ - pTab = 0; + if( cnt==0 && zDb==0 ){ + pTab = 0; +#ifndef SQLITE_OMIT_TRIGGER + if( pParse->pTriggerTab!=0 ){ + int op = pParse->eTriggerOp; + assert( op==TK_DELETE || op==TK_UPDATE || op==TK_INSERT ); + if( pParse->bReturning ){ + if( (pNC->ncFlags & NC_UBaseReg)!=0 + && ALWAYS(zTab==0 + || sqlite3StrICmp(zTab,pParse->pTriggerTab->zName)==0 + || isValidSchemaTableName(zTab, pParse->pTriggerTab, 0)) + ){ + pExpr->iTable = op!=TK_DELETE; + pTab = pParse->pTriggerTab; + } + }else if( op!=TK_DELETE && zTab && sqlite3StrICmp("new",zTab) == 0 ){ + pExpr->iTable = 1; + pTab = pParse->pTriggerTab; + }else if( op!=TK_INSERT && zTab && sqlite3StrICmp("old",zTab)==0 ){ + pExpr->iTable = 0; + pTab = pParse->pTriggerTab; + } } +#endif /* SQLITE_OMIT_TRIGGER */ +#ifndef SQLITE_OMIT_UPSERT + if( (pNC->ncFlags & NC_UUpsert)!=0 && zTab!=0 ){ + Upsert *pUpsert = pNC->uNC.pUpsert; + if( pUpsert && sqlite3StrICmp("excluded",zTab)==0 ){ + pTab = pUpsert->pUpsertSrc->a[0].pSTab; + pExpr->iTable = EXCLUDED_TABLE_NUMBER; + } + } +#endif /* SQLITE_OMIT_UPSERT */ - if( pTab ){ + if( pTab ){ int iCol; + u8 hCol = sqlite3StrIHash(zCol); pSchema = pTab->pSchema; cntTab++; for(iCol=0, pCol=pTab->aCol; iColnCol; iCol++, pCol++){ - if( sqlite3StrICmp(pCol->zName, zCol)==0 ){ + if( pCol->hName==hCol + && sqlite3StrICmp(pCol->zCnName, zCol)==0 + ){ if( iCol==pTab->iPKey ){ iCol = -1; } @@ -87789,38 +107644,72 @@ static int lookupName( } if( iColnCol ){ cnt++; - if( iCol<0 ){ - pExpr->affinity = SQLITE_AFF_INTEGER; - }else if( pExpr->iTable==0 ){ - testcase( iCol==31 ); - testcase( iCol==32 ); - pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<iTable==EXCLUDED_TABLE_NUMBER ){ + testcase( iCol==(-1) ); + assert( ExprUseYTab(pExpr) ); + if( IN_RENAME_OBJECT ){ + pExpr->iColumn = iCol; + pExpr->y.pTab = pTab; + eNewExprOp = TK_COLUMN; + }else{ + pExpr->iTable = pNC->uNC.pUpsert->regData + + sqlite3TableColumnToStorage(pTab, iCol); + eNewExprOp = TK_REGISTER; + } + }else +#endif /* SQLITE_OMIT_UPSERT */ + { + assert( ExprUseYTab(pExpr) ); + pExpr->y.pTab = pTab; + if( pParse->bReturning ){ + eNewExprOp = TK_REGISTER; + pExpr->op2 = TK_COLUMN; + pExpr->iColumn = iCol; + pExpr->iTable = pNC->uNC.iBaseReg + (pTab->nCol+1)*pExpr->iTable + + sqlite3TableColumnToStorage(pTab, iCol) + 1; + }else{ + pExpr->iColumn = (i16)iCol; + eNewExprOp = TK_TRIGGER; +#ifndef SQLITE_OMIT_TRIGGER + if( iCol<0 ){ + pExpr->affExpr = SQLITE_AFF_INTEGER; + }else if( pExpr->iTable==0 ){ + testcase( iCol==31 ); + testcase( iCol==32 ); + pParse->oldmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<newmask |= (iCol>=32 ? 0xffffffff : (((u32)1)<iColumn = (i16)iCol; - pExpr->pTab = pTab; - isTrigger = 1; } } } -#endif /* !defined(SQLITE_OMIT_TRIGGER) */ +#endif /* !defined(SQLITE_OMIT_TRIGGER) || !defined(SQLITE_OMIT_UPSERT) */ /* ** Perhaps the name is a reference to the ROWID */ if( cnt==0 - && cntTab==1 + && cntTab>=1 && pMatch - && (pNC->ncFlags & NC_IdxExpr)==0 + && (pNC->ncFlags & (NC_IdxExpr|NC_GenCol))==0 && sqlite3IsRowid(zCol) - && VisibleRowid(pMatch->pTab) + && ALWAYS(VisibleRowid(pMatch->pSTab) || pMatch->fg.isNestedFrom) ){ - cnt = 1; - pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; + cnt = cntTab; +#if SQLITE_ALLOW_ROWID_IN_VIEW+0==2 + if( pMatch->pSTab!=0 && IsView(pMatch->pSTab) ){ + eNewExprOp = TK_NULL; + } +#endif + if( pMatch->fg.isNestedFrom==0 ) pExpr->iColumn = -1; + pExpr->affExpr = SQLITE_AFF_INTEGER; } /* @@ -87841,39 +107730,56 @@ static int lookupName( ** is supported for backwards compatibility only. Hence, we issue a warning ** on sqlite3_log() whenever the capability is used. */ - if( (pEList = pNC->pEList)!=0 + if( cnt==0 + && (pNC->ncFlags & NC_UEList)!=0 && zTab==0 - && cnt==0 ){ + pEList = pNC->uNC.pEList; + assert( pEList!=0 ); for(j=0; jnExpr; j++){ - char *zAs = pEList->a[j].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + char *zAs = pEList->a[j].zEName; + if( pEList->a[j].fg.eEName==ENAME_NAME + && sqlite3_stricmp(zAs, zCol)==0 + ){ Expr *pOrig; assert( pExpr->pLeft==0 && pExpr->pRight==0 ); - assert( pExpr->x.pList==0 ); - assert( pExpr->x.pSelect==0 ); + assert( ExprUseXList(pExpr)==0 || pExpr->x.pList==0 ); + assert( ExprUseXSelect(pExpr)==0 || pExpr->x.pSelect==0 ); pOrig = pEList->a[j].pExpr; if( (pNC->ncFlags&NC_AllowAgg)==0 && ExprHasProperty(pOrig, EP_Agg) ){ sqlite3ErrorMsg(pParse, "misuse of aliased aggregate %s", zAs); return WRC_Abort; } - resolveAlias(pParse, pEList, j, pExpr, "", nSubquery); + if( ExprHasProperty(pOrig, EP_Win) + && ((pNC->ncFlags&NC_AllowWin)==0 || pNC!=pTopNC ) + ){ + sqlite3ErrorMsg(pParse, "misuse of aliased window function %s",zAs); + return WRC_Abort; + } + if( sqlite3ExprVectorSize(pOrig)!=1 ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return WRC_Abort; + } + resolveAlias(pParse, pEList, j, pExpr, nSubquery); cnt = 1; pMatch = 0; assert( zTab==0 && zDb==0 ); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pExpr); + } goto lookupname_end; } - } + } } /* Advance to the next name context. The loop will exit when either ** we have a match (cnt>0) or when we run out of name contexts. */ - if( cnt==0 ){ - pNC = pNC->pNext; - nSubquery++; - } - } + if( cnt ) break; + pNC = pNC->pNext; + nSubquery++; + }while( pNC ); + /* ** If X and Y are NULL (in other words if only the column name Z is @@ -87885,59 +107791,132 @@ static int lookupName( ** Because no reference was made to outer contexts, the pNC->nRef ** fields are not changed in any context. */ - if( cnt==0 && zTab==0 && ExprHasProperty(pExpr,EP_DblQuoted) ){ - pExpr->op = TK_STRING; - pExpr->pTab = 0; - return WRC_Prune; + if( cnt==0 && zTab==0 ){ + assert( pExpr->op==TK_ID ); + if( ExprHasProperty(pExpr,EP_DblQuoted) + && areDoubleQuotedStringsEnabled(db, pTopNC) + ){ + /* If a double-quoted identifier does not match any known column name, + ** then treat it as a string. + ** + ** This hack was added in the early days of SQLite in a misguided attempt + ** to be compatible with MySQL 3.x, which used double-quotes for strings. + ** I now sorely regret putting in this hack. The effect of this hack is + ** that misspelled identifier names are silently converted into strings + ** rather than causing an error, to the frustration of countless + ** programmers. To all those frustrated programmers, my apologies. + ** + ** Someday, I hope to get rid of this hack. Unfortunately there is + ** a huge amount of legacy SQL that uses it. So for now, we just + ** issue a warning. + */ + sqlite3_log(SQLITE_WARNING, + "double-quoted string literal: \"%w\"", zCol); +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3VdbeAddDblquoteStr(db, pParse->pVdbe, zCol); +#endif + pExpr->op = TK_STRING; + memset(&pExpr->y, 0, sizeof(pExpr->y)); + return WRC_Prune; + } + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } } /* - ** cnt==0 means there was not match. cnt>1 means there were two or - ** more matches. Either way, we have an error. + ** cnt==0 means there was not match. + ** cnt>1 means there were two or more matches. + ** + ** cnt==0 is always an error. cnt>1 is often an error, but might + ** be multiple matches for a NATURAL LEFT JOIN or a LEFT JOIN USING. */ + assert( pFJMatch==0 || cnt>0 ); + assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); if( cnt!=1 ){ const char *zErr; + if( pFJMatch ){ + if( pFJMatch->nExpr==cnt-1 ){ + if( ExprHasProperty(pExpr,EP_Leaf) ){ + ExprClearProperty(pExpr,EP_Leaf); + }else{ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + } + extendFJMatch(pParse, &pFJMatch, pMatch, pExpr->iColumn); + pExpr->op = TK_FUNCTION; + pExpr->u.zToken = "coalesce"; + pExpr->x.pList = pFJMatch; + cnt = 1; + goto lookupname_end; + }else{ + sqlite3ExprListDelete(db, pFJMatch); + pFJMatch = 0; + } + } zErr = cnt==0 ? "no such column" : "ambiguous column name"; if( zDb ){ sqlite3ErrorMsg(pParse, "%s: %s.%s.%s", zErr, zDb, zTab, zCol); }else if( zTab ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zErr, zTab, zCol); + }else if( cnt==0 && ExprHasProperty(pRight,EP_DblQuoted) ){ + sqlite3ErrorMsg(pParse, "%s: \"%s\" - should this be a" + " string literal in single-quotes?", + zErr, zCol); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zErr, zCol); } + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); pParse->checkSchema = 1; - pTopNC->nErr++; + pTopNC->nNcErr++; + eNewExprOp = TK_NULL; + } + assert( pFJMatch==0 ); + + /* Remove all substructure from pExpr */ + if( !ExprHasProperty(pExpr,(EP_TokenOnly|EP_Leaf)) ){ + sqlite3ExprDelete(db, pExpr->pLeft); + pExpr->pLeft = 0; + sqlite3ExprDelete(db, pExpr->pRight); + pExpr->pRight = 0; + ExprSetProperty(pExpr, EP_Leaf); } /* If a column from a table in pSrcList is referenced, then record ** this fact in the pSrcList.a[].colUsed bitmask. Column 0 causes - ** bit 0 to be set. Column 1 sets bit 1. And so forth. If the - ** column number is greater than the number of bits in the bitmask - ** then set the high-order bit of the bitmask. + ** bit 0 to be set. Column 1 sets bit 1. And so forth. Bit 63 is + ** set if the 63rd or any subsequent column is used. + ** + ** The colUsed mask is an optimization used to help determine if an + ** index is a covering index. The correct answer is still obtained + ** if the mask contains extra set bits. However, it is important to + ** avoid setting bits beyond the maximum column number of the table. + ** (See ticket [b92e5e8ec2cdbaa1]). + ** + ** If a generated column is referenced, set bits for every column + ** of the table. */ - if( pExpr->iColumn>=0 && pMatch!=0 ){ - int n = pExpr->iColumn; - testcase( n==BMS-1 ); - if( n>=BMS ){ - n = BMS-1; + if( pMatch ){ + if( pExpr->iColumn>=0 ){ + pMatch->colUsed |= sqlite3ExprColUsed(pExpr); + }else{ + pMatch->fg.rowidUsed = 1; } - assert( pMatch->iCursor==pExpr->iTable ); - pMatch->colUsed |= ((Bitmask)1)<pLeft); - pExpr->pLeft = 0; - sqlite3ExprDelete(db, pExpr->pRight); - pExpr->pRight = 0; - pExpr->op = (isTrigger ? TK_TRIGGER : TK_COLUMN); + pExpr->op = eNewExprOp; lookupname_end: if( cnt==1 ){ assert( pNC!=0 ); - if( !ExprHasProperty(pExpr, EP_Alias) ){ +#ifndef SQLITE_OMIT_AUTHORIZATION + if( pParse->db->xAuth + && (pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER) + ){ sqlite3AuthRead(pParse, pExpr, pSchema, pNC->pSrcList); } +#endif /* Increment the nRef value on all name contexts from TopNC up to ** the point where the name matched. */ for(;;){ @@ -87959,18 +107938,27 @@ lookupname_end: SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSrc, int iCol){ Expr *p = sqlite3ExprAlloc(db, TK_COLUMN, 0, 0); if( p ){ - struct SrcList_item *pItem = &pSrc->a[iSrc]; - p->pTab = pItem->pTab; + SrcItem *pItem = &pSrc->a[iSrc]; + Table *pTab; + assert( ExprUseYTab(p) ); + pTab = p->y.pTab = pItem->pSTab; p->iTable = pItem->iCursor; - if( p->pTab->iPKey==iCol ){ + if( p->y.pTab->iPKey==iCol ){ p->iColumn = -1; }else{ p->iColumn = (ynVar)iCol; - testcase( iCol==BMS ); - testcase( iCol==BMS-1 ); - pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + if( (pTab->tabFlags & TF_HasGenerated)!=0 + && (pTab->aCol[iCol].colFlags & COLFLAG_GENERATED)!=0 + ){ + testcase( pTab->nCol==63 ); + testcase( pTab->nCol==64 ); + pItem->colUsed = pTab->nCol>=64 ? ALLBITS : MASKBIT(pTab->nCol)-1; + }else{ + testcase( iCol==BMS ); + testcase( iCol==BMS-1 ); + pItem->colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); + } } - ExprSetProperty(p, EP_Resolved); } return p; } @@ -87978,23 +107966,41 @@ SQLITE_PRIVATE Expr *sqlite3CreateColumnExpr(sqlite3 *db, SrcList *pSrc, int iSr /* ** Report an error that an expression is not valid for some set of ** pNC->ncFlags values determined by validMask. +** +** static void notValid( +** Parse *pParse, // Leave error message here +** NameContext *pNC, // The name context +** const char *zMsg, // Type of error +** int validMask, // Set of contexts for which prohibited +** Expr *pExpr // Invalidate this expression on error +** ){...} +** +** As an optimization, since the conditional is almost always false +** (because errors are rare), the conditional is moved outside of the +** function call using a macro. */ -static void notValid( - Parse *pParse, /* Leave error message here */ - NameContext *pNC, /* The name context */ - const char *zMsg, /* Type of error */ - int validMask /* Set of contexts for which prohibited */ +static void notValidImpl( + Parse *pParse, /* Leave error message here */ + NameContext *pNC, /* The name context */ + const char *zMsg, /* Type of error */ + Expr *pExpr, /* Invalidate this expression on error */ + Expr *pError /* Associate error with this expression */ ){ - assert( (validMask&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr))==0 ); - if( (pNC->ncFlags & validMask)!=0 ){ - const char *zIn = "partial index WHERE clauses"; - if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; + const char *zIn = "partial index WHERE clauses"; + if( pNC->ncFlags & NC_IdxExpr ) zIn = "index expressions"; #ifndef SQLITE_OMIT_CHECK - else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; + else if( pNC->ncFlags & NC_IsCheck ) zIn = "CHECK constraints"; #endif - sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); - } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pNC->ncFlags & NC_GenCol ) zIn = "generated columns"; +#endif + sqlite3ErrorMsg(pParse, "%s prohibited in %s", zMsg, zIn); + if( pExpr ) pExpr->op = TK_NULL; + sqlite3RecordErrorOffsetOfExpr(pParse->db, pError); } +#define sqlite3ResolveNotValid(P,N,M,X,E,R) \ + assert( ((X)&~(NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol))==0 ); \ + if( ((N)->ncFlags & (X))!=0 ) notValidImpl(P,N,M,E,R); /* ** Expression p should encode a floating point value between 1.0 and 0.0. @@ -88004,6 +108010,7 @@ static void notValid( static int exprProbability(Expr *p){ double r = -1.0; if( p->op!=TK_FLOAT ) return -1; + assert( !ExprHasProperty(p, EP_IntValue) ); sqlite3AtoF(p->u.zToken, &r, sqlite3Strlen30(p->u.zToken), SQLITE_UTF8); assert( r>=0.0 ); if( r>1.0 ) return -1; @@ -88030,8 +108037,6 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ pParse = pNC->pParse; assert( pParse==pWalker->pParse ); - if( ExprHasProperty(pExpr, EP_Resolved) ) return WRC_Prune; - ExprSetProperty(pExpr, EP_Resolved); #ifndef NDEBUG if( pNC->pSrcList && pNC->pSrcList->nAlloc>0 ){ SrcList *pSrcList = pNC->pSrcList; @@ -88043,76 +108048,160 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif switch( pExpr->op ){ -#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) && !defined(SQLITE_OMIT_SUBQUERY) /* The special operator TK_ROW means use the rowid for the first ** column in the FROM clause. This is used by the LIMIT and ORDER BY - ** clause processing on UPDATE and DELETE statements. + ** clause processing on UPDATE and DELETE statements, and by + ** UPDATE ... FROM statement processing. */ case TK_ROW: { SrcList *pSrcList = pNC->pSrcList; - struct SrcList_item *pItem; - assert( pSrcList && pSrcList->nSrc==1 ); - pItem = pSrcList->a; + SrcItem *pItem; + assert( pSrcList && pSrcList->nSrc>=1 ); + pItem = pSrcList->a; pExpr->op = TK_COLUMN; - pExpr->pTab = pItem->pTab; + assert( ExprUseYTab(pExpr) ); + pExpr->y.pTab = pItem->pSTab; pExpr->iTable = pItem->iCursor; - pExpr->iColumn = -1; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->iColumn--; + pExpr->affExpr = SQLITE_AFF_INTEGER; break; } -#endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) - && !defined(SQLITE_OMIT_SUBQUERY) */ - /* A lone identifier is the name of a column. + /* An optimization: Attempt to convert + ** + ** "expr IS NOT NULL" --> "TRUE" + ** "expr IS NULL" --> "FALSE" + ** + ** if we can prove that "expr" is never NULL. Call this the + ** "NOT NULL strength reduction optimization". + ** + ** If this optimization occurs, also restore the NameContext ref-counts + ** to the state they where in before the "column" LHS expression was + ** resolved. This prevents "column" from being counted as having been + ** referenced, which might prevent a SELECT from being erroneously + ** marked as correlated. + ** + ** 2024-03-28: Beware of aggregates. A bare column of aggregated table + ** can still evaluate to NULL even though it is marked as NOT NULL. + ** Example: + ** + ** CREATE TABLE t1(a INT NOT NULL); + ** SELECT a, a IS NULL, a IS NOT NULL, count(*) FROM t1; + ** + ** The "a IS NULL" and "a IS NOT NULL" expressions cannot be optimized + ** here because at the time this case is hit, we do not yet know whether + ** or not t1 is being aggregated. We have to assume the worst and omit + ** the optimization. The only time it is safe to apply this optimization + ** is within the WHERE clause. */ - case TK_ID: { - return lookupName(pParse, 0, 0, pExpr->u.zToken, pNC, pExpr); + case TK_NOTNULL: + case TK_ISNULL: { + int anRef[8]; + NameContext *p; + int i; + for(i=0, p=pNC; p && ipNext, i++){ + anRef[i] = p->nRef; + } + sqlite3WalkExpr(pWalker, pExpr->pLeft); + if( IN_RENAME_OBJECT ) return WRC_Prune; + if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ + /* The expression can be NULL. So the optimization does not apply */ + return WRC_Prune; + } + + for(i=0, p=pNC; p; p=p->pNext, i++){ + if( (p->ncFlags & NC_Where)==0 ){ + return WRC_Prune; /* Not in a WHERE clause. Unsafe to optimize. */ + } + } + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x80000 ){ + sqlite3DebugPrintf( + "NOT NULL strength reduction converts the following to %d:\n", + pExpr->op==TK_NOTNULL + ); + sqlite3ShowExpr(pExpr); + } +#endif /* TREETRACE_ENABLED */ + pExpr->u.iValue = (pExpr->op==TK_NOTNULL); + pExpr->flags |= EP_IntValue; + pExpr->op = TK_INTEGER; + for(i=0, p=pNC; p && ipNext, i++){ + p->nRef = anRef[i]; + } + sqlite3ExprDelete(pParse->db, pExpr->pLeft); + pExpr->pLeft = 0; + return WRC_Prune; } - - /* A table name and column name: ID.ID + + /* A column name: ID + ** Or table name and column name: ID.ID ** Or a database, table and column: ID.ID.ID + ** + ** The TK_ID and TK_OUT cases are combined so that there will only + ** be one call to lookupName(). Then the compiler will in-line + ** lookupName() for a size reduction and performance increase. */ + case TK_ID: case TK_DOT: { - const char *zColumn; const char *zTable; const char *zDb; Expr *pRight; - /* if( pSrcList==0 ) break; */ - notValid(pParse, pNC, "the \".\" operator", NC_IdxExpr); - /*notValid(pParse, pNC, "the \".\" operator", NC_PartIdx|NC_IsCheck, 1);*/ - pRight = pExpr->pRight; - if( pRight->op==TK_ID ){ + if( pExpr->op==TK_ID ){ zDb = 0; - zTable = pExpr->pLeft->u.zToken; - zColumn = pRight->u.zToken; + zTable = 0; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pRight = pExpr; }else{ - assert( pRight->op==TK_DOT ); - zDb = pExpr->pLeft->u.zToken; - zTable = pRight->pLeft->u.zToken; - zColumn = pRight->pRight->u.zToken; + Expr *pLeft = pExpr->pLeft; + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "the \".\" operator", + NC_IdxExpr|NC_GenCol, 0, pExpr); + pRight = pExpr->pRight; + if( pRight->op==TK_ID ){ + zDb = 0; + }else{ + assert( pRight->op==TK_DOT ); + assert( !ExprHasProperty(pRight, EP_IntValue) ); + zDb = pLeft->u.zToken; + pLeft = pRight->pLeft; + pRight = pRight->pRight; + } + assert( ExprUseUToken(pLeft) && ExprUseUToken(pRight) ); + zTable = pLeft->u.zToken; + assert( ExprUseYTab(pExpr) ); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, (void*)pExpr, (void*)pRight); + sqlite3RenameTokenRemap(pParse, (void*)&pExpr->y.pTab, (void*)pLeft); + } } - return lookupName(pParse, zDb, zTable, zColumn, pNC, pExpr); + return lookupName(pParse, zDb, zTable, pRight, pNC, pExpr); } /* Resolve function names */ case TK_FUNCTION: { - ExprList *pList = pExpr->x.pList; /* The argument list */ - int n = pList ? pList->nExpr : 0; /* Number of arguments */ + ExprList *pList; /* The argument list */ + int n; /* Number of arguments */ int no_such_func = 0; /* True if no such function exists */ int wrong_num_args = 0; /* True if wrong number of arguments */ int is_agg = 0; /* True if is an aggregate function */ - int auth; /* Authorization to use the function */ - int nId; /* Number of characters in function name */ const char *zId; /* The function name. */ FuncDef *pDef; /* Information about the function */ u8 enc = ENC(pParse->db); /* The database encoding */ - - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - notValid(pParse, pNC, "functions", NC_PartIdx); + int savedAllowFlags = (pNC->ncFlags & (NC_AllowAgg | NC_AllowWin)); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = (IsWindowFunc(pExpr) ? pExpr->y.pWin : 0); +#endif + assert( !ExprHasProperty(pExpr, EP_xIsSelect|EP_IntValue) ); + assert( pExpr->pLeft==0 || pExpr->pLeft->op==TK_ORDER ); + pList = pExpr->x.pList; + n = pList ? pList->nExpr : 0; zId = pExpr->u.zToken; - nId = sqlite3Strlen30(zId); pDef = sqlite3FindFunction(pParse->db, zId, n, enc, 0); if( pDef==0 ){ pDef = sqlite3FindFunction(pParse->db, zId, -2, enc, 0); @@ -88124,14 +108213,14 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ }else{ is_agg = pDef->xFinalize!=0; if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ - ExprSetProperty(pExpr, EP_Unlikely|EP_Skip); + ExprSetProperty(pExpr, EP_Unlikely); if( n==2 ){ pExpr->iTable = exprProbability(pList->a[1].pExpr); if( pExpr->iTable<0 ){ sqlite3ErrorMsg(pParse, - "second argument to likelihood() must be a " - "constant between 0.0 and 1.0"); - pNC->nErr++; + "second argument to %#T() must be a " + "constant between 0.0 and 1.0", pExpr); + pNC->nNcErr++; } }else{ /* EVIDENCE-OF: R-61304-29449 The unlikely(X) function is @@ -88144,66 +108233,205 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ ** to likelihood(X,0.9375). */ /* TUNING: unlikely() probability is 0.0625. likely() is 0.9375 */ pExpr->iTable = pDef->zName[0]=='u' ? 8388608 : 125829120; - } + } } #ifndef SQLITE_OMIT_AUTHORIZATION - auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0, pDef->zName, 0); - if( auth!=SQLITE_OK ){ - if( auth==SQLITE_DENY ){ - sqlite3ErrorMsg(pParse, "not authorized to use function: %s", - pDef->zName); - pNC->nErr++; + { + int auth = sqlite3AuthCheck(pParse, SQLITE_FUNCTION, 0,pDef->zName,0); + if( auth!=SQLITE_OK ){ + if( auth==SQLITE_DENY ){ + sqlite3ErrorMsg(pParse, "not authorized to use function: %#T", + pExpr); + pNC->nNcErr++; + } + pExpr->op = TK_NULL; + return WRC_Prune; } - pExpr->op = TK_NULL; - return WRC_Prune; } #endif + + /* If the function may call sqlite3_value_subtype(), then set the + ** EP_SubtArg flag on all of its argument expressions. This prevents + ** where.c from replacing the expression with a value read from an + ** index on the same expression, which will not have the correct + ** subtype. Also set the flag if the function expression itself is + ** an EP_SubtArg expression. In this case subtypes are required as + ** the function may return a value with a subtype back to its + ** caller using sqlite3_result_value(). */ + if( (pDef->funcFlags & SQLITE_SUBTYPE) + || ExprHasProperty(pExpr, EP_SubtArg) + ){ + int ii; + for(ii=0; iia[ii].pExpr, EP_SubtArg); + } + } + if( pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG) ){ /* For the purposes of the EP_ConstFunc flag, date and time ** functions and other functions that change slowly are considered - ** constant because they are constant for the duration of one query */ + ** constant because they are constant for the duration of one query. + ** This allows them to be factored out of inner loops. */ ExprSetProperty(pExpr,EP_ConstFunc); } if( (pDef->funcFlags & SQLITE_FUNC_CONSTANT)==0 ){ - /* Date/time functions that use 'now', and other functions like + /* Clearly non-deterministic functions like random(), but also + ** date/time functions that use 'now', and other functions like ** sqlite_version() that might change over time cannot be used - ** in an index. */ - notValid(pParse, pNC, "non-deterministic functions", NC_IdxExpr); + ** in an index or generated column. Curiously, they can be used + ** in a CHECK constraint. SQLServer, MySQL, and PostgreSQL all + ** all this. */ + sqlite3ResolveNotValid(pParse, pNC, "non-deterministic functions", + NC_IdxExpr|NC_PartIdx|NC_GenCol, 0, pExpr); + }else{ + assert( (NC_SelfRef & 0xff)==NC_SelfRef ); /* Must fit in 8 bits */ + pExpr->op2 = pNC->ncFlags & NC_SelfRef; + if( pNC->ncFlags & NC_FromDDL ) ExprSetProperty(pExpr, EP_FromDDL); + } + if( (pDef->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && pParse->nested==0 + && (pParse->db->mDbFlags & DBFLAG_InternalFunc)==0 + ){ + /* Internal-use-only functions are disallowed unless the + ** SQL is being compiled using sqlite3NestedParse() or + ** the SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test-control has be + ** used to activate internal functions for testing purposes */ + no_such_func = 1; + pDef = 0; + }else + if( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 + && !IN_RENAME_OBJECT + ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } } - if( is_agg && (pNC->ncFlags & NC_AllowAgg)==0 ){ - sqlite3ErrorMsg(pParse, "misuse of aggregate function %.*s()", nId,zId); - pNC->nErr++; - is_agg = 0; - }else if( no_such_func && pParse->db->init.busy==0 ){ - sqlite3ErrorMsg(pParse, "no such function: %.*s", nId, zId); - pNC->nErr++; - }else if( wrong_num_args ){ - sqlite3ErrorMsg(pParse,"wrong number of arguments to function %.*s()", - nId, zId); - pNC->nErr++; + + if( 0==IN_RENAME_OBJECT ){ +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( is_agg==0 || (pDef->funcFlags & SQLITE_FUNC_MINMAX) + || (pDef->xValue==0 && pDef->xInverse==0) + || (pDef->xValue && pDef->xInverse && pDef->xSFunc && pDef->xFinalize) + ); + if( pDef && pDef->xValue==0 && pWin ){ + sqlite3ErrorMsg(pParse, + "%#T() may not be used as a window function", pExpr + ); + pNC->nNcErr++; + }else if( + (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) + || (is_agg && (pDef->funcFlags&SQLITE_FUNC_WINDOW) && !pWin) + || (is_agg && pWin && (pNC->ncFlags & NC_AllowWin)==0) + ){ + const char *zType; + if( (pDef->funcFlags & SQLITE_FUNC_WINDOW) || pWin ){ + zType = "window"; + }else{ + zType = "aggregate"; + } + sqlite3ErrorMsg(pParse, "misuse of %s function %#T()",zType,pExpr); + pNC->nNcErr++; + is_agg = 0; + } +#else + if( (is_agg && (pNC->ncFlags & NC_AllowAgg)==0) ){ + sqlite3ErrorMsg(pParse,"misuse of aggregate function %#T()",pExpr); + pNC->nNcErr++; + is_agg = 0; + } +#endif + else if( no_such_func && pParse->db->init.busy==0 +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + && pParse->explain==0 +#endif + ){ + sqlite3ErrorMsg(pParse, "no such function: %#T", pExpr); + pNC->nNcErr++; + }else if( wrong_num_args ){ + sqlite3ErrorMsg(pParse,"wrong number of arguments to function %#T()", + pExpr); + pNC->nNcErr++; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else if( is_agg==0 && ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ErrorMsg(pParse, + "FILTER may not be used with non-aggregate %#T()", + pExpr + ); + pNC->nNcErr++; + } +#endif + else if( is_agg==0 && pExpr->pLeft ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + pNC->nNcErr++; + } + if( is_agg ){ + /* Window functions may not be arguments of aggregate functions. + ** Or arguments of other window functions. But aggregate functions + ** may be arguments for window functions. */ +#ifndef SQLITE_OMIT_WINDOWFUNC + pNC->ncFlags &= ~(NC_AllowWin | (!pWin ? NC_AllowAgg : 0)); +#else + pNC->ncFlags &= ~NC_AllowAgg; +#endif + } + } + else if( ExprHasProperty(pExpr, EP_WinFunc) || pExpr->pLeft ){ + is_agg = 1; } - if( is_agg ) pNC->ncFlags &= ~NC_AllowAgg; sqlite3WalkExprList(pWalker, pList); if( is_agg ){ - NameContext *pNC2 = pNC; - pExpr->op = TK_AGG_FUNCTION; - pExpr->op2 = 0; - while( pNC2 && !sqlite3FunctionUsesThisSrc(pExpr, pNC2->pSrcList) ){ - pExpr->op2++; - pNC2 = pNC2->pNext; + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3WalkExprList(pWalker, pExpr->pLeft->x.pList); } - assert( pDef!=0 ); - if( pNC2 ){ - assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); - testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); - pNC2->ncFlags |= NC_HasAgg | (pDef->funcFlags & SQLITE_FUNC_MINMAX); - +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin && pParse->nErr==0 ){ + Select *pSel = pNC->pWinSelect; + assert( ExprUseYWin(pExpr) && pWin==pExpr->y.pWin ); + if( IN_RENAME_OBJECT==0 ){ + sqlite3WindowUpdate(pParse, pSel ? pSel->pWinDefn : 0, pWin, pDef); + if( pParse->db->mallocFailed ) break; + } + sqlite3WalkExprList(pWalker, pWin->pPartition); + sqlite3WalkExprList(pWalker, pWin->pOrderBy); + sqlite3WalkExpr(pWalker, pWin->pFilter); + sqlite3WindowLink(pSel, pWin); + pNC->ncFlags |= NC_HasWin; + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + NameContext *pNC2; /* For looping up thru outer contexts */ + pExpr->op = TK_AGG_FUNCTION; + pExpr->op2 = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(pWalker, pExpr->y.pWin->pFilter); + } +#endif + pNC2 = pNC; + while( pNC2 + && sqlite3ReferencesSrcList(pParse, pExpr, pNC2->pSrcList)==0 + ){ + pExpr->op2 += (1 + pNC2->nNestedSelect); + pNC2 = pNC2->pNext; + } + assert( pDef!=0 || IN_RENAME_OBJECT ); + if( pNC2 && pDef ){ + pExpr->op2 += pNC2->nNestedSelect; + assert( SQLITE_FUNC_MINMAX==NC_MinMaxAgg ); + assert( SQLITE_FUNC_ANYORDER==NC_OrderAgg ); + testcase( (pDef->funcFlags & SQLITE_FUNC_MINMAX)!=0 ); + testcase( (pDef->funcFlags & SQLITE_FUNC_ANYORDER)!=0 ); + pNC2->ncFlags |= NC_HasAgg + | ((pDef->funcFlags^SQLITE_FUNC_ANYORDER) + & (SQLITE_FUNC_MINMAX|SQLITE_FUNC_ANYORDER)); + } } - pNC->ncFlags |= NC_AllowAgg; + pNC->ncFlags |= savedAllowFlags; } /* FIX ME: Compute pExpr->affinity based on the expected return - ** type of the function + ** type of the function */ return WRC_Prune; } @@ -88213,24 +108441,92 @@ static int resolveExprStep(Walker *pWalker, Expr *pExpr){ #endif case TK_IN: { testcase( pExpr->op==TK_IN ); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( ExprUseXSelect(pExpr) ){ int nRef = pNC->nRef; - notValid(pParse, pNC, "subqueries", NC_IsCheck|NC_PartIdx|NC_IdxExpr); - sqlite3WalkSelect(pWalker, pExpr->x.pSelect); + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + assert( pExpr->x.pSelect ); + if( pNC->ncFlags & NC_SelfRef ){ + notValidImpl(pParse, pNC, "subqueries", pExpr, pExpr); + }else{ + sqlite3WalkSelect(pWalker, pExpr->x.pSelect); + } assert( pNC->nRef>=nRef ); if( nRef!=pNC->nRef ){ ExprSetProperty(pExpr, EP_VarSelect); - pNC->ncFlags |= NC_VarSelect; + pExpr->x.pSelect->selFlags |= SF_Correlated; } + pNC->ncFlags |= NC_Subquery; } break; } case TK_VARIABLE: { - notValid(pParse, pNC, "parameters", NC_IsCheck|NC_PartIdx|NC_IdxExpr); + testcase( pNC->ncFlags & NC_IsCheck ); + testcase( pNC->ncFlags & NC_PartIdx ); + testcase( pNC->ncFlags & NC_IdxExpr ); + testcase( pNC->ncFlags & NC_GenCol ); + sqlite3ResolveNotValid(pParse, pNC, "parameters", + NC_IsCheck|NC_PartIdx|NC_IdxExpr|NC_GenCol, pExpr, pExpr); + break; + } + case TK_IS: + case TK_ISNOT: { + Expr *pRight = sqlite3ExprSkipCollateAndLikely(pExpr->pRight); + assert( !ExprHasProperty(pExpr, EP_Reduced) ); + /* Handle special cases of "x IS TRUE", "x IS FALSE", "x IS NOT TRUE", + ** and "x IS NOT FALSE". */ + if( ALWAYS(pRight) && (pRight->op==TK_ID || pRight->op==TK_TRUEFALSE) ){ + int rc = resolveExprStep(pWalker, pRight); + if( rc==WRC_Abort ) return WRC_Abort; + if( pRight->op==TK_TRUEFALSE ){ + pExpr->op2 = pExpr->op; + pExpr->op = TK_TRUTH; + return WRC_Continue; + } + } + /* no break */ deliberate_fall_through + } + case TK_BETWEEN: + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: { + int nLeft, nRight; + if( pParse->db->mallocFailed ) break; + assert( pExpr->pLeft!=0 ); + nLeft = sqlite3ExprVectorSize(pExpr->pLeft); + if( pExpr->op==TK_BETWEEN ){ + assert( ExprUseXList(pExpr) ); + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[0].pExpr); + if( nRight==nLeft ){ + nRight = sqlite3ExprVectorSize(pExpr->x.pList->a[1].pExpr); + } + }else{ + assert( pExpr->pRight!=0 ); + nRight = sqlite3ExprVectorSize(pExpr->pRight); + } + if( nLeft!=nRight ){ + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_BETWEEN ); + sqlite3ErrorMsg(pParse, "row value misused"); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); + } break; } } - return (pParse->nErr || pParse->db->mallocFailed) ? WRC_Abort : WRC_Continue; + assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + return pParse->nErr ? WRC_Abort : WRC_Continue; } /* @@ -88255,10 +108551,13 @@ static int resolveAsName( UNUSED_PARAMETER(pParse); if( pE->op==TK_ID ){ - char *zCol = pE->u.zToken; + const char *zCol; + assert( !ExprHasProperty(pE, EP_IntValue) ); + zCol = pE->u.zToken; for(i=0; inExpr; i++){ - char *zAs = pEList->a[i].zName; - if( zAs!=0 && sqlite3StrICmp(zAs, zCol)==0 ){ + if( pEList->a[i].fg.eEName==ENAME_NAME + && sqlite3_stricmp(pEList->a[i].zEName, zCol)==0 + ){ return i+1; } } @@ -88296,7 +108595,7 @@ static int resolveOrderByTermToExprList( int rc; /* Return code from subprocedures */ u8 savedSuppErr; /* Saved value of db->suppressErr */ - assert( sqlite3ExprIsInteger(pE, &i)==0 ); + assert( sqlite3ExprIsInteger(pE, &i, 0)==0 ); pEList = pSelect->pEList; /* Resolve all names in the ORDER BY term expression @@ -88304,9 +108603,9 @@ static int resolveOrderByTermToExprList( memset(&nc, 0, sizeof(nc)); nc.pParse = pParse; nc.pSrcList = pSelect->pSrc; - nc.pEList = pEList; - nc.ncFlags = NC_AllowAgg; - nc.nErr = 0; + nc.uNC.pEList = pEList; + nc.ncFlags = NC_AllowAgg|NC_UEList|NC_NoSelect; + nc.nNcErr = 0; db = pParse->db; savedSuppErr = db->suppressErr; db->suppressErr = 1; @@ -88319,7 +108618,7 @@ static int resolveOrderByTermToExprList( ** result-set entry. */ for(i=0; inExpr; i++){ - if( sqlite3ExprCompare(pEList->a[i].pExpr, pE, -1)<2 ){ + if( sqlite3ExprCompare(0, pEList->a[i].pExpr, pE, -1)<2 ){ return i+1; } } @@ -88335,11 +108634,13 @@ static void resolveOutOfRangeError( Parse *pParse, /* The error context into which to write the error */ const char *zType, /* "ORDER" or "GROUP" */ int i, /* The index (1-based) of the term out of range */ - int mx /* Largest permissible value of i */ + int mx, /* Largest permissible value of i */ + Expr *pError /* Associate the error with the expression */ ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "%r %s BY term out of range - should be " "between 1 and %d", i, zType, mx); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pError); } /* @@ -88370,14 +108671,12 @@ static int resolveCompoundOrderBy( pOrderBy = pSelect->pOrderBy; if( pOrderBy==0 ) return 0; db = pParse->db; -#if SQLITE_MAX_COLUMN if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in ORDER BY clause"); return 1; } -#endif for(i=0; inExpr; i++){ - pOrderBy->a[i].done = 0; + pOrderBy->a[i].fg.done = 0; } pSelect->pNext = 0; while( pSelect->pPrior ){ @@ -88392,43 +108691,60 @@ static int resolveCompoundOrderBy( for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ int iCol = -1; Expr *pE, *pDup; - if( pItem->done ) continue; - pE = sqlite3ExprSkipCollate(pItem->pExpr); - if( sqlite3ExprIsInteger(pE, &iCol) ){ + if( pItem->fg.done ) continue; + pE = sqlite3ExprSkipCollateAndLikely(pItem->pExpr); + if( NEVER(pE==0) ) continue; + if( sqlite3ExprIsInteger(pE, &iCol, 0) ){ if( iCol<=0 || iCol>pEList->nExpr ){ - resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr); + resolveOutOfRangeError(pParse, "ORDER", i+1, pEList->nExpr, pE); return 1; } }else{ iCol = resolveAsName(pParse, pEList, pE); if( iCol==0 ){ + /* Now test if expression pE matches one of the values returned + ** by pSelect. In the usual case this is done by duplicating the + ** expression, resolving any symbols in it, and then comparing + ** it against each expression returned by the SELECT statement. + ** Once the comparisons are finished, the duplicate expression + ** is deleted. + ** + ** If this is running as part of an ALTER TABLE operation and + ** the symbols resolve successfully, also resolve the symbols in the + ** actual expression. This allows the code in alter.c to modify + ** column references within the ORDER BY expression as required. */ pDup = sqlite3ExprDup(db, pE, 0); if( !db->mallocFailed ){ assert(pDup); iCol = resolveOrderByTermToExprList(pParse, pSelect, pDup); + if( IN_RENAME_OBJECT && iCol>0 ){ + resolveOrderByTermToExprList(pParse, pSelect, pE); + } } sqlite3ExprDelete(db, pDup); } } if( iCol>0 ){ /* Convert the ORDER BY term into an integer column number iCol, - ** taking care to preserve the COLLATE clause if it exists */ - Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); - if( pNew==0 ) return 1; - pNew->flags |= EP_IntValue; - pNew->u.iValue = iCol; - if( pItem->pExpr==pE ){ - pItem->pExpr = pNew; - }else{ - Expr *pParent = pItem->pExpr; - assert( pParent->op==TK_COLLATE ); - while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; - assert( pParent->pLeft==pE ); - pParent->pLeft = pNew; + ** taking care to preserve the COLLATE clause if it exists. */ + if( !IN_RENAME_OBJECT ){ + Expr *pNew = sqlite3Expr(db, TK_INTEGER, 0); + if( pNew==0 ) return 1; + pNew->flags |= EP_IntValue; + pNew->u.iValue = iCol; + if( pItem->pExpr==pE ){ + pItem->pExpr = pNew; + }else{ + Expr *pParent = pItem->pExpr; + assert( pParent->op==TK_COLLATE ); + while( pParent->pLeft->op==TK_COLLATE ) pParent = pParent->pLeft; + assert( pParent->pLeft==pE ); + pParent->pLeft = pNew; + } + sqlite3ExprDelete(db, pE); + pItem->u.x.iOrderByCol = (u16)iCol; } - sqlite3ExprDelete(db, pE); - pItem->u.x.iOrderByCol = (u16)iCol; - pItem->done = 1; + pItem->fg.done = 1; }else{ moreToDo = 1; } @@ -88436,7 +108752,7 @@ static int resolveCompoundOrderBy( pSelect = pSelect->pNext; } for(i=0; inExpr; i++){ - if( pOrderBy->a[i].done==0 ){ + if( pOrderBy->a[i].fg.done==0 ){ sqlite3ErrorMsg(pParse, "%r ORDER BY term does not match any " "column in the result set", i+1); return 1; @@ -88466,28 +108782,55 @@ SQLITE_PRIVATE int sqlite3ResolveOrderGroupBy( ExprList *pEList; struct ExprList_item *pItem; - if( pOrderBy==0 || pParse->db->mallocFailed ) return 0; -#if SQLITE_MAX_COLUMN + if( pOrderBy==0 || pParse->db->mallocFailed || IN_RENAME_OBJECT ) return 0; if( pOrderBy->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many terms in %s BY clause", zType); return 1; } -#endif pEList = pSelect->pEList; assert( pEList!=0 ); /* sqlite3SelectNew() guarantees this */ for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ if( pItem->u.x.iOrderByCol ){ if( pItem->u.x.iOrderByCol>pEList->nExpr ){ - resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr); + resolveOutOfRangeError(pParse, zType, i+1, pEList->nExpr, 0); return 1; } - resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr, - zType,0); + resolveAlias(pParse, pEList, pItem->u.x.iOrderByCol-1, pItem->pExpr,0); } } return 0; } +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Walker callback for windowRemoveExprFromSelect(). +*/ +static int resolveRemoveWindowsCb(Walker *pWalker, Expr *pExpr){ + UNUSED_PARAMETER(pWalker); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + sqlite3WindowUnlinkFromSelect(pWin); + } + return WRC_Continue; +} + +/* +** Remove any Window objects owned by the expression pExpr from the +** Select.pWin list of Select object pSelect. +*/ +static void windowRemoveExprFromSelect(Select *pSelect, Expr *pExpr){ + if( pSelect->pWin ){ + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.xExprCallback = resolveRemoveWindowsCb; + sWalker.u.pSelect = pSelect; + sqlite3WalkExpr(&sWalker, pExpr); + } +} +#else +# define windowRemoveExprFromSelect(a, b) +#endif /* SQLITE_OMIT_WINDOWFUNC */ + /* ** pOrderBy is an ORDER BY or GROUP BY clause in SELECT statement pSelect. ** The Name context of the SELECT statement is pNC. zType is either @@ -88518,12 +108861,13 @@ static int resolveOrderGroupBy( Parse *pParse; /* Parsing context */ int nResult; /* Number of terms in the result set */ - if( pOrderBy==0 ) return 0; + assert( pOrderBy!=0 ); nResult = pSelect->pEList->nExpr; pParse = pNC->pParse; for(i=0, pItem=pOrderBy->a; inExpr; i++, pItem++){ Expr *pE = pItem->pExpr; - Expr *pE2 = sqlite3ExprSkipCollate(pE); + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pE); + if( NEVER(pE2==0) ) continue; if( zType[0]!='G' ){ iCol = resolveAsName(pParse, pSelect->pEList, pE2); if( iCol>0 ){ @@ -88535,12 +108879,12 @@ static int resolveOrderGroupBy( continue; } } - if( sqlite3ExprIsInteger(pE2, &iCol) ){ + if( sqlite3ExprIsInteger(pE2, &iCol, 0) ){ /* The ORDER BY term is an integer constant. Again, set the column ** number so that sqlite3ResolveOrderGroupBy() will convert the ** order-by term to a copy of the result-set expression */ if( iCol<1 || iCol>0xffff ){ - resolveOutOfRangeError(pParse, zType, i+1, nResult); + resolveOutOfRangeError(pParse, zType, i+1, nResult, pE2); return 1; } pItem->u.x.iOrderByCol = (u16)iCol; @@ -88553,7 +108897,11 @@ static int resolveOrderGroupBy( return 1; } for(j=0; jpEList->nExpr; j++){ - if( sqlite3ExprCompare(pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ + if( sqlite3ExprCompare(0, pE, pSelect->pEList->a[j].pExpr, -1)==0 ){ + /* Since this expression is being changed into a reference + ** to an identical expression in the result set, remove all Window + ** objects belonging to the expression from the Select.pWin list. */ + windowRemoveExprFromSelect(pSelect, pE); pItem->u.x.iOrderByCol = j+1; } } @@ -88574,7 +108922,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ExprList *pGroupBy; /* The GROUP BY clause */ Select *pLeftmost; /* Left-most of SELECT of a compound */ sqlite3 *db; /* Database connection */ - + assert( p!=0 ); if( p->selFlags & SF_Resolved ){ @@ -88594,7 +108942,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ if( (p->selFlags & SF_Expanded)==0 ){ sqlite3SelectPrep(pParse, p, pOuterNC); - return (pParse->nErr || db->mallocFailed) ? WRC_Abort : WRC_Prune; + return pParse->nErr ? WRC_Abort : WRC_Prune; } isCompound = p->pPrior!=0; @@ -88610,8 +108958,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ */ memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; - if( sqlite3ResolveExprNames(&sNC, p->pLimit) || - sqlite3ResolveExprNames(&sNC, p->pOffset) ){ + sNC.pWinSelect = p; + if( sqlite3ResolveExprNames(&sNC, p->pLimit) ){ return WRC_Abort; } @@ -88622,69 +108970,76 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** moves the pOrderBy down to the sub-query. It will be moved back ** after the names have been resolved. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + assert( p->pSrc->a[0].u4.pSubq!=0 ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); assert( p->pSrc->nSrc==1 && p->pOrderBy ); assert( pSub->pPrior && pSub->pOrderBy==0 ); pSub->pOrderBy = p->pOrderBy; p->pOrderBy = 0; } - - /* Recursively resolve names in all subqueries + + /* Recursively resolve names in all subqueries in the FROM clause */ + if( pOuterNC ) pOuterNC->nNestedSelect++; for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; - if( pItem->pSelect ){ - NameContext *pNC; /* Used to iterate name contexts */ - int nRef = 0; /* Refcount for pOuterNC and outer contexts */ + SrcItem *pItem = &p->pSrc->a[i]; + assert( pItem->zName!=0 + || pItem->fg.isSubquery ); /* Test of tag-20240424-1*/ + if( pItem->fg.isSubquery + && (pItem->u4.pSubq->pSelect->selFlags & SF_Resolved)==0 + ){ + int nRef = pOuterNC ? pOuterNC->nRef : 0; const char *zSavedContext = pParse->zAuthContext; - /* Count the total number of references to pOuterNC and all of its - ** parent contexts. After resolving references to expressions in - ** pItem->pSelect, check if this value has changed. If so, then - ** SELECT statement pItem->pSelect must be correlated. Set the - ** pItem->fg.isCorrelated flag if this is the case. */ - for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef += pNC->nRef; - if( pItem->zName ) pParse->zAuthContext = pItem->zName; - sqlite3ResolveSelectNames(pParse, pItem->pSelect, pOuterNC); + sqlite3ResolveSelectNames(pParse, pItem->u4.pSubq->pSelect, pOuterNC); pParse->zAuthContext = zSavedContext; - if( pParse->nErr || db->mallocFailed ) return WRC_Abort; + if( pParse->nErr ) return WRC_Abort; + assert( db->mallocFailed==0 ); - for(pNC=pOuterNC; pNC; pNC=pNC->pNext) nRef -= pNC->nRef; - assert( pItem->fg.isCorrelated==0 && nRef<=0 ); - pItem->fg.isCorrelated = (nRef!=0); + /* If the number of references to the outer context changed when + ** expressions in the sub-select were resolved, the sub-select + ** is correlated. It is not required to check the refcount on any + ** but the innermost outer context object, as lookupName() increments + ** the refcount on all contexts between the current one and the + ** context containing the column when it resolves a name. */ + if( pOuterNC ){ + assert( pItem->fg.isCorrelated==0 && pOuterNC->nRef>=nRef ); + pItem->fg.isCorrelated = (pOuterNC->nRef>nRef); + } } } - + if( pOuterNC && ALWAYS(pOuterNC->nNestedSelect>0) ){ + pOuterNC->nNestedSelect--; + } + /* Set up the local name-context to pass to sqlite3ResolveExprNames() to ** resolve the result-set expression list. */ - sNC.ncFlags = NC_AllowAgg; + sNC.ncFlags = NC_AllowAgg|NC_AllowWin; sNC.pSrcList = p->pSrc; sNC.pNext = pOuterNC; - + /* Resolve names in the result set. */ if( sqlite3ResolveExprListNames(&sNC, p->pEList) ) return WRC_Abort; - - /* If there are no aggregate functions in the result-set, and no GROUP BY + sNC.ncFlags &= ~NC_AllowWin; + + /* If there are no aggregate functions in the result-set, and no GROUP BY ** expression, do not allow aggregates in any of the other expressions. */ assert( (p->selFlags & SF_Aggregate)==0 ); pGroupBy = p->pGroupBy; if( pGroupBy || (sNC.ncFlags & NC_HasAgg)!=0 ){ assert( NC_MinMaxAgg==SF_MinMaxAgg ); - p->selFlags |= SF_Aggregate | (sNC.ncFlags&NC_MinMaxAgg); + assert( NC_OrderAgg==SF_OrderByReqd ); + p->selFlags |= SF_Aggregate | (sNC.ncFlags&(NC_MinMaxAgg|NC_OrderAgg)); }else{ sNC.ncFlags &= ~NC_AllowAgg; } - - /* If a HAVING clause is present, then there must be a GROUP BY clause. - */ - if( p->pHaving && !pGroupBy ){ - sqlite3ErrorMsg(pParse, "a GROUP BY clause is required before HAVING"); - return WRC_Abort; - } - + /* Add the output column list to the name-context before parsing the ** other expressions in the SELECT statement. This is so that ** expressions in the WHERE clause (etc.) can refer to expressions by @@ -88693,33 +109048,59 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** Minor point: If this is the case, then the expression will be ** re-evaluated for each reference to it. */ - sNC.pEList = p->pEList; - if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + assert( (sNC.ncFlags & (NC_UAggInfo|NC_UUpsert|NC_UBaseReg))==0 ); + sNC.uNC.pEList = p->pEList; + sNC.ncFlags |= NC_UEList; + if( p->pHaving ){ + if( (p->selFlags & SF_Aggregate)==0 ){ + sqlite3ErrorMsg(pParse, "HAVING clause on a non-aggregate query"); + return WRC_Abort; + } + if( sqlite3ResolveExprNames(&sNC, p->pHaving) ) return WRC_Abort; + } + sNC.ncFlags |= NC_Where; if( sqlite3ResolveExprNames(&sNC, p->pWhere) ) return WRC_Abort; + sNC.ncFlags &= ~NC_Where; /* Resolve names in table-valued-function arguments */ for(i=0; ipSrc->nSrc; i++){ - struct SrcList_item *pItem = &p->pSrc->a[i]; + SrcItem *pItem = &p->pSrc->a[i]; if( pItem->fg.isTabFunc - && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) + && sqlite3ResolveExprListNames(&sNC, pItem->u1.pFuncArg) ){ return WRC_Abort; } } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( IN_RENAME_OBJECT ){ + Window *pWin; + for(pWin=p->pWinDefn; pWin; pWin=pWin->pNextWin){ + if( sqlite3ResolveExprListNames(&sNC, pWin->pOrderBy) + || sqlite3ResolveExprListNames(&sNC, pWin->pPartition) + ){ + return WRC_Abort; + } + } + } +#endif + /* The ORDER BY and GROUP BY clauses may not refer to terms in - ** outer queries + ** outer queries */ sNC.pNext = 0; - sNC.ncFlags |= NC_AllowAgg; + sNC.ncFlags |= NC_AllowAgg|NC_AllowWin; - /* If this is a converted compound query, move the ORDER BY clause from + /* If this is a converted compound query, move the ORDER BY clause from ** the sub-query back to the parent query. At this point each term ** within the ORDER BY clause has been transformed to an integer value. ** These integers will be replaced by copies of the corresponding result ** set expressions by the call to resolveOrderGroupBy() below. */ if( p->selFlags & SF_Converted ){ - Select *pSub = p->pSrc->a[0].pSelect; + Select *pSub; + assert( p->pSrc->a[0].fg.isSubquery ); + pSub = p->pSrc->a[0].u4.pSubq->pSelect; + assert( pSub!=0 ); p->pOrderBy = pSub->pOrderBy; pSub->pOrderBy = 0; } @@ -88734,7 +109115,8 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** is not detected until much later, and so we need to go ahead and ** resolve those symbols on the incorrect ORDER BY for consistency. */ - if( isCompound<=nCompound /* Defer right-most ORDER BY of a compound */ + if( p->pOrderBy!=0 + && isCompound<=nCompound /* Defer right-most ORDER BY of a compound */ && resolveOrderGroupBy(&sNC, p, p->pOrderBy, "ORDER") ){ return WRC_Abort; @@ -88742,13 +109124,14 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ if( db->mallocFailed ){ return WRC_Abort; } - - /* Resolve the GROUP BY clause. At the same time, make sure + sNC.ncFlags &= ~NC_AllowWin; + + /* Resolve the GROUP BY clause. At the same time, make sure ** the GROUP BY clause does not contain aggregate functions. */ if( pGroupBy ){ struct ExprList_item *pItem; - + if( resolveOrderGroupBy(&sNC, p, pGroupBy, "GROUP") || db->mallocFailed ){ return WRC_Abort; } @@ -88790,7 +109173,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** checking on function usage and set a flag if any aggregate functions ** are seen. ** -** To resolve table columns references we look for nodes (or subtrees) of the +** To resolve table columns references we look for nodes (or subtrees) of the ** form X.Y.Z or Y.Z or just Z where ** ** X: The name of a database. Ex: "main" or "temp" or @@ -88822,7 +109205,7 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** ** SELECT a+b AS x, c+d AS y FROM t1 ORDER BY a+b; ** -** Function calls are checked to make sure that the function is +** Function calls are checked to make sure that the function is ** defined and that the correct number of arguments are specified. ** If the function is an aggregate function, then the NC_HasAgg flag is ** set and the opcode is changed from TK_FUNCTION to TK_AGG_FUNCTION. @@ -88832,67 +109215,96 @@ static int resolveSelectStep(Walker *pWalker, Select *p){ ** An error message is left in pParse if anything is amiss. The number ** if errors is returned. */ -SQLITE_PRIVATE int sqlite3ResolveExprNames( +SQLITE_PRIVATE int sqlite3ResolveExprNames( NameContext *pNC, /* Namespace to resolve expressions in. */ Expr *pExpr /* The expression to be analyzed. */ ){ - u16 savedHasAgg; + int savedHasAgg; Walker w; - if( pExpr==0 ) return 0; -#if SQLITE_MAX_EXPR_DEPTH>0 - { - Parse *pParse = pNC->pParse; - if( sqlite3ExprCheckHeight(pParse, pExpr->nHeight+pNC->pParse->nHeight) ){ - return 1; - } - pParse->nHeight += pExpr->nHeight; - } -#endif - savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg); - pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg); + if( pExpr==0 ) return SQLITE_OK; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); w.pParse = pNC->pParse; w.xExprCallback = resolveExprStep; - w.xSelectCallback = resolveSelectStep; + w.xSelectCallback = (pNC->ncFlags & NC_NoSelect) ? 0 : resolveSelectStep; w.xSelectCallback2 = 0; - w.walkerDepth = 0; - w.eCode = 0; w.u.pNC = pNC; - sqlite3WalkExpr(&w, pExpr); #if SQLITE_MAX_EXPR_DEPTH>0 - pNC->pParse->nHeight -= pExpr->nHeight; + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return SQLITE_ERROR; + } #endif - if( pNC->nErr>0 || w.pParse->nErr>0 ){ - ExprSetProperty(pExpr, EP_Error); - } - if( pNC->ncFlags & NC_HasAgg ){ - ExprSetProperty(pExpr, EP_Agg); - } + assert( pExpr!=0 ); + sqlite3WalkExprNN(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); pNC->ncFlags |= savedHasAgg; - return ExprHasProperty(pExpr, EP_Error); + return pNC->nNcErr>0 || w.pParse->nErr>0; } /* ** Resolve all names for all expression in an expression list. This is ** just like sqlite3ResolveExprNames() except that it works for an expression ** list rather than a single expression. +** +** The return value is SQLITE_OK (0) for success or SQLITE_ERROR (1) for a +** failure. */ -SQLITE_PRIVATE int sqlite3ResolveExprListNames( +SQLITE_PRIVATE int sqlite3ResolveExprListNames( NameContext *pNC, /* Namespace to resolve expressions in. */ ExprList *pList /* The expression list to be analyzed. */ ){ int i; - if( pList ){ - for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(pNC, pList->a[i].pExpr) ) return WRC_Abort; + int savedHasAgg = 0; + Walker w; + if( pList==0 ) return SQLITE_OK; + w.pParse = pNC->pParse; + w.xExprCallback = resolveExprStep; + w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; + w.u.pNC = pNC; + savedHasAgg = pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + if( pExpr==0 ) continue; +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight += pExpr->nHeight; + if( sqlite3ExprCheckHeight(w.pParse, w.pParse->nHeight) ){ + return SQLITE_ERROR; } +#endif + sqlite3WalkExprNN(&w, pExpr); +#if SQLITE_MAX_EXPR_DEPTH>0 + w.pParse->nHeight -= pExpr->nHeight; +#endif + assert( EP_Agg==NC_HasAgg ); + assert( EP_Win==NC_HasWin ); + testcase( pNC->ncFlags & NC_HasAgg ); + testcase( pNC->ncFlags & NC_HasWin ); + if( pNC->ncFlags & (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg) ){ + ExprSetProperty(pExpr, pNC->ncFlags & (NC_HasAgg|NC_HasWin) ); + savedHasAgg |= pNC->ncFlags & + (NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + pNC->ncFlags &= ~(NC_HasAgg|NC_MinMaxAgg|NC_HasWin|NC_OrderAgg); + } + if( w.pParse->nErr>0 ) return SQLITE_ERROR; } - return WRC_Continue; + pNC->ncFlags |= savedHasAgg; + return SQLITE_OK; } /* ** Resolve all names in all expressions of a SELECT and in all -** decendents of the SELECT, including compounds off of p->pPrior, +** descendants of the SELECT, including compounds off of p->pPrior, ** subqueries in expressions, and subqueries used as FROM clause ** terms. ** @@ -88910,47 +109322,65 @@ SQLITE_PRIVATE void sqlite3ResolveSelectNames( Walker w; assert( p!=0 ); - memset(&w, 0, sizeof(w)); w.xExprCallback = resolveExprStep; w.xSelectCallback = resolveSelectStep; + w.xSelectCallback2 = 0; w.pParse = pParse; w.u.pNC = pOuterNC; sqlite3WalkSelect(&w, p); } /* -** Resolve names in expressions that can only reference a single table: +** Resolve names in expressions that can only reference a single table +** or which cannot reference any tables at all. Examples: ** -** * CHECK constraints -** * WHERE clauses on partial indices +** "type" flag +** ------------ +** (1) CHECK constraints NC_IsCheck +** (2) WHERE clauses on partial indices NC_PartIdx +** (3) Expressions in indexes on expressions NC_IdxExpr +** (4) Expression arguments to VACUUM INTO. 0 +** (5) GENERATED ALWAYS as expressions NC_GenCol ** -** The Expr.iTable value for Expr.op==TK_COLUMN nodes of the expression -** is set to -1 and the Expr.iColumn value is set to the column number. +** In all cases except (4), the Expr.iTable value for Expr.op==TK_COLUMN +** nodes of the expression is set to -1 and the Expr.iColumn value is +** set to the column number. In case (4), TK_COLUMN nodes cause an error. ** ** Any errors cause an error message to be set in pParse. */ -SQLITE_PRIVATE void sqlite3ResolveSelfReference( - Parse *pParse, /* Parsing context */ - Table *pTab, /* The table being referenced */ - int type, /* NC_IsCheck or NC_PartIdx or NC_IdxExpr */ - Expr *pExpr, /* Expression to resolve. May be NULL. */ - ExprList *pList /* Expression list to resolve. May be NUL. */ +SQLITE_PRIVATE int sqlite3ResolveSelfReference( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table being referenced, or NULL */ + int type, /* NC_IsCheck, NC_PartIdx, NC_IdxExpr, NC_GenCol, or 0 */ + Expr *pExpr, /* Expression to resolve. May be NULL. */ + ExprList *pList /* Expression list to resolve. May be NULL. */ ){ SrcList sSrc; /* Fake SrcList for pParse->pNewTable */ NameContext sNC; /* Name context for pParse->pNewTable */ + int rc; - assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr ); + assert( type==0 || pTab!=0 ); + assert( type==NC_IsCheck || type==NC_PartIdx || type==NC_IdxExpr + || type==NC_GenCol || pTab==0 ); memset(&sNC, 0, sizeof(sNC)); memset(&sSrc, 0, sizeof(sSrc)); - sSrc.nSrc = 1; - sSrc.a[0].zName = pTab->zName; - sSrc.a[0].pTab = pTab; - sSrc.a[0].iCursor = -1; + if( pTab ){ + sSrc.nSrc = 1; + sSrc.a[0].zName = pTab->zName; + sSrc.a[0].pSTab = pTab; + sSrc.a[0].iCursor = -1; + if( pTab->pSchema!=pParse->db->aDb[1].pSchema ){ + /* Cause EP_FromDDL to be set on TK_FUNCTION nodes of non-TEMP + ** schema elements */ + type |= NC_FromDDL; + } + } sNC.pParse = pParse; sNC.pSrcList = &sSrc; - sNC.ncFlags = type; - if( sqlite3ResolveExprNames(&sNC, pExpr) ) return; - if( pList ) sqlite3ResolveExprListNames(&sNC, pList); + sNC.ncFlags = type | NC_IsDDL; + if( (rc = sqlite3ResolveExprNames(&sNC, pExpr))!=SQLITE_OK ) return rc; + if( pList ) rc = sqlite3ResolveExprListNames(&sNC, pList); + return rc; } /************** End of resolve.c *********************************************/ @@ -88971,11 +109401,23 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( */ /* #include "sqliteInt.h" */ +/* Forward declarations */ +static void exprCodeBetween(Parse*,Expr*,int,void(*)(Parse*,Expr*,int,int),int); +static int exprCodeVector(Parse *pParse, Expr *p, int *piToFree); + +/* +** Return the affinity character for a single column of a table. +*/ +SQLITE_PRIVATE char sqlite3TableColumnAffinity(const Table *pTab, int iCol){ + if( iCol<0 || NEVER(iCol>=pTab->nCol) ) return SQLITE_AFF_INTEGER; + return pTab->aCol[iCol].affinity; +} + /* ** Return the 'affinity' of the expression pExpr if any. ** ** If pExpr is a column, a reference to a column via an 'AS' alias, -** or a sub-select with a column as the return value, then the +** or a sub-select with a column as the return value, then the ** affinity of that column is returned. Otherwise, 0x00 is returned, ** indicating no affinity for the expression. ** @@ -88987,32 +109429,124 @@ SQLITE_PRIVATE void sqlite3ResolveSelfReference( ** SELECT a AS b FROM t1 WHERE b; ** SELECT * FROM t1 WHERE (select a from t1); */ -SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ +SQLITE_PRIVATE char sqlite3ExprAffinity(const Expr *pExpr){ int op; - pExpr = sqlite3ExprSkipCollate(pExpr); - if( pExpr->flags & EP_Generic ) return 0; op = pExpr->op; - if( op==TK_SELECT ){ - assert( pExpr->flags&EP_xIsSelect ); - return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); - } + while( 1 /* exit-by-break */ ){ + if( op==TK_COLUMN || (op==TK_AGG_COLUMN && pExpr->y.pTab!=0) ){ + assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); + return sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + } + if( op==TK_SELECT ){ + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( pExpr->x.pSelect->pEList!=0 ); + assert( pExpr->x.pSelect->pEList->a[0].pExpr!=0 ); + return sqlite3ExprAffinity(pExpr->x.pSelect->pEList->a[0].pExpr); + } #ifndef SQLITE_OMIT_CAST - if( op==TK_CAST ){ - assert( !ExprHasProperty(pExpr, EP_IntValue) ); - return sqlite3AffinityType(pExpr->u.zToken, 0); - } + if( op==TK_CAST ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + return sqlite3AffinityType(pExpr->u.zToken, 0); + } #endif - if( (op==TK_AGG_COLUMN || op==TK_COLUMN || op==TK_REGISTER) - && pExpr->pTab!=0 - ){ - /* op==TK_REGISTER && pExpr->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = pExpr->iColumn; - if( j<0 ) return SQLITE_AFF_INTEGER; - assert( pExpr->pTab && jpTab->nCol ); - return pExpr->pTab->aCol[j].affinity; + if( op==TK_SELECT_COLUMN ){ + assert( pExpr->pLeft!=0 && ExprUseXSelect(pExpr->pLeft) ); + assert( pExpr->iColumn < pExpr->iTable ); + assert( pExpr->iColumn >= 0 ); + assert( pExpr->iTable==pExpr->pLeft->x.pSelect->pEList->nExpr ); + return sqlite3ExprAffinity( + pExpr->pLeft->x.pSelect->pEList->a[pExpr->iColumn].pExpr + ); + } + if( op==TK_VECTOR ){ + assert( ExprUseXList(pExpr) ); + return sqlite3ExprAffinity(pExpr->x.pList->a[0].pExpr); + } + if( ExprHasProperty(pExpr, EP_Skip|EP_IfNullRow) ){ + assert( pExpr->op==TK_COLLATE + || pExpr->op==TK_IF_NULL_ROW + || (pExpr->op==TK_REGISTER && pExpr->op2==TK_IF_NULL_ROW) ); + pExpr = pExpr->pLeft; + op = pExpr->op; + continue; + } + if( op!=TK_REGISTER ) break; + op = pExpr->op2; + if( NEVER( op==TK_REGISTER ) ) break; } - return pExpr->affinity; + return pExpr->affExpr; +} + +/* +** Make a guess at all the possible datatypes of the result that could +** be returned by an expression. Return a bitmask indicating the answer: +** +** 0x01 Numeric +** 0x02 Text +** 0x04 Blob +** +** If the expression must return NULL, then 0x00 is returned. +*/ +SQLITE_PRIVATE int sqlite3ExprDataType(const Expr *pExpr){ + while( pExpr ){ + switch( pExpr->op ){ + case TK_COLLATE: + case TK_IF_NULL_ROW: + case TK_UPLUS: { + pExpr = pExpr->pLeft; + break; + } + case TK_NULL: { + pExpr = 0; + break; + } + case TK_STRING: { + return 0x02; + } + case TK_BLOB: { + return 0x04; + } + case TK_CONCAT: { + return 0x06; + } + case TK_VARIABLE: + case TK_AGG_FUNCTION: + case TK_FUNCTION: { + return 0x07; + } + case TK_COLUMN: + case TK_AGG_COLUMN: + case TK_SELECT: + case TK_CAST: + case TK_SELECT_COLUMN: + case TK_VECTOR: { + int aff = sqlite3ExprAffinity(pExpr); + if( aff>=SQLITE_AFF_NUMERIC ) return 0x05; + if( aff==SQLITE_AFF_TEXT ) return 0x06; + return 0x07; + } + case TK_CASE: { + int res = 0; + int ii; + ExprList *pList = pExpr->x.pList; + assert( ExprUseXList(pExpr) && pList!=0 ); + assert( pList->nExpr > 0); + for(ii=1; iinExpr; ii+=2){ + res |= sqlite3ExprDataType(pList->a[ii].pExpr); + } + if( pList->nExpr % 2 ){ + res |= sqlite3ExprDataType(pList->a[pList->nExpr-1].pExpr); + } + return res; + } + default: { + return 0x01; + } + } /* End of switch(op) */ + } /* End of while(pExpr) */ + return 0x00; } /* @@ -89024,7 +109558,7 @@ SQLITE_PRIVATE char sqlite3ExprAffinity(Expr *pExpr){ ** and the pExpr parameter is returned unchanged. */ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( - Parse *pParse, /* Parsing context */ + const Parse *pParse, /* Parsing context */ Expr *pExpr, /* Add the "COLLATE" clause to this expression */ const Token *pCollName, /* Name of collating sequence */ int dequote /* True to dequote pCollName */ @@ -89039,7 +109573,11 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateToken( } return pExpr; } -SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, const char *zC){ +SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString( + const Parse *pParse, /* Parsing context */ + Expr *pExpr, /* Add the "COLLATE" clause to this expression */ + const char *zC /* The collating sequence name */ +){ Token s; assert( zC!=0 ); sqlite3TokenInit(&s, (char*)zC); @@ -89047,21 +109585,34 @@ SQLITE_PRIVATE Expr *sqlite3ExprAddCollateString(Parse *pParse, Expr *pExpr, con } /* -** Skip over any TK_COLLATE operators and any unlikely() -** or likelihood() function at the root of an expression. +** Skip over any TK_COLLATE operators. */ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ while( pExpr && ExprHasProperty(pExpr, EP_Skip) ){ + assert( pExpr->op==TK_COLLATE ); + pExpr = pExpr->pLeft; + } + return pExpr; +} + +/* +** Skip over any TK_COLLATE operators and/or any unlikely() +** or likelihood() or likely() functions at the root of an +** expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSkipCollateAndLikely(Expr *pExpr){ + while( pExpr && ExprHasProperty(pExpr, EP_Skip|EP_Unlikely) ){ if( ExprHasProperty(pExpr, EP_Unlikely) ){ - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( ExprUseXList(pExpr) ); assert( pExpr->x.pList->nExpr>0 ); assert( pExpr->op==TK_FUNCTION ); pExpr = pExpr->x.pList->a[0].pExpr; - }else{ - assert( pExpr->op==TK_COLLATE ); + }else if( pExpr->op==TK_COLLATE ){ pExpr = pExpr->pLeft; + }else{ + break; } - } + } return pExpr; } @@ -89069,37 +109620,47 @@ SQLITE_PRIVATE Expr *sqlite3ExprSkipCollate(Expr *pExpr){ ** Return the collation sequence for the expression pExpr. If ** there is no defined collating sequence, return NULL. ** +** See also: sqlite3ExprNNCollSeq() +** +** The sqlite3ExprNNCollSeq() works the same exact that it returns the +** default collation if pExpr has no defined collation. +** ** The collating sequence might be determined by a COLLATE operator ** or by the presence of a column with a defined collating sequence. ** COLLATE operators take first precedence. Left operands take ** precedence over right operands. */ -SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ +SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, const Expr *pExpr){ sqlite3 *db = pParse->db; CollSeq *pColl = 0; - Expr *p = pExpr; + const Expr *p = pExpr; while( p ){ int op = p->op; - if( p->flags & EP_Generic ) break; + if( op==TK_REGISTER ) op = p->op2; + if( (op==TK_AGG_COLUMN && p->y.pTab!=0) + || op==TK_COLUMN || op==TK_TRIGGER + ){ + int j; + assert( ExprUseYTab(p) ); + assert( p->y.pTab!=0 ); + if( (j = p->iColumn)>=0 ){ + const char *zColl = sqlite3ColumnColl(&p->y.pTab->aCol[j]); + pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); + } + break; + } if( op==TK_CAST || op==TK_UPLUS ){ p = p->pLeft; continue; } - if( op==TK_COLLATE || (op==TK_REGISTER && p->op2==TK_COLLATE) ){ - pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); - break; + if( op==TK_VECTOR ){ + assert( ExprUseXList(p) ); + p = p->x.pList->a[0].pExpr; + continue; } - if( (op==TK_AGG_COLUMN || op==TK_COLUMN - || op==TK_REGISTER || op==TK_TRIGGER) - && p->pTab!=0 - ){ - /* op==TK_REGISTER && p->pTab!=0 happens when pExpr was originally - ** a TK_COLUMN but was previously evaluated and cached in a register */ - int j = p->iColumn; - if( j>=0 ){ - const char *zColl = p->pTab->aCol[j].zColl; - pColl = sqlite3FindCollSeq(db, ENC(db), zColl, 0); - } + if( op==TK_COLLATE ){ + assert( !ExprHasProperty(p, EP_IntValue) ); + pColl = sqlite3GetCollSeq(pParse, ENC(db), 0, p->u.zToken); break; } if( p->flags & EP_Collate ){ @@ -89108,13 +109669,10 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ }else{ Expr *pNext = p->pRight; /* The Expr.x union is never used at the same time as Expr.pRight */ - assert( p->x.pList==0 || p->pRight==0 ); - /* p->flags holds EP_Collate and p->pLeft->flags does not. And - ** p->x.pSelect cannot. So if p->x.pLeft exists, it must hold at - ** least one EP_Collate. Thus the following two ALWAYS. */ - if( p->x.pList!=0 && ALWAYS(!ExprHasProperty(p, EP_xIsSelect)) ){ + assert( !ExprUseXList(p) || p->x.pList==0 || p->pRight==0 ); + if( ExprUseXList(p) && p->x.pList!=0 && !db->mallocFailed ){ int i; - for(i=0; ALWAYS(ix.pList->nExpr); i++){ + for(i=0; ix.pList->nExpr; i++){ if( ExprHasProperty(p->x.pList->a[i].pExpr, EP_Collate) ){ pNext = p->x.pList->a[i].pExpr; break; @@ -89127,20 +109685,46 @@ SQLITE_PRIVATE CollSeq *sqlite3ExprCollSeq(Parse *pParse, Expr *pExpr){ break; } } - if( sqlite3CheckCollSeq(pParse, pColl) ){ + if( sqlite3CheckCollSeq(pParse, pColl) ){ pColl = 0; } return pColl; } +/* +** Return the collation sequence for the expression pExpr. If +** there is no defined collating sequence, return a pointer to the +** default collation sequence. +** +** See also: sqlite3ExprCollSeq() +** +** The sqlite3ExprCollSeq() routine works the same except that it +** returns NULL if there is no defined collation. +*/ +SQLITE_PRIVATE CollSeq *sqlite3ExprNNCollSeq(Parse *pParse, const Expr *pExpr){ + CollSeq *p = sqlite3ExprCollSeq(pParse, pExpr); + if( p==0 ) p = pParse->db->pDfltColl; + assert( p!=0 ); + return p; +} + +/* +** Return TRUE if the two expressions have equivalent collating sequences. +*/ +SQLITE_PRIVATE int sqlite3ExprCollSeqMatch(Parse *pParse, const Expr *pE1, const Expr *pE2){ + CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pE1); + CollSeq *pColl2 = sqlite3ExprNNCollSeq(pParse, pE2); + return sqlite3StrICmp(pColl1->zName, pColl2->zName)==0; +} + /* ** pExpr is an operand of a comparison operator. aff2 is the ** type affinity of the other operand. This routine returns the ** type affinity that should be used for the comparison operator. */ -SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ +SQLITE_PRIVATE char sqlite3CompareAffinity(const Expr *pExpr, char aff2){ char aff1 = sqlite3ExprAffinity(pExpr); - if( aff1 && aff2 ){ + if( aff1>SQLITE_AFF_NONE && aff2>SQLITE_AFF_NONE ){ /* Both sides of the comparison are columns. If one has numeric ** affinity, use that. Otherwise use no affinity. */ @@ -89149,15 +109733,10 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ }else{ return SQLITE_AFF_BLOB; } - }else if( !aff1 && !aff2 ){ - /* Neither side of the comparison is a column. Compare the - ** results directly. - */ - return SQLITE_AFF_BLOB; }else{ /* One side is a column, the other is not. Use the columns affinity. */ - assert( aff1==0 || aff2==0 ); - return (aff1 + aff2); + assert( aff1<=SQLITE_AFF_NONE || aff2<=SQLITE_AFF_NONE ); + return (aff1<=SQLITE_AFF_NONE ? aff2 : aff1) | SQLITE_AFF_NONE; } } @@ -89165,7 +109744,7 @@ SQLITE_PRIVATE char sqlite3CompareAffinity(Expr *pExpr, char aff2){ ** pExpr is a comparison operator. Return the type affinity that should ** be applied to both operands prior to doing the comparison. */ -static char comparisonAffinity(Expr *pExpr){ +static char comparisonAffinity(const Expr *pExpr){ char aff; assert( pExpr->op==TK_EQ || pExpr->op==TK_IN || pExpr->op==TK_LT || pExpr->op==TK_GT || pExpr->op==TK_GE || pExpr->op==TK_LE || @@ -89174,9 +109753,9 @@ static char comparisonAffinity(Expr *pExpr){ aff = sqlite3ExprAffinity(pExpr->pLeft); if( pExpr->pRight ){ aff = sqlite3CompareAffinity(pExpr->pRight, aff); - }else if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + }else if( ExprUseXSelect(pExpr) ){ aff = sqlite3CompareAffinity(pExpr->x.pSelect->pEList->a[0].pExpr, aff); - }else if( !aff ){ + }else if( aff==0 ){ aff = SQLITE_AFF_BLOB; } return aff; @@ -89188,23 +109767,26 @@ static char comparisonAffinity(Expr *pExpr){ ** if the index with affinity idx_affinity may be used to implement ** the comparison in pExpr. */ -SQLITE_PRIVATE int sqlite3IndexAffinityOk(Expr *pExpr, char idx_affinity){ +SQLITE_PRIVATE int sqlite3IndexAffinityOk(const Expr *pExpr, char idx_affinity){ char aff = comparisonAffinity(pExpr); - switch( aff ){ - case SQLITE_AFF_BLOB: - return 1; - case SQLITE_AFF_TEXT: - return idx_affinity==SQLITE_AFF_TEXT; - default: - return sqlite3IsNumericAffinity(idx_affinity); + if( affpRight, p->pLeft); + }else{ + return sqlite3BinaryCompareCollSeq(pParse, p->pLeft, p->pRight); + } +} + /* ** Generate code for a comparison operator. */ @@ -89252,13 +109850,19 @@ static int codeCompare( int opcode, /* The comparison opcode */ int in1, int in2, /* Register holding operands */ int dest, /* Jump here if true. */ - int jumpIfNull /* If true, jump if either operand is NULL */ + int jumpIfNull, /* If true, jump if either operand is NULL */ + int isCommuted /* The comparison has been commuted */ ){ int p5; int addr; CollSeq *p4; - p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + if( pParse->nErr ) return 0; + if( isCommuted ){ + p4 = sqlite3BinaryCompareCollSeq(pParse, pRight, pLeft); + }else{ + p4 = sqlite3BinaryCompareCollSeq(pParse, pLeft, pRight); + } p5 = binaryCompareP5(pLeft, pRight, jumpIfNull); addr = sqlite3VdbeAddOp4(pParse->pVdbe, opcode, in2, dest, in1, (void*)p4, P4_COLLSEQ); @@ -89266,6 +109870,302 @@ static int codeCompare( return addr; } +/* +** Return true if expression pExpr is a vector, or false otherwise. +** +** A vector is defined as any expression that results in two or more +** columns of result. Every TK_VECTOR node is an vector because the +** parser will not generate a TK_VECTOR with fewer than two entries. +** But a TK_SELECT might be either a vector or a scalar. It is only +** considered a vector if it has two or more result columns. +*/ +SQLITE_PRIVATE int sqlite3ExprIsVector(const Expr *pExpr){ + return sqlite3ExprVectorSize(pExpr)>1; +} + +/* +** If the expression passed as the only argument is of type TK_VECTOR +** return the number of expressions in the vector. Or, if the expression +** is a sub-select, return the number of columns in the sub-select. For +** any other type of expression, return 1. +*/ +SQLITE_PRIVATE int sqlite3ExprVectorSize(const Expr *pExpr){ + u8 op = pExpr->op; + if( op==TK_REGISTER ) op = pExpr->op2; + if( op==TK_VECTOR ){ + assert( ExprUseXList(pExpr) ); + return pExpr->x.pList->nExpr; + }else if( op==TK_SELECT ){ + assert( ExprUseXSelect(pExpr) ); + return pExpr->x.pSelect->pEList->nExpr; + }else{ + return 1; + } +} + +/* +** Return a pointer to a subexpression of pVector that is the i-th +** column of the vector (numbered starting with 0). The caller must +** ensure that i is within range. +** +** If pVector is really a scalar (and "scalar" here includes subqueries +** that return a single column!) then return pVector unmodified. +** +** pVector retains ownership of the returned subexpression. +** +** If the vector is a (SELECT ...) then the expression returned is +** just the expression for the i-th term of the result set, and may +** not be ready for evaluation because the table cursor has not yet +** been positioned. +*/ +SQLITE_PRIVATE Expr *sqlite3VectorFieldSubexpr(Expr *pVector, int i){ + assert( iop==TK_ERROR ); + if( sqlite3ExprIsVector(pVector) ){ + assert( pVector->op2==0 || pVector->op==TK_REGISTER ); + if( pVector->op==TK_SELECT || pVector->op2==TK_SELECT ){ + assert( ExprUseXSelect(pVector) ); + return pVector->x.pSelect->pEList->a[i].pExpr; + }else{ + assert( ExprUseXList(pVector) ); + return pVector->x.pList->a[i].pExpr; + } + } + return pVector; +} + +/* +** Compute and return a new Expr object which when passed to +** sqlite3ExprCode() will generate all necessary code to compute +** the iField-th column of the vector expression pVector. +** +** It is ok for pVector to be a scalar (as long as iField==0). +** In that case, this routine works like sqlite3ExprDup(). +** +** The caller owns the returned Expr object and is responsible for +** ensuring that the returned value eventually gets freed. +** +** The caller retains ownership of pVector. If pVector is a TK_SELECT, +** then the returned object will reference pVector and so pVector must remain +** valid for the life of the returned object. If pVector is a TK_VECTOR +** or a scalar expression, then it can be deleted as soon as this routine +** returns. +** +** A trick to cause a TK_SELECT pVector to be deleted together with +** the returned Expr object is to attach the pVector to the pRight field +** of the returned TK_SELECT_COLUMN Expr object. +*/ +SQLITE_PRIVATE Expr *sqlite3ExprForVectorField( + Parse *pParse, /* Parsing context */ + Expr *pVector, /* The vector. List of expressions or a sub-SELECT */ + int iField, /* Which column of the vector to return */ + int nField /* Total number of columns in the vector */ +){ + Expr *pRet; + if( pVector->op==TK_SELECT ){ + assert( ExprUseXSelect(pVector) ); + /* The TK_SELECT_COLUMN Expr node: + ** + ** pLeft: pVector containing TK_SELECT. Not deleted. + ** pRight: not used. But recursively deleted. + ** iColumn: Index of a column in pVector + ** iTable: 0 or the number of columns on the LHS of an assignment + ** pLeft->iTable: First in an array of register holding result, or 0 + ** if the result is not yet computed. + ** + ** sqlite3ExprDelete() specifically skips the recursive delete of + ** pLeft on TK_SELECT_COLUMN nodes. But pRight is followed, so pVector + ** can be attached to pRight to cause this node to take ownership of + ** pVector. Typically there will be multiple TK_SELECT_COLUMN nodes + ** with the same pLeft pointer to the pVector, but only one of them + ** will own the pVector. + */ + pRet = sqlite3PExpr(pParse, TK_SELECT_COLUMN, 0, 0); + if( pRet ){ + ExprSetProperty(pRet, EP_FullSize); + pRet->iTable = nField; + pRet->iColumn = iField; + pRet->pLeft = pVector; + } + }else{ + if( pVector->op==TK_VECTOR ){ + Expr **ppVector; + assert( ExprUseXList(pVector) ); + ppVector = &pVector->x.pList->a[iField].pExpr; + pVector = *ppVector; + if( IN_RENAME_OBJECT ){ + /* This must be a vector UPDATE inside a trigger */ + *ppVector = 0; + return pVector; + } + } + pRet = sqlite3ExprDup(pParse->db, pVector, 0); + } + return pRet; +} + +/* +** If expression pExpr is of type TK_SELECT, generate code to evaluate +** it. Return the register in which the result is stored (or, if the +** sub-select returns more than one column, the first in an array +** of registers in which the result is stored). +** +** If pExpr is not a TK_SELECT expression, return 0. +*/ +static int exprCodeSubselect(Parse *pParse, Expr *pExpr){ + int reg = 0; +#ifndef SQLITE_OMIT_SUBQUERY + if( pExpr->op==TK_SELECT ){ + reg = sqlite3CodeSubselect(pParse, pExpr); + } +#endif + return reg; +} + +/* +** Argument pVector points to a vector expression - either a TK_VECTOR +** or TK_SELECT that returns more than one column. This function returns +** the register number of a register that contains the value of +** element iField of the vector. +** +** If pVector is a TK_SELECT expression, then code for it must have +** already been generated using the exprCodeSubselect() routine. In this +** case parameter regSelect should be the first in an array of registers +** containing the results of the sub-select. +** +** If pVector is of type TK_VECTOR, then code for the requested field +** is generated. In this case (*pRegFree) may be set to the number of +** a temporary register to be freed by the caller before returning. +** +** Before returning, output parameter (*ppExpr) is set to point to the +** Expr object corresponding to element iElem of the vector. +*/ +static int exprVectorRegister( + Parse *pParse, /* Parse context */ + Expr *pVector, /* Vector to extract element from */ + int iField, /* Field to extract from pVector */ + int regSelect, /* First in array of registers */ + Expr **ppExpr, /* OUT: Expression element */ + int *pRegFree /* OUT: Temp register to free */ +){ + u8 op = pVector->op; + assert( op==TK_VECTOR || op==TK_REGISTER || op==TK_SELECT || op==TK_ERROR ); + if( op==TK_REGISTER ){ + *ppExpr = sqlite3VectorFieldSubexpr(pVector, iField); + return pVector->iTable+iField; + } + if( op==TK_SELECT ){ + assert( ExprUseXSelect(pVector) ); + *ppExpr = pVector->x.pSelect->pEList->a[iField].pExpr; + return regSelect+iField; + } + if( op==TK_VECTOR ){ + assert( ExprUseXList(pVector) ); + *ppExpr = pVector->x.pList->a[iField].pExpr; + return sqlite3ExprCodeTemp(pParse, *ppExpr, pRegFree); + } + return 0; +} + +/* +** Expression pExpr is a comparison between two vector values. Compute +** the result of the comparison (1, 0, or NULL) and write that +** result into register dest. +** +** The caller must satisfy the following preconditions: +** +** if pExpr->op==TK_IS: op==TK_EQ and p5==SQLITE_NULLEQ +** if pExpr->op==TK_ISNOT: op==TK_NE and p5==SQLITE_NULLEQ +** otherwise: op==pExpr->op and p5==0 +*/ +static void codeVectorCompare( + Parse *pParse, /* Code generator context */ + Expr *pExpr, /* The comparison operation */ + int dest, /* Write results into this register */ + u8 op, /* Comparison operator */ + u8 p5 /* SQLITE_NULLEQ or zero */ +){ + Vdbe *v = pParse->pVdbe; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + int nLeft = sqlite3ExprVectorSize(pLeft); + int i; + int regLeft = 0; + int regRight = 0; + u8 opx = op; + int addrCmp = 0; + int addrDone = sqlite3VdbeMakeLabel(pParse); + int isCommuted = ExprHasProperty(pExpr,EP_Commuted); + + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + if( pParse->nErr ) return; + if( nLeft!=sqlite3ExprVectorSize(pRight) ){ + sqlite3ErrorMsg(pParse, "row value misused"); + return; + } + assert( pExpr->op==TK_EQ || pExpr->op==TK_NE + || pExpr->op==TK_IS || pExpr->op==TK_ISNOT + || pExpr->op==TK_LT || pExpr->op==TK_GT + || pExpr->op==TK_LE || pExpr->op==TK_GE + ); + assert( pExpr->op==op || (pExpr->op==TK_IS && op==TK_EQ) + || (pExpr->op==TK_ISNOT && op==TK_NE) ); + assert( p5==0 || pExpr->op!=op ); + assert( p5==SQLITE_NULLEQ || pExpr->op==op ); + + if( op==TK_LE ) opx = TK_LT; + if( op==TK_GE ) opx = TK_GT; + if( op==TK_NE ) opx = TK_EQ; + + regLeft = exprCodeSubselect(pParse, pLeft); + regRight = exprCodeSubselect(pParse, pRight); + + sqlite3VdbeAddOp2(v, OP_Integer, 1, dest); + for(i=0; 1 /*Loop exits by "break"*/; i++){ + int regFree1 = 0, regFree2 = 0; + Expr *pL = 0, *pR = 0; + int r1, r2; + assert( i>=0 && i0 /* ** Check that argument nHeight is less than or equal to the maximum @@ -89276,7 +110176,7 @@ SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ int rc = SQLITE_OK; int mxHeight = pParse->db->aLimit[SQLITE_LIMIT_EXPR_DEPTH]; if( nHeight>mxHeight ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "Expression tree is too large (maximum depth %d)", mxHeight ); rc = SQLITE_ERROR; @@ -89293,14 +110193,14 @@ SQLITE_PRIVATE int sqlite3ExprCheckHeight(Parse *pParse, int nHeight){ ** to by pnHeight, the second parameter, then set *pnHeight to that ** value. */ -static void heightOfExpr(Expr *p, int *pnHeight){ +static void heightOfExpr(const Expr *p, int *pnHeight){ if( p ){ if( p->nHeight>*pnHeight ){ *pnHeight = p->nHeight; } } } -static void heightOfExprList(ExprList *p, int *pnHeight){ +static void heightOfExprList(const ExprList *p, int *pnHeight){ if( p ){ int i; for(i=0; inExpr; i++){ @@ -89308,34 +110208,34 @@ static void heightOfExprList(ExprList *p, int *pnHeight){ } } } -static void heightOfSelect(Select *p, int *pnHeight){ - if( p ){ +static void heightOfSelect(const Select *pSelect, int *pnHeight){ + const Select *p; + for(p=pSelect; p; p=p->pPrior){ heightOfExpr(p->pWhere, pnHeight); heightOfExpr(p->pHaving, pnHeight); heightOfExpr(p->pLimit, pnHeight); - heightOfExpr(p->pOffset, pnHeight); heightOfExprList(p->pEList, pnHeight); heightOfExprList(p->pGroupBy, pnHeight); heightOfExprList(p->pOrderBy, pnHeight); - heightOfSelect(p->pPrior, pnHeight); } } /* -** Set the Expr.nHeight variable in the structure passed as an -** argument. An expression with no children, Expr.pList or +** Set the Expr.nHeight variable in the structure passed as an +** argument. An expression with no children, Expr.pList or ** Expr.pSelect member has a height of 1. Any other expression -** has a height equal to the maximum height of any other +** has a height equal to the maximum height of any other ** referenced Expr plus one. ** ** Also propagate EP_Propagate flags up from Expr.x.pList to Expr.flags, ** if appropriate. */ static void exprSetHeight(Expr *p){ - int nHeight = 0; - heightOfExpr(p->pLeft, &nHeight); - heightOfExpr(p->pRight, &nHeight); - if( ExprHasProperty(p, EP_xIsSelect) ){ + int nHeight = p->pLeft ? p->pLeft->nHeight : 0; + if( NEVER(p->pRight) && p->pRight->nHeight>nHeight ){ + nHeight = p->pRight->nHeight; + } + if( ExprUseXSelect(p) ){ heightOfSelect(p->x.pSelect, &nHeight); }else if( p->x.pList ){ heightOfExprList(p->x.pList, &nHeight); @@ -89350,7 +110250,7 @@ static void exprSetHeight(Expr *p){ ** leave an error in pParse. ** ** Also propagate all EP_Propagate flags from the Expr.x.pList into -** Expr.flags. +** Expr.flags. */ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ if( pParse->nErr ) return; @@ -89362,7 +110262,7 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** Return the maximum height of any expression tree referenced ** by the select statement passed as an argument. */ -SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ +SQLITE_PRIVATE int sqlite3SelectExprHeight(const Select *p){ int nHeight = 0; heightOfSelect(p, &nHeight); return nHeight; @@ -89370,16 +110270,26 @@ SQLITE_PRIVATE int sqlite3SelectExprHeight(Select *p){ #else /* ABOVE: Height enforcement enabled. BELOW: Height enforcement off */ /* ** Propagate all EP_Propagate flags from the Expr.x.pList into -** Expr.flags. +** Expr.flags. */ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ - if( p && p->x.pList && !ExprHasProperty(p, EP_xIsSelect) ){ + if( pParse->nErr ) return; + if( p && ExprUseXList(p) && p->x.pList ){ p->flags |= EP_Propagate & sqlite3ExprListFlags(p->x.pList); } } #define exprSetHeight(y) #endif /* SQLITE_MAX_EXPR_DEPTH>0 */ +/* +** Set the error offset for an Expr node, if possible. +*/ +SQLITE_PRIVATE void sqlite3ExprSetErrorOffset(Expr *pExpr, int iOfst){ + if( pExpr==0 ) return; + if( NEVER(ExprUseWJoin(pExpr)) ) return; + pExpr->w.iOfst = iOfst; +} + /* ** This routine is the core allocator for Expr nodes. ** @@ -89394,14 +110304,15 @@ SQLITE_PRIVATE void sqlite3ExprSetHeightAndFlags(Parse *pParse, Expr *p){ ** appear to be quoted. If the quotes were of the form "..." (double-quotes) ** then the EP_DblQuoted flag is set on the expression node. ** -** Special case: If op==TK_INTEGER and pToken points to a string that -** can be translated into a 32-bit integer, then the token is not -** stored in u.zToken. Instead, the integer values is written -** into u.iValue and the EP_IntValue flag is set. No extra storage +** Special case (tag-20240227-a): If op==TK_INTEGER and pToken points to +** a string that can be translated into a 32-bit integer, then the token is +** not stored in u.zToken. Instead, the integer values is written +** into u.iValue and the EP_IntValue flag is set. No extra storage ** is allocated to hold the integer text and the dequote flag is ignored. +** See also tag-20240227-b. */ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( - sqlite3 *db, /* Handle for sqlite3DbMallocZero() (may be null) */ + sqlite3 *db, /* Handle for sqlite3DbMallocRawNN() */ int op, /* Expression opcode */ const Token *pToken, /* Token argument. Might be NULL */ int dequote /* True to dequote */ @@ -89414,7 +110325,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken ){ if( op!=TK_INTEGER || pToken->z==0 || sqlite3GetInt32(pToken->z, &iValue)==0 ){ - nExtra = pToken->n+1; + nExtra = pToken->n+1; /* tag-20240227-a */ assert( iValue>=0 ); } } @@ -89425,7 +110336,7 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( pNew->iAgg = -1; if( pToken ){ if( nExtra==0 ){ - pNew->flags |= EP_IntValue; + pNew->flags |= EP_IntValue|EP_Leaf|(iValue?EP_IsTrue:EP_IsFalse); pNew->u.iValue = iValue; }else{ pNew->u.zToken = (char*)&pNew[1]; @@ -89433,14 +110344,13 @@ SQLITE_PRIVATE Expr *sqlite3ExprAlloc( if( pToken->n ) memcpy(pNew->u.zToken, pToken->z, pToken->n); pNew->u.zToken[pToken->n] = 0; if( dequote && sqlite3Isquote(pNew->u.zToken[0]) ){ - if( pNew->u.zToken[0]=='"' ) pNew->flags |= EP_DblQuoted; - sqlite3Dequote(pNew->u.zToken); + sqlite3DequoteExpr(pNew); } } } #if SQLITE_MAX_EXPR_DEPTH>0 pNew->nHeight = 1; -#endif +#endif } return pNew; } @@ -89456,7 +110366,7 @@ SQLITE_PRIVATE Expr *sqlite3Expr( ){ Token x; x.z = zToken; - x.n = zToken ? sqlite3Strlen30(zToken) : 0; + x.n = sqlite3Strlen30(zToken); return sqlite3ExprAlloc(db, op, &x, 0); } @@ -89477,15 +110387,26 @@ SQLITE_PRIVATE void sqlite3ExprAttachSubtrees( sqlite3ExprDelete(db, pLeft); sqlite3ExprDelete(db, pRight); }else{ + assert( ExprUseXList(pRoot) ); + assert( pRoot->x.pSelect==0 ); if( pRight ){ pRoot->pRight = pRight; pRoot->flags |= EP_Propagate & pRight->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + pRoot->nHeight = pRight->nHeight+1; + }else{ + pRoot->nHeight = 1; +#endif } if( pLeft ){ pRoot->pLeft = pLeft; pRoot->flags |= EP_Propagate & pLeft->flags; +#if SQLITE_MAX_EXPR_DEPTH>0 + if( pLeft->nHeight>=pRoot->nHeight ){ + pRoot->nHeight = pLeft->nHeight+1; + } +#endif } - exprSetHeight(pRoot); } } @@ -89500,19 +110421,19 @@ SQLITE_PRIVATE Expr *sqlite3PExpr( Parse *pParse, /* Parsing context */ int op, /* Expression opcode */ Expr *pLeft, /* Left operand */ - Expr *pRight, /* Right operand */ - const Token *pToken /* Argument token */ + Expr *pRight /* Right operand */ ){ Expr *p; - if( op==TK_AND && pParse->nErr==0 ){ - /* Take advantage of short-circuit false optimization for AND */ - p = sqlite3ExprAnd(pParse->db, pLeft, pRight); - }else{ - p = sqlite3ExprAlloc(pParse->db, op & TKFLG_MASK, pToken, 1); + p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)); + if( p ){ + memset(p, 0, sizeof(Expr)); + p->op = op & 0xff; + p->iAgg = -1; sqlite3ExprAttachSubtrees(pParse->db, p, pLeft, pRight); - } - if( p ) { sqlite3ExprCheckHeight(pParse, p->nHeight); + }else{ + sqlite3ExprDelete(pParse->db, pLeft); + sqlite3ExprDelete(pParse->db, pRight); } return p; } @@ -89532,55 +110453,89 @@ SQLITE_PRIVATE void sqlite3PExprAddSelect(Parse *pParse, Expr *pExpr, Select *pS } } - /* -** If the expression is always either TRUE or FALSE (respectively), -** then return 1. If one cannot determine the truth value of the -** expression at compile-time return 0. +** Expression list pEList is a list of vector values. This function +** converts the contents of pEList to a VALUES(...) Select statement +** returning 1 row for each element of the list. For example, the +** expression list: ** -** This is an optimization. If is OK to return 0 here even if -** the expression really is always false or false (a false negative). -** But it is a bug to return 1 if the expression might have different -** boolean values in different circumstances (a false positive.) +** ( (1,2), (3,4) (5,6) ) ** -** Note that if the expression is part of conditional for a -** LEFT JOIN, then we cannot determine at compile-time whether or not -** is it true or false, so always return 0. +** is translated to the equivalent of: +** +** VALUES(1,2), (3,4), (5,6) +** +** Each of the vector values in pEList must contain exactly nElem terms. +** If a list element that is not a vector or does not contain nElem terms, +** an error message is left in pParse. +** +** This is used as part of processing IN(...) expressions with a list +** of vectors on the RHS. e.g. "... IN ((1,2), (3,4), (5,6))". */ -static int exprAlwaysTrue(Expr *p){ - int v = 0; - if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; - return v!=0; -} -static int exprAlwaysFalse(Expr *p){ - int v = 0; - if( ExprHasProperty(p, EP_FromJoin) ) return 0; - if( !sqlite3ExprIsInteger(p, &v) ) return 0; - return v==0; +SQLITE_PRIVATE Select *sqlite3ExprListToValues(Parse *pParse, int nElem, ExprList *pEList){ + int ii; + Select *pRet = 0; + assert( nElem>1 ); + for(ii=0; iinExpr; ii++){ + Select *pSel; + Expr *pExpr = pEList->a[ii].pExpr; + int nExprElem; + if( pExpr->op==TK_VECTOR ){ + assert( ExprUseXList(pExpr) ); + nExprElem = pExpr->x.pList->nExpr; + }else{ + nExprElem = 1; + } + if( nExprElem!=nElem ){ + sqlite3ErrorMsg(pParse, "IN(...) element has %d term%s - expected %d", + nExprElem, nExprElem>1?"s":"", nElem + ); + break; + } + assert( ExprUseXList(pExpr) ); + pSel = sqlite3SelectNew(pParse, pExpr->x.pList, 0, 0, 0, 0, 0, SF_Values,0); + pExpr->x.pList = 0; + if( pSel ){ + if( pRet ){ + pSel->op = TK_ALL; + pSel->pPrior = pRet; + } + pRet = pSel; + } + } + + if( pRet && pRet->pPrior ){ + pRet->selFlags |= SF_MultiValue; + } + sqlite3ExprListDelete(pParse->db, pEList); + return pRet; } /* ** Join two expressions using an AND operator. If either expression is ** NULL, then just return the other expression. ** -** If one side or the other of the AND is known to be false, then instead -** of returning an AND expression, just return a constant expression with -** a value of false. +** If one side or the other of the AND is known to be false, and neither side +** is part of an ON clause, then instead of returning an AND expression, +** just return a constant expression with a value of false. */ -SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ - if( pLeft==0 ){ +SQLITE_PRIVATE Expr *sqlite3ExprAnd(Parse *pParse, Expr *pLeft, Expr *pRight){ + sqlite3 *db = pParse->db; + if( pLeft==0 ){ return pRight; }else if( pRight==0 ){ return pLeft; - }else if( exprAlwaysFalse(pLeft) || exprAlwaysFalse(pRight) ){ - sqlite3ExprDelete(db, pLeft); - sqlite3ExprDelete(db, pRight); - return sqlite3ExprAlloc(db, TK_INTEGER, &sqlite3IntTokens[0], 0); }else{ - Expr *pNew = sqlite3ExprAlloc(db, TK_AND, 0, 0); - sqlite3ExprAttachSubtrees(db, pNew, pLeft, pRight); - return pNew; + u32 f = pLeft->flags | pRight->flags; + if( (f&(EP_OuterON|EP_InnerON|EP_IsFalse))==EP_IsFalse + && !IN_RENAME_OBJECT + ){ + sqlite3ExprDeferredDelete(pParse, pLeft); + sqlite3ExprDeferredDelete(pParse, pRight); + return sqlite3Expr(db, TK_INTEGER, "0"); + }else{ + return sqlite3PExpr(pParse, TK_AND, pLeft, pRight); + } } } @@ -89588,7 +110543,12 @@ SQLITE_PRIVATE Expr *sqlite3ExprAnd(sqlite3 *db, Expr *pLeft, Expr *pRight){ ** Construct a new expression node for a function with multiple ** arguments. */ -SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token *pToken){ +SQLITE_PRIVATE Expr *sqlite3ExprFunction( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* Argument list */ + const Token *pToken, /* Name of the function */ + int eDistinct /* SF_Distinct or SF_ALL or 0 */ +){ Expr *pNew; sqlite3 *db = pParse->db; assert( pToken ); @@ -89597,21 +110557,126 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * sqlite3ExprListDelete(db, pList); /* Avoid memory leak when malloc fails */ return 0; } + assert( !ExprHasProperty(pNew, EP_InnerON|EP_OuterON) ); + pNew->w.iOfst = (int)(pToken->z - pParse->zTail); + if( pList + && pList->nExpr > pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] + && !pParse->nested + ){ + sqlite3ErrorMsg(pParse, "too many arguments on function %T", pToken); + } pNew->x.pList = pList; - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + ExprSetProperty(pNew, EP_HasFunc); + assert( ExprUseXList(pNew) ); sqlite3ExprSetHeightAndFlags(pParse, pNew); + if( eDistinct==SF_Distinct ) ExprSetProperty(pNew, EP_Distinct); return pNew; } +/* +** Report an error when attempting to use an ORDER BY clause within +** the arguments of a non-aggregate function. +*/ +SQLITE_PRIVATE void sqlite3ExprOrderByAggregateError(Parse *pParse, Expr *p){ + sqlite3ErrorMsg(pParse, + "ORDER BY may not be used with non-aggregate %#T()", p + ); +} + +/* +** Attach an ORDER BY clause to a function call. +** +** functionname( arguments ORDER BY sortlist ) +** \_____________________/ \______/ +** pExpr pOrderBy +** +** The ORDER BY clause is inserted into a new Expr node of type TK_ORDER +** and added to the Expr.pLeft field of the parent TK_FUNCTION node. +*/ +SQLITE_PRIVATE void sqlite3ExprAddFunctionOrderBy( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The function call to which ORDER BY is to be added */ + ExprList *pOrderBy /* The ORDER BY clause to add */ +){ + Expr *pOB; + sqlite3 *db = pParse->db; + if( NEVER(pOrderBy==0) ){ + assert( db->mallocFailed ); + return; + } + if( pExpr==0 ){ + assert( db->mallocFailed ); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + assert( pExpr->op==TK_FUNCTION ); + assert( pExpr->pLeft==0 ); + assert( ExprUseXList(pExpr) ); + if( pExpr->x.pList==0 || NEVER(pExpr->x.pList->nExpr==0) ){ + /* Ignore ORDER BY on zero-argument aggregates */ + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pOrderBy); + return; + } + if( IsWindowFunc(pExpr) ){ + sqlite3ExprOrderByAggregateError(pParse, pExpr); + sqlite3ExprListDelete(db, pOrderBy); + return; + } + + pOB = sqlite3ExprAlloc(db, TK_ORDER, 0, 0); + if( pOB==0 ){ + sqlite3ExprListDelete(db, pOrderBy); + return; + } + pOB->x.pList = pOrderBy; + assert( ExprUseXList(pOB) ); + pExpr->pLeft = pOB; + ExprSetProperty(pOB, EP_FullSize); +} + +/* +** Check to see if a function is usable according to current access +** rules: +** +** SQLITE_FUNC_DIRECT - Only usable from top-level SQL +** +** SQLITE_FUNC_UNSAFE - Usable if TRUSTED_SCHEMA or from +** top-level SQL +** +** If the function is not usable, create an error. +*/ +SQLITE_PRIVATE void sqlite3ExprFunctionUsable( + Parse *pParse, /* Parsing and code generating context */ + const Expr *pExpr, /* The function invocation */ + const FuncDef *pDef /* The function being invoked */ +){ + assert( !IN_RENAME_OBJECT ); + assert( (pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE))!=0 ); + if( ExprHasProperty(pExpr, EP_FromDDL) ){ + if( (pDef->funcFlags & SQLITE_FUNC_DIRECT)!=0 + || (pParse->db->flags & SQLITE_TrustedSchema)==0 + ){ + /* Functions prohibited in triggers and views if: + ** (1) tagged with SQLITE_DIRECTONLY + ** (2) not tagged with SQLITE_INNOCUOUS (which means it + ** is tagged with SQLITE_FUNC_UNSAFE) and + ** SQLITE_DBCONFIG_TRUSTED_SCHEMA is off (meaning + ** that the schema is possibly tainted). + */ + sqlite3ErrorMsg(pParse, "unsafe use of %#T()", pExpr); + } + } +} + /* ** Assign a variable number to an expression that encodes a wildcard -** in the original SQL statement. +** in the original SQL statement. ** ** Wildcards consisting of a single "?" are assigned the next sequential ** variable number. ** ** Wildcards of the form "?nnn" are assigned the number "nnn". We make -** sure "nnn" is not too be to avoid a denial of service attack when +** sure "nnn" is not too big to avoid a denial of service attack when ** the SQL statement comes from an external source. ** ** Wildcards of the form ":aaa", "@aaa", or "$aaa" are assigned the same number @@ -89619,28 +110684,34 @@ SQLITE_PRIVATE Expr *sqlite3ExprFunction(Parse *pParse, ExprList *pList, Token * ** instance of the wildcard, the next sequential variable number is ** assigned. */ -SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ +SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr, u32 n){ sqlite3 *db = pParse->db; const char *z; + ynVar x; if( pExpr==0 ) return; assert( !ExprHasProperty(pExpr, EP_IntValue|EP_Reduced|EP_TokenOnly) ); z = pExpr->u.zToken; assert( z!=0 ); assert( z[0]!=0 ); + assert( n==(u32)sqlite3Strlen30(z) ); if( z[1]==0 ){ /* Wildcard of the form "?". Assign the next variable number */ assert( z[0]=='?' ); - pExpr->iColumn = (ynVar)(++pParse->nVar); + x = (ynVar)(++pParse->nVar); }else{ - ynVar x = 0; - u32 n = sqlite3Strlen30(z); + int doAdd = 0; if( z[0]=='?' ){ /* Wildcard of the form "?nnn". Convert "nnn" to an integer and ** use it as the variable number */ i64 i; - int bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); - pExpr->iColumn = x = (ynVar)i; + int bOk; + if( n==2 ){ /*OPTIMIZATION-IF-TRUE*/ + i = z[1]-'0'; /* The common case of ?N for a single digit N */ + bOk = 1; + }else{ + bOk = 0==sqlite3Atoi64(&z[1], &i, n-1, SQLITE_UTF8); + } testcase( i==0 ); testcase( i==1 ); testcase( i==db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]-1 ); @@ -89648,45 +110719,35 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ if( bOk==0 || i<1 || i>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "variable number must be between ?1 and ?%d", db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER]); - x = 0; + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); + return; } - if( i>pParse->nVar ){ - pParse->nVar = (int)i; + x = (ynVar)i; + if( x>pParse->nVar ){ + pParse->nVar = (int)x; + doAdd = 1; + }else if( sqlite3VListNumToName(pParse->pVList, x)==0 ){ + doAdd = 1; } }else{ /* Wildcards like ":aaa", "$aaa" or "@aaa". Reuse the same variable ** number as the prior appearance of the same name, or if the name ** has never appeared before, reuse the same variable number */ - ynVar i; - for(i=0; inzVar; i++){ - if( pParse->azVar[i] && strcmp(pParse->azVar[i],z)==0 ){ - pExpr->iColumn = x = (ynVar)i+1; - break; - } - } - if( x==0 ) x = pExpr->iColumn = (ynVar)(++pParse->nVar); - } - if( x>0 ){ - if( x>pParse->nzVar ){ - char **a; - a = sqlite3DbRealloc(db, pParse->azVar, x*sizeof(a[0])); - if( a==0 ){ - assert( db->mallocFailed ); /* Error reported through mallocFailed */ - return; - } - pParse->azVar = a; - memset(&a[pParse->nzVar], 0, (x-pParse->nzVar)*sizeof(a[0])); - pParse->nzVar = x; - } - if( z[0]!='?' || pParse->azVar[x-1]==0 ){ - sqlite3DbFree(db, pParse->azVar[x-1]); - pParse->azVar[x-1] = sqlite3DbStrNDup(db, z, n); + x = (ynVar)sqlite3VListNameToNum(pParse->pVList, z, n); + if( x==0 ){ + x = (ynVar)(++pParse->nVar); + doAdd = 1; } } - } - if( !pParse->nErr && pParse->nVar>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ + if( doAdd ){ + pParse->pVList = sqlite3VListAdd(db, pParse->pVList, z, n, x); + } + } + pExpr->iColumn = x; + if( x>db->aLimit[SQLITE_LIMIT_VARIABLE_NUMBER] ){ sqlite3ErrorMsg(pParse, "too many SQL variables"); + sqlite3RecordErrorOffsetOfExpr(pParse->db, pExpr); } } @@ -89695,34 +110756,107 @@ SQLITE_PRIVATE void sqlite3ExprAssignVarNumber(Parse *pParse, Expr *pExpr){ */ static SQLITE_NOINLINE void sqlite3ExprDeleteNN(sqlite3 *db, Expr *p){ assert( p!=0 ); - /* Sanity check: Assert that the IntValue is non-negative if it exists */ - assert( !ExprHasProperty(p, EP_IntValue) || p->u.iValue>=0 ); - if( !ExprHasProperty(p, EP_TokenOnly) ){ + assert( db!=0 ); +exprDeleteRestart: + assert( !ExprUseUValue(p) || p->u.iValue>=0 ); + assert( !ExprUseYWin(p) || !ExprUseYSub(p) ); + assert( !ExprUseYWin(p) || p->y.pWin!=0 || db->mallocFailed ); + assert( p->op!=TK_FUNCTION || !ExprUseYSub(p) ); +#ifdef SQLITE_DEBUG + if( ExprHasProperty(p, EP_Leaf) && !ExprHasProperty(p, EP_TokenOnly) ){ + assert( p->pLeft==0 ); + assert( p->pRight==0 ); + assert( !ExprUseXSelect(p) || p->x.pSelect==0 ); + assert( !ExprUseXList(p) || p->x.pList==0 ); + } +#endif + if( !ExprHasProperty(p, (EP_TokenOnly|EP_Leaf)) ){ /* The Expr.x union is never used at the same time as Expr.pRight */ - assert( p->x.pList==0 || p->pRight==0 ); - sqlite3ExprDelete(db, p->pLeft); - sqlite3ExprDelete(db, p->pRight); - if( ExprHasProperty(p, EP_MemToken) ) sqlite3DbFree(db, p->u.zToken); - if( ExprHasProperty(p, EP_xIsSelect) ){ + assert( (ExprUseXList(p) && p->x.pList==0) || p->pRight==0 ); + if( p->pRight ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); + sqlite3ExprDeleteNN(db, p->pRight); + }else if( ExprUseXSelect(p) ){ + assert( !ExprHasProperty(p, EP_WinFunc) ); sqlite3SelectDelete(db, p->x.pSelect); }else{ sqlite3ExprListDelete(db, p->x.pList); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + sqlite3WindowDelete(db, p->y.pWin); + } +#endif + } + if( p->pLeft && p->op!=TK_SELECT_COLUMN ){ + Expr *pLeft = p->pLeft; + if( !ExprHasProperty(p, EP_Static) + && !ExprHasProperty(pLeft, EP_Static) + ){ + /* Avoid unnecessary recursion on unary operators */ + sqlite3DbNNFreeNN(db, p); + p = pLeft; + goto exprDeleteRestart; + }else{ + sqlite3ExprDeleteNN(db, pLeft); + } } } if( !ExprHasProperty(p, EP_Static) ){ - sqlite3DbFree(db, p); + sqlite3DbNNFreeNN(db, p); } } SQLITE_PRIVATE void sqlite3ExprDelete(sqlite3 *db, Expr *p){ if( p ) sqlite3ExprDeleteNN(db, p); } +SQLITE_PRIVATE void sqlite3ExprDeleteGeneric(sqlite3 *db, void *p){ + if( ALWAYS(p) ) sqlite3ExprDeleteNN(db, (Expr*)p); +} /* -** Return the number of bytes allocated for the expression structure +** Clear both elements of an OnOrUsing object +*/ +SQLITE_PRIVATE void sqlite3ClearOnOrUsing(sqlite3 *db, OnOrUsing *p){ + if( p==0 ){ + /* Nothing to clear */ + }else if( p->pOn ){ + sqlite3ExprDeleteNN(db, p->pOn); + }else if( p->pUsing ){ + sqlite3IdListDelete(db, p->pUsing); + } +} + +/* +** Arrange to cause pExpr to be deleted when the pParse is deleted. +** This is similar to sqlite3ExprDelete() except that the delete is +** deferred until the pParse is deleted. +** +** The pExpr might be deleted immediately on an OOM error. +** +** Return 0 if the delete was successfully deferred. Return non-zero +** if the delete happened immediately because of an OOM. +*/ +SQLITE_PRIVATE int sqlite3ExprDeferredDelete(Parse *pParse, Expr *pExpr){ + return 0==sqlite3ParserAddCleanup(pParse, sqlite3ExprDeleteGeneric, pExpr); +} + +/* Invoke sqlite3RenameExprUnmap() and sqlite3ExprDelete() on the +** expression. +*/ +SQLITE_PRIVATE void sqlite3ExprUnmapAndDelete(Parse *pParse, Expr *p){ + if( p ){ + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, p); + } + sqlite3ExprDeleteNN(pParse->db, p); + } +} + +/* +** Return the number of bytes allocated for the expression structure ** passed as the first argument. This is always one of EXPR_FULLSIZE, ** EXPR_REDUCEDSIZE or EXPR_TOKENONLYSIZE. */ -static int exprStructSize(Expr *p){ +static int exprStructSize(const Expr *p){ if( ExprHasProperty(p, EP_TokenOnly) ) return EXPR_TOKENONLYSIZE; if( ExprHasProperty(p, EP_Reduced) ) return EXPR_REDUCEDSIZE; return EXPR_FULLSIZE; @@ -89733,14 +110867,14 @@ static int exprStructSize(Expr *p){ ** to store a copy of an expression or expression tree. They differ in ** how much of the tree is measured. ** -** dupedExprStructSize() Size of only the Expr structure +** dupedExprStructSize() Size of only the Expr structure ** dupedExprNodeSize() Size of Expr + space for token ** dupedExprSize() Expr + token + subtree components ** *************************************************************************** ** -** The dupedExprStructSize() function returns two values OR-ed together: -** (1) the space required for a copy of the Expr structure only and +** The dupedExprStructSize() function returns two values OR-ed together: +** (1) the space required for a copy of the Expr structure only and ** (2) the EP_xxx flags that indicate what the structure size should be. ** The return values is always one of: ** @@ -89755,25 +110889,24 @@ static int exprStructSize(Expr *p){ ** Note that with flags==EXPRDUP_REDUCE, this routines works on full-size ** (unreduced) Expr objects as they or originally constructed by the parser. ** During expression analysis, extra information is computed and moved into -** later parts of teh Expr object and that extra information might get chopped +** later parts of the Expr object and that extra information might get chopped ** off if the expression is reduced. Note also that it does not work to ** make an EXPRDUP_REDUCE copy of a reduced expression. It is only legal ** to reduce a pristine expression tree from the parser. The implementation ** of dupedExprStructSize() contain multiple assert() statements that attempt ** to enforce this constraint. */ -static int dupedExprStructSize(Expr *p, int flags){ +static int dupedExprStructSize(const Expr *p, int flags){ int nSize; assert( flags==EXPRDUP_REDUCE || flags==0 ); /* Only one flag value allowed */ assert( EXPR_FULLSIZE<=0xfff ); assert( (0xfff & (EP_Reduced|EP_TokenOnly))==0 ); - if( 0==flags ){ + if( 0==flags || ExprHasProperty(p, EP_FullSize) ){ nSize = EXPR_FULLSIZE; }else{ assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); - assert( !ExprHasProperty(p, EP_FromJoin) ); - assert( !ExprHasProperty(p, EP_MemToken) ); - assert( !ExprHasProperty(p, EP_NoReduce) ); + assert( !ExprHasProperty(p, EP_OuterON) ); + assert( !ExprHasVVAProperty(p, EP_NoReduce) ); if( p->pLeft || p->x.pList ){ nSize = EXPR_REDUCEDSIZE | EP_Reduced; }else{ @@ -89785,69 +110918,107 @@ static int dupedExprStructSize(Expr *p, int flags){ } /* -** This function returns the space in bytes required to store the copy +** This function returns the space in bytes required to store the copy ** of the Expr structure and a copy of the Expr.u.zToken string (if that ** string is defined.) */ -static int dupedExprNodeSize(Expr *p, int flags){ +static int dupedExprNodeSize(const Expr *p, int flags){ int nByte = dupedExprStructSize(p, flags) & 0xfff; if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nByte += sqlite3Strlen30(p->u.zToken)+1; + nByte += sqlite3Strlen30NN(p->u.zToken)+1; } return ROUND8(nByte); } /* -** Return the number of bytes required to create a duplicate of the -** expression passed as the first argument. The second argument is a -** mask containing EXPRDUP_XXX flags. +** Return the number of bytes required to create a duplicate of the +** expression passed as the first argument. ** ** The value returned includes space to create a copy of the Expr struct ** itself and the buffer referred to by Expr.u.zToken, if any. ** -** If the EXPRDUP_REDUCE flag is set, then the return value includes -** space to duplicate all Expr nodes in the tree formed by Expr.pLeft -** and Expr.pRight variables (but not for any structures pointed to or -** descended from the Expr.x.pList or Expr.x.pSelect variables). +** The return value includes space to duplicate all Expr nodes in the +** tree formed by Expr.pLeft and Expr.pRight, but not any other +** substructure such as Expr.x.pList, Expr.x.pSelect, and Expr.y.pWin. */ -static int dupedExprSize(Expr *p, int flags){ - int nByte = 0; - if( p ){ - nByte = dupedExprNodeSize(p, flags); - if( flags&EXPRDUP_REDUCE ){ - nByte += dupedExprSize(p->pLeft, flags) + dupedExprSize(p->pRight, flags); - } - } +static int dupedExprSize(const Expr *p){ + int nByte; + assert( p!=0 ); + nByte = dupedExprNodeSize(p, EXPRDUP_REDUCE); + if( p->pLeft ) nByte += dupedExprSize(p->pLeft); + if( p->pRight ) nByte += dupedExprSize(p->pRight); + assert( nByte==ROUND8(nByte) ); return nByte; } /* -** This function is similar to sqlite3ExprDup(), except that if pzBuffer -** is not NULL then *pzBuffer is assumed to point to a buffer large enough -** to store the copy of expression p, the copies of p->u.zToken -** (if applicable), and the copies of the p->pLeft and p->pRight expressions, -** if any. Before returning, *pzBuffer is set to the first byte past the -** portion of the buffer copied into by this function. +** An EdupBuf is a memory allocation used to stored multiple Expr objects +** together with their Expr.zToken content. This is used to help implement +** compression while doing sqlite3ExprDup(). The top-level Expr does the +** allocation for itself and many of its decendents, then passes an instance +** of the structure down into exprDup() so that they decendents can have +** access to that memory. */ -static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ +typedef struct EdupBuf EdupBuf; +struct EdupBuf { + u8 *zAlloc; /* Memory space available for storage */ +#ifdef SQLITE_DEBUG + u8 *zEnd; /* First byte past the end of memory */ +#endif +}; + +/* +** This function is similar to sqlite3ExprDup(), except that if pEdupBuf +** is not NULL then it points to memory that can be used to store a copy +** of the input Expr p together with its p->u.zToken (if any). pEdupBuf +** is updated with the new buffer tail prior to returning. +*/ +static Expr *exprDup( + sqlite3 *db, /* Database connection (for memory allocation) */ + const Expr *p, /* Expr tree to be duplicated */ + int dupFlags, /* EXPRDUP_REDUCE for compression. 0 if not */ + EdupBuf *pEdupBuf /* Preallocated storage space, or NULL */ +){ Expr *pNew; /* Value to return */ - u8 *zAlloc; /* Memory space from which to build Expr object */ + EdupBuf sEdupBuf; /* Memory space from which to build Expr object */ u32 staticFlag; /* EP_Static if space not obtained from malloc */ + int nToken = -1; /* Space needed for p->u.zToken. -1 means unknown */ assert( db!=0 ); assert( p ); assert( dupFlags==0 || dupFlags==EXPRDUP_REDUCE ); - assert( pzBuffer==0 || dupFlags==EXPRDUP_REDUCE ); + assert( pEdupBuf==0 || dupFlags==EXPRDUP_REDUCE ); /* Figure out where to write the new Expr structure. */ - if( pzBuffer ){ - zAlloc = *pzBuffer; + if( pEdupBuf ){ + sEdupBuf.zAlloc = pEdupBuf->zAlloc; +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = pEdupBuf->zEnd; +#endif staticFlag = EP_Static; + assert( sEdupBuf.zAlloc!=0 ); + assert( dupFlags==EXPRDUP_REDUCE ); }else{ - zAlloc = sqlite3DbMallocRawNN(db, dupedExprSize(p, dupFlags)); + int nAlloc; + if( dupFlags ){ + nAlloc = dupedExprSize(p); + }else if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30NN(p->u.zToken)+1; + nAlloc = ROUND8(EXPR_FULLSIZE + nToken); + }else{ + nToken = 0; + nAlloc = ROUND8(EXPR_FULLSIZE); + } + assert( nAlloc==ROUND8(nAlloc) ); + sEdupBuf.zAlloc = sqlite3DbMallocRawNN(db, nAlloc); +#ifdef SQLITE_DEBUG + sEdupBuf.zEnd = sEdupBuf.zAlloc ? sEdupBuf.zAlloc+nAlloc : 0; +#endif + staticFlag = 0; } - pNew = (Expr *)zAlloc; + pNew = (Expr *)sEdupBuf.zAlloc; + assert( EIGHT_BYTE_ALIGNMENT(pNew) ); if( pNew ){ /* Set nNewSize to the size allocated for the structure pointed to @@ -89856,76 +111027,105 @@ static Expr *exprDup(sqlite3 *db, Expr *p, int dupFlags, u8 **pzBuffer){ ** by the copy of the p->u.zToken string (if any). */ const unsigned nStructSize = dupedExprStructSize(p, dupFlags); - const int nNewSize = nStructSize & 0xfff; - int nToken; - if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ - nToken = sqlite3Strlen30(p->u.zToken) + 1; - }else{ - nToken = 0; + int nNewSize = nStructSize & 0xfff; + if( nToken<0 ){ + if( !ExprHasProperty(p, EP_IntValue) && p->u.zToken ){ + nToken = sqlite3Strlen30(p->u.zToken) + 1; + }else{ + nToken = 0; + } } if( dupFlags ){ + assert( (int)(sEdupBuf.zEnd - sEdupBuf.zAlloc) >= nNewSize+nToken ); assert( ExprHasProperty(p, EP_Reduced)==0 ); - memcpy(zAlloc, p, nNewSize); + memcpy(sEdupBuf.zAlloc, p, nNewSize); }else{ u32 nSize = (u32)exprStructSize(p); - memcpy(zAlloc, p, nSize); - if( nSize= + (int)EXPR_FULLSIZE+nToken ); + memcpy(sEdupBuf.zAlloc, p, nSize); + if( nSizeflags &= ~(EP_Reduced|EP_TokenOnly|EP_Static|EP_MemToken); + pNew->flags &= ~(EP_Reduced|EP_TokenOnly|EP_Static); pNew->flags |= nStructSize & (EP_Reduced|EP_TokenOnly); pNew->flags |= staticFlag; + ExprClearVVAProperties(pNew); + if( dupFlags ){ + ExprSetVVAProperty(pNew, EP_Immutable); + } /* Copy the p->u.zToken string, if any. */ - if( nToken ){ - char *zToken = pNew->u.zToken = (char*)&zAlloc[nNewSize]; + assert( nToken>=0 ); + if( nToken>0 ){ + char *zToken = pNew->u.zToken = (char*)&sEdupBuf.zAlloc[nNewSize]; memcpy(zToken, p->u.zToken, nToken); + nNewSize += nToken; } + sEdupBuf.zAlloc += ROUND8(nNewSize); + + if( ((p->flags|pNew->flags)&(EP_TokenOnly|EP_Leaf))==0 ){ - if( 0==((p->flags|pNew->flags) & EP_TokenOnly) ){ /* Fill in the pNew->x.pSelect or pNew->x.pList member. */ - if( ExprHasProperty(p, EP_xIsSelect) ){ + if( ExprUseXSelect(p) ){ pNew->x.pSelect = sqlite3SelectDup(db, p->x.pSelect, dupFlags); }else{ - pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, dupFlags); + pNew->x.pList = sqlite3ExprListDup(db, p->x.pList, + p->op!=TK_ORDER ? dupFlags : 0); } - } - /* Fill in pNew->pLeft and pNew->pRight. */ - if( ExprHasProperty(pNew, EP_Reduced|EP_TokenOnly) ){ - zAlloc += dupedExprNodeSize(p, dupFlags); - if( ExprHasProperty(pNew, EP_Reduced) ){ - pNew->pLeft = p->pLeft ? - exprDup(db, p->pLeft, EXPRDUP_REDUCE, &zAlloc) : 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(p, EP_WinFunc) ){ + pNew->y.pWin = sqlite3WindowDup(db, pNew, p->y.pWin); + assert( ExprHasProperty(pNew, EP_WinFunc) ); + } +#endif /* SQLITE_OMIT_WINDOWFUNC */ + + /* Fill in pNew->pLeft and pNew->pRight. */ + if( dupFlags ){ + if( p->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); + }else{ + pNew->pLeft = p->pLeft ? + exprDup(db, p->pLeft, EXPRDUP_REDUCE, &sEdupBuf) : 0; + } pNew->pRight = p->pRight ? - exprDup(db, p->pRight, EXPRDUP_REDUCE, &zAlloc) : 0; - } - if( pzBuffer ){ - *pzBuffer = zAlloc; - } - }else{ - if( !ExprHasProperty(p, EP_TokenOnly) ){ - pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + exprDup(db, p->pRight, EXPRDUP_REDUCE, &sEdupBuf) : 0; + }else{ + if( p->op==TK_SELECT_COLUMN ){ + pNew->pLeft = p->pLeft; + assert( p->pRight==0 + || p->pRight==p->pLeft + || ExprHasProperty(p->pLeft, EP_Subquery) ); + }else{ + pNew->pLeft = sqlite3ExprDup(db, p->pLeft, 0); + } pNew->pRight = sqlite3ExprDup(db, p->pRight, 0); } } } + if( pEdupBuf ) memcpy(pEdupBuf, &sEdupBuf, sizeof(sEdupBuf)); + assert( sEdupBuf.zAlloc <= sEdupBuf.zEnd ); return pNew; } /* -** Create and return a deep copy of the object passed as the second +** Create and return a deep copy of the object passed as the second ** argument. If an OOM condition is encountered, NULL is returned ** and the db->mallocFailed flag set. */ #ifndef SQLITE_OMIT_CTE -static With *withDup(sqlite3 *db, With *p){ +SQLITE_PRIVATE With *sqlite3WithDup(sqlite3 *db, With *p){ With *pRet = 0; if( p ){ - int nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); + sqlite3_int64 nByte = sizeof(*p) + sizeof(p->a[0]) * (p->nCte-1); pRet = sqlite3DbMallocZero(db, nByte); if( pRet ){ int i; @@ -89934,15 +111134,49 @@ static With *withDup(sqlite3 *db, With *p){ pRet->a[i].pSelect = sqlite3SelectDup(db, p->a[i].pSelect, 0); pRet->a[i].pCols = sqlite3ExprListDup(db, p->a[i].pCols, 0); pRet->a[i].zName = sqlite3DbStrDup(db, p->a[i].zName); + pRet->a[i].eM10d = p->a[i].eM10d; } } } return pRet; } #else -# define withDup(x,y) 0 +# define sqlite3WithDup(x,y) 0 #endif +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** The gatherSelectWindows() procedure and its helper routine +** gatherSelectWindowsCallback() are used to scan all the expressions +** an a newly duplicated SELECT statement and gather all of the Window +** objects found there, assembling them onto the linked list at Select->pWin. +*/ +static int gatherSelectWindowsCallback(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_WinFunc) ){ + Select *pSelect = pWalker->u.pSelect; + Window *pWin = pExpr->y.pWin; + assert( pWin ); + assert( IsWindowFunc(pExpr) ); + assert( pWin->ppThis==0 ); + sqlite3WindowLink(pSelect, pWin); + } + return WRC_Continue; +} +static int gatherSelectWindowsSelectCallback(Walker *pWalker, Select *p){ + return p==pWalker->u.pSelect ? WRC_Continue : WRC_Prune; +} +static void gatherSelectWindows(Select *p){ + Walker w; + w.xExprCallback = gatherSelectWindowsCallback; + w.xSelectCallback = gatherSelectWindowsSelectCallback; + w.xSelectCallback2 = 0; + w.pParse = 0; + w.u.pSelect = p; + sqlite3WalkSelect(&w, p); +} +#endif + + /* ** The following group of routines make deep copies of expressions, ** expression lists, ID lists, and select statements. The copies can @@ -89950,7 +111184,7 @@ static With *withDup(sqlite3 *db, With *p){ ** without effecting the originals. ** ** The expression list, ID, and source lists return by sqlite3ExprListDup(), -** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded +** sqlite3IdListDup(), and sqlite3SrcListDup() can not be further expanded ** by subsequent calls to sqlite*ListAppend() routines. ** ** Any tables that the SrcList might point to are not duplicated. @@ -89960,34 +111194,49 @@ static With *withDup(sqlite3 *db, With *p){ ** truncated version of the usual Expr structure that will be stored as ** part of the in-memory representation of the database schema. */ -SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, Expr *p, int flags){ +SQLITE_PRIVATE Expr *sqlite3ExprDup(sqlite3 *db, const Expr *p, int flags){ assert( flags==0 || flags==EXPRDUP_REDUCE ); return p ? exprDup(db, p, flags, 0) : 0; } -SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags){ +SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, const ExprList *p, int flags){ ExprList *pNew; - struct ExprList_item *pItem, *pOldItem; + struct ExprList_item *pItem; + const struct ExprList_item *pOldItem; int i; + Expr *pPriorSelectColOld = 0; + Expr *pPriorSelectColNew = 0; assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + pNew = sqlite3DbMallocRawNN(db, sqlite3DbMallocSize(db, p)); if( pNew==0 ) return 0; - pNew->nExpr = i = p->nExpr; - if( (flags & EXPRDUP_REDUCE)==0 ) for(i=1; inExpr; i+=i){} - pNew->a = pItem = sqlite3DbMallocRawNN(db, i*sizeof(p->a[0]) ); - if( pItem==0 ){ - sqlite3DbFree(db, pNew); - return 0; - } + pNew->nExpr = p->nExpr; + pNew->nAlloc = p->nAlloc; + pItem = pNew->a; pOldItem = p->a; for(i=0; inExpr; i++, pItem++, pOldItem++){ Expr *pOldExpr = pOldItem->pExpr; + Expr *pNewExpr; pItem->pExpr = sqlite3ExprDup(db, pOldExpr, flags); - pItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pItem->zSpan = sqlite3DbStrDup(db, pOldItem->zSpan); - pItem->sortOrder = pOldItem->sortOrder; - pItem->done = 0; - pItem->bSpanIsTab = pOldItem->bSpanIsTab; + if( pOldExpr + && pOldExpr->op==TK_SELECT_COLUMN + && (pNewExpr = pItem->pExpr)!=0 + ){ + if( pNewExpr->pRight ){ + pPriorSelectColOld = pOldExpr->pRight; + pPriorSelectColNew = pNewExpr->pRight; + pNewExpr->pLeft = pNewExpr->pRight; + }else{ + if( pOldExpr->pLeft!=pPriorSelectColOld ){ + pPriorSelectColOld = pOldExpr->pLeft; + pPriorSelectColNew = sqlite3ExprDup(db, pPriorSelectColOld, flags); + pNewExpr->pRight = pPriorSelectColNew; + } + pNewExpr->pLeft = pPriorSelectColNew; + } + } + pItem->zEName = sqlite3DbStrDup(db, pOldItem->zEName); + pItem->fg = pOldItem->fg; + pItem->fg.done = 0; pItem->u = pOldItem->u; } return pNew; @@ -89995,13 +111244,13 @@ SQLITE_PRIVATE ExprList *sqlite3ExprListDup(sqlite3 *db, ExprList *p, int flags) /* ** If cursors, triggers, views and subqueries are all omitted from -** the build, then none of the following routines, except for +** the build, then none of the following routines, except for ** sqlite3SelectDup(), can be called. sqlite3SelectDup() is sometimes ** called with a NULL argument. */ #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) \ || !defined(SQLITE_OMIT_SUBQUERY) -SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ +SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, const SrcList *p, int flags){ SrcList *pNew; int i; int nByte; @@ -90012,90 +111261,126 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListDup(sqlite3 *db, SrcList *p, int flags){ if( pNew==0 ) return 0; pNew->nSrc = pNew->nAlloc = p->nSrc; for(i=0; inSrc; i++){ - struct SrcList_item *pNewItem = &pNew->a[i]; - struct SrcList_item *pOldItem = &p->a[i]; + SrcItem *pNewItem = &pNew->a[i]; + const SrcItem *pOldItem = &p->a[i]; Table *pTab; - pNewItem->pSchema = pOldItem->pSchema; - pNewItem->zDatabase = sqlite3DbStrDup(db, pOldItem->zDatabase); + pNewItem->fg = pOldItem->fg; + if( pOldItem->fg.isSubquery ){ + Subquery *pNewSubq = sqlite3DbMallocRaw(db, sizeof(Subquery)); + if( pNewSubq==0 ){ + assert( db->mallocFailed ); + pNewItem->fg.isSubquery = 0; + }else{ + memcpy(pNewSubq, pOldItem->u4.pSubq, sizeof(*pNewSubq)); + pNewSubq->pSelect = sqlite3SelectDup(db, pNewSubq->pSelect, flags); + if( pNewSubq->pSelect==0 ){ + sqlite3DbFree(db, pNewSubq); + pNewSubq = 0; + pNewItem->fg.isSubquery = 0; + } + } + pNewItem->u4.pSubq = pNewSubq; + }else if( pOldItem->fg.fixedSchema ){ + pNewItem->u4.pSchema = pOldItem->u4.pSchema; + }else{ + pNewItem->u4.zDatabase = sqlite3DbStrDup(db, pOldItem->u4.zDatabase); + } pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); pNewItem->zAlias = sqlite3DbStrDup(db, pOldItem->zAlias); - pNewItem->fg = pOldItem->fg; pNewItem->iCursor = pOldItem->iCursor; - pNewItem->addrFillSub = pOldItem->addrFillSub; - pNewItem->regReturn = pOldItem->regReturn; if( pNewItem->fg.isIndexedBy ){ pNewItem->u1.zIndexedBy = sqlite3DbStrDup(db, pOldItem->u1.zIndexedBy); - } - pNewItem->pIBIndex = pOldItem->pIBIndex; - if( pNewItem->fg.isTabFunc ){ - pNewItem->u1.pFuncArg = + }else if( pNewItem->fg.isTabFunc ){ + pNewItem->u1.pFuncArg = sqlite3ExprListDup(db, pOldItem->u1.pFuncArg, flags); + }else{ + pNewItem->u1.nRow = pOldItem->u1.nRow; } - pTab = pNewItem->pTab = pOldItem->pTab; + pNewItem->u2 = pOldItem->u2; + if( pNewItem->fg.isCte ){ + pNewItem->u2.pCteUse->nUse++; + } + pTab = pNewItem->pSTab = pOldItem->pSTab; if( pTab ){ - pTab->nRef++; + pTab->nTabRef++; + } + if( pOldItem->fg.isUsing ){ + assert( pNewItem->fg.isUsing ); + pNewItem->u3.pUsing = sqlite3IdListDup(db, pOldItem->u3.pUsing); + }else{ + pNewItem->u3.pOn = sqlite3ExprDup(db, pOldItem->u3.pOn, flags); } - pNewItem->pSelect = sqlite3SelectDup(db, pOldItem->pSelect, flags); - pNewItem->pOn = sqlite3ExprDup(db, pOldItem->pOn, flags); - pNewItem->pUsing = sqlite3IdListDup(db, pOldItem->pUsing); pNewItem->colUsed = pOldItem->colUsed; } return pNew; } -SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, IdList *p){ +SQLITE_PRIVATE IdList *sqlite3IdListDup(sqlite3 *db, const IdList *p){ IdList *pNew; int i; assert( db!=0 ); if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + assert( p->eU4!=EU4_EXPR ); + pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew)+(p->nId-1)*sizeof(p->a[0]) ); if( pNew==0 ) return 0; pNew->nId = p->nId; - pNew->a = sqlite3DbMallocRawNN(db, p->nId*sizeof(p->a[0]) ); - if( pNew->a==0 ){ - sqlite3DbFree(db, pNew); - return 0; - } - /* Note that because the size of the allocation for p->a[] is not - ** necessarily a power of two, sqlite3IdListAppend() may not be called - ** on the duplicate created by this function. */ + pNew->eU4 = p->eU4; for(i=0; inId; i++){ struct IdList_item *pNewItem = &pNew->a[i]; - struct IdList_item *pOldItem = &p->a[i]; + const struct IdList_item *pOldItem = &p->a[i]; pNewItem->zName = sqlite3DbStrDup(db, pOldItem->zName); - pNewItem->idx = pOldItem->idx; + pNewItem->u4 = pOldItem->u4; } return pNew; } -SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ - Select *pNew, *pPrior; +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *pDup, int flags){ + Select *pRet = 0; + Select *pNext = 0; + Select **pp = &pRet; + const Select *p; + assert( db!=0 ); - if( p==0 ) return 0; - pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); - if( pNew==0 ) return 0; - pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); - pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); - pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); - pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); - pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); - pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); - pNew->op = p->op; - pNew->pPrior = pPrior = sqlite3SelectDup(db, p->pPrior, flags); - if( pPrior ) pPrior->pNext = pNew; - pNew->pNext = 0; - pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); - pNew->pOffset = sqlite3ExprDup(db, p->pOffset, flags); - pNew->iLimit = 0; - pNew->iOffset = 0; - pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; - pNew->addrOpenEphm[0] = -1; - pNew->addrOpenEphm[1] = -1; - pNew->nSelectRow = p->nSelectRow; - pNew->pWith = withDup(db, p->pWith); - sqlite3SelectSetName(pNew, p->zSelName); - return pNew; + for(p=pDup; p; p=p->pPrior){ + Select *pNew = sqlite3DbMallocRawNN(db, sizeof(*p) ); + if( pNew==0 ) break; + pNew->pEList = sqlite3ExprListDup(db, p->pEList, flags); + pNew->pSrc = sqlite3SrcListDup(db, p->pSrc, flags); + pNew->pWhere = sqlite3ExprDup(db, p->pWhere, flags); + pNew->pGroupBy = sqlite3ExprListDup(db, p->pGroupBy, flags); + pNew->pHaving = sqlite3ExprDup(db, p->pHaving, flags); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, flags); + pNew->op = p->op; + pNew->pNext = pNext; + pNew->pPrior = 0; + pNew->pLimit = sqlite3ExprDup(db, p->pLimit, flags); + pNew->iLimit = 0; + pNew->iOffset = 0; + pNew->selFlags = p->selFlags & ~SF_UsesEphemeral; + pNew->addrOpenEphm[0] = -1; + pNew->addrOpenEphm[1] = -1; + pNew->nSelectRow = p->nSelectRow; + pNew->pWith = sqlite3WithDup(db, p->pWith); +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = sqlite3WindowListDup(db, p->pWinDefn); + if( p->pWin && db->mallocFailed==0 ) gatherSelectWindows(pNew); +#endif + pNew->selId = p->selId; + if( db->mallocFailed ){ + /* Any prior OOM might have left the Select object incomplete. + ** Delete the whole thing rather than allow an incomplete Select + ** to be used by the code generator. */ + pNew->pNext = 0; + sqlite3SelectDelete(db, pNew); + break; + } + *pp = pNew; + pp = &pNew->pPrior; + pNext = pNew; + } + return pRet; } #else -SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ +SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, const Select *p, int flags){ assert( p==0 ); return 0; } @@ -90106,65 +111391,178 @@ SQLITE_PRIVATE Select *sqlite3SelectDup(sqlite3 *db, Select *p, int flags){ ** Add a new element to the end of an expression list. If pList is ** initially NULL, then create a new expression list. ** +** The pList argument must be either NULL or a pointer to an ExprList +** obtained from a prior call to sqlite3ExprListAppend(). +** ** If a memory allocation error occurs, the entire list is freed and ** NULL is returned. If non-NULL is returned, then it is guaranteed ** that the new entry was successfully appended. */ +static const struct ExprList_item zeroItem = {0}; +SQLITE_PRIVATE SQLITE_NOINLINE ExprList *sqlite3ExprListAppendNew( + sqlite3 *db, /* Database handle. Used for memory allocation */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + struct ExprList_item *pItem; + ExprList *pList; + + pList = sqlite3DbMallocRawNN(db, sizeof(ExprList)+sizeof(pList->a[0])*4 ); + if( pList==0 ){ + sqlite3ExprDelete(db, pExpr); + return 0; + } + pList->nAlloc = 4; + pList->nExpr = 1; + pItem = &pList->a[0]; + *pItem = zeroItem; + pItem->pExpr = pExpr; + return pList; +} +SQLITE_PRIVATE SQLITE_NOINLINE ExprList *sqlite3ExprListAppendGrow( + sqlite3 *db, /* Database handle. Used for memory allocation */ + ExprList *pList, /* List to which to append. Might be NULL */ + Expr *pExpr /* Expression to be appended. Might be NULL */ +){ + struct ExprList_item *pItem; + ExprList *pNew; + pList->nAlloc *= 2; + pNew = sqlite3DbRealloc(db, pList, + sizeof(*pList)+(pList->nAlloc-1)*sizeof(pList->a[0])); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pList); + sqlite3ExprDelete(db, pExpr); + return 0; + }else{ + pList = pNew; + } + pItem = &pList->a[pList->nExpr++]; + *pItem = zeroItem; + pItem->pExpr = pExpr; + return pList; +} SQLITE_PRIVATE ExprList *sqlite3ExprListAppend( Parse *pParse, /* Parsing context */ ExprList *pList, /* List to which to append. Might be NULL */ Expr *pExpr /* Expression to be appended. Might be NULL */ ){ - sqlite3 *db = pParse->db; - assert( db!=0 ); + struct ExprList_item *pItem; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(db, sizeof(ExprList) ); - if( pList==0 ){ - goto no_mem; - } - pList->nExpr = 0; - pList->a = sqlite3DbMallocRawNN(db, sizeof(pList->a[0])); - if( pList->a==0 ) goto no_mem; - }else if( (pList->nExpr & (pList->nExpr-1))==0 ){ - struct ExprList_item *a; - assert( pList->nExpr>0 ); - a = sqlite3DbRealloc(db, pList->a, pList->nExpr*2*sizeof(pList->a[0])); - if( a==0 ){ - goto no_mem; - } - pList->a = a; + return sqlite3ExprListAppendNew(pParse->db,pExpr); } - assert( pList->a!=0 ); - if( 1 ){ - struct ExprList_item *pItem = &pList->a[pList->nExpr++]; - memset(pItem, 0, sizeof(*pItem)); - pItem->pExpr = pExpr; + if( pList->nAllocnExpr+1 ){ + return sqlite3ExprListAppendGrow(pParse->db,pList,pExpr); } + pItem = &pList->a[pList->nExpr++]; + *pItem = zeroItem; + pItem->pExpr = pExpr; return pList; +} -no_mem: - /* Avoid leaking memory if malloc has failed. */ - sqlite3ExprDelete(db, pExpr); - sqlite3ExprListDelete(db, pList); - return 0; +/* +** pColumns and pExpr form a vector assignment which is part of the SET +** clause of an UPDATE statement. Like this: +** +** (a,b,c) = (expr1,expr2,expr3) +** Or: (a,b,c) = (SELECT x,y,z FROM ....) +** +** For each term of the vector assignment, append new entries to the +** expression list pList. In the case of a subquery on the RHS, append +** TK_SELECT_COLUMN expressions. +*/ +SQLITE_PRIVATE ExprList *sqlite3ExprListAppendVector( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + IdList *pColumns, /* List of names of LHS of the assignment */ + Expr *pExpr /* Vector expression to be appended. Might be NULL */ +){ + sqlite3 *db = pParse->db; + int n; + int i; + int iFirst = pList ? pList->nExpr : 0; + /* pColumns can only be NULL due to an OOM but an OOM will cause an + ** exit prior to this routine being invoked */ + if( NEVER(pColumns==0) ) goto vector_append_error; + if( pExpr==0 ) goto vector_append_error; + + /* If the RHS is a vector, then we can immediately check to see that + ** the size of the RHS and LHS match. But if the RHS is a SELECT, + ** wildcards ("*") in the result set of the SELECT must be expanded before + ** we can do the size check, so defer the size check until code generation. + */ + if( pExpr->op!=TK_SELECT && pColumns->nId!=(n=sqlite3ExprVectorSize(pExpr)) ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pColumns->nId, n); + goto vector_append_error; + } + + for(i=0; inId; i++){ + Expr *pSubExpr = sqlite3ExprForVectorField(pParse, pExpr, i, pColumns->nId); + assert( pSubExpr!=0 || db->mallocFailed ); + if( pSubExpr==0 ) continue; + pList = sqlite3ExprListAppend(pParse, pList, pSubExpr); + if( pList ){ + assert( pList->nExpr==iFirst+i+1 ); + pList->a[pList->nExpr-1].zEName = pColumns->a[i].zName; + pColumns->a[i].zName = 0; + } + } + + if( !db->mallocFailed && pExpr->op==TK_SELECT && ALWAYS(pList!=0) ){ + Expr *pFirst = pList->a[iFirst].pExpr; + assert( pFirst!=0 ); + assert( pFirst->op==TK_SELECT_COLUMN ); + + /* Store the SELECT statement in pRight so it will be deleted when + ** sqlite3ExprListDelete() is called */ + pFirst->pRight = pExpr; + pExpr = 0; + + /* Remember the size of the LHS in iTable so that we can check that + ** the RHS and LHS sizes match during code generation. */ + pFirst->iTable = pColumns->nId; + } + +vector_append_error: + sqlite3ExprUnmapAndDelete(pParse, pExpr); + sqlite3IdListDelete(db, pColumns); + return pList; } /* ** Set the sort order for the last element on the given ExprList. */ -SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ +SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder, int eNulls){ + struct ExprList_item *pItem; if( p==0 ) return; - assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC>=0 && SQLITE_SO_DESC>0 ); assert( p->nExpr>0 ); - if( iSortOrder<0 ){ - assert( p->a[p->nExpr-1].sortOrder==SQLITE_SO_ASC ); - return; + + assert( SQLITE_SO_UNDEFINED<0 && SQLITE_SO_ASC==0 && SQLITE_SO_DESC>0 ); + assert( iSortOrder==SQLITE_SO_UNDEFINED + || iSortOrder==SQLITE_SO_ASC + || iSortOrder==SQLITE_SO_DESC + ); + assert( eNulls==SQLITE_SO_UNDEFINED + || eNulls==SQLITE_SO_ASC + || eNulls==SQLITE_SO_DESC + ); + + pItem = &p->a[p->nExpr-1]; + assert( pItem->fg.bNulls==0 ); + if( iSortOrder==SQLITE_SO_UNDEFINED ){ + iSortOrder = SQLITE_SO_ASC; + } + pItem->fg.sortFlags = (u8)iSortOrder; + + if( eNulls!=SQLITE_SO_UNDEFINED ){ + pItem->fg.bNulls = 1; + if( iSortOrder!=eNulls ){ + pItem->fg.sortFlags |= KEYINFO_ORDER_BIGNULL; + } } - p->a[p->nExpr-1].sortOrder = (u8)iSortOrder; } /* -** Set the ExprList.a[].zName element of the most recently added item +** Set the ExprList.a[].zEName element of the most recently added item ** on the expression list. ** ** pList might be NULL following an OOM error. But pName should never be @@ -90174,17 +111572,27 @@ SQLITE_PRIVATE void sqlite3ExprListSetSortOrder(ExprList *p, int iSortOrder){ SQLITE_PRIVATE void sqlite3ExprListSetName( Parse *pParse, /* Parsing context */ ExprList *pList, /* List to which to add the span. */ - Token *pName, /* Name to be added */ + const Token *pName, /* Name to be added */ int dequote /* True to cause the name to be dequoted */ ){ assert( pList!=0 || pParse->db->mallocFailed!=0 ); + assert( pParse->eParseMode!=PARSE_MODE_UNMAP || dequote==0 ); if( pList ){ struct ExprList_item *pItem; assert( pList->nExpr>0 ); pItem = &pList->a[pList->nExpr-1]; - assert( pItem->zName==0 ); - pItem->zName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); - if( dequote ) sqlite3Dequote(pItem->zName); + assert( pItem->zEName==0 ); + assert( pItem->fg.eEName==ENAME_NAME ); + pItem->zEName = sqlite3DbStrNDup(pParse->db, pName->z, pName->n); + if( dequote ){ + /* If dequote==0, then pName->z does not point to part of a DDL + ** statement handled by the parser. And so no token need be added + ** to the token-map. */ + sqlite3Dequote(pItem->zEName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (const void*)pItem->zEName, pName); + } + } } } @@ -90199,17 +111607,18 @@ SQLITE_PRIVATE void sqlite3ExprListSetName( SQLITE_PRIVATE void sqlite3ExprListSetSpan( Parse *pParse, /* Parsing context */ ExprList *pList, /* List to which to add the span. */ - ExprSpan *pSpan /* The span to be added */ + const char *zStart, /* Start of the span */ + const char *zEnd /* End of the span */ ){ sqlite3 *db = pParse->db; assert( pList!=0 || db->mallocFailed!=0 ); if( pList ){ struct ExprList_item *pItem = &pList->a[pList->nExpr-1]; assert( pList->nExpr>0 ); - assert( db->mallocFailed || pItem->pExpr==pSpan->pExpr ); - sqlite3DbFree(db, pItem->zSpan); - pItem->zSpan = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); + if( pItem->zEName==0 ){ + pItem->zEName = sqlite3DbSpanDup(db, zStart, zEnd); + pItem->fg.eEName = ENAME_SPAN; + } } } @@ -90234,20 +111643,23 @@ SQLITE_PRIVATE void sqlite3ExprListCheckLength( ** Delete an entire expression list. */ static SQLITE_NOINLINE void exprListDeleteNN(sqlite3 *db, ExprList *pList){ - int i; - struct ExprList_item *pItem; - assert( pList->a!=0 || pList->nExpr==0 ); - for(pItem=pList->a, i=0; inExpr; i++, pItem++){ + int i = pList->nExpr; + struct ExprList_item *pItem = pList->a; + assert( pList->nExpr>0 ); + assert( db!=0 ); + do{ sqlite3ExprDelete(db, pItem->pExpr); - sqlite3DbFree(db, pItem->zName); - sqlite3DbFree(db, pItem->zSpan); - } - sqlite3DbFree(db, pList->a); - sqlite3DbFree(db, pList); + if( pItem->zEName ) sqlite3DbNNFreeNN(db, pItem->zEName); + pItem++; + }while( --i>0 ); + sqlite3DbNNFreeNN(db, pList); } SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ if( pList ) exprListDeleteNN(db, pList); } +SQLITE_PRIVATE void sqlite3ExprListDeleteGeneric(sqlite3 *db, void *pList){ + if( ALWAYS(pList) ) exprListDeleteNN(db, (ExprList*)pList); +} /* ** Return the bitwise-OR of all Expr.flags fields in the given @@ -90256,16 +111668,150 @@ SQLITE_PRIVATE void sqlite3ExprListDelete(sqlite3 *db, ExprList *pList){ SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ int i; u32 m = 0; - if( pList ){ - for(i=0; inExpr; i++){ - Expr *pExpr = pList->a[i].pExpr; - assert( pExpr!=0 ); - m |= pExpr->flags; - } + assert( pList!=0 ); + for(i=0; inExpr; i++){ + Expr *pExpr = pList->a[i].pExpr; + assert( pExpr!=0 ); + m |= pExpr->flags; } return m; } +/* +** This is a SELECT-node callback for the expression walker that +** always "fails". By "fail" in this case, we mean set +** pWalker->eCode to zero and abort. +** +** This callback is used by multiple expression walkers. +*/ +SQLITE_PRIVATE int sqlite3SelectWalkFail(Walker *pWalker, Select *NotUsed){ + UNUSED_PARAMETER(NotUsed); + pWalker->eCode = 0; + return WRC_Abort; +} + +/* +** Check the input string to see if it is "true" or "false" (in any case). +** +** If the string is.... Return +** "true" EP_IsTrue +** "false" EP_IsFalse +** anything else 0 +*/ +SQLITE_PRIVATE u32 sqlite3IsTrueOrFalse(const char *zIn){ + if( sqlite3StrICmp(zIn, "true")==0 ) return EP_IsTrue; + if( sqlite3StrICmp(zIn, "false")==0 ) return EP_IsFalse; + return 0; +} + + +/* +** If the input expression is an ID with the name "true" or "false" +** then convert it into an TK_TRUEFALSE term. Return non-zero if +** the conversion happened, and zero if the expression is unaltered. +*/ +SQLITE_PRIVATE int sqlite3ExprIdToTrueFalse(Expr *pExpr){ + u32 v; + assert( pExpr->op==TK_ID || pExpr->op==TK_STRING ); + if( !ExprHasProperty(pExpr, EP_Quoted|EP_IntValue) + && (v = sqlite3IsTrueOrFalse(pExpr->u.zToken))!=0 + ){ + pExpr->op = TK_TRUEFALSE; + ExprSetProperty(pExpr, v); + return 1; + } + return 0; +} + +/* +** The argument must be a TK_TRUEFALSE Expr node. Return 1 if it is TRUE +** and 0 if it is FALSE. +*/ +SQLITE_PRIVATE int sqlite3ExprTruthValue(const Expr *pExpr){ + pExpr = sqlite3ExprSkipCollateAndLikely((Expr*)pExpr); + assert( pExpr->op==TK_TRUEFALSE ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( sqlite3StrICmp(pExpr->u.zToken,"true")==0 + || sqlite3StrICmp(pExpr->u.zToken,"false")==0 ); + return pExpr->u.zToken[4]==0; +} + +/* +** If pExpr is an AND or OR expression, try to simplify it by eliminating +** terms that are always true or false. Return the simplified expression. +** Or return the original expression if no simplification is possible. +** +** Examples: +** +** (x<10) AND true => (x<10) +** (x<10) AND false => false +** (x<10) AND (y=22 OR false) => (x<10) AND (y=22) +** (x<10) AND (y=22 OR true) => (x<10) +** (y=22) OR true => true +*/ +SQLITE_PRIVATE Expr *sqlite3ExprSimplifiedAndOr(Expr *pExpr){ + assert( pExpr!=0 ); + if( pExpr->op==TK_AND || pExpr->op==TK_OR ){ + Expr *pRight = sqlite3ExprSimplifiedAndOr(pExpr->pRight); + Expr *pLeft = sqlite3ExprSimplifiedAndOr(pExpr->pLeft); + if( ExprAlwaysTrue(pLeft) || ExprAlwaysFalse(pRight) ){ + pExpr = pExpr->op==TK_AND ? pRight : pLeft; + }else if( ExprAlwaysTrue(pRight) || ExprAlwaysFalse(pLeft) ){ + pExpr = pExpr->op==TK_AND ? pLeft : pRight; + } + } + return pExpr; +} + +/* +** pExpr is a TK_FUNCTION node. Try to determine whether or not the +** function is a constant function. A function is constant if all of +** the following are true: +** +** (1) It is a scalar function (not an aggregate or window function) +** (2) It has either the SQLITE_FUNC_CONSTANT or SQLITE_FUNC_SLOCHNG +** property. +** (3) All of its arguments are constants +** +** This routine sets pWalker->eCode to 0 if pExpr is not a constant. +** It makes no changes to pWalker->eCode if pExpr is constant. In +** every case, it returns WRC_Abort. +** +** Called as a service subroutine from exprNodeIsConstant(). +*/ +static SQLITE_NOINLINE int exprNodeIsConstantFunction( + Walker *pWalker, + Expr *pExpr +){ + int n; /* Number of arguments */ + ExprList *pList; /* List of arguments */ + FuncDef *pDef; /* The function */ + sqlite3 *db; /* The database */ + + assert( pExpr->op==TK_FUNCTION ); + if( ExprHasProperty(pExpr, EP_TokenOnly) + || (pList = pExpr->x.pList)==0 + ){; + n = 0; + }else{ + n = pList->nExpr; + sqlite3WalkExprList(pWalker, pList); + if( pWalker->eCode==0 ) return WRC_Abort; + } + db = pWalker->pParse->db; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( pDef==0 + || pDef->xFinalize!=0 + || (pDef->funcFlags & (SQLITE_FUNC_CONSTANT|SQLITE_FUNC_SLOCHNG))==0 + || ExprHasProperty(pExpr, EP_WinFunc) + ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; +} + + /* ** These routines are Walker callbacks used to check expressions to ** see if they are "constant" for some definition of constant. The @@ -90282,21 +111828,23 @@ SQLITE_PRIVATE u32 sqlite3ExprListFlags(const ExprList *pList){ ** In all cases, the callbacks set Walker.eCode=0 and abort if the expression ** is found to not be a constant. ** -** The sqlite3ExprIsConstantOrFunction() is used for evaluating expressions -** in a CREATE TABLE statement. The Walker.eCode value is 5 when parsing -** an existing schema and 4 when processing a new statement. A bound -** parameter raises an error for new statements, but is silently converted -** to NULL for existing schemas. This allows sqlite_master tables that +** The sqlite3ExprIsConstantOrFunction() is used for evaluating DEFAULT +** expressions in a CREATE TABLE statement. The Walker.eCode value is 5 +** when parsing an existing schema out of the sqlite_schema table and 4 +** when processing a new CREATE TABLE statement. A bound parameter raises +** an error for new statements, but is silently converted +** to NULL for existing schemas. This allows sqlite_schema tables that ** contain a bound parameter because they were generated by older versions ** of SQLite to be parsed by newer versions of SQLite without raising a ** malformed schema error. */ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ + assert( pWalker->eCode>0 ); /* If pWalker->eCode is 2 then any term of the expression that comes from - ** the ON or USING clauses of a left join disqualifies the expression + ** the ON or USING clauses of an outer join disqualifies the expression ** from being considered constant. */ - if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_FromJoin) ){ + if( pWalker->eCode==2 && ExprHasProperty(pExpr, EP_OuterON) ){ pWalker->eCode = 0; return WRC_Abort; } @@ -90306,13 +111854,24 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ ** and either pWalker->eCode==4 or 5 or the function has the ** SQLITE_FUNC_CONST flag. */ case TK_FUNCTION: - if( pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc) ){ + if( (pWalker->eCode>=4 || ExprHasProperty(pExpr,EP_ConstFunc)) + && !ExprHasProperty(pExpr, EP_WinFunc) + ){ + if( pWalker->eCode==5 ) ExprSetProperty(pExpr, EP_FromDDL); return WRC_Continue; + }else if( pWalker->pParse ){ + return exprNodeIsConstantFunction(pWalker, pExpr); }else{ pWalker->eCode = 0; return WRC_Abort; } case TK_ID: + /* Convert "true" or "false" in a DEFAULT clause into the + ** appropriate TK_TRUEFALSE operator */ + if( sqlite3ExprIdToTrueFalse(pExpr) ){ + return WRC_Prune; + } + /* no break */ deliberate_fall_through case TK_COLUMN: case TK_AGG_FUNCTION: case TK_AGG_COLUMN: @@ -90320,17 +111879,28 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ testcase( pExpr->op==TK_COLUMN ); testcase( pExpr->op==TK_AGG_FUNCTION ); testcase( pExpr->op==TK_AGG_COLUMN ); + if( ExprHasProperty(pExpr, EP_FixedCol) && pWalker->eCode!=2 ){ + return WRC_Continue; + } if( pWalker->eCode==3 && pExpr->iTable==pWalker->u.iCur ){ return WRC_Continue; - }else{ - pWalker->eCode = 0; - return WRC_Abort; } + /* no break */ deliberate_fall_through + case TK_IF_NULL_ROW: + case TK_REGISTER: + case TK_DOT: + case TK_RAISE: + testcase( pExpr->op==TK_REGISTER ); + testcase( pExpr->op==TK_IF_NULL_ROW ); + testcase( pExpr->op==TK_DOT ); + testcase( pExpr->op==TK_RAISE ); + pWalker->eCode = 0; + return WRC_Abort; case TK_VARIABLE: if( pWalker->eCode==5 ){ /* Silently convert bound parameters that appear inside of CREATE ** statements into a NULL when parsing the CREATE statement text out - ** of the sqlite_master table */ + ** of the sqlite_schema table */ pExpr->op = TK_NULL; }else if( pWalker->eCode==4 ){ /* A bound parameter in a CREATE statement that originates from @@ -90338,25 +111908,22 @@ static int exprNodeIsConstant(Walker *pWalker, Expr *pExpr){ pWalker->eCode = 0; return WRC_Abort; } - /* Fall through */ + /* no break */ deliberate_fall_through default: - testcase( pExpr->op==TK_SELECT ); /* selectNodeIsConstant will disallow */ - testcase( pExpr->op==TK_EXISTS ); /* selectNodeIsConstant will disallow */ + testcase( pExpr->op==TK_SELECT ); /* sqlite3SelectWalkFail() disallows */ + testcase( pExpr->op==TK_EXISTS ); /* sqlite3SelectWalkFail() disallows */ return WRC_Continue; } } -static int selectNodeIsConstant(Walker *pWalker, Select *NotUsed){ - UNUSED_PARAMETER(NotUsed); - pWalker->eCode = 0; - return WRC_Abort; -} -static int exprIsConst(Expr *p, int initFlag, int iCur){ +static int exprIsConst(Parse *pParse, Expr *p, int initFlag){ Walker w; - memset(&w, 0, sizeof(w)); w.eCode = initFlag; + w.pParse = pParse; w.xExprCallback = exprNodeIsConstant; - w.xSelectCallback = selectNodeIsConstant; - w.u.iCur = iCur; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif sqlite3WalkExpr(&w, p); return w.eCode; } @@ -90368,19 +111935,48 @@ static int exprIsConst(Expr *p, int initFlag, int iCur){ ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is ** a constant. +** +** The pParse parameter may be NULL. But if it is NULL, there is no way +** to determine if function calls are constant or not, and hence all +** function calls will be considered to be non-constant. If pParse is +** not NULL, then a function call might be constant, depending on the +** function and on its parameters. */ -SQLITE_PRIVATE int sqlite3ExprIsConstant(Expr *p){ - return exprIsConst(p, 1, 0); +SQLITE_PRIVATE int sqlite3ExprIsConstant(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 1); } /* -** Walk an expression tree. Return non-zero if the expression is constant -** that does no originate from the ON or USING clauses of a join. -** Return 0 if it involves variables or function calls or terms from -** an ON or USING clause. +** Walk an expression tree. Return non-zero if +** +** (1) the expression is constant, and +** (2) the expression does originate in the ON or USING clause +** of a LEFT JOIN, and +** (3) the expression does not contain any EP_FixedCol TK_COLUMN +** operands created by the constant propagation optimization. +** +** When this routine returns true, it indicates that the expression +** can be added to the pParse->pConstExpr list and evaluated once when +** the prepared statement starts up. See sqlite3ExprCodeRunJustOnce(). */ -SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ - return exprIsConst(p, 2, 0); +static int sqlite3ExprIsConstantNotJoin(Parse *pParse, Expr *p){ + return exprIsConst(pParse, p, 2); +} + +/* +** This routine examines sub-SELECT statements as an expression is being +** walked as part of sqlite3ExprIsTableConstant(). Sub-SELECTs are considered +** constant as long as they are uncorrelated - meaning that they do not +** contain any terms from outer contexts. +*/ +static int exprSelectWalkTableConstant(Walker *pWalker, Select *pSelect){ + assert( pSelect!=0 ); + assert( pWalker->eCode==3 || pWalker->eCode==0 ); + if( (pSelect->selFlags & SF_Correlated)!=0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Prune; } /* @@ -90388,15 +111984,180 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantNotJoin(Expr *p){ ** for any single row of the table with cursor iCur. In other words, the ** expression must not refer to any non-deterministic function nor any ** table other than iCur. +** +** Consider uncorrelated subqueries to be constants if the bAllowSubq +** parameter is true. */ -SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ - return exprIsConst(p, 3, iCur); +static int sqlite3ExprIsTableConstant(Expr *p, int iCur, int bAllowSubq){ + Walker w; + w.eCode = 3; + w.pParse = 0; + w.xExprCallback = exprNodeIsConstant; + if( bAllowSubq ){ + w.xSelectCallback = exprSelectWalkTableConstant; + }else{ + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif + } + w.u.iCur = iCur; + sqlite3WalkExpr(&w, p); + return w.eCode; } /* -** Walk an expression tree. Return non-zero if the expression is constant -** or a function call with constant arguments. Return and 0 if there -** are any variables. +** Check pExpr to see if it is an constraint on the single data source +** pSrc = &pSrcList->a[iSrc]. In other words, check to see if pExpr +** constrains pSrc but does not depend on any other tables or data +** sources anywhere else in the query. Return true (non-zero) if pExpr +** is a constraint on pSrc only. +** +** This is an optimization. False negatives will perhaps cause slower +** queries, but false positives will yield incorrect answers. So when in +** doubt, return 0. +** +** To be an single-source constraint, the following must be true: +** +** (1) pExpr cannot refer to any table other than pSrc->iCursor. +** +** (2a) pExpr cannot use subqueries unless the bAllowSubq parameter is +** true and the subquery is non-correlated +** +** (2b) pExpr cannot use non-deterministic functions. +** +** (3) pSrc cannot be part of the left operand for a RIGHT JOIN. +** (Is there some way to relax this constraint?) +** +** (4) If pSrc is the right operand of a LEFT JOIN, then... +** (4a) pExpr must come from an ON clause.. +** (4b) and specifically the ON clause associated with the LEFT JOIN. +** +** (5) If pSrc is not the right operand of a LEFT JOIN or the left +** operand of a RIGHT JOIN, then pExpr must be from the WHERE +** clause, not an ON clause. +** +** (6) Either: +** +** (6a) pExpr does not originate in an ON or USING clause, or +** +** (6b) The ON or USING clause from which pExpr is derived is +** not to the left of a RIGHT JOIN (or FULL JOIN). +** +** Without this restriction, accepting pExpr as a single-table +** constraint might move the the ON/USING filter expression +** from the left side of a RIGHT JOIN over to the right side, +** which leads to incorrect answers. See also restriction (9) +** on push-down. +*/ +SQLITE_PRIVATE int sqlite3ExprIsSingleTableConstraint( + Expr *pExpr, /* The constraint */ + const SrcList *pSrcList, /* Complete FROM clause */ + int iSrc, /* Which element of pSrcList to use */ + int bAllowSubq /* Allow non-correlated subqueries */ +){ + const SrcItem *pSrc = &pSrcList->a[iSrc]; + if( pSrc->fg.jointype & JT_LTORJ ){ + return 0; /* rule (3) */ + } + if( pSrc->fg.jointype & JT_LEFT ){ + if( !ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* rule (4a) */ + if( pExpr->w.iJoin!=pSrc->iCursor ) return 0; /* rule (4b) */ + }else{ + if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; /* rule (5) */ + } + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) /* (6a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (6b) */ + ){ + int jj; + for(jj=0; jjw.iJoin==pSrcList->a[jj].iCursor ){ + if( (pSrcList->a[jj].fg.jointype & JT_LTORJ)!=0 ){ + return 0; /* restriction (6) */ + } + break; + } + } + } + /* Rules (1), (2a), and (2b) handled by the following: */ + return sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor, bAllowSubq); +} + + +/* +** sqlite3WalkExpr() callback used by sqlite3ExprIsConstantOrGroupBy(). +*/ +static int exprNodeIsConstantOrGroupBy(Walker *pWalker, Expr *pExpr){ + ExprList *pGroupBy = pWalker->u.pGroupBy; + int i; + + /* Check if pExpr is identical to any GROUP BY term. If so, consider + ** it constant. */ + for(i=0; inExpr; i++){ + Expr *p = pGroupBy->a[i].pExpr; + if( sqlite3ExprCompare(0, pExpr, p, -1)<2 ){ + CollSeq *pColl = sqlite3ExprNNCollSeq(pWalker->pParse, p); + if( sqlite3IsBinary(pColl) ){ + return WRC_Prune; + } + } + } + + /* Check if pExpr is a sub-select. If so, consider it variable. */ + if( ExprUseXSelect(pExpr) ){ + pWalker->eCode = 0; + return WRC_Abort; + } + + return exprNodeIsConstant(pWalker, pExpr); +} + +/* +** Walk the expression tree passed as the first argument. Return non-zero +** if the expression consists entirely of constants or copies of terms +** in pGroupBy that sort with the BINARY collation sequence. +** +** This routine is used to determine if a term of the HAVING clause can +** be promoted into the WHERE clause. In order for such a promotion to work, +** the value of the HAVING clause term must be the same for all members of +** a "group". The requirement that the GROUP BY term must be BINARY +** assumes that no other collating sequence will have a finer-grained +** grouping than binary. In other words (A=B COLLATE binary) implies +** A=B in every other collating sequence. The requirement that the +** GROUP BY be BINARY is stricter than necessary. It would also work +** to promote HAVING clauses that use the same alternative collating +** sequence as the GROUP BY term, but that is much harder to check, +** alternative collating sequences are uncommon, and this is only an +** optimization, so we take the easy way out and simply require the +** GROUP BY to use the BINARY collating sequence. +*/ +SQLITE_PRIVATE int sqlite3ExprIsConstantOrGroupBy(Parse *pParse, Expr *p, ExprList *pGroupBy){ + Walker w; + w.eCode = 1; + w.xExprCallback = exprNodeIsConstantOrGroupBy; + w.xSelectCallback = 0; + w.u.pGroupBy = pGroupBy; + w.pParse = pParse; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + +/* +** Walk an expression tree for the DEFAULT field of a column definition +** in a CREATE TABLE statement. Return non-zero if the expression is +** acceptable for use as a DEFAULT. That is to say, return non-zero if +** the expression is constant or a function call with constant arguments. +** Return and 0 if there are any variables. +** +** isInit is true when parsing from sqlite_schema. isInit is false when +** processing a new CREATE TABLE statement. When isInit is true, parameters +** (such as ? or $abc) in the expression are converted into NULL. When +** isInit is false, parameters raise an error. Parameters should not be +** allowed in a CREATE TABLE statement, but some legacy versions of SQLite +** allowed it, so we need to support it when reading sqlite_schema for +** backwards compatibility. +** +** If isInit is true, set EP_FromDDL on every TK_FUNCTION node. ** ** For the purposes of this function, a double-quoted string (ex: "abc") ** is considered a variable but a single-quoted string (ex: 'abc') is @@ -90404,7 +112165,7 @@ SQLITE_PRIVATE int sqlite3ExprIsTableConstant(Expr *p, int iCur){ */ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ assert( isInit==0 || isInit==1 ); - return exprIsConst(p, 4+isInit, 0); + return exprIsConst(0, p, 4+isInit); } #ifdef SQLITE_ENABLE_CURSOR_HINTS @@ -90414,10 +112175,12 @@ SQLITE_PRIVATE int sqlite3ExprIsConstantOrFunction(Expr *p, u8 isInit){ */ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ Walker w; - memset(&w, 0, sizeof(w)); w.eCode = 1; w.xExprCallback = sqlite3ExprWalkNoop; - w.xSelectCallback = selectNodeIsConstant; + w.xSelectCallback = sqlite3SelectWalkFail; +#ifdef SQLITE_DEBUG + w.xSelectCallback2 = sqlite3SelectWalkAssert2; +#endif sqlite3WalkExpr(&w, p); return w.eCode==0; } @@ -90428,9 +112191,14 @@ SQLITE_PRIVATE int sqlite3ExprContainsSubquery(Expr *p){ ** to fit in a 32-bit integer, return 1 and put the value of the integer ** in *pValue. If the expression is not an integer or if it is too big ** to fit in a signed 32-bit integer, return 0 and leave *pValue unchanged. +** +** If the pParse pointer is provided, then allow the expression p to be +** a parameter (TK_VARIABLE) that is bound to an integer. +** But if pParse is NULL, then p must be a pure integer literal. */ -SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ +SQLITE_PRIVATE int sqlite3ExprIsInteger(const Expr *p, int *pValue, Parse *pParse){ int rc = 0; + if( NEVER(p==0) ) return 0; /* Used to only happen following on OOM */ /* If an expression is an integer literal that fits in a signed 32-bit ** integer, then the EP_IntValue flag will have already been set */ @@ -90443,18 +112211,38 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ } switch( p->op ){ case TK_UPLUS: { - rc = sqlite3ExprIsInteger(p->pLeft, pValue); + rc = sqlite3ExprIsInteger(p->pLeft, pValue, 0); break; } case TK_UMINUS: { - int v; - if( sqlite3ExprIsInteger(p->pLeft, &v) ){ - assert( v!=(-2147483647-1) ); + int v = 0; + if( sqlite3ExprIsInteger(p->pLeft, &v, 0) ){ + assert( ((unsigned int)v)!=0x80000000 ); *pValue = -v; rc = 1; } break; } + case TK_VARIABLE: { + sqlite3_value *pVal; + if( pParse==0 ) break; + if( NEVER(pParse->pVdbe==0) ) break; + if( (pParse->db->flags & SQLITE_EnableQPSG)!=0 ) break; + sqlite3VdbeSetVarmask(pParse->pVdbe, p->iColumn); + pVal = sqlite3VdbeGetBoundValue(pParse->pReprepare, p->iColumn, + SQLITE_AFF_BLOB); + if( pVal ){ + if( sqlite3_value_type(pVal)==SQLITE_INTEGER ){ + sqlite3_int64 vv = sqlite3_value_int64(pVal); + if( vv == (vv & 0x7fffffff) ){ /* non-negative numbers only */ + *pValue = (int)vv; + rc = 1; + } + } + sqlite3ValueFree(pVal); + } + break; + } default: break; } return rc; @@ -90464,7 +112252,7 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ ** Return FALSE if there is no chance that the expression can be NULL. ** ** If the expression might be NULL or if the expression is too complex -** to tell return TRUE. +** to tell return TRUE. ** ** This routine is used as an optimization, to skip OP_IsNull opcodes ** when we know that a value cannot be NULL. Hence, a false positive @@ -90476,7 +112264,11 @@ SQLITE_PRIVATE int sqlite3ExprIsInteger(Expr *p, int *pValue){ */ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ u8 op; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + assert( p!=0 ); + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + p = p->pLeft; + assert( p!=0 ); + } op = p->op; if( op==TK_REGISTER ) op = p->op2; switch( op ){ @@ -90486,9 +112278,16 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ case TK_BLOB: return 0; case TK_COLUMN: - assert( p->pTab!=0 ); - return ExprHasProperty(p, EP_CanBeNull) || - (p->iColumn>=0 && p->pTab->aCol[p->iColumn].notNull==0); + assert( ExprUseYTab(p) ); + return ExprHasProperty(p, EP_CanBeNull) + || NEVER(p->y.pTab==0) /* Reference to column of index on expr */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + || (p->iColumn==XN_ROWID && IsView(p->y.pTab)) +#endif + || (p->iColumn>=0 + && p->y.pTab->aCol!=0 /* Possible due to prior error */ + && ALWAYS(p->iColumny.pTab->nCol) + && p->y.pTab->aCol[p->iColumn].notNull==0); default: return 1; } @@ -90506,27 +112305,30 @@ SQLITE_PRIVATE int sqlite3ExprCanBeNull(const Expr *p){ */ SQLITE_PRIVATE int sqlite3ExprNeedsNoAffinityChange(const Expr *p, char aff){ u8 op; + int unaryMinus = 0; if( aff==SQLITE_AFF_BLOB ) return 1; - while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ p = p->pLeft; } + while( p->op==TK_UPLUS || p->op==TK_UMINUS ){ + if( p->op==TK_UMINUS ) unaryMinus = 1; + p = p->pLeft; + } op = p->op; if( op==TK_REGISTER ) op = p->op2; switch( op ){ case TK_INTEGER: { - return aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC; + return aff>=SQLITE_AFF_NUMERIC; } case TK_FLOAT: { - return aff==SQLITE_AFF_REAL || aff==SQLITE_AFF_NUMERIC; + return aff>=SQLITE_AFF_NUMERIC; } case TK_STRING: { - return aff==SQLITE_AFF_TEXT; + return !unaryMinus && aff==SQLITE_AFF_TEXT; } case TK_BLOB: { - return 1; + return !unaryMinus; } case TK_COLUMN: { assert( p->iTable>=0 ); /* p cannot be part of a CHECK constraint */ - return p->iColumn<0 - && (aff==SQLITE_AFF_INTEGER || aff==SQLITE_AFF_NUMERIC); + return aff>=SQLITE_AFF_NUMERIC && p->iColumn<0; } default: { return 0; @@ -90545,20 +112347,41 @@ SQLITE_PRIVATE int sqlite3IsRowid(const char *z){ } /* -** pX is the RHS of an IN operator. If pX is a SELECT statement +** Return a pointer to a buffer containing a usable rowid alias for table +** pTab. An alias is usable if there is not an explicit user-defined column +** of the same name. +*/ +SQLITE_PRIVATE const char *sqlite3RowidAlias(Table *pTab){ + const char *azOpt[] = {"_ROWID_", "ROWID", "OID"}; + int ii; + assert( VisibleRowid(pTab) ); + for(ii=0; iinCol; iCol++){ + if( sqlite3_stricmp(azOpt[ii], pTab->aCol[iCol].zCnName)==0 ) break; + } + if( iCol==pTab->nCol ){ + return azOpt[ii]; + } + } + return 0; +} + +/* +** pX is the RHS of an IN operator. If pX is a SELECT statement ** that can be simplified to a direct table access, then return ** a pointer to the SELECT statement. If pX is not a SELECT statement, -** or if the SELECT statement needs to be manifested into a transient +** or if the SELECT statement needs to be materialized into a transient ** table, then return NULL. */ #ifndef SQLITE_OMIT_SUBQUERY -static Select *isCandidateForInOpt(Expr *pX){ +static Select *isCandidateForInOpt(const Expr *pX){ Select *p; SrcList *pSrc; ExprList *pEList; - Expr *pRes; Table *pTab; - if( !ExprHasProperty(pX, EP_xIsSelect) ) return 0; /* Not a subquery */ + int i; + if( !ExprUseXSelect(pX) ) return 0; /* Not a subquery */ if( ExprHasProperty(pX, EP_VarSelect) ) return 0; /* Correlated subq */ p = pX->x.pSelect; if( p->pPrior ) return 0; /* Not a compound SELECT */ @@ -90569,34 +112392,28 @@ static Select *isCandidateForInOpt(Expr *pX){ } assert( p->pGroupBy==0 ); /* Has no GROUP BY clause */ if( p->pLimit ) return 0; /* Has no LIMIT clause */ - assert( p->pOffset==0 ); /* No LIMIT means no OFFSET */ if( p->pWhere ) return 0; /* Has no WHERE clause */ pSrc = p->pSrc; assert( pSrc!=0 ); if( pSrc->nSrc!=1 ) return 0; /* Single term in FROM clause */ - if( pSrc->a[0].pSelect ) return 0; /* FROM is not a subquery or view */ - pTab = pSrc->a[0].pTab; + if( pSrc->a[0].fg.isSubquery) return 0;/* FROM is not a subquery or view */ + pTab = pSrc->a[0].pSTab; assert( pTab!=0 ); - assert( pTab->pSelect==0 ); /* FROM clause is not a view */ + assert( !IsView(pTab) ); /* FROM clause is not a view */ if( IsVirtual(pTab) ) return 0; /* FROM clause not a virtual table */ pEList = p->pEList; - if( pEList->nExpr!=1 ) return 0; /* One column in the result set */ - pRes = pEList->a[0].pExpr; - if( pRes->op!=TK_COLUMN ) return 0; /* Result is a column */ - assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + assert( pEList!=0 ); + /* All SELECT results must be columns. */ + for(i=0; inExpr; i++){ + Expr *pRes = pEList->a[i].pExpr; + if( pRes->op!=TK_COLUMN ) return 0; + assert( pRes->iTable==pSrc->a[0].iCursor ); /* Not a correlated subquery */ + } return p; } #endif /* SQLITE_OMIT_SUBQUERY */ -/* -** Code an OP_Once instruction and allocate space for its flag. Return the -** address of the new instruction. -*/ -SQLITE_PRIVATE int sqlite3CodeOnce(Parse *pParse){ - Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ - return sqlite3VdbeAddOp1(v, OP_Once, pParse->nOnce++); -} - +#ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code that checks the left-most column of index table iCur to see if ** it contains any NULL entries. Cause the register at regHasNull to be set @@ -90612,20 +112429,21 @@ static void sqlite3SetHasNullFlag(Vdbe *v, int iCur, int regHasNull){ VdbeComment((v, "first_entry_in(%d)", iCur)); sqlite3VdbeJumpHere(v, addr1); } +#endif #ifndef SQLITE_OMIT_SUBQUERY /* -** The argument is an IN operator with a list (not a subquery) on the +** The argument is an IN operator with a list (not a subquery) on the ** right-hand side. Return TRUE if that list is constant. */ -static int sqlite3InRhsIsConstant(Expr *pIn){ +static int sqlite3InRhsIsConstant(Parse *pParse, Expr *pIn){ Expr *pLHS; int res; assert( !ExprHasProperty(pIn, EP_xIsSelect) ); pLHS = pIn->pLeft; pIn->pLeft = 0; - res = sqlite3ExprIsConstant(pIn); + res = sqlite3ExprIsConstant(pParse, pIn); pIn->pLeft = pLHS; return res; } @@ -90641,7 +112459,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** all members of the RHS set, skipping duplicates. ** ** A cursor is opened on the b-tree object that is the RHS of the IN operator -** and pX->iTable is set to the index of that cursor. +** and the *piTab parameter is set to the index of that cursor. ** ** The returned value of this function indicates the b-tree type, as follows: ** @@ -90649,37 +112467,39 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** IN_INDEX_INDEX_ASC - The cursor was opened on an ascending index. ** IN_INDEX_INDEX_DESC - The cursor was opened on a descending index. ** IN_INDEX_EPH - The cursor was opened on a specially created and -** populated epheremal table. +** populated ephemeral table. ** IN_INDEX_NOOP - No cursor was allocated. The IN operator must be ** implemented as a sequence of comparisons. ** ** An existing b-tree might be used if the RHS expression pX is a simple ** subquery such as: ** -** SELECT FROM +** SELECT , ... FROM
            ** ** If the RHS of the IN operator is a list or a more complex subquery, then ** an ephemeral table might need to be generated from the RHS and then ** pX->iTable made to point to the ephemeral table instead of an -** existing table. +** existing table. In this case, the creation and initialization of the +** ephemeral table might be put inside of a subroutine, the EP_Subrtn flag +** will be set on pX and the pX->y.sub fields will be set to show where +** the subroutine is coded. ** -** The inFlags parameter must contain exactly one of the bits -** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP. If inFlags contains -** IN_INDEX_MEMBERSHIP, then the generated table will be used for a -** fast membership test. When the IN_INDEX_LOOP bit is set, the -** IN index will be used to loop over all values of the RHS of the -** IN operator. +** The inFlags parameter must contain, at a minimum, one of the bits +** IN_INDEX_MEMBERSHIP or IN_INDEX_LOOP but not both. If inFlags contains +** IN_INDEX_MEMBERSHIP, then the generated table will be used for a fast +** membership test. When the IN_INDEX_LOOP bit is set, the IN index will +** be used to loop over all values of the RHS of the IN operator. ** ** When IN_INDEX_LOOP is used (and the b-tree will be used to iterate ** through the set members) then the b-tree must not contain duplicates. -** An epheremal table must be used unless the selected is guaranteed -** to be unique - either because it is an INTEGER PRIMARY KEY or it -** has a UNIQUE constraint or UNIQUE index. +** An ephemeral table will be created unless the selected columns are guaranteed +** to be unique - either because it is an INTEGER PRIMARY KEY or due to +** a UNIQUE constraint or index. ** -** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used -** for fast set membership tests) then an epheremal table must -** be used unless is an INTEGER PRIMARY KEY or an index can -** be found with as its left-most column. +** When IN_INDEX_MEMBERSHIP is used (and the b-tree will be used +** for fast set membership tests) then an ephemeral table must +** be used unless is a single INTEGER PRIMARY KEY column or an +** index can be found with the specified as its left-most. ** ** If the IN_INDEX_NOOP_OK and IN_INDEX_MEMBERSHIP are both set and ** if the RHS of the IN operator is a list (not a subquery) then this @@ -90690,7 +112510,7 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** ** When the b-tree is being used for membership tests, the calling function ** might need to know whether or not the RHS side of the IN operator -** contains a NULL. If prRhsHasNull is not a NULL pointer and +** contains a NULL. If prRhsHasNull is not a NULL pointer and ** if there is any chance that the (...) might contain a NULL value at ** runtime, then a register is allocated and the register number written ** to *prRhsHasNull. If there is no chance that the (...) contains a @@ -90700,105 +112520,204 @@ static int sqlite3InRhsIsConstant(Expr *pIn){ ** the value in that register will be NULL if the b-tree contains one or more ** NULL values, and it will be some non-NULL value if the b-tree contains no ** NULL values. +** +** If the aiMap parameter is not NULL, it must point to an array containing +** one element for each column returned by the SELECT statement on the RHS +** of the IN(...) operator. The i'th entry of the array is populated with the +** offset of the index column that matches the i'th column returned by the +** SELECT. For example, if the expression and selected index are: +** +** (?,?,?) IN (SELECT a, b, c FROM t1) +** CREATE INDEX i1 ON t1(b, c, a); +** +** then aiMap[] is populated with {2, 0, 1}. */ #ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int *prRhsHasNull){ +SQLITE_PRIVATE int sqlite3FindInIndex( + Parse *pParse, /* Parsing context */ + Expr *pX, /* The IN expression */ + u32 inFlags, /* IN_INDEX_LOOP, _MEMBERSHIP, and/or _NOOP_OK */ + int *prRhsHasNull, /* Register holding NULL status. See notes */ + int *aiMap, /* Mapping from Index fields to RHS fields */ + int *piTab /* OUT: index to use */ +){ Select *p; /* SELECT to the right of IN operator */ int eType = 0; /* Type of RHS table. IN_INDEX_* */ - int iTab = pParse->nTab++; /* Cursor of the RHS table */ + int iTab; /* Cursor of the RHS table */ int mustBeUnique; /* True if RHS must be unique */ Vdbe *v = sqlite3GetVdbe(pParse); /* Virtual machine being coded */ assert( pX->op==TK_IN ); mustBeUnique = (inFlags & IN_INDEX_LOOP)!=0; + iTab = pParse->nTab++; + + /* If the RHS of this IN(...) operator is a SELECT, and if it matters + ** whether or not the SELECT result contains NULL values, check whether + ** or not NULL is actually possible (it may not be, for example, due + ** to NOT NULL constraints in the schema). If no NULL values are possible, + ** set prRhsHasNull to 0 before continuing. */ + if( prRhsHasNull && ExprUseXSelect(pX) ){ + int i; + ExprList *pEList = pX->x.pSelect->pEList; + for(i=0; inExpr; i++){ + if( sqlite3ExprCanBeNull(pEList->a[i].pExpr) ) break; + } + if( i==pEList->nExpr ){ + prRhsHasNull = 0; + } + } /* Check to see if an existing table or index can be used to - ** satisfy the query. This is preferable to generating a new - ** ephemeral table. - */ + ** satisfy the query. This is preferable to generating a new + ** ephemeral table. */ if( pParse->nErr==0 && (p = isCandidateForInOpt(pX))!=0 ){ sqlite3 *db = pParse->db; /* Database connection */ Table *pTab; /* Table
            . */ - Expr *pExpr; /* Expression */ - i16 iCol; /* Index of column */ - i16 iDb; /* Database idx for pTab */ + int iDb; /* Database idx for pTab */ + ExprList *pEList = p->pEList; + int nExpr = pEList->nExpr; assert( p->pEList!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pEList->a[0].pExpr!=0 ); /* Because of isCandidateForInOpt(p) */ assert( p->pSrc!=0 ); /* Because of isCandidateForInOpt(p) */ - pTab = p->pSrc->a[0].pTab; - pExpr = p->pEList->a[0].pExpr; - iCol = (i16)pExpr->iColumn; - + pTab = p->pSrc->a[0].pSTab; + /* Code an OP_Transaction and OP_TableLock for
            . */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 && iDbtnum, 0, pTab->zName); - /* This function is only called from two places. In both cases the vdbe - ** has already been allocated. So assume sqlite3GetVdbe() is always - ** successful here. - */ - assert(v); - if( iCol<0 ){ - int iAddr = sqlite3CodeOnce(pParse); + assert(v); /* sqlite3GetVdbe() has always been previously called */ + if( nExpr==1 && pEList->a[0].pExpr->iColumn<0 ){ + /* The "x IN (SELECT rowid FROM table)" case */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); eType = IN_INDEX_ROWID; - + ExplainQueryPlan((pParse, 0, + "USING ROWID SEARCH ON TABLE %s FOR IN-OPERATOR",pTab->zName)); sqlite3VdbeJumpHere(v, iAddr); }else{ Index *pIdx; /* Iterator variable */ + int affinity_ok = 1; + int i; - /* The collation sequence used by the comparison. If an index is to - ** be used in place of a temp-table, it must be ordered according - ** to this collation sequence. */ - CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pExpr); - - /* Check that the affinity that will be used to perform the - ** comparison is the same as the affinity of the column. If - ** it is not, it is not possible to use any index. - */ - int affinity_ok = sqlite3IndexAffinityOk(pX, pTab->aCol[iCol].affinity); - - for(pIdx=pTab->pIndex; pIdx && eType==0 && affinity_ok; pIdx=pIdx->pNext){ - if( (pIdx->aiColumn[0]==iCol) - && sqlite3FindCollSeq(db, ENC(db), pIdx->azColl[0], 0)==pReq - && (!mustBeUnique || (pIdx->nKeyCol==1 && IsUniqueIndex(pIdx))) - ){ - int iAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); - sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); - eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; - - if( prRhsHasNull && !pTab->aCol[iCol].notNull ){ - *prRhsHasNull = ++pParse->nMem; - sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); - } - sqlite3VdbeJumpHere(v, iAddr); + /* Check that the affinity that will be used to perform each + ** comparison is the same as the affinity of each column in table + ** on the RHS of the IN operator. If it not, it is not possible to + ** use any index of the RHS table. */ + for(i=0; ipLeft, i); + int iCol = pEList->a[i].pExpr->iColumn; + char idxaff = sqlite3TableColumnAffinity(pTab,iCol); /* RHS table */ + char cmpaff = sqlite3CompareAffinity(pLhs, idxaff); + testcase( cmpaff==SQLITE_AFF_BLOB ); + testcase( cmpaff==SQLITE_AFF_TEXT ); + switch( cmpaff ){ + case SQLITE_AFF_BLOB: + break; + case SQLITE_AFF_TEXT: + /* sqlite3CompareAffinity() only returns TEXT if one side or the + ** other has no affinity and the other side is TEXT. Hence, + ** the only way for cmpaff to be TEXT is for idxaff to be TEXT + ** and for the term on the LHS of the IN to have no affinity. */ + assert( idxaff==SQLITE_AFF_TEXT ); + break; + default: + affinity_ok = sqlite3IsNumericAffinity(idxaff); } } - } - } + + if( affinity_ok ){ + /* Search for an existing index that will work for this IN operator */ + for(pIdx=pTab->pIndex; pIdx && eType==0; pIdx=pIdx->pNext){ + Bitmask colUsed; /* Columns of the index used */ + Bitmask mCol; /* Mask for the current column */ + if( pIdx->nColumnpPartIdxWhere!=0 ) continue; + /* Maximum nColumn is BMS-2, not BMS-1, so that we can compute + ** BITMASK(nExpr) without overflowing */ + testcase( pIdx->nColumn==BMS-2 ); + testcase( pIdx->nColumn==BMS-1 ); + if( pIdx->nColumn>=BMS-1 ) continue; + if( mustBeUnique ){ + if( pIdx->nKeyCol>nExpr + ||(pIdx->nColumn>nExpr && !IsUniqueIndex(pIdx)) + ){ + continue; /* This index is not unique over the IN RHS columns */ + } + } + + colUsed = 0; /* Columns of index used so far */ + for(i=0; ipLeft, i); + Expr *pRhs = pEList->a[i].pExpr; + CollSeq *pReq = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + int j; + + for(j=0; jaiColumn[j]!=pRhs->iColumn ) continue; + assert( pIdx->azColl[j] ); + if( pReq!=0 && sqlite3StrICmp(pReq->zName, pIdx->azColl[j])!=0 ){ + continue; + } + break; + } + if( j==nExpr ) break; + mCol = MASKBIT(j); + if( mCol & colUsed ) break; /* Each column used only once */ + colUsed |= mCol; + if( aiMap ) aiMap[i] = j; + } + + assert( i==nExpr || colUsed!=(MASKBIT(nExpr)-1) ); + if( colUsed==(MASKBIT(nExpr)-1) ){ + /* If we reach this point, that means the index pIdx is usable */ + int iAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + ExplainQueryPlan((pParse, 0, + "USING INDEX %s FOR IN-OPERATOR",pIdx->zName)); + sqlite3VdbeAddOp3(v, OP_OpenRead, iTab, pIdx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIdx); + VdbeComment((v, "%s", pIdx->zName)); + assert( IN_INDEX_INDEX_DESC == IN_INDEX_INDEX_ASC+1 ); + eType = IN_INDEX_INDEX_ASC + pIdx->aSortOrder[0]; + + if( prRhsHasNull ){ +#ifdef SQLITE_ENABLE_COLUMN_USED_MASK + i64 mask = (1<nMem; + if( nExpr==1 ){ + sqlite3SetHasNullFlag(v, iTab, *prRhsHasNull); + } + } + sqlite3VdbeJumpHere(v, iAddr); + } + } /* End loop over indexes */ + } /* End if( affinity_ok ) */ + } /* End if not an rowid index */ + } /* End attempt to optimize using an index */ /* If no preexisting index is available for the IN clause ** and IN_INDEX_NOOP is an allowed reply ** and the RHS of the IN operator is a list, not a subquery - ** and the RHS is not contant or has two or fewer terms, + ** and the RHS is not constant or has two or fewer terms, ** then it is not worth creating an ephemeral table to evaluate ** the IN operator so return IN_INDEX_NOOP. */ if( eType==0 && (inFlags & IN_INDEX_NOOP_OK) - && !ExprHasProperty(pX, EP_xIsSelect) - && (!sqlite3InRhsIsConstant(pX) || pX->x.pList->nExpr<=2) + && ExprUseXList(pX) + && (!sqlite3InRhsIsConstant(pParse,pX) || pX->x.pList->nExpr<=2) ){ + pParse->nTab--; /* Back out the allocation of the unused cursor */ + iTab = -1; /* Cursor is not allocated */ eType = IN_INDEX_NOOP; } - if( eType==0 ){ /* Could not find an existing table or index to use as the RHS b-tree. @@ -90809,63 +112728,446 @@ SQLITE_PRIVATE int sqlite3FindInIndex(Parse *pParse, Expr *pX, u32 inFlags, int eType = IN_INDEX_EPH; if( inFlags & IN_INDEX_LOOP ){ pParse->nQueryLoop = 0; - if( pX->pLeft->iColumn<0 && !ExprHasProperty(pX, EP_xIsSelect) ){ - eType = IN_INDEX_ROWID; - } }else if( prRhsHasNull ){ *prRhsHasNull = rMayHaveNull = ++pParse->nMem; } - sqlite3CodeSubselect(pParse, pX, rMayHaveNull, eType==IN_INDEX_ROWID); + assert( pX->op==TK_IN ); + sqlite3CodeRhsOfIN(pParse, pX, iTab); + if( rMayHaveNull ){ + sqlite3SetHasNullFlag(v, iTab, rMayHaveNull); + } pParse->nQueryLoop = savedNQueryLoop; - }else{ - pX->iTable = iTab; } + + if( aiMap && eType!=IN_INDEX_INDEX_ASC && eType!=IN_INDEX_INDEX_DESC ){ + int i, n; + n = sqlite3ExprVectorSize(pX->pLeft); + for(i=0; ipLeft; + int nVal = sqlite3ExprVectorSize(pLeft); + Select *pSelect = ExprUseXSelect(pExpr) ? pExpr->x.pSelect : 0; + char *zRet; + + assert( pExpr->op==TK_IN ); + zRet = sqlite3DbMallocRaw(pParse->db, nVal+1); + if( zRet ){ + int i; + for(i=0; ipEList->a[i].pExpr, a); + }else{ + zRet[i] = a; + } + } + zRet[nVal] = '\0'; + } + return zRet; +} +#endif + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Load the Parse object passed as the first argument with an error +** message of the form: +** +** "sub-select returns N columns - expected M" +*/ +SQLITE_PRIVATE void sqlite3SubselectError(Parse *pParse, int nActual, int nExpect){ + if( pParse->nErr==0 ){ + const char *zFmt = "sub-select returns %d columns - expected %d"; + sqlite3ErrorMsg(pParse, zFmt, nActual, nExpect); + } +} +#endif + +/* +** Expression pExpr is a vector that has been used in a context where +** it is not permitted. If pExpr is a sub-select vector, this routine +** loads the Parse object with a message of the form: +** +** "sub-select returns N columns - expected 1" +** +** Or, if it is a regular scalar vector: +** +** "row value misused" +*/ +SQLITE_PRIVATE void sqlite3VectorErrorMsg(Parse *pParse, Expr *pExpr){ +#ifndef SQLITE_OMIT_SUBQUERY + if( ExprUseXSelect(pExpr) ){ + sqlite3SubselectError(pParse, pExpr->x.pSelect->pEList->nExpr, 1); + }else +#endif + { + sqlite3ErrorMsg(pParse, "row value misused"); + } +} + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Scan all previously generated bytecode looking for an OP_BeginSubrtn +** that is compatible with pExpr. If found, add the y.sub values +** to pExpr and return true. If not found, return false. +*/ +static int findCompatibleInRhsSubrtn( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* IN operator with RHS that we want to reuse */ + SubrtnSig *pNewSig /* Signature for the IN operator */ +){ + VdbeOp *pOp, *pEnd; + SubrtnSig *pSig; + Vdbe *v; + + if( pNewSig==0 ) return 0; + if( (pParse->mSubrtnSig & (1<<(pNewSig->selId&7)))==0 ) return 0; + assert( pExpr->op==TK_IN ); + assert( !ExprUseYSub(pExpr) ); + assert( ExprUseXSelect(pExpr) ); + assert( pExpr->x.pSelect!=0 ); + assert( (pExpr->x.pSelect->selFlags & SF_All)==0 ); + v = pParse->pVdbe; + assert( v!=0 ); + pOp = sqlite3VdbeGetOp(v, 1); + pEnd = sqlite3VdbeGetLastOp(v); + for(; pOpp4type!=P4_SUBRTNSIG ) continue; + assert( pOp->opcode==OP_BeginSubrtn ); + pSig = pOp->p4.pSubrtnSig; + assert( pSig!=0 ); + if( pNewSig->selId!=pSig->selId ) continue; + if( strcmp(pNewSig->zAff,pSig->zAff)!=0 ) continue; + pExpr->y.sub.iAddr = pSig->iAddr; + pExpr->y.sub.regReturn = pSig->regReturn; + pExpr->iTable = pSig->iTable; + ExprSetProperty(pExpr, EP_Subrtn); + return 1; + } + return 0; +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code that will construct an ephemeral table containing all terms +** in the RHS of an IN operator. The IN operator can be in either of two +** forms: ** -** (SELECT a FROM b) -- subquery -** EXISTS (SELECT a FROM b) -- EXISTS subquery ** x IN (4,5,11) -- IN operator with list on right-hand side ** x IN (SELECT a FROM b) -- IN operator with subquery on the right ** -** The pExpr parameter describes the expression that contains the IN -** operator or subquery. +** The pExpr parameter is the IN operator. The cursor number for the +** constructed ephemeral table is returned. The first time the ephemeral +** table is computed, the cursor number is also stored in pExpr->iTable, +** however the cursor number returned might not be the same, as it might +** have been duplicated using OP_OpenDup. ** -** If parameter isRowid is non-zero, then expression pExpr is guaranteed -** to be of the form " IN (?, ?, ?)", where is a reference -** to some integer key column of a table B-Tree. In this case, use an -** intkey B-Tree to store the set of IN(...) values instead of the usual -** (slower) variable length keys B-Tree. +** If the LHS expression ("x" in the examples) is a column value, or +** the SELECT statement returns a column value, then the affinity of that +** column is used to build the index keys. If both 'x' and the +** SELECT... statement are columns, then numeric affinity is used +** if either column has NUMERIC or INTEGER affinity. If neither +** 'x' nor the SELECT... statement are columns, then numeric affinity +** is used. +*/ +SQLITE_PRIVATE void sqlite3CodeRhsOfIN( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The IN operator */ + int iTab /* Use this cursor number */ +){ + int addrOnce = 0; /* Address of the OP_Once instruction at top */ + int addr; /* Address of OP_OpenEphemeral instruction */ + Expr *pLeft; /* the LHS of the IN operator */ + KeyInfo *pKeyInfo = 0; /* Key information */ + int nVal; /* Size of vector pLeft */ + Vdbe *v; /* The prepared statement under construction */ + + v = pParse->pVdbe; + assert( v!=0 ); + + /* The evaluation of the IN must be repeated every time it + ** is encountered if any of the following is true: + ** + ** * The right-hand side is a correlated subquery + ** * The right-hand side is an expression list containing variables + ** * We are inside a trigger + ** + ** If all of the above are false, then we can compute the RHS just once + ** and reuse it many names. + */ + if( !ExprHasProperty(pExpr, EP_VarSelect) && pParse->iSelfTab==0 ){ + /* Reuse of the RHS is allowed + ** + ** Compute a signature for the RHS of the IN operator to facility + ** finding and reusing prior instances of the same IN operator. + */ + SubrtnSig *pSig = 0; + assert( !ExprUseXSelect(pExpr) || pExpr->x.pSelect!=0 ); + if( ExprUseXSelect(pExpr) && (pExpr->x.pSelect->selFlags & SF_All)==0 ){ + pSig = sqlite3DbMallocRawNN(pParse->db, sizeof(pSig[0])); + if( pSig ){ + pSig->selId = pExpr->x.pSelect->selId; + pSig->zAff = exprINAffinity(pParse, pExpr); + } + } + + /* Check to see if there is a prior materialization of the RHS of + ** this IN operator. If there is, then make use of that prior + ** materialization rather than recomputing it. + */ + if( ExprHasProperty(pExpr, EP_Subrtn) + || findCompatibleInRhsSubrtn(pParse, pExpr, pSig) + ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + if( ExprUseXSelect(pExpr) ){ + ExplainQueryPlan((pParse, 0, "REUSE LIST SUBQUERY %d", + pExpr->x.pSelect->selId)); + } + assert( ExprUseYSub(pExpr) ); + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + assert( iTab!=pExpr->iTable ); + sqlite3VdbeAddOp2(v, OP_OpenDup, iTab, pExpr->iTable); + sqlite3VdbeJumpHere(v, addrOnce); + if( pSig ){ + sqlite3DbFree(pParse->db, pSig->zAff); + sqlite3DbFree(pParse->db, pSig); + } + return; + } + + /* Begin coding the subroutine */ + assert( !ExprUseYWin(pExpr) ); + ExprSetProperty(pExpr, EP_Subrtn); + assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; + if( pSig ){ + pSig->iAddr = pExpr->y.sub.iAddr; + pSig->regReturn = pExpr->y.sub.regReturn; + pSig->iTable = iTab; + pParse->mSubrtnSig = 1 << (pSig->selId&7); + sqlite3VdbeChangeP4(v, -1, (const char*)pSig, P4_SUBRTNSIG); + } + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + + /* Check to see if this is a vector IN operator */ + pLeft = pExpr->pLeft; + nVal = sqlite3ExprVectorSize(pLeft); + + /* Construct the ephemeral table that will contain the content of + ** RHS of the IN operator. + */ + pExpr->iTable = iTab; + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, nVal); +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + if( ExprUseXSelect(pExpr) ){ + VdbeComment((v, "Result of SELECT %u", pExpr->x.pSelect->selId)); + }else{ + VdbeComment((v, "RHS of IN operator")); + } +#endif + pKeyInfo = sqlite3KeyInfoAlloc(pParse->db, nVal, 1); + + if( ExprUseXSelect(pExpr) ){ + /* Case 1: expr IN (SELECT ...) + ** + ** Generate code to write the results of the select into the temporary + ** table allocated and opened above. + */ + Select *pSelect = pExpr->x.pSelect; + ExprList *pEList = pSelect->pEList; + + ExplainQueryPlan((pParse, 1, "%sLIST SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSelect->selId + )); + /* If the LHS and RHS of the IN operator do not match, that + ** error will have been caught long before we reach this point. */ + if( ALWAYS(pEList->nExpr==nVal) ){ + Select *pCopy; + SelectDest dest; + int i; + int rc; + int addrBloom = 0; + sqlite3SelectDestInit(&dest, SRT_Set, iTab); + dest.zAffSdst = exprINAffinity(pParse, pExpr); + pSelect->iLimit = 0; + if( addrOnce && OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ){ + int regBloom = ++pParse->nMem; + addrBloom = sqlite3VdbeAddOp2(v, OP_Blob, 10000, regBloom); + VdbeComment((v, "Bloom filter")); + dest.iSDParm2 = regBloom; + } + testcase( pSelect->selFlags & SF_Distinct ); + testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ + pCopy = sqlite3SelectDup(pParse->db, pSelect, 0); + rc = pParse->db->mallocFailed ? 1 :sqlite3Select(pParse, pCopy, &dest); + sqlite3SelectDelete(pParse->db, pCopy); + sqlite3DbFree(pParse->db, dest.zAffSdst); + if( addrBloom ){ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + if( dest.iSDParm2==0 ){ + sqlite3VdbeChangeToNoop(v, addrBloom); + }else{ + sqlite3VdbeGetOp(v, addrOnce)->p3 = dest.iSDParm2; + } + } + if( rc ){ + sqlite3KeyInfoUnref(pKeyInfo); + return; + } + assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ + assert( pEList!=0 ); + assert( pEList->nExpr>0 ); + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + for(i=0; iaColl[i] = sqlite3BinaryCompareCollSeq( + pParse, p, pEList->a[i].pExpr + ); + } + } + }else if( ALWAYS(pExpr->x.pList!=0) ){ + /* Case 2: expr IN (exprlist) + ** + ** For each expression, build an index key from the evaluation and + ** store it in the temporary table. If is a column, then use + ** that columns affinity when building index keys. If is not + ** a column, use numeric affinity. + */ + char affinity; /* Affinity of the LHS of the IN */ + int i; + ExprList *pList = pExpr->x.pList; + struct ExprList_item *pItem; + int r1, r2; + affinity = sqlite3ExprAffinity(pLeft); + if( affinity<=SQLITE_AFF_NONE ){ + affinity = SQLITE_AFF_BLOB; + }else if( affinity==SQLITE_AFF_REAL ){ + affinity = SQLITE_AFF_NUMERIC; + } + if( pKeyInfo ){ + assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); + pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); + } + + /* Loop through each expression in . */ + r1 = sqlite3GetTempReg(pParse); + r2 = sqlite3GetTempReg(pParse); + for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ + Expr *pE2 = pItem->pExpr; + + /* If the expression is not constant then we will need to + ** disable the test that was generated above that makes sure + ** this code only executes once. Because for a non-constant + ** expression we need to rerun this code each time. + */ + if( addrOnce && !sqlite3ExprIsConstant(pParse, pE2) ){ + sqlite3VdbeChangeToNoop(v, addrOnce-1); + sqlite3VdbeChangeToNoop(v, addrOnce); + ExprClearProperty(pExpr, EP_Subrtn); + addrOnce = 0; + } + + /* Evaluate the expression and insert it into the temp table */ + sqlite3ExprCode(pParse, pE2, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, r1, 1, r2, &affinity, 1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r2, r1, 1); + } + sqlite3ReleaseTempReg(pParse, r1); + sqlite3ReleaseTempReg(pParse, r2); + } + if( pKeyInfo ){ + sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); + } + if( addrOnce ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iTab); + sqlite3VdbeJumpHere(v, addrOnce); + /* Subroutine return */ + assert( ExprUseYSub(pExpr) ); + assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn + || pParse->nErr ); + sqlite3VdbeAddOp3(v, OP_Return, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr, 1); + VdbeCoverage(v); + sqlite3ClearTempRegCache(pParse); + } +} +#endif /* SQLITE_OMIT_SUBQUERY */ + +/* +** Generate code for scalar subqueries used as a subquery expression +** or EXISTS operator: ** -** If rMayHaveNull is non-zero, that means that the operation is an IN -** (not a SELECT or EXISTS) and that the RHS might contains NULLs. -** All this routine does is initialize the register given by rMayHaveNull -** to NULL. Calling routines will take care of changing this register -** value to non-NULL if the RHS is NULL-free. +** (SELECT a FROM b) -- subquery +** EXISTS (SELECT a FROM b) -- EXISTS subquery ** -** For a SELECT or EXISTS operator, return the register that holds the -** result. For IN operators or if an error occurs, the return value is 0. +** The pExpr parameter is the SELECT or EXISTS operator to be coded. +** +** Return the register that holds the result. For a multi-column SELECT, +** the result is stored in a contiguous array of registers and the +** return value is the register of the left-most result column. +** Return 0 if an error occurs. */ #ifndef SQLITE_OMIT_SUBQUERY -SQLITE_PRIVATE int sqlite3CodeSubselect( - Parse *pParse, /* Parsing context */ - Expr *pExpr, /* The IN, SELECT, or EXISTS operator */ - int rHasNullFlag, /* Register that records whether NULLs exist in RHS */ - int isRowid /* If true, LHS of IN operator is a rowid */ -){ - int jmpIfDynamic = -1; /* One-time test address */ - int rReg = 0; /* Register storing resulting */ - Vdbe *v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return 0; - sqlite3ExprCachePush(pParse); +SQLITE_PRIVATE int sqlite3CodeSubselect(Parse *pParse, Expr *pExpr){ + int addrOnce = 0; /* Address of OP_Once at top of subroutine */ + int rReg = 0; /* Register storing resulting */ + Select *pSel; /* SELECT statement to encode */ + SelectDest dest; /* How to deal with SELECT result */ + int nReg; /* Registers to allocate */ + Expr *pLimit; /* New limit expression */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ +#endif - /* This code must be run in its entirety every time it is encountered - ** if any of the following is true: + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + if( pParse->nErr ) return 0; + testcase( pExpr->op==TK_EXISTS ); + testcase( pExpr->op==TK_SELECT ); + assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); + assert( ExprUseXSelect(pExpr) ); + pSel = pExpr->x.pSelect; + + /* If this routine has already been coded, then invoke it as a + ** subroutine. */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + ExplainQueryPlan((pParse, 0, "REUSE SUBQUERY %d", pSel->selId)); + assert( ExprUseYSub(pExpr) ); + sqlite3VdbeAddOp2(v, OP_Gosub, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr); + return pExpr->iTable; + } + + /* Begin coding the subroutine */ + assert( !ExprUseYWin(pExpr) ); + assert( !ExprHasProperty(pExpr, EP_Reduced|EP_TokenOnly) ); + ExprSetProperty(pExpr, EP_Subrtn); + pExpr->y.sub.regReturn = ++pParse->nMem; + pExpr->y.sub.iAddr = + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pExpr->y.sub.regReturn) + 1; + + /* The evaluation of the EXISTS/SELECT must be repeated every time it + ** is encountered if any of the following is true: ** ** * The right-hand side is a correlated subquery ** * The right-hand side is an expression list containing variables @@ -90875,195 +113177,100 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** save the results, and reuse the same result on subsequent invocations. */ if( !ExprHasProperty(pExpr, EP_VarSelect) ){ - jmpIfDynamic = sqlite3CodeOnce(pParse); VdbeCoverage(v); + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } -#ifndef SQLITE_OMIT_EXPLAIN - if( pParse->explain==2 ){ - char *zMsg = sqlite3MPrintf(pParse->db, "EXECUTE %s%s SUBQUERY %d", - jmpIfDynamic>=0?"":"CORRELATED ", - pExpr->op==TK_IN?"LIST":"SCALAR", - pParse->iNextSelectId - ); - sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); + /* For a SELECT, generate code to put the values for all columns of + ** the first row into an array of registers and return the index of + ** the first register. + ** + ** If this is an EXISTS, write an integer 0 (not exists) or 1 (exists) + ** into a register and return that register number. + ** + ** In both cases, the query is augmented with "LIMIT 1". Any + ** preexisting limit is discarded in place of the new LIMIT 1. + */ + ExplainQueryPlan2(addrExplain, (pParse, 1, "%sSCALAR SUBQUERY %d", + addrOnce?"":"CORRELATED ", pSel->selId)); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, -1); + nReg = pExpr->op==TK_SELECT ? pSel->pEList->nExpr : 1; + sqlite3SelectDestInit(&dest, 0, pParse->nMem+1); + pParse->nMem += nReg; + if( pExpr->op==TK_SELECT ){ + dest.eDest = SRT_Mem; + dest.iSdst = dest.iSDParm; + dest.nSdst = nReg; + sqlite3VdbeAddOp3(v, OP_Null, 0, dest.iSDParm, dest.iSDParm+nReg-1); + VdbeComment((v, "Init subquery result")); + }else{ + dest.eDest = SRT_Exists; + sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); + VdbeComment((v, "Init EXISTS result")); } -#endif - - switch( pExpr->op ){ - case TK_IN: { - char affinity; /* Affinity of the LHS of the IN */ - int addr; /* Address of OP_OpenEphemeral instruction */ - Expr *pLeft = pExpr->pLeft; /* the LHS of the IN operator */ - KeyInfo *pKeyInfo = 0; /* Key information */ - - affinity = sqlite3ExprAffinity(pLeft); - - /* Whether this is an 'x IN(SELECT...)' or an 'x IN()' - ** expression it is handled the same way. An ephemeral table is - ** filled with single-field index keys representing the results - ** from the SELECT or the . - ** - ** If the 'x' expression is a column value, or the SELECT... - ** statement returns a column value, then the affinity of that - ** column is used to build the index keys. If both 'x' and the - ** SELECT... statement are columns, then numeric affinity is used - ** if either column has NUMERIC or INTEGER affinity. If neither - ** 'x' nor the SELECT... statement are columns, then numeric affinity - ** is used. - */ - pExpr->iTable = pParse->nTab++; - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pExpr->iTable, !isRowid); - pKeyInfo = isRowid ? 0 : sqlite3KeyInfoAlloc(pParse->db, 1, 1); - - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - /* Case 1: expr IN (SELECT ...) - ** - ** Generate code to write the results of the select into the temporary - ** table allocated and opened above. - */ - Select *pSelect = pExpr->x.pSelect; - SelectDest dest; - ExprList *pEList; - - assert( !isRowid ); - sqlite3SelectDestInit(&dest, SRT_Set, pExpr->iTable); - dest.affSdst = (u8)affinity; - assert( (pExpr->iTable&0x0000FFFF)==pExpr->iTable ); - pSelect->iLimit = 0; - testcase( pSelect->selFlags & SF_Distinct ); - testcase( pKeyInfo==0 ); /* Caused by OOM in sqlite3KeyInfoAlloc() */ - if( sqlite3Select(pParse, pSelect, &dest) ){ - sqlite3KeyInfoUnref(pKeyInfo); - return 0; - } - pEList = pSelect->pEList; - assert( pKeyInfo!=0 ); /* OOM will cause exit after sqlite3Select() */ - assert( pEList!=0 ); - assert( pEList->nExpr>0 ); - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - pKeyInfo->aColl[0] = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, - pEList->a[0].pExpr); - }else if( ALWAYS(pExpr->x.pList!=0) ){ - /* Case 2: expr IN (exprlist) - ** - ** For each expression, build an index key from the evaluation and - ** store it in the temporary table. If is a column, then use - ** that columns affinity when building index keys. If is not - ** a column, use numeric affinity. - */ - int i; - ExprList *pList = pExpr->x.pList; - struct ExprList_item *pItem; - int r1, r2, r3; - - if( !affinity ){ - affinity = SQLITE_AFF_BLOB; - } - if( pKeyInfo ){ - assert( sqlite3KeyInfoIsWriteable(pKeyInfo) ); - pKeyInfo->aColl[0] = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - } - - /* Loop through each expression in . */ - r1 = sqlite3GetTempReg(pParse); - r2 = sqlite3GetTempReg(pParse); - if( isRowid ) sqlite3VdbeAddOp2(v, OP_Null, 0, r2); - for(i=pList->nExpr, pItem=pList->a; i>0; i--, pItem++){ - Expr *pE2 = pItem->pExpr; - int iValToIns; - - /* If the expression is not constant then we will need to - ** disable the test that was generated above that makes sure - ** this code only executes once. Because for a non-constant - ** expression we need to rerun this code each time. - */ - if( jmpIfDynamic>=0 && !sqlite3ExprIsConstant(pE2) ){ - sqlite3VdbeChangeToNoop(v, jmpIfDynamic); - jmpIfDynamic = -1; - } - - /* Evaluate the expression and insert it into the temp table */ - if( isRowid && sqlite3ExprIsInteger(pE2, &iValToIns) ){ - sqlite3VdbeAddOp3(v, OP_InsertInt, pExpr->iTable, r2, iValToIns); - }else{ - r3 = sqlite3ExprCodeTarget(pParse, pE2, r1); - if( isRowid ){ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r3, - sqlite3VdbeCurrentAddr(v)+2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_Insert, pExpr->iTable, r2, r3); - }else{ - sqlite3VdbeAddOp4(v, OP_MakeRecord, r3, 1, r2, &affinity, 1); - sqlite3ExprCacheAffinityChange(pParse, r3, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pExpr->iTable, r2); - } - } - } - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ReleaseTempReg(pParse, r2); - } - if( pKeyInfo ){ - sqlite3VdbeChangeP4(v, addr, (void *)pKeyInfo, P4_KEYINFO); - } - break; - } - - case TK_EXISTS: - case TK_SELECT: - default: { - /* If this has to be a scalar SELECT. Generate code to put the - ** value of this select in a memory cell and record the number - ** of the memory cell in iColumn. If this is an EXISTS, write - ** an integer 0 (not exists) or 1 (exists) into a memory cell - ** and record that memory cell in iColumn. - */ - Select *pSel; /* SELECT statement to encode */ - SelectDest dest; /* How to deal with SELECt result */ - - testcase( pExpr->op==TK_EXISTS ); - testcase( pExpr->op==TK_SELECT ); - assert( pExpr->op==TK_EXISTS || pExpr->op==TK_SELECT ); - - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); - pSel = pExpr->x.pSelect; - sqlite3SelectDestInit(&dest, 0, ++pParse->nMem); - if( pExpr->op==TK_SELECT ){ - dest.eDest = SRT_Mem; - dest.iSdst = dest.iSDParm; - sqlite3VdbeAddOp2(v, OP_Null, 0, dest.iSDParm); - VdbeComment((v, "Init subquery result")); - }else{ - dest.eDest = SRT_Exists; - sqlite3VdbeAddOp2(v, OP_Integer, 0, dest.iSDParm); - VdbeComment((v, "Init EXISTS result")); - } - sqlite3ExprDelete(pParse->db, pSel->pLimit); - pSel->pLimit = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, - &sqlite3IntTokens[1]); - pSel->iLimit = 0; - pSel->selFlags &= ~SF_MultiValue; - if( sqlite3Select(pParse, pSel, &dest) ){ - return 0; - } - rReg = dest.iSDParm; - ExprSetVVAProperty(pExpr, EP_NoReduce); - break; + if( pSel->pLimit ){ + /* The subquery already has a limit. If the pre-existing limit is X + ** then make the new limit X<>0 so that the new limit is either 1 or 0 */ + sqlite3 *db = pParse->db; + pLimit = sqlite3Expr(db, TK_INTEGER, "0"); + if( pLimit ){ + pLimit->affExpr = SQLITE_AFF_NUMERIC; + pLimit = sqlite3PExpr(pParse, TK_NE, + sqlite3ExprDup(db, pSel->pLimit->pLeft, 0), pLimit); } + sqlite3ExprDeferredDelete(pParse, pSel->pLimit->pLeft); + pSel->pLimit->pLeft = pLimit; + }else{ + /* If there is no pre-existing limit add a limit of 1 */ + pLimit = sqlite3Expr(pParse->db, TK_INTEGER, "1"); + pSel->pLimit = sqlite3PExpr(pParse, TK_LIMIT, pLimit, 0); } - - if( rHasNullFlag ){ - sqlite3SetHasNullFlag(v, pExpr->iTable, rHasNullFlag); + pSel->iLimit = 0; + if( sqlite3Select(pParse, pSel, &dest) ){ + pExpr->op2 = pExpr->op; + pExpr->op = TK_ERROR; + return 0; } - - if( jmpIfDynamic>=0 ){ - sqlite3VdbeJumpHere(v, jmpIfDynamic); + pExpr->iTable = rReg = dest.iSDParm; + ExprSetVVAProperty(pExpr, EP_NoReduce); + if( addrOnce ){ + sqlite3VdbeJumpHere(v, addrOnce); } - sqlite3ExprCachePop(pParse); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); + /* Subroutine return */ + assert( ExprUseYSub(pExpr) ); + assert( sqlite3VdbeGetOp(v,pExpr->y.sub.iAddr-1)->opcode==OP_BeginSubrtn + || pParse->nErr ); + sqlite3VdbeAddOp3(v, OP_Return, pExpr->y.sub.regReturn, + pExpr->y.sub.iAddr, 1); + VdbeCoverage(v); + sqlite3ClearTempRegCache(pParse); return rReg; } #endif /* SQLITE_OMIT_SUBQUERY */ +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Expr pIn is an IN(...) expression. This function checks that the +** sub-select on the RHS of the IN() operator has the same number of +** columns as the vector on the LHS. Or, if the RHS of the IN() is not +** a sub-query, that the LHS is a vector of size 1. +*/ +SQLITE_PRIVATE int sqlite3ExprCheckIN(Parse *pParse, Expr *pIn){ + int nVector = sqlite3ExprVectorSize(pIn->pLeft); + if( ExprUseXSelect(pIn) && !pParse->db->mallocFailed ){ + if( nVector!=pIn->x.pSelect->pEList->nExpr ){ + sqlite3SubselectError(pParse, pIn->x.pSelect->pEList->nExpr, nVector); + return 1; + } + }else if( nVector!=1 ){ + sqlite3VectorErrorMsg(pParse, pIn->pLeft); + return 1; + } + return 0; +} +#endif + #ifndef SQLITE_OMIT_SUBQUERY /* ** Generate code for an IN expression. @@ -91071,16 +113278,24 @@ SQLITE_PRIVATE int sqlite3CodeSubselect( ** x IN (SELECT ...) ** x IN (value, value, ...) ** -** The left-hand side (LHS) is a scalar expression. The right-hand side (RHS) -** is an array of zero or more values. The expression is true if the LHS is -** contained within the RHS. The value of the expression is unknown (NULL) -** if the LHS is NULL or if the LHS is not contained within the RHS and the -** RHS contains one or more NULL values. +** The left-hand side (LHS) is a scalar or vector expression. The +** right-hand side (RHS) is an array of zero or more scalar values, or a +** subquery. If the RHS is a subquery, the number of result columns must +** match the number of columns in the vector on the LHS. If the RHS is +** a list of values, the LHS must be a scalar. ** -** This routine generates code that jumps to destIfFalse if the LHS is not +** The IN operator is true if the LHS value is contained within the RHS. +** The result is false if the LHS is definitely not in the RHS. The +** result is NULL if the presence of the LHS in the RHS cannot be +** determined due to NULLs. +** +** This routine generates code that jumps to destIfFalse if the LHS is not ** contained within the RHS. If due to NULLs we cannot determine if the LHS ** is contained in the RHS then jump to destIfNull. If the LHS is contained ** within the RHS then fall through. +** +** See the separate in-operator.md documentation file in the canonical +** SQLite source tree for additional information. */ static void sqlite3ExprCodeIN( Parse *pParse, /* Parsing and code generating context */ @@ -91089,67 +113304,129 @@ static void sqlite3ExprCodeIN( int destIfNull /* Jump here if the results are unknown due to NULLs */ ){ int rRhsHasNull = 0; /* Register that is true if RHS contains NULL values */ - char affinity; /* Comparison affinity to use */ int eType; /* Type of the RHS */ - int r1; /* Temporary use register */ + int rLhs; /* Register(s) holding the LHS values */ + int rLhsOrig; /* LHS values prior to reordering by aiMap[] */ Vdbe *v; /* Statement under construction */ + int *aiMap = 0; /* Map from vector field to index column */ + char *zAff = 0; /* Affinity string for comparisons */ + int nVector; /* Size of vectors for this IN operator */ + int iDummy; /* Dummy parameter to exprCodeVector() */ + Expr *pLeft; /* The LHS of the IN operator */ + int i; /* loop counter */ + int destStep2; /* Where to jump when NULLs seen in step 2 */ + int destStep6 = 0; /* Start of code for Step 6 */ + int addrTruthOp; /* Address of opcode that determines the IN is true */ + int destNotNull; /* Jump here if a comparison is not true in step 6 */ + int addrTop; /* Top of the step-6 loop */ + int iTab = 0; /* Index to use */ + u8 okConstFactor = pParse->okConstFactor; - /* Compute the RHS. After this step, the table with cursor - ** pExpr->iTable will contains the values that make up the RHS. - */ + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); + pLeft = pExpr->pLeft; + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + zAff = exprINAffinity(pParse, pExpr); + nVector = sqlite3ExprVectorSize(pExpr->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, nVector*sizeof(int)); + if( pParse->db->mallocFailed ) goto sqlite3ExprCodeIN_oom_error; + + /* Attempt to compute the RHS. After this step, if anything other than + ** IN_INDEX_NOOP is returned, the table opened with cursor iTab + ** contains the values that make up the RHS. If IN_INDEX_NOOP is returned, + ** the RHS has not yet been coded. */ v = pParse->pVdbe; assert( v!=0 ); /* OOM detected prior to this routine */ VdbeNoopComment((v, "begin IN expr")); eType = sqlite3FindInIndex(pParse, pExpr, IN_INDEX_MEMBERSHIP | IN_INDEX_NOOP_OK, - destIfFalse==destIfNull ? 0 : &rRhsHasNull); + destIfFalse==destIfNull ? 0 : &rRhsHasNull, + aiMap, &iTab); - /* Figure out the affinity to use to create a key from the results - ** of the expression. affinityStr stores a static string suitable for - ** P4 of OP_MakeRecord. - */ - affinity = comparisonAffinity(pExpr); + assert( pParse->nErr || nVector==1 || eType==IN_INDEX_EPH + || eType==IN_INDEX_INDEX_ASC || eType==IN_INDEX_INDEX_DESC + ); +#ifdef SQLITE_DEBUG + /* Confirm that aiMap[] contains nVector integer values between 0 and + ** nVector-1. */ + for(i=0; i from " IN (...)". - */ - sqlite3ExprCachePush(pParse); - r1 = sqlite3GetTempReg(pParse); - sqlite3ExprCode(pParse, pExpr->pLeft, r1); + /* Code the LHS, the from " IN (...)". If the LHS is a + ** vector, then it is stored in an array of nVector registers starting + ** at r1. + ** + ** sqlite3FindInIndex() might have reordered the fields of the LHS vector + ** so that the fields are in the same order as an existing index. The + ** aiMap[] array contains a mapping from the original LHS field order to + ** the field order that matches the RHS index. + ** + ** Avoid factoring the LHS of the IN(...) expression out of the loop, + ** even if it is constant, as OP_Affinity may be used on the register + ** by code generated below. */ + assert( pParse->okConstFactor==okConstFactor ); + pParse->okConstFactor = 0; + rLhsOrig = exprCodeVector(pParse, pLeft, &iDummy); + pParse->okConstFactor = okConstFactor; + for(i=0; ix.pList; - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - int labelOk = sqlite3VdbeMakeLabel(v); + ExprList *pList; + CollSeq *pColl; + int labelOk = sqlite3VdbeMakeLabel(pParse); int r2, regToFree; int regCkNull = 0; int ii; - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); + assert( ExprUseXList(pExpr) ); + pList = pExpr->x.pList; + pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); if( destIfNull!=destIfFalse ){ regCkNull = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_BitAnd, r1, r1, regCkNull); + sqlite3VdbeAddOp3(v, OP_BitAnd, rLhs, rLhs, regCkNull); } for(ii=0; iinExpr; ii++){ r2 = sqlite3ExprCodeTemp(pParse, pList->a[ii].pExpr, ®ToFree); if( regCkNull && sqlite3ExprCanBeNull(pList->a[ii].pExpr) ){ sqlite3VdbeAddOp3(v, OP_BitAnd, regCkNull, r2, regCkNull); } - if( iinExpr-1 || destIfNull!=destIfFalse ){ - sqlite3VdbeAddOp4(v, OP_Eq, r1, labelOk, r2, - (void*)pColl, P4_COLLSEQ); - VdbeCoverageIf(v, iinExpr-1); - VdbeCoverageIf(v, ii==pList->nExpr-1); - sqlite3VdbeChangeP5(v, affinity); - }else{ - assert( destIfNull==destIfFalse ); - sqlite3VdbeAddOp4(v, OP_Ne, r1, destIfFalse, r2, - (void*)pColl, P4_COLLSEQ); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, affinity | SQLITE_JUMPIFNULL); - } sqlite3ReleaseTempReg(pParse, regToFree); + if( iinExpr-1 || destIfNull!=destIfFalse ){ + int op = rLhs!=r2 ? OP_Eq : OP_NotNull; + sqlite3VdbeAddOp4(v, op, rLhs, labelOk, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, iinExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_Eq); + VdbeCoverageIf(v, iinExpr-1 && op==OP_NotNull); + VdbeCoverageIf(v, ii==pList->nExpr-1 && op==OP_NotNull); + sqlite3VdbeChangeP5(v, zAff[0]); + }else{ + int op = rLhs!=r2 ? OP_Ne : OP_IsNull; + assert( destIfNull==destIfFalse ); + sqlite3VdbeAddOp4(v, op, rLhs, destIfFalse, r2, + (void*)pColl, P4_COLLSEQ); + VdbeCoverageIf(v, op==OP_Ne); + VdbeCoverageIf(v, op==OP_IsNull); + sqlite3VdbeChangeP5(v, zAff[0] | SQLITE_JUMPIFNULL); + } } if( regCkNull ){ sqlite3VdbeAddOp2(v, OP_IsNull, regCkNull, destIfNull); VdbeCoverage(v); @@ -91157,78 +113434,122 @@ static void sqlite3ExprCodeIN( } sqlite3VdbeResolveLabel(v, labelOk); sqlite3ReleaseTempReg(pParse, regCkNull); + goto sqlite3ExprCodeIN_finished; + } + + /* Step 2: Check to see if the LHS contains any NULL columns. If the + ** LHS does contain NULLs then the result must be either FALSE or NULL. + ** We will then skip the binary search of the RHS. + */ + if( destIfNull==destIfFalse ){ + destStep2 = destIfFalse; }else{ - - /* If the LHS is NULL, then the result is either false or NULL depending - ** on whether the RHS is empty or not, respectively. - */ - if( sqlite3ExprCanBeNull(pExpr->pLeft) ){ - if( destIfNull==destIfFalse ){ - /* Shortcut for the common case where the false and NULL outcomes are - ** the same. */ - sqlite3VdbeAddOp2(v, OP_IsNull, r1, destIfNull); VdbeCoverage(v); - }else{ - int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, r1); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Rewind, pExpr->iTable, destIfFalse); - VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfNull); - sqlite3VdbeJumpHere(v, addr1); - } - } - - if( eType==IN_INDEX_ROWID ){ - /* In this case, the RHS is the ROWID of table b-tree - */ - sqlite3VdbeAddOp2(v, OP_MustBeInt, r1, destIfFalse); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, pExpr->iTable, destIfFalse, r1); + destStep2 = destStep6 = sqlite3VdbeMakeLabel(pParse); + } + for(i=0; ipLeft, i); + if( pParse->nErr ) goto sqlite3ExprCodeIN_oom_error; + if( sqlite3ExprCanBeNull(p) ){ + sqlite3VdbeAddOp2(v, OP_IsNull, rLhs+i, destStep2); VdbeCoverage(v); - }else{ - /* In this case, the RHS is an index b-tree. - */ - sqlite3VdbeAddOp4(v, OP_Affinity, r1, 1, 0, &affinity, 1); - - /* If the set membership test fails, then the result of the - ** "x IN (...)" expression must be either 0 or NULL. If the set - ** contains no NULL values, then the result is 0. If the set - ** contains one or more NULL values, then the result of the - ** expression is also NULL. - */ - assert( destIfFalse!=destIfNull || rRhsHasNull==0 ); - if( rRhsHasNull==0 ){ - /* This branch runs if it is known at compile time that the RHS - ** cannot contain NULL values. This happens as the result - ** of a "NOT NULL" constraint in the database schema. - ** - ** Also run this branch if NULL is equivalent to FALSE - ** for this particular IN operator. - */ - sqlite3VdbeAddOp4Int(v, OP_NotFound, pExpr->iTable, destIfFalse, r1, 1); - VdbeCoverage(v); - }else{ - /* In this branch, the RHS of the IN might contain a NULL and - ** the presence of a NULL on the RHS makes a difference in the - ** outcome. - */ - int addr1; - - /* First check to see if the LHS is contained in the RHS. If so, - ** then the answer is TRUE the presence of NULLs in the RHS does - ** not matter. If the LHS is not contained in the RHS, then the - ** answer is NULL if the RHS contains NULLs and the answer is - ** FALSE if the RHS is NULL-free. - */ - addr1 = sqlite3VdbeAddOp4Int(v, OP_Found, pExpr->iTable, 0, r1, 1); - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IsNull, rRhsHasNull, destIfNull); - VdbeCoverage(v); - sqlite3VdbeGoto(v, destIfFalse); - sqlite3VdbeJumpHere(v, addr1); - } } } - sqlite3ReleaseTempReg(pParse, r1); - sqlite3ExprCachePop(pParse); + + /* Step 3. The LHS is now known to be non-NULL. Do the binary search + ** of the RHS using the LHS as a probe. If found, the result is + ** true. + */ + if( eType==IN_INDEX_ROWID ){ + /* In this case, the RHS is the ROWID of table b-tree and so we also + ** know that the RHS is non-NULL. Hence, we combine steps 3 and 4 + ** into a single opcode. */ + sqlite3VdbeAddOp3(v, OP_SeekRowid, iTab, destIfFalse, rLhs); + VdbeCoverage(v); + addrTruthOp = sqlite3VdbeAddOp0(v, OP_Goto); /* Return True */ + }else{ + sqlite3VdbeAddOp4(v, OP_Affinity, rLhs, nVector, 0, zAff, nVector); + if( destIfFalse==destIfNull ){ + /* Combine Step 3 and Step 5 into a single opcode */ + if( ExprHasProperty(pExpr, EP_Subrtn) ){ + const VdbeOp *pOp = sqlite3VdbeGetOp(v, pExpr->y.sub.iAddr); + assert( pOp->opcode==OP_Once || pParse->nErr ); + if( pOp->opcode==OP_Once && pOp->p3>0 ){ + assert( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) ); + sqlite3VdbeAddOp4Int(v, OP_Filter, pOp->p3, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + } + } + sqlite3VdbeAddOp4Int(v, OP_NotFound, iTab, destIfFalse, + rLhs, nVector); VdbeCoverage(v); + goto sqlite3ExprCodeIN_finished; + } + /* Ordinary Step 3, for the case where FALSE and NULL are distinct */ + addrTruthOp = sqlite3VdbeAddOp4Int(v, OP_Found, iTab, 0, + rLhs, nVector); VdbeCoverage(v); + } + + /* Step 4. If the RHS is known to be non-NULL and we did not find + ** an match on the search above, then the result must be FALSE. + */ + if( rRhsHasNull && nVector==1 ){ + sqlite3VdbeAddOp2(v, OP_NotNull, rRhsHasNull, destIfFalse); + VdbeCoverage(v); + } + + /* Step 5. If we do not care about the difference between NULL and + ** FALSE, then just return false. + */ + if( destIfFalse==destIfNull ) sqlite3VdbeGoto(v, destIfFalse); + + /* Step 6: Loop through rows of the RHS. Compare each row to the LHS. + ** If any comparison is NULL, then the result is NULL. If all + ** comparisons are FALSE then the final result is FALSE. + ** + ** For a scalar LHS, it is sufficient to check just the first row + ** of the RHS. + */ + if( destStep6 ) sqlite3VdbeResolveLabel(v, destStep6); + addrTop = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, destIfFalse); + VdbeCoverage(v); + if( nVector>1 ){ + destNotNull = sqlite3VdbeMakeLabel(pParse); + }else{ + /* For nVector==1, combine steps 6 and 7 by immediately returning + ** FALSE if the first comparison is not NULL */ + destNotNull = destIfFalse; + } + for(i=0; i1 ){ + sqlite3VdbeResolveLabel(v, destNotNull); + sqlite3VdbeAddOp2(v, OP_Next, iTab, addrTop+1); + VdbeCoverage(v); + + /* Step 7: If we reach this point, we know that the result must + ** be false. */ + sqlite3VdbeAddOp2(v, OP_Goto, 0, destIfFalse); + } + + /* Jumps here in order to return true. */ + sqlite3VdbeJumpHere(v, addrTruthOp); + +sqlite3ExprCodeIN_finished: + if( rLhs!=rLhsOrig ) sqlite3ReleaseTempReg(pParse, rLhs); VdbeComment((v, "end IN expr")); +sqlite3ExprCodeIN_oom_error: + sqlite3DbFree(pParse->db, aiMap); + sqlite3DbFree(pParse->db, zAff); } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -91237,7 +113558,7 @@ static void sqlite3ExprCodeIN( ** Generate an instruction that will put the floating point ** value described by z[0..n-1] into register iMem. ** -** The z[] string will probably not be zero-terminated. But the +** The z[] string will probably not be zero-terminated. But the ** z[n] character is guaranteed to be something that does not look ** like the continuation of the number. */ @@ -91272,189 +113593,27 @@ static void codeInteger(Parse *pParse, Expr *pExpr, int negFlag, int iMem){ const char *z = pExpr->u.zToken; assert( z!=0 ); c = sqlite3DecOrHexToI64(z, &value); - if( c==0 || (c==2 && negFlag) ){ - if( negFlag ){ value = c==2 ? SMALLEST_INT64 : -value; } - sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); - }else{ + if( (c==3 && !negFlag) || (c==2) || (negFlag && value==SMALLEST_INT64)){ #ifdef SQLITE_OMIT_FLOATING_POINT - sqlite3ErrorMsg(pParse, "oversized integer: %s%s", negFlag ? "-" : "", z); + sqlite3ErrorMsg(pParse, "oversized integer: %s%#T", negFlag?"-":"",pExpr); #else #ifndef SQLITE_OMIT_HEX_INTEGER if( sqlite3_strnicmp(z,"0x",2)==0 ){ - sqlite3ErrorMsg(pParse, "hex literal too big: %s", z); + sqlite3ErrorMsg(pParse, "hex literal too big: %s%#T", + negFlag?"-":"",pExpr); }else #endif { codeReal(v, z, negFlag, iMem); } #endif + }else{ + if( negFlag ){ value = c==3 ? SMALLEST_INT64 : -value; } + sqlite3VdbeAddOp4Dup8(v, OP_Int64, 0, iMem, 0, (u8*)&value, P4_INT64); } } } -#if defined(SQLITE_DEBUG) -/* -** Verify the consistency of the column cache -*/ -static int cacheIsValid(Parse *pParse){ - int i, n; - for(i=n=0; iaColCache[i].iReg>0 ) n++; - } - return n==pParse->nColCache; -} -#endif - -/* -** Clear a cache entry. -*/ -static void cacheEntryClear(Parse *pParse, struct yColCache *p){ - if( p->tempReg ){ - if( pParse->nTempRegaTempReg) ){ - pParse->aTempReg[pParse->nTempReg++] = p->iReg; - } - p->tempReg = 0; - } - p->iReg = 0; - pParse->nColCache--; - assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); -} - - -/* -** Record in the column cache that a particular column from a -** particular table is stored in a particular register. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheStore(Parse *pParse, int iTab, int iCol, int iReg){ - int i; - int minLru; - int idxLru; - struct yColCache *p; - - /* Unless an error has occurred, register numbers are always positive. */ - assert( iReg>0 || pParse->nErr || pParse->db->mallocFailed ); - assert( iCol>=-1 && iCol<32768 ); /* Finite column numbers */ - - /* The SQLITE_ColumnCache flag disables the column cache. This is used - ** for testing only - to verify that SQLite always gets the same answer - ** with and without the column cache. - */ - if( OptimizationDisabled(pParse->db, SQLITE_ColumnCache) ) return; - - /* First replace any existing entry. - ** - ** Actually, the way the column cache is currently used, we are guaranteed - ** that the object will never already be in cache. Verify this guarantee. - */ -#ifndef NDEBUG - for(i=0, p=pParse->aColCache; iiReg==0 || p->iTable!=iTab || p->iColumn!=iCol ); - } -#endif - - /* Find an empty slot and replace it */ - for(i=0, p=pParse->aColCache; iiReg==0 ){ - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - pParse->nColCache++; - assert( pParse->db->mallocFailed || cacheIsValid(pParse) ); - return; - } - } - - /* Replace the last recently used */ - minLru = 0x7fffffff; - idxLru = -1; - for(i=0, p=pParse->aColCache; ilrulru; - } - } - if( ALWAYS(idxLru>=0) ){ - p = &pParse->aColCache[idxLru]; - p->iLevel = pParse->iCacheLevel; - p->iTable = iTab; - p->iColumn = iCol; - p->iReg = iReg; - p->tempReg = 0; - p->lru = pParse->iCacheCnt++; - assert( cacheIsValid(pParse) ); - return; - } -} - -/* -** Indicate that registers between iReg..iReg+nReg-1 are being overwritten. -** Purge the range of registers from the column cache. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheRemove(Parse *pParse, int iReg, int nReg){ - struct yColCache *p; - if( iReg<=0 || pParse->nColCache==0 ) return; - p = &pParse->aColCache[SQLITE_N_COLCACHE-1]; - while(1){ - if( p->iReg >= iReg && p->iReg < iReg+nReg ) cacheEntryClear(pParse, p); - if( p==pParse->aColCache ) break; - p--; - } -} - -/* -** Remember the current column cache context. Any new entries added -** added to the column cache after this call are removed when the -** corresponding pop occurs. -*/ -SQLITE_PRIVATE void sqlite3ExprCachePush(Parse *pParse){ - pParse->iCacheLevel++; -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("PUSH to %d\n", pParse->iCacheLevel); - } -#endif -} - -/* -** Remove from the column cache any entries that were added since the -** the previous sqlite3ExprCachePush operation. In other words, restore -** the cache to the state it was in prior the most recent Push. -*/ -SQLITE_PRIVATE void sqlite3ExprCachePop(Parse *pParse){ - int i; - struct yColCache *p; - assert( pParse->iCacheLevel>=1 ); - pParse->iCacheLevel--; -#ifdef SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("POP to %d\n", pParse->iCacheLevel); - } -#endif - for(i=0, p=pParse->aColCache; iiReg && p->iLevel>pParse->iCacheLevel ){ - cacheEntryClear(pParse, p); - } - } -} - -/* -** When a cached column is reused, make sure that its register is -** no longer available as a temp register. ticket #3879: that same -** register might be in the cache in multiple places, so be sure to -** get them all. -*/ -static void sqlite3ExprCachePinRegister(Parse *pParse, int iReg){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg==iReg ){ - p->tempReg = 0; - } - } -} /* Generate code that will load into register regOut a value that is ** appropriate for the iIdxCol-th column of index pIdx. @@ -91470,47 +113629,101 @@ SQLITE_PRIVATE void sqlite3ExprCodeLoadIndexColumn( if( iTabCol==XN_EXPR ){ assert( pIdx->aColExpr ); assert( pIdx->aColExpr->nExpr>iIdxCol ); - pParse->iSelfTab = iTabCur; + pParse->iSelfTab = iTabCur + 1; sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[iIdxCol].pExpr, regOut); + pParse->iSelfTab = 0; }else{ sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pIdx->pTable, iTabCur, iTabCol, regOut); } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** Generate code that will compute the value of generated column pCol +** and store the result in register regOut +*/ +SQLITE_PRIVATE void sqlite3ExprCodeGeneratedColumn( + Parse *pParse, /* Parsing context */ + Table *pTab, /* Table containing the generated column */ + Column *pCol, /* The generated column */ + int regOut /* Put the result in this register */ +){ + int iAddr; + Vdbe *v = pParse->pVdbe; + int nErr = pParse->nErr; + assert( v!=0 ); + assert( pParse->iSelfTab!=0 ); + if( pParse->iSelfTab>0 ){ + iAddr = sqlite3VdbeAddOp3(v, OP_IfNullRow, pParse->iSelfTab-1, 0, regOut); + }else{ + iAddr = 0; + } + sqlite3ExprCodeCopy(pParse, sqlite3ColumnExpr(pTab,pCol), regOut); + if( pCol->affinity>=SQLITE_AFF_TEXT ){ + sqlite3VdbeAddOp4(v, OP_Affinity, regOut, 1, 0, &pCol->affinity, 1); + } + if( iAddr ) sqlite3VdbeJumpHere(v, iAddr); + if( pParse->nErr>nErr ) pParse->db->errByteOffset = -1; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + /* ** Generate code to extract the value of the iCol-th column of a table. */ SQLITE_PRIVATE void sqlite3ExprCodeGetColumnOfTable( - Vdbe *v, /* The VDBE under construction */ + Vdbe *v, /* Parsing context */ Table *pTab, /* The table containing the value */ int iTabCur, /* The table cursor. Or the PK cursor for WITHOUT ROWID */ int iCol, /* Index of the column to extract */ int regOut /* Extract the value into this register */ ){ + Column *pCol; + assert( v!=0 ); + assert( pTab!=0 ); + assert( iCol!=XN_EXPR ); if( iCol<0 || iCol==pTab->iPKey ){ sqlite3VdbeAddOp2(v, OP_Rowid, iTabCur, regOut); + VdbeComment((v, "%s.rowid", pTab->zName)); }else{ - int op = IsVirtual(pTab) ? OP_VColumn : OP_Column; - int x = iCol; - if( !HasRowid(pTab) ){ - x = sqlite3ColumnOfIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + int op; + int x; + if( IsVirtual(pTab) ){ + op = OP_VColumn; + x = iCol; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( (pCol = &pTab->aCol[iCol])->colFlags & COLFLAG_VIRTUAL ){ + Parse *pParse = sqlite3VdbeParser(v); + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zCnName); + }else{ + int savedSelfTab = pParse->iSelfTab; + pCol->colFlags |= COLFLAG_BUSY; + pParse->iSelfTab = iTabCur+1; + sqlite3ExprCodeGeneratedColumn(pParse, pTab, pCol, regOut); + pParse->iSelfTab = savedSelfTab; + pCol->colFlags &= ~COLFLAG_BUSY; + } + return; +#endif + }else if( !HasRowid(pTab) ){ + testcase( iCol!=sqlite3TableColumnToStorage(pTab, iCol) ); + x = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), iCol); + op = OP_Column; + }else{ + x = sqlite3TableColumnToStorage(pTab,iCol); + testcase( x!=iCol ); + op = OP_Column; } sqlite3VdbeAddOp3(v, op, iTabCur, x, regOut); - } - if( iCol>=0 ){ sqlite3ColumnDefault(v, pTab, iCol, regOut); } } /* ** Generate code that will extract the iColumn-th column from -** table pTab and store the column value in a register. -** -** An effort is made to store the column value in register iReg. This -** is not garanteeed for GetColumn() - the result can be stored in -** any register. But the result is guaranteed to land in register iReg -** for GetColumnToReg(). +** table pTab and store the column value in register iReg. ** ** There must be an open cursor to pTab in iTable when this routine ** is called. If iColumn<0 then code is generated that extracts the rowid. @@ -91523,105 +113736,369 @@ SQLITE_PRIVATE int sqlite3ExprCodeGetColumn( int iReg, /* Store results here */ u8 p5 /* P5 value for OP_Column + FLAGS */ ){ - Vdbe *v = pParse->pVdbe; - int i; - struct yColCache *p; - - for(i=0, p=pParse->aColCache; iiReg>0 && p->iTable==iTable && p->iColumn==iColumn ){ - p->lru = pParse->iCacheCnt++; - sqlite3ExprCachePinRegister(pParse, p->iReg); - return p->iReg; - } - } - assert( v!=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iTable, iColumn, iReg); + assert( pParse->pVdbe!=0 ); + assert( (p5 & (OPFLAG_NOCHNG|OPFLAG_TYPEOFARG|OPFLAG_LENGTHARG))==p5 ); + assert( IsVirtual(pTab) || (p5 & OPFLAG_NOCHNG)==0 ); + sqlite3ExprCodeGetColumnOfTable(pParse->pVdbe, pTab, iTable, iColumn, iReg); if( p5 ){ - sqlite3VdbeChangeP5(v, p5); - }else{ - sqlite3ExprCacheStore(pParse, iTable, iColumn, iReg); + VdbeOp *pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); + if( pOp->opcode==OP_Column ) pOp->p5 = p5; + if( pOp->opcode==OP_VColumn ) pOp->p5 = (p5 & OPFLAG_NOCHNG); } return iReg; } -SQLITE_PRIVATE void sqlite3ExprCodeGetColumnToReg( - Parse *pParse, /* Parsing and code generating context */ - Table *pTab, /* Description of the table we are reading from */ - int iColumn, /* Index of the table column */ - int iTable, /* The cursor pointing to the table */ - int iReg /* Store results here */ -){ - int r1 = sqlite3ExprCodeGetColumn(pParse, pTab, iColumn, iTable, iReg, 0); - if( r1!=iReg ) sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, r1, iReg); -} - - -/* -** Clear all column cache entries. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheClear(Parse *pParse){ - int i; - struct yColCache *p; - -#if SQLITE_DEBUG - if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ - printf("CLEAR\n"); - } -#endif - for(i=0, p=pParse->aColCache; iiReg ){ - cacheEntryClear(pParse, p); - } - } -} - -/* -** Record the fact that an affinity change has occurred on iCount -** registers starting with iStart. -*/ -SQLITE_PRIVATE void sqlite3ExprCacheAffinityChange(Parse *pParse, int iStart, int iCount){ - sqlite3ExprCacheRemove(pParse, iStart, iCount); -} /* ** Generate code to move content from registers iFrom...iFrom+nReg-1 -** over to iTo..iTo+nReg-1. Keep the column cache up-to-date. +** over to iTo..iTo+nReg-1. */ SQLITE_PRIVATE void sqlite3ExprCodeMove(Parse *pParse, int iFrom, int iTo, int nReg){ - assert( iFrom>=iTo+nReg || iFrom+nReg<=iTo ); sqlite3VdbeAddOp3(pParse->pVdbe, OP_Move, iFrom, iTo, nReg); - sqlite3ExprCacheRemove(pParse, iFrom, nReg); } -#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) /* -** Return true if any register in the range iFrom..iTo (inclusive) -** is used as part of the column cache. -** -** This routine is used within assert() and testcase() macros only -** and does not appear in a normal build. +** Convert a scalar expression node to a TK_REGISTER referencing +** register iReg. The caller must ensure that iReg already contains +** the correct value for the expression. */ -static int usedAsColumnCache(Parse *pParse, int iFrom, int iTo){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg; - if( r>=iFrom && r<=iTo ) return 1; /*NO_TEST*/ +SQLITE_PRIVATE void sqlite3ExprToRegister(Expr *pExpr, int iReg){ + Expr *p = sqlite3ExprSkipCollateAndLikely(pExpr); + if( NEVER(p==0) ) return; + if( p->op==TK_REGISTER ){ + assert( p->iTable==iReg ); + }else{ + p->op2 = p->op; + p->op = TK_REGISTER; + p->iTable = iReg; + ExprClearProperty(p, EP_Skip); + } +} + +/* +** Evaluate an expression (either a vector or a scalar expression) and store +** the result in contiguous temporary registers. Return the index of +** the first register used to store the result. +** +** If the returned result register is a temporary scalar, then also write +** that register number into *piFreeable. If the returned result register +** is not a temporary or if the expression is a vector set *piFreeable +** to 0. +*/ +static int exprCodeVector(Parse *pParse, Expr *p, int *piFreeable){ + int iResult; + int nResult = sqlite3ExprVectorSize(p); + if( nResult==1 ){ + iResult = sqlite3ExprCodeTemp(pParse, p, piFreeable); + }else{ + *piFreeable = 0; + if( p->op==TK_SELECT ){ +#if SQLITE_OMIT_SUBQUERY + iResult = 0; +#else + iResult = sqlite3CodeSubselect(pParse, p); +#endif + }else{ + int i; + iResult = pParse->nMem+1; + pParse->nMem += nResult; + assert( ExprUseXList(p) ); + for(i=0; ix.pList->a[i].pExpr, i+iResult); + } + } + } + return iResult; +} + +/* +** If the last opcode is a OP_Copy, then set the do-not-merge flag (p5) +** so that a subsequent copy will not be merged into this one. +*/ +static void setDoNotMergeFlagOnCopy(Vdbe *v){ + if( sqlite3VdbeGetLastOp(v)->opcode==OP_Copy ){ + sqlite3VdbeChangeP5(v, 1); /* Tag trailing OP_Copy as not mergeable */ + } +} + +/* +** Generate code to implement special SQL functions that are implemented +** in-line rather than by using the usual callbacks. +*/ +static int exprCodeInlineFunction( + Parse *pParse, /* Parsing context */ + ExprList *pFarg, /* List of function arguments */ + int iFuncId, /* Function ID. One of the INTFUNC_... values */ + int target /* Store function result in this register */ +){ + int nFarg; + Vdbe *v = pParse->pVdbe; + assert( v!=0 ); + assert( pFarg!=0 ); + nFarg = pFarg->nExpr; + assert( nFarg>0 ); /* All in-line functions have at least one argument */ + switch( iFuncId ){ + case INLINEFUNC_coalesce: { + /* Attempt a direct implementation of the built-in COALESCE() and + ** IFNULL() functions. This avoids unnecessary evaluation of + ** arguments past the first non-NULL argument. + */ + int endCoalesce = sqlite3VdbeMakeLabel(pParse); + int i; + assert( nFarg>=2 ); + sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); + for(i=1; ia[i].pExpr, target); + } + setDoNotMergeFlagOnCopy(v); + sqlite3VdbeResolveLabel(v, endCoalesce); + break; + } + case INLINEFUNC_iif: { + Expr caseExpr; + memset(&caseExpr, 0, sizeof(caseExpr)); + caseExpr.op = TK_CASE; + caseExpr.x.pList = pFarg; + return sqlite3ExprCodeTarget(pParse, &caseExpr, target); + } +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + case INLINEFUNC_sqlite_offset: { + Expr *pArg = pFarg->a[0].pExpr; + if( pArg->op==TK_COLUMN && pArg->iTable>=0 ){ + sqlite3VdbeAddOp3(v, OP_Offset, pArg->iTable, pArg->iColumn, target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + break; + } +#endif + default: { + /* The UNLIKELY() function is a no-op. The result is the value + ** of the first argument. + */ + assert( nFarg==1 || nFarg==2 ); + target = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); + break; + } + + /*********************************************************************** + ** Test-only SQL functions that are only usable if enabled + ** via SQLITE_TESTCTRL_INTERNAL_FUNCTIONS + */ +#if !defined(SQLITE_UNTESTABLE) + case INLINEFUNC_expr_compare: { + /* Compare two expressions using sqlite3ExprCompare() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprCompare(0,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_expr_implies_expr: { + /* Compare two expressions using sqlite3ExprImpliesExpr() */ + assert( nFarg==2 ); + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesExpr(pParse,pFarg->a[0].pExpr, pFarg->a[1].pExpr,-1), + target); + break; + } + + case INLINEFUNC_implies_nonnull_row: { + /* Result of sqlite3ExprImpliesNonNullRow() */ + Expr *pA1; + assert( nFarg==2 ); + pA1 = pFarg->a[1].pExpr; + if( pA1->op==TK_COLUMN ){ + sqlite3VdbeAddOp2(v, OP_Integer, + sqlite3ExprImpliesNonNullRow(pFarg->a[0].pExpr,pA1->iTable,1), + target); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); + } + break; + } + + case INLINEFUNC_affinity: { + /* The AFFINITY() function evaluates to a string that describes + ** the type affinity of the argument. This is used for testing of + ** the SQLite type logic. + */ + const char *azAff[] = { "blob", "text", "numeric", "integer", + "real", "flexnum" }; + char aff; + assert( nFarg==1 ); + aff = sqlite3ExprAffinity(pFarg->a[0].pExpr); + assert( aff<=SQLITE_AFF_NONE + || (aff>=SQLITE_AFF_BLOB && aff<=SQLITE_AFF_FLEXNUM) ); + sqlite3VdbeLoadString(v, target, + (aff<=SQLITE_AFF_NONE) ? "none" : azAff[aff-SQLITE_AFF_BLOB]); + break; + } +#endif /* !defined(SQLITE_UNTESTABLE) */ + } + return target; +} + +/* +** Expression Node callback for sqlite3ExprCanReturnSubtype(). +** +** Only a function call is able to return a subtype. So if the node +** is not a function call, return WRC_Prune immediately. +** +** A function call is able to return a subtype if it has the +** SQLITE_RESULT_SUBTYPE property. +** +** Assume that every function is able to pass-through a subtype from +** one of its argument (using sqlite3_result_value()). Most functions +** are not this way, but we don't have a mechanism to distinguish those +** that are from those that are not, so assume they all work this way. +** That means that if one of its arguments is another function and that +** other function is able to return a subtype, then this function is +** able to return a subtype. +*/ +static int exprNodeCanReturnSubtype(Walker *pWalker, Expr *pExpr){ + int n; + FuncDef *pDef; + sqlite3 *db; + if( pExpr->op!=TK_FUNCTION ){ + return WRC_Prune; + } + assert( ExprUseXList(pExpr) ); + db = pWalker->pParse->db; + n = ALWAYS(pExpr->x.pList) ? pExpr->x.pList->nExpr : 0; + pDef = sqlite3FindFunction(db, pExpr->u.zToken, n, ENC(db), 0); + if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_RESULT_SUBTYPE)!=0 ){ + pWalker->eCode = 1; + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Return TRUE if expression pExpr is able to return a subtype. +** +** A TRUE return does not guarantee that a subtype will be returned. +** It only indicates that a subtype return is possible. False positives +** are acceptable as they only disable an optimization. False negatives, +** on the other hand, can lead to incorrect answers. +*/ +static int sqlite3ExprCanReturnSubtype(Parse *pParse, Expr *pExpr){ + Walker w; + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = exprNodeCanReturnSubtype; + sqlite3WalkExpr(&w, pExpr); + return w.eCode; +} + + +/* +** Check to see if pExpr is one of the indexed expressions on pParse->pIdxEpr. +** If it is, then resolve the expression by reading from the index and +** return the register into which the value has been read. If pExpr is +** not an indexed expression, then return negative. +*/ +static SQLITE_NOINLINE int sqlite3IndexedExprLookup( + Parse *pParse, /* The parsing context */ + Expr *pExpr, /* The expression to potentially bypass */ + int target /* Where to store the result of the expression */ +){ + IndexedExpr *p; + Vdbe *v; + for(p=pParse->pIdxEpr; p; p=p->pIENext){ + u8 exprAff; + int iDataCur = p->iDataCur; + if( iDataCur<0 ) continue; + if( pParse->iSelfTab ){ + if( p->iDataCur!=pParse->iSelfTab-1 ) continue; + iDataCur = -1; + } + if( sqlite3ExprCompare(0, pExpr, p->pExpr, iDataCur)!=0 ) continue; + assert( p->aff>=SQLITE_AFF_BLOB && p->aff<=SQLITE_AFF_NUMERIC ); + exprAff = sqlite3ExprAffinity(pExpr); + if( (exprAff<=SQLITE_AFF_BLOB && p->aff!=SQLITE_AFF_BLOB) + || (exprAff==SQLITE_AFF_TEXT && p->aff!=SQLITE_AFF_TEXT) + || (exprAff>=SQLITE_AFF_NUMERIC && p->aff!=SQLITE_AFF_NUMERIC) + ){ + /* Affinity mismatch on a generated column */ + continue; + } + + + /* Functions that might set a subtype should not be replaced by the + ** value taken from an expression index if they are themselves an + ** argument to another scalar function or aggregate. + ** https://sqlite.org/forum/forumpost/68d284c86b082c3e */ + if( ExprHasProperty(pExpr, EP_SubtArg) + && sqlite3ExprCanReturnSubtype(pParse, pExpr) + ){ + continue; + } + + v = pParse->pVdbe; + assert( v!=0 ); + if( p->bMaybeNullRow ){ + /* If the index is on a NULL row due to an outer join, then we + ** cannot extract the value from the index. The value must be + ** computed using the original expression. */ + int addr = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_IfNullRow, p->iIdxCur, addr+3, target); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + sqlite3VdbeGoto(v, 0); + p = pParse->pIdxEpr; + pParse->pIdxEpr = 0; + sqlite3ExprCode(pParse, pExpr, target); + pParse->pIdxEpr = p; + sqlite3VdbeJumpHere(v, addr+2); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, p->iIdxCur, p->iIdxCol, target); + VdbeComment((v, "%s expr-column %d", p->zIdxName, p->iIdxCol)); + } + return target; + } + return -1; /* Not found */ +} + + +/* +** Expresion pExpr is guaranteed to be a TK_COLUMN or equivalent. This +** function checks the Parse.pIdxPartExpr list to see if this column +** can be replaced with a constant value. If so, it generates code to +** put the constant value in a register (ideally, but not necessarily, +** register iTarget) and returns the register number. +** +** Or, if the TK_COLUMN cannot be replaced by a constant, zero is +** returned. +*/ +static int exprPartidxExprLookup(Parse *pParse, Expr *pExpr, int iTarget){ + IndexedExpr *p; + for(p=pParse->pIdxPartExpr; p; p=p->pIENext){ + if( pExpr->iColumn==p->iIdxCol && pExpr->iTable==p->iDataCur ){ + Vdbe *v = pParse->pVdbe; + int addr = 0; + int ret; + + if( p->bMaybeNullRow ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNullRow, p->iIdxCur); + } + ret = sqlite3ExprCodeTarget(pParse, p->pExpr, iTarget); + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Affinity, ret, 1, 0, + (const char*)&p->aff, 1); + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeChangeP3(v, addr, ret); + } + return ret; + } } return 0; } -#endif /* SQLITE_DEBUG || SQLITE_COVERAGE_TEST */ -/* -** Convert an expression node to a TK_REGISTER -*/ -static void exprToRegister(Expr *p, int iReg){ - p->op2 = p->op; - p->op = TK_REGISTER; - p->iTable = iReg; - ExprClearProperty(p, EP_Skip); -} - /* ** Generate code into the current Vdbe to evaluate the given ** expression. Attempt to store the results in register "target". @@ -91639,73 +114116,187 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) int inReg = target; /* Results stored in register inReg */ int regFree1 = 0; /* If non-zero free this temporary register */ int regFree2 = 0; /* If non-zero free this temporary register */ - int r1, r2, r3, r4; /* Various register numbers */ - sqlite3 *db = pParse->db; /* The database connection */ + int r1, r2; /* Various register numbers */ Expr tempX; /* Temporary expression node */ + int p5 = 0; assert( target>0 && target<=pParse->nMem ); - if( v==0 ){ - assert( pParse->db->mallocFailed ); - return 0; - } + assert( v!=0 ); +expr_code_doover: if( pExpr==0 ){ op = TK_NULL; + }else if( pParse->pIdxEpr!=0 + && !ExprHasProperty(pExpr, EP_Leaf) + && (r1 = sqlite3IndexedExprLookup(pParse, pExpr, target))>=0 + ){ + return r1; }else{ + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); op = pExpr->op; } + assert( op!=TK_ORDER ); switch( op ){ case TK_AGG_COLUMN: { AggInfo *pAggInfo = pExpr->pAggInfo; - struct AggInfo_col *pCol = &pAggInfo->aCol[pExpr->iAgg]; - if( !pAggInfo->directMode ){ - assert( pCol->iMem>0 ); - inReg = pCol->iMem; - break; - }else if( pAggInfo->useSortingIdx ){ - sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, - pCol->iSorterColumn, target); + struct AggInfo_col *pCol; + assert( pAggInfo!=0 ); + assert( pExpr->iAgg>=0 ); + if( pExpr->iAgg>=pAggInfo->nColumn ){ + /* Happens when the left table of a RIGHT JOIN is null and + ** is using an expression index */ + sqlite3VdbeAddOp2(v, OP_Null, 0, target); +#ifdef SQLITE_VDBE_COVERAGE + /* Verify that the OP_Null above is exercised by tests + ** tag-20230325-2 */ + sqlite3VdbeAddOp3(v, OP_NotNull, target, 1, 20230325); + VdbeCoverageNeverTaken(v); +#endif break; } + pCol = &pAggInfo->aCol[pExpr->iAgg]; + if( !pAggInfo->directMode ){ + return AggInfoColumnReg(pAggInfo, pExpr->iAgg); + }else if( pAggInfo->useSortingIdx ){ + Table *pTab = pCol->pTab; + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pCol->iSorterColumn, target); + if( pTab==0 ){ + /* No comment added */ + }else if( pCol->iColumn<0 ){ + VdbeComment((v,"%s.rowid",pTab->zName)); + }else{ + VdbeComment((v,"%s.%s", + pTab->zName, pTab->aCol[pCol->iColumn].zCnName)); + if( pTab->aCol[pCol->iColumn].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + } + } + return target; + }else if( pExpr->y.pTab==0 ){ + /* This case happens when the argument to an aggregate function + ** is rewritten by aggregateConvertIndexedExprRefToColumn() */ + sqlite3VdbeAddOp3(v, OP_Column, pExpr->iTable, pExpr->iColumn, target); + return target; + } /* Otherwise, fall thru into the TK_COLUMN case */ + /* no break */ deliberate_fall_through } case TK_COLUMN: { int iTab = pExpr->iTable; + int iReg; + if( ExprHasProperty(pExpr, EP_FixedCol) ){ + /* This COLUMN expression is really a constant due to WHERE clause + ** constraints, and that constant is coded by the pExpr->pLeft + ** expression. However, make sure the constant has the correct + ** datatype by applying the Affinity of the table column to the + ** constant. + */ + int aff; + iReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft,target); + assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); + aff = sqlite3TableColumnAffinity(pExpr->y.pTab, pExpr->iColumn); + if( aff>SQLITE_AFF_BLOB ){ + static const char zAff[] = "B\000C\000D\000E\000F"; + assert( SQLITE_AFF_BLOB=='A' ); + assert( SQLITE_AFF_TEXT=='B' ); + sqlite3VdbeAddOp4(v, OP_Affinity, iReg, 1, 0, + &zAff[(aff-'B')*2], P4_STATIC); + } + return iReg; + } if( iTab<0 ){ - if( pParse->ckBase>0 ){ - /* Generating CHECK constraints or inserting into partial index */ - inReg = pExpr->iColumn + pParse->ckBase; - break; + if( pParse->iSelfTab<0 ){ + /* Other columns in the same row for CHECK constraints or + ** generated columns or for inserting into partial index. + ** The row is unpacked into registers beginning at + ** 0-(pParse->iSelfTab). The rowid (if any) is in a register + ** immediately prior to the first column. + */ + Column *pCol; + Table *pTab; + int iSrc; + int iCol = pExpr->iColumn; + assert( ExprUseYTab(pExpr) ); + pTab = pExpr->y.pTab; + assert( pTab!=0 ); + assert( iCol>=XN_ROWID ); + assert( iColnCol ); + if( iCol<0 ){ + return -1-pParse->iSelfTab; + } + pCol = pTab->aCol + iCol; + testcase( iCol!=sqlite3TableColumnToStorage(pTab,iCol) ); + iSrc = sqlite3TableColumnToStorage(pTab, iCol) - pParse->iSelfTab; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + if( pCol->colFlags & COLFLAG_BUSY ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", + pCol->zCnName); + return 0; + } + pCol->colFlags |= COLFLAG_BUSY; + if( pCol->colFlags & COLFLAG_NOTAVAIL ){ + sqlite3ExprCodeGeneratedColumn(pParse, pTab, pCol, iSrc); + } + pCol->colFlags &= ~(COLFLAG_BUSY|COLFLAG_NOTAVAIL); + return iSrc; + }else +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + if( pCol->affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp2(v, OP_SCopy, iSrc, target); + sqlite3VdbeAddOp1(v, OP_RealAffinity, target); + return target; + }else{ + return iSrc; + } }else{ /* Coding an expression that is part of an index where column names ** in the index refer to the table to which the index belongs */ - iTab = pParse->iSelfTab; + iTab = pParse->iSelfTab - 1; } } - inReg = sqlite3ExprCodeGetColumn(pParse, pExpr->pTab, + else if( pParse->pIdxPartExpr + && 0!=(r1 = exprPartidxExprLookup(pParse, pExpr, target)) + ){ + return r1; + } + assert( ExprUseYTab(pExpr) ); + assert( pExpr->y.pTab!=0 ); + iReg = sqlite3ExprCodeGetColumn(pParse, pExpr->y.pTab, pExpr->iColumn, iTab, target, pExpr->op2); - break; + return iReg; } case TK_INTEGER: { codeInteger(pParse, pExpr, 0, target); - break; + return target; + } + case TK_TRUEFALSE: { + sqlite3VdbeAddOp2(v, OP_Integer, sqlite3ExprTruthValue(pExpr), target); + return target; } #ifndef SQLITE_OMIT_FLOATING_POINT case TK_FLOAT: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pExpr->u.zToken, 0, target); - break; + return target; } #endif case TK_STRING: { assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeLoadString(v, target, pExpr->u.zToken); - break; + return target; } - case TK_NULL: { + default: { + /* Make NULL the default case so that if a bug causes an illegal + ** Expr node to be passed into this function, it will be handled + ** sanely and not crash. But keep the assert() to bring the problem + ** to the attention of the developers. */ + assert( op==TK_NULL || op==TK_ERROR || pParse->db->mallocFailed ); sqlite3VdbeAddOp2(v, OP_Null, 0, target); - break; + return target; } #ifndef SQLITE_OMIT_BLOB_LITERAL case TK_BLOB: { @@ -91720,7 +114311,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( z[n]=='\'' ); zBlob = sqlite3HexToBlob(sqlite3VdbeDb(v), z, n); sqlite3VdbeAddOp4(v, OP_Blob, n/2, target, 0, zBlob, P4_DYNAMIC); - break; + return target; } #endif case TK_VARIABLE: { @@ -91728,65 +114319,57 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( pExpr->u.zToken!=0 ); assert( pExpr->u.zToken[0]!=0 ); sqlite3VdbeAddOp2(v, OP_Variable, pExpr->iColumn, target); - if( pExpr->u.zToken[1]!=0 ){ - assert( pExpr->u.zToken[0]=='?' - || strcmp(pExpr->u.zToken, pParse->azVar[pExpr->iColumn-1])==0 ); - sqlite3VdbeChangeP4(v, -1, pParse->azVar[pExpr->iColumn-1], P4_STATIC); - } - break; + return target; } case TK_REGISTER: { - inReg = pExpr->iTable; - break; + return pExpr->iTable; } #ifndef SQLITE_OMIT_CAST case TK_CAST: { /* Expressions of the form: CAST(pLeft AS token) */ - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - if( inReg!=target ){ - sqlite3VdbeAddOp2(v, OP_SCopy, inReg, target); - inReg = target; - } + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( inReg==target ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); sqlite3VdbeAddOp2(v, OP_Cast, target, sqlite3AffinityType(pExpr->u.zToken, 0)); - testcase( usedAsColumnCache(pParse, inReg, inReg) ); - sqlite3ExprCacheAffinityChange(pParse, inReg, 1); - break; + return inReg; } #endif /* SQLITE_OMIT_CAST */ + case TK_IS: + case TK_ISNOT: + op = (op==TK_IS) ? TK_EQ : TK_NE; + p5 = SQLITE_NULLEQ; + /* fall-through */ case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2); - assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); - assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); - assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); - assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); - assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); - assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - break; - } - case TK_IS: - case TK_ISNOT: { - testcase( op==TK_IS ); - testcase( op==TK_ISNOT ); - r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); - op = (op==TK_IS) ? TK_EQ : TK_NE; - codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, inReg, SQLITE_STOREP2 | SQLITE_NULLEQ); - VdbeCoverageIf(v, op==TK_EQ); - VdbeCoverageIf(v, op==TK_NE); - testcase( regFree1==0 ); - testcase( regFree2==0 ); + Expr *pLeft = pExpr->pLeft; + if( sqlite3ExprIsVector(pLeft) ){ + codeVectorCompare(pParse, pExpr, target, op, p5); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); + r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); + sqlite3VdbeAddOp2(v, OP_Integer, 1, inReg); + codeCompare(pParse, pLeft, pExpr->pRight, op, r1, r2, + sqlite3VdbeCurrentAddr(v)+2, p5, + ExprHasProperty(pExpr,EP_Commuted)); + assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); + assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); + assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); + assert(TK_GE==OP_Ge); testcase(op==OP_Ge); VdbeCoverageIf(v,op==OP_Ge); + assert(TK_EQ==OP_Eq); testcase(op==OP_Eq); VdbeCoverageIf(v,op==OP_Eq); + assert(TK_NE==OP_Ne); testcase(op==OP_Ne); VdbeCoverageIf(v,op==OP_Ne); + if( p5==SQLITE_NULLEQ ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, inReg); + }else{ + sqlite3VdbeAddOp3(v, OP_ZeroOrNull, r1, inReg, r2); + } + testcase( regFree1==0 ); + testcase( regFree2==0 ); + } break; } case TK_AND: @@ -91799,7 +114382,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) case TK_BITOR: case TK_SLASH: case TK_LSHIFT: - case TK_RSHIFT: + case TK_RSHIFT: case TK_CONCAT: { assert( TK_AND==OP_And ); testcase( op==TK_AND ); assert( TK_OR==OP_Or ); testcase( op==TK_OR ); @@ -91824,21 +114407,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( pLeft ); if( pLeft->op==TK_INTEGER ){ codeInteger(pParse, pLeft, 1, target); + return target; #ifndef SQLITE_OMIT_FLOATING_POINT }else if( pLeft->op==TK_FLOAT ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); codeReal(v, pLeft->u.zToken, 1, target); + return target; #endif }else{ tempX.op = TK_INTEGER; tempX.flags = EP_IntValue|EP_TokenOnly; tempX.u.iValue = 0; + ExprClearVVAProperties(&tempX); r1 = sqlite3ExprCodeTemp(pParse, &tempX, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free2); sqlite3VdbeAddOp3(v, OP_Subtract, r2, r1, target); testcase( regFree2==0 ); } - inReg = target; break; } case TK_BITNOT: @@ -91847,10 +114432,21 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) assert( TK_NOT==OP_Not ); testcase( op==TK_NOT ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); testcase( regFree1==0 ); - inReg = target; sqlite3VdbeAddOp2(v, op, r1, inReg); break; } + case TK_TRUTH: { + int isTrue; /* IS TRUE or IS NOT TRUE */ + int bNormal; /* IS TRUE or IS FALSE */ + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + testcase( regFree1==0 ); + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + bNormal = pExpr->op2==TK_IS; + testcase( isTrue && bNormal); + testcase( !isTrue && bNormal); + sqlite3VdbeAddOp4Int(v, OP_IsTrue, r1, inReg, !isTrue, isTrue ^ bNormal); + break; + } case TK_ISNULL: case TK_NOTNULL: { int addr; @@ -91868,11 +114464,14 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } case TK_AGG_FUNCTION: { AggInfo *pInfo = pExpr->pAggInfo; - if( pInfo==0 ){ + if( pInfo==0 + || NEVER(pExpr->iAgg<0) + || NEVER(pExpr->iAgg>=pInfo->nFunc) + ){ assert( !ExprHasProperty(pExpr, EP_IntValue) ); - sqlite3ErrorMsg(pParse, "misuse of aggregate: %s()", pExpr->u.zToken); + sqlite3ErrorMsg(pParse, "misuse of aggregate: %#T()", pExpr); }else{ - inReg = pInfo->aFunc[pExpr->iAgg].iMem; + return AggInfoFuncReg(pInfo, pExpr->iAgg); } break; } @@ -91883,55 +114482,50 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) const char *zId; /* The function name */ u32 constMask = 0; /* Mask of function arguments that are constant */ int i; /* Loop counter */ + sqlite3 *db = pParse->db; /* The database connection */ u8 enc = ENC(db); /* The text encoding used by this database */ CollSeq *pColl = 0; /* A collating sequence */ - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - if( ExprHasProperty(pExpr, EP_TokenOnly) ){ - pFarg = 0; - }else{ - pFarg = pExpr->x.pList; +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + return pExpr->y.pWin->regResult; } +#endif + + if( ConstFactorOk(pParse) + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ + /* SQL functions can be expensive. So try to avoid running them + ** multiple times if we know they always give the same result */ + return sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); + } + assert( !ExprHasProperty(pExpr, EP_TokenOnly) ); + assert( ExprUseXList(pExpr) ); + pFarg = pExpr->x.pList; nFarg = pFarg ? pFarg->nExpr : 0; assert( !ExprHasProperty(pExpr, EP_IntValue) ); zId = pExpr->u.zToken; pDef = sqlite3FindFunction(db, zId, nFarg, enc, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 && pParse->explain ){ + pDef = sqlite3FindFunction(db, "unknown", nFarg, enc, 0); + } +#endif if( pDef==0 || pDef->xFinalize!=0 ){ - sqlite3ErrorMsg(pParse, "unknown function: %s()", zId); + sqlite3ErrorMsg(pParse, "unknown function: %#T()", pExpr); break; } - - /* Attempt a direct implementation of the built-in COALESCE() and - ** IFNULL() functions. This avoids unnecessary evaluation of - ** arguments past the first non-NULL argument. - */ - if( pDef->funcFlags & SQLITE_FUNC_COALESCE ){ - int endCoalesce = sqlite3VdbeMakeLabel(v); - assert( nFarg>=2 ); - sqlite3ExprCode(pParse, pFarg->a[0].pExpr, target); - for(i=1; ia[i].pExpr, target); - sqlite3ExprCachePop(pParse); - } - sqlite3VdbeResolveLabel(v, endCoalesce); - break; - } - - /* The UNLIKELY() function is a no-op. The result is the value - ** of the first argument. - */ - if( pDef->funcFlags & SQLITE_FUNC_UNLIKELY ){ - assert( nFarg>=1 ); - inReg = sqlite3ExprCodeTarget(pParse, pFarg->a[0].pExpr, target); - break; + if( (pDef->funcFlags & SQLITE_FUNC_INLINE)!=0 && ALWAYS(pFarg!=0) ){ + assert( (pDef->funcFlags & SQLITE_FUNC_UNSAFE)==0 ); + assert( (pDef->funcFlags & SQLITE_FUNC_DIRECT)==0 ); + return exprCodeInlineFunction(pParse, pFarg, + SQLITE_PTR_TO_INT(pDef->pUserData), target); + }else if( pDef->funcFlags & (SQLITE_FUNC_DIRECT|SQLITE_FUNC_UNSAFE) ){ + sqlite3ExprFunctionUsable(pParse, pExpr, pDef); } for(i=0; ia[i].pExpr) ){ + if( i<32 && sqlite3ExprIsConstant(pParse, pFarg->a[i].pExpr) ){ testcase( i==31 ); constMask |= MASKBIT32(i); } @@ -91947,10 +114541,10 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) r1 = sqlite3GetTempRange(pParse, nFarg); } - /* For length() and typeof() functions with a column argument, + /* For length() and typeof() and octet_length() functions, ** set the P5 parameter to the OP_Column opcode to OPFLAG_LENGTHARG - ** or OPFLAG_TYPEOFARG respectively, to avoid unnecessary data - ** loading. + ** or OPFLAG_TYPEOFARG or OPFLAG_BYTELENARG respectively, to avoid + ** unnecessary data loading. */ if( (pDef->funcFlags & (SQLITE_FUNC_LENGTH|SQLITE_FUNC_TYPEOF))!=0 ){ u8 exprOp; @@ -91960,16 +114554,16 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) if( exprOp==TK_COLUMN || exprOp==TK_AGG_COLUMN ){ assert( SQLITE_FUNC_LENGTH==OPFLAG_LENGTHARG ); assert( SQLITE_FUNC_TYPEOF==OPFLAG_TYPEOFARG ); - testcase( pDef->funcFlags & OPFLAG_LENGTHARG ); - pFarg->a[0].pExpr->op2 = - pDef->funcFlags & (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG); + assert( SQLITE_FUNC_BYTELEN==OPFLAG_BYTELENARG ); + assert( (OPFLAG_LENGTHARG|OPFLAG_TYPEOFARG)==OPFLAG_BYTELENARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_LENGTHARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_TYPEOFARG ); + testcase( (pDef->funcFlags & OPFLAG_BYTELENARG)==OPFLAG_BYTELENARG); + pFarg->a[0].pExpr->op2 = pDef->funcFlags & OPFLAG_BYTELENARG; } } - sqlite3ExprCachePush(pParse); /* Ticket 2ea2425d34be */ - sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, - SQLITE_ECEL_DUP|SQLITE_ECEL_FACTOR); - sqlite3ExprCachePop(pParse); /* Ticket 2ea2425d34be */ + sqlite3ExprCodeExprList(pParse, pFarg, r1, 0, SQLITE_ECEL_FACTOR); }else{ r1 = 0; } @@ -91982,46 +114576,74 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** see if it is a column in a virtual table. This is done because ** the left operand of infix functions (the operand we want to ** control overloading) ends up as the second argument to the - ** function. The expression "A glob B" is equivalent to + ** function. The expression "A glob B" is equivalent to ** "glob(B,A). We want to use the A in "A glob B" to test ** for function overloading. But we use the B term in "glob(B,A)". */ - if( nFarg>=2 && (pExpr->flags & EP_InfixFunc) ){ + if( nFarg>=2 && ExprHasProperty(pExpr, EP_InfixFunc) ){ pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[1].pExpr); }else if( nFarg>0 ){ pDef = sqlite3VtabOverloadFunction(db, pDef, nFarg, pFarg->a[0].pExpr); } #endif if( pDef->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - if( !pColl ) pColl = db->pDfltColl; + if( !pColl ) pColl = db->pDfltColl; sqlite3VdbeAddOp4(v, OP_CollSeq, 0, 0, 0, (char *)pColl, P4_COLLSEQ); } - sqlite3VdbeAddOp4(v, OP_Function0, constMask, r1, target, - (char*)pDef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nFarg); - if( nFarg && constMask==0 ){ - sqlite3ReleaseTempRange(pParse, r1, nFarg); + sqlite3VdbeAddFunctionCall(pParse, constMask, r1, target, nFarg, + pDef, pExpr->op2); + if( nFarg ){ + if( constMask==0 ){ + sqlite3ReleaseTempRange(pParse, r1, nFarg); + }else{ + sqlite3VdbeReleaseRegisters(pParse, r1, nFarg, constMask, 1); + } } - break; + return target; } #ifndef SQLITE_OMIT_SUBQUERY case TK_EXISTS: case TK_SELECT: { + int nCol; testcase( op==TK_EXISTS ); testcase( op==TK_SELECT ); - inReg = sqlite3CodeSubselect(pParse, pExpr, 0, 0); + if( pParse->db->mallocFailed ){ + return 0; + }else if( op==TK_SELECT + && ALWAYS( ExprUseXSelect(pExpr) ) + && (nCol = pExpr->x.pSelect->pEList->nExpr)!=1 + ){ + sqlite3SubselectError(pParse, nCol, 1); + }else{ + return sqlite3CodeSubselect(pParse, pExpr); + } break; } + case TK_SELECT_COLUMN: { + int n; + Expr *pLeft = pExpr->pLeft; + if( pLeft->iTable==0 || pParse->withinRJSubrtn > pLeft->op2 ){ + pLeft->iTable = sqlite3CodeSubselect(pParse, pLeft); + pLeft->op2 = pParse->withinRJSubrtn; + } + assert( pLeft->op==TK_SELECT || pLeft->op==TK_ERROR ); + n = sqlite3ExprVectorSize(pLeft); + if( pExpr->iTable!=n ){ + sqlite3ErrorMsg(pParse, "%d columns assigned %d values", + pExpr->iTable, n); + } + return pLeft->iTable + pExpr->iColumn; + } case TK_IN: { - int destIfFalse = sqlite3VdbeMakeLabel(v); - int destIfNull = sqlite3VdbeMakeLabel(v); + int destIfFalse = sqlite3VdbeMakeLabel(pParse); + int destIfNull = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp2(v, OP_Null, 0, target); sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); sqlite3VdbeAddOp2(v, OP_Integer, 1, target); sqlite3VdbeResolveLabel(v, destIfFalse); sqlite3VdbeAddOp2(v, OP_AddImm, target, 0); sqlite3VdbeResolveLabel(v, destIfNull); - break; + return target; } #endif /* SQLITE_OMIT_SUBQUERY */ @@ -92038,35 +114660,30 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** Z is stored in pExpr->pList->a[1].pExpr. */ case TK_BETWEEN: { - Expr *pLeft = pExpr->pLeft; - struct ExprList_item *pLItem = pExpr->x.pList->a; - Expr *pRight = pLItem->pExpr; - - r1 = sqlite3ExprCodeTemp(pParse, pLeft, ®Free1); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree1==0 ); - testcase( regFree2==0 ); - r3 = sqlite3GetTempReg(pParse); - r4 = sqlite3GetTempReg(pParse); - codeCompare(pParse, pLeft, pRight, OP_Ge, - r1, r2, r3, SQLITE_STOREP2); VdbeCoverage(v); - pLItem++; - pRight = pLItem->pExpr; - sqlite3ReleaseTempReg(pParse, regFree2); - r2 = sqlite3ExprCodeTemp(pParse, pRight, ®Free2); - testcase( regFree2==0 ); - codeCompare(pParse, pLeft, pRight, OP_Le, r1, r2, r4, SQLITE_STOREP2); - VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_And, r3, r4, target); - sqlite3ReleaseTempReg(pParse, r3); - sqlite3ReleaseTempReg(pParse, r4); - break; + exprCodeBetween(pParse, pExpr, target, 0, 0); + return target; + } + case TK_COLLATE: { + if( !ExprHasProperty(pExpr, EP_Collate) ){ + /* A TK_COLLATE Expr node without the EP_Collate tag is a so-called + ** "SOFT-COLLATE" that is added to constraints that are pushed down + ** from outer queries into sub-queries by the WHERE-clause push-down + ** optimization. Clear subtypes as subtypes may not cross a subquery + ** boundary. + */ + assert( pExpr->pLeft ); + sqlite3ExprCode(pParse, pExpr->pLeft, target); + sqlite3VdbeAddOp1(v, OP_ClrSubtype, target); + return target; + }else{ + pExpr = pExpr->pLeft; + goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. */ + } } case TK_SPAN: - case TK_COLLATE: case TK_UPLUS: { - inReg = sqlite3ExprCodeTarget(pParse, pExpr->pLeft, target); - break; + pExpr = pExpr->pLeft; + goto expr_code_doover; /* 2018-04-28: Prevent deep recursion. OSSFuzz. */ } case TK_TRIGGER: { @@ -92079,7 +114696,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** ** The expression is implemented using an OP_Param opcode. The p1 ** parameter is set to 0 for an old.rowid reference, or to (i+1) - ** to reference another column of the old.* pseudo-table, where + ** to reference another column of the old.* pseudo-table, where ** i is the index of the column. For a new.rowid reference, p1 is ** set to (n+1), where n is the number of columns in each pseudo-table. ** For a reference to any other column in the new.* pseudo-table, p1 @@ -92093,21 +114710,27 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** ** p1==0 -> old.rowid p1==3 -> new.rowid ** p1==1 -> old.a p1==4 -> new.a - ** p1==2 -> old.b p1==5 -> new.b + ** p1==2 -> old.b p1==5 -> new.b */ - Table *pTab = pExpr->pTab; - int p1 = pExpr->iTable * (pTab->nCol+1) + 1 + pExpr->iColumn; + Table *pTab; + int iCol; + int p1; + + assert( ExprUseYTab(pExpr) ); + pTab = pExpr->y.pTab; + iCol = pExpr->iColumn; + p1 = pExpr->iTable * (pTab->nCol+1) + 1 + + sqlite3TableColumnToStorage(pTab, iCol); assert( pExpr->iTable==0 || pExpr->iTable==1 ); - assert( pExpr->iColumn>=-1 && pExpr->iColumnnCol ); - assert( pTab->iPKey<0 || pExpr->iColumn!=pTab->iPKey ); + assert( iCol>=-1 && iColnCol ); + assert( pTab->iPKey<0 || iCol!=pTab->iPKey ); assert( p1>=0 && p1<(pTab->nCol*2+2) ); sqlite3VdbeAddOp2(v, OP_Param, p1, target); - VdbeComment((v, "%s.%s -> $%d", + VdbeComment((v, "r[%d]=%s.%s", target, (pExpr->iTable ? "new" : "old"), - (pExpr->iColumn<0 ? "rowid" : pExpr->pTab->aCol[pExpr->iColumn].zName), - target + (pExpr->iColumn<0 ? "rowid" : pExpr->y.pTab->aCol[iCol].zCnName) )); #ifndef SQLITE_OMIT_FLOATING_POINT @@ -92116,15 +114739,57 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** ** EVIDENCE-OF: R-60985-57662 SQLite will convert the value back to ** floating point when extracting it from the record. */ - if( pExpr->iColumn>=0 - && pTab->aCol[pExpr->iColumn].affinity==SQLITE_AFF_REAL - ){ + if( iCol>=0 && pTab->aCol[iCol].affinity==SQLITE_AFF_REAL ){ sqlite3VdbeAddOp1(v, OP_RealAffinity, target); } #endif break; } + case TK_VECTOR: { + sqlite3ErrorMsg(pParse, "row value misused"); + break; + } + + /* TK_IF_NULL_ROW Expr nodes are inserted ahead of expressions + ** that derive from the right-hand table of a LEFT JOIN. The + ** Expr.iTable value is the table number for the right-hand table. + ** The expression is only evaluated if that table is not currently + ** on a LEFT JOIN NULL row. + */ + case TK_IF_NULL_ROW: { + int addrINR; + u8 okConstFactor = pParse->okConstFactor; + AggInfo *pAggInfo = pExpr->pAggInfo; + if( pAggInfo ){ + assert( pExpr->iAgg>=0 && pExpr->iAggnColumn ); + if( !pAggInfo->directMode ){ + inReg = AggInfoColumnReg(pAggInfo, pExpr->iAgg); + break; + } + if( pExpr->pAggInfo->useSortingIdx ){ + sqlite3VdbeAddOp3(v, OP_Column, pAggInfo->sortingIdxPTab, + pAggInfo->aCol[pExpr->iAgg].iSorterColumn, + target); + inReg = target; + break; + } + } + addrINR = sqlite3VdbeAddOp3(v, OP_IfNullRow, pExpr->iTable, 0, target); + /* The OP_IfNullRow opcode above can overwrite the result register with + ** NULL. So we have to ensure that the result register is not a value + ** that is suppose to be a constant. Two defenses are needed: + ** (1) Temporarily disable factoring of constant expressions + ** (2) Make sure the computed value really is stored in register + ** "target" and not someplace else. + */ + pParse->okConstFactor = 0; /* note (1) above */ + sqlite3ExprCode(pParse, pExpr->pLeft, target); + assert( target==inReg ); + pParse->okConstFactor = okConstFactor; + sqlite3VdbeJumpHere(v, addrINR); + break; + } /* ** Form A: @@ -92147,7 +114812,7 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) ** or if there is no matching Ei, the ELSE term Y, or if there is ** no ELSE term, NULL. */ - default: assert( op==TK_CASE ); { + case TK_CASE: { int endLabel; /* GOTO label for end of CASE stmt */ int nextCase; /* GOTO label for next WHEN clause */ int nExpr; /* 2x number of WHEN terms */ @@ -92157,21 +114822,27 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) Expr opCompare; /* The X==Ei expression */ Expr *pX; /* The X expression */ Expr *pTest = 0; /* X==Ei (form A) or just Ei (form B) */ - VVA_ONLY( int iCacheLevel = pParse->iCacheLevel; ) + Expr *pDel = 0; + sqlite3 *db = pParse->db; - assert( !ExprHasProperty(pExpr, EP_xIsSelect) && pExpr->x.pList ); + assert( ExprUseXList(pExpr) && pExpr->x.pList!=0 ); assert(pExpr->x.pList->nExpr > 0); pEList = pExpr->x.pList; aListelem = pEList->a; nExpr = pEList->nExpr; - endLabel = sqlite3VdbeMakeLabel(v); + endLabel = sqlite3VdbeMakeLabel(pParse); if( (pX = pExpr->pLeft)!=0 ){ - tempX = *pX; + pDel = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDel); + break; + } testcase( pX->op==TK_COLUMN ); - exprToRegister(&tempX, sqlite3ExprCodeTemp(pParse, pX, ®Free1)); + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); testcase( regFree1==0 ); + memset(&opCompare, 0, sizeof(opCompare)); opCompare.op = TK_EQ; - opCompare.pLeft = &tempX; + opCompare.pLeft = pDel; pTest = &opCompare; /* Ticket b351d95f9cd5ef17e9d9dbae18f5ca8611190001: ** The value in regFree1 might get SCopy-ed into the file result. @@ -92180,59 +114851,55 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) regFree1 = 0; } for(i=0; iop==TK_COLUMN ); sqlite3ExprIfFalse(pParse, pTest, nextCase, SQLITE_JUMPIFNULL); testcase( aListelem[i+1].pExpr->op==TK_COLUMN ); sqlite3ExprCode(pParse, aListelem[i+1].pExpr, target); sqlite3VdbeGoto(v, endLabel); - sqlite3ExprCachePop(pParse); sqlite3VdbeResolveLabel(v, nextCase); } if( (nExpr&1)!=0 ){ - sqlite3ExprCachePush(pParse); sqlite3ExprCode(pParse, pEList->a[nExpr-1].pExpr, target); - sqlite3ExprCachePop(pParse); }else{ sqlite3VdbeAddOp2(v, OP_Null, 0, target); } - assert( db->mallocFailed || pParse->nErr>0 - || pParse->iCacheLevel==iCacheLevel ); + sqlite3ExprDelete(db, pDel); + setDoNotMergeFlagOnCopy(v); sqlite3VdbeResolveLabel(v, endLabel); break; } #ifndef SQLITE_OMIT_TRIGGER case TK_RAISE: { - assert( pExpr->affinity==OE_Rollback - || pExpr->affinity==OE_Abort - || pExpr->affinity==OE_Fail - || pExpr->affinity==OE_Ignore + assert( pExpr->affExpr==OE_Rollback + || pExpr->affExpr==OE_Abort + || pExpr->affExpr==OE_Fail + || pExpr->affExpr==OE_Ignore ); - if( !pParse->pTriggerTab ){ + if( !pParse->pTriggerTab && !pParse->nested ){ sqlite3ErrorMsg(pParse, "RAISE() may only be used within a trigger-program"); return 0; } - if( pExpr->affinity==OE_Abort ){ + if( pExpr->affExpr==OE_Abort ){ sqlite3MayAbort(pParse); } assert( !ExprHasProperty(pExpr, EP_IntValue) ); - if( pExpr->affinity==OE_Ignore ){ - sqlite3VdbeAddOp4( - v, OP_Halt, SQLITE_OK, OE_Ignore, 0, pExpr->u.zToken,0); + if( pExpr->affExpr==OE_Ignore ){ + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, OE_Ignore); VdbeCoverage(v); }else{ - sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_TRIGGER, - pExpr->affinity, pExpr->u.zToken, 0, 0); + r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeAddOp3(v, OP_Halt, + pParse->pTriggerTab ? SQLITE_CONSTRAINT_TRIGGER : SQLITE_ERROR, + pExpr->affExpr, r1); } - break; } #endif @@ -92243,25 +114910,67 @@ SQLITE_PRIVATE int sqlite3ExprCodeTarget(Parse *pParse, Expr *pExpr, int target) } /* -** Factor out the code of the given expression to initialization time. +** Generate code that will evaluate expression pExpr just one time +** per prepared statement execution. +** +** If the expression uses functions (that might throw an exception) then +** guard them with an OP_Once opcode to ensure that the code is only executed +** once. If no functions are involved, then factor the code out and put it at +** the end of the prepared statement in the initialization section. +** +** If regDest>0 then the result is always stored in that register and the +** result is not reusable. If regDest<0 then this routine is free to +** store the value wherever it wants. The register where the expression +** is stored is returned. When regDest<0, two identical expressions might +** code to the same register, if they do not contain function calls and hence +** are factored out into the initialization section at the end of the +** prepared statement. */ -SQLITE_PRIVATE void sqlite3ExprCodeAtInit( +SQLITE_PRIVATE int sqlite3ExprCodeRunJustOnce( Parse *pParse, /* Parsing context */ Expr *pExpr, /* The expression to code when the VDBE initializes */ - int regDest, /* Store the value in this register */ - u8 reusable /* True if this expression is reusable */ + int regDest /* Store the value in this register */ ){ ExprList *p; assert( ConstFactorOk(pParse) ); + assert( regDest!=0 ); p = pParse->pConstExpr; - pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); - p = sqlite3ExprListAppend(pParse, p, pExpr); - if( p ){ - struct ExprList_item *pItem = &p->a[p->nExpr-1]; - pItem->u.iConstExprReg = regDest; - pItem->reusable = reusable; + if( regDest<0 && p ){ + struct ExprList_item *pItem; + int i; + for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ + if( pItem->fg.reusable + && sqlite3ExprCompare(0,pItem->pExpr,pExpr,-1)==0 + ){ + return pItem->u.iConstExprReg; + } + } } - pParse->pConstExpr = p; + pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pExpr!=0 && ExprHasProperty(pExpr, EP_HasFunc) ){ + Vdbe *v = pParse->pVdbe; + int addr; + assert( v ); + addr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + pParse->okConstFactor = 0; + if( !pParse->db->mallocFailed ){ + if( regDest<0 ) regDest = ++pParse->nMem; + sqlite3ExprCode(pParse, pExpr, regDest); + } + pParse->okConstFactor = 1; + sqlite3ExprDelete(pParse->db, pExpr); + sqlite3VdbeJumpHere(v, addr); + }else{ + p = sqlite3ExprListAppend(pParse, p, pExpr); + if( p ){ + struct ExprList_item *pItem = &p->a[p->nExpr-1]; + pItem->fg.reusable = regDest<0; + if( regDest<0 ) regDest = ++pParse->nMem; + pItem->u.iConstExprReg = regDest; + } + pParse->pConstExpr = p; + } + return regDest; } /* @@ -92279,24 +114988,14 @@ SQLITE_PRIVATE void sqlite3ExprCodeAtInit( */ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ int r2; - pExpr = sqlite3ExprSkipCollate(pExpr); + pExpr = sqlite3ExprSkipCollateAndLikely(pExpr); if( ConstFactorOk(pParse) + && ALWAYS(pExpr!=0) && pExpr->op!=TK_REGISTER - && sqlite3ExprIsConstantNotJoin(pExpr) + && sqlite3ExprIsConstantNotJoin(pParse, pExpr) ){ - ExprList *p = pParse->pConstExpr; - int i; *pReg = 0; - if( p ){ - struct ExprList_item *pItem; - for(pItem=p->a, i=p->nExpr; i>0; pItem++, i--){ - if( pItem->reusable && sqlite3ExprCompare(pItem->pExpr,pExpr,-1)==0 ){ - return pItem->u.iConstExprReg; - } - } - } - r2 = ++pParse->nMem; - sqlite3ExprCodeAtInit(pParse, pExpr, r2, 1); + r2 = sqlite3ExprCodeRunJustOnce(pParse, pExpr, -1); }else{ int r1 = sqlite3GetTempReg(pParse); r2 = sqlite3ExprCodeTarget(pParse, pExpr, r1); @@ -92318,15 +115017,23 @@ SQLITE_PRIVATE int sqlite3ExprCodeTemp(Parse *pParse, Expr *pExpr, int *pReg){ SQLITE_PRIVATE void sqlite3ExprCode(Parse *pParse, Expr *pExpr, int target){ int inReg; + assert( pExpr==0 || !ExprHasVVAProperty(pExpr,EP_Immutable) ); assert( target>0 && target<=pParse->nMem ); - if( pExpr && pExpr->op==TK_REGISTER ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_Copy, pExpr->iTable, target); - }else{ - inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); - assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); - if( inReg!=target && pParse->pVdbe ){ - sqlite3VdbeAddOp2(pParse->pVdbe, OP_SCopy, inReg, target); + assert( pParse->pVdbe!=0 || pParse->db->mallocFailed ); + if( pParse->pVdbe==0 ) return; + inReg = sqlite3ExprCodeTarget(pParse, pExpr, target); + if( inReg!=target ){ + u8 op; + Expr *pX = sqlite3ExprSkipCollateAndLikely(pExpr); + testcase( pX!=pExpr ); + if( ALWAYS(pX) + && (ExprHasProperty(pX,EP_Subquery) || pX->op==TK_REGISTER) + ){ + op = OP_Copy; + }else{ + op = OP_SCopy; } + sqlite3VdbeAddOp2(pParse->pVdbe, op, inReg, target); } } @@ -92349,42 +115056,20 @@ SQLITE_PRIVATE void sqlite3ExprCodeCopy(Parse *pParse, Expr *pExpr, int target){ ** might choose to code the expression at initialization time. */ SQLITE_PRIVATE void sqlite3ExprCodeFactorable(Parse *pParse, Expr *pExpr, int target){ - if( pParse->okConstFactor && sqlite3ExprIsConstant(pExpr) ){ - sqlite3ExprCodeAtInit(pParse, pExpr, target, 0); + if( pParse->okConstFactor && sqlite3ExprIsConstantNotJoin(pParse,pExpr) ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target); }else{ - sqlite3ExprCode(pParse, pExpr, target); + sqlite3ExprCodeCopy(pParse, pExpr, target); } } -/* -** Generate code that evaluates the given expression and puts the result -** in register target. -** -** Also make a copy of the expression results into another "cache" register -** and modify the expression so that the next time it is evaluated, -** the result is a copy of the cache register. -** -** This routine is used for expressions that are used multiple -** times. They are evaluated once and the results of the expression -** are reused. -*/ -SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int target){ - Vdbe *v = pParse->pVdbe; - int iMem; - - assert( target>0 ); - assert( pExpr->op!=TK_REGISTER ); - sqlite3ExprCode(pParse, pExpr, target); - iMem = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_Copy, target, iMem); - exprToRegister(pExpr, iMem); -} - /* ** Generate code that pushes the value of every element of the given ** expression list into a sequence of registers beginning at target. ** -** Return the number of elements evaluated. +** Return the number of elements evaluated. The number returned will +** usually be pList->nExpr but might be reduced if SQLITE_ECEL_OMITREF +** is defined. ** ** The SQLITE_ECEL_DUP flag prevents the arguments from being ** filled using OP_SCopy. OP_Copy must be used instead. @@ -92395,6 +115080,8 @@ SQLITE_PRIVATE void sqlite3ExprCodeAndCache(Parse *pParse, Expr *pExpr, int targ ** The SQLITE_ECEL_REF flag means that expressions in the list with ** ExprList.a[].u.x.iOrderByCol>0 have already been evaluated and stored ** in registers at srcReg, and so the value can be copied from there. +** If SQLITE_ECEL_OMITREF is also set, then the values with u.x.iOrderByCol>0 +** are simply omitted rather than being copied from srcReg. */ SQLITE_PRIVATE int sqlite3ExprCodeExprList( Parse *pParse, /* Parsing context */ @@ -92414,18 +115101,32 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( if( !ConstFactorOk(pParse) ) flags &= ~SQLITE_ECEL_FACTOR; for(pItem=pList->a, i=0; ipExpr; - if( (flags & SQLITE_ECEL_REF)!=0 && (j = pList->a[i].u.x.iOrderByCol)>0 ){ - sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); - }else if( (flags & SQLITE_ECEL_FACTOR)!=0 && sqlite3ExprIsConstant(pExpr) ){ - sqlite3ExprCodeAtInit(pParse, pExpr, target+i, 0); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pItem->fg.bSorterRef ){ + i--; + n--; + }else +#endif + if( (flags & SQLITE_ECEL_REF)!=0 && (j = pItem->u.x.iOrderByCol)>0 ){ + if( flags & SQLITE_ECEL_OMITREF ){ + i--; + n--; + }else{ + sqlite3VdbeAddOp2(v, copyOp, j+srcReg-1, target+i); + } + }else if( (flags & SQLITE_ECEL_FACTOR)!=0 + && sqlite3ExprIsConstantNotJoin(pParse,pExpr) + ){ + sqlite3ExprCodeRunJustOnce(pParse, pExpr, target+i); }else{ int inReg = sqlite3ExprCodeTarget(pParse, pExpr, target+i); if( inReg!=target+i ){ VdbeOp *pOp; if( copyOp==OP_Copy - && (pOp=sqlite3VdbeGetOp(v, -1))->opcode==OP_Copy + && (pOp=sqlite3VdbeGetLastOp(v))->opcode==OP_Copy && pOp->p1+pOp->p3+1==inReg && pOp->p2+pOp->p3+1==target+i + && pOp->p5==0 /* The do-not-merge flag must be clear */ ){ pOp->p3++; }else{ @@ -92442,54 +115143,77 @@ SQLITE_PRIVATE int sqlite3ExprCodeExprList( ** ** x BETWEEN y AND z ** -** The above is equivalent to +** The above is equivalent to ** ** x>=y AND x<=z ** ** Code it as such, taking care to do the common subexpression ** elimination of x. +** +** The xJumpIf parameter determines details: +** +** NULL: Store the boolean result in reg[dest] +** sqlite3ExprIfTrue: Jump to dest if true +** sqlite3ExprIfFalse: Jump to dest if false +** +** The jumpIfNull parameter is ignored if xJumpIf is NULL. */ static void exprCodeBetween( Parse *pParse, /* Parsing and code generating context */ Expr *pExpr, /* The BETWEEN expression */ - int dest, /* Jump here if the jump is taken */ - int jumpIfTrue, /* Take the jump if the BETWEEN is true */ + int dest, /* Jump destination or storage location */ + void (*xJump)(Parse*,Expr*,int,int), /* Action to take */ int jumpIfNull /* Take the jump if the BETWEEN is NULL */ ){ Expr exprAnd; /* The AND operator in x>=y AND x<=z */ Expr compLeft; /* The x>=y term */ Expr compRight; /* The x<=z term */ - Expr exprX; /* The x subexpression */ int regFree1 = 0; /* Temporary use register */ + Expr *pDel = 0; + sqlite3 *db = pParse->db; - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - exprX = *pExpr->pLeft; - exprAnd.op = TK_AND; - exprAnd.pLeft = &compLeft; - exprAnd.pRight = &compRight; - compLeft.op = TK_GE; - compLeft.pLeft = &exprX; - compLeft.pRight = pExpr->x.pList->a[0].pExpr; - compRight.op = TK_LE; - compRight.pLeft = &exprX; - compRight.pRight = pExpr->x.pList->a[1].pExpr; - exprToRegister(&exprX, sqlite3ExprCodeTemp(pParse, &exprX, ®Free1)); - if( jumpIfTrue ){ - sqlite3ExprIfTrue(pParse, &exprAnd, dest, jumpIfNull); - }else{ - sqlite3ExprIfFalse(pParse, &exprAnd, dest, jumpIfNull); + memset(&compLeft, 0, sizeof(Expr)); + memset(&compRight, 0, sizeof(Expr)); + memset(&exprAnd, 0, sizeof(Expr)); + + assert( ExprUseXList(pExpr) ); + pDel = sqlite3ExprDup(db, pExpr->pLeft, 0); + if( db->mallocFailed==0 ){ + exprAnd.op = TK_AND; + exprAnd.pLeft = &compLeft; + exprAnd.pRight = &compRight; + compLeft.op = TK_GE; + compLeft.pLeft = pDel; + compLeft.pRight = pExpr->x.pList->a[0].pExpr; + compRight.op = TK_LE; + compRight.pLeft = pDel; + compRight.pRight = pExpr->x.pList->a[1].pExpr; + sqlite3ExprToRegister(pDel, exprCodeVector(pParse, pDel, ®Free1)); + if( xJump ){ + xJump(pParse, &exprAnd, dest, jumpIfNull); + }else{ + /* Mark the expression is being from the ON or USING clause of a join + ** so that the sqlite3ExprCodeTarget() routine will not attempt to move + ** it into the Parse.pConstExpr list. We should use a new bit for this, + ** for clarity, but we are out of bits in the Expr.flags field so we + ** have to reuse the EP_OuterON bit. Bummer. */ + pDel->flags |= EP_OuterON; + sqlite3ExprCodeTarget(pParse, &exprAnd, dest); + } + sqlite3ReleaseTempReg(pParse, regFree1); } - sqlite3ReleaseTempReg(pParse, regFree1); + sqlite3ExprDelete(db, pDel); /* Ensure adequate test coverage */ - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue==0 && jumpIfNull!=0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull==0 && regFree1!=0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1==0 ); - testcase( jumpIfTrue!=0 && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfTrue && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull==0 && regFree1!=0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1==0 ); + testcase( xJump==sqlite3ExprIfFalse && jumpIfNull!=0 && regFree1!=0 ); + testcase( xJump==0 ); } /* @@ -92516,24 +115240,26 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( NEVER(pExpr==0) ) return; /* No way this can happen */ + assert( !ExprHasVVAProperty(pExpr, EP_Immutable) ); op = pExpr->op; switch( op ){ - case TK_AND: { - int d2 = sqlite3VdbeMakeLabel(v); - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2,jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse); - break; - } + case TK_AND: case TK_OR: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse); + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfTrue(pParse, pAlt, dest, jumpIfNull); + }else if( op==TK_AND ){ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + }else{ + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfTrue(pParse, pExpr->pRight, dest, jumpIfNull); + } break; } case TK_NOT: { @@ -92541,24 +115267,42 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + }else{ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? SQLITE_JUMPIFNULL : 0); + } + break; + } case TK_IS: case TK_ISNOT: testcase( op==TK_IS ); testcase( op==TK_ISNOT ); op = (op==TK_IS) ? TK_EQ : TK_NE; jumpIfNull = SQLITE_NULLEQ; - /* Fall thru */ + /* no break */ deliberate_fall_through case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, jumpIfNull); + r1, r2, dest, jumpIfNull, ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); @@ -92578,6 +115322,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int assert( TK_ISNULL==OP_IsNull ); testcase( op==TK_ISNULL ); assert( TK_NOTNULL==OP_NotNull ); testcase( op==TK_NOTNULL ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); VdbeCoverageIf(v, op==TK_ISNULL); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -92586,12 +115331,12 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 1, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfTrue, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY case TK_IN: { - int destIfFalse = sqlite3VdbeMakeLabel(v); + int destIfFalse = sqlite3VdbeMakeLabel(pParse); int destIfNull = jumpIfNull ? dest : destIfFalse; sqlite3ExprCodeIN(pParse, pExpr, destIfFalse, destIfNull); sqlite3VdbeGoto(v, dest); @@ -92600,9 +115345,10 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } #endif default: { - if( exprAlwaysTrue(pExpr) ){ + default_expr: + if( ExprAlwaysTrue(pExpr) ){ sqlite3VdbeGoto(v, dest); - }else if( exprAlwaysFalse(pExpr) ){ + }else if( ExprAlwaysFalse(pExpr) ){ /* No-op */ }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); @@ -92615,7 +115361,7 @@ SQLITE_PRIVATE void sqlite3ExprIfTrue(Parse *pParse, Expr *pExpr, int dest, int } } sqlite3ReleaseTempReg(pParse, regFree1); - sqlite3ReleaseTempReg(pParse, regFree2); + sqlite3ReleaseTempReg(pParse, regFree2); } /* @@ -92637,6 +115383,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert( jumpIfNull==SQLITE_JUMPIFNULL || jumpIfNull==0 ); if( NEVER(v==0) ) return; /* Existence of VDBE checked by caller */ if( pExpr==0 ) return; + assert( !ExprHasVVAProperty(pExpr,EP_Immutable) ); /* The value of pExpr->op and op are related as follows: ** @@ -92670,22 +115417,23 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int assert( pExpr->op!=TK_GE || op==OP_Lt ); switch( pExpr->op ){ - case TK_AND: { - testcase( jumpIfNull==0 ); - sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3ExprCachePop(pParse); - break; - } + case TK_AND: case TK_OR: { - int d2 = sqlite3VdbeMakeLabel(v); - testcase( jumpIfNull==0 ); - sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, jumpIfNull^SQLITE_JUMPIFNULL); - sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); - sqlite3VdbeResolveLabel(v, d2); - sqlite3ExprCachePop(pParse); + Expr *pAlt = sqlite3ExprSimplifiedAndOr(pExpr); + if( pAlt!=pExpr ){ + sqlite3ExprIfFalse(pParse, pAlt, dest, jumpIfNull); + }else if( pExpr->op==TK_AND ){ + testcase( jumpIfNull==0 ); + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, jumpIfNull); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + }else{ + int d2 = sqlite3VdbeMakeLabel(pParse); + testcase( jumpIfNull==0 ); + sqlite3ExprIfTrue(pParse, pExpr->pLeft, d2, + jumpIfNull^SQLITE_JUMPIFNULL); + sqlite3ExprIfFalse(pParse, pExpr->pRight, dest, jumpIfNull); + sqlite3VdbeResolveLabel(v, d2); + } break; } case TK_NOT: { @@ -92693,24 +115441,45 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, jumpIfNull); break; } + case TK_TRUTH: { + int isNot; /* IS NOT TRUE or IS NOT FALSE */ + int isTrue; /* IS TRUE or IS NOT TRUE */ + testcase( jumpIfNull==0 ); + isNot = pExpr->op2==TK_ISNOT; + isTrue = sqlite3ExprTruthValue(pExpr->pRight); + testcase( isTrue && isNot ); + testcase( !isTrue && isNot ); + if( isTrue ^ isNot ){ + /* IS TRUE and IS NOT FALSE */ + sqlite3ExprIfFalse(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + + }else{ + /* IS FALSE and IS NOT TRUE */ + sqlite3ExprIfTrue(pParse, pExpr->pLeft, dest, + isNot ? 0 : SQLITE_JUMPIFNULL); + } + break; + } case TK_IS: case TK_ISNOT: testcase( pExpr->op==TK_IS ); testcase( pExpr->op==TK_ISNOT ); op = (pExpr->op==TK_IS) ? TK_NE : TK_EQ; jumpIfNull = SQLITE_NULLEQ; - /* Fall thru */ + /* no break */ deliberate_fall_through case TK_LT: case TK_LE: case TK_GT: case TK_GE: case TK_NE: case TK_EQ: { + if( sqlite3ExprIsVector(pExpr->pLeft) ) goto default_expr; testcase( jumpIfNull==0 ); r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); r2 = sqlite3ExprCodeTemp(pParse, pExpr->pRight, ®Free2); codeCompare(pParse, pExpr->pLeft, pExpr->pRight, op, - r1, r2, dest, jumpIfNull); + r1, r2, dest, jumpIfNull,ExprHasProperty(pExpr,EP_Commuted)); assert(TK_LT==OP_Lt); testcase(op==OP_Lt); VdbeCoverageIf(v,op==OP_Lt); assert(TK_LE==OP_Le); testcase(op==OP_Le); VdbeCoverageIf(v,op==OP_Le); assert(TK_GT==OP_Gt); testcase(op==OP_Gt); VdbeCoverageIf(v,op==OP_Gt); @@ -92728,6 +115497,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int case TK_ISNULL: case TK_NOTNULL: { r1 = sqlite3ExprCodeTemp(pParse, pExpr->pLeft, ®Free1); + sqlite3VdbeTypeofColumn(v, r1); sqlite3VdbeAddOp2(v, op, r1, dest); testcase( op==TK_ISNULL ); VdbeCoverageIf(v, op==TK_ISNULL); testcase( op==TK_NOTNULL ); VdbeCoverageIf(v, op==TK_NOTNULL); @@ -92736,7 +115506,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } case TK_BETWEEN: { testcase( jumpIfNull==0 ); - exprCodeBetween(pParse, pExpr, dest, 0, jumpIfNull); + exprCodeBetween(pParse, pExpr, dest, sqlite3ExprIfFalse, jumpIfNull); break; } #ifndef SQLITE_OMIT_SUBQUERY @@ -92744,7 +115514,7 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int if( jumpIfNull ){ sqlite3ExprCodeIN(pParse, pExpr, dest, dest); }else{ - int destIfNull = sqlite3VdbeMakeLabel(v); + int destIfNull = sqlite3VdbeMakeLabel(pParse); sqlite3ExprCodeIN(pParse, pExpr, dest, destIfNull); sqlite3VdbeResolveLabel(v, destIfNull); } @@ -92752,9 +115522,10 @@ SQLITE_PRIVATE void sqlite3ExprIfFalse(Parse *pParse, Expr *pExpr, int dest, int } #endif default: { - if( exprAlwaysFalse(pExpr) ){ + default_expr: + if( ExprAlwaysFalse(pExpr) ){ sqlite3VdbeGoto(v, dest); - }else if( exprAlwaysTrue(pExpr) ){ + }else if( ExprAlwaysTrue(pExpr) ){ /* no-op */ }else{ r1 = sqlite3ExprCodeTemp(pParse, pExpr, ®Free1); @@ -92784,6 +115555,45 @@ SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,i sqlite3ExprDelete(db, pCopy); } +/* +** Expression pVar is guaranteed to be an SQL variable. pExpr may be any +** type of expression. +** +** If pExpr is a simple SQL value - an integer, real, string, blob +** or NULL value - then the VDBE currently being prepared is configured +** to re-prepare each time a new value is bound to variable pVar. +** +** Additionally, if pExpr is a simple SQL value and the value is the +** same as that currently bound to variable pVar, non-zero is returned. +** Otherwise, if the values are not the same or if pExpr is not a simple +** SQL value, zero is returned. +*/ +static int exprCompareVariable( + const Parse *pParse, + const Expr *pVar, + const Expr *pExpr +){ + int res = 0; + int iVar; + sqlite3_value *pL, *pR = 0; + + sqlite3ValueFromExpr(pParse->db, pExpr, SQLITE_UTF8, SQLITE_AFF_BLOB, &pR); + if( pR ){ + iVar = pVar->iColumn; + sqlite3VdbeSetVarmask(pParse->pVdbe, iVar); + pL = sqlite3VdbeGetBoundValue(pParse->pReprepare, iVar, SQLITE_AFF_BLOB); + if( pL ){ + if( sqlite3_value_type(pL)==SQLITE_TEXT ){ + sqlite3_value_text(pL); /* Make sure the encoding is UTF-8 */ + } + res = 0==sqlite3MemCompare(pL, pR, 0); + } + sqlite3ValueFree(pR); + sqlite3ValueFree(pL); + } + + return res; +} /* ** Do a deep comparison of two expression trees. Return 0 if the two @@ -92806,12 +115616,27 @@ SQLITE_PRIVATE void sqlite3ExprIfFalseDup(Parse *pParse, Expr *pExpr, int dest,i ** this routine is used, it does not hurt to get an extra 2 - that ** just might result in some slightly slower code. But returning ** an incorrect 0 or 1 could lead to a malfunction. +** +** If pParse is not NULL then TK_VARIABLE terms in pA with bindings in +** pParse->pReprepare can be matched against literals in pB. The +** pParse->pVdbe->expmask bitmask is updated for each variable referenced. +** If pParse is NULL (the normal case) then any TK_VARIABLE term in +** Argument pParse should normally be NULL. If it is not NULL and pA or +** pB causes a return value of 2. */ -SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ +SQLITE_PRIVATE int sqlite3ExprCompare( + const Parse *pParse, + const Expr *pA, + const Expr *pB, + int iTab +){ u32 combinedFlags; if( pA==0 || pB==0 ){ return pB==pA ? 0 : 2; } + if( pParse && pA->op==TK_VARIABLE && exprCompareVariable(pParse, pA, pB) ){ + return 0; + } combinedFlags = pA->flags | pB->flags; if( combinedFlags & EP_IntValue ){ if( (pA->flags&pB->flags&EP_IntValue)!=0 && pA->u.iValue==pB->u.iValue ){ @@ -92819,40 +115644,76 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ } return 2; } - if( pA->op!=pB->op ){ - if( pA->op==TK_COLLATE && sqlite3ExprCompare(pA->pLeft, pB, iTab)<2 ){ + if( pA->op!=pB->op || pA->op==TK_RAISE ){ + if( pA->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA->pLeft,pB,iTab)<2 ){ return 1; } - if( pB->op==TK_COLLATE && sqlite3ExprCompare(pA, pB->pLeft, iTab)<2 ){ + if( pB->op==TK_COLLATE && sqlite3ExprCompare(pParse, pA,pB->pLeft,iTab)<2 ){ return 1; } - return 2; + if( pA->op==TK_AGG_COLUMN && pB->op==TK_COLUMN + && pB->iTable<0 && pA->iTable==iTab + ){ + /* fall through */ + }else{ + return 2; + } } - if( pA->op!=TK_COLUMN && pA->op!=TK_AGG_COLUMN && pA->u.zToken ){ - if( pA->op==TK_FUNCTION ){ + assert( !ExprHasProperty(pA, EP_IntValue) ); + assert( !ExprHasProperty(pB, EP_IntValue) ); + if( pA->u.zToken ){ + if( pA->op==TK_FUNCTION || pA->op==TK_AGG_FUNCTION ){ if( sqlite3StrICmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; - }else if( strcmp(pA->u.zToken,pB->u.zToken)!=0 ){ - return pA->op==TK_COLLATE ? 1 : 2; +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( pA->op==pB->op ); + if( ExprHasProperty(pA,EP_WinFunc)!=ExprHasProperty(pB,EP_WinFunc) ){ + return 2; + } + if( ExprHasProperty(pA,EP_WinFunc) ){ + if( sqlite3WindowCompare(pParse, pA->y.pWin, pB->y.pWin, 1)!=0 ){ + return 2; + } + } +#endif + }else if( pA->op==TK_NULL ){ + return 0; + }else if( pA->op==TK_COLLATE ){ + if( sqlite3_stricmp(pA->u.zToken,pB->u.zToken)!=0 ) return 2; + }else + if( pB->u.zToken!=0 + && pA->op!=TK_COLUMN + && pA->op!=TK_AGG_COLUMN + && strcmp(pA->u.zToken,pB->u.zToken)!=0 + ){ + return 2; } } - if( (pA->flags & EP_Distinct)!=(pB->flags & EP_Distinct) ) return 2; + if( (pA->flags & (EP_Distinct|EP_Commuted)) + != (pB->flags & (EP_Distinct|EP_Commuted)) ) return 2; if( ALWAYS((combinedFlags & EP_TokenOnly)==0) ){ if( combinedFlags & EP_xIsSelect ) return 2; - if( sqlite3ExprCompare(pA->pLeft, pB->pLeft, iTab) ) return 2; - if( sqlite3ExprCompare(pA->pRight, pB->pRight, iTab) ) return 2; + if( (combinedFlags & EP_FixedCol)==0 + && sqlite3ExprCompare(pParse, pA->pLeft, pB->pLeft, iTab) ) return 2; + if( sqlite3ExprCompare(pParse, pA->pRight, pB->pRight, iTab) ) return 2; if( sqlite3ExprListCompare(pA->x.pList, pB->x.pList, iTab) ) return 2; - if( ALWAYS((combinedFlags & EP_Reduced)==0) && pA->op!=TK_STRING ){ + if( pA->op!=TK_STRING + && pA->op!=TK_TRUEFALSE + && ALWAYS((combinedFlags & EP_Reduced)==0) + ){ if( pA->iColumn!=pB->iColumn ) return 2; - if( pA->iTable!=pB->iTable - && (pA->iTable!=iTab || NEVER(pB->iTable>=0)) ) return 2; + if( pA->op2!=pB->op2 && pA->op==TK_TRUTH ) return 2; + if( pA->op!=TK_IN && pA->iTable!=pB->iTable && pA->iTable!=iTab ){ + return 2; + } } } return 0; } /* -** Compare two ExprList objects. Return 0 if they are identical and -** non-zero if they differ in any way. +** Compare two ExprList objects. Return 0 if they are identical, 1 +** if they are certainly different, or 2 if it is not possible to +** determine if they are identical or not. ** ** If any subelement of pB has Expr.iTable==(-1) then it is allowed ** to compare equal to an equivalent element in pA with Expr.iTable==iTab. @@ -92865,16 +115726,106 @@ SQLITE_PRIVATE int sqlite3ExprCompare(Expr *pA, Expr *pB, int iTab){ ** Two NULL pointers are considered to be the same. But a NULL pointer ** always differs from a non-NULL pointer. */ -SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ +SQLITE_PRIVATE int sqlite3ExprListCompare(const ExprList *pA, const ExprList *pB, int iTab){ int i; if( pA==0 && pB==0 ) return 0; if( pA==0 || pB==0 ) return 1; if( pA->nExpr!=pB->nExpr ) return 1; for(i=0; inExpr; i++){ + int res; Expr *pExprA = pA->a[i].pExpr; Expr *pExprB = pB->a[i].pExpr; - if( pA->a[i].sortOrder!=pB->a[i].sortOrder ) return 1; - if( sqlite3ExprCompare(pExprA, pExprB, iTab) ) return 1; + if( pA->a[i].fg.sortFlags!=pB->a[i].fg.sortFlags ) return 1; + if( (res = sqlite3ExprCompare(0, pExprA, pExprB, iTab)) ) return res; + } + return 0; +} + +/* +** Like sqlite3ExprCompare() except COLLATE operators at the top-level +** are ignored. +*/ +SQLITE_PRIVATE int sqlite3ExprCompareSkip(Expr *pA,Expr *pB, int iTab){ + return sqlite3ExprCompare(0, + sqlite3ExprSkipCollate(pA), + sqlite3ExprSkipCollate(pB), + iTab); +} + +/* +** Return non-zero if Expr p can only be true if pNN is not NULL. +** +** Or if seenNot is true, return non-zero if Expr p can only be +** non-NULL if pNN is not NULL +*/ +static int exprImpliesNotNull( + const Parse *pParse,/* Parsing context */ + const Expr *p, /* The expression to be checked */ + const Expr *pNN, /* The expression that is NOT NULL */ + int iTab, /* Table being evaluated */ + int seenNot /* Return true only if p can be any non-NULL value */ +){ + assert( p ); + assert( pNN ); + if( sqlite3ExprCompare(pParse, p, pNN, iTab)==0 ){ + return pNN->op!=TK_NULL; + } + switch( p->op ){ + case TK_IN: { + if( seenNot && ExprHasProperty(p, EP_xIsSelect) ) return 0; + assert( ExprUseXSelect(p) || (p->x.pList!=0 && p->x.pList->nExpr>0) ); + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BETWEEN: { + ExprList *pList; + assert( ExprUseXList(p) ); + pList = p->x.pList; + assert( pList!=0 ); + assert( pList->nExpr==2 ); + if( seenNot ) return 0; + if( exprImpliesNotNull(pParse, pList->a[0].pExpr, pNN, iTab, 1) + || exprImpliesNotNull(pParse, pList->a[1].pExpr, pNN, iTab, 1) + ){ + return 1; + } + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: + case TK_PLUS: + case TK_MINUS: + case TK_BITOR: + case TK_LSHIFT: + case TK_RSHIFT: + case TK_CONCAT: + seenNot = 1; + /* no break */ deliberate_fall_through + case TK_STAR: + case TK_REM: + case TK_BITAND: + case TK_SLASH: { + if( exprImpliesNotNull(pParse, p->pRight, pNN, iTab, seenNot) ) return 1; + /* no break */ deliberate_fall_through + } + case TK_SPAN: + case TK_COLLATE: + case TK_UPLUS: + case TK_UMINUS: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, seenNot); + } + case TK_TRUTH: { + if( seenNot ) return 0; + if( p->op2!=TK_IS ) return 0; + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } + case TK_BITNOT: + case TK_NOT: { + return exprImpliesNotNull(pParse, p->pLeft, pNN, iTab, 1); + } } return 0; } @@ -92890,90 +115841,459 @@ SQLITE_PRIVATE int sqlite3ExprListCompare(ExprList *pA, ExprList *pB, int iTab){ ** pE1: x!=123 pE2: x IS NOT NULL Result: true ** pE1: x!=?1 pE2: x IS NOT NULL Result: true ** pE1: x IS NULL pE2: x IS NOT NULL Result: false -** pE1: x IS ?2 pE2: x IS NOT NULL Reuslt: false +** pE1: x IS ?2 pE2: x IS NOT NULL Result: false ** ** When comparing TK_COLUMN nodes between pE1 and pE2, if pE2 has ** Expr.iTable<0 then assume a table number given by iTab. ** +** If pParse is not NULL, then the values of bound variables in pE1 are +** compared against literal values in pE2 and pParse->pVdbe->expmask is +** modified to record which bound variables are referenced. If pParse +** is NULL, then false will be returned if pE1 contains any bound variables. +** ** When in doubt, return false. Returning true might give a performance ** improvement. Returning false might cause a performance reduction, but ** it will always give the correct answer and is hence always safe. */ -SQLITE_PRIVATE int sqlite3ExprImpliesExpr(Expr *pE1, Expr *pE2, int iTab){ - if( sqlite3ExprCompare(pE1, pE2, iTab)==0 ){ +SQLITE_PRIVATE int sqlite3ExprImpliesExpr( + const Parse *pParse, + const Expr *pE1, + const Expr *pE2, + int iTab +){ + if( sqlite3ExprCompare(pParse, pE1, pE2, iTab)==0 ){ return 1; } if( pE2->op==TK_OR - && (sqlite3ExprImpliesExpr(pE1, pE2->pLeft, iTab) - || sqlite3ExprImpliesExpr(pE1, pE2->pRight, iTab) ) + && (sqlite3ExprImpliesExpr(pParse, pE1, pE2->pLeft, iTab) + || sqlite3ExprImpliesExpr(pParse, pE1, pE2->pRight, iTab) ) ){ return 1; } if( pE2->op==TK_NOTNULL - && sqlite3ExprCompare(pE1->pLeft, pE2->pLeft, iTab)==0 - && (pE1->op!=TK_ISNULL && pE1->op!=TK_IS) + && exprImpliesNotNull(pParse, pE1, pE2->pLeft, iTab, 0) ){ return 1; } return 0; } +/* This is a helper function to impliesNotNullRow(). In this routine, +** set pWalker->eCode to one only if *both* of the input expressions +** separately have the implies-not-null-row property. +*/ +static void bothImplyNotNullRow(Walker *pWalker, Expr *pE1, Expr *pE2){ + if( pWalker->eCode==0 ){ + sqlite3WalkExpr(pWalker, pE1); + if( pWalker->eCode ){ + pWalker->eCode = 0; + sqlite3WalkExpr(pWalker, pE2); + } + } +} + +/* +** This is the Expr node callback for sqlite3ExprImpliesNonNullRow(). +** If the expression node requires that the table at pWalker->iCur +** have one or more non-NULL column, then set pWalker->eCode to 1 and abort. +** +** pWalker->mWFlags is non-zero if this inquiry is being undertaking on +** behalf of a RIGHT JOIN (or FULL JOIN). That makes a difference when +** evaluating terms in the ON clause of an inner join. +** +** This routine controls an optimization. False positives (setting +** pWalker->eCode to 1 when it should not be) are deadly, but false-negatives +** (never setting pWalker->eCode) is a harmless missed optimization. +*/ +static int impliesNotNullRow(Walker *pWalker, Expr *pExpr){ + testcase( pExpr->op==TK_AGG_COLUMN ); + testcase( pExpr->op==TK_AGG_FUNCTION ); + if( ExprHasProperty(pExpr, EP_OuterON) ) return WRC_Prune; + if( ExprHasProperty(pExpr, EP_InnerON) && pWalker->mWFlags ){ + /* If iCur is used in an inner-join ON clause to the left of a + ** RIGHT JOIN, that does *not* mean that the table must be non-null. + ** But it is difficult to check for that condition precisely. + ** To keep things simple, any use of iCur from any inner-join is + ** ignored while attempting to simplify a RIGHT JOIN. */ + return WRC_Prune; + } + switch( pExpr->op ){ + case TK_ISNOT: + case TK_ISNULL: + case TK_NOTNULL: + case TK_IS: + case TK_VECTOR: + case TK_FUNCTION: + case TK_TRUTH: + case TK_CASE: + testcase( pExpr->op==TK_ISNOT ); + testcase( pExpr->op==TK_ISNULL ); + testcase( pExpr->op==TK_NOTNULL ); + testcase( pExpr->op==TK_IS ); + testcase( pExpr->op==TK_VECTOR ); + testcase( pExpr->op==TK_FUNCTION ); + testcase( pExpr->op==TK_TRUTH ); + testcase( pExpr->op==TK_CASE ); + return WRC_Prune; + + case TK_COLUMN: + if( pWalker->u.iCur==pExpr->iTable ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Prune; + + case TK_OR: + case TK_AND: + /* Both sides of an AND or OR must separately imply non-null-row. + ** Consider these cases: + ** 1. NOT (x AND y) + ** 2. x OR y + ** If only one of x or y is non-null-row, then the overall expression + ** can be true if the other arm is false (case 1) or true (case 2). + */ + testcase( pExpr->op==TK_OR ); + testcase( pExpr->op==TK_AND ); + bothImplyNotNullRow(pWalker, pExpr->pLeft, pExpr->pRight); + return WRC_Prune; + + case TK_IN: + /* Beware of "x NOT IN ()" and "x NOT IN (SELECT 1 WHERE false)", + ** both of which can be true. But apart from these cases, if + ** the left-hand side of the IN is NULL then the IN itself will be + ** NULL. */ + if( ExprUseXList(pExpr) && ALWAYS(pExpr->x.pList->nExpr>0) ){ + sqlite3WalkExpr(pWalker, pExpr->pLeft); + } + return WRC_Prune; + + case TK_BETWEEN: + /* In "x NOT BETWEEN y AND z" either x must be non-null-row or else + ** both y and z must be non-null row */ + assert( ExprUseXList(pExpr) ); + assert( pExpr->x.pList->nExpr==2 ); + sqlite3WalkExpr(pWalker, pExpr->pLeft); + bothImplyNotNullRow(pWalker, pExpr->x.pList->a[0].pExpr, + pExpr->x.pList->a[1].pExpr); + return WRC_Prune; + + /* Virtual tables are allowed to use constraints like x=NULL. So + ** a term of the form x=y does not prove that y is not null if x + ** is the column of a virtual table */ + case TK_EQ: + case TK_NE: + case TK_LT: + case TK_LE: + case TK_GT: + case TK_GE: { + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + testcase( pExpr->op==TK_EQ ); + testcase( pExpr->op==TK_NE ); + testcase( pExpr->op==TK_LT ); + testcase( pExpr->op==TK_LE ); + testcase( pExpr->op==TK_GT ); + testcase( pExpr->op==TK_GE ); + /* The y.pTab=0 assignment in wherecode.c always happens after the + ** impliesNotNullRow() test */ + assert( pLeft->op!=TK_COLUMN || ExprUseYTab(pLeft) ); + assert( pRight->op!=TK_COLUMN || ExprUseYTab(pRight) ); + if( (pLeft->op==TK_COLUMN + && ALWAYS(pLeft->y.pTab!=0) + && IsVirtual(pLeft->y.pTab)) + || (pRight->op==TK_COLUMN + && ALWAYS(pRight->y.pTab!=0) + && IsVirtual(pRight->y.pTab)) + ){ + return WRC_Prune; + } + /* no break */ deliberate_fall_through + } + default: + return WRC_Continue; + } +} + +/* +** Return true (non-zero) if expression p can only be true if at least +** one column of table iTab is non-null. In other words, return true +** if expression p will always be NULL or false if every column of iTab +** is NULL. +** +** False negatives are acceptable. In other words, it is ok to return +** zero even if expression p will never be true of every column of iTab +** is NULL. A false negative is merely a missed optimization opportunity. +** +** False positives are not allowed, however. A false positive may result +** in an incorrect answer. +** +** Terms of p that are marked with EP_OuterON (and hence that come from +** the ON or USING clauses of OUTER JOINS) are excluded from the analysis. +** +** This routine is used to check if a LEFT JOIN can be converted into +** an ordinary JOIN. The p argument is the WHERE clause. If the WHERE +** clause requires that some column of the right table of the LEFT JOIN +** be non-NULL, then the LEFT JOIN can be safely converted into an +** ordinary join. +*/ +SQLITE_PRIVATE int sqlite3ExprImpliesNonNullRow(Expr *p, int iTab, int isRJ){ + Walker w; + p = sqlite3ExprSkipCollateAndLikely(p); + if( p==0 ) return 0; + if( p->op==TK_NOTNULL ){ + p = p->pLeft; + }else{ + while( p->op==TK_AND ){ + if( sqlite3ExprImpliesNonNullRow(p->pLeft, iTab, isRJ) ) return 1; + p = p->pRight; + } + } + w.xExprCallback = impliesNotNullRow; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + w.eCode = 0; + w.mWFlags = isRJ!=0; + w.u.iCur = iTab; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + /* ** An instance of the following structure is used by the tree walker -** to count references to table columns in the arguments of an -** aggregate function, in order to implement the -** sqlite3FunctionThisSrc() routine. +** to determine if an expression can be evaluated by reference to the +** index only, without having to do a search for the corresponding +** table entry. The IdxCover.pIdx field is the index. IdxCover.iCur +** is the cursor for the table. */ -struct SrcCount { - SrcList *pSrc; /* One particular FROM clause in a nested query */ - int nThis; /* Number of references to columns in pSrcList */ - int nOther; /* Number of references to columns in other FROM clauses */ +struct IdxCover { + Index *pIdx; /* The index to be tested for coverage */ + int iCur; /* Cursor number for the table corresponding to the index */ }; /* -** Count the number of references to columns. +** Check to see if there are references to columns in table +** pWalker->u.pIdxCover->iCur can be satisfied using the index +** pWalker->u.pIdxCover->pIdx. */ -static int exprSrcCount(Walker *pWalker, Expr *pExpr){ - /* The NEVER() on the second term is because sqlite3FunctionUsesThisSrc() - ** is always called before sqlite3ExprAnalyzeAggregates() and so the - ** TK_COLUMNs have not yet been converted into TK_AGG_COLUMN. If - ** sqlite3FunctionUsesThisSrc() is used differently in the future, the - ** NEVER() will need to be removed. */ - if( pExpr->op==TK_COLUMN || NEVER(pExpr->op==TK_AGG_COLUMN) ){ +static int exprIdxCover(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pWalker->u.pIdxCover->iCur + && sqlite3TableColumnToIndex(pWalker->u.pIdxCover->pIdx, pExpr->iColumn)<0 + ){ + pWalker->eCode = 1; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Determine if an index pIdx on table with cursor iCur contains will +** the expression pExpr. Return true if the index does cover the +** expression and false if the pExpr expression references table columns +** that are not found in the index pIdx. +** +** An index covering an expression means that the expression can be +** evaluated using only the index and without having to lookup the +** corresponding table entry. +*/ +SQLITE_PRIVATE int sqlite3ExprCoveredByIndex( + Expr *pExpr, /* The index to be tested */ + int iCur, /* The cursor number for the corresponding table */ + Index *pIdx /* The index that might be used for coverage */ +){ + Walker w; + struct IdxCover xcov; + memset(&w, 0, sizeof(w)); + xcov.iCur = iCur; + xcov.pIdx = pIdx; + w.xExprCallback = exprIdxCover; + w.u.pIdxCover = &xcov; + sqlite3WalkExpr(&w, pExpr); + return !w.eCode; +} + + +/* Structure used to pass information throughout the Walker in order to +** implement sqlite3ReferencesSrcList(). +*/ +struct RefSrcList { + sqlite3 *db; /* Database connection used for sqlite3DbRealloc() */ + SrcList *pRef; /* Looking for references to these tables */ + i64 nExclude; /* Number of tables to exclude from the search */ + int *aiExclude; /* Cursor IDs for tables to exclude from the search */ +}; + +/* +** Walker SELECT callbacks for sqlite3ReferencesSrcList(). +** +** When entering a new subquery on the pExpr argument, add all FROM clause +** entries for that subquery to the exclude list. +** +** When leaving the subquery, remove those entries from the exclude list. +*/ +static int selectRefEnter(Walker *pWalker, Select *pSelect){ + struct RefSrcList *p = pWalker->u.pRefSrcList; + SrcList *pSrc = pSelect->pSrc; + i64 i, j; + int *piNew; + if( pSrc->nSrc==0 ) return WRC_Continue; + j = p->nExclude; + p->nExclude += pSrc->nSrc; + piNew = sqlite3DbRealloc(p->db, p->aiExclude, p->nExclude*sizeof(int)); + if( piNew==0 ){ + p->nExclude = 0; + return WRC_Abort; + }else{ + p->aiExclude = piNew; + } + for(i=0; inSrc; i++, j++){ + p->aiExclude[j] = pSrc->a[i].iCursor; + } + return WRC_Continue; +} +static void selectRefLeave(Walker *pWalker, Select *pSelect){ + struct RefSrcList *p = pWalker->u.pRefSrcList; + SrcList *pSrc = pSelect->pSrc; + if( p->nExclude ){ + assert( p->nExclude>=pSrc->nSrc ); + p->nExclude -= pSrc->nSrc; + } +} + +/* This is the Walker EXPR callback for sqlite3ReferencesSrcList(). +** +** Set the 0x01 bit of pWalker->eCode if there is a reference to any +** of the tables shown in RefSrcList.pRef. +** +** Set the 0x02 bit of pWalker->eCode if there is a reference to a +** table is in neither RefSrcList.pRef nor RefSrcList.aiExclude. +*/ +static int exprRefToSrcList(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN + || pExpr->op==TK_AGG_COLUMN + ){ int i; - struct SrcCount *p = pWalker->u.pSrcCount; - SrcList *pSrc = p->pSrc; + struct RefSrcList *p = pWalker->u.pRefSrcList; + SrcList *pSrc = p->pRef; int nSrc = pSrc ? pSrc->nSrc : 0; for(i=0; iiTable==pSrc->a[i].iCursor ) break; + if( pExpr->iTable==pSrc->a[i].iCursor ){ + pWalker->eCode |= 1; + return WRC_Continue; + } } - if( inThis++; - }else{ - p->nOther++; + for(i=0; inExclude && p->aiExclude[i]!=pExpr->iTable; i++){} + if( i>=p->nExclude ){ + pWalker->eCode |= 2; } } return WRC_Continue; } /* -** Determine if any of the arguments to the pExpr Function reference -** pSrcList. Return true if they do. Also return true if the function -** has no arguments or has only constant arguments. Return false if pExpr -** references columns but not columns of tables found in pSrcList. +** Check to see if pExpr references any tables in pSrcList. +** Possible return values: +** +** 1 pExpr does references a table in pSrcList. +** +** 0 pExpr references some table that is not defined in either +** pSrcList or in subqueries of pExpr itself. +** +** -1 pExpr only references no tables at all, or it only +** references tables defined in subqueries of pExpr itself. +** +** As currently used, pExpr is always an aggregate function call. That +** fact is exploited for efficiency. */ -SQLITE_PRIVATE int sqlite3FunctionUsesThisSrc(Expr *pExpr, SrcList *pSrcList){ +SQLITE_PRIVATE int sqlite3ReferencesSrcList(Parse *pParse, Expr *pExpr, SrcList *pSrcList){ Walker w; - struct SrcCount cnt; - assert( pExpr->op==TK_AGG_FUNCTION ); + struct RefSrcList x; + assert( pParse->db!=0 ); memset(&w, 0, sizeof(w)); - w.xExprCallback = exprSrcCount; - w.u.pSrcCount = &cnt; - cnt.pSrc = pSrcList; - cnt.nThis = 0; - cnt.nOther = 0; + memset(&x, 0, sizeof(x)); + w.xExprCallback = exprRefToSrcList; + w.xSelectCallback = selectRefEnter; + w.xSelectCallback2 = selectRefLeave; + w.u.pRefSrcList = &x; + x.db = pParse->db; + x.pRef = pSrcList; + assert( pExpr->op==TK_AGG_FUNCTION ); + assert( ExprUseXList(pExpr) ); sqlite3WalkExprList(&w, pExpr->x.pList); - return cnt.nThis>0 || cnt.nOther==0; + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + assert( pExpr->pLeft->x.pList!=0 ); + sqlite3WalkExprList(&w, pExpr->pLeft->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3WalkExpr(&w, pExpr->y.pWin->pFilter); + } +#endif + if( x.aiExclude ) sqlite3DbNNFreeNN(pParse->db, x.aiExclude); + if( w.eCode & 0x01 ){ + return 1; + }else if( w.eCode ){ + return 0; + }else{ + return -1; + } +} + +/* +** This is a Walker expression node callback. +** +** For Expr nodes that contain pAggInfo pointers, make sure the AggInfo +** object that is referenced does not refer directly to the Expr. If +** it does, make a copy. This is done because the pExpr argument is +** subject to change. +** +** The copy is scheduled for deletion using the sqlite3ExprDeferredDelete() +** which builds on the sqlite3ParserAddCleanup() mechanism. +*/ +static int agginfoPersistExprCb(Walker *pWalker, Expr *pExpr){ + if( ALWAYS(!ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced)) + && pExpr->pAggInfo!=0 + ){ + AggInfo *pAggInfo = pExpr->pAggInfo; + int iAgg = pExpr->iAgg; + Parse *pParse = pWalker->pParse; + sqlite3 *db = pParse->db; + assert( iAgg>=0 ); + if( pExpr->op!=TK_AGG_FUNCTION ){ + if( iAggnColumn + && pAggInfo->aCol[iAgg].pCExpr==pExpr + ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ + pAggInfo->aCol[iAgg].pCExpr = pExpr; + } + } + }else{ + assert( pExpr->op==TK_AGG_FUNCTION ); + if( ALWAYS(iAggnFunc) + && pAggInfo->aFunc[iAgg].pFExpr==pExpr + ){ + pExpr = sqlite3ExprDup(db, pExpr, 0); + if( pExpr && !sqlite3ExprDeferredDelete(pParse, pExpr) ){ + pAggInfo->aFunc[iAgg].pFExpr = pExpr; + } + } + } + } + return WRC_Continue; +} + +/* +** Initialize a Walker object so that will persist AggInfo entries referenced +** by the tree that is walked. +*/ +SQLITE_PRIVATE void sqlite3AggInfoPersistWalkerInit(Walker *pWalker, Parse *pParse){ + memset(pWalker, 0, sizeof(*pWalker)); + pWalker->pParse = pParse; + pWalker->xExprCallback = agginfoPersistExprCb; + pWalker->xSelectCallback = sqlite3SelectWalkNoop; } /* @@ -92990,7 +116310,7 @@ static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ &i ); return i; -} +} /* ** Add a new element to the pAggInfo->aFunc[] array. Return the index of @@ -92999,14 +116319,82 @@ static int addAggInfoColumn(sqlite3 *db, AggInfo *pInfo){ static int addAggInfoFunc(sqlite3 *db, AggInfo *pInfo){ int i; pInfo->aFunc = sqlite3ArrayAllocate( - db, + db, pInfo->aFunc, sizeof(pInfo->aFunc[0]), &pInfo->nFunc, &i ); return i; -} +} + +/* +** Search the AggInfo object for an aCol[] entry that has iTable and iColumn. +** Return the index in aCol[] of the entry that describes that column. +** +** If no prior entry is found, create a new one and return -1. The +** new column will have an index of pAggInfo->nColumn-1. +*/ +static void findOrCreateAggInfoColumn( + Parse *pParse, /* Parsing context */ + AggInfo *pAggInfo, /* The AggInfo object to search and/or modify */ + Expr *pExpr /* Expr describing the column to find or insert */ +){ + struct AggInfo_col *pCol; + int k; + + assert( pAggInfo->iFirstReg==0 ); + pCol = pAggInfo->aCol; + for(k=0; knColumn; k++, pCol++){ + if( pCol->pCExpr==pExpr ) return; + if( pCol->iTable==pExpr->iTable + && pCol->iColumn==pExpr->iColumn + && pExpr->op!=TK_IF_NULL_ROW + ){ + goto fix_up_expr; + } + } + k = addAggInfoColumn(pParse->db, pAggInfo); + if( k<0 ){ + /* OOM on resize */ + assert( pParse->db->mallocFailed ); + return; + } + pCol = &pAggInfo->aCol[k]; + assert( ExprUseYTab(pExpr) ); + pCol->pTab = pExpr->y.pTab; + pCol->iTable = pExpr->iTable; + pCol->iColumn = pExpr->iColumn; + pCol->iSorterColumn = -1; + pCol->pCExpr = pExpr; + if( pAggInfo->pGroupBy && pExpr->op!=TK_IF_NULL_ROW ){ + int j, n; + ExprList *pGB = pAggInfo->pGroupBy; + struct ExprList_item *pTerm = pGB->a; + n = pGB->nExpr; + for(j=0; jpExpr; + if( pE->op==TK_COLUMN + && pE->iTable==pExpr->iTable + && pE->iColumn==pExpr->iColumn + ){ + pCol->iSorterColumn = j; + break; + } + } + } + if( pCol->iSorterColumn<0 ){ + pCol->iSorterColumn = pAggInfo->nSortingColumn++; + } +fix_up_expr: + ExprSetVVAProperty(pExpr, EP_NoReduce); + assert( pExpr->pAggInfo==0 || pExpr->pAggInfo==pAggInfo ); + pExpr->pAggInfo = pAggInfo; + if( pExpr->op==TK_COLUMN ){ + pExpr->op = TK_AGG_COLUMN; + } + pExpr->iAgg = (i16)k; +} /* ** This is the xExprCallback for a tree walker. It is used to @@ -93018,88 +116406,80 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ NameContext *pNC = pWalker->u.pNC; Parse *pParse = pNC->pParse; SrcList *pSrcList = pNC->pSrcList; - AggInfo *pAggInfo = pNC->pAggInfo; + AggInfo *pAggInfo = pNC->uNC.pAggInfo; + assert( pNC->ncFlags & NC_UAggInfo ); + assert( pAggInfo->iFirstReg==0 ); switch( pExpr->op ){ + default: { + IndexedExpr *pIEpr; + Expr tmp; + assert( pParse->iSelfTab==0 ); + if( (pNC->ncFlags & NC_InAggFunc)==0 ) break; + if( pParse->pIdxEpr==0 ) break; + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + int iDataCur = pIEpr->iDataCur; + if( iDataCur<0 ) continue; + if( sqlite3ExprCompare(0, pExpr, pIEpr->pExpr, iDataCur)==0 ) break; + } + if( pIEpr==0 ) break; + if( NEVER(!ExprUseYTab(pExpr)) ) break; + for(i=0; inSrc; i++){ + if( pSrcList->a[0].iCursor==pIEpr->iDataCur ) break; + } + if( i>=pSrcList->nSrc ) break; + if( NEVER(pExpr->pAggInfo!=0) ) break; /* Resolved by outer context */ + if( pParse->nErr ){ return WRC_Abort; } + + /* If we reach this point, it means that expression pExpr can be + ** translated into a reference to an index column as described by + ** pIEpr. + */ + memset(&tmp, 0, sizeof(tmp)); + tmp.op = TK_AGG_COLUMN; + tmp.iTable = pIEpr->iIdxCur; + tmp.iColumn = pIEpr->iIdxCol; + findOrCreateAggInfoColumn(pParse, pAggInfo, &tmp); + if( pParse->nErr ){ return WRC_Abort; } + assert( pAggInfo->aCol!=0 ); + assert( tmp.iAggnColumn ); + pAggInfo->aCol[tmp.iAgg].pCExpr = pExpr; + pExpr->pAggInfo = pAggInfo; + pExpr->iAgg = tmp.iAgg; + return WRC_Prune; + } + case TK_IF_NULL_ROW: case TK_AGG_COLUMN: case TK_COLUMN: { testcase( pExpr->op==TK_AGG_COLUMN ); testcase( pExpr->op==TK_COLUMN ); + testcase( pExpr->op==TK_IF_NULL_ROW ); /* Check to see if the column is in one of the tables in the FROM ** clause of the aggregate query */ if( ALWAYS(pSrcList!=0) ){ - struct SrcList_item *pItem = pSrcList->a; + SrcItem *pItem = pSrcList->a; for(i=0; inSrc; i++, pItem++){ - struct AggInfo_col *pCol; assert( !ExprHasProperty(pExpr, EP_TokenOnly|EP_Reduced) ); if( pExpr->iTable==pItem->iCursor ){ - /* If we reach this point, it means that pExpr refers to a table - ** that is in the FROM clause of the aggregate query. - ** - ** Make an entry for the column in pAggInfo->aCol[] if there - ** is not an entry there already. - */ - int k; - pCol = pAggInfo->aCol; - for(k=0; knColumn; k++, pCol++){ - if( pCol->iTable==pExpr->iTable && - pCol->iColumn==pExpr->iColumn ){ - break; - } - } - if( (k>=pAggInfo->nColumn) - && (k = addAggInfoColumn(pParse->db, pAggInfo))>=0 - ){ - pCol = &pAggInfo->aCol[k]; - pCol->pTab = pExpr->pTab; - pCol->iTable = pExpr->iTable; - pCol->iColumn = pExpr->iColumn; - pCol->iMem = ++pParse->nMem; - pCol->iSorterColumn = -1; - pCol->pExpr = pExpr; - if( pAggInfo->pGroupBy ){ - int j, n; - ExprList *pGB = pAggInfo->pGroupBy; - struct ExprList_item *pTerm = pGB->a; - n = pGB->nExpr; - for(j=0; jpExpr; - if( pE->op==TK_COLUMN && pE->iTable==pExpr->iTable && - pE->iColumn==pExpr->iColumn ){ - pCol->iSorterColumn = j; - break; - } - } - } - if( pCol->iSorterColumn<0 ){ - pCol->iSorterColumn = pAggInfo->nSortingColumn++; - } - } - /* There is now an entry for pExpr in pAggInfo->aCol[] (either - ** because it was there before or because we just created it). - ** Convert the pExpr to be a TK_AGG_COLUMN referring to that - ** pAggInfo->aCol[] entry. - */ - ExprSetVVAProperty(pExpr, EP_NoReduce); - pExpr->pAggInfo = pAggInfo; - pExpr->op = TK_AGG_COLUMN; - pExpr->iAgg = (i16)k; + findOrCreateAggInfoColumn(pParse, pAggInfo, pExpr); break; } /* endif pExpr->iTable==pItem->iCursor */ } /* end loop over pSrcList */ } - return WRC_Prune; + return WRC_Continue; } case TK_AGG_FUNCTION: { if( (pNC->ncFlags & NC_InAggFunc)==0 && pWalker->walkerDepth==pExpr->op2 + && pExpr->pAggInfo==0 ){ - /* Check to see if pExpr is a duplicate of another aggregate + /* Check to see if pExpr is a duplicate of another aggregate ** function that is already in the pAggInfo structure */ struct AggInfo_func *pItem = pAggInfo->aFunc; for(i=0; inFunc; i++, pItem++){ - if( sqlite3ExprCompare(pItem->pExpr, pExpr, -1)==0 ){ + if( NEVER(pItem->pFExpr==pExpr) ) break; + if( sqlite3ExprCompare(0, pItem->pFExpr, pExpr, -1)==0 ){ break; } } @@ -93109,15 +116489,44 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ u8 enc = ENC(pParse->db); i = addAggInfoFunc(pParse->db, pAggInfo); if( i>=0 ){ + int nArg; assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); pItem = &pAggInfo->aFunc[i]; - pItem->pExpr = pExpr; - pItem->iMem = ++pParse->nMem; - assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pItem->pFExpr = pExpr; + assert( ExprUseUToken(pExpr) ); + nArg = pExpr->x.pList ? pExpr->x.pList->nExpr : 0; pItem->pFunc = sqlite3FindFunction(pParse->db, - pExpr->u.zToken, - pExpr->x.pList ? pExpr->x.pList->nExpr : 0, enc, 0); - if( pExpr->flags & EP_Distinct ){ + pExpr->u.zToken, nArg, enc, 0); + assert( pItem->bOBUnique==0 ); + if( pExpr->pLeft + && (pItem->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL)==0 + ){ + /* The NEEDCOLL test above causes any ORDER BY clause on + ** aggregate min() or max() to be ignored. */ + ExprList *pOBList; + assert( nArg>0 ); + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + pItem->iOBTab = pParse->nTab++; + pOBList = pExpr->pLeft->x.pList; + assert( pOBList->nExpr>0 ); + assert( pItem->bOBUnique==0 ); + if( pOBList->nExpr==1 + && nArg==1 + && sqlite3ExprCompare(0,pOBList->a[0].pExpr, + pExpr->x.pList->a[0].pExpr,0)==0 + ){ + pItem->bOBPayload = 0; + pItem->bOBUnique = ExprHasProperty(pExpr, EP_Distinct); + }else{ + pItem->bOBPayload = 1; + } + pItem->bUseSubtype = + (pItem->pFunc->funcFlags & SQLITE_SUBTYPE)!=0; + }else{ + pItem->iOBTab = -1; + } + if( ExprHasProperty(pExpr, EP_Distinct) && !pItem->bOBUnique ){ pItem->iDistinct = pParse->nTab++; }else{ pItem->iDistinct = -1; @@ -93138,11 +116547,6 @@ static int analyzeAggregate(Walker *pWalker, Expr *pExpr){ } return WRC_Continue; } -static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ - UNUSED_PARAMETER(pWalker); - UNUSED_PARAMETER(pSelect); - return WRC_Continue; -} /* ** Analyze the pExpr expression looking for aggregate functions and @@ -93155,10 +116559,12 @@ static int analyzeAggregatesInSelect(Walker *pWalker, Select *pSelect){ */ SQLITE_PRIVATE void sqlite3ExprAnalyzeAggregates(NameContext *pNC, Expr *pExpr){ Walker w; - memset(&w, 0, sizeof(w)); w.xExprCallback = analyzeAggregate; - w.xSelectCallback = analyzeAggregatesInSelect; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + w.walkerDepth = 0; w.u.pNC = pNC; + w.pParse = 0; assert( pNC->pSrcList!=0 ); sqlite3WalkExpr(&w, pExpr); } @@ -93192,34 +116598,25 @@ SQLITE_PRIVATE int sqlite3GetTempReg(Parse *pParse){ /* ** Deallocate a register, making available for reuse for some other ** purpose. -** -** If a register is currently being used by the column cache, then -** the deallocation is deferred until the column cache line that uses -** the register becomes stale. */ SQLITE_PRIVATE void sqlite3ReleaseTempReg(Parse *pParse, int iReg){ - if( iReg && pParse->nTempRegaTempReg) ){ - int i; - struct yColCache *p; - for(i=0, p=pParse->aColCache; iiReg==iReg ){ - p->tempReg = 1; - return; - } + if( iReg ){ + sqlite3VdbeReleaseRegisters(pParse, iReg, 1, 0, 0); + if( pParse->nTempRegaTempReg) ){ + pParse->aTempReg[pParse->nTempReg++] = iReg; } - pParse->aTempReg[pParse->nTempReg++] = iReg; } } /* -** Allocate or deallocate a block of nReg consecutive registers +** Allocate or deallocate a block of nReg consecutive registers. */ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ int i, n; + if( nReg==1 ) return sqlite3GetTempReg(pParse); i = pParse->iRangeReg; n = pParse->nRangeReg; if( nReg<=n ){ - assert( !usedAsColumnCache(pParse, i, i+n-1) ); pParse->iRangeReg += nReg; pParse->nRangeReg -= nReg; }else{ @@ -93229,7 +116626,11 @@ SQLITE_PRIVATE int sqlite3GetTempRange(Parse *pParse, int nReg){ return i; } SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ - sqlite3ExprCacheRemove(pParse, iReg, nReg); + if( nReg==1 ){ + sqlite3ReleaseTempReg(pParse, iReg); + return; + } + sqlite3VdbeReleaseRegisters(pParse, iReg, nReg, 0, 0); if( nReg>pParse->nRangeReg ){ pParse->nRangeReg = nReg; pParse->iRangeReg = iReg; @@ -93238,12 +116639,48 @@ SQLITE_PRIVATE void sqlite3ReleaseTempRange(Parse *pParse, int iReg, int nReg){ /* ** Mark all temporary registers as being unavailable for reuse. +** +** Always invoke this procedure after coding a subroutine or co-routine +** that might be invoked from other parts of the code, to ensure that +** the sub/co-routine does not use registers in common with the code that +** invokes the sub/co-routine. */ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ pParse->nTempReg = 0; pParse->nRangeReg = 0; } +/* +** Make sure sufficient registers have been allocated so that +** iReg is a valid register number. +*/ +SQLITE_PRIVATE void sqlite3TouchRegister(Parse *pParse, int iReg){ + if( pParse->nMemnMem = iReg; +} + +#if defined(SQLITE_ENABLE_STAT4) || defined(SQLITE_DEBUG) +/* +** Return the latest reusable register in the set of all registers. +** The value returned is no less than iMin. If any register iMin or +** greater is in permanent use, then return one more than that last +** permanent register. +*/ +SQLITE_PRIVATE int sqlite3FirstAvailableRegister(Parse *pParse, int iMin){ + const ExprList *pList = pParse->pConstExpr; + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].u.iConstExprReg>=iMin ){ + iMin = pList->a[i].u.iConstExprReg + 1; + } + } + } + pParse->nTempReg = 0; + pParse->nRangeReg = 0; + return iMin; +} +#endif /* SQLITE_ENABLE_STAT4 || SQLITE_DEBUG */ + /* ** Validate that no temporary register falls within the range of ** iFirst..iLast, inclusive. This routine is only call from within assert() @@ -93253,8 +116690,8 @@ SQLITE_PRIVATE void sqlite3ClearTempRegCache(Parse *pParse){ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ int i; if( pParse->nRangeReg>0 - && pParse->iRangeReg+pParse->nRangeRegiRangeReg>=iFirst + && pParse->iRangeReg+pParse->nRangeReg > iFirst + && pParse->iRangeReg <= iLast ){ return 0; } @@ -93263,6 +116700,14 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ return 0; } } + if( pParse->pConstExpr ){ + ExprList *pList = pParse->pConstExpr; + for(i=0; inExpr; i++){ + int iReg = pList->a[i].u.iConstExprReg; + if( iReg==0 ) continue; + if( iReg>=iFirst && iReg<=iLast ) return 0; + } + } return 1; } #endif /* SQLITE_DEBUG */ @@ -93291,352 +116736,6 @@ SQLITE_PRIVATE int sqlite3NoTempsInRange(Parse *pParse, int iFirst, int iLast){ */ #ifndef SQLITE_OMIT_ALTERTABLE - -/* -** This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TABLE or -** CREATE INDEX command. The second is a table name. The table name in -** the CREATE TABLE or CREATE INDEX statement is replaced with the third -** argument and the result returned. Examples: -** -** sqlite_rename_table('CREATE TABLE abc(a, b, c)', 'def') -** -> 'CREATE TABLE def(a, b, c)' -** -** sqlite_rename_table('CREATE INDEX i ON abc(a)', 'def') -** -> 'CREATE INDEX i ON def(a, b, c)' -*/ -static void renameTableFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - - sqlite3 *db = sqlite3_context_db_handle(context); - - UNUSED_PARAMETER(NotUsed); - - /* The principle used to locate the table name in the CREATE TABLE - ** statement is that the table name is the first non-space token that - ** is immediately followed by a TK_LP or TK_USING token. - */ - if( zSql ){ - do { - if( !*zCsr ){ - /* Ran out of input before finding an opening bracket. Return NULL. */ - return; - } - - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; - - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - } while( token==TK_SPACE ); - assert( len>0 ); - } while( token!=TK_LP && token!=TK_USING ); - - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), - zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); - } -} - -/* -** This C function implements an SQL user function that is used by SQL code -** generated by the ALTER TABLE ... RENAME command to modify the definition -** of any foreign key constraints that use the table being renamed as the -** parent table. It is passed three arguments: -** -** 1) The complete text of the CREATE TABLE statement being modified, -** 2) The old name of the table being renamed, and -** 3) The new name of the table being renamed. -** -** It returns the new CREATE TABLE statement. For example: -** -** sqlite_rename_parent('CREATE TABLE t1(a REFERENCES t2)', 't2', 't3') -** -> 'CREATE TABLE t1(a REFERENCES t3)' -*/ -#ifndef SQLITE_OMIT_FOREIGN_KEY -static void renameParentFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - sqlite3 *db = sqlite3_context_db_handle(context); - char *zOutput = 0; - char *zResult; - unsigned char const *zInput = sqlite3_value_text(argv[0]); - unsigned char const *zOld = sqlite3_value_text(argv[1]); - unsigned char const *zNew = sqlite3_value_text(argv[2]); - - unsigned const char *z; /* Pointer to token */ - int n; /* Length of token z */ - int token; /* Type of token */ - - UNUSED_PARAMETER(NotUsed); - if( zInput==0 || zOld==0 ) return; - for(z=zInput; *z; z=z+n){ - n = sqlite3GetToken(z, &token); - if( token==TK_REFERENCES ){ - char *zParent; - do { - z += n; - n = sqlite3GetToken(z, &token); - }while( token==TK_SPACE ); - - if( token==TK_ILLEGAL ) break; - zParent = sqlite3DbStrNDup(db, (const char *)z, n); - if( zParent==0 ) break; - sqlite3Dequote(zParent); - if( 0==sqlite3StrICmp((const char *)zOld, zParent) ){ - char *zOut = sqlite3MPrintf(db, "%s%.*s\"%w\"", - (zOutput?zOutput:""), (int)(z-zInput), zInput, (const char *)zNew - ); - sqlite3DbFree(db, zOutput); - zOutput = zOut; - zInput = &z[n]; - } - sqlite3DbFree(db, zParent); - } - } - - zResult = sqlite3MPrintf(db, "%s%s", (zOutput?zOutput:""), zInput), - sqlite3_result_text(context, zResult, -1, SQLITE_DYNAMIC); - sqlite3DbFree(db, zOutput); -} -#endif - -#ifndef SQLITE_OMIT_TRIGGER -/* This function is used by SQL generated to implement the -** ALTER TABLE command. The first argument is the text of a CREATE TRIGGER -** statement. The second is a table name. The table name in the CREATE -** TRIGGER statement is replaced with the third argument and the result -** returned. This is analagous to renameTableFunc() above, except for CREATE -** TRIGGER, not CREATE INDEX and CREATE TABLE. -*/ -static void renameTriggerFunc( - sqlite3_context *context, - int NotUsed, - sqlite3_value **argv -){ - unsigned char const *zSql = sqlite3_value_text(argv[0]); - unsigned char const *zTableName = sqlite3_value_text(argv[1]); - - int token; - Token tname; - int dist = 3; - unsigned char const *zCsr = zSql; - int len = 0; - char *zRet; - sqlite3 *db = sqlite3_context_db_handle(context); - - UNUSED_PARAMETER(NotUsed); - - /* The principle used to locate the table name in the CREATE TRIGGER - ** statement is that the table name is the first token that is immediately - ** preceded by either TK_ON or TK_DOT and immediately followed by one - ** of TK_WHEN, TK_BEGIN or TK_FOR. - */ - if( zSql ){ - do { - - if( !*zCsr ){ - /* Ran out of input before finding the table name. Return NULL. */ - return; - } - - /* Store the token that zCsr points to in tname. */ - tname.z = (char*)zCsr; - tname.n = len; - - /* Advance zCsr to the next token. Store that token type in 'token', - ** and its length in 'len' (to be used next iteration of this loop). - */ - do { - zCsr += len; - len = sqlite3GetToken(zCsr, &token); - }while( token==TK_SPACE ); - assert( len>0 ); - - /* Variable 'dist' stores the number of tokens read since the most - ** recent TK_DOT or TK_ON. This means that when a WHEN, FOR or BEGIN - ** token is read and 'dist' equals 2, the condition stated above - ** to be met. - ** - ** Note that ON cannot be a database, table or column name, so - ** there is no need to worry about syntax like - ** "CREATE TRIGGER ... ON ON.ON BEGIN ..." etc. - */ - dist++; - if( token==TK_DOT || token==TK_ON ){ - dist = 0; - } - } while( dist!=2 || (token!=TK_WHEN && token!=TK_FOR && token!=TK_BEGIN) ); - - /* Variable tname now contains the token that is the old table-name - ** in the CREATE TRIGGER statement. - */ - zRet = sqlite3MPrintf(db, "%.*s\"%w\"%s", (int)(((u8*)tname.z) - zSql), - zSql, zTableName, tname.z+tname.n); - sqlite3_result_text(context, zRet, -1, SQLITE_DYNAMIC); - } -} -#endif /* !SQLITE_OMIT_TRIGGER */ - -/* -** Register built-in functions used to help implement ALTER TABLE -*/ -SQLITE_PRIVATE void sqlite3AlterFunctions(void){ - static FuncDef aAlterTableFuncs[] = { - FUNCTION(sqlite_rename_table, 2, 0, 0, renameTableFunc), -#ifndef SQLITE_OMIT_TRIGGER - FUNCTION(sqlite_rename_trigger, 2, 0, 0, renameTriggerFunc), -#endif -#ifndef SQLITE_OMIT_FOREIGN_KEY - FUNCTION(sqlite_rename_parent, 3, 0, 0, renameParentFunc), -#endif - }; - sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); -} - -/* -** This function is used to create the text of expressions of the form: -** -** name= OR name= OR ... -** -** If argument zWhere is NULL, then a pointer string containing the text -** "name=" is returned, where is the quoted version -** of the string passed as argument zConstant. The returned buffer is -** allocated using sqlite3DbMalloc(). It is the responsibility of the -** caller to ensure that it is eventually freed. -** -** If argument zWhere is not NULL, then the string returned is -** " OR name=", where is the contents of zWhere. -** In this case zWhere is passed to sqlite3DbFree() before returning. -** -*/ -static char *whereOrName(sqlite3 *db, char *zWhere, char *zConstant){ - char *zNew; - if( !zWhere ){ - zNew = sqlite3MPrintf(db, "name=%Q", zConstant); - }else{ - zNew = sqlite3MPrintf(db, "%s OR name=%Q", zWhere, zConstant); - sqlite3DbFree(db, zWhere); - } - return zNew; -} - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) -/* -** Generate the text of a WHERE expression which can be used to select all -** tables that have foreign key constraints that refer to table pTab (i.e. -** constraints for which pTab is the parent table) from the sqlite_master -** table. -*/ -static char *whereForeignKeys(Parse *pParse, Table *pTab){ - FKey *p; - char *zWhere = 0; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - zWhere = whereOrName(pParse->db, zWhere, p->pFrom->zName); - } - return zWhere; -} -#endif - -/* -** Generate the text of a WHERE expression which can be used to select all -** temporary triggers on table pTab from the sqlite_temp_master table. If -** table pTab has no temporary triggers, or is itself stored in the -** temporary database, NULL is returned. -*/ -static char *whereTempTriggers(Parse *pParse, Table *pTab){ - Trigger *pTrig; - char *zWhere = 0; - const Schema *pTempSchema = pParse->db->aDb[1].pSchema; /* Temp db schema */ - - /* If the table is not located in the temp-db (in which case NULL is - ** returned, loop through the tables list of triggers. For each trigger - ** that is not part of the temp-db schema, add a clause to the WHERE - ** expression being built up in zWhere. - */ - if( pTab->pSchema!=pTempSchema ){ - sqlite3 *db = pParse->db; - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - if( pTrig->pSchema==pTempSchema ){ - zWhere = whereOrName(db, zWhere, pTrig->zName); - } - } - } - if( zWhere ){ - char *zNew = sqlite3MPrintf(pParse->db, "type='trigger' AND (%s)", zWhere); - sqlite3DbFree(pParse->db, zWhere); - zWhere = zNew; - } - return zWhere; -} - -/* -** Generate code to drop and reload the internal representation of table -** pTab from the database, including triggers and temporary triggers. -** Argument zName is the name of the table in the database schema at -** the time the generated code is executed. This can be different from -** pTab->zName if this function is being called to code part of an -** "ALTER TABLE RENAME TO" statement. -*/ -static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ - Vdbe *v; - char *zWhere; - int iDb; /* Index of database containing pTab */ -#ifndef SQLITE_OMIT_TRIGGER - Trigger *pTrig; -#endif - - v = sqlite3GetVdbe(pParse); - if( NEVER(v==0) ) return; - assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); - iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - assert( iDb>=0 ); - -#ifndef SQLITE_OMIT_TRIGGER - /* Drop any table triggers from the internal schema. */ - for(pTrig=sqlite3TriggerList(pParse, pTab); pTrig; pTrig=pTrig->pNext){ - int iTrigDb = sqlite3SchemaToIndex(pParse->db, pTrig->pSchema); - assert( iTrigDb==iDb || iTrigDb==1 ); - sqlite3VdbeAddOp4(v, OP_DropTrigger, iTrigDb, 0, 0, pTrig->zName, 0); - } -#endif - - /* Drop the table and index from the internal schema. */ - sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); - - /* Reload the table, index and permanent trigger schemas. */ - zWhere = sqlite3MPrintf(pParse->db, "tbl_name=%Q", zName); - if( !zWhere ) return; - sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); - -#ifndef SQLITE_OMIT_TRIGGER - /* Now, if the table is not stored in the temp database, reload any temp - ** triggers. Don't use IN(...) in case SQLITE_OMIT_SUBQUERY is defined. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3VdbeAddParseSchemaOp(v, 1, zWhere); - } -#endif -} - /* ** Parameter zName is the name of a table that is about to be altered ** (either with ALTER TABLE ... RENAME TO or ALTER TABLE ... ADD COLUMN). @@ -93645,17 +116744,98 @@ static void reloadTableSchema(Parse *pParse, Table *pTab, const char *zName){ ** ** Or, if zName is not a system table, zero is returned. */ -static int isSystemTable(Parse *pParse, const char *zName){ - if( sqlite3Strlen30(zName)>6 && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ - sqlite3ErrorMsg(pParse, "table %s may not be altered", zName); +static int isAlterableTable(Parse *pParse, Table *pTab){ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) +#ifndef SQLITE_OMIT_VIRTUALTABLE + || (pTab->tabFlags & TF_Eponymous)!=0 + || ( (pTab->tabFlags & TF_Shadow)!=0 + && sqlite3ReadOnlyShadowTables(pParse->db) + ) +#endif + ){ + sqlite3ErrorMsg(pParse, "table %s may not be altered", pTab->zName); return 1; } return 0; } /* -** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" -** command. +** Generate code to verify that the schemas of database zDb and, if +** bTemp is not true, database "temp", can still be parsed. This is +** called at the end of the generation of an ALTER TABLE ... RENAME ... +** statement to ensure that the operation has not rendered any schema +** objects unusable. +*/ +static void renameTestSchema( + Parse *pParse, /* Parse context */ + const char *zDb, /* Name of db to verify schema of */ + int bTemp, /* True if this is the temp db */ + const char *zWhen, /* "when" part of error message */ + int bNoDQS /* Do not allow DQS in the schema */ +){ + pParse->colNamesSet = 1; + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM \"%w\"." LEGACY_SCHEMA_TABLE " " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, %d, %Q, %d)=NULL ", + zDb, + zDb, bTemp, zWhen, bNoDQS + ); + + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "SELECT 1 " + "FROM temp." LEGACY_SCHEMA_TABLE " " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + " AND sqlite_rename_test(%Q, sql, type, name, 1, %Q, %d)=NULL ", + zDb, zWhen, bNoDQS + ); + } +} + +/* +** Generate VM code to replace any double-quoted strings (but not double-quoted +** identifiers) within the "sql" column of the sqlite_schema table in +** database zDb with their single-quoted equivalents. If argument bTemp is +** not true, similarly update all SQL statements in the sqlite_schema table +** of the temp db. +*/ +static void renameFixQuotes(Parse *pParse, const char *zDb, int bTemp){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE + " SET sql = sqlite_rename_quotefix(%Q, sql)" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" , zDb, zDb + ); + if( bTemp==0 ){ + sqlite3NestedParse(pParse, + "UPDATE temp." LEGACY_SCHEMA_TABLE + " SET sql = sqlite_rename_quotefix('temp', sql)" + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + " AND sql NOT LIKE 'create virtual%%'" + ); + } +} + +/* +** Generate code to reload the schema for database iDb. And, if iDb!=1, for +** the temp database as well. +*/ +static void renameReloadSchema(Parse *pParse, int iDb, u16 p5){ + Vdbe *v = pParse->pVdbe; + if( v ){ + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, iDb, 0, p5); + if( iDb!=1 ) sqlite3VdbeAddParseSchemaOp(pParse->pVdbe, 1, 0, p5); + } +} + +/* +** Generate code to implement the "ALTER TABLE xxx RENAME TO yyy" +** command. */ SQLITE_PRIVATE void sqlite3AlterRenameTable( Parse *pParse, /* Parser context. */ @@ -93665,18 +116845,13 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( int iDb; /* Database that contains the table */ char *zDb; /* Name of database iDb */ Table *pTab; /* Table being renamed */ - char *zName = 0; /* NULL-terminated version of pName */ + char *zName = 0; /* NULL-terminated version of pName */ sqlite3 *db = pParse->db; /* Database connection */ int nTabName; /* Number of UTF-8 characters in zTabName */ const char *zTabName; /* Original name of the table */ Vdbe *v; -#ifndef SQLITE_OMIT_TRIGGER - char *zWhere = 0; /* Where clause to locate temp triggers */ -#endif VTable *pVTab = 0; /* Non-zero if this is a v-tab with an xRename() */ - int savedDbFlags; /* Saved value of db->flags */ - savedDbFlags = db->flags; if( NEVER(db->mallocFailed) ) goto exit_rename_table; assert( pSrc->nSrc==1 ); assert( sqlite3BtreeHoldsAllMutexes(pParse->db) ); @@ -93684,8 +116859,7 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); if( !pTab ) goto exit_rename_table; iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - zDb = db->aDb[iDb].zName; - db->flags |= SQLITE_PreferBuiltin; + zDb = db->aDb[iDb].zDbSName; /* Get a NULL terminated version of the new table name. */ zName = sqlite3NameFromToken(db, pName); @@ -93694,8 +116868,11 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( /* Check that a table or index named 'zName' does not already exist ** in database iDb. If so, this is an error. */ - if( sqlite3FindTable(db, zName, zDb) || sqlite3FindIndex(db, zName, zDb) ){ - sqlite3ErrorMsg(pParse, + if( sqlite3FindTable(db, zName, zDb) + || sqlite3FindIndex(db, zName, zDb) + || sqlite3IsShadowTableOf(db, pTab, zName) + ){ + sqlite3ErrorMsg(pParse, "there is already another table or index with this name: %s", zName); goto exit_rename_table; } @@ -93703,15 +116880,15 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( /* Make sure it is not a system table being altered, or a reserved name ** that the table is being renamed to. */ - if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ goto exit_rename_table; } - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ goto - exit_rename_table; + if( SQLITE_OK!=sqlite3CheckObjectName(pParse,zName,"table",zName) ){ + goto exit_rename_table; } #ifndef SQLITE_OMIT_VIEW - if( pTab->pSelect ){ + if( IsView(pTab) ){ sqlite3ErrorMsg(pParse, "view %s may not be altered", pTab->zName); goto exit_rename_table; } @@ -93736,17 +116913,73 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( } #endif - /* Begin a transaction for database iDb. - ** Then modify the schema cookie (since the ALTER TABLE modifies the - ** schema). Open a statement transaction if the table is a virtual - ** table. - */ + /* Begin a transaction for database iDb. Then modify the schema cookie + ** (since the ALTER TABLE modifies the schema). Call sqlite3MayAbort(), + ** as the scalar functions (e.g. sqlite_rename_table()) invoked by the + ** nested SQL may raise an exception. */ v = sqlite3GetVdbe(pParse); if( v==0 ){ goto exit_rename_table; } - sqlite3BeginWriteOperation(pParse, pVTab!=0, iDb); - sqlite3ChangeCookie(pParse, iDb); + sqlite3MayAbort(pParse); + + /* figure out how many UTF-8 characters are in zName */ + zTabName = pTab->zName; + nTabName = sqlite3Utf8CharLen(zTabName, -1); + + /* Rewrite all CREATE TABLE, INDEX, TRIGGER or VIEW statements in + ** the schema to use the new table name. */ + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, %d) " + "WHERE (type!='index' OR tbl_name=%Q COLLATE nocase)" + "AND name NOT LIKE 'sqliteX_%%' ESCAPE 'X'" + , zDb, zDb, zTabName, zName, (iDb==1), zTabName + ); + + /* Update the tbl_name and name columns of the sqlite_schema table + ** as required. */ + sqlite3NestedParse(pParse, + "UPDATE %Q." LEGACY_SCHEMA_TABLE " SET " + "tbl_name = %Q, " + "name = CASE " + "WHEN type='table' THEN %Q " + "WHEN name LIKE 'sqliteX_autoindex%%' ESCAPE 'X' " + " AND type='index' THEN " + "'sqlite_autoindex_' || %Q || substr(name,%d+18) " + "ELSE name END " + "WHERE tbl_name=%Q COLLATE nocase AND " + "(type='table' OR type='index' OR type='trigger');", + zDb, + zName, zName, zName, + nTabName, zTabName + ); + +#ifndef SQLITE_OMIT_AUTOINCREMENT + /* If the sqlite_sequence table exists in this database, then update + ** it with the new table name. + */ + if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ + sqlite3NestedParse(pParse, + "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", + zDb, zName, pTab->zName); + } +#endif + + /* If the table being renamed is not itself part of the temp database, + ** edit view and trigger definitions within the temp database + ** as required. */ + if( iDb!=1 ){ + sqlite3NestedParse(pParse, + "UPDATE sqlite_temp_schema SET " + "sql = sqlite_rename_table(%Q, type, name, sql, %Q, %Q, 1), " + "tbl_name = " + "CASE WHEN tbl_name=%Q COLLATE nocase AND " + " sqlite_rename_test(%Q, sql, type, name, 1, 'after rename', 0) " + "THEN %Q ELSE tbl_name END " + "WHERE type IN ('view', 'trigger')" + , zDb, zTabName, zName, zTabName, zDb, zName); + } /* If this is a virtual table, invoke the xRename() function if ** one is defined. The xRename() callback will modify the names @@ -93758,99 +116991,31 @@ SQLITE_PRIVATE void sqlite3AlterRenameTable( int i = ++pParse->nMem; sqlite3VdbeLoadString(v, i, zName); sqlite3VdbeAddOp4(v, OP_VRename, i, 0, 0,(const char*)pVTab, P4_VTAB); - sqlite3MayAbort(pParse); } #endif - /* figure out how many UTF-8 characters are in zName */ - zTabName = pTab->zName; - nTabName = sqlite3Utf8CharLen(zTabName, -1); - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - /* If foreign-key support is enabled, rewrite the CREATE TABLE - ** statements corresponding to all child tables of foreign key constraints - ** for which the renamed table is the parent table. */ - if( (zWhere=whereForeignKeys(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".%s SET " - "sql = sqlite_rename_parent(sql, %Q, %Q) " - "WHERE %s;", zDb, SCHEMA_TABLE(iDb), zTabName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } - } -#endif - - /* Modify the sqlite_master table to use the new table name. */ - sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET " -#ifdef SQLITE_OMIT_TRIGGER - "sql = sqlite_rename_table(sql, %Q), " -#else - "sql = CASE " - "WHEN type = 'trigger' THEN sqlite_rename_trigger(sql, %Q)" - "ELSE sqlite_rename_table(sql, %Q) END, " -#endif - "tbl_name = %Q, " - "name = CASE " - "WHEN type='table' THEN %Q " - "WHEN name LIKE 'sqlite_autoindex%%' AND type='index' THEN " - "'sqlite_autoindex_' || %Q || substr(name,%d+18) " - "ELSE name END " - "WHERE tbl_name=%Q COLLATE nocase AND " - "(type='table' OR type='index' OR type='trigger');", - zDb, SCHEMA_TABLE(iDb), zName, zName, zName, -#ifndef SQLITE_OMIT_TRIGGER - zName, -#endif - zName, nTabName, zTabName - ); - -#ifndef SQLITE_OMIT_AUTOINCREMENT - /* If the sqlite_sequence table exists in this database, then update - ** it with the new table name. - */ - if( sqlite3FindTable(db, "sqlite_sequence", zDb) ){ - sqlite3NestedParse(pParse, - "UPDATE \"%w\".sqlite_sequence set name = %Q WHERE name = %Q", - zDb, zName, pTab->zName); - } -#endif - -#ifndef SQLITE_OMIT_TRIGGER - /* If there are TEMP triggers on this table, modify the sqlite_temp_master - ** table. Don't do this if the table being ALTERed is itself located in - ** the temp database. - */ - if( (zWhere=whereTempTriggers(pParse, pTab))!=0 ){ - sqlite3NestedParse(pParse, - "UPDATE sqlite_temp_master SET " - "sql = sqlite_rename_trigger(sql, %Q), " - "tbl_name = %Q " - "WHERE %s;", zName, zName, zWhere); - sqlite3DbFree(db, zWhere); - } -#endif - -#if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - if( db->flags&SQLITE_ForeignKeys ){ - FKey *p; - for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - Table *pFrom = p->pFrom; - if( pFrom!=pTab ){ - reloadTableSchema(pParse, p->pFrom, pFrom->zName); - } - } - } -#endif - - /* Drop and reload the internal table schema. */ - reloadTableSchema(pParse, pTab, zName); + renameReloadSchema(pParse, iDb, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iDb==1, "after rename", 0); exit_rename_table: sqlite3SrcListDelete(db, pSrc); sqlite3DbFree(db, zName); - db->flags = savedDbFlags; +} + +/* +** Write code that will raise an error if the table described by +** zDb and zTab is not empty. +*/ +static void sqlite3ErrorIfNotEmpty( + Parse *pParse, /* Parsing context */ + const char *zDb, /* Schema holding the table */ + const char *zTab, /* Table to check for empty */ + const char *zErr /* Error message text */ +){ + sqlite3NestedParse(pParse, + "SELECT raise(ABORT,%Q) FROM \"%w\".\"%w\"", + zErr, zDb, zTab + ); } /* @@ -93871,20 +117036,22 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ Column *pCol; /* The new column */ Expr *pDflt; /* Default value for the new column */ sqlite3 *db; /* The database connection; */ - Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ + Vdbe *v; /* The prepared statement under construction */ + int r1; /* Temporary registers */ db = pParse->db; - if( pParse->nErr || db->mallocFailed ) return; - assert( v!=0 ); + assert( db->pParse==pParse ); + if( pParse->nErr ) return; + assert( db->mallocFailed==0 ); pNew = pParse->pNewTable; assert( pNew ); assert( sqlite3BtreeHoldsAllMutexes(db) ); iDb = sqlite3SchemaToIndex(db, pNew->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; zTab = &pNew->zName[16]; /* Skip the "sqlite_altertab_" prefix on the name */ pCol = &pNew->aCol[pNew->nCol-1]; - pDflt = pCol->pDflt; + pDflt = sqlite3ColumnExpr(pNew, pCol); pTab = sqlite3FindTable(db, zTab, zDb); assert( pTab ); @@ -93895,14 +117062,6 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ } #endif - /* If the default value for the new column was specified with a - ** literal NULL, then set pDflt to 0. This simplifies checking - ** for an SQL NULL default below. - */ - assert( pDflt==0 || pDflt->op==TK_SPAN ); - if( pDflt && pDflt->pLeft->op==TK_NULL ){ - pDflt = 0; - } /* Check that the new column is not specified as PRIMARY KEY or UNIQUE. ** If there is a NOT NULL constraint, then the default value for the @@ -93913,84 +117072,125 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ return; } if( pNew->pIndex ){ - sqlite3ErrorMsg(pParse, "Cannot add a UNIQUE column"); + sqlite3ErrorMsg(pParse, + "Cannot add a UNIQUE column"); return; } - if( (db->flags&SQLITE_ForeignKeys) && pNew->pFKey && pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a REFERENCES column with non-NULL default value"); - return; - } - if( pCol->notNull && !pDflt ){ - sqlite3ErrorMsg(pParse, - "Cannot add a NOT NULL column with default value NULL"); - return; + if( (pCol->colFlags & COLFLAG_GENERATED)==0 ){ + /* If the default value for the new column was specified with a + ** literal NULL, then set pDflt to 0. This simplifies checking + ** for an SQL NULL default below. + */ + assert( pDflt==0 || pDflt->op==TK_SPAN ); + if( pDflt && pDflt->pLeft->op==TK_NULL ){ + pDflt = 0; + } + assert( IsOrdinaryTable(pNew) ); + if( (db->flags&SQLITE_ForeignKeys) && pNew->u.tab.pFKey && pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a REFERENCES column with non-NULL default value"); + } + if( pCol->notNull && !pDflt ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a NOT NULL column with default value NULL"); + } + + + /* Ensure the default expression is something that sqlite3ValueFromExpr() + ** can handle (i.e. not CURRENT_TIME etc.) + */ + if( pDflt ){ + sqlite3_value *pVal = 0; + int rc; + rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); + assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); + if( rc!=SQLITE_OK ){ + assert( db->mallocFailed == 1 ); + return; + } + if( !pVal ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, + "Cannot add a column with non-constant default"); + } + sqlite3ValueFree(pVal); + } + }else if( pCol->colFlags & COLFLAG_STORED ){ + sqlite3ErrorIfNotEmpty(pParse, zDb, zTab, "cannot add a STORED column"); } - /* Ensure the default expression is something that sqlite3ValueFromExpr() - ** can handle (i.e. not CURRENT_TIME etc.) - */ - if( pDflt ){ - sqlite3_value *pVal = 0; - int rc; - rc = sqlite3ValueFromExpr(db, pDflt, SQLITE_UTF8, SQLITE_AFF_BLOB, &pVal); - assert( rc==SQLITE_OK || rc==SQLITE_NOMEM ); - if( rc!=SQLITE_OK ){ - assert( db->mallocFailed == 1 ); - return; - } - if( !pVal ){ - sqlite3ErrorMsg(pParse, "Cannot add a column with non-constant default"); - return; - } - sqlite3ValueFree(pVal); - } /* Modify the CREATE TABLE statement. */ zCol = sqlite3DbStrNDup(db, (char*)pColDef->z, pColDef->n); if( zCol ){ char *zEnd = &zCol[pColDef->n-1]; - int savedDbFlags = db->flags; while( zEnd>zCol && (*zEnd==';' || sqlite3Isspace(*zEnd)) ){ *zEnd-- = '\0'; } - db->flags |= SQLITE_PreferBuiltin; - sqlite3NestedParse(pParse, - "UPDATE \"%w\".%s SET " - "sql = substr(sql,1,%d) || ', ' || %Q || substr(sql,%d) " - "WHERE type = 'table' AND name = %Q", - zDb, SCHEMA_TABLE(iDb), pNew->addColOffset, zCol, pNew->addColOffset+1, + /* substr() operations on characters, but addColOffset is in bytes. So we + ** have to use printf() to translate between these units: */ + assert( IsOrdinaryTable(pTab) ); + assert( IsOrdinaryTable(pNew) ); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = printf('%%.%ds, ',sql) || %Q" + " || substr(sql,1+length(printf('%%.%ds',sql))) " + "WHERE type = 'table' AND name = %Q", + zDb, pNew->u.tab.addColOffset, zCol, pNew->u.tab.addColOffset, zTab ); sqlite3DbFree(db, zCol); - db->flags = savedDbFlags; } - /* If the default value of the new column is NULL, then the file - ** format to 2. If the default value of the new column is not NULL, - ** the file format be 3. Back when this feature was first added - ** in 2006, we went to the trouble to upgrade the file format to the - ** minimum support values. But 10-years on, we can assume that all - ** extent versions of SQLite support file-format 4, so we always and - ** unconditionally upgrade to 4. - */ - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, - SQLITE_MAX_FILE_FORMAT); + v = sqlite3GetVdbe(pParse); + if( v ){ + /* Make sure the schema version is at least 3. But do not upgrade + ** from less than 3 to 4, as that will corrupt any preexisting DESC + ** index. + */ + r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_ReadCookie, iDb, r1, BTREE_FILE_FORMAT); + sqlite3VdbeUsesBtree(v, iDb); + sqlite3VdbeAddOp2(v, OP_AddImm, r1, -2); + sqlite3VdbeAddOp2(v, OP_IfPos, r1, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_FILE_FORMAT, 3); + sqlite3ReleaseTempReg(pParse, r1); - /* Reload the schema of the modified table. */ - reloadTableSchema(pParse, pTab, pTab->zName); + /* Reload the table definition */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterAdd); + + /* Verify that constraints are still satisfied */ + if( pNew->pCheck!=0 + || (pCol->notNull && (pCol->colFlags & COLFLAG_GENERATED)!=0) + || (pTab->tabFlags & TF_Strict)!=0 + ){ + sqlite3NestedParse(pParse, + "SELECT CASE WHEN quick_check GLOB 'CHECK*'" + " THEN raise(ABORT,'CHECK constraint failed')" + " WHEN quick_check GLOB 'non-* value in*'" + " THEN raise(ABORT,'type mismatch on DEFAULT')" + " ELSE raise(ABORT,'NOT NULL constraint failed')" + " END" + " FROM pragma_quick_check(%Q,%Q)" + " WHERE quick_check GLOB 'CHECK*'" + " OR quick_check GLOB 'NULL*'" + " OR quick_check GLOB 'non-* value in*'", + zTab, zDb + ); + } + } } /* ** This function is called by the parser after the table-name in -** an "ALTER TABLE ADD" statement is parsed. Argument +** an "ALTER TABLE ADD" statement is parsed. Argument ** pSrc is the full-name of the table being altered. ** ** This routine makes a (partial) copy of the Table structure ** for the table being altered and sets Parse.pNewTable to point ** to it. Routines called by the parser as the column definition -** is parsed (i.e. sqlite3AddColumn()) add the new Column data to -** the copy. The copy of the Table structure is deleted by tokenize.c +** is parsed (i.e. sqlite3AddColumn()) add the new Column data to +** the copy. The copy of the Table structure is deleted by tokenize.c ** after parsing is finished. ** ** Routine sqlite3AlterFinishAddColumn() will be called to complete @@ -93999,7 +117199,6 @@ SQLITE_PRIVATE void sqlite3AlterFinishAddColumn(Parse *pParse, Token *pColDef){ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ Table *pNew; Table *pTab; - Vdbe *v; int iDb; int i; int nAlloc; @@ -94020,15 +117219,17 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ #endif /* Make sure this is not an attempt to ALTER a view. */ - if( pTab->pSelect ){ + if( IsView(pTab) ){ sqlite3ErrorMsg(pParse, "Cannot add a column to a view"); goto exit_begin_add_column; } - if( SQLITE_OK!=isSystemTable(pParse, pTab->zName) ){ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ){ goto exit_begin_add_column; } - assert( pTab->addColOffset>0 ); + sqlite3MayAbort(pParse); + assert( IsOrdinaryTable(pTab) ); + assert( pTab->u.tab.addColOffset>0 ); iDb = sqlite3SchemaToIndex(db, pTab->pSchema); /* Put a copy of the Table struct in Parse.pNewTable for the @@ -94041,7 +117242,7 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ pNew = (Table*)sqlite3DbMallocZero(db, sizeof(Table)); if( !pNew ) goto exit_begin_add_column; pParse->pNewTable = pNew; - pNew->nRef = 1; + pNew->nTabRef = 1; pNew->nCol = pTab->nCol; assert( pNew->nCol>0 ); nAlloc = (((pNew->nCol-1)/8)*8)+8; @@ -94055,24 +117256,1779 @@ SQLITE_PRIVATE void sqlite3AlterBeginAddColumn(Parse *pParse, SrcList *pSrc){ memcpy(pNew->aCol, pTab->aCol, sizeof(Column)*pNew->nCol); for(i=0; inCol; i++){ Column *pCol = &pNew->aCol[i]; - pCol->zName = sqlite3DbStrDup(db, pCol->zName); - pCol->zColl = 0; - pCol->pDflt = 0; + pCol->zCnName = sqlite3DbStrDup(db, pCol->zCnName); + pCol->hName = sqlite3StrIHash(pCol->zCnName); } + assert( IsOrdinaryTable(pNew) ); + pNew->u.tab.pDfltList = sqlite3ExprListDup(db, pTab->u.tab.pDfltList, 0); pNew->pSchema = db->aDb[iDb].pSchema; - pNew->addColOffset = pTab->addColOffset; - pNew->nRef = 1; - - /* Begin a transaction and increment the schema cookie. */ - sqlite3BeginWriteOperation(pParse, 0, iDb); - v = sqlite3GetVdbe(pParse); - if( !v ) goto exit_begin_add_column; - sqlite3ChangeCookie(pParse, iDb); + pNew->u.tab.addColOffset = pTab->u.tab.addColOffset; + assert( pNew->nTabRef==1 ); exit_begin_add_column: sqlite3SrcListDelete(db, pSrc); return; } + +/* +** Parameter pTab is the subject of an ALTER TABLE ... RENAME COLUMN +** command. This function checks if the table is a view or virtual +** table (columns of views or virtual tables may not be renamed). If so, +** it loads an error message into pParse and returns non-zero. +** +** Or, if pTab is not a view or virtual table, zero is returned. +*/ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) +static int isRealTable(Parse *pParse, Table *pTab, int bDrop){ + const char *zType = 0; +#ifndef SQLITE_OMIT_VIEW + if( IsView(pTab) ){ + zType = "view"; + } +#endif +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pTab) ){ + zType = "virtual table"; + } +#endif + if( zType ){ + sqlite3ErrorMsg(pParse, "cannot %s %s \"%s\"", + (bDrop ? "drop column from" : "rename columns of"), + zType, pTab->zName + ); + return 1; + } + return 0; +} +#else /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ +# define isRealTable(x,y,z) (0) +#endif + +/* +** Handles the following parser reduction: +** +** cmd ::= ALTER TABLE pSrc RENAME COLUMN pOld TO pNew +*/ +SQLITE_PRIVATE void sqlite3AlterRenameColumn( + Parse *pParse, /* Parsing context */ + SrcList *pSrc, /* Table being altered. pSrc->nSrc==1 */ + Token *pOld, /* Name of column being changed */ + Token *pNew /* New column name */ +){ + sqlite3 *db = pParse->db; /* Database connection */ + Table *pTab; /* Table being updated */ + int iCol; /* Index of column being renamed */ + char *zOld = 0; /* Old column name */ + char *zNew = 0; /* New column name */ + const char *zDb; /* Name of schema containing the table */ + int iSchema; /* Index of the schema */ + int bQuote; /* True to quote the new name */ + + /* Locate the table to be altered */ + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_rename_column; + + /* Cannot alter a system table */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_rename_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 0) ) goto exit_rename_column; + + /* Which schema holds the table to be altered */ + iSchema = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iSchema>=0 ); + zDb = db->aDb[iSchema].zDbSName; + +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, 0) ){ + goto exit_rename_column; + } +#endif + + /* Make sure the old name really is a column name in the table to be + ** altered. Set iCol to be the index of the column being renamed */ + zOld = sqlite3NameFromToken(db, pOld); + if( !zOld ) goto exit_rename_column; + for(iCol=0; iColnCol; iCol++){ + if( 0==sqlite3StrICmp(pTab->aCol[iCol].zCnName, zOld) ) break; + } + if( iCol==pTab->nCol ){ + sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pOld); + goto exit_rename_column; + } + + /* Ensure the schema contains no double-quoted strings */ + renameTestSchema(pParse, zDb, iSchema==1, "", 0); + renameFixQuotes(pParse, zDb, iSchema==1); + + /* Do the rename operation using a recursive UPDATE statement that + ** uses the sqlite_rename_column() SQL function to compute the new + ** CREATE statement text for the sqlite_schema table. + */ + sqlite3MayAbort(pParse); + zNew = sqlite3NameFromToken(db, pNew); + if( !zNew ) goto exit_rename_column; + assert( pNew->n>0 ); + bQuote = sqlite3Isquote(pNew->z[0]); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, %d) " + "WHERE name NOT LIKE 'sqliteX_%%' ESCAPE 'X' " + " AND (type != 'index' OR tbl_name = %Q)", + zDb, + zDb, pTab->zName, iCol, zNew, bQuote, iSchema==1, + pTab->zName + ); + + sqlite3NestedParse(pParse, + "UPDATE temp." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_rename_column(sql, type, name, %Q, %Q, %d, %Q, %d, 1) " + "WHERE type IN ('trigger', 'view')", + zDb, pTab->zName, iCol, zNew, bQuote + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iSchema, INITFLAG_AlterRename); + renameTestSchema(pParse, zDb, iSchema==1, "after rename", 1); + + exit_rename_column: + sqlite3SrcListDelete(db, pSrc); + sqlite3DbFree(db, zOld); + sqlite3DbFree(db, zNew); + return; +} + +/* +** Each RenameToken object maps an element of the parse tree into +** the token that generated that element. The parse tree element +** might be one of: +** +** * A pointer to an Expr that represents an ID +** * The name of a table column in Column.zName +** +** A list of RenameToken objects can be constructed during parsing. +** Each new object is created by sqlite3RenameTokenMap(). +** As the parse tree is transformed, the sqlite3RenameTokenRemap() +** routine is used to keep the mapping current. +** +** After the parse finishes, renameTokenFind() routine can be used +** to look up the actual token value that created some element in +** the parse tree. +*/ +struct RenameToken { + const void *p; /* Parse tree element created by token t */ + Token t; /* The token that created parse tree element p */ + RenameToken *pNext; /* Next is a list of all RenameToken objects */ +}; + +/* +** The context of an ALTER TABLE RENAME COLUMN operation that gets passed +** down into the Walker. +*/ +typedef struct RenameCtx RenameCtx; +struct RenameCtx { + RenameToken *pList; /* List of tokens to overwrite */ + int nList; /* Number of tokens in pList */ + int iCol; /* Index of column being renamed */ + Table *pTab; /* Table being ALTERed */ + const char *zOld; /* Old column name */ +}; + +#ifdef SQLITE_DEBUG +/* +** This function is only for debugging. It performs two tasks: +** +** 1. Checks that pointer pPtr does not already appear in the +** rename-token list. +** +** 2. Dereferences each pointer in the rename-token list. +** +** The second is most effective when debugging under valgrind or +** address-sanitizer or similar. If any of these pointers no longer +** point to valid objects, an exception is raised by the memory-checking +** tool. +** +** The point of this is to prevent comparisons of invalid pointer values. +** Even though this always seems to work, it is undefined according to the +** C standard. Example of undefined comparison: +** +** sqlite3_free(x); +** if( x==y ) ... +** +** Technically, as x no longer points into a valid object or to the byte +** following a valid object, it may not be used in comparison operations. +*/ +static void renameTokenCheckAll(Parse *pParse, const void *pPtr){ + assert( pParse==pParse->db->pParse ); + assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); + if( pParse->nErr==0 ){ + const RenameToken *p; + u32 i = 1; + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p ){ + assert( p->p!=pPtr ); + i += *(u8*)(p->p) | 1; + } + } + assert( i>0 ); + } +} +#else +# define renameTokenCheckAll(x,y) +#endif + +/* +** Remember that the parser tree element pPtr was created using +** the token pToken. +** +** In other words, construct a new RenameToken object and add it +** to the list of RenameToken objects currently being built up +** in pParse->pRename. +** +** The pPtr argument is returned so that this routine can be used +** with tail recursion in tokenExpr() routine, for a small performance +** improvement. +*/ +SQLITE_PRIVATE const void *sqlite3RenameTokenMap( + Parse *pParse, + const void *pPtr, + const Token *pToken +){ + RenameToken *pNew; + assert( pPtr || pParse->db->mallocFailed ); + renameTokenCheckAll(pParse, pPtr); + if( ALWAYS(pParse->eParseMode!=PARSE_MODE_UNMAP) ){ + pNew = sqlite3DbMallocZero(pParse->db, sizeof(RenameToken)); + if( pNew ){ + pNew->p = pPtr; + pNew->t = *pToken; + pNew->pNext = pParse->pRename; + pParse->pRename = pNew; + } + } + + return pPtr; +} + +/* +** It is assumed that there is already a RenameToken object associated +** with parse tree element pFrom. This function remaps the associated token +** to parse tree element pTo. +*/ +SQLITE_PRIVATE void sqlite3RenameTokenRemap(Parse *pParse, const void *pTo, const void *pFrom){ + RenameToken *p; + renameTokenCheckAll(pParse, pTo); + for(p=pParse->pRename; p; p=p->pNext){ + if( p->p==pFrom ){ + p->p = pTo; + break; + } + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapExprCb(Walker *pWalker, Expr *pExpr){ + Parse *pParse = pWalker->pParse; + sqlite3RenameTokenRemap(pParse, 0, (const void*)pExpr); + if( ExprUseYTab(pExpr) ){ + sqlite3RenameTokenRemap(pParse, 0, (const void*)&pExpr->y.pTab); + } + return WRC_Continue; +} + +/* +** Iterate through the Select objects that are part of WITH clauses attached +** to select statement pSelect. +*/ +static void renameWalkWith(Walker *pWalker, Select *pSelect){ + With *pWith = pSelect->pWith; + if( pWith ){ + Parse *pParse = pWalker->pParse; + int i; + With *pCopy = 0; + assert( pWith->nCte>0 ); + if( (pWith->a[0].pSelect->selFlags & SF_Expanded)==0 ){ + /* Push a copy of the With object onto the with-stack. We use a copy + ** here as the original will be expanded and resolved (flags SF_Expanded + ** and SF_Resolved) below. And the parser code that uses the with-stack + ** fails if the Select objects on it have already been expanded and + ** resolved. */ + pCopy = sqlite3WithDup(pParse->db, pWith); + pCopy = sqlite3WithPush(pParse, pCopy, 1); + } + for(i=0; inCte; i++){ + Select *p = pWith->a[i].pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + if( pCopy ) sqlite3SelectPrep(sNC.pParse, p, &sNC); + if( sNC.pParse->db->mallocFailed ) return; + sqlite3WalkSelect(pWalker, p); + sqlite3RenameExprlistUnmap(pParse, pWith->a[i].pCols); + } + if( pCopy && pParse->pWith==pCopy ){ + pParse->pWith = pCopy->pOuter; + } + } +} + +/* +** Unmap all tokens in the IdList object passed as the second argument. +*/ +static void unmapColumnIdlistNames( + Parse *pParse, + const IdList *pIdList +){ + int ii; + assert( pIdList!=0 ); + for(ii=0; iinId; ii++){ + sqlite3RenameTokenRemap(pParse, 0, (const void*)pIdList->a[ii].zName); + } +} + +/* +** Walker callback used by sqlite3RenameExprUnmap(). +*/ +static int renameUnmapSelectCb(Walker *pWalker, Select *p){ + Parse *pParse = pWalker->pParse; + int i; + if( pParse->nErr ) return WRC_Abort; + testcase( p->selFlags & SF_View ); + testcase( p->selFlags & SF_CopyCte ); + if( p->selFlags & (SF_View|SF_CopyCte) ){ + return WRC_Prune; + } + if( ALWAYS(p->pEList) ){ + ExprList *pList = p->pEList; + for(i=0; inExpr; i++){ + if( pList->a[i].zEName && pList->a[i].fg.eEName==ENAME_NAME ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pList->a[i].zEName); + } + } + } + if( ALWAYS(p->pSrc) ){ /* Every Select as a SrcList, even if it is empty */ + SrcList *pSrc = p->pSrc; + for(i=0; inSrc; i++){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pSrc->a[i].zName); + if( pSrc->a[i].fg.isUsing==0 ){ + sqlite3WalkExpr(pWalker, pSrc->a[i].u3.pOn); + }else{ + unmapColumnIdlistNames(pParse, pSrc->a[i].u3.pUsing); + } + } + } + + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** Remove all nodes that are part of expression pExpr from the rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprUnmap(Parse *pParse, Expr *pExpr){ + u8 eMode = pParse->eParseMode; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sWalker.xSelectCallback = renameUnmapSelectCb; + pParse->eParseMode = PARSE_MODE_UNMAP; + sqlite3WalkExpr(&sWalker, pExpr); + pParse->eParseMode = eMode; +} + +/* +** Remove all nodes that are part of expression-list pEList from the +** rename list. +*/ +SQLITE_PRIVATE void sqlite3RenameExprlistUnmap(Parse *pParse, ExprList *pEList){ + if( pEList ){ + int i; + Walker sWalker; + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = renameUnmapExprCb; + sqlite3WalkExprList(&sWalker, pEList); + for(i=0; inExpr; i++){ + if( ALWAYS(pEList->a[i].fg.eEName==ENAME_NAME) ){ + sqlite3RenameTokenRemap(pParse, 0, (void*)pEList->a[i].zEName); + } + } + } +} + +/* +** Free the list of RenameToken objects given in the second argument +*/ +static void renameTokenFree(sqlite3 *db, RenameToken *pToken){ + RenameToken *pNext; + RenameToken *p; + for(p=pToken; p; p=pNext){ + pNext = p->pNext; + sqlite3DbFree(db, p); + } +} + +/* +** Search the Parse object passed as the first argument for a RenameToken +** object associated with parse tree element pPtr. If found, return a pointer +** to it. Otherwise, return NULL. +** +** If the second argument passed to this function is not NULL and a matching +** RenameToken object is found, remove it from the Parse object and add it to +** the list maintained by the RenameCtx object. +*/ +static RenameToken *renameTokenFind( + Parse *pParse, + struct RenameCtx *pCtx, + const void *pPtr +){ + RenameToken **pp; + if( NEVER(pPtr==0) ){ + return 0; + } + for(pp=&pParse->pRename; (*pp); pp=&(*pp)->pNext){ + if( (*pp)->p==pPtr ){ + RenameToken *pToken = *pp; + if( pCtx ){ + *pp = pToken->pNext; + pToken->pNext = pCtx->pList; + pCtx->pList = pToken; + pCtx->nList++; + } + return pToken; + } + } + return 0; +} + +/* +** This is a Walker select callback. It does nothing. It is only required +** because without a dummy callback, sqlite3WalkExpr() and similar do not +** descend into sub-select statements. +*/ +static int renameColumnSelectCb(Walker *pWalker, Select *p){ + if( p->selFlags & (SF_View|SF_CopyCte) ){ + testcase( p->selFlags & SF_View ); + testcase( p->selFlags & SF_CopyCte ); + return WRC_Prune; + } + renameWalkWith(pWalker, p); + return WRC_Continue; +} + +/* +** This is a Walker expression callback. +** +** For every TK_COLUMN node in the expression tree, search to see +** if the column being references is the column being renamed by an +** ALTER TABLE statement. If it is, then attach its associated +** RenameToken object to the list of RenameToken objects being +** constructed in RenameCtx object at pWalker->u.pRename. +*/ +static int renameColumnExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_TRIGGER + && pExpr->iColumn==p->iCol + && pWalker->pParse->pTriggerTab==p->pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + }else if( pExpr->op==TK_COLUMN + && pExpr->iColumn==p->iCol + && ALWAYS(ExprUseYTab(pExpr)) + && p->pTab==pExpr->y.pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)pExpr); + } + return WRC_Continue; +} + +/* +** The RenameCtx contains a list of tokens that reference a column that +** is being renamed by an ALTER TABLE statement. Return the "last" +** RenameToken in the RenameCtx and remove that RenameToken from the +** RenameContext. "Last" means the last RenameToken encountered when +** the input SQL is parsed from left to right. Repeated calls to this routine +** return all column name tokens in the order that they are encountered +** in the SQL statement. +*/ +static RenameToken *renameColumnTokenNext(RenameCtx *pCtx){ + RenameToken *pBest = pCtx->pList; + RenameToken *pToken; + RenameToken **pp; + + for(pToken=pBest->pNext; pToken; pToken=pToken->pNext){ + if( pToken->t.z>pBest->t.z ) pBest = pToken; + } + for(pp=&pCtx->pList; *pp!=pBest; pp=&(*pp)->pNext); + *pp = pBest->pNext; + + return pBest; +} + +/* +** An error occurred while parsing or otherwise processing a database +** object (either pParse->pNewTable, pNewIndex or pNewTrigger) as part of an +** ALTER TABLE RENAME COLUMN program. The error message emitted by the +** sub-routine is currently stored in pParse->zErrMsg. This function +** adds context to the error message and then stores it in pCtx. +*/ +static void renameColumnParseError( + sqlite3_context *pCtx, + const char *zWhen, + sqlite3_value *pType, + sqlite3_value *pObject, + Parse *pParse +){ + const char *zT = (const char*)sqlite3_value_text(pType); + const char *zN = (const char*)sqlite3_value_text(pObject); + char *zErr; + + zErr = sqlite3MPrintf(pParse->db, "error in %s %s%s%s: %s", + zT, zN, (zWhen[0] ? " " : ""), zWhen, + pParse->zErrMsg + ); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3DbFree(pParse->db, zErr); +} + +/* +** For each name in the the expression-list pEList (i.e. each +** pEList->a[i].zName) that matches the string in zOld, extract the +** corresponding rename-token from Parse object pParse and add it +** to the RenameCtx pCtx. +*/ +static void renameColumnElistNames( + Parse *pParse, + RenameCtx *pCtx, + const ExprList *pEList, + const char *zOld +){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + const char *zName = pEList->a[i].zEName; + if( ALWAYS(pEList->a[i].fg.eEName==ENAME_NAME) + && ALWAYS(zName!=0) + && 0==sqlite3_stricmp(zName, zOld) + ){ + renameTokenFind(pParse, pCtx, (const void*)zName); + } + } + } +} + +/* +** For each name in the the id-list pIdList (i.e. each pIdList->a[i].zName) +** that matches the string in zOld, extract the corresponding rename-token +** from Parse object pParse and add it to the RenameCtx pCtx. +*/ +static void renameColumnIdlistNames( + Parse *pParse, + RenameCtx *pCtx, + const IdList *pIdList, + const char *zOld +){ + if( pIdList ){ + int i; + for(i=0; inId; i++){ + const char *zName = pIdList->a[i].zName; + if( 0==sqlite3_stricmp(zName, zOld) ){ + renameTokenFind(pParse, pCtx, (const void*)zName); + } + } + } +} + + +/* +** Parse the SQL statement zSql using Parse object (*p). The Parse object +** is initialized by this function before it is used. +*/ +static int renameParseSql( + Parse *p, /* Memory to use for Parse object */ + const char *zDb, /* Name of schema SQL belongs to */ + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL to parse */ + int bTemp /* True if SQL is from temp schema */ +){ + int rc; + + sqlite3ParseObjectInit(p, db); + if( zSql==0 ){ + return SQLITE_NOMEM; + } + if( sqlite3StrNICmp(zSql,"CREATE ",7)!=0 ){ + return SQLITE_CORRUPT_BKPT; + } + db->init.iDb = bTemp ? 1 : sqlite3FindDbName(db, zDb); + p->eParseMode = PARSE_MODE_RENAME; + p->db = db; + p->nQueryLoop = 1; + rc = sqlite3RunParser(p, zSql); + if( db->mallocFailed ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK + && NEVER(p->pNewTable==0 && p->pNewIndex==0 && p->pNewTrigger==0) + ){ + rc = SQLITE_CORRUPT_BKPT; + } + +#ifdef SQLITE_DEBUG + /* Ensure that all mappings in the Parse.pRename list really do map to + ** a part of the input string. */ + if( rc==SQLITE_OK ){ + int nSql = sqlite3Strlen30(zSql); + RenameToken *pToken; + for(pToken=p->pRename; pToken; pToken=pToken->pNext){ + assert( pToken->t.z>=zSql && &pToken->t.z[pToken->t.n]<=&zSql[nSql] ); + } + } +#endif + + db->init.iDb = 0; + return rc; +} + +/* +** This function edits SQL statement zSql, replacing each token identified +** by the linked list pRename with the text of zNew. If argument bQuote is +** true, then zNew is always quoted first. If no error occurs, the result +** is loaded into context object pCtx as the result. +** +** Or, if an error occurs (i.e. an OOM condition), an error is left in +** pCtx and an SQLite error code returned. +*/ +static int renameEditSql( + sqlite3_context *pCtx, /* Return result here */ + RenameCtx *pRename, /* Rename context */ + const char *zSql, /* SQL statement to edit */ + const char *zNew, /* New token text */ + int bQuote /* True to always quote token */ +){ + i64 nNew = sqlite3Strlen30(zNew); + i64 nSql = sqlite3Strlen30(zSql); + sqlite3 *db = sqlite3_context_db_handle(pCtx); + int rc = SQLITE_OK; + char *zQuot = 0; + char *zOut; + i64 nQuot = 0; + char *zBuf1 = 0; + char *zBuf2 = 0; + + if( zNew ){ + /* Set zQuot to point to a buffer containing a quoted copy of the + ** identifier zNew. If the corresponding identifier in the original + ** ALTER TABLE statement was quoted (bQuote==1), then set zNew to + ** point to zQuot so that all substitutions are made using the + ** quoted version of the new column name. */ + zQuot = sqlite3MPrintf(db, "\"%w\" ", zNew); + if( zQuot==0 ){ + return SQLITE_NOMEM; + }else{ + nQuot = sqlite3Strlen30(zQuot)-1; + } + + assert( nQuot>=nNew ); + zOut = sqlite3DbMallocZero(db, nSql + pRename->nList*nQuot + 1); + }else{ + zOut = (char*)sqlite3DbMallocZero(db, (nSql*2+1) * 3); + if( zOut ){ + zBuf1 = &zOut[nSql*2+1]; + zBuf2 = &zOut[nSql*4+2]; + } + } + + /* At this point pRename->pList contains a list of RenameToken objects + ** corresponding to all tokens in the input SQL that must be replaced + ** with the new column name, or with single-quoted versions of themselves. + ** All that remains is to construct and return the edited SQL string. */ + if( zOut ){ + int nOut = nSql; + memcpy(zOut, zSql, nSql); + while( pRename->pList ){ + int iOff; /* Offset of token to replace in zOut */ + u32 nReplace; + const char *zReplace; + RenameToken *pBest = renameColumnTokenNext(pRename); + + if( zNew ){ + if( bQuote==0 && sqlite3IsIdChar(*pBest->t.z) ){ + nReplace = nNew; + zReplace = zNew; + }else{ + nReplace = nQuot; + zReplace = zQuot; + if( pBest->t.z[pBest->t.n]=='"' ) nReplace++; + } + }else{ + /* Dequote the double-quoted token. Then requote it again, this time + ** using single quotes. If the character immediately following the + ** original token within the input SQL was a single quote ('), then + ** add another space after the new, single-quoted version of the + ** token. This is so that (SELECT "string"'alias') maps to + ** (SELECT 'string' 'alias'), and not (SELECT 'string''alias'). */ + memcpy(zBuf1, pBest->t.z, pBest->t.n); + zBuf1[pBest->t.n] = 0; + sqlite3Dequote(zBuf1); + sqlite3_snprintf(nSql*2, zBuf2, "%Q%s", zBuf1, + pBest->t.z[pBest->t.n]=='\'' ? " " : "" + ); + zReplace = zBuf2; + nReplace = sqlite3Strlen30(zReplace); + } + + iOff = pBest->t.z - zSql; + if( pBest->t.n!=nReplace ){ + memmove(&zOut[iOff + nReplace], &zOut[iOff + pBest->t.n], + nOut - (iOff + pBest->t.n) + ); + nOut += nReplace - pBest->t.n; + zOut[nOut] = '\0'; + } + memcpy(&zOut[iOff], zReplace, nReplace); + sqlite3DbFree(db, pBest); + } + + sqlite3_result_text(pCtx, zOut, -1, SQLITE_TRANSIENT); + sqlite3DbFree(db, zOut); + }else{ + rc = SQLITE_NOMEM; + } + + sqlite3_free(zQuot); + return rc; +} + +/* +** Set all pEList->a[].fg.eEName fields in the expression-list to val. +*/ +static void renameSetENames(ExprList *pEList, int val){ + if( pEList ){ + int i; + for(i=0; inExpr; i++){ + assert( val==ENAME_NAME || pEList->a[i].fg.eEName==ENAME_NAME ); + pEList->a[i].fg.eEName = val; + } + } +} + +/* +** Resolve all symbols in the trigger at pParse->pNewTrigger, assuming +** it was read from the schema of database zDb. Return SQLITE_OK if +** successful. Otherwise, return an SQLite error code and leave an error +** message in the Parse object. +*/ +static int renameResolveTrigger(Parse *pParse){ + sqlite3 *db = pParse->db; + Trigger *pNew = pParse->pNewTrigger; + TriggerStep *pStep; + NameContext sNC; + int rc = SQLITE_OK; + + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + assert( pNew->pTabSchema ); + pParse->pTriggerTab = sqlite3FindTable(db, pNew->table, + db->aDb[sqlite3SchemaToIndex(db, pNew->pTabSchema)].zDbSName + ); + pParse->eTriggerOp = pNew->op; + /* ALWAYS() because if the table of the trigger does not exist, the + ** error would have been hit before this point */ + if( ALWAYS(pParse->pTriggerTab) ){ + rc = sqlite3ViewGetColumnNames(pParse, pParse->pTriggerTab)!=0; + } + + /* Resolve symbols in WHEN clause */ + if( rc==SQLITE_OK && pNew->pWhen ){ + rc = sqlite3ResolveExprNames(&sNC, pNew->pWhen); + } + + for(pStep=pNew->step_list; rc==SQLITE_OK && pStep; pStep=pStep->pNext){ + if( pStep->pSelect ){ + sqlite3SelectPrep(pParse, pStep->pSelect, &sNC); + if( pParse->nErr ) rc = pParse->rc; + } + if( rc==SQLITE_OK && pStep->zTarget ){ + SrcList *pSrc = sqlite3TriggerStepSrc(pParse, pStep); + if( pSrc ){ + Select *pSel = sqlite3SelectNew( + pParse, pStep->pExprList, pSrc, 0, 0, 0, 0, 0, 0 + ); + if( pSel==0 ){ + pStep->pExprList = 0; + pSrc = 0; + rc = SQLITE_NOMEM; + }else{ + /* pStep->pExprList contains an expression-list used for an UPDATE + ** statement. So the a[].zEName values are the RHS of the + ** "= " clauses of the UPDATE statement. So, before + ** running SelectPrep(), change all the eEName values in + ** pStep->pExprList to ENAME_SPAN (from their current value of + ** ENAME_NAME). This is to prevent any ids in ON() clauses that are + ** part of pSrc from being incorrectly resolved against the + ** a[].zEName values as if they were column aliases. */ + renameSetENames(pStep->pExprList, ENAME_SPAN); + sqlite3SelectPrep(pParse, pSel, 0); + renameSetENames(pStep->pExprList, ENAME_NAME); + rc = pParse->nErr ? SQLITE_ERROR : SQLITE_OK; + assert( pStep->pExprList==0 || pStep->pExprList==pSel->pEList ); + assert( pSrc==pSel->pSrc ); + if( pStep->pExprList ) pSel->pEList = 0; + pSel->pSrc = 0; + sqlite3SelectDelete(db, pSel); + } + if( pStep->pFrom ){ + int i; + for(i=0; ipFrom->nSrc && rc==SQLITE_OK; i++){ + SrcItem *p = &pStep->pFrom->a[i]; + if( p->fg.isSubquery ){ + assert( p->u4.pSubq!=0 ); + sqlite3SelectPrep(pParse, p->u4.pSubq->pSelect, 0); + } + } + } + + if( db->mallocFailed ){ + rc = SQLITE_NOMEM; + } + sNC.pSrcList = pSrc; + if( rc==SQLITE_OK && pStep->pWhere ){ + rc = sqlite3ResolveExprNames(&sNC, pStep->pWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprListNames(&sNC, pStep->pExprList); + } + assert( !pStep->pUpsert || (!pStep->pWhere && !pStep->pExprList) ); + if( pStep->pUpsert && rc==SQLITE_OK ){ + Upsert *pUpsert = pStep->pUpsert; + pUpsert->pUpsertSrc = pSrc; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc==SQLITE_OK ){ + ExprList *pUpsertSet = pUpsert->pUpsertSet; + rc = sqlite3ResolveExprListNames(&sNC, pUpsertSet); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertWhere); + } + if( rc==SQLITE_OK ){ + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + } + sNC.ncFlags = 0; + } + sNC.pSrcList = 0; + sqlite3SrcListDelete(db, pSrc); + }else{ + rc = SQLITE_NOMEM; + } + } + } + return rc; +} + +/* +** Invoke sqlite3WalkExpr() or sqlite3WalkSelect() on all Select or Expr +** objects that are part of the trigger passed as the second argument. +*/ +static void renameWalkTrigger(Walker *pWalker, Trigger *pTrigger){ + TriggerStep *pStep; + + /* Find tokens to edit in WHEN clause */ + sqlite3WalkExpr(pWalker, pTrigger->pWhen); + + /* Find tokens to edit in trigger steps */ + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + sqlite3WalkSelect(pWalker, pStep->pSelect); + sqlite3WalkExpr(pWalker, pStep->pWhere); + sqlite3WalkExprList(pWalker, pStep->pExprList); + if( pStep->pUpsert ){ + Upsert *pUpsert = pStep->pUpsert; + sqlite3WalkExprList(pWalker, pUpsert->pUpsertTarget); + sqlite3WalkExprList(pWalker, pUpsert->pUpsertSet); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertWhere); + sqlite3WalkExpr(pWalker, pUpsert->pUpsertTargetWhere); + } + if( pStep->pFrom ){ + int i; + SrcList *pFrom = pStep->pFrom; + for(i=0; inSrc; i++){ + if( pFrom->a[i].fg.isSubquery ){ + assert( pFrom->a[i].u4.pSubq!=0 ); + sqlite3WalkSelect(pWalker, pFrom->a[i].u4.pSubq->pSelect); + } + } + } + } +} + +/* +** Free the contents of Parse object (*pParse). Do not free the memory +** occupied by the Parse object itself. +*/ +static void renameParseCleanup(Parse *pParse){ + sqlite3 *db = pParse->db; + Index *pIdx; + if( pParse->pVdbe ){ + sqlite3VdbeFinalize(pParse->pVdbe); + } + sqlite3DeleteTable(db, pParse->pNewTable); + while( (pIdx = pParse->pNewIndex)!=0 ){ + pParse->pNewIndex = pIdx->pNext; + sqlite3FreeIndex(db, pIdx); + } + sqlite3DeleteTrigger(db, pParse->pNewTrigger); + sqlite3DbFree(db, pParse->zErrMsg); + renameTokenFree(db, pParse->pRename); + sqlite3ParseObjectReset(pParse); +} + +/* +** SQL function: +** +** sqlite_rename_column(SQL,TYPE,OBJ,DB,TABLE,COL,NEWNAME,QUOTE,TEMP) +** +** 0. zSql: SQL statement to rewrite +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3. Database: Database name (e.g. "main") +** 4. Table: Table name +** 5. iCol: Index of column to rename +** 6. zNew: New column name +** 7. bQuote: Non-zero if the new column name should be quoted. +** 8. bTemp: True if zSql comes from temp schema +** +** Do a column rename operation on the CREATE statement given in zSql. +** The iCol-th column (left-most is 0) of table zTable is renamed from zCol +** into zNew. The name should be quoted if bQuote is true. +** +** This function is used internally by the ALTER TABLE RENAME COLUMN command. +** It is only accessible to SQL created using sqlite3NestedParse(). It is +** not reachable from ordinary SQL passed into sqlite3_prepare() unless the +** SQLITE_TESTCTRL_INTERNAL_FUNCTIONS test setting is enabled. +*/ +static void renameColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + RenameCtx sCtx; + const char *zSql = (const char*)sqlite3_value_text(argv[0]); + const char *zDb = (const char*)sqlite3_value_text(argv[3]); + const char *zTable = (const char*)sqlite3_value_text(argv[4]); + int iCol = sqlite3_value_int(argv[5]); + const char *zNew = (const char*)sqlite3_value_text(argv[6]); + int bQuote = sqlite3_value_int(argv[7]); + int bTemp = sqlite3_value_int(argv[8]); + const char *zOld; + int rc; + Parse sParse; + Walker sWalker; + Index *pIdx; + int i; + Table *pTab; +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; +#endif + + UNUSED_PARAMETER(NotUsed); + if( zSql==0 ) return; + if( zTable==0 ) return; + if( zNew==0 ) return; + if( iCol<0 ) return; + sqlite3BtreeEnterAll(db); + pTab = sqlite3FindTable(db, zTable, zDb); + if( pTab==0 || iCol>=pTab->nCol ){ + sqlite3BtreeLeaveAll(db); + return; + } + zOld = pTab->aCol[iCol].zCnName; + memset(&sCtx, 0, sizeof(sCtx)); + sCtx.iCol = ((iCol==pTab->iPKey) ? -1 : iCol); + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = 0; +#endif + rc = renameParseSql(&sParse, zDb, db, zSql, bTemp); + + /* Find tokens that need to be replaced. */ + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameColumnExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + sCtx.pTab = pTab; + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + if( sParse.pNewTable ){ + if( IsView(sParse.pNewTable) ){ + Select *pSelect = sParse.pNewTable->u.view.pSelect; + pSelect->selFlags &= ~SF_View; + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + }else if( IsOrdinaryTable(sParse.pNewTable) ){ + /* A regular table */ + int bFKOnly = sqlite3_stricmp(zTable, sParse.pNewTable->zName); + FKey *pFKey; + sCtx.pTab = sParse.pNewTable; + if( bFKOnly==0 ){ + if( iColnCol ){ + renameTokenFind( + &sParse, &sCtx, (void*)sParse.pNewTable->aCol[iCol].zCnName + ); + } + if( sCtx.iCol<0 ){ + renameTokenFind(&sParse, &sCtx, (void*)&sParse.pNewTable->iPKey); + } + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); + for(pIdx=sParse.pNewTable->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } + for(pIdx=sParse.pNewIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3WalkExprList(&sWalker, pIdx->aColExpr); + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + Expr *pExpr = sqlite3ColumnExpr(sParse.pNewTable, + &sParse.pNewTable->aCol[i]); + sqlite3WalkExpr(&sWalker, pExpr); + } +#endif + } + + assert( IsOrdinaryTable(sParse.pNewTable) ); + for(pFKey=sParse.pNewTable->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + for(i=0; inCol; i++){ + if( bFKOnly==0 && pFKey->aCol[i].iFrom==iCol ){ + renameTokenFind(&sParse, &sCtx, (void*)&pFKey->aCol[i]); + } + if( 0==sqlite3_stricmp(pFKey->zTo, zTable) + && 0==sqlite3_stricmp(pFKey->aCol[i].zCol, zOld) + ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->aCol[i].zCol); + } + } + } + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ + /* A trigger */ + TriggerStep *pStep; + rc = renameResolveTrigger(&sParse); + if( rc!=SQLITE_OK ) goto renameColumnFunc_done; + + for(pStep=sParse.pNewTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget ){ + Table *pTarget = sqlite3LocateTable(&sParse, 0, pStep->zTarget, zDb); + if( pTarget==pTab ){ + if( pStep->pUpsert ){ + ExprList *pUpsertSet = pStep->pUpsert->pUpsertSet; + renameColumnElistNames(&sParse, &sCtx, pUpsertSet, zOld); + } + renameColumnIdlistNames(&sParse, &sCtx, pStep->pIdList, zOld); + renameColumnElistNames(&sParse, &sCtx, pStep->pExprList, zOld); + } + } + } + + + /* Find tokens to edit in UPDATE OF clause */ + if( sParse.pTriggerTab==pTab ){ + renameColumnIdlistNames(&sParse, &sCtx,sParse.pNewTrigger->pColumns,zOld); + } + + /* Find tokens to edit in various expressions and selects */ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } + + assert( rc==SQLITE_OK ); + rc = renameEditSql(context, &sCtx, zSql, zNew, bQuote); + +renameColumnFunc_done: + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_ERROR && sqlite3WritableSchema(db) ){ + sqlite3_result_value(context, argv[0]); + }else if( sParse.zErrMsg ){ + renameColumnParseError(context, "", argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + sqlite3BtreeLeaveAll(db); +} + +/* +** Walker expression callback used by "RENAME TABLE". +*/ +static int renameTableExprCb(Walker *pWalker, Expr *pExpr){ + RenameCtx *p = pWalker->u.pRename; + if( pExpr->op==TK_COLUMN + && ALWAYS(ExprUseYTab(pExpr)) + && p->pTab==pExpr->y.pTab + ){ + renameTokenFind(pWalker->pParse, p, (void*)&pExpr->y.pTab); + } + return WRC_Continue; +} + +/* +** Walker select callback used by "RENAME TABLE". +*/ +static int renameTableSelectCb(Walker *pWalker, Select *pSelect){ + int i; + RenameCtx *p = pWalker->u.pRename; + SrcList *pSrc = pSelect->pSrc; + if( pSelect->selFlags & (SF_View|SF_CopyCte) ){ + testcase( pSelect->selFlags & SF_View ); + testcase( pSelect->selFlags & SF_CopyCte ); + return WRC_Prune; + } + if( NEVER(pSrc==0) ){ + assert( pWalker->pParse->db->mallocFailed ); + return WRC_Abort; + } + for(i=0; inSrc; i++){ + SrcItem *pItem = &pSrc->a[i]; + if( pItem->pSTab==p->pTab ){ + renameTokenFind(pWalker->pParse, p, pItem->zName); + } + } + renameWalkWith(pWalker, pSelect); + + return WRC_Continue; +} + + +/* +** This C function implements an SQL user function that is used by SQL code +** generated by the ALTER TABLE ... RENAME command to modify the definition +** of any foreign key constraints that use the table being renamed as the +** parent table. It is passed three arguments: +** +** 0: The database containing the table being renamed. +** 1. type: Type of object ("table", "view" etc.) +** 2. object: Name of object +** 3: The complete text of the schema statement being modified, +** 4: The old name of the table being renamed, and +** 5: The new name of the table being renamed. +** 6: True if the schema statement comes from the temp db. +** +** It returns the new schema statement. For example: +** +** sqlite_rename_table('main', 'CREATE TABLE t1(a REFERENCES t2)','t2','t3',0) +** -> 'CREATE TABLE t1(a REFERENCES t3)' +*/ +static void renameTableFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + const char *zDb = (const char*)sqlite3_value_text(argv[0]); + const char *zInput = (const char*)sqlite3_value_text(argv[3]); + const char *zOld = (const char*)sqlite3_value_text(argv[4]); + const char *zNew = (const char*)sqlite3_value_text(argv[5]); + int bTemp = sqlite3_value_int(argv[6]); + UNUSED_PARAMETER(NotUsed); + + if( zInput && zOld && zNew ){ + Parse sParse; + int rc; + int bQuote = 1; + RenameCtx sCtx; + Walker sWalker; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + memset(&sCtx, 0, sizeof(RenameCtx)); + sCtx.pTab = sqlite3FindTable(db, zOld, zDb); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameTableExprCb; + sWalker.xSelectCallback = renameTableSelectCb; + sWalker.u.pRename = &sCtx; + + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); + + if( rc==SQLITE_OK ){ + int isLegacy = (db->flags & SQLITE_LegacyAlter); + if( sParse.pNewTable ){ + Table *pTab = sParse.pNewTable; + + if( IsView(pTab) ){ + if( isLegacy==0 ){ + Select *pSelect = pTab->u.view.pSelect; + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + + assert( pSelect->selFlags & SF_View ); + pSelect->selFlags &= ~SF_View; + sqlite3SelectPrep(&sParse, pTab->u.view.pSelect, &sNC); + if( sParse.nErr ){ + rc = sParse.rc; + }else{ + sqlite3WalkSelect(&sWalker, pTab->u.view.pSelect); + } + } + }else{ + /* Modify any FK definitions to point to the new table. */ +#ifndef SQLITE_OMIT_FOREIGN_KEY + if( (isLegacy==0 || (db->flags & SQLITE_ForeignKeys)) + && !IsVirtual(pTab) + ){ + FKey *pFKey; + assert( IsOrdinaryTable(pTab) ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + if( sqlite3_stricmp(pFKey->zTo, zOld)==0 ){ + renameTokenFind(&sParse, &sCtx, (void*)pFKey->zTo); + } + } + } +#endif + + /* If this is the table being altered, fix any table refs in CHECK + ** expressions. Also update the name that appears right after the + ** "CREATE [VIRTUAL] TABLE" bit. */ + if( sqlite3_stricmp(zOld, pTab->zName)==0 ){ + sCtx.pTab = pTab; + if( isLegacy==0 ){ + sqlite3WalkExprList(&sWalker, pTab->pCheck); + } + renameTokenFind(&sParse, &sCtx, pTab->zName); + } + } + } + + else if( sParse.pNewIndex ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewIndex->zName); + if( isLegacy==0 ){ + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + } + } + +#ifndef SQLITE_OMIT_TRIGGER + else{ + Trigger *pTrigger = sParse.pNewTrigger; + TriggerStep *pStep; + if( 0==sqlite3_stricmp(sParse.pNewTrigger->table, zOld) + && sCtx.pTab->pSchema==pTrigger->pTabSchema + ){ + renameTokenFind(&sParse, &sCtx, sParse.pNewTrigger->table); + } + + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, pTrigger); + for(pStep=pTrigger->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget && 0==sqlite3_stricmp(pStep->zTarget, zOld) ){ + renameTokenFind(&sParse, &sCtx, pStep->zTarget); + } + if( pStep->pFrom ){ + int i; + for(i=0; ipFrom->nSrc; i++){ + SrcItem *pItem = &pStep->pFrom->a[i]; + if( 0==sqlite3_stricmp(pItem->zName, zOld) ){ + renameTokenFind(&sParse, &sCtx, pItem->zName); + } + } + } + } + } + } + } +#endif + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, zNew, bQuote); + } + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_ERROR && sqlite3WritableSchema(db) ){ + sqlite3_result_value(context, argv[3]); + }else if( sParse.zErrMsg ){ + renameColumnParseError(context, "", argv[1], argv[2], &sParse); + }else{ + sqlite3_result_error_code(context, rc); + } + } + + renameParseCleanup(&sParse); + renameTokenFree(db, sCtx.pList); + sqlite3BtreeLeaveAll(db); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + } + + return; +} + +static int renameQuotefixExprCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_STRING && (pExpr->flags & EP_DblQuoted) ){ + renameTokenFind(pWalker->pParse, pWalker->u.pRename, (const void*)pExpr); + } + return WRC_Continue; +} + +/* SQL function: sqlite_rename_quotefix(DB,SQL) +** +** Rewrite the DDL statement "SQL" so that any string literals that use +** double-quotes use single quotes instead. +** +** Two arguments must be passed: +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement to edit. +** +** The returned value is the modified SQL statement. For example, given +** the database schema: +** +** CREATE TABLE t1(a, b, c); +** +** SELECT sqlite_rename_quotefix('main', +** 'CREATE VIEW v1 AS SELECT "a", "string" FROM t1' +** ); +** +** returns the string: +** +** CREATE VIEW v1 AS SELECT "a", 'string' FROM t1 +** +** If there is a error in the input SQL, then raise an error, except +** if PRAGMA writable_schema=ON, then just return the input string +** unmodified following an error. +*/ +static void renameQuotefixFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + sqlite3BtreeEnterAll(db); + + UNUSED_PARAMETER(NotUsed); + if( zDb && zInput ){ + int rc; + Parse sParse; + rc = renameParseSql(&sParse, zDb, db, zInput, 0); + + if( rc==SQLITE_OK ){ + RenameCtx sCtx; + Walker sWalker; + + /* Walker to find tokens that need to be replaced. */ + memset(&sCtx, 0, sizeof(RenameCtx)); + memset(&sWalker, 0, sizeof(Walker)); + sWalker.pParse = &sParse; + sWalker.xExprCallback = renameQuotefixExprCb; + sWalker.xSelectCallback = renameColumnSelectCb; + sWalker.u.pRename = &sCtx; + + if( sParse.pNewTable ){ + if( IsView(sParse.pNewTable) ){ + Select *pSelect = sParse.pNewTable->u.view.pSelect; + pSelect->selFlags &= ~SF_View; + sParse.rc = SQLITE_OK; + sqlite3SelectPrep(&sParse, pSelect, 0); + rc = (db->mallocFailed ? SQLITE_NOMEM : sParse.rc); + if( rc==SQLITE_OK ){ + sqlite3WalkSelect(&sWalker, pSelect); + } + }else{ + int i; + sqlite3WalkExprList(&sWalker, sParse.pNewTable->pCheck); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + for(i=0; inCol; i++){ + sqlite3WalkExpr(&sWalker, + sqlite3ColumnExpr(sParse.pNewTable, + &sParse.pNewTable->aCol[i])); + } +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + } + }else if( sParse.pNewIndex ){ + sqlite3WalkExprList(&sWalker, sParse.pNewIndex->aColExpr); + sqlite3WalkExpr(&sWalker, sParse.pNewIndex->pPartIdxWhere); + }else{ +#ifndef SQLITE_OMIT_TRIGGER + rc = renameResolveTrigger(&sParse); + if( rc==SQLITE_OK ){ + renameWalkTrigger(&sWalker, sParse.pNewTrigger); + } +#endif /* SQLITE_OMIT_TRIGGER */ + } + + if( rc==SQLITE_OK ){ + rc = renameEditSql(context, &sCtx, zInput, 0, 0); + } + renameTokenFree(db, sCtx.pList); + } + if( rc!=SQLITE_OK ){ + if( sqlite3WritableSchema(db) && rc==SQLITE_ERROR ){ + sqlite3_result_value(context, argv[1]); + }else{ + sqlite3_result_error_code(context, rc); + } + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + + sqlite3BtreeLeaveAll(db); +} + +/* Function: sqlite_rename_test(DB,SQL,TYPE,NAME,ISTEMP,WHEN,DQS) +** +** An SQL user function that checks that there are no parse or symbol +** resolution problems in a CREATE TRIGGER|TABLE|VIEW|INDEX statement. +** After an ALTER TABLE .. RENAME operation is performed and the schema +** reloaded, this function is called on each SQL statement in the schema +** to ensure that it is still usable. +** +** 0: Database name ("main", "temp" etc.). +** 1: SQL statement. +** 2: Object type ("view", "table", "trigger" or "index"). +** 3: Object name. +** 4: True if object is from temp schema. +** 5: "when" part of error message. +** 6: True to disable the DQS quirk when parsing SQL. +** +** The return value is computed as follows: +** +** A. If an error is seen and not in PRAGMA writable_schema=ON mode, +** then raise the error. +** B. Else if a trigger is created and the the table that the trigger is +** attached to is in database zDb, then return 1. +** C. Otherwise return NULL. +*/ +static void renameTableTest( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + char const *zDb = (const char*)sqlite3_value_text(argv[0]); + char const *zInput = (const char*)sqlite3_value_text(argv[1]); + int bTemp = sqlite3_value_int(argv[4]); + int isLegacy = (db->flags & SQLITE_LegacyAlter); + char const *zWhen = (const char*)sqlite3_value_text(argv[5]); + int bNoDQS = sqlite3_value_int(argv[6]); + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + + if( zDb && zInput ){ + int rc; + Parse sParse; + int flags = db->flags; + if( bNoDQS ) db->flags &= ~(SQLITE_DqsDML|SQLITE_DqsDDL); + rc = renameParseSql(&sParse, zDb, db, zInput, bTemp); + db->flags |= (flags & (SQLITE_DqsDML|SQLITE_DqsDDL)); + if( rc==SQLITE_OK ){ + if( isLegacy==0 && sParse.pNewTable && IsView(sParse.pNewTable) ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = &sParse; + sqlite3SelectPrep(&sParse, sParse.pNewTable->u.view.pSelect, &sNC); + if( sParse.nErr ) rc = sParse.rc; + } + + else if( sParse.pNewTrigger ){ + if( isLegacy==0 ){ + rc = renameResolveTrigger(&sParse); + } + if( rc==SQLITE_OK ){ + int i1 = sqlite3SchemaToIndex(db, sParse.pNewTrigger->pTabSchema); + int i2 = sqlite3FindDbName(db, zDb); + if( i1==i2 ){ + /* Handle output case B */ + sqlite3_result_int(context, 1); + } + } + } + } + + if( rc!=SQLITE_OK && zWhen && !sqlite3WritableSchema(db) ){ + /* Output case A */ + renameColumnParseError(context, zWhen, argv[2], argv[3],&sParse); + } + renameParseCleanup(&sParse); + } + +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif +} + +/* +** The implementation of internal UDF sqlite_drop_column(). +** +** Arguments: +** +** argv[0]: An integer - the index of the schema containing the table +** argv[1]: CREATE TABLE statement to modify. +** argv[2]: An integer - the index of the column to remove. +** +** The value returned is a string containing the CREATE TABLE statement +** with column argv[2] removed. +*/ +static void dropColumnFunc( + sqlite3_context *context, + int NotUsed, + sqlite3_value **argv +){ + sqlite3 *db = sqlite3_context_db_handle(context); + int iSchema = sqlite3_value_int(argv[0]); + const char *zSql = (const char*)sqlite3_value_text(argv[1]); + int iCol = sqlite3_value_int(argv[2]); + const char *zDb = db->aDb[iSchema].zDbSName; + int rc; + Parse sParse; + RenameToken *pCol; + Table *pTab; + const char *zEnd; + char *zNew = 0; + +#ifndef SQLITE_OMIT_AUTHORIZATION + sqlite3_xauth xAuth = db->xAuth; + db->xAuth = 0; +#endif + + UNUSED_PARAMETER(NotUsed); + rc = renameParseSql(&sParse, zDb, db, zSql, iSchema==1); + if( rc!=SQLITE_OK ) goto drop_column_done; + pTab = sParse.pNewTable; + if( pTab==0 || pTab->nCol==1 || iCol>=pTab->nCol ){ + /* This can happen if the sqlite_schema table is corrupt */ + rc = SQLITE_CORRUPT_BKPT; + goto drop_column_done; + } + + pCol = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol].zCnName); + if( iColnCol-1 ){ + RenameToken *pEnd; + pEnd = renameTokenFind(&sParse, 0, (void*)pTab->aCol[iCol+1].zCnName); + zEnd = (const char*)pEnd->t.z; + }else{ + assert( IsOrdinaryTable(pTab) ); + zEnd = (const char*)&zSql[pTab->u.tab.addColOffset]; + while( ALWAYS(pCol->t.z[0]!=0) && pCol->t.z[0]!=',' ) pCol->t.z--; + } + + zNew = sqlite3MPrintf(db, "%.*s%s", pCol->t.z-zSql, zSql, zEnd); + sqlite3_result_text(context, zNew, -1, SQLITE_TRANSIENT); + sqlite3_free(zNew); + +drop_column_done: + renameParseCleanup(&sParse); +#ifndef SQLITE_OMIT_AUTHORIZATION + db->xAuth = xAuth; +#endif + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(context, rc); + } +} + +/* +** This function is called by the parser upon parsing an +** +** ALTER TABLE pSrc DROP COLUMN pName +** +** statement. Argument pSrc contains the possibly qualified name of the +** table being edited, and token pName the name of the column to drop. +*/ +SQLITE_PRIVATE void sqlite3AlterDropColumn(Parse *pParse, SrcList *pSrc, const Token *pName){ + sqlite3 *db = pParse->db; /* Database handle */ + Table *pTab; /* Table to modify */ + int iDb; /* Index of db containing pTab in aDb[] */ + const char *zDb; /* Database containing pTab ("main" etc.) */ + char *zCol = 0; /* Name of column to drop */ + int iCol; /* Index of column zCol in pTab->aCol[] */ + + /* Look up the table being altered. */ + assert( pParse->pNewTable==0 ); + assert( sqlite3BtreeHoldsAllMutexes(db) ); + if( NEVER(db->mallocFailed) ) goto exit_drop_column; + pTab = sqlite3LocateTableItem(pParse, 0, &pSrc->a[0]); + if( !pTab ) goto exit_drop_column; + + /* Make sure this is not an attempt to ALTER a view, virtual table or + ** system table. */ + if( SQLITE_OK!=isAlterableTable(pParse, pTab) ) goto exit_drop_column; + if( SQLITE_OK!=isRealTable(pParse, pTab, 1) ) goto exit_drop_column; + + /* Find the index of the column being dropped. */ + zCol = sqlite3NameFromToken(db, pName); + if( zCol==0 ){ + assert( db->mallocFailed ); + goto exit_drop_column; + } + iCol = sqlite3ColumnIndex(pTab, zCol); + if( iCol<0 ){ + sqlite3ErrorMsg(pParse, "no such column: \"%T\"", pName); + goto exit_drop_column; + } + + /* Do not allow the user to drop a PRIMARY KEY column or a column + ** constrained by a UNIQUE constraint. */ + if( pTab->aCol[iCol].colFlags & (COLFLAG_PRIMKEY|COLFLAG_UNIQUE) ){ + sqlite3ErrorMsg(pParse, "cannot drop %s column: \"%s\"", + (pTab->aCol[iCol].colFlags&COLFLAG_PRIMKEY) ? "PRIMARY KEY" : "UNIQUE", + zCol + ); + goto exit_drop_column; + } + + /* Do not allow the number of columns to go to zero */ + if( pTab->nCol<=1 ){ + sqlite3ErrorMsg(pParse, "cannot drop column \"%s\": no other columns exist",zCol); + goto exit_drop_column; + } + + /* Edit the sqlite_schema table */ + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + assert( iDb>=0 ); + zDb = db->aDb[iDb].zDbSName; +#ifndef SQLITE_OMIT_AUTHORIZATION + /* Invoke the authorization callback. */ + if( sqlite3AuthCheck(pParse, SQLITE_ALTER_TABLE, zDb, pTab->zName, zCol) ){ + goto exit_drop_column; + } +#endif + renameTestSchema(pParse, zDb, iDb==1, "", 0); + renameFixQuotes(pParse, zDb, iDb==1); + sqlite3NestedParse(pParse, + "UPDATE \"%w\"." LEGACY_SCHEMA_TABLE " SET " + "sql = sqlite_drop_column(%d, sql, %d) " + "WHERE (type=='table' AND tbl_name=%Q COLLATE nocase)" + , zDb, iDb, iCol, pTab->zName + ); + + /* Drop and reload the database schema. */ + renameReloadSchema(pParse, iDb, INITFLAG_AlterDrop); + renameTestSchema(pParse, zDb, iDb==1, "after drop column", 1); + + /* Edit rows of table on disk */ + if( pParse->nErr==0 && (pTab->aCol[iCol].colFlags & COLFLAG_VIRTUAL)==0 ){ + int i; + int addr; + int reg; + int regRec; + Index *pPk = 0; + int nField = 0; /* Number of non-virtual columns after drop */ + int iCur; + Vdbe *v = sqlite3GetVdbe(pParse); + iCur = pParse->nTab++; + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenWrite); + addr = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + reg = ++pParse->nMem; + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, reg); + pParse->nMem += pTab->nCol; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + pParse->nMem += pPk->nColumn; + for(i=0; inKeyCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, iCur, i, reg+i+1); + } + nField = pPk->nKeyCol; + } + regRec = ++pParse->nMem; + for(i=0; inCol; i++){ + if( i!=iCol && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + int regOut; + if( pPk ){ + int iPos = sqlite3TableColumnToIndex(pPk, i); + int iColPos = sqlite3TableColumnToIndex(pPk, iCol); + if( iPosnKeyCol ) continue; + regOut = reg+1+iPos-(iPos>iColPos); + }else{ + regOut = reg+1+nField; + } + if( i==pTab->iPKey ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regOut); + }else{ + char aff = pTab->aCol[i].affinity; + if( aff==SQLITE_AFF_REAL ){ + pTab->aCol[i].affinity = SQLITE_AFF_NUMERIC; + } + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, i, regOut); + pTab->aCol[i].affinity = aff; + } + nField++; + } + } + if( nField==0 ){ + /* dbsqlfuzz 5f09e7bcc78b4954d06bf9f2400d7715f48d1fef */ + pParse->nMem++; + sqlite3VdbeAddOp2(v, OP_Null, 0, reg+1); + nField = 1; + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg+1, nField, regRec); + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iCur, regRec, reg+1, pPk->nKeyCol); + }else{ + sqlite3VdbeAddOp3(v, OP_Insert, iCur, regRec, reg); + } + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + + sqlite3VdbeAddOp2(v, OP_Next, iCur, addr+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addr); + } + +exit_drop_column: + sqlite3DbFree(db, zCol); + sqlite3SrcListDelete(db, pSrc); +} + +/* +** Register built-in functions used to help implement ALTER TABLE +*/ +SQLITE_PRIVATE void sqlite3AlterFunctions(void){ + static FuncDef aAlterTableFuncs[] = { + INTERNAL_FUNCTION(sqlite_rename_column, 9, renameColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_table, 7, renameTableFunc), + INTERNAL_FUNCTION(sqlite_rename_test, 7, renameTableTest), + INTERNAL_FUNCTION(sqlite_drop_column, 3, dropColumnFunc), + INTERNAL_FUNCTION(sqlite_rename_quotefix,2, renameQuotefixFunc), + }; + sqlite3InsertBuiltinFuncs(aAlterTableFuncs, ArraySize(aAlterTableFuncs)); +} #endif /* SQLITE_ALTER_TABLE */ /************** End of alter.c ***********************************************/ @@ -94106,13 +119062,13 @@ exit_begin_add_column: ** is between 3.6.18 and 3.7.8, inclusive, and unless SQLite is compiled ** with SQLITE_ENABLE_STAT2. The sqlite_stat2 table is deprecated. ** The sqlite_stat2 table is superseded by sqlite_stat3, which is only -** created and used by SQLite versions 3.7.9 and later and with +** created and used by SQLite versions 3.7.9 through 3.29.0 when ** SQLITE_ENABLE_STAT3 defined. The functionality of sqlite_stat3 -** is a superset of sqlite_stat2. The sqlite_stat4 is an enhanced -** version of sqlite_stat3 and is only available when compiled with -** SQLITE_ENABLE_STAT4 and in SQLite versions 3.8.1 and later. It is -** not possible to enable both STAT3 and STAT4 at the same time. If they -** are both enabled, then STAT4 takes precedence. +** is a superset of sqlite_stat2 and is also now deprecated. The +** sqlite_stat4 is an enhanced version of sqlite_stat3 and is only +** available when compiled with SQLITE_ENABLE_STAT4 and in SQLite +** versions 3.8.1 and later. STAT4 is the only variant that is still +** supported. ** ** For most applications, sqlite_stat1 provides all the statistics required ** for the query planner to make good choices. @@ -94128,7 +119084,7 @@ exit_begin_add_column: ** integer is the average number of rows in the index that have the same ** value in the first column of the index. The third integer is the average ** number of rows in the index that have the same value for the first two -** columns. The N-th integer (for N>1) is the average number of rows in +** columns. The N-th integer (for N>1) is the average number of rows in ** the index which have the same value for the first N-1 columns. For ** a K-column index, there will be K+1 integers in the stat column. If ** the index is unique, then the last integer will be 1. @@ -94138,7 +119094,7 @@ exit_begin_add_column: ** must be separated from the last integer by a single space. If the ** "unordered" keyword is present, then the query planner assumes that ** the index is unordered and will not use the index for a range query. -** +** ** If the sqlite_stat1.idx column is NULL, then the sqlite_stat1.stat ** column contains a single integer which is the (estimated) number of ** rows in the table identified by sqlite_stat1.tbl. @@ -94196,9 +119152,9 @@ exit_begin_add_column: ** number of entries that are strictly less than the sample. The first ** integer in nLt contains the number of entries in the index where the ** left-most column is less than the left-most column of the sample. -** The K-th integer in the nLt entry is the number of index entries +** The K-th integer in the nLt entry is the number of index entries ** where the first K columns are less than the first K columns of the -** sample. The nDLt column is like nLt except that it contains the +** sample. The nDLt column is like nLt except that it contains the ** number of distinct entries in the index that are less than the ** sample. ** @@ -94223,17 +119179,11 @@ exit_begin_add_column: #if defined(SQLITE_ENABLE_STAT4) # define IsStat4 1 -# define IsStat3 0 -#elif defined(SQLITE_ENABLE_STAT3) -# define IsStat4 0 -# define IsStat3 1 #else # define IsStat4 0 -# define IsStat3 0 # undef SQLITE_STAT4_SAMPLES # define SQLITE_STAT4_SAMPLES 1 #endif -#define IsStat34 (IsStat3+IsStat4) /* 1 for STAT3 or STAT4. 0 otherwise */ /* ** This routine generates code that opens the sqlite_statN tables. @@ -94262,21 +119212,22 @@ static void openStatTable( { "sqlite_stat1", "tbl,idx,stat" }, #if defined(SQLITE_ENABLE_STAT4) { "sqlite_stat4", "tbl,idx,neq,nlt,ndlt,sample" }, - { "sqlite_stat3", 0 }, -#elif defined(SQLITE_ENABLE_STAT3) - { "sqlite_stat3", "tbl,idx,neq,nlt,ndlt,sample" }, - { "sqlite_stat4", 0 }, #else - { "sqlite_stat3", 0 }, { "sqlite_stat4", 0 }, #endif + { "sqlite_stat3", 0 }, }; int i; sqlite3 *db = pParse->db; Db *pDb; Vdbe *v = sqlite3GetVdbe(pParse); - int aRoot[ArraySize(aTable)]; + u32 aRoot[ArraySize(aTable)]; u8 aCreateTbl[ArraySize(aTable)]; +#ifdef SQLITE_ENABLE_STAT4 + const int nToOpen = OptimizationEnabled(db,SQLITE_Stat4) ? 2 : 1; +#else + const int nToOpen = 1; +#endif if( v==0 ) return; assert( sqlite3BtreeHoldsAllMutexes(db) ); @@ -94289,41 +119240,45 @@ static void openStatTable( for(i=0; izName))==0 ){ - if( aTable[i].zCols ){ - /* The sqlite_statN table does not exist. Create it. Note that a - ** side-effect of the CREATE TABLE statement is to leave the rootpage - ** of the new table in register pParse->regRoot. This is important + aCreateTbl[i] = 0; + if( (pStat = sqlite3FindTable(db, zTab, pDb->zDbSName))==0 ){ + if( iregRoot. This is important ** because the OpenWrite opcode below will be needing it. */ sqlite3NestedParse(pParse, - "CREATE TABLE %Q.%s(%s)", pDb->zName, zTab, aTable[i].zCols + "CREATE TABLE %Q.%s(%s)", pDb->zDbSName, zTab, aTable[i].zCols ); - aRoot[i] = pParse->regRoot; + aRoot[i] = (u32)pParse->regRoot; aCreateTbl[i] = OPFLAG_P2ISREG; } }else{ - /* The table already exists. If zWhere is not NULL, delete all entries + /* The table already exists. If zWhere is not NULL, delete all entries ** associated with the table zWhere. If zWhere is NULL, delete the ** entire contents of the table. */ aRoot[i] = pStat->tnum; - aCreateTbl[i] = 0; sqlite3TableLock(pParse, iDb, aRoot[i], 1, zTab); if( zWhere ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.%s WHERE %s=%Q", - pDb->zName, zTab, zWhereType, zWhere + pDb->zDbSName, zTab, zWhereType, zWhere ); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + }else if( db->xPreUpdateCallback ){ + sqlite3NestedParse(pParse, "DELETE FROM %Q.%s", pDb->zDbSName, zTab); +#endif }else{ /* The sqlite_stat[134] table already exists. Delete all rows. */ - sqlite3VdbeAddOp2(v, OP_Clear, aRoot[i], iDb); + sqlite3VdbeAddOp2(v, OP_Clear, (int)aRoot[i], iDb); } } } /* Open the sqlite_stat[134] tables for writing. */ - for(i=0; aTable[i].zCols; i++){ + for(i=0; inRowid ){ sqlite3DbFree(db, p->u.aRowid); @@ -94388,8 +119349,8 @@ static void sampleClear(sqlite3 *db, Stat4Sample *p){ /* Initialize the BLOB value of a ROWID */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowid(sqlite3 *db, StatSample *p, int n, const u8 *pData){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); p->u.aRowid = sqlite3DbMallocRawNN(db, n); @@ -94404,8 +119365,8 @@ static void sampleSetRowid(sqlite3 *db, Stat4Sample *p, int n, const u8 *pData){ /* Initialize the INTEGER value of a ROWID. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleSetRowidInt64(sqlite3 *db, StatSample *p, i64 iRowid){ assert( db!=0 ); if( p->nRowid ) sqlite3DbFree(db, p->u.aRowid); p->nRowid = 0; @@ -94417,8 +119378,8 @@ static void sampleSetRowidInt64(sqlite3 *db, Stat4Sample *p, i64 iRowid){ /* ** Copy the contents of object (*pFrom) into (*pTo). */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ +#ifdef SQLITE_ENABLE_STAT4 +static void sampleCopy(StatAccum *p, StatSample *pTo, StatSample *pFrom){ pTo->isPSample = pFrom->isPSample; pTo->iCol = pFrom->iCol; pTo->iHash = pFrom->iHash; @@ -94434,40 +119395,41 @@ static void sampleCopy(Stat4Accum *p, Stat4Sample *pTo, Stat4Sample *pFrom){ #endif /* -** Reclaim all memory of a Stat4Accum structure. +** Reclaim all memory of a StatAccum structure. */ -static void stat4Destructor(void *pOld){ - Stat4Accum *p = (Stat4Accum*)pOld; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - int i; - for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); - for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); - sampleClear(p->db, &p->current); +static void statAccumDestructor(void *pOld){ + StatAccum *p = (StatAccum*)pOld; +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ){ + int i; + for(i=0; inCol; i++) sampleClear(p->db, p->aBest+i); + for(i=0; imxSample; i++) sampleClear(p->db, p->a+i); + sampleClear(p->db, &p->current); + } #endif sqlite3DbFree(p->db, p); } /* -** Implementation of the stat_init(N,K,C) SQL function. The three parameters +** Implementation of the stat_init(N,K,C,L) SQL function. The four parameters ** are: ** N: The number of columns in the index including the rowid/pk (note 1) ** K: The number of columns in the index excluding the rowid/pk. -** C: The number of rows in the index (note 2) +** C: Estimated number of rows in the index +** L: A limit on the number of rows to scan, or 0 for no-limit ** ** Note 1: In the special case of the covering index that implements a ** WITHOUT ROWID table, N is the number of PRIMARY KEY columns, not the ** total number of columns in the table. ** -** Note 2: C is only used for STAT3 and STAT4. -** ** For indexes on ordinary rowid tables, N==K+1. But for indexes on ** WITHOUT ROWID tables, N=K+P where P is the number of columns in the ** PRIMARY KEY of the table. The covering index that implements the ** original WITHOUT ROWID table as N==K as a special case. ** -** This routine allocates the Stat4Accum object in heap memory. The return -** value is a pointer to the Stat4Accum object. The datatype of the -** return value is BLOB, but it is really just a pointer to the Stat4Accum +** This routine allocates the StatAccum object in heap memory. The return +** value is a pointer to the StatAccum object. The datatype of the +** return value is BLOB, but it is really just a pointer to the StatAccum ** object. */ static void statInit( @@ -94475,14 +119437,15 @@ static void statInit( int argc, sqlite3_value **argv ){ - Stat4Accum *p; + StatAccum *p; int nCol; /* Number of columns in index being sampled */ int nKeyCol; /* Number of key columns */ int nColUp; /* nCol rounded up for alignment */ int n; /* Bytes of space to allocate */ - sqlite3 *db; /* Database connection */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - int mxSample = SQLITE_STAT4_SAMPLES; + sqlite3 *db = sqlite3_context_db_handle(context); /* Database connection */ +#ifdef SQLITE_ENABLE_STAT4 + /* Maximum number of samples. 0 if STAT4 data is not collected */ + int mxSample = OptimizationEnabled(db,SQLITE_Stat4) ?SQLITE_STAT4_SAMPLES :0; #endif /* Decode the three function arguments */ @@ -94494,17 +119457,17 @@ static void statInit( assert( nKeyCol<=nCol ); assert( nKeyCol>0 ); - /* Allocate the space required for the Stat4Accum object */ - n = sizeof(*p) - + sizeof(tRowcnt)*nColUp /* Stat4Accum.anEq */ - + sizeof(tRowcnt)*nColUp /* Stat4Accum.anDLt */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - + sizeof(tRowcnt)*nColUp /* Stat4Accum.anLt */ - + sizeof(Stat4Sample)*(nCol+mxSample) /* Stat4Accum.aBest[], a[] */ - + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample) + /* Allocate the space required for the StatAccum object */ + n = sizeof(*p) + + sizeof(tRowcnt)*nColUp; /* StatAccum.anDLt */ +#ifdef SQLITE_ENABLE_STAT4 + n += sizeof(tRowcnt)*nColUp; /* StatAccum.anEq */ + if( mxSample ){ + n += sizeof(tRowcnt)*nColUp /* StatAccum.anLt */ + + sizeof(StatSample)*(nCol+mxSample) /* StatAccum.aBest[], a[] */ + + sizeof(tRowcnt)*3*nColUp*(nCol+mxSample); + } #endif - ; - db = sqlite3_context_db_handle(context); p = sqlite3DbMallocZero(db, n); if( p==0 ){ sqlite3_result_error_nomem(context); @@ -94512,25 +119475,28 @@ static void statInit( } p->db = db; + p->nEst = sqlite3_value_int64(argv[2]); p->nRow = 0; + p->nLimit = sqlite3_value_int64(argv[3]); p->nCol = nCol; p->nKeyCol = nKeyCol; + p->nSkipAhead = 0; p->current.anDLt = (tRowcnt*)&p[1]; - p->current.anEq = &p->current.anDLt[nColUp]; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - { +#ifdef SQLITE_ENABLE_STAT4 + p->current.anEq = &p->current.anDLt[nColUp]; + p->mxSample = p->nLimit==0 ? mxSample : 0; + if( mxSample ){ u8 *pSpace; /* Allocated space not yet assigned */ int i; /* Used to iterate through p->aSample[] */ p->iGet = -1; - p->mxSample = mxSample; - p->nPSample = (tRowcnt)(sqlite3_value_int64(argv[2])/(mxSample/3+1) + 1); + p->nPSample = (tRowcnt)(p->nEst/(mxSample/3+1) + 1); p->current.anLt = &p->current.anEq[nColUp]; p->iPrn = 0x689e962d*(u32)nCol ^ 0xd0944565*(u32)sqlite3_value_int(argv[2]); - - /* Set up the Stat4Accum.a[] and aBest[] arrays */ - p->a = (struct Stat4Sample*)&p->current.anLt[nColUp]; + + /* Set up the StatAccum.a[] and aBest[] arrays */ + p->a = (struct StatSample*)&p->current.anLt[nColUp]; p->aBest = &p->a[mxSample]; pSpace = (u8*)(&p->a[mxSample+nCol]); for(i=0; i<(mxSample+nCol); i++){ @@ -94539,7 +119505,7 @@ static void statInit( p->a[i].anDLt = (tRowcnt *)pSpace; pSpace += (sizeof(tRowcnt) * nColUp); } assert( (pSpace - (u8*)p)==n ); - + for(i=0; iaBest[i].iCol = i; } @@ -94550,35 +119516,36 @@ static void statInit( ** only the pointer (the 2nd parameter) matters. The size of the object ** (given by the 3rd parameter) is never used and can be any positive ** value. */ - sqlite3_result_blob(context, p, sizeof(*p), stat4Destructor); + sqlite3_result_blob(context, p, sizeof(*p), statAccumDestructor); } static const FuncDef statInitFuncdef = { - 2+IsStat34, /* nArg */ + 4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ statInit, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_init", /* zName */ {0} }; #ifdef SQLITE_ENABLE_STAT4 /* -** pNew and pOld are both candidate non-periodic samples selected for -** the same column (pNew->iCol==pOld->iCol). Ignoring this column and +** pNew and pOld are both candidate non-periodic samples selected for +** the same column (pNew->iCol==pOld->iCol). Ignoring this column and ** considering only any trailing columns and the sample hash value, this ** function returns true if sample pNew is to be preferred over pOld. ** In other words, if we assume that the cardinalities of the selected ** column for pNew and pOld are equal, is pNew to be preferred over pOld. ** ** This function assumes that for each argument sample, the contents of -** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. +** the anEq[] array from pSample->anEq[pSample->iCol+1] onwards are valid. */ static int sampleIsBetterPost( - Stat4Accum *pAccum, - Stat4Sample *pNew, - Stat4Sample *pOld + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld ){ int nCol = pAccum->nCol; int i; @@ -94592,17 +119559,17 @@ static int sampleIsBetterPost( } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Return true if pNew is to be preferred over pOld. ** ** This function assumes that for each argument sample, the contents of -** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. +** the anEq[] array from pSample->anEq[pSample->iCol] onwards are valid. */ static int sampleIsBetter( - Stat4Accum *pAccum, - Stat4Sample *pNew, - Stat4Sample *pOld + StatAccum *pAccum, + StatSample *pNew, + StatSample *pOld ){ tRowcnt nEqNew = pNew->anEq[pNew->iCol]; tRowcnt nEqOld = pOld->anEq[pOld->iCol]; @@ -94611,39 +119578,41 @@ static int sampleIsBetter( assert( IsStat4 || (pNew->iCol==0 && pOld->iCol==0) ); if( (nEqNew>nEqOld) ) return 1; -#ifdef SQLITE_ENABLE_STAT4 if( nEqNew==nEqOld ){ if( pNew->iColiCol ) return 1; return (pNew->iCol==pOld->iCol && sampleIsBetterPost(pAccum, pNew, pOld)); } return 0; -#else - return (nEqNew==nEqOld && pNew->iHash>pOld->iHash); -#endif } /* ** Copy the contents of sample *pNew into the p->a[] array. If necessary, ** remove the least desirable sample from p->a[] to make room. */ -static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ - Stat4Sample *pSample = 0; +static void sampleInsert(StatAccum *p, StatSample *pNew, int nEqZero){ + StatSample *pSample = 0; int i; assert( IsStat4 || nEqZero==0 ); -#ifdef SQLITE_ENABLE_STAT4 + /* StatAccum.nMaxEqZero is set to the maximum number of leading 0 + ** values in the anEq[] array of any sample in StatAccum.a[]. In + ** other words, if nMaxEqZero is n, then it is guaranteed that there + ** are no samples with StatSample.anEq[m]==0 for (m>=n). */ + if( nEqZero>p->nMaxEqZero ){ + p->nMaxEqZero = nEqZero; + } if( pNew->isPSample==0 ){ - Stat4Sample *pUpgrade = 0; + StatSample *pUpgrade = 0; assert( pNew->anEq[pNew->iCol]>0 ); - /* This sample is being added because the prefix that ends in column + /* This sample is being added because the prefix that ends in column ** iCol occurs many times in the table. However, if we have already ** added a sample that shares this prefix, there is no need to add ** this one. Instead, upgrade the priority of the highest priority ** existing sample that shares this prefix. */ for(i=p->nSample-1; i>=0; i--){ - Stat4Sample *pOld = &p->a[i]; + StatSample *pOld = &p->a[i]; if( pOld->anEq[pNew->iCol]==0 ){ if( pOld->isPSample ) return; assert( pOld->iCol>pNew->iCol ); @@ -94659,11 +119628,10 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ goto find_new_min; } } -#endif /* If necessary, remove sample iMin to make room for the new sample. */ if( p->nSample>=p->mxSample ){ - Stat4Sample *pMin = &p->a[p->iMin]; + StatSample *pMin = &p->a[p->iMin]; tRowcnt *anEq = pMin->anEq; tRowcnt *anLt = pMin->anLt; tRowcnt *anDLt = pMin->anDLt; @@ -94680,10 +119648,8 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ /* The "rows less-than" for the rowid column must be greater than that ** for the last sample in the p->a[] array. Otherwise, the samples would ** be out of order. */ -#ifdef SQLITE_ENABLE_STAT4 - assert( p->nSample==0 + assert( p->nSample==0 || pNew->anLt[p->nCol-1] > p->a[p->nSample-1].anLt[p->nCol-1] ); -#endif /* Insert the new sample */ pSample = &p->a[p->nSample]; @@ -94693,9 +119659,7 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ /* Zero the first nEqZero entries in the anEq[] array. */ memset(pSample->anEq, 0, sizeof(tRowcnt)*nEqZero); -#ifdef SQLITE_ENABLE_STAT4 - find_new_min: -#endif +find_new_min: if( p->nSample>=p->mxSample ){ int iMin = -1; for(i=0; imxSample; i++){ @@ -94708,79 +119672,66 @@ static void sampleInsert(Stat4Accum *p, Stat4Sample *pNew, int nEqZero){ p->iMin = iMin; } } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ +#ifdef SQLITE_ENABLE_STAT4 /* ** Field iChng of the index being scanned has changed. So at this point ** p->current contains a sample that reflects the previous row of the ** index. The value of anEq[iChng] and subsequent anEq[] elements are ** correct at this point. */ -static void samplePushPrevious(Stat4Accum *p, int iChng){ -#ifdef SQLITE_ENABLE_STAT4 +static void samplePushPrevious(StatAccum *p, int iChng){ int i; /* Check if any samples from the aBest[] array should be pushed ** into IndexSample.a[] at this point. */ for(i=(p->nCol-2); i>=iChng; i--){ - Stat4Sample *pBest = &p->aBest[i]; + StatSample *pBest = &p->aBest[i]; pBest->anEq[i] = p->current.anEq[i]; if( p->nSamplemxSample || sampleIsBetter(p, pBest, &p->a[p->iMin]) ){ sampleInsert(p, pBest, i); } } - /* Update the anEq[] fields of any samples already collected. */ + /* Check that no sample contains an anEq[] entry with an index of + ** p->nMaxEqZero or greater set to zero. */ for(i=p->nSample-1; i>=0; i--){ int j; - for(j=iChng; jnCol; j++){ - if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; - } + for(j=p->nMaxEqZero; jnCol; j++) assert( p->a[i].anEq[j]>0 ); } -#endif -#if defined(SQLITE_ENABLE_STAT3) && !defined(SQLITE_ENABLE_STAT4) - if( iChng==0 ){ - tRowcnt nLt = p->current.anLt[0]; - tRowcnt nEq = p->current.anEq[0]; - - /* Check if this is to be a periodic sample. If so, add it. */ - if( (nLt/p->nPSample)!=(nLt+nEq)/p->nPSample ){ - p->current.isPSample = 1; - sampleInsert(p, &p->current, 0); - p->current.isPSample = 0; - }else - - /* Or if it is a non-periodic sample. Add it in this case too. */ - if( p->nSamplemxSample - || sampleIsBetter(p, &p->current, &p->a[p->iMin]) - ){ - sampleInsert(p, &p->current, 0); + /* Update the anEq[] fields of any samples already collected. */ + if( iChngnMaxEqZero ){ + for(i=p->nSample-1; i>=0; i--){ + int j; + for(j=iChng; jnCol; j++){ + if( p->a[i].anEq[j]==0 ) p->a[i].anEq[j] = p->current.anEq[j]; + } } + p->nMaxEqZero = iChng; } -#endif - -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 - UNUSED_PARAMETER( p ); - UNUSED_PARAMETER( iChng ); -#endif } +#endif /* SQLITE_ENABLE_STAT4 */ /* ** Implementation of the stat_push SQL function: stat_push(P,C,R) ** Arguments: ** -** P Pointer to the Stat4Accum object created by stat_init() +** P Pointer to the StatAccum object created by stat_init() ** C Index of left-most column to differ from previous row ** R Rowid for the current row. Might be a key record for ** WITHOUT ROWID tables. ** -** This SQL function always returns NULL. It's purpose it to accumulate -** statistical data and/or samples in the Stat4Accum object about the -** index being analyzed. The stat_get() SQL function will later be used to -** extract relevant information for constructing the sqlite_statN tables. +** The purpose of this routine is to collect statistical data and/or +** samples from the index being analyzed into the StatAccum object. +** The stat_get() SQL function will be used afterwards to +** retrieve the information gathered. ** -** The R parameter is only used for STAT3 and STAT4 +** This SQL function usually returns NULL, but might return an integer +** if it wants the byte-code to do special processing. +** +** The R parameter is only used for STAT4 */ static void statPush( sqlite3_context *context, @@ -94790,7 +119741,7 @@ static void statPush( int i; /* The three function arguments */ - Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); int iChng = sqlite3_value_int(argv[1]); UNUSED_PARAMETER( argc ); @@ -94800,39 +119751,44 @@ static void statPush( if( p->nRow==0 ){ /* This is the first call to this function. Do initialization. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; inCol; i++) p->current.anEq[i] = 1; +#endif }else{ /* Second and subsequent calls get processed here */ - samplePushPrevious(p, iChng); +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) samplePushPrevious(p, iChng); +#endif /* Update anDLt[], anLt[] and anEq[] to reflect the values that apply ** to the current row of the index. */ +#ifdef SQLITE_ENABLE_STAT4 for(i=0; icurrent.anEq[i]++; } +#endif for(i=iChng; inCol; i++){ p->current.anDLt[i]++; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - p->current.anLt[i] += p->current.anEq[i]; -#endif +#ifdef SQLITE_ENABLE_STAT4 + if( p->mxSample ) p->current.anLt[i] += p->current.anEq[i]; p->current.anEq[i] = 1; +#endif } } + p->nRow++; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ - sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); - }else{ - sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), - sqlite3_value_blob(argv[2])); - } - p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; -#endif - #ifdef SQLITE_ENABLE_STAT4 - { - tRowcnt nLt = p->current.anLt[p->nCol-1]; + if( p->mxSample ){ + tRowcnt nLt; + if( sqlite3_value_type(argv[2])==SQLITE_INTEGER ){ + sampleSetRowidInt64(p->db, &p->current, sqlite3_value_int64(argv[2])); + }else{ + sampleSetRowid(p->db, &p->current, sqlite3_value_bytes(argv[2]), + sqlite3_value_blob(argv[2])); + } + p->current.iHash = p->iPrn = p->iPrn*1103515245 + 12345; + nLt = p->current.anLt[p->nCol-1]; /* Check if this is to be a periodic sample. If so, add it. */ if( (nLt/p->nPSample)!=(nLt+1)/p->nPSample ){ p->current.isPSample = 1; @@ -94848,16 +119804,22 @@ static void statPush( sampleCopy(p, &p->aBest[i], &p->current); } } - } + }else #endif + if( p->nLimit && p->nRow>(tRowcnt)p->nLimit*(p->nSkipAhead+1) ){ + p->nSkipAhead++; + sqlite3_result_int(context, p->current.anDLt[0]>0); + } } + static const FuncDef statPushFuncdef = { - 2+IsStat34, /* nArg */ + 2+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ statPush, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_push", /* zName */ {0} }; @@ -94871,12 +119833,18 @@ static const FuncDef statPushFuncdef = { /* ** Implementation of the stat_get(P,J) SQL function. This routine is ** used to query statistical information that has been gathered into -** the Stat4Accum object by prior calls to stat_push(). The P parameter -** has type BLOB but it is really just a pointer to the Stat4Accum object. +** the StatAccum object by prior calls to stat_push(). The P parameter +** has type BLOB but it is really just a pointer to the StatAccum object. ** The content to returned is determined by the parameter J ** which is one of the STAT_GET_xxxx values defined above. ** -** If neither STAT3 nor STAT4 are enabled, then J is always +** The stat_get(P,J) function is not available to generic SQL. It is +** inserted as part of a manually constructed bytecode program. (See +** the callStatGet() routine below.) It is guaranteed that the P +** parameter will always be a pointer to a StatAccum object, never a +** NULL. +** +** If STAT4 is not enabled, then J is always ** STAT_GET_STAT1 and is hence omitted and this routine becomes ** a one-parameter function, stat_get(P), that always returns the ** stat1 table entry information. @@ -94886,15 +119854,16 @@ static void statGet( int argc, sqlite3_value **argv ){ - Stat4Accum *p = (Stat4Accum*)sqlite3_value_blob(argv[0]); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* STAT3 and STAT4 have a parameter on this routine. */ + StatAccum *p = (StatAccum*)sqlite3_value_blob(argv[0]); +#ifdef SQLITE_ENABLE_STAT4 + /* STAT4 has a parameter on this routine. */ int eCall = sqlite3_value_int(argv[1]); assert( argc==2 ); - assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ + assert( eCall==STAT_GET_STAT1 || eCall==STAT_GET_NEQ || eCall==STAT_GET_ROWID || eCall==STAT_GET_NLT - || eCall==STAT_GET_NDLT + || eCall==STAT_GET_NDLT ); + assert( eCall==STAT_GET_STAT1 || p->mxSample ); if( eCall==STAT_GET_STAT1 ) #else assert( argc==1 ); @@ -94903,54 +119872,54 @@ static void statGet( /* Return the value to store in the "stat" column of the sqlite_stat1 ** table for this index. ** - ** The value is a string composed of a list of integers describing - ** the index. The first integer in the list is the total number of - ** entries in the index. There is one additional integer in the list + ** The value is a string composed of a list of integers describing + ** the index. The first integer in the list is the total number of + ** entries in the index. There is one additional integer in the list ** for each indexed column. This additional integer is an estimate of - ** the number of rows matched by a stabbing query on the index using + ** the number of rows matched by a equality query on the index using ** a key with the corresponding number of fields. In other words, - ** if the index is on columns (a,b) and the sqlite_stat1 value is + ** if the index is on columns (a,b) and the sqlite_stat1 value is ** "100 10 2", then SQLite estimates that: ** ** * the index contains 100 rows, ** * "WHERE a=?" matches 10 rows, and ** * "WHERE a=? AND b=?" matches 2 rows. ** - ** If D is the count of distinct values and K is the total number of - ** rows, then each estimate is computed as: + ** If D is the count of distinct values and K is the total number of + ** rows, then each estimate is usually computed as: ** ** I = (K+D-1)/D + ** + ** In other words, I is K/D rounded up to the next whole integer. + ** However, if I is between 1.0 and 1.1 (in other words if I is + ** close to 1.0 but just a little larger) then do not round up but + ** instead keep the I value at 1.0. */ - char *z; - int i; + sqlite3_str sStat; /* Text of the constructed "stat" line */ + int i; /* Loop counter */ - char *zRet = sqlite3MallocZero( (p->nKeyCol+1)*25 ); - if( zRet==0 ){ - sqlite3_result_error_nomem(context); - return; - } - - sqlite3_snprintf(24, zRet, "%llu", (u64)p->nRow); - z = zRet + sqlite3Strlen30(zRet); + sqlite3StrAccumInit(&sStat, 0, 0, 0, (p->nKeyCol+1)*100); + sqlite3_str_appendf(&sStat, "%llu", + p->nSkipAhead ? (u64)p->nEst : (u64)p->nRow); for(i=0; inKeyCol; i++){ u64 nDistinct = p->current.anDLt[i] + 1; u64 iVal = (p->nRow + nDistinct - 1) / nDistinct; - sqlite3_snprintf(24, z, " %llu", iVal); - z += sqlite3Strlen30(z); - assert( p->current.anEq[i] ); + if( iVal==2 && p->nRow*10 <= nDistinct*11 ) iVal = 1; + sqlite3_str_appendf(&sStat, " %llu", iVal); +#ifdef SQLITE_ENABLE_STAT4 + assert( p->current.anEq[i] || p->nRow==0 ); +#endif } - assert( z[0]=='\0' && z>zRet ); - - sqlite3_result_text(context, zRet, -1, sqlite3_free); + sqlite3ResultStrAccum(context, &sStat); } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 else if( eCall==STAT_GET_ROWID ){ if( p->iGet<0 ){ samplePushPrevious(p, 0); p->iGet = 0; } if( p->iGetnSample ){ - Stat4Sample *pS = p->a + p->iGet; + StatSample *pS = p->a + p->iGet; if( pS->nRowid==0 ){ sqlite3_result_int64(context, pS->u.iRowid); }else{ @@ -94960,67 +119929,81 @@ static void statGet( } }else{ tRowcnt *aCnt = 0; + sqlite3_str sStat; + int i; assert( p->iGetnSample ); switch( eCall ){ case STAT_GET_NEQ: aCnt = p->a[p->iGet].anEq; break; case STAT_GET_NLT: aCnt = p->a[p->iGet].anLt; break; default: { - aCnt = p->a[p->iGet].anDLt; + aCnt = p->a[p->iGet].anDLt; p->iGet++; break; } } - - if( IsStat3 ){ - sqlite3_result_int64(context, (i64)aCnt[0]); - }else{ - char *zRet = sqlite3MallocZero(p->nCol * 25); - if( zRet==0 ){ - sqlite3_result_error_nomem(context); - }else{ - int i; - char *z = zRet; - for(i=0; inCol; i++){ - sqlite3_snprintf(24, z, "%llu ", (u64)aCnt[i]); - z += sqlite3Strlen30(z); - } - assert( z[0]=='\0' && z>zRet ); - z[-1] = '\0'; - sqlite3_result_text(context, zRet, -1, sqlite3_free); - } + sqlite3StrAccumInit(&sStat, 0, 0, 0, p->nCol*100); + for(i=0; inCol; i++){ + sqlite3_str_appendf(&sStat, "%llu ", (u64)aCnt[i]); } + if( sStat.nChar ) sStat.nChar--; + sqlite3ResultStrAccum(context, &sStat); } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ #ifndef SQLITE_DEBUG UNUSED_PARAMETER( argc ); #endif } static const FuncDef statGetFuncdef = { - 1+IsStat34, /* nArg */ + 1+IsStat4, /* nArg */ SQLITE_UTF8, /* funcFlags */ 0, /* pUserData */ 0, /* pNext */ statGet, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "stat_get", /* zName */ {0} }; -static void callStatGet(Vdbe *v, int regStat4, int iParam, int regOut){ - assert( regOut!=regStat4 && regOut!=regStat4+1 ); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Integer, iParam, regStat4+1); +static void callStatGet(Parse *pParse, int regStat, int iParam, int regOut){ +#ifdef SQLITE_ENABLE_STAT4 + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Integer, iParam, regStat+1); #elif SQLITE_DEBUG assert( iParam==STAT_GET_STAT1 ); #else UNUSED_PARAMETER( iParam ); #endif - sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4, regOut, - (char*)&statGetFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 1 + IsStat34); + assert( regOut!=regStat && regOut!=regStat+1 ); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat, regOut, 1+IsStat4, + &statGetFuncdef, 0); } +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS +/* Add a comment to the most recent VDBE opcode that is the name +** of the k-th column of the pIdx index. +*/ +static void analyzeVdbeCommentIndexWithColumnName( + Vdbe *v, /* Prepared statement under construction */ + Index *pIdx, /* Index whose column is being loaded */ + int k /* Which column index */ +){ + int i; /* Index of column in the table */ + assert( k>=0 && knColumn ); + i = pIdx->aiColumn[k]; + if( NEVER(i==XN_ROWID) ){ + VdbeComment((v,"%s.rowid",pIdx->zName)); + }else if( i==XN_EXPR ){ + assert( pIdx->bHasExpr ); + VdbeComment((v,"%s.expr(%d)",pIdx->zName, k)); + }else{ + VdbeComment((v,"%s.%s", pIdx->zName, pIdx->pTable->aCol[i].zCnName)); + } +} +#else +# define analyzeVdbeCommentIndexWithColumnName(a,b,c) +#endif /* SQLITE_DEBUG */ + /* ** Generate code to do an analysis of all indices associated with ** a single table. @@ -95043,27 +120026,33 @@ static void analyzeOneTable( int iDb; /* Index of database containing pTab */ u8 needTableCnt = 1; /* True to count the table */ int regNewRowid = iMem++; /* Rowid for the inserted record */ - int regStat4 = iMem++; /* Register to hold Stat4Accum object */ + int regStat = iMem++; /* Register to hold StatAccum object */ int regChng = iMem++; /* Index of changed index field */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 int regRowid = iMem++; /* Rowid argument passed to stat_push() */ -#endif int regTemp = iMem++; /* Temporary use register */ + int regTemp2 = iMem++; /* Second temporary use register */ int regTabname = iMem++; /* Register containing table name */ int regIdxname = iMem++; /* Register containing index name */ int regStat1 = iMem++; /* Value for the stat column of sqlite_stat1 */ int regPrev = iMem; /* MUST BE LAST (see below) */ +#ifdef SQLITE_ENABLE_STAT4 + int doOnce = 1; /* Flag for a one-time computation */ +#endif +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + Table *pStat1 = 0; +#endif - pParse->nMem = MAX(pParse->nMem, iMem); + sqlite3TouchRegister(pParse, iMem); + assert( sqlite3NoTempsInRange(pParse, regNewRowid, iMem) ); v = sqlite3GetVdbe(pParse); if( v==0 || NEVER(pTab==0) ){ return; } - if( pTab->tnum==0 ){ + if( !IsOrdinaryTable(pTab) ){ /* Do not gather statistics on views or virtual tables */ return; } - if( sqlite3_strlike("sqlite_%", pTab->zName, 0)==0 ){ + if( sqlite3_strlike("sqlite\\_%", pTab->zName, '\\')==0 ){ /* Do not gather statistics on system tables */ return; } @@ -95073,12 +120062,24 @@ static void analyzeOneTable( assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_ANALYZE, pTab->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif - /* Establish a read-lock on the table at the shared-cache level. +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( db->xPreUpdateCallback ){ + pStat1 = (Table*)sqlite3DbMallocZero(db, sizeof(Table) + 13); + if( pStat1==0 ) return; + pStat1->zName = (char*)&pStat1[1]; + memcpy(pStat1->zName, "sqlite_stat1", 13); + pStat1->nCol = 3; + pStat1->iPKey = -1; + sqlite3VdbeAddOp4(pParse->pVdbe, OP_Noop, 0, 0, 0,(char*)pStat1,P4_DYNAMIC); + } +#endif + + /* Establish a read-lock on the table at the shared-cache level. ** Open a read-only cursor on the table. Also allocate a cursor number ** to use for scanning indexes (iIdxCur). No index cursor is opened at ** this time though. */ @@ -95091,7 +120092,7 @@ static void analyzeOneTable( for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ int nCol; /* Number of columns in pIdx. "N" */ - int addrRewind; /* Address of "OP_Rewind iIdxCur" */ + int addrGotoEnd; /* Address of "OP_Rewind iIdxCur" */ int addrNextRow; /* Address of "next_row:" */ const char *zIdxName; /* Name of the index */ int nColTest; /* Number of columns to test for changes */ @@ -95115,9 +120116,14 @@ static void analyzeOneTable( /* ** Pseudo-code for loop that calls stat_push(): ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() ** goto chng_addr_0; ** ** next_row: @@ -95144,11 +120150,11 @@ static void analyzeOneTable( ** end_of_scan: */ - /* Make sure there are enough memory cells allocated to accommodate + /* Make sure there are enough memory cells allocated to accommodate ** the regPrev array and a trailing rowid (the rowid slot is required - ** when building a record to insert into the sample column of + ** when building a record to insert into the sample column of ** the sqlite_stat4 table. */ - pParse->nMem = MAX(pParse->nMem, regPrev+nColTest); + sqlite3TouchRegister(pParse, regPrev+nColTest); /* Open a read-only cursor on the index being analyzed. */ assert( iDb==sqlite3SchemaToIndex(db, pIdx->pSchema) ); @@ -95156,40 +120162,41 @@ static void analyzeOneTable( sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "%s", pIdx->zName)); - /* Invoke the stat_init() function. The arguments are: - ** + /* Implementation of the following: + ** + ** regChng = 0 + ** Rewind csr + ** if eof(csr){ + ** stat_init() with count = 0; + ** goto end_of_scan; + ** } + ** count() + ** stat_init() + ** goto chng_addr_0; + */ + assert( regTemp2==regStat+4 ); + sqlite3VdbeAddOp2(v, OP_Integer, db->nAnalysisLimit, regTemp2); + + /* Arguments to stat_init(): ** (1) the number of columns in the index including the rowid ** (or for a WITHOUT ROWID table, the number of PK columns), ** (2) the number of columns in the key without the rowid/pk - ** (3) the number of rows in the index, - ** - ** - ** The third argument is only used for STAT3 and STAT4 - */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur, regStat4+3); -#endif - sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat4+1); - sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regStat4+2); - sqlite3VdbeAddOp4(v, OP_Function0, 0, regStat4+1, regStat4, - (char*)&statInitFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2+IsStat34); - - /* Implementation of the following: - ** - ** Rewind csr - ** if eof(csr) goto end_of_scan; - ** regChng = 0 - ** goto next_push_0; - ** - */ - addrRewind = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); + ** (3) estimated number of rows in the index. */ + sqlite3VdbeAddOp2(v, OP_Integer, nCol, regStat+1); + assert( regRowid==regStat+2 ); + sqlite3VdbeAddOp2(v, OP_Integer, pIdx->nKeyCol, regRowid); + sqlite3VdbeAddOp3(v, OP_Count, iIdxCur, regTemp, + OptimizationDisabled(db, SQLITE_Stat4)); + sqlite3VdbeAddFunctionCall(pParse, 0, regStat+1, regStat, 4, + &statInitFuncdef, 0); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_Rewind, iIdxCur); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Integer, 0, regChng); addrNextRow = sqlite3VdbeCurrentAddr(v); if( nColTest>0 ){ - int endDistinctTest = sqlite3VdbeMakeLabel(v); + int endDistinctTest = sqlite3VdbeMakeLabel(pParse); int *aGotoChng; /* Array of jump instruction addresses */ aGotoChng = sqlite3DbMallocRawNN(db, sizeof(int)*nColTest); if( aGotoChng==0 ) continue; @@ -95208,7 +120215,7 @@ static void analyzeOneTable( addrNextRow = sqlite3VdbeCurrentAddr(v); if( nColTest==1 && pIdx->nKeyCol==1 && IsUniqueIndex(pIdx) ){ /* For a single-column UNIQUE index, once we have found a non-NULL - ** row, we know that all the rest will be distinct, so skip + ** row, we know that all the rest will be distinct, so skip ** subsequent distinctness tests. */ sqlite3VdbeAddOp2(v, OP_NotNull, regPrev, endDistinctTest); VdbeCoverage(v); @@ -95217,15 +120224,16 @@ static void analyzeOneTable( char *pColl = (char*)sqlite3LocateCollSeq(pParse, pIdx->azColl[i]); sqlite3VdbeAddOp2(v, OP_Integer, i, regChng); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, i, regTemp); - aGotoChng[i] = + analyzeVdbeCommentIndexWithColumnName(v,pIdx,i); + aGotoChng[i] = sqlite3VdbeAddOp4(v, OP_Ne, regTemp, 0, regPrev+i, pColl, P4_COLLSEQ); sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_Integer, nColTest, regChng); sqlite3VdbeGoto(v, endDistinctTest); - - + + /* ** chng_addr_0: ** regPrev(0) = idx(0) @@ -95237,53 +120245,78 @@ static void analyzeOneTable( for(i=0; ipTable); - int j, k, regKey; - regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); - for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); - assert( k>=0 && knCol ); - sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); - VdbeComment((v, "%s", pTab->aCol[pPk->aiColumn[j]].zName)); +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) ){ + assert( regRowid==(regStat+2) ); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, regRowid); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); + int j, k, regKey; + regKey = sqlite3GetTempRange(pParse, pPk->nKeyCol); + for(j=0; jnKeyCol; j++){ + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); + assert( k>=0 && knColumn ); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, regKey+j); + analyzeVdbeCommentIndexWithColumnName(v,pIdx,k); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); + sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regKey, pPk->nKeyCol, regRowid); - sqlite3ReleaseTempRange(pParse, regKey, pPk->nKeyCol); } #endif - assert( regChng==(regStat4+1) ); - sqlite3VdbeAddOp4(v, OP_Function0, 1, regStat4, regTemp, - (char*)&statPushFuncdef, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, 2+IsStat34); - sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + assert( regChng==(regStat+1) ); + { + sqlite3VdbeAddFunctionCall(pParse, 1, regStat, regTemp, 2+IsStat4, + &statPushFuncdef, 0); + if( db->nAnalysisLimit ){ + int j1, j2, j3; + j1 = sqlite3VdbeAddOp1(v, OP_IsNull, regTemp); VdbeCoverage(v); + j2 = sqlite3VdbeAddOp1(v, OP_If, regTemp); VdbeCoverage(v); + j3 = sqlite3VdbeAddOp4Int(v, OP_SeekGT, iIdxCur, 0, regPrev, 1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j1); + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, j2); + sqlite3VdbeJumpHere(v, j3); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, iIdxCur, addrNextRow); VdbeCoverage(v); + } + } /* Add the entry to the stat1 table. */ - callStatGet(v, regStat4, STAT_GET_STAT1, regStat1); + if( pIdx->pPartIdxWhere ){ + /* Partial indexes might get a zero-entry in sqlite_stat1. But + ** an empty table is omitted from sqlite_stat1. */ + sqlite3VdbeJumpHere(v, addrGotoEnd); + addrGotoEnd = 0; + } + callStatGet(pParse, regStat, STAT_GET_STAT1, regStat1); assert( "BBB"[0]==SQLITE_AFF_TEXT ); sqlite3VdbeAddOp4(v, OP_MakeRecord, regTabname, 3, regTemp, "BBB", 0); sqlite3VdbeAddOp2(v, OP_NewRowid, iStatCur, regNewRowid); sqlite3VdbeAddOp3(v, OP_Insert, iStatCur, regTemp, regNewRowid); +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + sqlite3VdbeChangeP4(v, -1, (char*)pStat1, P4_TABLE); +#endif sqlite3VdbeChangeP5(v, OPFLAG_APPEND); - /* Add the entries to the stat3 or stat4 table. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - { + /* Add the entries to the stat4 table. */ +#ifdef SQLITE_ENABLE_STAT4 + if( OptimizationEnabled(db, SQLITE_Stat4) && db->nAnalysisLimit==0 ){ int regEq = regStat1; int regLt = regStat1+1; int regDLt = regStat1+2; @@ -95294,38 +120327,66 @@ static void analyzeOneTable( int addrIsNull; u8 seekOp = HasRowid(pTab) ? OP_NotExists : OP_NotFound; - pParse->nMem = MAX(pParse->nMem, regCol+nCol); + /* No STAT4 data is generated if the number of rows is zero */ + if( addrGotoEnd==0 ){ + sqlite3VdbeAddOp2(v, OP_Cast, regStat1, SQLITE_AFF_INTEGER); + addrGotoEnd = sqlite3VdbeAddOp1(v, OP_IfNot, regStat1); + VdbeCoverage(v); + } + + if( doOnce ){ + int mxCol = nCol; + Index *pX; + + /* Compute the maximum number of columns in any index */ + for(pX=pTab->pIndex; pX; pX=pX->pNext){ + int nColX; /* Number of columns in pX */ + if( !HasRowid(pTab) && IsPrimaryKeyIndex(pX) ){ + nColX = pX->nKeyCol; + }else{ + nColX = pX->nColumn; + } + if( nColX>mxCol ) mxCol = nColX; + } + + /* Allocate space to compute results for the largest index */ + sqlite3TouchRegister(pParse, regCol+mxCol); + doOnce = 0; +#ifdef SQLITE_DEBUG + /* Verify that the call to sqlite3ClearTempRegCache() below + ** really is needed. + ** https://sqlite.org/forum/forumpost/83cb4a95a0 (2023-03-25) + */ + testcase( !sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); +#endif + sqlite3ClearTempRegCache(pParse); /* tag-20230325-1 */ + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+mxCol) ); + } + assert( sqlite3NoTempsInRange(pParse, regEq, regCol+nCol) ); addrNext = sqlite3VdbeCurrentAddr(v); - callStatGet(v, regStat4, STAT_GET_ROWID, regSampleRowid); + callStatGet(pParse, regStat, STAT_GET_ROWID, regSampleRowid); addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regSampleRowid); VdbeCoverage(v); - callStatGet(v, regStat4, STAT_GET_NEQ, regEq); - callStatGet(v, regStat4, STAT_GET_NLT, regLt); - callStatGet(v, regStat4, STAT_GET_NDLT, regDLt); + callStatGet(pParse, regStat, STAT_GET_NEQ, regEq); + callStatGet(pParse, regStat, STAT_GET_NLT, regLt); + callStatGet(pParse, regStat, STAT_GET_NDLT, regDLt); sqlite3VdbeAddOp4Int(v, seekOp, iTabCur, addrNext, regSampleRowid, 0); - /* We know that the regSampleRowid row exists because it was read by - ** the previous loop. Thus the not-found jump of seekOp will never - ** be taken */ - VdbeCoverageNeverTaken(v); -#ifdef SQLITE_ENABLE_STAT3 - sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iTabCur, 0, regSample); -#else + VdbeCoverage(v); for(i=0; itblHash); k; k=sqliteHashNext(k)){ Table *pTab = (Table*)sqliteHashData(k); analyzeOneTable(pParse, pTab, 0, iStatCur, iMem, iTab); +#ifdef SQLITE_ENABLE_STAT4 + iMem = sqlite3FirstAvailableRegister(pParse, iMem); +#else + assert( iMem==sqlite3FirstAvailableRegister(pParse,iMem) ); +#endif } loadAnalysis(pParse, iDb); } @@ -95443,27 +120512,14 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ if( i==1 ) continue; /* Do not analyze the TEMP database */ analyzeDatabase(pParse, i); } - }else if( pName2->n==0 ){ - /* Form 2: Analyze the database or table named */ - iDb = sqlite3FindDb(db, pName1); - if( iDb>=0 ){ - analyzeDatabase(pParse, iDb); - }else{ - z = sqlite3NameFromToken(db, pName1); - if( z ){ - if( (pIdx = sqlite3FindIndex(db, z, 0))!=0 ){ - analyzeTable(pParse, pIdx->pTable, pIdx); - }else if( (pTab = sqlite3LocateTable(pParse, 0, z, 0))!=0 ){ - analyzeTable(pParse, pTab, 0); - } - sqlite3DbFree(db, z); - } - } + }else if( pName2->n==0 && (iDb = sqlite3FindDb(db, pName1))>=0 ){ + /* Analyze the schema named as the argument */ + analyzeDatabase(pParse, iDb); }else{ - /* Form 3: Analyze the fully qualified table name */ + /* Form 3: Analyze the table or index named as an argument */ iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pTableName); if( iDb>=0 ){ - zDb = db->aDb[iDb].zName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; z = sqlite3NameFromToken(db, pTableName); if( z ){ if( (pIdx = sqlite3FindIndex(db, z, zDb))!=0 ){ @@ -95473,10 +120529,11 @@ SQLITE_PRIVATE void sqlite3Analyze(Parse *pParse, Token *pName1, Token *pName2){ } sqlite3DbFree(db, z); } - } + } + } + if( db->nSqlExec==0 && (v = sqlite3GetVdbe(pParse))!=0 ){ + sqlite3VdbeAddOp0(v, OP_Expire); } - v = sqlite3GetVdbe(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Expire); } /* @@ -95506,7 +120563,7 @@ static void decodeIntArray( int i; tRowcnt v; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( z==0 ) z = ""; #else assert( z!=0 ); @@ -95517,7 +120574,7 @@ static void decodeIntArray( v = v*10 + c - '0'; z++; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 if( aOut ) aOut[i] = v; if( aLog ) aLog[i] = sqlite3LogEst(v); #else @@ -95528,7 +120585,7 @@ static void decodeIntArray( #endif if( *z==' ' ) z++; } -#ifndef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifndef SQLITE_ENABLE_STAT4 assert( pIndex!=0 ); { #else if( pIndex ){ @@ -95539,7 +120596,9 @@ static void decodeIntArray( if( sqlite3_strglob("unordered*", z)==0 ){ pIndex->bUnordered = 1; }else if( sqlite3_strglob("sz=[0-9]*", z)==0 ){ - pIndex->szIdxRow = sqlite3LogEst(sqlite3Atoi(z+3)); + int sz = sqlite3Atoi(z+3); + if( sz<2 ) sz = 2; + pIndex->szIdxRow = sqlite3LogEst(sz); }else if( sqlite3_strglob("noskipscan*", z)==0 ){ pIndex->noSkipScan = 1; } @@ -95551,12 +120610,22 @@ static void decodeIntArray( while( z[0]!=0 && z[0]!=' ' ) z++; while( z[0]==' ' ) z++; } + + /* Set the bLowQual flag if the peak number of rows obtained + ** from a full equality match is so large that a full table scan + ** seems likely to be faster than using the index. + */ + if( aLog[0] > 66 /* Index has more than 100 rows */ + && aLog[0] <= aLog[nOut-1] /* And only a single value seen */ + ){ + pIndex->bLowQual = 1; + } } } /* ** This callback is invoked once for each index when reading the -** sqlite_stat1 table. +** sqlite_stat1 table. ** ** argv[0] = name of the table ** argv[1] = name of the index (might be NULL) @@ -95593,8 +120662,8 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ if( pIndex ){ tRowcnt *aiRowEst = 0; int nCol = pIndex->nKeyCol+1; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* Index.aiRowEst may already be set here if there are duplicate +#ifdef SQLITE_ENABLE_STAT4 + /* Index.aiRowEst may already be set here if there are duplicate ** sqlite_stat1 entries for this index. In that case just clobber ** the old data with the new instead of allocating a new array. */ if( pIndex->aiRowEst==0 ){ @@ -95605,7 +120674,11 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif pIndex->bUnordered = 0; decodeIntArray((char*)z, nCol, aiRowEst, pIndex->aiRowLogEst, pIndex); - if( pIndex->pPartIdxWhere==0 ) pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pIndex->hasStat1 = 1; + if( pIndex->pPartIdxWhere==0 ){ + pTable->nRowLogEst = pIndex->aiRowLogEst[0]; + pTable->tabFlags |= TF_HasStat1; + } }else{ Index fakeIdx; fakeIdx.szIdxRow = pTable->szTabRow; @@ -95614,6 +120687,7 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ #endif decodeIntArray((char*)z, 1, 0, &pTable->nRowLogEst, &fakeIdx); pTable->szTabRow = fakeIdx.szIdxRow; + pTable->tabFlags |= TF_HasStat1; } return 0; @@ -95624,7 +120698,9 @@ static int analysisLoader(void *pData, int argc, char **argv, char **NotUsed){ ** and its contents. */ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + assert( db!=0 ); + assert( pIdx!=0 ); +#ifdef SQLITE_ENABLE_STAT4 if( pIdx->aSample ){ int j; for(j=0; jnSample; j++){ @@ -95633,20 +120709,20 @@ SQLITE_PRIVATE void sqlite3DeleteIndexSamples(sqlite3 *db, Index *pIdx){ } sqlite3DbFree(db, pIdx->aSample); } - if( db && db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ pIdx->nSample = 0; pIdx->aSample = 0; } #else UNUSED_PARAMETER(db); UNUSED_PARAMETER(pIdx); -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Populate the pIdx->aAvgEq[] array based on the samples currently -** stored in pIdx->aSample[]. +** stored in pIdx->aSample[]. */ static void initAvgEq(Index *pIdx){ if( pIdx ){ @@ -95682,19 +120758,19 @@ static void initAvgEq(Index *pIdx){ pIdx->nRowEst0 = nRow; /* Set nSum to the number of distinct (iCol+1) field prefixes that - ** occur in the stat4 table for this index. Set sumEq to the sum of - ** the nEq values for column iCol for the same set (adding the value + ** occur in the stat4 table for this index. Set sumEq to the sum of + ** the nEq values for column iCol for the same set (adding the value ** only once where there exist duplicate prefixes). */ for(i=0; inSample-1) - || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] + || aSample[i].anDLt[iCol]!=aSample[i+1].anDLt[iCol] ){ sumEq += aSample[i].anEq[iCol]; nSum100 += 100; } } - if( nDist100>nSum100 ){ + if( nDist100>nSum100 && sumEqnSample==0 ); - /* Index.nSample is non-zero at this point if data has already been - ** loaded from the stat4 table. In this case ignore stat3 data. */ - if( pIdx==0 || pIdx->nSample ) continue; - if( bStat3==0 ){ - assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); - if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ - nIdxCol = pIdx->nKeyCol; - }else{ - nIdxCol = pIdx->nColumn; - } + assert( pIdx==0 || pIdx->nSample==0 ); + if( pIdx==0 ) continue; + if( pIdx->aSample!=0 ){ + /* The same index appears in sqlite_stat4 under multiple names */ + continue; + } + assert( !HasRowid(pIdx->pTable) || pIdx->nColumn==pIdx->nKeyCol+1 ); + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nIdxCol = pIdx->nKeyCol; + }else{ + nIdxCol = pIdx->nColumn; } pIdx->nSampleCol = nIdxCol; - nByte = sizeof(IndexSample) * nSample; + pIdx->mxSample = nSample; + nByte = ROUND8(sizeof(IndexSample) * nSample); nByte += sizeof(tRowcnt) * nIdxCol * 3 * nSample; nByte += nIdxCol * sizeof(tRowcnt); /* Space for Index.aAvgEq[] */ @@ -95791,8 +120867,12 @@ static int loadStatTbl( sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; } - pSpace = (tRowcnt*)&pIdx->aSample[nSample]; + pPtr = (u8*)pIdx->aSample; + pPtr += ROUND8(nSample*sizeof(pIdx->aSample[0])); + pSpace = (tRowcnt*)pPtr; + assert( EIGHT_BYTE_ALIGNMENT( pSpace ) ); pIdx->aAvgEq = pSpace; pSpace += nIdxCol; + pIdx->pTable->tabFlags |= TF_HasStat4; for(i=0; iaSample[i].anEq = pSpace; pSpace += nIdxCol; pIdx->aSample[i].anLt = pSpace; pSpace += nIdxCol; @@ -95820,10 +120900,14 @@ static int loadStatTbl( if( zIndex==0 ) continue; pIdx = findIndexOrPrimaryKey(db, zIndex, zDb); if( pIdx==0 ) continue; - /* This next condition is true if data has already been loaded from - ** the sqlite_stat4 table. In this case ignore stat3 data. */ + if( pIdx->nSample>=pIdx->mxSample ){ + /* Too many slots used because the same index appears in + ** sqlite_stat4 using multiple names */ + continue; + } + /* This next condition is true if data has already been loaded from + ** the sqlite_stat4 table. */ nCol = pIdx->nSampleCol; - if( bStat3 && nCol>1 ) continue; if( pIdx!=pPrevIdx ){ initAvgEq(pPrevIdx); pPrevIdx = pIdx; @@ -95833,19 +120917,22 @@ static int loadStatTbl( decodeIntArray((char*)sqlite3_column_text(pStmt,2),nCol,pSample->anLt,0,0); decodeIntArray((char*)sqlite3_column_text(pStmt,3),nCol,pSample->anDLt,0,0); - /* Take a copy of the sample. Add two 0x00 bytes the end of the buffer. + /* Take a copy of the sample. Add 8 extra 0x00 bytes the end of the buffer. ** This is in case the sample record is corrupted. In that case, the ** sqlite3VdbeRecordCompare() may read up to two varints past the ** end of the allocated buffer before it realizes it is dealing with - ** a corrupt record. Adding the two 0x00 bytes prevents this from causing + ** a corrupt record. Or it might try to read a large integer from the + ** buffer. In any case, eight 0x00 bytes prevents this from causing ** a buffer overread. */ pSample->n = sqlite3_column_bytes(pStmt, 4); - pSample->p = sqlite3DbMallocZero(db, pSample->n + 2); + pSample->p = sqlite3DbMallocZero(db, pSample->n + 8); if( pSample->p==0 ){ sqlite3_finalize(pStmt); return SQLITE_NOMEM_BKPT; } - memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + if( pSample->n ){ + memcpy(pSample->p, sqlite3_column_blob(pStmt, 4), pSample->n); + } pIdx->nSample++; } rc = sqlite3_finalize(pStmt); @@ -95854,45 +120941,40 @@ static int loadStatTbl( } /* -** Load content from the sqlite_stat4 and sqlite_stat3 tables into +** Load content from the sqlite_stat4 table into ** the Index.aSample[] arrays of all indices. */ static int loadStat4(sqlite3 *db, const char *zDb){ int rc = SQLITE_OK; /* Result codes from subroutines */ + const Table *pStat4; assert( db->lookaside.bDisable ); - if( sqlite3FindTable(db, "sqlite_stat4", zDb) ){ - rc = loadStatTbl(db, 0, - "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx", + if( OptimizationEnabled(db, SQLITE_Stat4) + && (pStat4 = sqlite3FindTable(db, "sqlite_stat4", zDb))!=0 + && IsOrdinaryTable(pStat4) + ){ + rc = loadStatTbl(db, + "SELECT idx,count(*) FROM %Q.sqlite_stat4 GROUP BY idx COLLATE nocase", "SELECT idx,neq,nlt,ndlt,sample FROM %Q.sqlite_stat4", zDb ); } - - if( rc==SQLITE_OK && sqlite3FindTable(db, "sqlite_stat3", zDb) ){ - rc = loadStatTbl(db, 1, - "SELECT idx,count(*) FROM %Q.sqlite_stat3 GROUP BY idx", - "SELECT idx,neq,nlt,ndlt,sqlite_record(sample) FROM %Q.sqlite_stat3", - zDb - ); - } - return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* -** Load the content of the sqlite_stat1 and sqlite_stat3/4 tables. The +** Load the content of the sqlite_stat1 and sqlite_stat4 tables. The ** contents of sqlite_stat1 are used to populate the Index.aiRowEst[] -** arrays. The contents of sqlite_stat3/4 are used to populate the +** arrays. The contents of sqlite_stat4 are used to populate the ** Index.aSample[] arrays. ** ** If the sqlite_stat1 table is not present in the database, SQLITE_ERROR -** is returned. In this case, even if SQLITE_ENABLE_STAT3/4 was defined -** during compilation and the sqlite_stat3/4 table is present, no data is +** is returned. In this case, even if SQLITE_ENABLE_STAT4 was defined +** during compilation and the sqlite_stat4 table is present, no data is ** read from it. ** -** If SQLITE_ENABLE_STAT3/4 was defined during compilation and the +** If SQLITE_ENABLE_STAT4 was defined during compilation and the ** sqlite_stat4 table is not present in the database, SQLITE_ERROR is ** returned. However, in this case, data is read from the sqlite_stat1 ** table (if it is present) before returning. @@ -95906,16 +120988,22 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ HashElem *i; char *zSql; int rc = SQLITE_OK; + Schema *pSchema = db->aDb[iDb].pSchema; + const Table *pStat1; assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pBt!=0 ); /* Clear any prior statistics */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + pTab->tabFlags &= ~TF_HasStat1; + } + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - pIdx->aiRowLogEst[0] = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + pIdx->hasStat1 = 0; +#ifdef SQLITE_ENABLE_STAT4 sqlite3DeleteIndexSamples(db, pIdx); pIdx->aSample = 0; #endif @@ -95923,9 +121011,11 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Load new statistics out of the sqlite_stat1 table */ sInfo.db = db; - sInfo.zDatabase = db->aDb[iDb].zName; - if( sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)!=0 ){ - zSql = sqlite3MPrintf(db, + sInfo.zDatabase = db->aDb[iDb].zDbSName; + if( (pStat1 = sqlite3FindTable(db, "sqlite_stat1", sInfo.zDatabase)) + && IsOrdinaryTable(pStat1) + ){ + zSql = sqlite3MPrintf(db, "SELECT tbl,idx,stat FROM %Q.sqlite_stat1", sInfo.zDatabase); if( zSql==0 ){ rc = SQLITE_NOMEM_BKPT; @@ -95937,19 +121027,19 @@ SQLITE_PRIVATE int sqlite3AnalysisLoad(sqlite3 *db, int iDb){ /* Set appropriate defaults on all indexes not in the sqlite_stat1 table */ assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); - if( pIdx->aiRowLogEst[0]==0 ) sqlite3DefaultRowEst(pIdx); + if( !pIdx->hasStat1 ) sqlite3DefaultRowEst(pIdx); } /* Load the statistics from the sqlite_stat4 table. */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - if( rc==SQLITE_OK && OptimizationEnabled(db, SQLITE_Stat34) ){ - db->lookaside.bDisable++; +#ifdef SQLITE_ENABLE_STAT4 + if( rc==SQLITE_OK ){ + DisableLookaside; rc = loadStat4(db, sInfo.zDatabase); - db->lookaside.bDisable--; + EnableLookaside; } - for(i=sqliteHashFirst(&db->aDb[iDb].pSchema->idxHash);i;i=sqliteHashNext(i)){ + for(i=sqliteHashFirst(&pSchema->idxHash); i; i=sqliteHashNext(i)){ Index *pIdx = sqliteHashData(i); sqlite3_free(pIdx->aiRowEst); pIdx->aiRowEst = 0; @@ -96014,6 +121104,17 @@ static int resolveAttachExpr(NameContext *pName, Expr *pExpr) return rc; } +/* +** Return true if zName points to a name that may be used to refer to +** database iDb attached to handle db. +*/ +SQLITE_PRIVATE int sqlite3DbIsNamed(sqlite3 *db, int iDb, const char *zName){ + return ( + sqlite3StrICmp(db->aDb[iDb].zDbSName, zName)==0 + || (iDb==0 && sqlite3StrICmp("main", zName)==0) + ); +} + /* ** An SQL user-function registered to do the work of an ATTACH statement. The ** three arguments to the function come directly from an attach statement: @@ -96024,6 +121125,10 @@ static int resolveAttachExpr(NameContext *pName, Expr *pExpr) ** ** If the optional "KEY z" syntax is omitted, an SQL NULL is passed as the ** third argument. +** +** If the db->init.reopenMemdb flags is set, then instead of attaching a +** new database, close the database on db->init.iDb and reopen it as an +** empty MemDB. */ static void attachFunc( sqlite3_context *context, @@ -96038,149 +121143,151 @@ static void attachFunc( char *zPath = 0; char *zErr = 0; unsigned int flags; - Db *aNew; + Db *aNew; /* New array of Db pointers */ + Db *pNew = 0; /* Db object for the newly attached database */ char *zErrDyn = 0; sqlite3_vfs *pVfs; UNUSED_PARAMETER(NotUsed); - zFile = (const char *)sqlite3_value_text(argv[0]); zName = (const char *)sqlite3_value_text(argv[1]); if( zFile==0 ) zFile = ""; if( zName==0 ) zName = ""; - /* Check for the following errors: - ** - ** * Too many attached databases, - ** * Transaction currently open - ** * Specified database name already being used. - */ - if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ - zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", - db->aLimit[SQLITE_LIMIT_ATTACHED] - ); - goto attach_error; - } - if( !db->autoCommit ){ - zErrDyn = sqlite3MPrintf(db, "cannot ATTACH database within transaction"); - goto attach_error; - } - for(i=0; inDb; i++){ - char *z = db->aDb[i].zName; - assert( z && zName ); - if( sqlite3StrICmp(z, zName)==0 ){ - zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); +#ifndef SQLITE_OMIT_DESERIALIZE +# define REOPEN_AS_MEMDB(db) (db->init.reopenMemdb) +#else +# define REOPEN_AS_MEMDB(db) (0) +#endif + + if( REOPEN_AS_MEMDB(db) ){ + /* This is not a real ATTACH. Instead, this routine is being called + ** from sqlite3_deserialize() to close database db->init.iDb and + ** reopen it as a MemDB */ + Btree *pNewBt = 0; + pVfs = sqlite3_vfs_find("memdb"); + if( pVfs==0 ) return; + rc = sqlite3BtreeOpen(pVfs, "x\0", db, &pNewBt, 0, SQLITE_OPEN_MAIN_DB); + if( rc==SQLITE_OK ){ + Schema *pNewSchema = sqlite3SchemaGet(db, pNewBt); + if( pNewSchema ){ + /* Both the Btree and the new Schema were allocated successfully. + ** Close the old db and update the aDb[] slot with the new memdb + ** values. */ + pNew = &db->aDb[db->init.iDb]; + if( ALWAYS(pNew->pBt) ) sqlite3BtreeClose(pNew->pBt); + pNew->pBt = pNewBt; + pNew->pSchema = pNewSchema; + }else{ + sqlite3BtreeClose(pNewBt); + rc = SQLITE_NOMEM; + } + } + if( rc ) goto attach_error; + }else{ + /* This is a real ATTACH + ** + ** Check for the following errors: + ** + ** * Too many attached databases, + ** * Transaction currently open + ** * Specified database name already being used. + */ + if( db->nDb>=db->aLimit[SQLITE_LIMIT_ATTACHED]+2 ){ + zErrDyn = sqlite3MPrintf(db, "too many attached databases - max %d", + db->aLimit[SQLITE_LIMIT_ATTACHED] + ); goto attach_error; } - } + for(i=0; inDb; i++){ + assert( zName ); + if( sqlite3DbIsNamed(db, i, zName) ){ + zErrDyn = sqlite3MPrintf(db, "database %s is already in use", zName); + goto attach_error; + } + } - /* Allocate the new entry in the db->aDb[] array and initialize the schema - ** hash tables. - */ - if( db->aDb==db->aDbStatic ){ - aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); - if( aNew==0 ) return; - memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); - }else{ - aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); - if( aNew==0 ) return; - } - db->aDb = aNew; - aNew = &db->aDb[db->nDb]; - memset(aNew, 0, sizeof(*aNew)); + /* Allocate the new entry in the db->aDb[] array and initialize the schema + ** hash tables. + */ + if( db->aDb==db->aDbStatic ){ + aNew = sqlite3DbMallocRawNN(db, sizeof(db->aDb[0])*3 ); + if( aNew==0 ) return; + memcpy(aNew, db->aDb, sizeof(db->aDb[0])*2); + }else{ + aNew = sqlite3DbRealloc(db, db->aDb, sizeof(db->aDb[0])*(db->nDb+1) ); + if( aNew==0 ) return; + } + db->aDb = aNew; + pNew = &db->aDb[db->nDb]; + memset(pNew, 0, sizeof(*pNew)); - /* Open the database file. If the btree is successfully opened, use - ** it to obtain the database schema. At this point the schema may - ** or may not be initialized. - */ - flags = db->openFlags; - rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); - if( rc!=SQLITE_OK ){ - if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); - return; + /* Open the database file. If the btree is successfully opened, use + ** it to obtain the database schema. At this point the schema may + ** or may not be initialized. + */ + flags = db->openFlags; + rc = sqlite3ParseUri(db->pVfs->zName, zFile, &flags, &pVfs, &zPath, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); + return; + } + assert( pVfs ); + flags |= SQLITE_OPEN_MAIN_DB; + rc = sqlite3BtreeOpen(pVfs, zPath, db, &pNew->pBt, 0, flags); + db->nDb++; + pNew->zDbSName = sqlite3DbStrDup(db, zName); } - assert( pVfs ); - flags |= SQLITE_OPEN_MAIN_DB; - rc = sqlite3BtreeOpen(pVfs, zPath, db, &aNew->pBt, 0, flags); - sqlite3_free( zPath ); - db->nDb++; + db->noSharedCache = 0; if( rc==SQLITE_CONSTRAINT ){ rc = SQLITE_ERROR; zErrDyn = sqlite3MPrintf(db, "database is already attached"); }else if( rc==SQLITE_OK ){ Pager *pPager; - aNew->pSchema = sqlite3SchemaGet(db, aNew->pBt); - if( !aNew->pSchema ){ + pNew->pSchema = sqlite3SchemaGet(db, pNew->pBt); + if( !pNew->pSchema ){ rc = SQLITE_NOMEM_BKPT; - }else if( aNew->pSchema->file_format && aNew->pSchema->enc!=ENC(db) ){ - zErrDyn = sqlite3MPrintf(db, + }else if( pNew->pSchema->file_format && pNew->pSchema->enc!=ENC(db) ){ + zErrDyn = sqlite3MPrintf(db, "attached databases must use the same text encoding as main database"); rc = SQLITE_ERROR; } - sqlite3BtreeEnter(aNew->pBt); - pPager = sqlite3BtreePager(aNew->pBt); + sqlite3BtreeEnter(pNew->pBt); + pPager = sqlite3BtreePager(pNew->pBt); sqlite3PagerLockingMode(pPager, db->dfltLockMode); - sqlite3BtreeSecureDelete(aNew->pBt, + sqlite3BtreeSecureDelete(pNew->pBt, sqlite3BtreeSecureDelete(db->aDb[0].pBt,-1) ); #ifndef SQLITE_OMIT_PAGER_PRAGMAS - sqlite3BtreeSetPagerFlags(aNew->pBt, + sqlite3BtreeSetPagerFlags(pNew->pBt, PAGER_SYNCHRONOUS_FULL | (db->flags & PAGER_FLAGS_MASK)); #endif - sqlite3BtreeLeave(aNew->pBt); + sqlite3BtreeLeave(pNew->pBt); } - aNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - aNew->zName = sqlite3DbStrDup(db, zName); - if( rc==SQLITE_OK && aNew->zName==0 ){ + pNew->safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; + if( rc==SQLITE_OK && pNew->zDbSName==0 ){ rc = SQLITE_NOMEM_BKPT; } - - -#ifdef SQLITE_HAS_CODEC - if( rc==SQLITE_OK ){ - extern int sqlite3CodecAttach(sqlite3*, int, const void*, int); - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); - int nKey; - char *zKey; - int t = sqlite3_value_type(argv[2]); - switch( t ){ - case SQLITE_INTEGER: - case SQLITE_FLOAT: - zErrDyn = sqlite3DbStrDup(db, "Invalid key value"); - rc = SQLITE_ERROR; - break; - - case SQLITE_TEXT: - case SQLITE_BLOB: - nKey = sqlite3_value_bytes(argv[2]); - zKey = (char *)sqlite3_value_blob(argv[2]); - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); - break; - - case SQLITE_NULL: - /* No key specified. Use the key from the main database */ - sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey>0 || sqlite3BtreeGetOptimalReserve(db->aDb[0].pBt)>0 ){ - rc = sqlite3CodecAttach(db, db->nDb-1, zKey, nKey); - } - break; - } - } -#endif + sqlite3_free_filename( zPath ); /* If the file was opened successfully, read the schema for the new database. - ** If this fails, or if opening the file failed, then close the file and - ** remove the entry from the db->aDb[] array. i.e. put everything back the way - ** we found it. + ** If this fails, or if opening the file failed, then close the file and + ** remove the entry from the db->aDb[] array. i.e. put everything back the + ** way we found it. */ if( rc==SQLITE_OK ){ sqlite3BtreeEnterAll(db); - rc = sqlite3Init(db, &zErrDyn); + db->init.iDb = 0; + db->mDbFlags &= ~(DBFLAG_SchemaKnownOk); + if( !REOPEN_AS_MEMDB(db) ){ + rc = sqlite3Init(db, &zErrDyn); + } sqlite3BtreeLeaveAll(db); + assert( zErrDyn==0 || rc!=SQLITE_OK ); } #ifdef SQLITE_USER_AUTHENTICATION - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && !REOPEN_AS_MEMDB(db) ){ u8 newAuth = 0; rc = sqlite3UserAuthCheckLogin(db, zName, &newAuth); if( newAuthauth.authLevel ){ @@ -96189,25 +121296,27 @@ static void attachFunc( } #endif if( rc ){ - int iDb = db->nDb - 1; - assert( iDb>=2 ); - if( db->aDb[iDb].pBt ){ - sqlite3BtreeClose(db->aDb[iDb].pBt); - db->aDb[iDb].pBt = 0; - db->aDb[iDb].pSchema = 0; - } - sqlite3ResetAllSchemasOfConnection(db); - db->nDb = iDb; - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - sqlite3OomFault(db); - sqlite3DbFree(db, zErrDyn); - zErrDyn = sqlite3MPrintf(db, "out of memory"); - }else if( zErrDyn==0 ){ - zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + if( ALWAYS(!REOPEN_AS_MEMDB(db)) ){ + int iDb = db->nDb - 1; + assert( iDb>=2 ); + if( db->aDb[iDb].pBt ){ + sqlite3BtreeClose(db->aDb[iDb].pBt); + db->aDb[iDb].pBt = 0; + db->aDb[iDb].pSchema = 0; + } + sqlite3ResetAllSchemasOfConnection(db); + db->nDb = iDb; + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + sqlite3DbFree(db, zErrDyn); + zErrDyn = sqlite3MPrintf(db, "out of memory"); + }else if( zErrDyn==0 ){ + zErrDyn = sqlite3MPrintf(db, "unable to open database: %s", zFile); + } } goto attach_error; } - + return; attach_error: @@ -96236,6 +121345,7 @@ static void detachFunc( sqlite3 *db = sqlite3_context_db_handle(context); int i; Db *pDb = 0; + HashElem *pEntry; char zErr[128]; UNUSED_PARAMETER(NotUsed); @@ -96244,7 +121354,7 @@ static void detachFunc( for(i=0; inDb; i++){ pDb = &db->aDb[i]; if( pDb->pBt==0 ) continue; - if( sqlite3StrICmp(pDb->zName, zName)==0 ) break; + if( sqlite3DbIsNamed(db, i, zName) ) break; } if( i>=db->nDb ){ @@ -96255,16 +121365,25 @@ static void detachFunc( sqlite3_snprintf(sizeof(zErr),zErr, "cannot detach database %s", zName); goto detach_error; } - if( !db->autoCommit ){ - sqlite3_snprintf(sizeof(zErr), zErr, - "cannot DETACH database within transaction"); - goto detach_error; - } - if( sqlite3BtreeIsInReadTrans(pDb->pBt) || sqlite3BtreeIsInBackup(pDb->pBt) ){ + if( sqlite3BtreeTxnState(pDb->pBt)!=SQLITE_TXN_NONE + || sqlite3BtreeIsInBackup(pDb->pBt) + ){ sqlite3_snprintf(sizeof(zErr),zErr, "database %s is locked", zName); goto detach_error; } + /* If any TEMP triggers reference the schema being detached, move those + ** triggers to reference the TEMP schema itself. */ + assert( db->aDb[1].pSchema ); + pEntry = sqliteHashFirst(&db->aDb[1].pSchema->trigHash); + while( pEntry ){ + Trigger *pTrig = (Trigger*)sqliteHashData(pEntry); + if( pTrig->pTabSchema==pDb->pSchema ){ + pTrig->pTabSchema = pTrig->pSchema; + } + pEntry = sqliteHashNext(pEntry); + } + sqlite3BtreeClose(pDb->pBt); pDb->pBt = 0; pDb->pSchema = 0; @@ -96294,21 +121413,25 @@ static void codeAttach( sqlite3* db = pParse->db; int regArgs; + if( SQLITE_OK!=sqlite3ReadSchema(pParse) ) goto attach_end; + + if( pParse->nErr ) goto attach_end; memset(&sName, 0, sizeof(NameContext)); sName.pParse = pParse; - if( - SQLITE_OK!=(rc = resolveAttachExpr(&sName, pFilename)) || - SQLITE_OK!=(rc = resolveAttachExpr(&sName, pDbname)) || - SQLITE_OK!=(rc = resolveAttachExpr(&sName, pKey)) + if( + SQLITE_OK!=resolveAttachExpr(&sName, pFilename) || + SQLITE_OK!=resolveAttachExpr(&sName, pDbname) || + SQLITE_OK!=resolveAttachExpr(&sName, pKey) ){ goto attach_end; } #ifndef SQLITE_OMIT_AUTHORIZATION - if( pAuthArg ){ + if( ALWAYS(pAuthArg) ){ char *zAuthArg; if( pAuthArg->op==TK_STRING ){ + assert( !ExprHasProperty(pAuthArg, EP_IntValue) ); zAuthArg = pAuthArg->u.zToken; }else{ zAuthArg = 0; @@ -96329,18 +121452,15 @@ static void codeAttach( assert( v || db->mallocFailed ); if( v ){ - sqlite3VdbeAddOp4(v, OP_Function0, 0, regArgs+3-pFunc->nArg, regArgs+3, - (char *)pFunc, P4_FUNCDEF); - assert( pFunc->nArg==-1 || (pFunc->nArg&0xff)==pFunc->nArg ); - sqlite3VdbeChangeP5(v, (u8)(pFunc->nArg)); - + sqlite3VdbeAddFunctionCall(pParse, 0, regArgs+3-pFunc->nArg, regArgs+3, + pFunc->nArg, pFunc, 0); /* Code an OP_Expire. For an ATTACH statement, set P1 to true (expire this ** statement only). For DETACH, set it to false (expire all existing ** statements). */ sqlite3VdbeAddOp1(v, OP_Expire, (type==SQLITE_ATTACH)); } - + attach_end: sqlite3ExprDelete(db, pFilename); sqlite3ExprDelete(db, pDbname); @@ -96360,6 +121480,7 @@ SQLITE_PRIVATE void sqlite3Detach(Parse *pParse, Expr *pDbname){ 0, /* pNext */ detachFunc, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "sqlite_detach", /* zName */ {0} }; @@ -96379,6 +121500,7 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p 0, /* pNext */ attachFunc, /* xSFunc */ 0, /* xFinalize */ + 0, 0, /* xValue, xInverse */ "sqlite_attach", /* zName */ {0} }; @@ -96386,6 +121508,70 @@ SQLITE_PRIVATE void sqlite3Attach(Parse *pParse, Expr *p, Expr *pDbname, Expr *p } #endif /* SQLITE_OMIT_ATTACH */ +/* +** Expression callback used by sqlite3FixAAAA() routines. +*/ +static int fixExprCb(Walker *p, Expr *pExpr){ + DbFixer *pFix = p->u.pFix; + if( !pFix->bTemp ) ExprSetProperty(pExpr, EP_FromDDL); + if( pExpr->op==TK_VARIABLE ){ + if( pFix->pParse->db->init.busy ){ + pExpr->op = TK_NULL; + }else{ + sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); + return WRC_Abort; + } + } + return WRC_Continue; +} + +/* +** Select callback used by sqlite3FixAAAA() routines. +*/ +static int fixSelectCb(Walker *p, Select *pSelect){ + DbFixer *pFix = p->u.pFix; + int i; + SrcItem *pItem; + sqlite3 *db = pFix->pParse->db; + int iDb = sqlite3FindDbName(db, pFix->zDb); + SrcList *pList = pSelect->pSrc; + + if( NEVER(pList==0) ) return WRC_Continue; + for(i=0, pItem=pList->a; inSrc; i++, pItem++){ + if( pFix->bTemp==0 && pItem->fg.isSubquery==0 ){ + if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + if( iDb!=sqlite3FindDbName(db, pItem->u4.zDatabase) ){ + sqlite3ErrorMsg(pFix->pParse, + "%s %T cannot reference objects in database %s", + pFix->zType, pFix->pName, pItem->u4.zDatabase); + return WRC_Abort; + } + sqlite3DbFree(db, pItem->u4.zDatabase); + pItem->fg.notCte = 1; + pItem->fg.hadSchema = 1; + } + pItem->u4.pSchema = pFix->pSchema; + pItem->fg.fromDDL = 1; + pItem->fg.fixedSchema = 1; + } +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) + if( pList->a[i].fg.isUsing==0 + && sqlite3WalkExpr(&pFix->w, pList->a[i].u3.pOn) + ){ + return WRC_Abort; + } +#endif + } + if( pSelect->pWith ){ + for(i=0; ipWith->nCte; i++){ + if( sqlite3WalkSelect(p, pSelect->pWith->a[i].pSelect) ){ + return WRC_Abort; + } + } + } + return WRC_Continue; +} + /* ** Initialize a DbFixer structure. This routine must be called prior ** to passing the structure to one of the sqliteFixAAAA() routines below. @@ -96397,16 +121583,21 @@ SQLITE_PRIVATE void sqlite3FixInit( const char *zType, /* "view", "trigger", or "index" */ const Token *pName /* Name of the view, trigger, or index */ ){ - sqlite3 *db; - - db = pParse->db; + sqlite3 *db = pParse->db; assert( db->nDb>iDb ); pFix->pParse = pParse; - pFix->zDb = db->aDb[iDb].zName; + pFix->zDb = db->aDb[iDb].zDbSName; pFix->pSchema = db->aDb[iDb].pSchema; pFix->zType = zType; pFix->pName = pName; - pFix->bVarOnly = (iDb==1); + pFix->bTemp = (iDb==1); + pFix->w.pParse = pParse; + pFix->w.xExprCallback = fixExprCb; + pFix->w.xSelectCallback = fixSelectCb; + pFix->w.xSelectCallback2 = sqlite3WalkWinDefnDummyCallback; + pFix->w.walkerDepth = 0; + pFix->w.eCode = 0; + pFix->w.u.pFix = pFix; } /* @@ -96427,104 +121618,27 @@ SQLITE_PRIVATE int sqlite3FixSrcList( DbFixer *pFix, /* Context of the fixation */ SrcList *pList /* The Source list to check and modify */ ){ - int i; - const char *zDb; - struct SrcList_item *pItem; - - if( NEVER(pList==0) ) return 0; - zDb = pFix->zDb; - for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pFix->bVarOnly==0 ){ - if( pItem->zDatabase && sqlite3StrICmp(pItem->zDatabase, zDb) ){ - sqlite3ErrorMsg(pFix->pParse, - "%s %T cannot reference objects in database %s", - pFix->zType, pFix->pName, pItem->zDatabase); - return 1; - } - sqlite3DbFree(pFix->pParse->db, pItem->zDatabase); - pItem->zDatabase = 0; - pItem->pSchema = pFix->pSchema; - } -#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) - if( sqlite3FixSelect(pFix, pItem->pSelect) ) return 1; - if( sqlite3FixExpr(pFix, pItem->pOn) ) return 1; -#endif + int res = 0; + if( pList ){ + Select s; + memset(&s, 0, sizeof(s)); + s.pSrc = pList; + res = sqlite3WalkSelect(&pFix->w, &s); } - return 0; + return res; } #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_TRIGGER) SQLITE_PRIVATE int sqlite3FixSelect( DbFixer *pFix, /* Context of the fixation */ Select *pSelect /* The SELECT statement to be fixed to one database */ ){ - while( pSelect ){ - if( sqlite3FixExprList(pFix, pSelect->pEList) ){ - return 1; - } - if( sqlite3FixSrcList(pFix, pSelect->pSrc) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pWhere) ){ - return 1; - } - if( sqlite3FixExprList(pFix, pSelect->pGroupBy) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pHaving) ){ - return 1; - } - if( sqlite3FixExprList(pFix, pSelect->pOrderBy) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pLimit) ){ - return 1; - } - if( sqlite3FixExpr(pFix, pSelect->pOffset) ){ - return 1; - } - pSelect = pSelect->pPrior; - } - return 0; + return sqlite3WalkSelect(&pFix->w, pSelect); } SQLITE_PRIVATE int sqlite3FixExpr( DbFixer *pFix, /* Context of the fixation */ Expr *pExpr /* The expression to be fixed to one database */ ){ - while( pExpr ){ - if( pExpr->op==TK_VARIABLE ){ - if( pFix->pParse->db->init.busy ){ - pExpr->op = TK_NULL; - }else{ - sqlite3ErrorMsg(pFix->pParse, "%s cannot use variables", pFix->zType); - return 1; - } - } - if( ExprHasProperty(pExpr, EP_TokenOnly) ) break; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - if( sqlite3FixSelect(pFix, pExpr->x.pSelect) ) return 1; - }else{ - if( sqlite3FixExprList(pFix, pExpr->x.pList) ) return 1; - } - if( sqlite3FixExpr(pFix, pExpr->pRight) ){ - return 1; - } - pExpr = pExpr->pLeft; - } - return 0; -} -SQLITE_PRIVATE int sqlite3FixExprList( - DbFixer *pFix, /* Context of the fixation */ - ExprList *pList /* The expression to be fixed to one database */ -){ - int i; - struct ExprList_item *pItem; - if( pList==0 ) return 0; - for(i=0, pItem=pList->a; inExpr; i++, pItem++){ - if( sqlite3FixExpr(pFix, pItem->pExpr) ){ - return 1; - } - } - return 0; + return sqlite3WalkExpr(&pFix->w, pExpr); } #endif @@ -96534,17 +121648,30 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( TriggerStep *pStep /* The trigger step be fixed to one database */ ){ while( pStep ){ - if( sqlite3FixSelect(pFix, pStep->pSelect) ){ + if( sqlite3WalkSelect(&pFix->w, pStep->pSelect) + || sqlite3WalkExpr(&pFix->w, pStep->pWhere) + || sqlite3WalkExprList(&pFix->w, pStep->pExprList) + || sqlite3FixSrcList(pFix, pStep->pFrom) + ){ return 1; } - if( sqlite3FixExpr(pFix, pStep->pWhere) ){ - return 1; - } - if( sqlite3FixExprList(pFix, pStep->pExprList) ){ - return 1; +#ifndef SQLITE_OMIT_UPSERT + { + Upsert *pUp; + for(pUp=pStep->pUpsert; pUp; pUp=pUp->pNextUpsert){ + if( sqlite3WalkExprList(&pFix->w, pUp->pUpsertTarget) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertTargetWhere) + || sqlite3WalkExprList(&pFix->w, pUp->pUpsertSet) + || sqlite3WalkExpr(&pFix->w, pUp->pUpsertWhere) + ){ + return 1; + } + } } +#endif pStep = pStep->pNext; } + return 0; } #endif @@ -96620,7 +121747,7 @@ SQLITE_PRIVATE int sqlite3FixTriggerStep( ** Setting the auth function to NULL disables this hook. The default ** setting of the auth function is NULL. */ -SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( +SQLITE_API int sqlite3_set_authorizer( sqlite3 *db, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pArg @@ -96631,7 +121758,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_set_authorizer( sqlite3_mutex_enter(db->mutex); db->xAuth = (sqlite3_xauth)xAuth; db->pAuthArg = pArg; - sqlite3ExpirePreparedStatements(db); + if( db->xAuth ) sqlite3ExpirePreparedStatements(db, 1); sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } @@ -96660,21 +121787,20 @@ SQLITE_PRIVATE int sqlite3AuthReadCol( const char *zCol, /* Column name */ int iDb /* Index of containing database. */ ){ - sqlite3 *db = pParse->db; /* Database handle */ - char *zDb = db->aDb[iDb].zName; /* Name of attached database */ - int rc; /* Auth callback return code */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zDb = db->aDb[iDb].zDbSName; /* Schema name of attached database */ + int rc; /* Auth callback return code */ + if( db->init.busy ) return SQLITE_OK; rc = db->xAuth(db->pAuthArg, SQLITE_READ, zTab,zCol,zDb,pParse->zAuthContext #ifdef SQLITE_USER_AUTHENTICATION ,db->auth.zAuthUser #endif ); if( rc==SQLITE_DENY ){ - if( db->nDb>2 || iDb!=0 ){ - sqlite3ErrorMsg(pParse, "access to %s.%s.%s is prohibited",zDb,zTab,zCol); - }else{ - sqlite3ErrorMsg(pParse, "access to %s.%s is prohibited", zTab, zCol); - } + char *z = sqlite3_mprintf("%s.%s", zTab, zCol); + if( db->nDb>2 || iDb!=0 ) z = sqlite3_mprintf("%s.%z", zDb, z); + sqlite3ErrorMsg(pParse, "access to %z is prohibited", z); pParse->rc = SQLITE_AUTH; }else if( rc!=SQLITE_IGNORE && rc!=SQLITE_OK ){ sqliteAuthBadReturnCode(pParse); @@ -96684,10 +121810,10 @@ SQLITE_PRIVATE int sqlite3AuthReadCol( /* ** The pExpr should be a TK_COLUMN expression. The table referred to -** is in pTabList or else it is the NEW or OLD table of a trigger. +** is in pTabList or else it is the NEW or OLD table of a trigger. ** Check to see if it is OK to read this particular column. ** -** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN +** If the auth function returns SQLITE_IGNORE, change the TK_COLUMN ** instruction into a TK_NULL. If the auth function returns SQLITE_DENY, ** then generate an error. */ @@ -96697,14 +121823,15 @@ SQLITE_PRIVATE void sqlite3AuthRead( Schema *pSchema, /* The schema of the expression */ SrcList *pTabList /* All table that pExpr might refer to */ ){ - sqlite3 *db = pParse->db; Table *pTab = 0; /* The table being read */ const char *zCol; /* Name of the column of the table */ int iSrc; /* Index in pTabList->a[] of table being read */ int iDb; /* The index of the database the expression refers to */ int iCol; /* Index of column in table */ - if( db->xAuth==0 ) return; + assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); + assert( !IN_RENAME_OBJECT ); + assert( pParse->db->xAuth!=0 ); iDb = sqlite3SchemaToIndex(pParse->db, pSchema); if( iDb<0 ){ /* An attempt to read a column out of a subquery or other @@ -96712,31 +121839,30 @@ SQLITE_PRIVATE void sqlite3AuthRead( return; } - assert( pExpr->op==TK_COLUMN || pExpr->op==TK_TRIGGER ); if( pExpr->op==TK_TRIGGER ){ pTab = pParse->pTriggerTab; }else{ assert( pTabList ); - for(iSrc=0; ALWAYS(iSrcnSrc); iSrc++){ + for(iSrc=0; iSrcnSrc; iSrc++){ if( pExpr->iTable==pTabList->a[iSrc].iCursor ){ - pTab = pTabList->a[iSrc].pTab; + pTab = pTabList->a[iSrc].pSTab; break; } } } iCol = pExpr->iColumn; - if( NEVER(pTab==0) ) return; + if( pTab==0 ) return; if( iCol>=0 ){ assert( iColnCol ); - zCol = pTab->aCol[iCol].zName; + zCol = pTab->aCol[iCol].zCnName; }else if( pTab->iPKey>=0 ){ assert( pTab->iPKeynCol ); - zCol = pTab->aCol[pTab->iPKey].zName; + zCol = pTab->aCol[pTab->iPKey].zCnName; }else{ zCol = "ROWID"; } - assert( iDb>=0 && iDbnDb ); + assert( iDb>=0 && iDbdb->nDb ); if( SQLITE_IGNORE==sqlite3AuthReadCol(pParse, pTab->zName, zCol, iDb) ){ pExpr->op = TK_NULL; } @@ -96758,16 +121884,25 @@ SQLITE_PRIVATE int sqlite3AuthCheck( sqlite3 *db = pParse->db; int rc; - /* Don't do any authorization checks if the database is initialising + /* Don't do any authorization checks if the database is initializing ** or if the parser is being invoked from within sqlite3_declare_vtab. */ - if( db->init.busy || IN_DECLARE_VTAB ){ + assert( !IN_RENAME_OBJECT || db->xAuth==0 ); + if( db->xAuth==0 || db->init.busy || IN_SPECIAL_PARSE ){ return SQLITE_OK; } - if( db->xAuth==0 ){ - return SQLITE_OK; - } + /* EVIDENCE-OF: R-43249-19882 The third through sixth parameters to the + ** callback are either NULL pointers or zero-terminated strings that + ** contain additional details about the action to be authorized. + ** + ** The following testcase() macros show that any of the 3rd through 6th + ** parameters can be either NULL or a string. */ + testcase( zArg1==0 ); + testcase( zArg2==0 ); + testcase( zArg3==0 ); + testcase( pParse->zAuthContext==0 ); + rc = db->xAuth(db->pAuthArg, code, zArg1, zArg2, zArg3, pParse->zAuthContext #ifdef SQLITE_USER_AUTHENTICATION ,db->auth.zAuthUser @@ -96790,7 +121925,7 @@ SQLITE_PRIVATE int sqlite3AuthCheck( */ SQLITE_PRIVATE void sqlite3AuthContextPush( Parse *pParse, - AuthContext *pContext, + AuthContext *pContext, const char *zContext ){ assert( pParse ); @@ -96846,14 +121981,14 @@ SQLITE_PRIVATE void sqlite3AuthContextPop(AuthContext *pContext){ ** codeTableLocks() functions. */ struct TableLock { - int iDb; /* The database containing the table to be locked */ - int iTab; /* The root page of the table to be locked */ - u8 isWriteLock; /* True for write lock. False for a read lock */ - const char *zName; /* Name of the table */ + int iDb; /* The database containing the table to be locked */ + Pgno iTab; /* The root page of the table to be locked */ + u8 isWriteLock; /* True for write lock. False for a read lock */ + const char *zLockName; /* Name of the table */ }; /* -** Record the fact that we want to lock a table at run-time. +** Record the fact that we want to lock a table at run-time. ** ** The table to be locked has root page iTab and is found in database iDb. ** A read or a write lock can be taken depending on isWritelock. @@ -96862,19 +121997,20 @@ struct TableLock { ** code to make the lock occur is generated by a later call to ** codeTableLocks() which occurs during sqlite3FinishCoding(). */ -SQLITE_PRIVATE void sqlite3TableLock( +static SQLITE_NOINLINE void lockTable( Parse *pParse, /* Parsing context */ int iDb, /* Index of the database containing the table to lock */ - int iTab, /* Root page number of the table to be locked */ + Pgno iTab, /* Root page number of the table to be locked */ u8 isWriteLock, /* True for a write lock */ const char *zName /* Name of the table to be locked */ ){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); + Parse *pToplevel; int i; int nBytes; TableLock *p; assert( iDb>=0 ); + pToplevel = sqlite3ParseToplevel(pParse); for(i=0; inTableLock; i++){ p = &pToplevel->aTableLock[i]; if( p->iDb==iDb && p->iTab==iTab ){ @@ -96891,12 +122027,23 @@ SQLITE_PRIVATE void sqlite3TableLock( p->iDb = iDb; p->iTab = iTab; p->isWriteLock = isWriteLock; - p->zName = zName; + p->zLockName = zName; }else{ pToplevel->nTableLock = 0; sqlite3OomFault(pToplevel->db); } } +SQLITE_PRIVATE void sqlite3TableLock( + Parse *pParse, /* Parsing context */ + int iDb, /* Index of the database containing the table to lock */ + Pgno iTab, /* Root page number of the table to be locked */ + u8 isWriteLock, /* True for a write lock */ + const char *zName /* Name of the table to be locked */ +){ + if( iDb==1 ) return; + if( !sqlite3BtreeSharable(pParse->db->aDb[iDb].pBt) ) return; + lockTable(pParse, iDb, iTab, isWriteLock, zName); +} /* ** Code an OP_TableLock instruction for each table locked by the @@ -96904,16 +122051,14 @@ SQLITE_PRIVATE void sqlite3TableLock( */ static void codeTableLocks(Parse *pParse){ int i; - Vdbe *pVdbe; - - pVdbe = sqlite3GetVdbe(pParse); - assert( pVdbe!=0 ); /* sqlite3GetVdbe cannot fail: VDBE already allocated */ + Vdbe *pVdbe = pParse->pVdbe; + assert( pVdbe!=0 ); for(i=0; inTableLock; i++){ TableLock *p = &pParse->aTableLock[i]; int p1 = p->iDb; sqlite3VdbeAddOp4(pVdbe, OP_TableLock, p1, p->iTab, p->isWriteLock, - p->zName, P4_STATIC); + p->zLockName, P4_STATIC); } } #else @@ -96946,31 +122091,61 @@ SQLITE_PRIVATE int sqlite3DbMaskAllZero(yDbMask m){ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ sqlite3 *db; Vdbe *v; + int iDb, i; assert( pParse->pToplevel==0 ); db = pParse->db; + assert( db->pParse==pParse ); if( pParse->nested ) return; - if( db->mallocFailed || pParse->nErr ){ - if( pParse->rc==SQLITE_OK ) pParse->rc = SQLITE_ERROR; + if( pParse->nErr ){ + if( db->mallocFailed ) pParse->rc = SQLITE_NOMEM; return; } + assert( db->mallocFailed==0 ); /* Begin by generating some termination code at the end of the ** vdbe program */ - v = sqlite3GetVdbe(pParse); - assert( !pParse->isMultiWrite + v = pParse->pVdbe; + if( v==0 ){ + if( db->init.busy ){ + pParse->rc = SQLITE_DONE; + return; + } + v = sqlite3GetVdbe(pParse); + if( v==0 ) pParse->rc = SQLITE_ERROR; + } + assert( !pParse->isMultiWrite || sqlite3VdbeAssertMayAbort(v, pParse->mayAbort)); if( v ){ - while( sqlite3VdbeDeletePriorOpcode(v, OP_Close) ){} + if( pParse->bReturning ){ + Returning *pReturning = pParse->u1.pReturning; + int addrRewind; + int reg; + + if( pReturning->nRetCol ){ + sqlite3VdbeAddOp0(v, OP_FkCheck); + addrRewind = + sqlite3VdbeAddOp1(v, OP_Rewind, pReturning->iRetCur); + VdbeCoverage(v); + reg = pReturning->iRetReg; + for(i=0; inRetCol; i++){ + sqlite3VdbeAddOp3(v, OP_Column, pReturning->iRetCur, i, reg+i); + } + sqlite3VdbeAddOp2(v, OP_ResultRow, reg, i); + sqlite3VdbeAddOp2(v, OP_Next, pReturning->iRetCur, addrRewind+1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrRewind); + } + } sqlite3VdbeAddOp0(v, OP_Halt); -#if SQLITE_USER_AUTHENTICATION +#if SQLITE_USER_AUTHENTICATION && !defined(SQLITE_OMIT_SHARED_CACHE) if( pParse->nTableLock>0 && db->init.busy==0 ){ sqlite3UserAuthInit(db); if( db->auth.authLevelrc = SQLITE_AUTH_USER; sqlite3ErrorMsg(pParse, "user not authenticated"); + pParse->rc = SQLITE_AUTH_USER; return; } } @@ -96982,118 +122157,125 @@ SQLITE_PRIVATE void sqlite3FinishCoding(Parse *pParse){ ** transaction on each used database and to verify the schema cookie ** on each used database. */ - if( db->mallocFailed==0 - && (DbMaskNonZero(pParse->cookieMask) || pParse->pConstExpr) - ){ - int iDb, i; - assert( sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); - sqlite3VdbeJumpHere(v, 0); - for(iDb=0; iDbnDb; iDb++){ - if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; - sqlite3VdbeUsesBtree(v, iDb); - sqlite3VdbeAddOp4Int(v, - OP_Transaction, /* Opcode */ - iDb, /* P1 */ - DbMaskTest(pParse->writeMask,iDb), /* P2 */ - pParse->cookieValue[iDb], /* P3 */ - db->aDb[iDb].pSchema->iGeneration /* P4 */ - ); - if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); - VdbeComment((v, - "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); - } + assert( pParse->nErr>0 || sqlite3VdbeGetOp(v, 0)->opcode==OP_Init ); + sqlite3VdbeJumpHere(v, 0); + assert( db->nDb>0 ); + iDb = 0; + do{ + Schema *pSchema; + if( DbMaskTest(pParse->cookieMask, iDb)==0 ) continue; + sqlite3VdbeUsesBtree(v, iDb); + pSchema = db->aDb[iDb].pSchema; + sqlite3VdbeAddOp4Int(v, + OP_Transaction, /* Opcode */ + iDb, /* P1 */ + DbMaskTest(pParse->writeMask,iDb), /* P2 */ + pSchema->schema_cookie, /* P3 */ + pSchema->iGeneration /* P4 */ + ); + if( db->init.busy==0 ) sqlite3VdbeChangeP5(v, 1); + VdbeComment((v, + "usesStmtJournal=%d", pParse->mayAbort && pParse->isMultiWrite)); + }while( ++iDbnDb ); #ifndef SQLITE_OMIT_VIRTUALTABLE - for(i=0; inVtabLock; i++){ - char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); - sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); - } - pParse->nVtabLock = 0; + for(i=0; inVtabLock; i++){ + char *vtab = (char *)sqlite3GetVTable(db, pParse->apVtabLock[i]); + sqlite3VdbeAddOp4(v, OP_VBegin, 0, 0, 0, vtab, P4_VTAB); + } + pParse->nVtabLock = 0; #endif - /* Once all the cookies have been verified and transactions opened, - ** obtain the required table-locks. This is a no-op unless the - ** shared-cache feature is enabled. - */ - codeTableLocks(pParse); +#ifndef SQLITE_OMIT_SHARED_CACHE + /* Once all the cookies have been verified and transactions opened, + ** obtain the required table-locks. This is a no-op unless the + ** shared-cache feature is enabled. + */ + if( pParse->nTableLock ) codeTableLocks(pParse); +#endif - /* Initialize any AUTOINCREMENT data structures required. - */ - sqlite3AutoincrementBegin(pParse); + /* Initialize any AUTOINCREMENT data structures required. + */ + if( pParse->pAinc ) sqlite3AutoincrementBegin(pParse); - /* Code constant expressions that where factored out of inner loops */ - if( pParse->pConstExpr ){ - ExprList *pEL = pParse->pConstExpr; - pParse->okConstFactor = 0; - for(i=0; inExpr; i++){ - sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); - } + /* Code constant expressions that were factored out of inner loops. + */ + if( pParse->pConstExpr ){ + ExprList *pEL = pParse->pConstExpr; + pParse->okConstFactor = 0; + for(i=0; inExpr; i++){ + assert( pEL->a[i].u.iConstExprReg>0 ); + sqlite3ExprCode(pParse, pEL->a[i].pExpr, pEL->a[i].u.iConstExprReg); } - - /* Finally, jump back to the beginning of the executable code. */ - sqlite3VdbeGoto(v, 1); } - } + if( pParse->bReturning ){ + Returning *pRet = pParse->u1.pReturning; + if( pRet->nRetCol ){ + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRet->iRetCur, pRet->nRetCol); + } + } + + /* Finally, jump back to the beginning of the executable code. */ + sqlite3VdbeGoto(v, 1); + } /* Get the VDBE program ready for execution */ - if( v && pParse->nErr==0 && !db->mallocFailed ){ - assert( pParse->iCacheLevel==0 ); /* Disables and re-enables match */ + assert( v!=0 || pParse->nErr ); + assert( db->mallocFailed==0 || pParse->nErr ); + if( pParse->nErr==0 ){ /* A minimum of one cursor is required if autoincrement is used * See ticket [a696379c1f08866] */ - if( pParse->pAinc!=0 && pParse->nTab==0 ) pParse->nTab = 1; + assert( pParse->pAinc==0 || pParse->nTab>0 ); sqlite3VdbeMakeReady(v, pParse); pParse->rc = SQLITE_DONE; }else{ pParse->rc = SQLITE_ERROR; } - - /* We are done with this Parse object. There is no need to de-initialize it */ -#if 0 - pParse->colNamesSet = 0; - pParse->nTab = 0; - pParse->nMem = 0; - pParse->nSet = 0; - pParse->nVar = 0; - DbMaskZero(pParse->cookieMask); -#endif } /* ** Run the parser and code generator recursively in order to generate ** code for the SQL statement given onto the end of the pParse context -** currently under construction. When the parser is run recursively -** this way, the final OP_Halt is not appended and other initialization -** and finalization steps are omitted because those are handling by the -** outermost parser. +** currently under construction. Notes: ** -** Not everything is nestable. This facility is designed to permit -** INSERT, UPDATE, and DELETE operations against SQLITE_MASTER. Use -** care if you decide to try to use this routine for some other purposes. +** * The final OP_Halt is not appended and other initialization +** and finalization steps are omitted because those are handling by the +** outermost parser. +** +** * Built-in SQL functions always take precedence over application-defined +** SQL functions. In other words, it is not possible to override a +** built-in function. */ SQLITE_PRIVATE void sqlite3NestedParse(Parse *pParse, const char *zFormat, ...){ va_list ap; char *zSql; - char *zErrMsg = 0; sqlite3 *db = pParse->db; -# define SAVE_SZ (sizeof(Parse) - offsetof(Parse,nVar)) - char saveBuf[SAVE_SZ]; + u32 savedDbFlags = db->mDbFlags; + char saveBuf[PARSE_TAIL_SZ]; if( pParse->nErr ) return; + if( pParse->eParseMode ) return; assert( pParse->nested<10 ); /* Nesting should only be of limited depth */ va_start(ap, zFormat); zSql = sqlite3VMPrintf(db, zFormat, ap); va_end(ap); if( zSql==0 ){ - return; /* A malloc must have failed */ + /* This can result either from an OOM or because the formatted string + ** exceeds SQLITE_LIMIT_LENGTH. In the latter case, we need to set + ** an error */ + if( !db->mallocFailed ) pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; + return; } pParse->nested++; - memcpy(saveBuf, &pParse->nVar, SAVE_SZ); - memset(&pParse->nVar, 0, SAVE_SZ); - sqlite3RunParser(pParse, zSql, &zErrMsg); - sqlite3DbFree(db, zErrMsg); + memcpy(saveBuf, PARSE_TAIL(pParse), PARSE_TAIL_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); + db->mDbFlags |= DBFLAG_PreferBuiltin; + sqlite3RunParser(pParse, zSql); + db->mDbFlags = savedDbFlags; sqlite3DbFree(db, zSql); - memcpy(&pParse->nVar, saveBuf, SAVE_SZ); + memcpy(PARSE_TAIL(pParse), saveBuf, PARSE_TAIL_SZ); pParse->nested--; } @@ -97132,12 +122314,57 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha return 0; } #endif - for(i=OMIT_TEMPDB; inDb; i++){ - int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDatabase!=0 && sqlite3StrICmp(zDatabase, db->aDb[j].zName) ) continue; - assert( sqlite3SchemaMutexHeld(db, j, 0) ); - p = sqlite3HashFind(&db->aDb[j].pSchema->tblHash, zName); - if( p ) break; + if( zDatabase ){ + for(i=0; inDb; i++){ + if( sqlite3StrICmp(zDatabase, db->aDb[i].zDbSName)==0 ) break; + } + if( i>=db->nDb ){ + /* No match against the official names. But always match "main" + ** to schema 0 as a legacy fallback. */ + if( sqlite3StrICmp(zDatabase,"main")==0 ){ + i = 0; + }else{ + return 0; + } + } + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p==0 && sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( i==1 ){ + if( sqlite3StrICmp(zName+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 + || sqlite3StrICmp(zName+7, &PREFERRED_SCHEMA_TABLE[7])==0 + || sqlite3StrICmp(zName+7, &LEGACY_SCHEMA_TABLE[7])==0 + ){ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + LEGACY_TEMP_SCHEMA_TABLE); + } + }else{ + if( sqlite3StrICmp(zName+7, &PREFERRED_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, + LEGACY_SCHEMA_TABLE); + } + } + } + }else{ + /* Match against TEMP first */ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, zName); + if( p ) return p; + /* The main database is second */ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, zName); + if( p ) return p; + /* Attached databases are in order of attachment */ + for(i=2; inDb; i++){ + assert( sqlite3SchemaMutexHeld(db, i, 0) ); + p = sqlite3HashFind(&db->aDb[i].pSchema->tblHash, zName); + if( p ) break; + } + if( p==0 && sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( sqlite3StrICmp(zName+7, &PREFERRED_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[0].pSchema->tblHash, LEGACY_SCHEMA_TABLE); + }else if( sqlite3StrICmp(zName+7, &PREFERRED_TEMP_SCHEMA_TABLE[7])==0 ){ + p = sqlite3HashFind(&db->aDb[1].pSchema->tblHash, + LEGACY_TEMP_SCHEMA_TABLE); + } + } } return p; } @@ -97154,38 +122381,53 @@ SQLITE_PRIVATE Table *sqlite3FindTable(sqlite3 *db, const char *zName, const cha */ SQLITE_PRIVATE Table *sqlite3LocateTable( Parse *pParse, /* context in which to report errors */ - int isView, /* True if looking for a VIEW rather than a TABLE */ + u32 flags, /* LOCATE_VIEW or LOCATE_NOERR */ const char *zName, /* Name of the table we are looking for */ const char *zDbase /* Name of the database. Might be NULL */ ){ Table *p; + sqlite3 *db = pParse->db; /* Read the database schema. If an error occurs, leave an error message ** and code in pParse and return NULL. */ - if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ + if( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 + && SQLITE_OK!=sqlite3ReadSchema(pParse) + ){ return 0; } - p = sqlite3FindTable(pParse->db, zName, zDbase); + p = sqlite3FindTable(db, zName, zDbase); if( p==0 ){ - const char *zMsg = isView ? "no such view" : "no such table"; #ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3FindDbName(pParse->db, zDbase)<1 ){ - /* If zName is the not the name of a table in the schema created using - ** CREATE, then check to see if it is the name of an virtual table that - ** can be an eponymous virtual table. */ - Module *pMod = (Module*)sqlite3HashFind(&pParse->db->aModule, zName); + /* If zName is the not the name of a table in the schema created using + ** CREATE, then check to see if it is the name of an virtual table that + ** can be an eponymous virtual table. */ + if( (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)==0 && db->init.busy==0 ){ + Module *pMod = (Module*)sqlite3HashFind(&db->aModule, zName); + if( pMod==0 && sqlite3_strnicmp(zName, "pragma_", 7)==0 ){ + pMod = sqlite3PragmaVtabRegister(db, zName); + } if( pMod && sqlite3VtabEponymousTableInit(pParse, pMod) ){ + testcase( pMod->pEpoTab==0 ); return pMod->pEpoTab; } } #endif + if( flags & LOCATE_NOERR ) return 0; + pParse->checkSchema = 1; + }else if( IsVirtual(p) && (pParse->prepFlags & SQLITE_PREPARE_NO_VTAB)!=0 ){ + p = 0; + } + + if( p==0 ){ + const char *zMsg = flags & LOCATE_VIEW ? "no such view" : "no such table"; if( zDbase ){ sqlite3ErrorMsg(pParse, "%s: %s.%s", zMsg, zDbase, zName); }else{ sqlite3ErrorMsg(pParse, "%s: %s", zMsg, zName); } - pParse->checkSchema = 1; + }else{ + assert( HasRowid(p) || p->iPKey<0 ); } return p; @@ -97201,23 +122443,39 @@ SQLITE_PRIVATE Table *sqlite3LocateTable( ** sqlite3FixSrcList() for details. */ SQLITE_PRIVATE Table *sqlite3LocateTableItem( - Parse *pParse, - int isView, - struct SrcList_item *p + Parse *pParse, + u32 flags, + SrcItem *p ){ const char *zDb; - assert( p->pSchema==0 || p->zDatabase==0 ); - if( p->pSchema ){ - int iDb = sqlite3SchemaToIndex(pParse->db, p->pSchema); - zDb = pParse->db->aDb[iDb].zName; + if( p->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, p->u4.pSchema); + zDb = pParse->db->aDb[iDb].zDbSName; }else{ - zDb = p->zDatabase; + assert( !p->fg.isSubquery ); + zDb = p->u4.zDatabase; } - return sqlite3LocateTable(pParse, isView, p->zName, zDb); + return sqlite3LocateTable(pParse, flags, p->zName, zDb); } /* -** Locate the in-memory structure that describes +** Return the preferred table name for system tables. Translate legacy +** names into the new preferred names, as appropriate. +*/ +SQLITE_PRIVATE const char *sqlite3PreferredTableName(const char *zName){ + if( sqlite3StrNICmp(zName, "sqlite_", 7)==0 ){ + if( sqlite3StrICmp(zName+7, &LEGACY_SCHEMA_TABLE[7])==0 ){ + return PREFERRED_SCHEMA_TABLE; + } + if( sqlite3StrICmp(zName+7, &LEGACY_TEMP_SCHEMA_TABLE[7])==0 ){ + return PREFERRED_TEMP_SCHEMA_TABLE; + } + } + return zName; +} + +/* +** Locate the in-memory structure that describes ** a particular index given the name of that index ** and the name of the database that contains the index. ** Return NULL if not found. @@ -97237,7 +122495,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ Schema *pSchema = db->aDb[j].pSchema; assert( pSchema ); - if( zDb && sqlite3StrICmp(zDb, db->aDb[j].zName) ) continue; + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); p = sqlite3HashFind(&pSchema->idxHash, zName); if( p ) break; @@ -97248,7 +122506,7 @@ SQLITE_PRIVATE Index *sqlite3FindIndex(sqlite3 *db, const char *zName, const cha /* ** Reclaim the memory used by an index */ -static void freeIndex(sqlite3 *db, Index *p){ +SQLITE_PRIVATE void sqlite3FreeIndex(sqlite3 *db, Index *p){ #ifndef SQLITE_OMIT_ANALYZE sqlite3DeleteIndexSamples(db, p); #endif @@ -97256,7 +122514,7 @@ static void freeIndex(sqlite3 *db, Index *p){ sqlite3ExprListDelete(db, p->aColExpr); sqlite3DbFree(db, p->zColAff); if( p->isResized ) sqlite3DbFree(db, (void *)p->azColl); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 sqlite3_free(p->aiRowEst); #endif sqlite3DbFree(db, p); @@ -97288,9 +122546,9 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteIndex(sqlite3 *db, int iDb, const char p->pNext = pIndex->pNext; } } - freeIndex(db, pIndex); + sqlite3FreeIndex(db, pIndex); } - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; } /* @@ -97306,8 +122564,8 @@ SQLITE_PRIVATE void sqlite3CollapseDatabaseArray(sqlite3 *db){ for(i=j=2; inDb; i++){ struct Db *pDb = &db->aDb[i]; if( pDb->pBt==0 ){ - sqlite3DbFree(db, pDb->zName); - pDb->zName = 0; + sqlite3DbFree(db, pDb->zDbSName); + pDb->zDbSName = 0; continue; } if( jnSchemaLock is not zero. +** Deferred resets may be run by calling with iDb<0. */ SQLITE_PRIVATE void sqlite3ResetOneSchema(sqlite3 *db, int iDb){ - Db *pDb; + int i; assert( iDbnDb ); - /* Case 1: Reset the single schema identified by iDb */ - pDb = &db->aDb[iDb]; - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - assert( pDb->pSchema!=0 ); - sqlite3SchemaClear(pDb->pSchema); - - /* If any database other than TEMP is reset, then also reset TEMP - ** since TEMP might be holding triggers that reference tables in the - ** other database. - */ - if( iDb!=1 ){ - pDb = &db->aDb[1]; - assert( pDb->pSchema!=0 ); - sqlite3SchemaClear(pDb->pSchema); + if( iDb>=0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + DbSetProperty(db, iDb, DB_ResetWanted); + DbSetProperty(db, 1, DB_ResetWanted); + db->mDbFlags &= ~DBFLAG_SchemaKnownOk; + } + + if( db->nSchemaLock==0 ){ + for(i=0; inDb; i++){ + if( DbHasProperty(db, i, DB_ResetWanted) ){ + sqlite3SchemaClear(db->aDb[i].pSchema); + } + } } - return; } /* @@ -97359,20 +122616,104 @@ SQLITE_PRIVATE void sqlite3ResetAllSchemasOfConnection(sqlite3 *db){ for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; if( pDb->pSchema ){ - sqlite3SchemaClear(pDb->pSchema); + if( db->nSchemaLock==0 ){ + sqlite3SchemaClear(pDb->pSchema); + }else{ + DbSetProperty(db, i, DB_ResetWanted); + } } } - db->flags &= ~SQLITE_InternChanges; + db->mDbFlags &= ~(DBFLAG_SchemaChange|DBFLAG_SchemaKnownOk); sqlite3VtabUnlockList(db); sqlite3BtreeLeaveAll(db); - sqlite3CollapseDatabaseArray(db); + if( db->nSchemaLock==0 ){ + sqlite3CollapseDatabaseArray(db); + } } /* ** This routine is called when a commit occurs. */ SQLITE_PRIVATE void sqlite3CommitInternalChanges(sqlite3 *db){ - db->flags &= ~SQLITE_InternChanges; + db->mDbFlags &= ~DBFLAG_SchemaChange; +} + +/* +** Set the expression associated with a column. This is usually +** the DEFAULT value, but might also be the expression that computes +** the value for a generated column. +*/ +SQLITE_PRIVATE void sqlite3ColumnSetExpr( + Parse *pParse, /* Parsing context */ + Table *pTab, /* The table containing the column */ + Column *pCol, /* The column to receive the new DEFAULT expression */ + Expr *pExpr /* The new default expression */ +){ + ExprList *pList; + assert( IsOrdinaryTable(pTab) ); + pList = pTab->u.tab.pDfltList; + if( pCol->iDflt==0 + || NEVER(pList==0) + || NEVER(pList->nExpriDflt) + ){ + pCol->iDflt = pList==0 ? 1 : pList->nExpr+1; + pTab->u.tab.pDfltList = sqlite3ExprListAppend(pParse, pList, pExpr); + }else{ + sqlite3ExprDelete(pParse->db, pList->a[pCol->iDflt-1].pExpr); + pList->a[pCol->iDflt-1].pExpr = pExpr; + } +} + +/* +** Return the expression associated with a column. The expression might be +** the DEFAULT clause or the AS clause of a generated column. +** Return NULL if the column has no associated expression. +*/ +SQLITE_PRIVATE Expr *sqlite3ColumnExpr(Table *pTab, Column *pCol){ + if( pCol->iDflt==0 ) return 0; + if( !IsOrdinaryTable(pTab) ) return 0; + if( NEVER(pTab->u.tab.pDfltList==0) ) return 0; + if( NEVER(pTab->u.tab.pDfltList->nExpriDflt) ) return 0; + return pTab->u.tab.pDfltList->a[pCol->iDflt-1].pExpr; +} + +/* +** Set the collating sequence name for a column. +*/ +SQLITE_PRIVATE void sqlite3ColumnSetColl( + sqlite3 *db, + Column *pCol, + const char *zColl +){ + i64 nColl; + i64 n; + char *zNew; + assert( zColl!=0 ); + n = sqlite3Strlen30(pCol->zCnName) + 1; + if( pCol->colFlags & COLFLAG_HASTYPE ){ + n += sqlite3Strlen30(pCol->zCnName+n) + 1; + } + nColl = sqlite3Strlen30(zColl) + 1; + zNew = sqlite3DbRealloc(db, pCol->zCnName, nColl+n); + if( zNew ){ + pCol->zCnName = zNew; + memcpy(pCol->zCnName + n, zColl, nColl); + pCol->colFlags |= COLFLAG_HASCOLL; + } +} + +/* +** Return the collating sequence name for a column +*/ +SQLITE_PRIVATE const char *sqlite3ColumnColl(Column *pCol){ + const char *z; + if( (pCol->colFlags & COLFLAG_HASCOLL)==0 ) return 0; + z = pCol->zCnName; + while( *z ){ z++; } + if( pCol->colFlags & COLFLAG_HASTYPE ){ + do{ z++; }while( *z ); + } + return z+1; } /* @@ -97383,13 +122724,23 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ int i; Column *pCol; assert( pTable!=0 ); + assert( db!=0 ); if( (pCol = pTable->aCol)!=0 ){ for(i=0; inCol; i++, pCol++){ - sqlite3DbFree(db, pCol->zName); - sqlite3ExprDelete(db, pCol->pDflt); - sqlite3DbFree(db, pCol->zColl); + assert( pCol->zCnName==0 || pCol->hName==sqlite3StrIHash(pCol->zCnName) ); + sqlite3DbFree(db, pCol->zCnName); + } + sqlite3DbNNFreeNN(db, pTable->aCol); + if( IsOrdinaryTable(pTable) ){ + sqlite3ExprListDelete(db, pTable->u.tab.pDfltList); + } + if( db->pnBytesFreed==0 ){ + pTable->aCol = 0; + pTable->nCol = 0; + if( IsOrdinaryTable(pTable) ){ + pTable->u.tab.pDfltList = 0; + } } - sqlite3DbFree(db, pTable->aCol); } } @@ -97399,10 +122750,10 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ ** ** This routine just deletes the data structure. It does not unlink ** the table data structure from the hash table. But it does destroy -** memory structures of the indices and foreign keys associated with +** memory structures of the indices and foreign keys associated with ** the table. ** -** The db parameter is optional. It is needed if the Table object +** The db parameter is optional. It is needed if the Table object ** contains lookaside memory. (Table objects in the schema do not use ** lookaside memory, but some ephemeral Table objects do.) Or the ** db parameter can be used with db->pnBytesFreed to measure the memory @@ -97410,53 +122761,72 @@ SQLITE_PRIVATE void sqlite3DeleteColumnNames(sqlite3 *db, Table *pTable){ */ static void SQLITE_NOINLINE deleteTable(sqlite3 *db, Table *pTable){ Index *pIndex, *pNext; - TESTONLY( int nLookaside; ) /* Used to verify lookaside not used for schema */ +#ifdef SQLITE_DEBUG /* Record the number of outstanding lookaside allocations in schema Tables - ** prior to doing any free() operations. Since schema Tables do not use - ** lookaside, this number should not change. */ - TESTONLY( nLookaside = (db && (pTable->tabFlags & TF_Ephemeral)==0) ? - db->lookaside.nOut : 0 ); + ** prior to doing any free() operations. Since schema Tables do not use + ** lookaside, this number should not change. + ** + ** If malloc has already failed, it may be that it failed while allocating + ** a Table object that was going to be marked ephemeral. So do not check + ** that no lookaside memory is used in this case either. */ + int nLookaside = 0; + assert( db!=0 ); + if( !db->mallocFailed && (pTable->tabFlags & TF_Ephemeral)==0 ){ + nLookaside = sqlite3LookasideUsed(db, 0); + } +#endif /* Delete all indices associated with this table. */ for(pIndex = pTable->pIndex; pIndex; pIndex=pNext){ pNext = pIndex->pNext; - assert( pIndex->pSchema==pTable->pSchema ); - if( !db || db->pnBytesFreed==0 ){ - char *zName = pIndex->zName; + assert( pIndex->pSchema==pTable->pSchema + || (IsVirtual(pTable) && pIndex->idxType!=SQLITE_IDXTYPE_APPDEF) ); + if( db->pnBytesFreed==0 && !IsVirtual(pTable) ){ + char *zName = pIndex->zName; TESTONLY ( Index *pOld = ) sqlite3HashInsert( &pIndex->pSchema->idxHash, zName, 0 ); assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); assert( pOld==pIndex || pOld==0 ); } - freeIndex(db, pIndex); + sqlite3FreeIndex(db, pIndex); } - /* Delete any foreign keys attached to this table. */ - sqlite3FkDelete(db, pTable); + if( IsOrdinaryTable(pTable) ){ + sqlite3FkDelete(db, pTable); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( IsVirtual(pTable) ){ + sqlite3VtabClear(db, pTable); + } +#endif + else{ + assert( IsView(pTable) ); + sqlite3SelectDelete(db, pTable->u.view.pSelect); + } /* Delete the Table structure itself. */ sqlite3DeleteColumnNames(db, pTable); sqlite3DbFree(db, pTable->zName); sqlite3DbFree(db, pTable->zColAff); - sqlite3SelectDelete(db, pTable->pSelect); sqlite3ExprListDelete(db, pTable->pCheck); -#ifndef SQLITE_OMIT_VIRTUALTABLE - sqlite3VtabClear(db, pTable); -#endif sqlite3DbFree(db, pTable); /* Verify that no lookaside memory was used by schema tables */ - assert( nLookaside==0 || nLookaside==db->lookaside.nOut ); + assert( nLookaside==0 || nLookaside==sqlite3LookasideUsed(db,0) ); } SQLITE_PRIVATE void sqlite3DeleteTable(sqlite3 *db, Table *pTable){ /* Do not delete the table until the reference count reaches zero. */ + assert( db!=0 ); if( !pTable ) return; - if( ((!db || db->pnBytesFreed==0) && (--pTable->nRef)>0) ) return; + if( db->pnBytesFreed==0 && (--pTable->nTabRef)>0 ) return; deleteTable(db, pTable); } +SQLITE_PRIVATE void sqlite3DeleteTableGeneric(sqlite3 *db, void *pTable){ + sqlite3DeleteTable(db, (Table*)pTable); +} /* @@ -97475,7 +122845,7 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char pDb = &db->aDb[iDb]; p = sqlite3HashInsert(&pDb->pSchema->tblHash, zTabName, 0); sqlite3DeleteTable(db, p); - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; } /* @@ -97491,10 +122861,10 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTable(sqlite3 *db, int iDb, const char ** are not \000 terminated and are not persistent. The returned string ** is \000 terminated and is persistent. */ -SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ +SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, const Token *pName){ char *zName; if( pName ){ - zName = sqlite3DbStrNDup(db, (char*)pName->z, pName->n); + zName = sqlite3DbStrNDup(db, (const char*)pName->z, pName->n); sqlite3Dequote(zName); }else{ zName = 0; @@ -97503,13 +122873,13 @@ SQLITE_PRIVATE char *sqlite3NameFromToken(sqlite3 *db, Token *pName){ } /* -** Open the sqlite_master table stored in database number iDb for +** Open the sqlite_schema table stored in database number iDb for ** writing. The table is opened using cursor 0. */ -SQLITE_PRIVATE void sqlite3OpenMasterTable(Parse *p, int iDb){ +SQLITE_PRIVATE void sqlite3OpenSchemaTable(Parse *p, int iDb){ Vdbe *v = sqlite3GetVdbe(p); - sqlite3TableLock(p, iDb, MASTER_ROOT, 1, SCHEMA_TABLE(iDb)); - sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, MASTER_ROOT, iDb, 5); + sqlite3TableLock(p, iDb, SCHEMA_ROOT, 1, LEGACY_SCHEMA_TABLE); + sqlite3VdbeAddOp4Int(v, OP_OpenWrite, 0, SCHEMA_ROOT, iDb, 5); if( p->nTab==0 ){ p->nTab = 1; } @@ -97526,7 +122896,10 @@ SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ if( zName ){ Db *pDb; for(i=(db->nDb-1), pDb=&db->aDb[i]; i>=0; i--, pDb--){ - if( 0==sqlite3StrICmp(pDb->zName, zName) ) break; + if( 0==sqlite3_stricmp(pDb->zDbSName, zName) ) break; + /* "main" is always an acceptable alias for the primary database + ** even if it has been renamed using SQLITE_DBCONFIG_MAINDBNAME. */ + if( i==0 && 0==sqlite3_stricmp("main", zName) ) break; } } return i; @@ -97535,7 +122908,7 @@ SQLITE_PRIVATE int sqlite3FindDbName(sqlite3 *db, const char *zName){ /* ** The token *pName contains the name of a database (either "main" or ** "temp" or the name of an attached db). This routine returns the -** index of the named database in db->aDb[], or -1 if the named db +** index of the named database in db->aDb[], or -1 if the named db ** does not exist. */ SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){ @@ -97551,7 +122924,7 @@ SQLITE_PRIVATE int sqlite3FindDb(sqlite3 *db, Token *pName){ ** pName1 and pName2. If the table name was fully qualified, for example: ** ** CREATE TABLE xxx.yyy (...); -** +** ** Then pName1 is set to "xxx" and pName2 "yyy". On the other hand if ** the table name is not fully qualified, i.e.: ** @@ -97585,26 +122958,70 @@ SQLITE_PRIVATE int sqlite3TwoPartName( return -1; } }else{ - assert( db->init.iDb==0 || db->init.busy ); + assert( db->init.iDb==0 || db->init.busy || IN_SPECIAL_PARSE + || (db->mDbFlags & DBFLAG_Vacuum)!=0); iDb = db->init.iDb; *pUnqual = pName1; } return iDb; } +/* +** True if PRAGMA writable_schema is ON +*/ +SQLITE_PRIVATE int sqlite3WritableSchema(sqlite3 *db){ + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==0 ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_WriteSchema ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + SQLITE_Defensive ); + testcase( (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))== + (SQLITE_WriteSchema|SQLITE_Defensive) ); + return (db->flags&(SQLITE_WriteSchema|SQLITE_Defensive))==SQLITE_WriteSchema; +} + /* ** This routine is used to check if the UTF-8 string zName is a legal ** unqualified name for a new schema object (table, index, view or ** trigger). All names are legal except those that begin with the string ** "sqlite_" (in upper, lower or mixed case). This portion of the namespace ** is reserved for internal use. +** +** When parsing the sqlite_schema table, this routine also checks to +** make sure the "type", "name", and "tbl_name" columns are consistent +** with the SQL. */ -SQLITE_PRIVATE int sqlite3CheckObjectName(Parse *pParse, const char *zName){ - if( !pParse->db->init.busy && pParse->nested==0 - && (pParse->db->flags & SQLITE_WriteSchema)==0 - && 0==sqlite3StrNICmp(zName, "sqlite_", 7) ){ - sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", zName); - return SQLITE_ERROR; +SQLITE_PRIVATE int sqlite3CheckObjectName( + Parse *pParse, /* Parsing context */ + const char *zName, /* Name of the object to check */ + const char *zType, /* Type of this object */ + const char *zTblName /* Parent table name for triggers and indexes */ +){ + sqlite3 *db = pParse->db; + if( sqlite3WritableSchema(db) + || db->init.imposterTable + || !sqlite3Config.bExtraSchemaChecks + ){ + /* Skip these error checks for writable_schema=ON */ + return SQLITE_OK; + } + if( db->init.busy ){ + if( sqlite3_stricmp(zType, db->init.azInit[0]) + || sqlite3_stricmp(zName, db->init.azInit[1]) + || sqlite3_stricmp(zTblName, db->init.azInit[2]) + ){ + sqlite3ErrorMsg(pParse, ""); /* corruptSchema() will supply the error */ + return SQLITE_ERROR; + } + }else{ + if( (pParse->nested==0 && 0==sqlite3StrNICmp(zName, "sqlite_", 7)) + || (sqlite3ReadOnlyShadowTables(db) && sqlite3ShadowTableName(db, zName)) + ){ + sqlite3ErrorMsg(pParse, "object name reserved for internal use: %s", + zName); + return SQLITE_ERROR; + } + } return SQLITE_OK; } @@ -97619,10 +123036,12 @@ SQLITE_PRIVATE Index *sqlite3PrimaryKeyIndex(Table *pTab){ } /* -** Return the column of index pIdx that corresponds to table -** column iCol. Return -1 if not found. +** Convert an table column number into a index column number. That is, +** for the column iCol in the table (as defined by the CREATE TABLE statement) +** find the (first) offset of that column in index pIdx. Or return -1 +** if column iCol is not used in index pIdx. */ -SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ +SQLITE_PRIVATE i16 sqlite3TableColumnToIndex(Index *pIdx, i16 iCol){ int i; for(i=0; inColumn; i++){ if( iCol==pIdx->aiColumn[i] ) return i; @@ -97630,6 +123049,101 @@ SQLITE_PRIVATE i16 sqlite3ColumnOfIndex(Index *pIdx, i16 iCol){ return -1; } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a storage column number into a table column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. The true column number +** is the index (0,1,2,...) of the column in the CREATE TABLE statement. +** +** The storage column number is less than the table column number if +** and only there are VIRTUAL columns to the left. +** +** If SQLITE_OMIT_GENERATED_COLUMNS, this routine is a no-op macro. +*/ +SQLITE_PRIVATE i16 sqlite3StorageColumnToTable(Table *pTab, i16 iCol){ + if( pTab->tabFlags & TF_HasVirtual ){ + int i; + for(i=0; i<=iCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) iCol++; + } + } + return iCol; +} +#endif + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* Convert a table column number into a storage column number. +** +** The storage column number (0,1,2,....) is the index of the value +** as it appears in the record on disk. Or, if the input column is +** the N-th virtual column (zero-based) then the storage number is +** the number of non-virtual columns in the table plus N. +** +** The true column number is the index (0,1,2,...) of the column in +** the CREATE TABLE statement. +** +** If the input column is a VIRTUAL column, then it should not appear +** in storage. But the value sometimes is cached in registers that +** follow the range of registers used to construct storage. This +** avoids computing the same VIRTUAL column multiple times, and provides +** values for use by OP_Param opcodes in triggers. Hence, if the +** input column is a VIRTUAL table, put it after all the other columns. +** +** In the following, N means "normal column", S means STORED, and +** V means VIRTUAL. Suppose the CREATE TABLE has columns like this: +** +** CREATE TABLE ex(N,S,V,N,S,V,N,S,V); +** -- 0 1 2 3 4 5 6 7 8 +** +** Then the mapping from this function is as follows: +** +** INPUTS: 0 1 2 3 4 5 6 7 8 +** OUTPUTS: 0 1 6 2 3 7 4 5 8 +** +** So, in other words, this routine shifts all the virtual columns to +** the end. +** +** If SQLITE_OMIT_GENERATED_COLUMNS then there are no virtual columns and +** this routine is a no-op macro. If the pTab does not have any virtual +** columns, then this routine is no-op that always return iCol. If iCol +** is negative (indicating the ROWID column) then this routine return iCol. +*/ +SQLITE_PRIVATE i16 sqlite3TableColumnToStorage(Table *pTab, i16 iCol){ + int i; + i16 n; + assert( iColnCol ); + if( (pTab->tabFlags & TF_HasVirtual)==0 || iCol<0 ) return iCol; + for(i=0, n=0; iaCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) n++; + } + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ){ + /* iCol is a virtual column itself */ + return pTab->nNVCol + i - n; + }else{ + /* iCol is a normal or stored column */ + return n; + } +} +#endif + +/* +** Insert a single OP_JournalMode query opcode in order to force the +** prepared statement to return false for sqlite3_stmt_readonly(). This +** is used by CREATE TABLE IF NOT EXISTS and similar if the table already +** exists, so that the prepared statement for CREATE TABLE IF NOT EXISTS +** will return false for sqlite3_stmt_readonly() even if that statement +** is a read-only no-op. +*/ +static void sqlite3ForceNotReadOnly(Parse *pParse){ + int iReg = ++pParse->nMem; + Vdbe *v = sqlite3GetVdbe(pParse); + if( v ){ + sqlite3VdbeAddOp3(v, OP_JournalMode, 0, iReg, PAGER_JOURNALMODE_QUERY); + sqlite3VdbeUsesBtree(v, 0); + } +} + /* ** Begin constructing a new table representation in memory. This is ** the first of several action routines that get called in response @@ -97663,7 +123177,7 @@ SQLITE_PRIVATE void sqlite3StartTable( Token *pName; /* Unqualified name of the table to create */ if( db->init.busy && db->init.newTnum==1 ){ - /* Special case: Parsing the sqlite_master or sqlite_temp_master schema */ + /* Special case: Parsing the sqlite_schema or sqlite_temp_schema schema */ iDb = db->init.iDb; zName = sqlite3DbStrDup(db, SCHEMA_TABLE(iDb)); pName = pName1; @@ -97672,17 +123186,20 @@ SQLITE_PRIVATE void sqlite3StartTable( iDb = sqlite3TwoPartName(pParse, pName1, pName2, &pName); if( iDb<0 ) return; if( !OMIT_TEMPDB && isTemp && pName2->n>0 && iDb!=1 ){ - /* If creating a temp table, the name may not be qualified. Unless + /* If creating a temp table, the name may not be qualified. Unless ** the database name is "temp" anyway. */ sqlite3ErrorMsg(pParse, "temporary table name must be unqualified"); return; } if( !OMIT_TEMPDB && isTemp ) iDb = 1; zName = sqlite3NameFromToken(db, pName); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)zName, pName); + } } pParse->sNameToken = *pName; if( zName==0 ) return; - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( sqlite3CheckObjectName(pParse, zName, isView?"view":"table", zName) ){ goto begin_table_error; } if( db->init.iDb==1 ) isTemp = 1; @@ -97696,7 +123213,7 @@ SQLITE_PRIVATE void sqlite3StartTable( SQLITE_CREATE_VIEW, SQLITE_CREATE_TEMP_VIEW }; - char *zDb = db->aDb[iDb].zName; + char *zDb = db->aDb[iDb].zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(isTemp), 0, zDb) ){ goto begin_table_error; } @@ -97714,18 +123231,20 @@ SQLITE_PRIVATE void sqlite3StartTable( ** and types will be used, so there is no need to test for namespace ** collisions. */ - if( !IN_DECLARE_VTAB ){ - char *zDb = db->aDb[iDb].zName; + if( !IN_SPECIAL_PARSE ){ + char *zDb = db->aDb[iDb].zDbSName; if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto begin_table_error; } pTable = sqlite3FindTable(db, zName, zDb); if( pTable ){ if( !noErr ){ - sqlite3ErrorMsg(pParse, "table %T already exists", pName); + sqlite3ErrorMsg(pParse, "%s %T already exists", + (IsView(pTable)? "view" : "table"), pName); }else{ assert( !db->init.busy || CORRUPT_DB ); sqlite3CodeVerifySchema(pParse, iDb); + sqlite3ForceNotReadOnly(pParse); } goto begin_table_error; } @@ -97745,27 +123264,20 @@ SQLITE_PRIVATE void sqlite3StartTable( pTable->zName = zName; pTable->iPKey = -1; pTable->pSchema = db->aDb[iDb].pSchema; - pTable->nRef = 1; + pTable->nTabRef = 1; +#ifdef SQLITE_DEFAULT_ROWEST + pTable->nRowLogEst = sqlite3LogEst(SQLITE_DEFAULT_ROWEST); +#else pTable->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); +#endif assert( pParse->pNewTable==0 ); pParse->pNewTable = pTable; - /* If this is the magic sqlite_sequence table used by autoincrement, - ** then record a pointer to this table in the main database structure - ** so that INSERT can find the table easily. - */ -#ifndef SQLITE_OMIT_AUTOINCREMENT - if( !pParse->nested && strcmp(zName, "sqlite_sequence")==0 ){ - assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - pTable->pSchema->pSeqTab = pTable; - } -#endif - /* Begin generating the code that will insert the table record into - ** the SQLITE_MASTER table. Note in particular that we must go ahead + ** the schema table. Note in particular that we must go ahead ** and allocate the record number for the table entry now. Before any ** PRIMARY KEY or UNIQUE keywords are parsed. Those keywords will cause - ** indices to be created and the table record must come before the + ** indices to be created and the table record must come before the ** indices. Hence, the record number for the table must be allocated ** now. */ @@ -97783,7 +123295,7 @@ SQLITE_PRIVATE void sqlite3StartTable( } #endif - /* If the file format and encoding in the database have not been set, + /* If the file format and encoding in the database have not been set, ** set them now. */ reg1 = pParse->regRowid = ++pParse->nMem; @@ -97798,7 +123310,7 @@ SQLITE_PRIVATE void sqlite3StartTable( sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_TEXT_ENCODING, ENC(db)); sqlite3VdbeJumpHere(v, addr1); - /* This just creates a place-holder record in the sqlite_master table. + /* This just creates a place-holder record in the sqlite_schema table. ** The record created does not contain anything yet. It will be replaced ** by the real entry in code generated at sqlite3EndTable(). ** @@ -97813,9 +123325,11 @@ SQLITE_PRIVATE void sqlite3StartTable( }else #endif { - pParse->addrCrTab = sqlite3VdbeAddOp2(v, OP_CreateTable, iDb, reg2); + assert( !pParse->bReturning ); + pParse->u1.addrCrTab = + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, reg2, BTREE_INTKEY); } - sqlite3OpenMasterTable(pParse, iDb); + sqlite3OpenSchemaTable(pParse, iDb); sqlite3VdbeAddOp2(v, OP_NewRowid, 0, reg1); sqlite3VdbeAddOp4(v, OP_Blob, 6, reg3, 0, nullRow, P4_STATIC); sqlite3VdbeAddOp3(v, OP_Insert, 0, reg3, reg1); @@ -97828,6 +123342,7 @@ SQLITE_PRIVATE void sqlite3StartTable( /* If an error occurs, we jump here */ begin_table_error: + pParse->checkSchema = 1; sqlite3DbFree(db, zName); return; } @@ -97837,14 +123352,84 @@ begin_table_error: */ #if SQLITE_ENABLE_HIDDEN_COLUMNS SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ - if( sqlite3_strnicmp(pCol->zName, "__hidden__", 10)==0 ){ + if( sqlite3_strnicmp(pCol->zCnName, "__hidden__", 10)==0 ){ pCol->colFlags |= COLFLAG_HIDDEN; + if( pTab ) pTab->tabFlags |= TF_HasHidden; }else if( pTab && pCol!=pTab->aCol && (pCol[-1].colFlags & COLFLAG_HIDDEN) ){ pTab->tabFlags |= TF_OOOHidden; } } #endif +/* +** Clean up the data structures associated with the RETURNING clause. +*/ +static void sqlite3DeleteReturning(sqlite3 *db, void *pArg){ + Returning *pRet = (Returning*)pArg; + Hash *pHash; + pHash = &(db->aDb[1].pSchema->trigHash); + sqlite3HashInsert(pHash, pRet->zName, 0); + sqlite3ExprListDelete(db, pRet->pReturnEL); + sqlite3DbFree(db, pRet); +} + +/* +** Add the RETURNING clause to the parse currently underway. +** +** This routine creates a special TEMP trigger that will fire for each row +** of the DML statement. That TEMP trigger contains a single SELECT +** statement with a result set that is the argument of the RETURNING clause. +** The trigger has the Trigger.bReturning flag and an opcode of +** TK_RETURNING instead of TK_SELECT, so that the trigger code generator +** knows to handle it specially. The TEMP trigger is automatically +** removed at the end of the parse. +** +** When this routine is called, we do not yet know if the RETURNING clause +** is attached to a DELETE, INSERT, or UPDATE, so construct it as a +** RETURNING trigger instead. It will then be converted into the appropriate +** type on the first call to sqlite3TriggersExist(). +*/ +SQLITE_PRIVATE void sqlite3AddReturning(Parse *pParse, ExprList *pList){ + Returning *pRet; + Hash *pHash; + sqlite3 *db = pParse->db; + if( pParse->pNewTrigger ){ + sqlite3ErrorMsg(pParse, "cannot use RETURNING in a trigger"); + }else{ + assert( pParse->bReturning==0 || pParse->ifNotExists ); + } + pParse->bReturning = 1; + pRet = sqlite3DbMallocZero(db, sizeof(*pRet)); + if( pRet==0 ){ + sqlite3ExprListDelete(db, pList); + return; + } + pParse->u1.pReturning = pRet; + pRet->pParse = pParse; + pRet->pReturnEL = pList; + sqlite3ParserAddCleanup(pParse, sqlite3DeleteReturning, pRet); + testcase( pParse->earlyCleanup ); + if( db->mallocFailed ) return; + sqlite3_snprintf(sizeof(pRet->zName), pRet->zName, + "sqlite_returning_%p", pParse); + pRet->retTrig.zName = pRet->zName; + pRet->retTrig.op = TK_RETURNING; + pRet->retTrig.tr_tm = TRIGGER_AFTER; + pRet->retTrig.bReturning = 1; + pRet->retTrig.pSchema = db->aDb[1].pSchema; + pRet->retTrig.pTabSchema = db->aDb[1].pSchema; + pRet->retTrig.step_list = &pRet->retTStep; + pRet->retTStep.op = TK_RETURNING; + pRet->retTStep.pTrig = &pRet->retTrig; + pRet->retTStep.pExprList = pList; + pHash = &(db->aDb[1].pSchema->trigHash); + assert( sqlite3HashFind(pHash, pRet->zName)==0 + || pParse->nErr || pParse->ifNotExists ); + if( sqlite3HashInsert(pHash, pRet->zName, &pRet->retTrig) + ==&pRet->retTrig ){ + sqlite3OomFault(db); + } +} /* ** Add a new column to the table currently being constructed. @@ -97854,60 +123439,110 @@ SQLITE_PRIVATE void sqlite3ColumnPropertiesFromName(Table *pTab, Column *pCol){ ** first to get things going. Then this routine is called for each ** column. */ -SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ +SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token sName, Token sType){ Table *p; int i; char *z; char *zType; Column *pCol; sqlite3 *db = pParse->db; + u8 hName; + Column *aNew; + u8 eType = COLTYPE_CUSTOM; + u8 szEst = 1; + char affinity = SQLITE_AFF_BLOB; + if( (p = pParse->pNewTable)==0 ) return; -#if SQLITE_MAX_COLUMN if( p->nCol+1>db->aLimit[SQLITE_LIMIT_COLUMN] ){ sqlite3ErrorMsg(pParse, "too many columns on %s", p->zName); return; } -#endif - z = sqlite3DbMallocRaw(db, pName->n + pType->n + 2); + if( !IN_RENAME_OBJECT ) sqlite3DequoteToken(&sName); + + /* Because keywords GENERATE ALWAYS can be converted into identifiers + ** by the parser, we can sometimes end up with a typename that ends + ** with "generated always". Check for this case and omit the surplus + ** text. */ + if( sType.n>=16 + && sqlite3_strnicmp(sType.z+(sType.n-6),"always",6)==0 + ){ + sType.n -= 6; + while( ALWAYS(sType.n>0) && sqlite3Isspace(sType.z[sType.n-1]) ) sType.n--; + if( sType.n>=9 + && sqlite3_strnicmp(sType.z+(sType.n-9),"generated",9)==0 + ){ + sType.n -= 9; + while( sType.n>0 && sqlite3Isspace(sType.z[sType.n-1]) ) sType.n--; + } + } + + /* Check for standard typenames. For standard typenames we will + ** set the Column.eType field rather than storing the typename after + ** the column name, in order to save space. */ + if( sType.n>=3 ){ + sqlite3DequoteToken(&sType); + for(i=0; i0) ); if( z==0 ) return; - memcpy(z, pName->z, pName->n); - z[pName->n] = 0; + if( IN_RENAME_OBJECT ) sqlite3RenameTokenMap(pParse, (void*)z, &sName); + memcpy(z, sName.z, sName.n); + z[sName.n] = 0; sqlite3Dequote(z); + hName = sqlite3StrIHash(z); for(i=0; inCol; i++){ - if( sqlite3_stricmp(z, p->aCol[i].zName)==0 ){ + if( p->aCol[i].hName==hName && sqlite3StrICmp(z, p->aCol[i].zCnName)==0 ){ sqlite3ErrorMsg(pParse, "duplicate column name: %s", z); sqlite3DbFree(db, z); return; } } - if( (p->nCol & 0x7)==0 ){ - Column *aNew; - aNew = sqlite3DbRealloc(db,p->aCol,(p->nCol+8)*sizeof(p->aCol[0])); - if( aNew==0 ){ - sqlite3DbFree(db, z); - return; - } - p->aCol = aNew; + aNew = sqlite3DbRealloc(db,p->aCol,((i64)p->nCol+1)*sizeof(p->aCol[0])); + if( aNew==0 ){ + sqlite3DbFree(db, z); + return; } + p->aCol = aNew; pCol = &p->aCol[p->nCol]; memset(pCol, 0, sizeof(p->aCol[0])); - pCol->zName = z; + pCol->zCnName = z; + pCol->hName = hName; sqlite3ColumnPropertiesFromName(p, pCol); - - if( pType->n==0 ){ + + if( sType.n==0 ){ /* If there is no type specified, columns have the default affinity - ** 'BLOB'. */ - pCol->affinity = SQLITE_AFF_BLOB; - pCol->szEst = 1; + ** 'BLOB' with a default size of 4 bytes. */ + pCol->affinity = affinity; + pCol->eCType = eType; + pCol->szEst = szEst; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( affinity==SQLITE_AFF_BLOB ){ + if( 4>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } + } +#endif }else{ zType = z + sqlite3Strlen30(z) + 1; - memcpy(zType, pType->z, pType->n); - zType[pType->n] = 0; + memcpy(zType, sType.z, sType.n); + zType[sType.n] = 0; sqlite3Dequote(zType); - pCol->affinity = sqlite3AffinityType(zType, &pCol->szEst); + pCol->affinity = sqlite3AffinityType(zType, pCol); pCol->colFlags |= COLFLAG_HASTYPE; } p->nCol++; + p->nNVCol++; pParse->constraintName.n = 0; } @@ -97919,20 +123554,35 @@ SQLITE_PRIVATE void sqlite3AddColumn(Parse *pParse, Token *pName, Token *pType){ */ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ Table *p; + Column *pCol; p = pParse->pNewTable; if( p==0 || NEVER(p->nCol<1) ) return; - p->aCol[p->nCol-1].notNull = (u8)onError; + pCol = &p->aCol[p->nCol-1]; + pCol->notNull = (u8)onError; + p->tabFlags |= TF_HasNotNull; + + /* Set the uniqNotNull flag on any UNIQUE or PK indexes already created + ** on this column. */ + if( pCol->colFlags & COLFLAG_UNIQUE ){ + Index *pIdx; + for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ + assert( pIdx->nKeyCol==1 && pIdx->onError!=OE_None ); + if( pIdx->aiColumn[0]==p->nCol-1 ){ + pIdx->uniqNotNull = 1; + } + } + } } /* ** Scan the column type name zType (length nType) and return the ** associated affinity type. ** -** This routine does a case-independent search of zType for the +** This routine does a case-independent search of zType for the ** substrings in the following table. If one of the substrings is ** found, the corresponding affinity is returned. If zType contains -** more than one of the substrings, entries toward the top of -** the table take priority. For example, if zType is 'BLOBINT', +** more than one of the substrings, entries toward the top of +** the table take priority. For example, if zType is 'BLOBINT', ** SQLITE_AFF_INTEGER is returned. ** ** Substring | Affinity @@ -97949,14 +123599,15 @@ SQLITE_PRIVATE void sqlite3AddNotNull(Parse *pParse, int onError){ ** If none of the substrings in the above table are found, ** SQLITE_AFF_NUMERIC is returned. */ -SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ +SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, Column *pCol){ u32 h = 0; char aff = SQLITE_AFF_NUMERIC; const char *zChar = 0; assert( zIn!=0 ); while( zIn[0] ){ - h = (h<<8) + sqlite3UpperToLower[(*zIn)&0xff]; + u8 x = *(u8*)zIn; + h = (h<<8) + sqlite3UpperToLower[x]; zIn++; if( h==(('c'<<24)+('h'<<16)+('a'<<8)+'r') ){ /* CHAR */ aff = SQLITE_AFF_TEXT; @@ -97986,27 +123637,32 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ } } - /* If pszEst is not NULL, store an estimate of the field size. The + /* If pCol is not NULL, store an estimate of the field size. The ** estimate is scaled so that the size of an integer is 1. */ - if( pszEst ){ - *pszEst = 1; /* default size is approx 4 bytes */ + if( pCol ){ + int v = 0; /* default size is approx 4 bytes */ if( aff r=(k/4+1) */ sqlite3GetInt32(zChar, &v); - v = v/4 + 1; - if( v>255 ) v = 255; - *pszEst = v; /* BLOB(k), VARCHAR(k), CHAR(k) -> r=(k/4+1) */ break; } zChar++; } }else{ - *pszEst = 5; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ + v = 16; /* BLOB, TEXT, CLOB -> r=5 (approx 20 bytes)*/ } } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( v>=sqlite3GlobalConfig.szSorterRef ){ + pCol->colFlags |= COLFLAG_SORTERREF; + } +#endif + v = v/4 + 1; + if( v>255 ) v = 255; + pCol->szEst = v; } return aff; } @@ -98021,39 +123677,52 @@ SQLITE_PRIVATE char sqlite3AffinityType(const char *zIn, u8 *pszEst){ ** This routine is called by the parser while in the middle of ** parsing a CREATE TABLE statement. */ -SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ +SQLITE_PRIVATE void sqlite3AddDefaultValue( + Parse *pParse, /* Parsing context */ + Expr *pExpr, /* The parsed expression of the default value */ + const char *zStart, /* Start of the default value text */ + const char *zEnd /* First character past end of default value text */ +){ Table *p; Column *pCol; sqlite3 *db = pParse->db; p = pParse->pNewTable; if( p!=0 ){ + int isInit = db->init.busy && db->init.iDb!=1; pCol = &(p->aCol[p->nCol-1]); - if( !sqlite3ExprIsConstantOrFunction(pSpan->pExpr, db->init.busy) ){ + if( !sqlite3ExprIsConstantOrFunction(pExpr, isInit) ){ sqlite3ErrorMsg(pParse, "default value of column [%s] is not constant", - pCol->zName); + pCol->zCnName); +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + }else if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, "cannot use DEFAULT on a generated column"); +#endif }else{ /* A copy of pExpr is used instead of the original, as pExpr contains - ** tokens that point to volatile memory. The 'span' of the expression - ** is required by pragma table_info. + ** tokens that point to volatile memory. */ - Expr x; - sqlite3ExprDelete(db, pCol->pDflt); + Expr x, *pDfltExpr; memset(&x, 0, sizeof(x)); x.op = TK_SPAN; - x.u.zToken = sqlite3DbStrNDup(db, (char*)pSpan->zStart, - (int)(pSpan->zEnd - pSpan->zStart)); - x.pLeft = pSpan->pExpr; + x.u.zToken = sqlite3DbSpanDup(db, zStart, zEnd); + x.pLeft = pExpr; x.flags = EP_Skip; - pCol->pDflt = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); + pDfltExpr = sqlite3ExprDup(db, &x, EXPRDUP_REDUCE); sqlite3DbFree(db, x.u.zToken); + sqlite3ColumnSetExpr(pParse, p, pCol, pDfltExpr); } } - sqlite3ExprDelete(db, pSpan->pExpr); + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprUnmap(pParse, pExpr); + } + sqlite3ExprDelete(db, pExpr); } /* ** Backwards Compatibility Hack: -** +** ** Historical versions of SQLite accepted strings as column names in ** indexes and PRIMARY KEY constraints and in UNIQUE constraints. Example: ** @@ -98064,7 +123733,7 @@ SQLITE_PRIVATE void sqlite3AddDefaultValue(Parse *pParse, ExprSpan *pSpan){ ** accept it. This routine does the necessary conversion. It converts ** the expression given in its argument from a TK_STRING into a TK_ID ** if the expression is just a TK_STRING with an optional COLLATE clause. -** If the epxression is anything other than TK_STRING, the expression is +** If the expression is anything other than TK_STRING, the expression is ** unchanged. */ static void sqlite3StringToId(Expr *p){ @@ -98076,7 +123745,22 @@ static void sqlite3StringToId(Expr *p){ } /* -** Designate the PRIMARY KEY for the table. pList is a list of names +** Tag the given column as being part of the PRIMARY KEY +*/ +static void makeColumnPartOfPrimaryKey(Parse *pParse, Column *pCol){ + pCol->colFlags |= COLFLAG_PRIMKEY; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "generated columns cannot be part of the PRIMARY KEY"); + } +#endif +} + +/* +** Designate the PRIMARY KEY for the table. pList is a list of names ** of columns that form the primary key. If pList is NULL, then the ** most recently added column of the table is the primary key. ** @@ -98104,9 +123788,9 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( Column *pCol = 0; int iCol = -1, i; int nTerm; - if( pTab==0 || IN_DECLARE_VTAB ) goto primary_key_exit; + if( pTab==0 ) goto primary_key_exit; if( pTab->tabFlags & TF_HasPrimaryKey ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "table \"%s\" has more than one primary key", pTab->zName); goto primary_key_exit; } @@ -98114,7 +123798,7 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( if( pList==0 ){ iCol = pTab->nCol - 1; pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + makeColumnPartOfPrimaryKey(pParse, pCol); nTerm = 1; }else{ nTerm = pList->nExpr; @@ -98123,11 +123807,13 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( assert( pCExpr!=0 ); sqlite3StringToId(pCExpr); if( pCExpr->op==TK_ID ){ - const char *zCName = pCExpr->u.zToken; + const char *zCName; + assert( !ExprHasProperty(pCExpr, EP_IntValue) ); + zCName = pCExpr->u.zToken; for(iCol=0; iColnCol; iCol++){ - if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zName)==0 ){ + if( sqlite3StrICmp(zCName, pTab->aCol[iCol].zCnName)==0 ){ pCol = &pTab->aCol[iCol]; - pCol->colFlags |= COLFLAG_PRIMKEY; + makeColumnPartOfPrimaryKey(pParse, pCol); break; } } @@ -98136,26 +123822,27 @@ SQLITE_PRIVATE void sqlite3AddPrimaryKey( } if( nTerm==1 && pCol - && sqlite3StrICmp(sqlite3ColumnType(pCol,""), "INTEGER")==0 + && pCol->eCType==COLTYPE_INTEGER && sortOrder!=SQLITE_SO_DESC ){ + if( IN_RENAME_OBJECT && pList ){ + Expr *pCExpr = sqlite3ExprSkipCollate(pList->a[0].pExpr); + sqlite3RenameTokenRemap(pParse, &pTab->iPKey, pCExpr); + } pTab->iPKey = iCol; pTab->keyConf = (u8)onError; assert( autoInc==0 || autoInc==1 ); pTab->tabFlags |= autoInc*TF_Autoincrement; - if( pList ) pParse->iPkSortOrder = pList->a[0].sortOrder; + if( pList ) pParse->iPkSortOrder = pList->a[0].fg.sortFlags; + (void)sqlite3HasExplicitNulls(pParse, pList); }else if( autoInc ){ #ifndef SQLITE_OMIT_AUTOINCREMENT sqlite3ErrorMsg(pParse, "AUTOINCREMENT is only allowed on an " "INTEGER PRIMARY KEY"); #endif }else{ - Index *p; - p = sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, - 0, sortOrder, 0); - if( p ){ - p->idxType = SQLITE_IDXTYPE_PRIMARYKEY; - } + sqlite3CreateIndex(pParse, 0, 0, 0, pList, onError, 0, + 0, sortOrder, 0, SQLITE_IDXTYPE_PRIMARYKEY); pList = 0; } @@ -98168,8 +123855,10 @@ primary_key_exit: ** Add a new CHECK constraint to the table currently under construction. */ SQLITE_PRIVATE void sqlite3AddCheckConstraint( - Parse *pParse, /* Parsing context */ - Expr *pCheckExpr /* The check expression */ + Parse *pParse, /* Parsing context */ + Expr *pCheckExpr, /* The check expression */ + const char *zStart, /* Opening "(" */ + const char *zEnd /* Closing ")" */ ){ #ifndef SQLITE_OMIT_CHECK Table *pTab = pParse->pNewTable; @@ -98180,6 +123869,13 @@ SQLITE_PRIVATE void sqlite3AddCheckConstraint( pTab->pCheck = sqlite3ExprListAppend(pParse, pTab->pCheck, pCheckExpr); if( pParse->constraintName.n ){ sqlite3ExprListSetName(pParse, pTab->pCheck, &pParse->constraintName, 1); + }else{ + Token t; + for(zStart++; sqlite3Isspace(zStart[0]); zStart++){} + while( sqlite3Isspace(zEnd[-1]) ){ zEnd--; } + t.z = zStart; + t.n = (int)(zEnd - t.z); + sqlite3ExprListSetName(pParse, pTab->pCheck, &t, 1); } }else #endif @@ -98198,7 +123894,7 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ char *zColl; /* Dequoted name of collation sequence */ sqlite3 *db; - if( (p = pParse->pNewTable)==0 ) return; + if( (p = pParse->pNewTable)==0 || IN_RENAME_OBJECT ) return; i = p->nCol-1; db = pParse->db; zColl = sqlite3NameFromToken(db, pToken); @@ -98206,9 +123902,8 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ if( sqlite3LocateCollSeq(pParse, zColl) ){ Index *pIdx; - sqlite3DbFree(db, p->aCol[i].zColl); - p->aCol[i].zColl = zColl; - + sqlite3ColumnSetColl(db, &p->aCol[i], zColl); + /* If the column is declared as " PRIMARY KEY COLLATE ", ** then an index may have been created on this column before the ** collation type was added. Correct this if it is the case. @@ -98216,49 +123911,73 @@ SQLITE_PRIVATE void sqlite3AddCollateType(Parse *pParse, Token *pToken){ for(pIdx=p->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->nKeyCol==1 ); if( pIdx->aiColumn[0]==i ){ - pIdx->azColl[0] = p->aCol[i].zColl; + pIdx->azColl[0] = sqlite3ColumnColl(&p->aCol[i]); } } - }else{ - sqlite3DbFree(db, zColl); } + sqlite3DbFree(db, zColl); } -/* -** This function returns the collation sequence for database native text -** encoding identified by the string zName, length nName. -** -** If the requested collation sequence is not available, or not available -** in the database native encoding, the collation factory is invoked to -** request it. If the collation factory does not supply such a sequence, -** and the sequence is available in another text encoding, then that is -** returned instead. -** -** If no versions of the requested collations sequence are available, or -** another error occurs, NULL is returned and an error message written into -** pParse. -** -** This routine is a wrapper around sqlite3FindCollSeq(). This routine -** invokes the collation factory if the named collation cannot be found -** and generates an error message. -** -** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() +/* Change the most recently parsed column to be a GENERATED ALWAYS AS +** column. */ -SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ - sqlite3 *db = pParse->db; - u8 enc = ENC(db); - u8 initbusy = db->init.busy; - CollSeq *pColl; - - pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); - if( !initbusy && (!pColl || !pColl->xCmp) ){ - pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); +SQLITE_PRIVATE void sqlite3AddGenerated(Parse *pParse, Expr *pExpr, Token *pType){ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + u8 eType = COLFLAG_VIRTUAL; + Table *pTab = pParse->pNewTable; + Column *pCol; + if( pTab==0 ){ + /* generated column in an CREATE TABLE IF NOT EXISTS that already exists */ + goto generated_done; } + pCol = &(pTab->aCol[pTab->nCol-1]); + if( IN_DECLARE_VTAB ){ + sqlite3ErrorMsg(pParse, "virtual tables cannot use computed columns"); + goto generated_done; + } + if( pCol->iDflt>0 ) goto generated_error; + if( pType ){ + if( pType->n==7 && sqlite3StrNICmp("virtual",pType->z,7)==0 ){ + /* no-op */ + }else if( pType->n==6 && sqlite3StrNICmp("stored",pType->z,6)==0 ){ + eType = COLFLAG_STORED; + }else{ + goto generated_error; + } + } + if( eType==COLFLAG_VIRTUAL ) pTab->nNVCol--; + pCol->colFlags |= eType; + assert( TF_HasVirtual==COLFLAG_VIRTUAL ); + assert( TF_HasStored==COLFLAG_STORED ); + pTab->tabFlags |= eType; + if( pCol->colFlags & COLFLAG_PRIMKEY ){ + makeColumnPartOfPrimaryKey(pParse, pCol); /* For the error message */ + } + if( ALWAYS(pExpr) && pExpr->op==TK_ID ){ + /* The value of a generated column needs to be a real expression, not + ** just a reference to another column, in order for covering index + ** optimizations to work correctly. So if the value is not an expression, + ** turn it into one by adding a unary "+" operator. */ + pExpr = sqlite3PExpr(pParse, TK_UPLUS, pExpr, 0); + } + if( pExpr && pExpr->op!=TK_RAISE ) pExpr->affExpr = pCol->affinity; + sqlite3ColumnSetExpr(pParse, pTab, pCol, pExpr); + pExpr = 0; + goto generated_done; - return pColl; +generated_error: + sqlite3ErrorMsg(pParse, "error in generated column \"%s\"", + pCol->zCnName); +generated_done: + sqlite3ExprDelete(pParse->db, pExpr); +#else + /* Throw and error for the GENERATED ALWAYS AS clause if the + ** SQLITE_OMIT_GENERATED_COLUMNS compile-time option is used. */ + sqlite3ErrorMsg(pParse, "generated columns not supported"); + sqlite3ExprDelete(pParse->db, pExpr); +#endif } - /* ** Generate code that will increment the schema cookie. ** @@ -98274,13 +123993,16 @@ SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ ** set back to prior value. But schema changes are infrequent ** and the probability of hitting the same cookie value is only ** 1 chance in 2^32. So we're safe enough. +** +** IMPLEMENTATION-OF: R-34230-56049 SQLite automatically increments +** the schema-version whenever the schema changes. */ SQLITE_PRIVATE void sqlite3ChangeCookie(Parse *pParse, int iDb){ sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, - db->aDb[iDb].pSchema->schema_cookie+1); + sqlite3VdbeAddOp3(v, OP_SetCookie, iDb, BTREE_SCHEMA_VERSION, + (int)(1+(unsigned)db->aDb[iDb].pSchema->schema_cookie)); } /* @@ -98300,14 +124022,14 @@ static int identLength(const char *z){ } /* -** The first parameter is a pointer to an output buffer. The second +** The first parameter is a pointer to an output buffer. The second ** parameter is a pointer to an integer that contains the offset at ** which to write into the output buffer. This function copies the ** nul-terminated string pointed to by the third parameter, zSignedIdent, ** to the specified offset in the buffer and updates *pIdx to refer ** to the first byte after the last byte written before returning. -** -** If the string zSignedIdent consists entirely of alpha-numeric +** +** If the string zSignedIdent consists entirely of alphanumeric ** characters, does not begin with a digit and is not an SQL keyword, ** then it is copied to the output buffer exactly as it is. Otherwise, ** it is quoted using double-quotes. @@ -98347,10 +124069,10 @@ static char *createTableStmt(sqlite3 *db, Table *p){ Column *pCol; n = 0; for(pCol = p->aCol, i=0; inCol; i++, pCol++){ - n += identLength(pCol->zName) + 5; + n += identLength(pCol->zCnName) + 5; } n += identLength(p->zName); - if( n<50 ){ + if( n<50 ){ zSep = ""; zSep2 = ","; zEnd = ")"; @@ -98375,7 +124097,8 @@ static char *createTableStmt(sqlite3 *db, Table *p){ /* SQLITE_AFF_TEXT */ " TEXT", /* SQLITE_AFF_NUMERIC */ " NUM", /* SQLITE_AFF_INTEGER */ " INT", - /* SQLITE_AFF_REAL */ " REAL" + /* SQLITE_AFF_REAL */ " REAL", + /* SQLITE_AFF_FLEXNUM */ " NUM", }; int len; const char *zType; @@ -98383,7 +124106,7 @@ static char *createTableStmt(sqlite3 *db, Table *p){ sqlite3_snprintf(n-k, &zStmt[k], zSep); k += sqlite3Strlen30(&zStmt[k]); zSep = zSep2; - identPut(zStmt, &k, pCol->zName); + identPut(zStmt, &k, pCol->zCnName); assert( pCol->affinity-SQLITE_AFF_BLOB >= 0 ); assert( pCol->affinity-SQLITE_AFF_BLOB < ArraySize(azType) ); testcase( pCol->affinity==SQLITE_AFF_BLOB ); @@ -98391,10 +124114,12 @@ static char *createTableStmt(sqlite3 *db, Table *p){ testcase( pCol->affinity==SQLITE_AFF_NUMERIC ); testcase( pCol->affinity==SQLITE_AFF_INTEGER ); testcase( pCol->affinity==SQLITE_AFF_REAL ); - + testcase( pCol->affinity==SQLITE_AFF_FLEXNUM ); + zType = azType[pCol->affinity - SQLITE_AFF_BLOB]; len = sqlite3Strlen30(zType); - assert( pCol->affinity==SQLITE_AFF_BLOB + assert( pCol->affinity==SQLITE_AFF_BLOB + || pCol->affinity==SQLITE_AFF_FLEXNUM || pCol->affinity==sqlite3AffinityType(zType, 0) ); memcpy(&zStmt[k], zType, len); k += len; @@ -98413,12 +124138,15 @@ static int resizeIndexObject(sqlite3 *db, Index *pIdx, int N){ int nByte; if( pIdx->nColumn>=N ) return SQLITE_OK; assert( pIdx->isResized==0 ); - nByte = (sizeof(char*) + sizeof(i16) + 1)*N; + nByte = (sizeof(char*) + sizeof(LogEst) + sizeof(i16) + 1)*N; zExtra = sqlite3DbMallocZero(db, nByte); if( zExtra==0 ) return SQLITE_NOMEM_BKPT; memcpy(zExtra, pIdx->azColl, sizeof(char*)*pIdx->nColumn); pIdx->azColl = (const char**)zExtra; zExtra += sizeof(char*)*N; + memcpy(zExtra, pIdx->aiRowLogEst, sizeof(LogEst)*(pIdx->nKeyCol+1)); + pIdx->aiRowLogEst = (LogEst*)zExtra; + zExtra += sizeof(LogEst)*N; memcpy(zExtra, pIdx->aiColumn, sizeof(i16)*pIdx->nColumn); pIdx->aiColumn = (i16*)zExtra; zExtra += sizeof(i16)*N; @@ -98453,18 +124181,93 @@ static void estimateIndexWidth(Index *pIdx){ for(i=0; inColumn; i++){ i16 x = pIdx->aiColumn[i]; assert( xpTable->nCol ); - wIndex += x<0 ? 1 : aCol[pIdx->aiColumn[i]].szEst; + wIndex += x<0 ? 1 : aCol[x].szEst; } pIdx->szIdxRow = sqlite3LogEst(wIndex*4); } -/* Return true if value x is found any of the first nCol entries of aiCol[] +/* Return true if column number x is any of the first nCol entries of aiCol[]. +** This is used to determine if the column number x appears in any of the +** first nCol entries of an index. */ static int hasColumn(const i16 *aiCol, int nCol, int x){ - while( nCol-- > 0 ) if( x==*(aiCol++) ) return 1; + while( nCol-- > 0 ){ + if( x==*(aiCol++) ){ + return 1; + } + } return 0; } +/* +** Return true if any of the first nKey entries of index pIdx exactly +** match the iCol-th entry of pPk. pPk is always a WITHOUT ROWID +** PRIMARY KEY index. pIdx is an index on the same table. pIdx may +** or may not be the same index as pPk. +** +** The first nKey entries of pIdx are guaranteed to be ordinary columns, +** not a rowid or expression. +** +** This routine differs from hasColumn() in that both the column and the +** collating sequence must match for this routine, but for hasColumn() only +** the column name must match. +*/ +static int isDupColumn(Index *pIdx, int nKey, Index *pPk, int iCol){ + int i, j; + assert( nKey<=pIdx->nColumn ); + assert( iColnColumn,pPk->nKeyCol) ); + assert( pPk->idxType==SQLITE_IDXTYPE_PRIMARYKEY ); + assert( pPk->pTable->tabFlags & TF_WithoutRowid ); + assert( pPk->pTable==pIdx->pTable ); + testcase( pPk==pIdx ); + j = pPk->aiColumn[iCol]; + assert( j!=XN_ROWID && j!=XN_EXPR ); + for(i=0; iaiColumn[i]>=0 || j>=0 ); + if( pIdx->aiColumn[i]==j + && sqlite3StrICmp(pIdx->azColl[i], pPk->azColl[iCol])==0 + ){ + return 1; + } + } + return 0; +} + +/* Recompute the colNotIdxed field of the Index. +** +** colNotIdxed is a bitmask that has a 0 bit representing each indexed +** columns that are within the first 63 columns of the table and a 1 for +** all other bits (all columns that are not in the index). The +** high-order bit of colNotIdxed is always 1. All unindexed columns +** of the table have a 1. +** +** 2019-10-24: For the purpose of this computation, virtual columns are +** not considered to be covered by the index, even if they are in the +** index, because we do not trust the logic in whereIndexExprTrans() to be +** able to find all instances of a reference to the indexed table column +** and convert them into references to the index. Hence we always want +** the actual table at hand in order to recompute the virtual column, if +** necessary. +** +** The colNotIdxed mask is AND-ed with the SrcList.a[].colUsed mask +** to determine if the index is covering index. +*/ +static void recomputeColumnsNotIndexed(Index *pIdx){ + Bitmask m = 0; + int j; + Table *pTab = pIdx->pTable; + for(j=pIdx->nColumn-1; j>=0; j--){ + int x = pIdx->aiColumn[j]; + if( x>=0 && (pTab->aCol[x].colFlags & COLFLAG_VIRTUAL)==0 ){ + testcase( x==BMS-1 ); + testcase( x==BMS-2 ); + if( xcolNotIdxed = ~m; + assert( (pIdx->colNotIdxed>>63)==1 ); /* See note-20221022-a */ +} + /* ** This routine runs at the end of parsing a CREATE TABLE statement that ** has a WITHOUT ROWID clause. The job of this routine is to convert both @@ -98472,66 +124275,85 @@ static int hasColumn(const i16 *aiCol, int nCol, int x){ ** are appropriate for a WITHOUT ROWID table instead of a rowid table. ** Changes include: ** -** (1) Convert the OP_CreateTable into an OP_CreateIndex. There is -** no rowid btree for a WITHOUT ROWID. Instead, the canonical -** data storage is a covering index btree. -** (2) Bypass the creation of the sqlite_master table entry +** (1) Set all columns of the PRIMARY KEY schema object to be NOT NULL. +** (2) Convert P3 parameter of the OP_CreateBtree from BTREE_INTKEY +** into BTREE_BLOBKEY. +** (3) Bypass the creation of the sqlite_schema table entry ** for the PRIMARY KEY as the primary key index is now -** identified by the sqlite_master table entry of the table itself. -** (3) Set the Index.tnum of the PRIMARY KEY Index object in the +** identified by the sqlite_schema table entry of the table itself. +** (4) Set the Index.tnum of the PRIMARY KEY Index object in the ** schema to the rootpage from the main table. -** (4) Set all columns of the PRIMARY KEY schema object to be NOT NULL. ** (5) Add all table columns to the PRIMARY KEY Index object ** so that the PRIMARY KEY is a covering index. The surplus -** columns are part of KeyInfo.nXField and are not used for +** columns are part of KeyInfo.nAllField and are not used for ** sorting or lookup or uniqueness checks. ** (6) Replace the rowid tail on all automatically generated UNIQUE ** indices with the PRIMARY KEY columns. +** +** For virtual tables, only (1) is performed. */ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ Index *pIdx; Index *pPk; int nPk; + int nExtra; int i, j; sqlite3 *db = pParse->db; Vdbe *v = pParse->pVdbe; - /* Convert the OP_CreateTable opcode that would normally create the - ** root-page for the table into an OP_CreateIndex opcode. The index - ** created will become the PRIMARY KEY index. + /* Mark every PRIMARY KEY column as NOT NULL (except for imposter tables) */ - if( pParse->addrCrTab ){ + if( !db->init.imposterTable ){ + for(i=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + && (pTab->aCol[i].notNull==OE_None) + ){ + pTab->aCol[i].notNull = OE_Abort; + } + } + pTab->tabFlags |= TF_HasNotNull; + } + + /* Convert the P3 operand of the OP_CreateBtree opcode from BTREE_INTKEY + ** into BTREE_BLOBKEY. + */ + assert( !pParse->bReturning ); + if( pParse->u1.addrCrTab ){ assert( v ); - sqlite3VdbeChangeOpcode(v, pParse->addrCrTab, OP_CreateIndex); + sqlite3VdbeChangeP3(v, pParse->u1.addrCrTab, BTREE_BLOBKEY); } /* Locate the PRIMARY KEY index. Or, if this table was originally - ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. + ** an INTEGER PRIMARY KEY table, create a new PRIMARY KEY index. */ if( pTab->iPKey>=0 ){ ExprList *pList; Token ipkToken; - sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zName); - pList = sqlite3ExprListAppend(pParse, 0, + sqlite3TokenInit(&ipkToken, pTab->aCol[pTab->iPKey].zCnName); + pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &ipkToken, 0)); - if( pList==0 ) return; - pList->a[0].sortOrder = pParse->iPkSortOrder; + if( pList==0 ){ + pTab->tabFlags &= ~TF_WithoutRowid; + return; + } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pList->a[0].pExpr, &pTab->iPKey); + } + pList->a[0].fg.sortFlags = pParse->iPkSortOrder; assert( pParse->pNewTable==pTab ); - pPk = sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0); - if( pPk==0 ) return; - pPk->idxType = SQLITE_IDXTYPE_PRIMARYKEY; pTab->iPKey = -1; + sqlite3CreateIndex(pParse, 0, 0, 0, pList, pTab->keyConf, 0, 0, 0, 0, + SQLITE_IDXTYPE_PRIMARYKEY); + if( pParse->nErr ){ + pTab->tabFlags &= ~TF_WithoutRowid; + return; + } + assert( db->mallocFailed==0 ); + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk->nKeyCol==1 ); }else{ pPk = sqlite3PrimaryKeyIndex(pTab); - - /* Bypass the creation of the PRIMARY KEY btree and the sqlite_master - ** table entry. This is only required if currently generating VDBE - ** code for a CREATE TABLE (not when parsing one as part of reading - ** a database schema). */ - if( v ){ - assert( db->init.busy==0 ); - sqlite3VdbeChangeOpcode(v, pPk->tnum, OP_Goto); - } + assert( pPk!=0 ); /* ** Remove all redundant columns from the PRIMARY KEY. For example, change @@ -98539,25 +124361,29 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** code assumes the PRIMARY KEY contains no repeated columns. */ for(i=j=1; inKeyCol; i++){ - if( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ){ + if( isDupColumn(pPk, j, pPk, i) ){ pPk->nColumn--; }else{ + testcase( hasColumn(pPk->aiColumn, j, pPk->aiColumn[i]) ); + pPk->azColl[j] = pPk->azColl[i]; + pPk->aSortOrder[j] = pPk->aSortOrder[i]; pPk->aiColumn[j++] = pPk->aiColumn[i]; } } pPk->nKeyCol = j; } - pPk->isCovering = 1; assert( pPk!=0 ); - nPk = pPk->nKeyCol; + pPk->isCovering = 1; + if( !db->init.imposterTable ) pPk->uniqNotNull = 1; + nPk = pPk->nColumn = pPk->nKeyCol; - /* Make sure every column of the PRIMARY KEY is NOT NULL. (Except, - ** do not enforce this for imposter tables.) */ - if( !db->init.imposterTable ){ - for(i=0; iaCol[pPk->aiColumn[i]].notNull = OE_Abort; - } - pPk->uniqNotNull = 1; + /* Bypass the creation of the PRIMARY KEY btree and the sqlite_schema + ** table entry. This is only required if currently generating VDBE + ** code for a CREATE TABLE (not when parsing one as part of reading + ** a database schema). */ + if( v && pPk->tnum>0 ){ + assert( db->init.busy==0 ); + sqlite3VdbeChangeOpcode(v, (int)pPk->tnum, OP_Goto); } /* The root page of the PRIMARY KEY is the table root page */ @@ -98570,7 +124396,10 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ int n; if( IsPrimaryKeyIndex(pIdx) ) continue; for(i=n=0; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ) n++; + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); + n++; + } } if( n==0 ){ /* This index is a superset of the primary key */ @@ -98579,9 +124408,14 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ } if( resizeIndexObject(db, pIdx, pIdx->nKeyCol+n) ) return; for(i=0, j=pIdx->nKeyCol; iaiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ){ + if( !isDupColumn(pIdx, pIdx->nKeyCol, pPk, i) ){ + testcase( hasColumn(pIdx->aiColumn, pIdx->nKeyCol, pPk->aiColumn[i]) ); pIdx->aiColumn[j] = pPk->aiColumn[i]; pIdx->azColl[j] = pPk->azColl[i]; + if( pPk->aSortOrder[i] ){ + /* See ticket https://www.sqlite.org/src/info/bba7b69f9849b5bf */ + pIdx->bAscKeyBug = 1; + } j++; } } @@ -98591,22 +124425,133 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ /* Add all table columns to the PRIMARY KEY index */ - if( nPknCol ){ - if( resizeIndexObject(db, pPk, pTab->nCol) ) return; - for(i=0, j=nPk; inCol; i++){ - if( !hasColumn(pPk->aiColumn, j, i) ){ - assert( jnColumn ); - pPk->aiColumn[j] = i; - pPk->azColl[j] = sqlite3StrBINARY; - j++; - } + nExtra = 0; + for(i=0; inCol; i++){ + if( !hasColumn(pPk->aiColumn, nPk, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ) nExtra++; + } + if( resizeIndexObject(db, pPk, nPk+nExtra) ) return; + for(i=0, j=nPk; inCol; i++){ + if( !hasColumn(pPk->aiColumn, j, i) + && (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 + ){ + assert( jnColumn ); + pPk->aiColumn[j] = i; + pPk->azColl[j] = sqlite3StrBINARY; + j++; + } + } + assert( pPk->nColumn==j ); + assert( pTab->nNVCol<=j ); + recomputeColumnsNotIndexed(pPk); +} + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if pTab is a virtual table and zName is a shadow table name +** for that virtual table. +*/ +SQLITE_PRIVATE int sqlite3IsShadowTableOf(sqlite3 *db, Table *pTab, const char *zName){ + int nName; /* Length of zName */ + Module *pMod; /* Module for the virtual table */ + + if( !IsVirtual(pTab) ) return 0; + nName = sqlite3Strlen30(pTab->zName); + if( sqlite3_strnicmp(zName, pTab->zName, nName)!=0 ) return 0; + if( zName[nName]!='_' ) return 0; + pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->u.vtab.azArg[0]); + if( pMod==0 ) return 0; + if( pMod->pModule->iVersion<3 ) return 0; + if( pMod->pModule->xShadowName==0 ) return 0; + return pMod->pModule->xShadowName(zName+nName+1); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Table pTab is a virtual table. If it the virtual table implementation +** exists and has an xShadowName method, then loop over all other ordinary +** tables within the same schema looking for shadow tables of pTab, and mark +** any shadow tables seen using the TF_Shadow flag. +*/ +SQLITE_PRIVATE void sqlite3MarkAllShadowTablesOf(sqlite3 *db, Table *pTab){ + int nName; /* Length of pTab->zName */ + Module *pMod; /* Module for the virtual table */ + HashElem *k; /* For looping through the symbol table */ + + assert( IsVirtual(pTab) ); + pMod = (Module*)sqlite3HashFind(&db->aModule, pTab->u.vtab.azArg[0]); + if( pMod==0 ) return; + if( NEVER(pMod->pModule==0) ) return; + if( pMod->pModule->iVersion<3 ) return; + if( pMod->pModule->xShadowName==0 ) return; + assert( pTab->zName!=0 ); + nName = sqlite3Strlen30(pTab->zName); + for(k=sqliteHashFirst(&pTab->pSchema->tblHash); k; k=sqliteHashNext(k)){ + Table *pOther = sqliteHashData(k); + assert( pOther->zName!=0 ); + if( !IsOrdinaryTable(pOther) ) continue; + if( pOther->tabFlags & TF_Shadow ) continue; + if( sqlite3StrNICmp(pOther->zName, pTab->zName, nName)==0 + && pOther->zName[nName]=='_' + && pMod->pModule->xShadowName(pOther->zName+nName+1) + ){ + pOther->tabFlags |= TF_Shadow; } - assert( pPk->nColumn==j ); - assert( pTab->nCol==j ); - }else{ - pPk->nColumn = pTab->nCol; } } +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if zName is a shadow table name in the current database +** connection. +** +** zName is temporarily modified while this routine is running, but is +** restored to its original value prior to this routine returning. +*/ +SQLITE_PRIVATE int sqlite3ShadowTableName(sqlite3 *db, const char *zName){ + char *zTail; /* Pointer to the last "_" in zName */ + Table *pTab; /* Table that zName is a shadow of */ + zTail = strrchr(zName, '_'); + if( zTail==0 ) return 0; + *zTail = 0; + pTab = sqlite3FindTable(db, zName, 0); + *zTail = '_'; + if( pTab==0 ) return 0; + if( !IsVirtual(pTab) ) return 0; + return sqlite3IsShadowTableOf(db, pTab, zName); +} +#endif /* ifndef SQLITE_OMIT_VIRTUALTABLE */ + + +#ifdef SQLITE_DEBUG +/* +** Mark all nodes of an expression as EP_Immutable, indicating that +** they should not be changed. Expressions attached to a table or +** index definition are tagged this way to help ensure that we do +** not pass them into code generator routines by mistake. +*/ +static int markImmutableExprStep(Walker *pWalker, Expr *pExpr){ + (void)pWalker; + ExprSetVVAProperty(pExpr, EP_Immutable); + return WRC_Continue; +} +static void markExprListImmutable(ExprList *pList){ + if( pList ){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = markImmutableExprStep; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + sqlite3WalkExprList(&w, pList); + } +} +#else +#define markExprListImmutable(X) /* no-op */ +#endif /* SQLITE_DEBUG */ + /* ** This routine is called to report the final ")" that terminates @@ -98616,15 +124561,15 @@ static void convertToWithoutRowidTable(Parse *pParse, Table *pTab){ ** is added to the internal hash tables, assuming no errors have ** occurred. ** -** An entry for the table is made in the master table on disk, unless +** An entry for the table is made in the schema table on disk, unless ** this is a temporary table or db->init.busy==1. When db->init.busy==1 -** it means we are reading the sqlite_master table because we just -** connected to the database or because the sqlite_master table has +** it means we are reading the sqlite_schema table because we just +** connected to the database or because the sqlite_schema table has ** recently changed, so the entry for this table already exists in -** the sqlite_master table. We do not want to create it again. +** the sqlite_schema table. We do not want to create it again. ** ** If the pSelect argument is not NULL, it means that this routine -** was called to create a table generated from a +** was called to create a table generated from a ** "CREATE TABLE ... AS SELECT ..." statement. The column names of ** the new table will match the result set of the SELECT. */ @@ -98632,7 +124577,7 @@ SQLITE_PRIVATE void sqlite3EndTable( Parse *pParse, /* Parse context */ Token *pCons, /* The ',' token after the last column defn. */ Token *pEnd, /* The ')' before options in the CREATE TABLE */ - u8 tabOpts, /* Extra table options. Usually 0. */ + u32 tabOpts, /* Extra table options. Usually 0. */ Select *pSelect /* Select from a "CREATE ... AS SELECT" */ ){ Table *p; /* The new table */ @@ -98643,26 +124588,74 @@ SQLITE_PRIVATE void sqlite3EndTable( if( pEnd==0 && pSelect==0 ){ return; } - assert( !db->mallocFailed ); p = pParse->pNewTable; if( p==0 ) return; - assert( !db->init.busy || !pSelect ); + if( pSelect==0 && sqlite3ShadowTableName(db, p->zName) ){ + p->tabFlags |= TF_Shadow; + } /* If the db->init.busy is 1 it means we are reading the SQL off the - ** "sqlite_master" or "sqlite_temp_master" table on the disk. + ** "sqlite_schema" or "sqlite_temp_schema" table on the disk. ** So do not write to the disk again. Extract the root page number ** for the table from the db->init.newTnum field. (The page number ** should have been put there by the sqliteOpenCb routine.) ** - ** If the root page number is 1, that means this is the sqlite_master + ** If the root page number is 1, that means this is the sqlite_schema ** table itself. So mark it read-only. */ if( db->init.busy ){ + if( pSelect || (!IsOrdinaryTable(p) && db->init.newTnum) ){ + sqlite3ErrorMsg(pParse, ""); + return; + } p->tnum = db->init.newTnum; if( p->tnum==1 ) p->tabFlags |= TF_Readonly; } + /* Special processing for tables that include the STRICT keyword: + ** + ** * Do not allow custom column datatypes. Every column must have + ** a datatype that is one of INT, INTEGER, REAL, TEXT, or BLOB. + ** + ** * If a PRIMARY KEY is defined, other than the INTEGER PRIMARY KEY, + ** then all columns of the PRIMARY KEY must have a NOT NULL + ** constraint. + */ + if( tabOpts & TF_Strict ){ + int ii; + p->tabFlags |= TF_Strict; + for(ii=0; iinCol; ii++){ + Column *pCol = &p->aCol[ii]; + if( pCol->eCType==COLTYPE_CUSTOM ){ + if( pCol->colFlags & COLFLAG_HASTYPE ){ + sqlite3ErrorMsg(pParse, + "unknown datatype for %s.%s: \"%s\"", + p->zName, pCol->zCnName, sqlite3ColumnType(pCol, "") + ); + }else{ + sqlite3ErrorMsg(pParse, "missing datatype for %s.%s", + p->zName, pCol->zCnName); + } + return; + }else if( pCol->eCType==COLTYPE_ANY ){ + pCol->affinity = SQLITE_AFF_BLOB; + } + if( (pCol->colFlags & COLFLAG_PRIMKEY)!=0 + && p->iPKey!=ii + && pCol->notNull == OE_None + ){ + pCol->notNull = OE_Abort; + p->tabFlags |= TF_HasNotNull; + } + } + } + + assert( (p->tabFlags & TF_HasPrimaryKey)==0 + || p->iPKey>=0 || sqlite3PrimaryKeyIndex(p)!=0 ); + assert( (p->tabFlags & TF_HasPrimaryKey)!=0 + || (p->iPKey<0 && sqlite3PrimaryKeyIndex(p)==0) ); + /* Special processing for WITHOUT ROWID Tables */ if( tabOpts & TF_WithoutRowid ){ if( (p->tabFlags & TF_Autoincrement) ){ @@ -98672,12 +124665,11 @@ SQLITE_PRIVATE void sqlite3EndTable( } if( (p->tabFlags & TF_HasPrimaryKey)==0 ){ sqlite3ErrorMsg(pParse, "PRIMARY KEY missing on table %s", p->zName); - }else{ - p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; - convertToWithoutRowidTable(pParse, p); + return; } + p->tabFlags |= TF_WithoutRowid | TF_NoVisibleRowid; + convertToWithoutRowidTable(pParse, p); } - iDb = sqlite3SchemaToIndex(db, p->pSchema); #ifndef SQLITE_OMIT_CHECK @@ -98685,8 +124677,47 @@ SQLITE_PRIVATE void sqlite3EndTable( */ if( p->pCheck ){ sqlite3ResolveSelfReference(pParse, p, NC_IsCheck, 0, p->pCheck); + if( pParse->nErr ){ + /* If errors are seen, delete the CHECK constraints now, else they might + ** actually be used if PRAGMA writable_schema=ON is set. */ + sqlite3ExprListDelete(db, p->pCheck); + p->pCheck = 0; + }else{ + markExprListImmutable(p->pCheck); + } } #endif /* !defined(SQLITE_OMIT_CHECK) */ +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( p->tabFlags & TF_HasGenerated ){ + int ii, nNG = 0; + testcase( p->tabFlags & TF_HasVirtual ); + testcase( p->tabFlags & TF_HasStored ); + for(ii=0; iinCol; ii++){ + u32 colFlags = p->aCol[ii].colFlags; + if( (colFlags & COLFLAG_GENERATED)!=0 ){ + Expr *pX = sqlite3ColumnExpr(p, &p->aCol[ii]); + testcase( colFlags & COLFLAG_VIRTUAL ); + testcase( colFlags & COLFLAG_STORED ); + if( sqlite3ResolveSelfReference(pParse, p, NC_GenCol, pX, 0) ){ + /* If there are errors in resolving the expression, change the + ** expression to a NULL. This prevents code generators that operate + ** on the expression from inserting extra parts into the expression + ** tree that have been allocated from lookaside memory, which is + ** illegal in a schema and will lead to errors or heap corruption + ** when the database connection closes. */ + sqlite3ColumnSetExpr(pParse, p, &p->aCol[ii], + sqlite3ExprAlloc(db, TK_NULL, 0, 0)); + } + }else{ + nNG++; + } + } + if( nNG==0 ){ + sqlite3ErrorMsg(pParse, "must have at least one non-generated column"); + return; + } + } +#endif /* Estimate the average row size for the table and for all implied indices */ estimateTableWidth(p); @@ -98695,7 +124726,7 @@ SQLITE_PRIVATE void sqlite3EndTable( } /* If not initializing, then create a record for the new table - ** in the SQLITE_MASTER table of the database. + ** in the schema table of the database. ** ** If this is a TEMPORARY table, write the entry into the auxiliary ** file instead of into the main database file. @@ -98712,10 +124743,10 @@ SQLITE_PRIVATE void sqlite3EndTable( sqlite3VdbeAddOp1(v, OP_Close, 0); - /* + /* ** Initialize zType for the new view or table. */ - if( p->pSelect==0 ){ + if( IsOrdinaryTable(p) ){ /* A regular table */ zType = "table"; zType2 = "TABLE"; @@ -98748,39 +124779,45 @@ SQLITE_PRIVATE void sqlite3EndTable( int regRowid; /* Rowid of the next row to insert */ int addrInsLoop; /* Top of the loop for inserting rows */ Table *pSelTab; /* A table that describes the SELECT results */ + int iCsr; /* Write cursor on the new table */ + if( IN_SPECIAL_PARSE ){ + pParse->rc = SQLITE_ERROR; + pParse->nErr++; + return; + } + iCsr = pParse->nTab++; regYield = ++pParse->nMem; regRec = ++pParse->nMem; regRowid = ++pParse->nMem; - assert(pParse->nTab==1); sqlite3MayAbort(pParse); - sqlite3VdbeAddOp3(v, OP_OpenWrite, 1, pParse->regRoot, iDb); + sqlite3VdbeAddOp3(v, OP_OpenWrite, iCsr, pParse->regRoot, iDb); sqlite3VdbeChangeP5(v, OPFLAG_P2ISREG); - pParse->nTab = 2; addrTop = sqlite3VdbeCurrentAddr(v) + 1; sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - sqlite3Select(pParse, pSelect, &dest); - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); if( pParse->nErr ) return; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSelect, SQLITE_AFF_BLOB); if( pSelTab==0 ) return; assert( p->aCol==0 ); - p->nCol = pSelTab->nCol; + p->nCol = p->nNVCol = pSelTab->nCol; p->aCol = pSelTab->aCol; pSelTab->nCol = 0; pSelTab->aCol = 0; sqlite3DeleteTable(db, pSelTab); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + sqlite3Select(pParse, pSelect, &dest); + if( pParse->nErr ) return; + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); addrInsLoop = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); sqlite3VdbeAddOp3(v, OP_MakeRecord, dest.iSdst, dest.nSdst, regRec); sqlite3TableAffinity(v, p, 0); - sqlite3VdbeAddOp2(v, OP_NewRowid, 1, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, 1, regRec, regRowid); + sqlite3VdbeAddOp2(v, OP_NewRowid, iCsr, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, iCsr, regRec, regRowid); sqlite3VdbeGoto(v, addrInsLoop); sqlite3VdbeJumpHere(v, addrInsLoop); - sqlite3VdbeAddOp1(v, OP_Close, 1); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); } /* Compute the complete text of the CREATE statement */ @@ -98790,20 +124827,20 @@ SQLITE_PRIVATE void sqlite3EndTable( Token *pEnd2 = tabOpts ? &pParse->sLastToken : pEnd; n = (int)(pEnd2->z - pParse->sNameToken.z); if( pEnd2->z[0]!=';' ) n += pEnd2->n; - zStmt = sqlite3MPrintf(db, + zStmt = sqlite3MPrintf(db, "CREATE %s %.*s", zType2, n, pParse->sNameToken.z ); } - /* A slot for the record has already been allocated in the - ** SQLITE_MASTER table. We just need to update that slot with all + /* A slot for the record has already been allocated in the + ** schema table. We just need to update that slot with all ** the information we've collected. */ sqlite3NestedParse(pParse, - "UPDATE %Q.%s " - "SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q " - "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + "UPDATE %Q." LEGACY_SCHEMA_TABLE + " SET type='%s', name=%Q, tbl_name=%Q, rootpage=#%d, sql=%Q" + " WHERE rowid=#%d", + db->aDb[iDb].zDbSName, zType, p->zName, p->zName, @@ -98818,13 +124855,13 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Check to see if we need to create an sqlite_sequence table for ** keeping track of autoincrement keys. */ - if( p->tabFlags & TF_Autoincrement ){ + if( (p->tabFlags & TF_Autoincrement)!=0 && !IN_SPECIAL_PARSE ){ Db *pDb = &db->aDb[iDb]; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( pDb->pSchema->pSeqTab==0 ){ sqlite3NestedParse(pParse, "CREATE TABLE %Q.sqlite_sequence(name,seq)", - pDb->zName + pDb->zDbSName ); } } @@ -98832,9 +124869,16 @@ SQLITE_PRIVATE void sqlite3EndTable( /* Reparse everything to update our internal data structures */ sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName)); - } + sqlite3MPrintf(db, "tbl_name='%q' AND type!='trigger'", p->zName),0); + /* Test for cycles in generated columns and illegal expressions + ** in CHECK constraints and in DEFAULT clauses. */ + if( p->tabFlags & TF_HasGenerated ){ + sqlite3VdbeAddOp4(v, OP_SqlExec, 0x0001, 0, 0, + sqlite3MPrintf(db, "SELECT*FROM\"%w\".\"%w\"", + db->aDb[iDb].zDbSName, p->zName), P4_DYNAMIC); + } + } /* Add the table to the in-memory representation of the database. */ @@ -98842,6 +124886,7 @@ SQLITE_PRIVATE void sqlite3EndTable( Table *pOld; Schema *pSchema = p->pSchema; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( HasRowid(p) || p->iPKey<0 ); pOld = sqlite3HashInsert(&pSchema->tblHash, p->zName, p); if( pOld ){ assert( p==pOld ); /* Malloc must have failed inside HashInsert() */ @@ -98849,21 +124894,29 @@ SQLITE_PRIVATE void sqlite3EndTable( return; } pParse->pNewTable = 0; - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; -#ifndef SQLITE_OMIT_ALTERTABLE - if( !p->pSelect ){ - const char *zName = (const char *)pParse->sNameToken.z; - int nName; - assert( !pSelect && pCons && pEnd ); - if( pCons->z==0 ){ - pCons = pEnd; - } - nName = (int)((const char *)pCons->z - zName); - p->addColOffset = 13 + sqlite3Utf8CharLen(zName, nName); + /* If this is the magic sqlite_sequence table used by autoincrement, + ** then record a pointer to this table in the main database structure + ** so that INSERT can find the table easily. */ + assert( !pParse->nested ); +#ifndef SQLITE_OMIT_AUTOINCREMENT + if( strcmp(p->zName, "sqlite_sequence")==0 ){ + assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + p->pSchema->pSeqTab = p; } #endif } + +#ifndef SQLITE_OMIT_ALTERTABLE + if( !pSelect && IsOrdinaryTable(p) ){ + assert( pCons && pEnd ); + if( pCons->z==0 ){ + pCons = pEnd; + } + p->u.tab.addColOffset = 13 + (int)(pCons->z - pParse->sNameToken.z); + } +#endif } #ifndef SQLITE_OMIT_VIEW @@ -98896,6 +124949,19 @@ SQLITE_PRIVATE void sqlite3CreateView( sqlite3StartTable(pParse, pName1, pName2, isTemp, 1, 0, noErr); p = pParse->pNewTable; if( p==0 || pParse->nErr ) goto create_view_fail; + + /* Legacy versions of SQLite allowed the use of the magic "rowid" column + ** on a view, even though views do not have rowids. The following flag + ** setting fixes this problem. But the fix can be disabled by compiling + ** with -DSQLITE_ALLOW_ROWID_IN_VIEW in case there are legacy apps that + ** depend upon the old buggy behavior. The ability can also be toggled + ** using sqlite3_config(SQLITE_CONFIG_ROWID_IN_VIEW,...) */ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + p->tabFlags |= sqlite3Config.mNoVisibleRowid; /* Optional. Allow by default */ +#else + p->tabFlags |= TF_NoVisibleRowid; /* Never allow rowid in view */ +#endif + sqlite3TwoPartName(pParse, pName1, pName2, &pName); iDb = sqlite3SchemaToIndex(db, p->pSchema); sqlite3FixInit(&sFix, pParse, iDb, "view", pName); @@ -98906,15 +124972,22 @@ SQLITE_PRIVATE void sqlite3CreateView( ** allocated rather than point to the input string - which means that ** they will persist after the current sqlite3_exec() call returns. */ - p->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + pSelect->selFlags |= SF_View; + if( IN_RENAME_OBJECT ){ + p->u.view.pSelect = pSelect; + pSelect = 0; + }else{ + p->u.view.pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } p->pCheck = sqlite3ExprListDup(db, pCNames, EXPRDUP_REDUCE); + p->eTabType = TABTYP_VIEW; if( db->mallocFailed ) goto create_view_fail; /* Locate the end of the CREATE VIEW statement. Make sEnd point to ** the end. */ sEnd = pParse->sLastToken; - assert( sEnd.z[0]!=0 ); + assert( sEnd.z[0]!=0 || sEnd.n==0 ); if( sEnd.z[0]!=';' ){ sEnd.z += sEnd.n; } @@ -98926,11 +124999,14 @@ SQLITE_PRIVATE void sqlite3CreateView( sEnd.z = &z[n-1]; sEnd.n = 1; - /* Use sqlite3EndTable() to add the view to the SQLITE_MASTER table */ + /* Use sqlite3EndTable() to add the view to the schema table */ sqlite3EndTable(pParse, 0, &sEnd, 0, 0); create_view_fail: sqlite3SelectDelete(db, pSelect); + if( IN_RENAME_OBJECT ){ + sqlite3RenameExprlistUnmap(pParse, pCNames); + } sqlite3ExprListDelete(db, pCNames); return; } @@ -98939,31 +125015,39 @@ create_view_fail: #if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) /* ** The Table structure pTable is really a VIEW. Fill in the names of -** the columns of the view in the pTable structure. Return the number -** of errors. If an error is seen leave an error message in pParse->zErrMsg. +** the columns of the view in the pTable structure. Return non-zero if +** there are errors. If an error is seen an error message is left +** in pParse->zErrMsg. */ -SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ +static SQLITE_NOINLINE int viewGetColumnNames(Parse *pParse, Table *pTable){ Table *pSelTab; /* A fake table from which we get the result set */ Select *pSel; /* Copy of the SELECT that implements the view */ int nErr = 0; /* Number of errors encountered */ - int n; /* Temporarily holds the number of cursors assigned */ sqlite3 *db = pParse->db; /* Database connection for malloc errors */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + int rc; +#endif +#ifndef SQLITE_OMIT_AUTHORIZATION sqlite3_xauth xAuth; /* Saved xAuth pointer */ +#endif assert( pTable ); #ifndef SQLITE_OMIT_VIRTUALTABLE - if( sqlite3VtabCallConnect(pParse, pTable) ){ - return SQLITE_ERROR; + if( IsVirtual(pTable) ){ + db->nSchemaLock++; + rc = sqlite3VtabCallConnect(pParse, pTable); + db->nSchemaLock--; + return rc; } - if( IsVirtual(pTable) ) return 0; #endif #ifndef SQLITE_OMIT_VIEW /* A positive nCol means the columns names for this view are - ** already known. + ** already known. This routine is not called unless either the + ** table is virtual or nCol is zero. */ - if( pTable->nCol>0 ) return 0; + assert( pTable->nCol<=0 ); /* A negative nCol is a special marker meaning that we are currently ** trying to compute the column names. If we enter this routine with @@ -98975,7 +125059,7 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** Actually, the error above is now caught prior to reaching this point. ** But the following test is still important as it does come up ** in the following: - ** + ** ** CREATE TABLE main.ex1(a); ** CREATE TEMP VIEW ex1 AS SELECT a FROM ex1; ** SELECT * FROM temp.ex1; @@ -98993,60 +125077,75 @@ SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ ** to be permanent. So the computation is done on a copy of the SELECT ** statement that defines the view. */ - assert( pTable->pSelect ); - pSel = sqlite3SelectDup(db, pTable->pSelect, 0); + assert( IsView(pTable) ); + pSel = sqlite3SelectDup(db, pTable->u.view.pSelect, 0); if( pSel ){ - n = pParse->nTab; + u8 eParseMode = pParse->eParseMode; + int nTab = pParse->nTab; + int nSelect = pParse->nSelect; + pParse->eParseMode = PARSE_MODE_NORMAL; sqlite3SrcListAssignCursors(pParse, pSel->pSrc); pTable->nCol = -1; - db->lookaside.bDisable++; + DisableLookaside; #ifndef SQLITE_OMIT_AUTHORIZATION xAuth = db->xAuth; db->xAuth = 0; - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); db->xAuth = xAuth; #else - pSelTab = sqlite3ResultSetOfSelect(pParse, pSel); + pSelTab = sqlite3ResultSetOfSelect(pParse, pSel, SQLITE_AFF_NONE); #endif - pParse->nTab = n; - if( pTable->pCheck ){ + pParse->nTab = nTab; + pParse->nSelect = nSelect; + if( pSelTab==0 ){ + pTable->nCol = 0; + nErr++; + }else if( pTable->pCheck ){ /* CREATE VIEW name(arglist) AS ... ** The names of the columns in the table are taken from ** arglist which is stored in pTable->pCheck. The pCheck field ** normally holds CHECK constraints on an ordinary table, but for ** a VIEW it holds the list of column names. */ - sqlite3ColumnsFromExprList(pParse, pTable->pCheck, + sqlite3ColumnsFromExprList(pParse, pTable->pCheck, &pTable->nCol, &pTable->aCol); - if( db->mallocFailed==0 - && pParse->nErr==0 + if( pParse->nErr==0 && pTable->nCol==pSel->pEList->nExpr ){ - sqlite3SelectAddColumnTypeAndCollation(pParse, pTable, pSel); + assert( db->mallocFailed==0 ); + sqlite3SubqueryColumnTypes(pParse, pTable, pSel, SQLITE_AFF_NONE); } - }else if( pSelTab ){ + }else{ /* CREATE VIEW name AS... without an argument list. Construct ** the column names from the SELECT statement that defines the view. */ assert( pTable->aCol==0 ); pTable->nCol = pSelTab->nCol; pTable->aCol = pSelTab->aCol; + pTable->tabFlags |= (pSelTab->tabFlags & COLFLAG_NOINSERT); pSelTab->nCol = 0; pSelTab->aCol = 0; assert( sqlite3SchemaMutexHeld(db, 0, pTable->pSchema) ); - }else{ - pTable->nCol = 0; - nErr++; } + pTable->nNVCol = pTable->nCol; sqlite3DeleteTable(db, pSelTab); sqlite3SelectDelete(db, pSel); - db->lookaside.bDisable--; + EnableLookaside; + pParse->eParseMode = eParseMode; } else { nErr++; } pTable->pSchema->schemaFlags |= DB_UnresetViews; + if( db->mallocFailed ){ + sqlite3DeleteColumnNames(db, pTable); + } #endif /* SQLITE_OMIT_VIEW */ - return nErr; + return nErr + pParse->nErr; +} +SQLITE_PRIVATE int sqlite3ViewGetColumnNames(Parse *pParse, Table *pTable){ + assert( pTable!=0 ); + if( !IsVirtual(pTable) && pTable->nCol>0 ) return 0; + return viewGetColumnNames(pParse, pTable); } #endif /* !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) */ @@ -99060,10 +125159,8 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ if( !DbHasProperty(db, idx, DB_UnresetViews) ) return; for(i=sqliteHashFirst(&db->aDb[idx].pSchema->tblHash); i;i=sqliteHashNext(i)){ Table *pTab = sqliteHashData(i); - if( pTab->pSelect ){ + if( IsView(pTab) ){ sqlite3DeleteColumnNames(db, pTab); - pTab->aCol = 0; - pTab->nCol = 0; } } DbClearProperty(db, idx, DB_UnresetViews); @@ -99082,7 +125179,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ ** on tables and/or indices that are the process of being deleted. ** If you are unlucky, one of those deleted indices or tables might ** have the same rootpage number as the real table or index that is -** being moved. So we cannot stop searching after the first match +** being moved. So we cannot stop searching after the first match ** because the first match might be for one of the deleted indices ** or tables and not the table/index that is actually being moved. ** We must continue looping until all tables and indices with @@ -99090,7 +125187,7 @@ static void sqliteViewResetAll(sqlite3 *db, int idx){ ** in order to be certain that we got the right one. */ #ifndef SQLITE_OMIT_AUTOVACUUM -SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iTo){ +SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, Pgno iFrom, Pgno iTo){ HashElem *pElem; Hash *pHash; Db *pDb; @@ -99116,51 +125213,44 @@ SQLITE_PRIVATE void sqlite3RootPageMoved(sqlite3 *db, int iDb, int iFrom, int iT /* ** Write code to erase the table with root-page iTable from database iDb. -** Also write code to modify the sqlite_master table and internal schema +** Also write code to modify the sqlite_schema table and internal schema ** if a root-page of another table is moved by the btree-layer whilst ** erasing iTable (this can happen with an auto-vacuum database). -*/ +*/ static void destroyRootPage(Parse *pParse, int iTable, int iDb){ Vdbe *v = sqlite3GetVdbe(pParse); int r1 = sqlite3GetTempReg(pParse); - assert( iTable>1 ); + if( iTable<2 ) sqlite3ErrorMsg(pParse, "corrupt schema"); sqlite3VdbeAddOp3(v, OP_Destroy, iTable, r1, iDb); sqlite3MayAbort(pParse); #ifndef SQLITE_OMIT_AUTOVACUUM /* OP_Destroy stores an in integer r1. If this integer ** is non-zero, then it is the root page number of a table moved to - ** location iTable. The following code modifies the sqlite_master table to + ** location iTable. The following code modifies the sqlite_schema table to ** reflect this. ** ** The "#NNN" in the SQL is a special constant that means whatever value ** is in register NNN. See grammar rules associated with the TK_REGISTER ** token for additional information. */ - sqlite3NestedParse(pParse, - "UPDATE %Q.%s SET rootpage=%d WHERE #%d AND rootpage=#%d", - pParse->db->aDb[iDb].zName, SCHEMA_TABLE(iDb), iTable, r1, r1); + sqlite3NestedParse(pParse, + "UPDATE %Q." LEGACY_SCHEMA_TABLE + " SET rootpage=%d WHERE #%d AND rootpage=#%d", + pParse->db->aDb[iDb].zDbSName, iTable, r1, r1); #endif sqlite3ReleaseTempReg(pParse, r1); } /* ** Write VDBE code to erase table pTab and all associated indices on disk. -** Code to update the sqlite_master tables and internal schema definitions +** Code to update the sqlite_schema tables and internal schema definitions ** in case a root-page belonging to another table is moved by the btree layer ** is also added (this can happen with an auto-vacuum database). */ static void destroyTable(Parse *pParse, Table *pTab){ -#ifdef SQLITE_OMIT_AUTOVACUUM - Index *pIdx; - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - destroyRootPage(pParse, pTab->tnum, iDb); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - destroyRootPage(pParse, pIdx->tnum, iDb); - } -#else /* If the database may be auto-vacuum capable (if SQLITE_OMIT_AUTOVACUUM ** is not defined), then it is important to call OP_Destroy on the - ** table and index root-pages in order, starting with the numerically + ** table and index root-pages in order, starting with the numerically ** largest root-page number. This guarantees that none of the root-pages ** to be destroyed is relocated by an earlier OP_Destroy. i.e. if the ** following were coded: @@ -99170,22 +125260,22 @@ static void destroyTable(Parse *pParse, Table *pTab){ ** OP_Destroy 5 0 ** ** and root page 5 happened to be the largest root-page number in the - ** database, then root page 5 would be moved to page 4 by the + ** database, then root page 5 would be moved to page 4 by the ** "OP_Destroy 4 0" opcode. The subsequent "OP_Destroy 5 0" would hit ** a free-list page. */ - int iTab = pTab->tnum; - int iDestroyed = 0; + Pgno iTab = pTab->tnum; + Pgno iDestroyed = 0; while( 1 ){ Index *pIdx; - int iLargest = 0; + Pgno iLargest = 0; if( iDestroyed==0 || iTabpIndex; pIdx; pIdx=pIdx->pNext){ - int iIdx = pIdx->tnum; + Pgno iIdx = pIdx->tnum; assert( pIdx->pSchema==pTab->pSchema ); if( (iDestroyed==0 || (iIdxiLargest ){ iLargest = iIdx; @@ -99200,7 +125290,6 @@ static void destroyTable(Parse *pParse, Table *pTab){ iDestroyed = iLargest; } } -#endif } /* @@ -99214,7 +125303,7 @@ static void sqlite3ClearStatTables( const char *zName /* Name of index or table */ ){ int i; - const char *zDbName = pParse->db->aDb[iDb].zName; + const char *zDbName = pParse->db->aDb[iDb].zDbSName; for(i=1; i<=4; i++){ char zTab[24]; sqlite3_snprintf(sizeof(zTab),zTab,"sqlite_stat%d",i); @@ -99247,12 +125336,12 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in #endif /* Drop all triggers associated with the table being dropped. Code - ** is generated to remove entries from sqlite_master and/or - ** sqlite_temp_master if required. + ** is generated to remove entries from sqlite_schema and/or + ** sqlite_temp_schema if required. */ pTrigger = sqlite3TriggerList(pParse, pTab); while( pTrigger ){ - assert( pTrigger->pSchema==pTab->pSchema || + assert( pTrigger->pSchema==pTab->pSchema || pTrigger->pSchema==db->aDb[1].pSchema ); sqlite3DropTriggerPtr(pParse, pTrigger); pTrigger = pTrigger->pNext; @@ -99267,21 +125356,22 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in if( pTab->tabFlags & TF_Autoincrement ){ sqlite3NestedParse(pParse, "DELETE FROM %Q.sqlite_sequence WHERE name=%Q", - pDb->zName, pTab->zName + pDb->zDbSName, pTab->zName ); } #endif - /* Drop all SQLITE_MASTER table and index entries that refer to the - ** table. The program name loops through the master table and deletes + /* Drop all entries in the schema table that refer to the + ** table. The program name loops through the schema table and deletes ** every row that refers to a table of the same name as the one being ** dropped. Triggers are handled separately because a trigger can be ** created in the temp database that refers to a table in another ** database. */ - sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE tbl_name=%Q and type!='trigger'", - pDb->zName, SCHEMA_TABLE(iDb), pTab->zName); + sqlite3NestedParse(pParse, + "DELETE FROM %Q." LEGACY_SCHEMA_TABLE + " WHERE tbl_name=%Q and type!='trigger'", + pDb->zDbSName, pTab->zName); if( !isView && !IsVirtual(pTab) ){ destroyTable(pParse, pTab); } @@ -99291,12 +125381,48 @@ SQLITE_PRIVATE void sqlite3CodeDropTable(Parse *pParse, Table *pTab, int iDb, in */ if( IsVirtual(pTab) ){ sqlite3VdbeAddOp4(v, OP_VDestroy, iDb, 0, 0, pTab->zName, 0); + sqlite3MayAbort(pParse); } sqlite3VdbeAddOp4(v, OP_DropTable, iDb, 0, 0, pTab->zName, 0); sqlite3ChangeCookie(pParse, iDb); sqliteViewResetAll(db, iDb); } +/* +** Return TRUE if shadow tables should be read-only in the current +** context. +*/ +SQLITE_PRIVATE int sqlite3ReadOnlyShadowTables(sqlite3 *db){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( (db->flags & SQLITE_Defensive)!=0 + && db->pVtabCtx==0 + && db->nVdbeExec==0 + && !sqlite3VtabInSync(db) + ){ + return 1; + } +#endif + return 0; +} + +/* +** Return true if it is not allowed to drop the given table +*/ +static int tableMayNotBeDropped(sqlite3 *db, Table *pTab){ + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 ){ + if( sqlite3StrNICmp(pTab->zName+7, "stat", 4)==0 ) return 0; + if( sqlite3StrNICmp(pTab->zName+7, "parameters", 10)==0 ) return 0; + return 1; + } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + return 1; + } + if( pTab->tabFlags & TF_Eponymous ){ + return 1; + } + return 0; +} + /* ** This routine is called to do the work of a DROP TABLE statement. ** pName is the name of the table to be dropped. @@ -99312,13 +125438,19 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } assert( pParse->nErr==0 ); assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( sqlite3ReadSchema(pParse) ) goto exit_drop_table; if( noErr ) db->suppressErr++; + assert( isView==0 || isView==LOCATE_VIEW ); pTab = sqlite3LocateTableItem(pParse, isView, &pName->a[0]); if( noErr ) db->suppressErr--; if( pTab==0 ){ - if( noErr ) sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + if( noErr ){ + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); + sqlite3ForceNotReadOnly(pParse); + } goto exit_drop_table; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); @@ -99334,7 +125466,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, { int code; const char *zTab = SCHEMA_TABLE(iDb); - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zArg2 = 0; if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb)){ goto exit_drop_table; @@ -99365,8 +125497,7 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, } } #endif - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 - && sqlite3StrNICmp(pTab->zName, "sqlite_stat", 11)!=0 ){ + if( tableMayNotBeDropped(db, pTab) ){ sqlite3ErrorMsg(pParse, "table %s may not be dropped", pTab->zName); goto exit_drop_table; } @@ -99375,24 +125506,26 @@ SQLITE_PRIVATE void sqlite3DropTable(Parse *pParse, SrcList *pName, int isView, /* Ensure DROP TABLE is not used on a view, and DROP VIEW is not used ** on a table. */ - if( isView && pTab->pSelect==0 ){ + if( isView && !IsView(pTab) ){ sqlite3ErrorMsg(pParse, "use DROP TABLE to delete table %s", pTab->zName); goto exit_drop_table; } - if( !isView && pTab->pSelect ){ + if( !isView && IsView(pTab) ){ sqlite3ErrorMsg(pParse, "use DROP VIEW to delete view %s", pTab->zName); goto exit_drop_table; } #endif - /* Generate code to remove the table from the master table + /* Generate code to remove the table from the schema table ** on disk. */ v = sqlite3GetVdbe(pParse); if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); - sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); - sqlite3FkDropTable(pParse, pName, pTab); + if( !isView ){ + sqlite3ClearStatTables(pParse, iDb, "tbl", pTab->zName); + sqlite3FkDropTable(pParse, pName, pTab); + } sqlite3CodeDropTable(pParse, pTab, iDb, isView); } @@ -99428,7 +125561,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( FKey *pFKey = 0; FKey *pNextTo; Table *p = pParse->pNewTable; - int nByte; + i64 nByte; int i; int nCol; char *z; @@ -99441,7 +125574,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( if( pToCol && pToCol->nExpr!=1 ){ sqlite3ErrorMsg(pParse, "foreign key on %s" " should reference only one column of table %T", - p->aCol[iCol].zName, pTo); + p->aCol[iCol].zCnName, pTo); goto fk_end; } nCol = 1; @@ -99456,7 +125589,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( nByte = sizeof(*pFKey) + (nCol-1)*sizeof(pFKey->aCol[0]) + pTo->n + 1; if( pToCol ){ for(i=0; inExpr; i++){ - nByte += sqlite3Strlen30(pToCol->a[i].zName) + 1; + nByte += sqlite3Strlen30(pToCol->a[i].zEName) + 1; } } pFKey = sqlite3DbMallocZero(db, nByte ); @@ -99464,9 +125597,13 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( goto fk_end; } pFKey->pFrom = p; - pFKey->pNextFrom = p->pFKey; + assert( IsOrdinaryTable(p) ); + pFKey->pNextFrom = p->u.tab.pFKey; z = (char*)&pFKey->aCol[nCol]; pFKey->zTo = z; + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, (void*)z, pTo); + } memcpy(z, pTo->z, pTo->n); z[pTo->n] = 0; sqlite3Dequote(z); @@ -99478,24 +125615,30 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( for(i=0; inCol; j++){ - if( sqlite3StrICmp(p->aCol[j].zName, pFromCol->a[i].zName)==0 ){ + if( sqlite3StrICmp(p->aCol[j].zCnName, pFromCol->a[i].zEName)==0 ){ pFKey->aCol[i].iFrom = j; break; } } if( j>=p->nCol ){ - sqlite3ErrorMsg(pParse, - "unknown column \"%s\" in foreign key definition", - pFromCol->a[i].zName); + sqlite3ErrorMsg(pParse, + "unknown column \"%s\" in foreign key definition", + pFromCol->a[i].zEName); goto fk_end; } + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, &pFKey->aCol[i], pFromCol->a[i].zEName); + } } } if( pToCol ){ for(i=0; ia[i].zName); + int n = sqlite3Strlen30(pToCol->a[i].zEName); pFKey->aCol[i].zCol = z; - memcpy(z, pToCol->a[i].zName, n); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, z, pToCol->a[i].zEName); + } + memcpy(z, pToCol->a[i].zEName, n); z[n] = 0; z += n+1; } @@ -99505,7 +125648,7 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( pFKey->aAction[1] = (u8)((flags >> 8 ) & 0xff); /* ON UPDATE action */ assert( sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); - pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, + pNextTo = (FKey *)sqlite3HashInsert(&p->pSchema->fkeyHash, pFKey->zTo, (void *)pFKey ); if( pNextTo==pFKey ){ @@ -99520,7 +125663,8 @@ SQLITE_PRIVATE void sqlite3CreateForeignKey( /* Link the foreign key to the table as the last step. */ - p->pFKey = pFKey; + assert( IsOrdinaryTable(p) ); + p->u.tab.pFKey = pFKey; pFKey = 0; fk_end: @@ -99541,7 +125685,9 @@ SQLITE_PRIVATE void sqlite3DeferForeignKey(Parse *pParse, int isDeferred){ #ifndef SQLITE_OMIT_FOREIGN_KEY Table *pTab; FKey *pFKey; - if( (pTab = pParse->pNewTable)==0 || (pFKey = pTab->pFKey)==0 ) return; + if( (pTab = pParse->pNewTable)==0 ) return; + if( NEVER(!IsOrdinaryTable(pTab)) ) return; + if( (pFKey = pTab->u.tab.pFKey)==0 ) return; assert( isDeferred==0 || isDeferred==1 ); /* EV: R-30323-21917 */ pFKey->isDeferred = (u8)isDeferred; #endif @@ -99565,7 +125711,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ int iSorter; /* Cursor opened by OpenSorter (if in use) */ int addr1; /* Address of top of loop */ int addr2; /* Address to jump to for next iteration */ - int tnum; /* Root page of index */ + Pgno tnum; /* Root page of index */ int iPartIdxLabel; /* Jump to this label to skip a row */ Vdbe *v; /* Generate code into this virtual machine */ KeyInfo *pKey; /* KeyInfo for index */ @@ -99575,7 +125721,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ #ifndef SQLITE_OMIT_AUTHORIZATION if( sqlite3AuthCheck(pParse, SQLITE_REINDEX, pIndex->zName, 0, - db->aDb[iDb].zName ) ){ + db->aDb[iDb].zDbSName ) ){ return; } #endif @@ -99586,12 +125732,12 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ v = sqlite3GetVdbe(pParse); if( v==0 ) return; if( memRootPage>=0 ){ - tnum = memRootPage; + tnum = (Pgno)memRootPage; }else{ tnum = pIndex->tnum; } pKey = sqlite3KeyInfoOfIndex(pParse, pIndex); - assert( pKey!=0 || db->mallocFailed || pParse->nErr ); + assert( pKey!=0 || pParse->nErr ); /* Open the sorter cursor if we are to use one. */ iSorter = pParse->nTab++; @@ -99603,6 +125749,7 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3OpenTable(pParse, iTab, iDb, pTab, OP_OpenRead); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iTab, 0); VdbeCoverage(v); regRecord = sqlite3GetTempReg(pParse); + sqlite3MultiWrite(pParse); sqlite3GenerateIndexKey(pParse,pIndex,iTab,regRecord,0,&iPartIdxLabel,0,0); sqlite3VdbeAddOp2(v, OP_SorterInsert, iSorter, regRecord); @@ -99610,24 +125757,42 @@ static void sqlite3RefillIndex(Parse *pParse, Index *pIndex, int memRootPage){ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); if( memRootPage<0 ) sqlite3VdbeAddOp2(v, OP_Clear, tnum, iDb); - sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, tnum, iDb, + sqlite3VdbeAddOp4(v, OP_OpenWrite, iIdx, (int)tnum, iDb, (char *)pKey, P4_KEYINFO); sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR|((memRootPage>=0)?OPFLAG_P2ISREG:0)); addr1 = sqlite3VdbeAddOp2(v, OP_SorterSort, iSorter, 0); VdbeCoverage(v); if( IsUniqueIndex(pIndex) ){ - int j2 = sqlite3VdbeCurrentAddr(v) + 3; - sqlite3VdbeGoto(v, j2); + int j2 = sqlite3VdbeGoto(v, 1); addr2 = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeVerifyAbortable(v, OE_Abort); sqlite3VdbeAddOp4Int(v, OP_SorterCompare, iSorter, j2, regRecord, pIndex->nKeyCol); VdbeCoverage(v); sqlite3UniqueConstraint(pParse, OE_Abort, pIndex); + sqlite3VdbeJumpHere(v, j2); }else{ + /* Most CREATE INDEX and REINDEX statements that are not UNIQUE can not + ** abort. The exception is if one of the indexed expressions contains a + ** user function that throws an exception when it is evaluated. But the + ** overhead of adding a statement journal to a CREATE INDEX statement is + ** very small (since most of the pages written do not contain content that + ** needs to be restored if the statement aborts), so we call + ** sqlite3MayAbort() for all CREATE INDEX statements. */ + sqlite3MayAbort(pParse); addr2 = sqlite3VdbeCurrentAddr(v); } sqlite3VdbeAddOp3(v, OP_SorterData, iSorter, regRecord, iIdx); - sqlite3VdbeAddOp3(v, OP_Last, iIdx, 0, -1); - sqlite3VdbeAddOp3(v, OP_IdxInsert, iIdx, regRecord, 0); + if( !pIndex->bAscKeyBug ){ + /* This OP_SeekEnd opcode makes index insert for a REINDEX go much + ** faster by avoiding unnecessary seeks. But the optimization does + ** not work for UNIQUE constraint indexes on WITHOUT ROWID tables + ** with DESC primary keys, since those indexes have there keys in + ** a different order from the main table. + ** See ticket: https://www.sqlite.org/src/info/bba7b69f9849b5bf + */ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iIdx); + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdx, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3VdbeAddOp2(v, OP_SorterNext, iSorter, addr2); VdbeCoverage(v); @@ -99674,8 +125839,29 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( } /* -** Create a new index for an SQL table. pName1.pName2 is the name of the index -** and pTblList is the name of the table that is to be indexed. Both will +** If expression list pList contains an expression that was parsed with +** an explicit "NULLS FIRST" or "NULLS LAST" clause, leave an error in +** pParse and return non-zero. Otherwise, return zero. +*/ +SQLITE_PRIVATE int sqlite3HasExplicitNulls(Parse *pParse, ExprList *pList){ + if( pList ){ + int i; + for(i=0; inExpr; i++){ + if( pList->a[i].fg.bNulls ){ + u8 sf = pList->a[i].fg.sortFlags; + sqlite3ErrorMsg(pParse, "unsupported use of NULLS %s", + (sf==0 || sf==3) ? "FIRST" : "LAST" + ); + return 1; + } + } + } + return 0; +} + +/* +** Create a new index for an SQL table. pName1.pName2 is the name of the index +** and pTblList is the name of the table that is to be indexed. Both will ** be NULL for a primary key or an index that is created to satisfy a ** UNIQUE constraint. If pTable and pIndex are NULL, use pParse->pNewTable ** as the table to be indexed. pParse->pNewTable is a table that is @@ -99683,13 +125869,9 @@ SQLITE_PRIVATE Index *sqlite3AllocateIndexObject( ** ** pList is a list of columns to be indexed. pList will be NULL if this ** is a primary key or unique-constraint on the most recent column added -** to the table currently under construction. -** -** If the index is created successfully, return a pointer to the new Index -** structure. This is used by sqlite3AddPrimaryKey() to mark the index -** as the tables primary key (Index.idxType==SQLITE_IDXTYPE_PRIMARYKEY) +** to the table currently under construction. */ -SQLITE_PRIVATE Index *sqlite3CreateIndex( +SQLITE_PRIVATE void sqlite3CreateIndex( Parse *pParse, /* All information about this parse */ Token *pName1, /* First part of index name. May be NULL */ Token *pName2, /* Second part of index name. May be NULL */ @@ -99699,9 +125881,9 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( Token *pStart, /* The CREATE token that begins this statement */ Expr *pPIWhere, /* WHERE clause for partial indices */ int sortOrder, /* Sort order of primary key when pList==NULL */ - int ifNotExist /* Omit error if index already exists */ + int ifNotExist, /* Omit error if index already exists */ + u8 idxType /* The index type */ ){ - Index *pRet = 0; /* Pointer to return */ Table *pTab = 0; /* Table to be indexed */ Index *pIndex = 0; /* The index to be created */ char *zName = 0; /* Name of the index */ @@ -99719,19 +125901,27 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( char *zExtra = 0; /* Extra space after the Index object */ Index *pPk = 0; /* PRIMARY KEY index for WITHOUT ROWID tables */ - if( db->mallocFailed || IN_DECLARE_VTAB || pParse->nErr>0 ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ + goto exit_create_index; + } + assert( db->mallocFailed==0 ); + if( IN_DECLARE_VTAB && idxType!=SQLITE_IDXTYPE_PRIMARYKEY ){ goto exit_create_index; } if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_create_index; } + if( sqlite3HasExplicitNulls(pParse, pList) ){ + goto exit_create_index; + } /* ** Find the table that is to be indexed. Return early if not found. */ if( pTblName!=0 ){ - /* Use the two-part index name to determine the database + /* Use the two-part index name to determine the database ** to search for the table. 'Fix' the table name to this db ** before looking up the table. */ @@ -99743,7 +125933,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( #ifndef SQLITE_OMIT_TEMPDB /* If the index name was unqualified, check if the table ** is a temp table. If so, set the database to 1. Do not do this - ** if initialising a database schema. + ** if initializing a database schema. */ if( !db->init.busy ){ pTab = sqlite3SrcListLookup(pParse, pTblName); @@ -99763,7 +125953,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( assert( db->mallocFailed==0 || pTab==0 ); if( pTab==0 ) goto exit_create_index; if( iDb==1 && db->aDb[iDb].pSchema!=pTab->pSchema ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "cannot create a TEMP index on non-TEMP table \"%s\"", pTab->zName); goto exit_create_index; @@ -99779,18 +125969,18 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pDb = &db->aDb[iDb]; assert( pTab!=0 ); - assert( pParse->nErr==0 ); - if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 + if( sqlite3StrNICmp(pTab->zName, "sqlite_", 7)==0 && db->init.busy==0 + && pTblName!=0 #if SQLITE_USER_AUTHENTICATION && sqlite3UserAuthTable(pTab->zName)==0 #endif - && sqlite3StrNICmp(&pTab->zName[7],"altertab_",9)!=0 ){ + ){ sqlite3ErrorMsg(pParse, "table %s may not be indexed", pTab->zName); goto exit_create_index; } #ifndef SQLITE_OMIT_VIEW - if( pTab->pSelect ){ + if( IsView(pTab) ){ sqlite3ErrorMsg(pParse, "views may not be indexed"); goto exit_create_index; } @@ -99804,10 +125994,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* ** Find the name of the index. Make sure there is not already another - ** index or table with the same name. + ** index or table with the same name. ** ** Exception: If we are reading the names of permanent indices from the - ** sqlite_master table (because some other process changed the schema) and + ** sqlite_schema table (because some other process changed the schema) and ** one of the index names collides with the name of a temporary table or ** index, then we will continue to process this index. ** @@ -99819,24 +126009,27 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( zName = sqlite3NameFromToken(db, pName); if( zName==0 ) goto exit_create_index; assert( pName->z!=0 ); - if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( SQLITE_OK!=sqlite3CheckObjectName(pParse, zName,"index",pTab->zName) ){ goto exit_create_index; } - if( !db->init.busy ){ - if( sqlite3FindTable(db, zName, 0)!=0 ){ - sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + if( !IN_RENAME_OBJECT ){ + if( !db->init.busy ){ + if( sqlite3FindTable(db, zName, pDb->zDbSName)!=0 ){ + sqlite3ErrorMsg(pParse, "there is already a table named %s", zName); + goto exit_create_index; + } + } + if( sqlite3FindIndex(db, zName, pDb->zDbSName)!=0 ){ + if( !ifNotExist ){ + sqlite3ErrorMsg(pParse, "index %s already exists", zName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + sqlite3ForceNotReadOnly(pParse); + } goto exit_create_index; } } - if( sqlite3FindIndex(db, zName, pDb->zName)!=0 ){ - if( !ifNotExist ){ - sqlite3ErrorMsg(pParse, "index %s already exists", zName); - }else{ - assert( !db->init.busy ); - sqlite3CodeVerifySchema(pParse, iDb); - } - goto exit_create_index; - } }else{ int n; Index *pLoop; @@ -99845,13 +126038,20 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( zName==0 ){ goto exit_create_index; } + + /* Automatic index names generated from within sqlite3_declare_vtab() + ** must have names that are distinct from normal automatic index names. + ** The following statement converts "sqlite3_autoindex..." into + ** "sqlite3_butoindex..." in order to make the names distinct. + ** The "vtab_err.test" test demonstrates the need of this statement. */ + if( IN_SPECIAL_PARSE ) zName[7]++; } /* Check for authorization to create an index. */ #ifndef SQLITE_OMIT_AUTHORIZATION - { - const char *zDb = pDb->zName; + if( !IN_RENAME_OBJECT ){ + const char *zDb = pDb->zDbSName; if( sqlite3AuthCheck(pParse, SQLITE_INSERT, SCHEMA_TABLE(iDb), 0, zDb) ){ goto exit_create_index; } @@ -99869,14 +126069,17 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( */ if( pList==0 ){ Token prevCol; - sqlite3TokenInit(&prevCol, pTab->aCol[pTab->nCol-1].zName); + Column *pCol = &pTab->aCol[pTab->nCol-1]; + pCol->colFlags |= COLFLAG_UNIQUE; + sqlite3TokenInit(&prevCol, pCol->zCnName); pList = sqlite3ExprListAppend(pParse, 0, sqlite3ExprAlloc(db, TK_ID, &prevCol, 0)); if( pList==0 ) goto exit_create_index; assert( pList->nExpr==1 ); - sqlite3ExprListSetSortOrder(pList, sortOrder); + sqlite3ExprListSetSortOrder(pList, sortOrder, SQLITE_SO_UNDEFINED); }else{ sqlite3ExprListCheckLength(pParse, pList, "index"); + if( pParse->nErr ) goto exit_create_index; } /* Figure out how many bytes of space are required to store explicitly @@ -99886,15 +126089,17 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( Expr *pExpr = pList->a[i].pExpr; assert( pExpr!=0 ); if( pExpr->op==TK_COLLATE ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); nExtra += (1 + sqlite3Strlen30(pExpr->u.zToken)); } } - /* - ** Allocate the index structure. + /* + ** Allocate the index structure. */ nName = sqlite3Strlen30(zName); nExtraCol = pPk ? pPk->nKeyCol : 1; + assert( pList->nExpr + nExtraCol <= 32767 /* Fits in i16 */ ); pIndex = sqlite3AllocateIndexObject(db, pList->nExpr + nExtraCol, nName + nExtra + 1, &zExtra); if( db->mallocFailed ){ @@ -99908,7 +126113,7 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( pIndex->pTable = pTab; pIndex->onError = (u8)onError; pIndex->uniqNotNull = onError!=OE_None; - pIndex->idxType = pName ? SQLITE_IDXTYPE_APPDEF : SQLITE_IDXTYPE_UNIQUE; + pIndex->idxType = idxType; pIndex->pSchema = db->aDb[iDb].pSchema; pIndex->nKeyCol = pList->nExpr; if( pPIWhere ){ @@ -99935,7 +126140,12 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( ** TODO: Issue a warning if the table primary key is used as part of the ** index key. */ - for(i=0, pListItem=pList->a; inExpr; i++, pListItem++){ + pListItem = pList->a; + if( IN_RENAME_OBJECT ){ + pIndex->aColExpr = pList; + pList = 0; + } + for(i=0; inKeyCol; i++, pListItem++){ Expr *pCExpr; /* The i-th index expression */ int requestedSortOrder; /* ASC or DESC on the i-th expression */ const char *zColl; /* Collation sequence name */ @@ -99951,29 +126161,33 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( goto exit_create_index; } if( pIndex->aColExpr==0 ){ - ExprList *pCopy = sqlite3ExprListDup(db, pList, 0); - pIndex->aColExpr = pCopy; - if( !db->mallocFailed ){ - assert( pCopy!=0 ); - pListItem = &pCopy->a[i]; - } + pIndex->aColExpr = pList; + pList = 0; } j = XN_EXPR; pIndex->aiColumn[i] = XN_EXPR; pIndex->uniqNotNull = 0; + pIndex->bHasExpr = 1; }else{ j = pCExpr->iColumn; assert( j<=0x7fff ); if( j<0 ){ j = pTab->iPKey; - }else if( pTab->aCol[j].notNull==0 ){ - pIndex->uniqNotNull = 0; + }else{ + if( pTab->aCol[j].notNull==0 ){ + pIndex->uniqNotNull = 0; + } + if( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ){ + pIndex->bHasVCol = 1; + pIndex->bHasExpr = 1; + } } pIndex->aiColumn[i] = (i16)j; } zColl = 0; if( pListItem->pExpr->op==TK_COLLATE ){ int nColl; + assert( !ExprHasProperty(pListItem->pExpr, EP_IntValue) ); zColl = pListItem->pExpr->u.zToken; nColl = sqlite3Strlen30(zColl) + 1; assert( nExtra>=nColl ); @@ -99982,14 +126196,14 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( zExtra += nColl; nExtra -= nColl; }else if( j>=0 ){ - zColl = pTab->aCol[j].zColl; + zColl = sqlite3ColumnColl(&pTab->aCol[j]); } if( !zColl ) zColl = sqlite3StrBINARY; if( !db->init.busy && !sqlite3LocateCollSeq(pParse, zColl) ){ goto exit_create_index; } pIndex->azColl[i] = zColl; - requestedSortOrder = pListItem->sortOrder & sortOrderMask; + requestedSortOrder = pListItem->fg.sortFlags & sortOrderMask; pIndex->aSortOrder[i] = (u8)requestedSortOrder; } @@ -100001,9 +126215,10 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( for(j=0; jnKeyCol; j++){ int x = pPk->aiColumn[j]; assert( x>=0 ); - if( hasColumn(pIndex->aiColumn, pIndex->nKeyCol, x) ){ - pIndex->nColumn--; + if( isDupColumn(pIndex, pIndex->nKeyCol, pPk, j) ){ + pIndex->nColumn--; }else{ + testcase( hasColumn(pIndex->aiColumn,pIndex->nKeyCol,x) ); pIndex->aiColumn[i] = x; pIndex->azColl[i] = pPk->azColl[j]; pIndex->aSortOrder[i] = pPk->aSortOrder[j]; @@ -100020,13 +126235,14 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( /* If this index contains every column of its table, then mark ** it as a covering index */ - assert( HasRowid(pTab) - || pTab->iPKey<0 || sqlite3ColumnOfIndex(pIndex, pTab->iPKey)>=0 ); + assert( HasRowid(pTab) + || pTab->iPKey<0 || sqlite3TableColumnToIndex(pIndex, pTab->iPKey)>=0 ); + recomputeColumnsNotIndexed(pIndex); if( pTblName!=0 && pIndex->nColumn>=pTab->nCol ){ pIndex->isCovering = 1; for(j=0; jnCol; j++){ if( j==pTab->iPKey ) continue; - if( sqlite3ColumnOfIndex(pIndex,j)>=0 ) continue; + if( sqlite3TableColumnToIndex(pIndex,j)>=0 ) continue; pIndex->isCovering = 0; break; } @@ -100075,149 +126291,179 @@ SQLITE_PRIVATE Index *sqlite3CreateIndex( if( pIdx->onError!=pIndex->onError ){ /* This constraint creates the same index as a previous ** constraint specified somewhere in the CREATE TABLE statement. - ** However the ON CONFLICT clauses are different. If both this + ** However the ON CONFLICT clauses are different. If both this ** constraint and the previous equivalent constraint have explicit ** ON CONFLICT clauses this is an error. Otherwise, use the ** explicitly specified behavior for the index. */ if( !(pIdx->onError==OE_Default || pIndex->onError==OE_Default) ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "conflicting ON CONFLICT clauses specified", 0); } if( pIdx->onError==OE_Default ){ pIdx->onError = pIndex->onError; } } - pRet = pIdx; + if( idxType==SQLITE_IDXTYPE_PRIMARYKEY ) pIdx->idxType = idxType; + if( IN_RENAME_OBJECT ){ + pIndex->pNext = pParse->pNewIndex; + pParse->pNewIndex = pIndex; + pIndex = 0; + } goto exit_create_index; } } } - /* Link the new Index structure to its table and to the other - ** in-memory database structures. - */ - assert( pParse->nErr==0 ); - if( db->init.busy ){ - Index *p; - assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); - p = sqlite3HashInsert(&pIndex->pSchema->idxHash, - pIndex->zName, pIndex); - if( p ){ - assert( p==pIndex ); /* Malloc must have failed */ - sqlite3OomFault(db); - goto exit_create_index; - } - db->flags |= SQLITE_InternChanges; - if( pTblName!=0 ){ - pIndex->tnum = db->init.newTnum; - } - } + if( !IN_RENAME_OBJECT ){ - /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the - ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then - ** emit code to allocate the index rootpage on disk and make an entry for - ** the index in the sqlite_master table and populate the index with - ** content. But, do not do this if we are simply reading the sqlite_master - ** table to parse the schema, or if this index is the PRIMARY KEY index - ** of a WITHOUT ROWID table. - ** - ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY - ** or UNIQUE index in a CREATE TABLE statement. Since the table - ** has just been created, it contains no data and the index initialization - ** step can be skipped. - */ - else if( HasRowid(pTab) || pTblName!=0 ){ - Vdbe *v; - char *zStmt; - int iMem = ++pParse->nMem; - - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto exit_create_index; - - sqlite3BeginWriteOperation(pParse, 1, iDb); - - /* Create the rootpage for the index using CreateIndex. But before - ** doing so, code a Noop instruction and store its address in - ** Index.tnum. This is required in case this index is actually a - ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In - ** that case the convertToWithoutRowidTable() routine will replace - ** the Noop with a Goto to jump over the VDBE code generated below. */ - pIndex->tnum = sqlite3VdbeAddOp0(v, OP_Noop); - sqlite3VdbeAddOp2(v, OP_CreateIndex, iDb, iMem); - - /* Gather the complete text of the CREATE INDEX statement into - ** the zStmt variable + /* Link the new Index structure to its table and to the other + ** in-memory database structures. */ - if( pStart ){ - int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; - if( pName->z[n-1]==';' ) n--; - /* A named index with an explicit CREATE INDEX statement */ - zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", - onError==OE_None ? "" : " UNIQUE", n, pName->z); - }else{ - /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ - /* zStmt = sqlite3MPrintf(""); */ - zStmt = 0; - } - - /* Add an entry in sqlite_master for this index - */ - sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('index',%Q,%Q,#%d,%Q);", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), - pIndex->zName, - pTab->zName, - iMem, - zStmt - ); - sqlite3DbFree(db, zStmt); - - /* Fill the index with data and reparse the schema. Code an OP_Expire - ** to invalidate all pre-compiled statements. - */ - if( pTblName ){ - sqlite3RefillIndex(pParse, pIndex, iMem); - sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName)); - sqlite3VdbeAddOp1(v, OP_Expire, 0); - } - - sqlite3VdbeJumpHere(v, pIndex->tnum); - } - - /* When adding an index to the list of indices for a table, make - ** sure all indices labeled OE_Replace come after all those labeled - ** OE_Ignore. This is necessary for the correct constraint check - ** processing (in sqlite3GenerateConstraintChecks()) as part of - ** UPDATE and INSERT statements. - */ - if( db->init.busy || pTblName==0 ){ - if( onError!=OE_Replace || pTab->pIndex==0 - || pTab->pIndex->onError==OE_Replace){ - pIndex->pNext = pTab->pIndex; - pTab->pIndex = pIndex; - }else{ - Index *pOther = pTab->pIndex; - while( pOther->pNext && pOther->pNext->onError!=OE_Replace ){ - pOther = pOther->pNext; + assert( pParse->nErr==0 ); + if( db->init.busy ){ + Index *p; + assert( !IN_SPECIAL_PARSE ); + assert( sqlite3SchemaMutexHeld(db, 0, pIndex->pSchema) ); + if( pTblName!=0 ){ + pIndex->tnum = db->init.newTnum; + if( sqlite3IndexHasDuplicateRootPage(pIndex) ){ + sqlite3ErrorMsg(pParse, "invalid rootpage"); + pParse->rc = SQLITE_CORRUPT_BKPT; + goto exit_create_index; + } } - pIndex->pNext = pOther->pNext; - pOther->pNext = pIndex; + p = sqlite3HashInsert(&pIndex->pSchema->idxHash, + pIndex->zName, pIndex); + if( p ){ + assert( p==pIndex ); /* Malloc must have failed */ + sqlite3OomFault(db); + goto exit_create_index; + } + db->mDbFlags |= DBFLAG_SchemaChange; } - pRet = pIndex; + + /* If this is the initial CREATE INDEX statement (or CREATE TABLE if the + ** index is an implied index for a UNIQUE or PRIMARY KEY constraint) then + ** emit code to allocate the index rootpage on disk and make an entry for + ** the index in the sqlite_schema table and populate the index with + ** content. But, do not do this if we are simply reading the sqlite_schema + ** table to parse the schema, or if this index is the PRIMARY KEY index + ** of a WITHOUT ROWID table. + ** + ** If pTblName==0 it means this index is generated as an implied PRIMARY KEY + ** or UNIQUE index in a CREATE TABLE statement. Since the table + ** has just been created, it contains no data and the index initialization + ** step can be skipped. + */ + else if( HasRowid(pTab) || pTblName!=0 ){ + Vdbe *v; + char *zStmt; + int iMem = ++pParse->nMem; + + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto exit_create_index; + + sqlite3BeginWriteOperation(pParse, 1, iDb); + + /* Create the rootpage for the index using CreateIndex. But before + ** doing so, code a Noop instruction and store its address in + ** Index.tnum. This is required in case this index is actually a + ** PRIMARY KEY and the table is actually a WITHOUT ROWID table. In + ** that case the convertToWithoutRowidTable() routine will replace + ** the Noop with a Goto to jump over the VDBE code generated below. */ + pIndex->tnum = (Pgno)sqlite3VdbeAddOp0(v, OP_Noop); + sqlite3VdbeAddOp3(v, OP_CreateBtree, iDb, iMem, BTREE_BLOBKEY); + + /* Gather the complete text of the CREATE INDEX statement into + ** the zStmt variable + */ + assert( pName!=0 || pStart==0 ); + if( pStart ){ + int n = (int)(pParse->sLastToken.z - pName->z) + pParse->sLastToken.n; + if( pName->z[n-1]==';' ) n--; + /* A named index with an explicit CREATE INDEX statement */ + zStmt = sqlite3MPrintf(db, "CREATE%s INDEX %.*s", + onError==OE_None ? "" : " UNIQUE", n, pName->z); + }else{ + /* An automatic index created by a PRIMARY KEY or UNIQUE constraint */ + /* zStmt = sqlite3MPrintf(""); */ + zStmt = 0; + } + + /* Add an entry in sqlite_schema for this index + */ + sqlite3NestedParse(pParse, + "INSERT INTO %Q." LEGACY_SCHEMA_TABLE " VALUES('index',%Q,%Q,#%d,%Q);", + db->aDb[iDb].zDbSName, + pIndex->zName, + pTab->zName, + iMem, + zStmt + ); + sqlite3DbFree(db, zStmt); + + /* Fill the index with data and reparse the schema. Code an OP_Expire + ** to invalidate all pre-compiled statements. + */ + if( pTblName ){ + sqlite3RefillIndex(pParse, pIndex, iMem); + sqlite3ChangeCookie(pParse, iDb); + sqlite3VdbeAddParseSchemaOp(v, iDb, + sqlite3MPrintf(db, "name='%q' AND type='index'", pIndex->zName), 0); + sqlite3VdbeAddOp2(v, OP_Expire, 0, 1); + } + + sqlite3VdbeJumpHere(v, (int)pIndex->tnum); + } + } + if( db->init.busy || pTblName==0 ){ + pIndex->pNext = pTab->pIndex; + pTab->pIndex = pIndex; + pIndex = 0; + } + else if( IN_RENAME_OBJECT ){ + assert( pParse->pNewIndex==0 ); + pParse->pNewIndex = pIndex; pIndex = 0; } /* Clean up before exiting */ exit_create_index: - if( pIndex ) freeIndex(db, pIndex); + if( pIndex ) sqlite3FreeIndex(db, pIndex); + if( pTab ){ + /* Ensure all REPLACE indexes on pTab are at the end of the pIndex list. + ** The list was already ordered when this routine was entered, so at this + ** point at most a single index (the newly added index) will be out of + ** order. So we have to reorder at most one index. */ + Index **ppFrom; + Index *pThis; + for(ppFrom=&pTab->pIndex; (pThis = *ppFrom)!=0; ppFrom=&pThis->pNext){ + Index *pNext; + if( pThis->onError!=OE_Replace ) continue; + while( (pNext = pThis->pNext)!=0 && pNext->onError!=OE_Replace ){ + *ppFrom = pNext; + pThis->pNext = pNext->pNext; + pNext->pNext = pThis; + ppFrom = &pNext->pNext; + } + break; + } +#ifdef SQLITE_DEBUG + /* Verify that all REPLACE indexes really are now at the end + ** of the index list. In other words, no other index type ever + ** comes after a REPLACE index on the list. */ + for(pThis = pTab->pIndex; pThis; pThis=pThis->pNext){ + assert( pThis->onError!=OE_Replace + || pThis->pNext==0 + || pThis->pNext->onError==OE_Replace ); + } +#endif + } sqlite3ExprDelete(db, pPIWhere); sqlite3ExprListDelete(db, pList); sqlite3SrcListDelete(db, pTblName); sqlite3DbFree(db, zName); - return pRet; } /* @@ -100239,17 +126485,33 @@ exit_create_index: ** are based on typical values found in actual indices. */ SQLITE_PRIVATE void sqlite3DefaultRowEst(Index *pIdx){ - /* 10, 9, 8, 7, 6 */ - LogEst aVal[] = { 33, 32, 30, 28, 26 }; + /* 10, 9, 8, 7, 6 */ + static const LogEst aVal[] = { 33, 32, 30, 28, 26 }; LogEst *a = pIdx->aiRowLogEst; + LogEst x; int nCopy = MIN(ArraySize(aVal), pIdx->nKeyCol); int i; - /* Set the first entry (number of rows in the index) to the estimated - ** number of rows in the table. Or 10, if the estimated number of rows - ** in the table is less than that. */ - a[0] = pIdx->pTable->nRowLogEst; - if( a[0]<33 ) a[0] = 33; assert( 33==sqlite3LogEst(10) ); + /* Indexes with default row estimates should not have stat1 data */ + assert( !pIdx->hasStat1 ); + + /* Set the first entry (number of rows in the index) to the estimated + ** number of rows in the table, or half the number of rows in the table + ** for a partial index. + ** + ** 2020-05-27: If some of the stat data is coming from the sqlite_stat1 + ** table but other parts we are having to guess at, then do not let the + ** estimated number of rows in the table be less than 1000 (LogEst 99). + ** Failure to do this can cause the indexes for which we do not have + ** stat1 data to be ignored by the query planner. + */ + x = pIdx->pTable->nRowLogEst; + assert( 99==sqlite3LogEst(1000) ); + if( x<99 ){ + pIdx->pTable->nRowLogEst = x = 99; + } + if( pIdx->pPartIdxWhere!=0 ){ x -= 10; assert( 10==sqlite3LogEst(2) ); } + a[0] = x; /* Estimate that a[1] is 10, a[2] is 9, a[3] is 8, a[4] is 7, a[5] is ** 6 and each subsequent value (if any) is 5. */ @@ -100272,20 +126534,23 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists sqlite3 *db = pParse->db; int iDb; - assert( pParse->nErr==0 ); /* Never called with prior errors */ if( db->mallocFailed ){ goto exit_drop_index; } + assert( pParse->nErr==0 ); /* Never called with prior non-OOM errors */ assert( pName->nSrc==1 ); + assert( pName->a[0].fg.fixedSchema==0 ); + assert( pName->a[0].fg.isSubquery==0 ); if( SQLITE_OK!=sqlite3ReadSchema(pParse) ){ goto exit_drop_index; } - pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].zDatabase); + pIndex = sqlite3FindIndex(db, pName->a[0].zName, pName->a[0].u4.zDatabase); if( pIndex==0 ){ if( !ifExists ){ - sqlite3ErrorMsg(pParse, "no such index: %S", pName, 0); + sqlite3ErrorMsg(pParse, "no such index: %S", pName->a); }else{ - sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].zDatabase); + sqlite3CodeVerifyNamedSchema(pParse, pName->a[0].u4.zDatabase); + sqlite3ForceNotReadOnly(pParse); } pParse->checkSchema = 1; goto exit_drop_index; @@ -100300,25 +126565,25 @@ SQLITE_PRIVATE void sqlite3DropIndex(Parse *pParse, SrcList *pName, int ifExists { int code = SQLITE_DROP_INDEX; Table *pTab = pIndex->pTable; - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); if( sqlite3AuthCheck(pParse, SQLITE_DELETE, zTab, 0, zDb) ){ goto exit_drop_index; } - if( !OMIT_TEMPDB && iDb ) code = SQLITE_DROP_TEMP_INDEX; + if( !OMIT_TEMPDB && iDb==1 ) code = SQLITE_DROP_TEMP_INDEX; if( sqlite3AuthCheck(pParse, code, pIndex->zName, pTab->zName, zDb) ){ goto exit_drop_index; } } #endif - /* Generate code to remove the index and from the master table */ + /* Generate code to remove the index and from the schema table */ v = sqlite3GetVdbe(pParse); if( v ){ sqlite3BeginWriteOperation(pParse, 1, iDb); sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q AND type='index'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pIndex->zName + "DELETE FROM %Q." LEGACY_SCHEMA_TABLE " WHERE name=%Q AND type='index'", + db->aDb[iDb].zDbSName, pIndex->zName ); sqlite3ClearStatTables(pParse, iDb, "idx", pIndex->zName); sqlite3ChangeCookie(pParse, iDb); @@ -100355,9 +126620,9 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( int *pIdx /* Write the index of a new slot here */ ){ char *z; - int n = *pnEntry; + sqlite3_int64 n = *pIdx = *pnEntry; if( (n & (n-1))==0 ){ - int sz = (n==0) ? 1 : 2*n; + sqlite3_int64 sz = (n==0) ? 1 : 2*n; void *pNew = sqlite3DbRealloc(db, pArray, sz*szEntry); if( pNew==0 ){ *pIdx = -1; @@ -100367,7 +126632,6 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( } z = (char*)pArray; memset(&z[n * szEntry], 0, szEntry); - *pIdx = n; ++*pnEntry; return pArray; } @@ -100378,24 +126642,27 @@ SQLITE_PRIVATE void *sqlite3ArrayAllocate( ** ** A new IdList is returned, or NULL if malloc() fails. */ -SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pToken){ +SQLITE_PRIVATE IdList *sqlite3IdListAppend(Parse *pParse, IdList *pList, Token *pToken){ + sqlite3 *db = pParse->db; int i; if( pList==0 ){ pList = sqlite3DbMallocZero(db, sizeof(IdList) ); if( pList==0 ) return 0; + }else{ + IdList *pNew; + pNew = sqlite3DbRealloc(db, pList, + sizeof(IdList) + pList->nId*sizeof(pList->a)); + if( pNew==0 ){ + sqlite3IdListDelete(db, pList); + return 0; + } + pList = pNew; } - pList->a = sqlite3ArrayAllocate( - db, - pList->a, - sizeof(pList->a[0]), - &pList->nId, - &i - ); - if( i<0 ){ - sqlite3IdListDelete(db, pList); - return 0; - } + i = pList->nId++; pList->a[i].zName = sqlite3NameFromToken(db, pToken); + if( IN_RENAME_OBJECT && pList->a[i].zName ){ + sqlite3RenameTokenMap(pParse, (void*)pList->a[i].zName, pToken); + } return pList; } @@ -100404,12 +126671,13 @@ SQLITE_PRIVATE IdList *sqlite3IdListAppend(sqlite3 *db, IdList *pList, Token *pT */ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ int i; + assert( db!=0 ); if( pList==0 ) return; + assert( pList->eU4!=EU4_EXPR ); /* EU4_EXPR mode is not currently used */ for(i=0; inId; i++){ sqlite3DbFree(db, pList->a[i].zName); } - sqlite3DbFree(db, pList->a); - sqlite3DbFree(db, pList); + sqlite3DbNNFreeNN(db, pList); } /* @@ -100418,13 +126686,25 @@ SQLITE_PRIVATE void sqlite3IdListDelete(sqlite3 *db, IdList *pList){ */ SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ int i; - if( pList==0 ) return -1; + assert( pList!=0 ); for(i=0; inId; i++){ if( sqlite3StrICmp(pList->a[i].zName, zName)==0 ) return i; } return -1; } +/* +** Maximum size of a SrcList object. +** The SrcList object is used to represent the FROM clause of a +** SELECT statement, and the query planner cannot deal with more +** than 64 tables in a join. So any value larger than 64 here +** is sufficient for most uses. Smaller values, like say 10, are +** appropriate for small and memory-limited applications. +*/ +#ifndef SQLITE_MAX_SRCLIST +# define SQLITE_MAX_SRCLIST 200 +#endif + /* ** Expand the space allocated for the given SrcList object by ** creating nExtra new slots beginning at iStart. iStart is zero based. @@ -100441,11 +126721,12 @@ SQLITE_PRIVATE int sqlite3IdListIndex(IdList *pList, const char *zName){ ** the iStart value would be 0. The result then would ** be: nil, nil, nil, A, B. ** -** If a memory allocation fails the SrcList is unchanged. The -** db->mallocFailed flag will be set to true. +** If a memory allocation fails or the SrcList becomes too large, leave +** the original SrcList unchanged, return NULL, and leave an error message +** in pParse. */ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( - sqlite3 *db, /* Database connection to notify of OOM errors */ + Parse *pParse, /* Parsing context into which errors are reported */ SrcList *pSrc, /* The SrcList to be enlarged */ int nExtra, /* Number of new slots to add to pSrc->a[] */ int iStart /* Index in pSrc->a[] of first new slot */ @@ -100461,17 +126742,23 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( /* Allocate additional space if needed */ if( (u32)pSrc->nSrc+nExtra>pSrc->nAlloc ){ SrcList *pNew; - int nAlloc = pSrc->nSrc+nExtra; - int nGot; + sqlite3_int64 nAlloc = 2*(sqlite3_int64)pSrc->nSrc+nExtra; + sqlite3 *db = pParse->db; + + if( pSrc->nSrc+nExtra>=SQLITE_MAX_SRCLIST ){ + sqlite3ErrorMsg(pParse, "too many FROM clause terms, max: %d", + SQLITE_MAX_SRCLIST); + return 0; + } + if( nAlloc>SQLITE_MAX_SRCLIST ) nAlloc = SQLITE_MAX_SRCLIST; pNew = sqlite3DbRealloc(db, pSrc, sizeof(*pSrc) + (nAlloc-1)*sizeof(pSrc->a[0]) ); if( pNew==0 ){ assert( db->mallocFailed ); - return pSrc; + return 0; } pSrc = pNew; - nGot = (sqlite3DbMallocSize(db, pNew) - sizeof(*pSrc))/sizeof(pSrc->a[0])+1; - pSrc->nAlloc = nGot; + pSrc->nAlloc = nAlloc; } /* Move existing slots that come after the newly inserted slots @@ -100496,7 +126783,8 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( ** Append a new table name to the given SrcList. Create a new SrcList if ** need be. A new entry is created in the SrcList even if pTable is NULL. ** -** A SrcList is returned, or NULL if there is an OOM error. The returned +** A SrcList is returned, or NULL if there is an OOM error or if the +** SrcList grows to large. The returned ** SrcList might be the same as the SrcList that was input or it might be ** a new one. If an OOM error does occurs, then the prior value of pList ** that is input to this routine is automatically freed. @@ -100505,7 +126793,7 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( ** database name prefix. Like this: "database.table". The pDatabase ** points to the table name and the pTable points to the database name. ** The SrcList.a[].zName field is filled with the table name which might -** come from pTable (if pDatabase is NULL) or from pDatabase. +** come from pTable (if pDatabase is NULL) or from pDatabase. ** SrcList.a[].zDatabase is filled with the database name from pTable, ** or with NULL if no database is specified. ** @@ -100527,36 +126815,46 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListEnlarge( ** before being added to the SrcList. */ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( - sqlite3 *db, /* Connection to notify of malloc failures */ + Parse *pParse, /* Parsing context, in which errors are reported */ SrcList *pList, /* Append to this SrcList. NULL creates a new SrcList */ Token *pTable, /* Table to append */ Token *pDatabase /* Database of the table */ ){ - struct SrcList_item *pItem; + SrcItem *pItem; + sqlite3 *db; assert( pDatabase==0 || pTable!=0 ); /* Cannot have C without B */ - assert( db!=0 ); + assert( pParse!=0 ); + assert( pParse->db!=0 ); + db = pParse->db; if( pList==0 ){ - pList = sqlite3DbMallocRawNN(db, sizeof(SrcList) ); + pList = sqlite3DbMallocRawNN(pParse->db, sizeof(SrcList) ); if( pList==0 ) return 0; pList->nAlloc = 1; - pList->nSrc = 0; - } - pList = sqlite3SrcListEnlarge(db, pList, 1, pList->nSrc); - if( db->mallocFailed ){ - sqlite3SrcListDelete(db, pList); - return 0; + pList->nSrc = 1; + memset(&pList->a[0], 0, sizeof(pList->a[0])); + pList->a[0].iCursor = -1; + }else{ + SrcList *pNew = sqlite3SrcListEnlarge(pParse, pList, 1, pList->nSrc); + if( pNew==0 ){ + sqlite3SrcListDelete(db, pList); + return 0; + }else{ + pList = pNew; + } } pItem = &pList->a[pList->nSrc-1]; if( pDatabase && pDatabase->z==0 ){ pDatabase = 0; } + assert( pItem->fg.fixedSchema==0 ); + assert( pItem->fg.isSubquery==0 ); if( pDatabase ){ - Token *pTemp = pDatabase; - pDatabase = pTable; - pTable = pTemp; + pItem->zName = sqlite3NameFromToken(db, pDatabase); + pItem->u4.zDatabase = sqlite3NameFromToken(db, pTable); + }else{ + pItem->zName = sqlite3NameFromToken(db, pTable); + pItem->u4.zDatabase = 0; } - pItem->zName = sqlite3NameFromToken(db, pTable); - pItem->zDatabase = sqlite3NameFromToken(db, pDatabase); return pList; } @@ -100565,40 +126863,130 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppend( */ SQLITE_PRIVATE void sqlite3SrcListAssignCursors(Parse *pParse, SrcList *pList){ int i; - struct SrcList_item *pItem; - assert(pList || pParse->db->mallocFailed ); - if( pList ){ + SrcItem *pItem; + assert( pList || pParse->db->mallocFailed ); + if( ALWAYS(pList) ){ for(i=0, pItem=pList->a; inSrc; i++, pItem++){ - if( pItem->iCursor>=0 ) break; + if( pItem->iCursor>=0 ) continue; pItem->iCursor = pParse->nTab++; - if( pItem->pSelect ){ - sqlite3SrcListAssignCursors(pParse, pItem->pSelect->pSrc); + if( pItem->fg.isSubquery ){ + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + assert( pItem->u4.pSubq->pSelect->pSrc!=0 ); + sqlite3SrcListAssignCursors(pParse, pItem->u4.pSubq->pSelect->pSrc); } } } } +/* +** Delete a Subquery object and its substructure. +*/ +SQLITE_PRIVATE void sqlite3SubqueryDelete(sqlite3 *db, Subquery *pSubq){ + assert( pSubq!=0 && pSubq->pSelect!=0 ); + sqlite3SelectDelete(db, pSubq->pSelect); + sqlite3DbFree(db, pSubq); +} + +/* +** Remove a Subquery from a SrcItem. Return the associated Select object. +** The returned Select becomes the responsibility of the caller. +*/ +SQLITE_PRIVATE Select *sqlite3SubqueryDetach(sqlite3 *db, SrcItem *pItem){ + Select *pSel; + assert( pItem!=0 ); + assert( pItem->fg.isSubquery ); + pSel = pItem->u4.pSubq->pSelect; + sqlite3DbFree(db, pItem->u4.pSubq); + pItem->u4.pSubq = 0; + pItem->fg.isSubquery = 0; + return pSel; +} + /* ** Delete an entire SrcList including all its substructure. */ SQLITE_PRIVATE void sqlite3SrcListDelete(sqlite3 *db, SrcList *pList){ int i; - struct SrcList_item *pItem; + SrcItem *pItem; + assert( db!=0 ); if( pList==0 ) return; for(pItem=pList->a, i=0; inSrc; i++, pItem++){ - sqlite3DbFree(db, pItem->zDatabase); - sqlite3DbFree(db, pItem->zName); - sqlite3DbFree(db, pItem->zAlias); + + /* Check invariants on SrcItem */ + assert( !pItem->fg.isIndexedBy || !pItem->fg.isTabFunc ); + assert( !pItem->fg.isCte || !pItem->fg.isIndexedBy ); + assert( !pItem->fg.fixedSchema || !pItem->fg.isSubquery ); + assert( !pItem->fg.isSubquery || (pItem->u4.pSubq!=0 && + pItem->u4.pSubq->pSelect!=0) ); + + if( pItem->zName ) sqlite3DbNNFreeNN(db, pItem->zName); + if( pItem->zAlias ) sqlite3DbNNFreeNN(db, pItem->zAlias); + if( pItem->fg.isSubquery ){ + sqlite3SubqueryDelete(db, pItem->u4.pSubq); + }else if( pItem->fg.fixedSchema==0 && pItem->u4.zDatabase!=0 ){ + sqlite3DbNNFreeNN(db, pItem->u4.zDatabase); + } if( pItem->fg.isIndexedBy ) sqlite3DbFree(db, pItem->u1.zIndexedBy); if( pItem->fg.isTabFunc ) sqlite3ExprListDelete(db, pItem->u1.pFuncArg); - sqlite3DeleteTable(db, pItem->pTab); - sqlite3SelectDelete(db, pItem->pSelect); - sqlite3ExprDelete(db, pItem->pOn); - sqlite3IdListDelete(db, pItem->pUsing); + sqlite3DeleteTable(db, pItem->pSTab); + if( pItem->fg.isUsing ){ + sqlite3IdListDelete(db, pItem->u3.pUsing); + }else if( pItem->u3.pOn ){ + sqlite3ExprDelete(db, pItem->u3.pOn); + } } - sqlite3DbFree(db, pList); + sqlite3DbNNFreeNN(db, pList); } +/* +** Attach a Subquery object to pItem->uv.pSubq. Set the +** pSelect value but leave all the other values initialized +** to zero. +** +** A copy of the Select object is made if dupSelect is true, and the +** SrcItem takes responsibility for deleting the copy. If dupSelect is +** false, ownership of the Select passes to the SrcItem. Either way, +** the SrcItem will take responsibility for deleting the Select. +** +** When dupSelect is zero, that means the Select might get deleted right +** away if there is an OOM error. Beware. +** +** Return non-zero on success. Return zero on an OOM error. +*/ +SQLITE_PRIVATE int sqlite3SrcItemAttachSubquery( + Parse *pParse, /* Parsing context */ + SrcItem *pItem, /* Item to which the subquery is to be attached */ + Select *pSelect, /* The subquery SELECT. Must be non-NULL */ + int dupSelect /* If true, attach a copy of pSelect, not pSelect itself.*/ +){ + Subquery *p; + assert( pSelect!=0 ); + assert( pItem->fg.isSubquery==0 ); + if( pItem->fg.fixedSchema ){ + pItem->u4.pSchema = 0; + pItem->fg.fixedSchema = 0; + }else if( pItem->u4.zDatabase!=0 ){ + sqlite3DbFree(pParse->db, pItem->u4.zDatabase); + pItem->u4.zDatabase = 0; + } + if( dupSelect ){ + pSelect = sqlite3SelectDup(pParse->db, pSelect, 0); + if( pSelect==0 ) return 0; + } + p = pItem->u4.pSubq = sqlite3DbMallocRawNN(pParse->db, sizeof(Subquery)); + if( p==0 ){ + sqlite3SelectDelete(pParse->db, pSelect); + return 0; + } + pItem->fg.isSubquery = 1; + p->pSelect = pSelect; + assert( offsetof(Subquery, pSelect)==0 ); + memset(((char*)p)+sizeof(p->pSelect), 0, sizeof(*p)-sizeof(p->pSelect)); + return 1; +} + + /* ** This routine is called by the parser to add a new term to the ** end of a growing FROM clause. The "p" parameter is the part of @@ -100622,68 +127010,112 @@ SQLITE_PRIVATE SrcList *sqlite3SrcListAppendFromTerm( Token *pDatabase, /* Name of the database containing pTable */ Token *pAlias, /* The right-hand side of the AS subexpression */ Select *pSubquery, /* A subquery used in place of a table name */ - Expr *pOn, /* The ON clause of a join */ - IdList *pUsing /* The USING clause of a join */ + OnOrUsing *pOnUsing /* Either the ON clause or the USING clause */ ){ - struct SrcList_item *pItem; + SrcItem *pItem; sqlite3 *db = pParse->db; - if( !p && (pOn || pUsing) ){ - sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", - (pOn ? "ON" : "USING") + if( !p && pOnUsing!=0 && (pOnUsing->pOn || pOnUsing->pUsing) ){ + sqlite3ErrorMsg(pParse, "a JOIN clause is required before %s", + (pOnUsing->pOn ? "ON" : "USING") ); goto append_from_error; } - p = sqlite3SrcListAppend(db, p, pTable, pDatabase); - if( p==0 || NEVER(p->nSrc==0) ){ + p = sqlite3SrcListAppend(pParse, p, pTable, pDatabase); + if( p==0 ){ goto append_from_error; } + assert( p->nSrc>0 ); pItem = &p->a[p->nSrc-1]; + assert( (pTable==0)==(pDatabase==0) ); + assert( pItem->zName==0 || pDatabase!=0 ); + if( IN_RENAME_OBJECT && pItem->zName ){ + Token *pToken = (ALWAYS(pDatabase) && pDatabase->z) ? pDatabase : pTable; + sqlite3RenameTokenMap(pParse, pItem->zName, pToken); + } assert( pAlias!=0 ); if( pAlias->n ){ pItem->zAlias = sqlite3NameFromToken(db, pAlias); } - pItem->pSelect = pSubquery; - pItem->pOn = pOn; - pItem->pUsing = pUsing; + assert( pSubquery==0 || pDatabase==0 ); + if( pSubquery ){ + if( sqlite3SrcItemAttachSubquery(pParse, pItem, pSubquery, 0) ){ + if( pSubquery->selFlags & SF_NestedFrom ){ + pItem->fg.isNestedFrom = 1; + } + } + } + assert( pOnUsing==0 || pOnUsing->pOn==0 || pOnUsing->pUsing==0 ); + assert( pItem->fg.isUsing==0 ); + if( pOnUsing==0 ){ + pItem->u3.pOn = 0; + }else if( pOnUsing->pUsing ){ + pItem->fg.isUsing = 1; + pItem->u3.pUsing = pOnUsing->pUsing; + }else{ + pItem->u3.pOn = pOnUsing->pOn; + } return p; - append_from_error: +append_from_error: assert( p==0 ); - sqlite3ExprDelete(db, pOn); - sqlite3IdListDelete(db, pUsing); + sqlite3ClearOnOrUsing(db, pOnUsing); sqlite3SelectDelete(db, pSubquery); return 0; } /* -** Add an INDEXED BY or NOT INDEXED clause to the most recently added +** Add an INDEXED BY or NOT INDEXED clause to the most recently added ** element of the source-list passed as the second argument. */ SQLITE_PRIVATE void sqlite3SrcListIndexedBy(Parse *pParse, SrcList *p, Token *pIndexedBy){ assert( pIndexedBy!=0 ); - if( p && ALWAYS(p->nSrc>0) ){ - struct SrcList_item *pItem = &p->a[p->nSrc-1]; + if( p && pIndexedBy->n>0 ){ + SrcItem *pItem; + assert( p->nSrc>0 ); + pItem = &p->a[p->nSrc-1]; assert( pItem->fg.notIndexed==0 ); assert( pItem->fg.isIndexedBy==0 ); assert( pItem->fg.isTabFunc==0 ); if( pIndexedBy->n==1 && !pIndexedBy->z ){ - /* A "NOT INDEXED" clause was supplied. See parse.y + /* A "NOT INDEXED" clause was supplied. See parse.y ** construct "indexed_opt" for details. */ pItem->fg.notIndexed = 1; }else{ pItem->u1.zIndexedBy = sqlite3NameFromToken(pParse->db, pIndexedBy); - pItem->fg.isIndexedBy = (pItem->u1.zIndexedBy!=0); + pItem->fg.isIndexedBy = 1; + assert( pItem->fg.isCte==0 ); /* No collision on union u2 */ } } } +/* +** Append the contents of SrcList p2 to SrcList p1 and return the resulting +** SrcList. Or, if an error occurs, return NULL. In all cases, p1 and p2 +** are deleted by this function. +*/ +SQLITE_PRIVATE SrcList *sqlite3SrcListAppendList(Parse *pParse, SrcList *p1, SrcList *p2){ + assert( p1 && p1->nSrc==1 ); + if( p2 ){ + SrcList *pNew = sqlite3SrcListEnlarge(pParse, p1, p2->nSrc, 1); + if( pNew==0 ){ + sqlite3SrcListDelete(pParse->db, p2); + }else{ + p1 = pNew; + memcpy(&p1->a[1], p2->a, p2->nSrc*sizeof(SrcItem)); + sqlite3DbFree(pParse->db, p2); + p1->a[0].fg.jointype |= (JT_LTORJ & p1->a[1].fg.jointype); + } + } + return p1; +} + /* ** Add the list of function arguments to the SrcList entry for a ** table-valued-function. */ SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList *pList){ if( p ){ - struct SrcList_item *pItem = &p->a[p->nSrc-1]; + SrcItem *pItem = &p->a[p->nSrc-1]; assert( pItem->fg.notIndexed==0 ); assert( pItem->fg.isIndexedBy==0 ); assert( pItem->fg.isTabFunc==0 ); @@ -100708,14 +127140,34 @@ SQLITE_PRIVATE void sqlite3SrcListFuncArgs(Parse *pParse, SrcList *p, ExprList * ** The operator is "natural cross join". The A and B operands are stored ** in p->a[0] and p->a[1], respectively. The parser initially stores the ** operator with A. This routine shifts that operator over to B. +** +** Additional changes: +** +** * All tables to the left of the right-most RIGHT JOIN are tagged with +** JT_LTORJ (mnemonic: Left Table Of Right Join) so that the +** code generator can easily tell that the table is part of +** the left operand of at least one RIGHT JOIN. */ -SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(SrcList *p){ - if( p ){ - int i; - for(i=p->nSrc-1; i>0; i--){ - p->a[i].fg.jointype = p->a[i-1].fg.jointype; - } +SQLITE_PRIVATE void sqlite3SrcListShiftJoinType(Parse *pParse, SrcList *p){ + (void)pParse; + if( p && p->nSrc>1 ){ + int i = p->nSrc-1; + u8 allFlags = 0; + do{ + allFlags |= p->a[i].fg.jointype = p->a[i-1].fg.jointype; + }while( (--i)>0 ); p->a[0].fg.jointype = 0; + + /* All terms to the left of a RIGHT JOIN should be tagged with the + ** JT_LTORJ flags */ + if( allFlags & JT_RIGHT ){ + for(i=p->nSrc-1; ALWAYS(i>0) && (p->a[i].fg.jointype&JT_RIGHT)==0; i--){} + i--; + assert( i>=0 ); + do{ + p->a[i].fg.jointype |= JT_LTORJ; + }while( (--i)>=0 ); + } } } @@ -100737,7 +127189,16 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ if( !v ) return; if( type!=TK_DEFERRED ){ for(i=0; inDb; i++){ - sqlite3VdbeAddOp2(v, OP_Transaction, i, (type==TK_EXCLUSIVE)+1); + int eTxnType; + Btree *pBt = db->aDb[i].pBt; + if( pBt && sqlite3BtreeIsReadonly(pBt) ){ + eTxnType = 0; /* Read txn */ + }else if( type==TK_EXCLUSIVE ){ + eTxnType = 2; /* Exclusive txn */ + }else{ + eTxnType = 1; /* Write txn */ + } + sqlite3VdbeAddOp2(v, OP_Transaction, i, eTxnType); sqlite3VdbeUsesBtree(v, i); } } @@ -100745,42 +127206,31 @@ SQLITE_PRIVATE void sqlite3BeginTransaction(Parse *pParse, int type){ } /* -** Generate VDBE code for a COMMIT statement. +** Generate VDBE code for a COMMIT or ROLLBACK statement. +** Code for ROLLBACK is generated if eType==TK_ROLLBACK. Otherwise +** code is generated for a COMMIT. */ -SQLITE_PRIVATE void sqlite3CommitTransaction(Parse *pParse){ +SQLITE_PRIVATE void sqlite3EndTransaction(Parse *pParse, int eType){ Vdbe *v; + int isRollback; assert( pParse!=0 ); assert( pParse->db!=0 ); - if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "COMMIT", 0, 0) ){ + assert( eType==TK_COMMIT || eType==TK_END || eType==TK_ROLLBACK ); + isRollback = eType==TK_ROLLBACK; + if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, + isRollback ? "ROLLBACK" : "COMMIT", 0, 0) ){ return; } v = sqlite3GetVdbe(pParse); if( v ){ - sqlite3VdbeAddOp1(v, OP_AutoCommit, 1); - } -} - -/* -** Generate VDBE code for a ROLLBACK statement. -*/ -SQLITE_PRIVATE void sqlite3RollbackTransaction(Parse *pParse){ - Vdbe *v; - - assert( pParse!=0 ); - assert( pParse->db!=0 ); - if( sqlite3AuthCheck(pParse, SQLITE_TRANSACTION, "ROLLBACK", 0, 0) ){ - return; - } - v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, 1); + sqlite3VdbeAddOp2(v, OP_AutoCommit, 1, isRollback); } } /* ** This function is called by the parser when it parses a command to create, -** release or rollback an SQL savepoint. +** release or rollback an SQL savepoint. */ SQLITE_PRIVATE void sqlite3Savepoint(Parse *pParse, int op, Token *pName){ char *zName = sqlite3NameFromToken(pParse->db, pName); @@ -100807,7 +127257,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ if( db->aDb[1].pBt==0 && !pParse->explain ){ int rc; Btree *pBt; - static const int flags = + static const int flags = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_EXCLUSIVE | @@ -100823,7 +127273,7 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ } db->aDb[1].pBt = pBt; assert( db->aDb[1].pSchema ); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, -1, 0) ){ + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize, 0, 0) ){ sqlite3OomFault(db); return 1; } @@ -100837,25 +127287,25 @@ SQLITE_PRIVATE int sqlite3OpenTempDatabase(Parse *pParse){ ** will occur at the end of the top-level VDBE and will be generated ** later, by sqlite3FinishCoding(). */ -SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ - Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3 *db = pToplevel->db; - - assert( iDb>=0 && iDbnDb ); - assert( db->aDb[iDb].pBt!=0 || iDb==1 ); - assert( iDb=0 && iDbdb->nDb ); + assert( pToplevel->db->aDb[iDb].pBt!=0 || iDb==1 ); + assert( iDbdb, iDb, 0) ); if( DbMaskTest(pToplevel->cookieMask, iDb)==0 ){ DbMaskSet(pToplevel->cookieMask, iDb); - pToplevel->cookieValue[iDb] = db->aDb[iDb].pSchema->schema_cookie; if( !OMIT_TEMPDB && iDb==1 ){ sqlite3OpenTempDatabase(pToplevel); } } } +SQLITE_PRIVATE void sqlite3CodeVerifySchema(Parse *pParse, int iDb){ + sqlite3CodeVerifySchemaAtToplevel(sqlite3ParseToplevel(pParse), iDb); +} + /* -** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each +** If argument zDb is NULL, then call sqlite3CodeVerifySchema() for each ** attached database. Otherwise, invoke it for the database named zDb only. */ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb){ @@ -100863,7 +127313,7 @@ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb) int i; for(i=0; inDb; i++){ Db *pDb = &db->aDb[i]; - if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zName)) ){ + if( pDb->pBt && (!zDb || 0==sqlite3StrICmp(zDb, pDb->zDbSName)) ){ sqlite3CodeVerifySchema(pParse, i); } } @@ -100884,7 +127334,7 @@ SQLITE_PRIVATE void sqlite3CodeVerifyNamedSchema(Parse *pParse, const char *zDb) */ SQLITE_PRIVATE void sqlite3BeginWriteOperation(Parse *pParse, int setStatement, int iDb){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - sqlite3CodeVerifySchema(pParse, iDb); + sqlite3CodeVerifySchemaAtToplevel(pToplevel, iDb); DbMaskSet(pToplevel->writeMask, iDb); pToplevel->isMultiWrite |= setStatement; } @@ -100901,9 +127351,9 @@ SQLITE_PRIVATE void sqlite3MultiWrite(Parse *pParse){ pToplevel->isMultiWrite = 1; } -/* +/* ** The code generator calls this routine if is discovers that it is -** possible to abort a statement prior to completion. In order to +** possible to abort a statement prior to completion. In order to ** perform this abort without corrupting the database, we need to make ** sure that the statement is protected by a statement transaction. ** @@ -100912,7 +127362,7 @@ SQLITE_PRIVATE void sqlite3MultiWrite(Parse *pParse){ ** such that the abort must occur after the multiwrite. This makes ** some statements involving the REPLACE conflict resolution algorithm ** go a little faster. But taking advantage of this time dependency -** makes it more difficult to prove that the code is correct (in +** makes it more difficult to prove that the code is correct (in ** particular, it prevents us from writing an effective ** implementation of sqlite3AssertMayAbort()) and so we have chosen ** to take the safe route and skip the optimization. @@ -100935,8 +127385,10 @@ SQLITE_PRIVATE void sqlite3HaltConstraint( i8 p4type, /* P4_STATIC or P4_TRANSIENT */ u8 p5Errmsg /* P5_ErrMsg type */ ){ - Vdbe *v = sqlite3GetVdbe(pParse); - assert( (errCode&0xff)==SQLITE_CONSTRAINT ); + Vdbe *v; + assert( pParse->pVdbe!=0 ); + v = sqlite3GetVdbe(pParse); + assert( (errCode&0xff)==SQLITE_CONSTRAINT || pParse->nested ); if( onError==OE_Abort ){ sqlite3MayAbort(pParse); } @@ -100957,21 +127409,24 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( StrAccum errMsg; Table *pTab = pIdx->pTable; - sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, 200); + sqlite3StrAccumInit(&errMsg, pParse->db, 0, 0, + pParse->db->aLimit[SQLITE_LIMIT_LENGTH]); if( pIdx->aColExpr ){ - sqlite3XPrintf(&errMsg, "index '%q'", pIdx->zName); + sqlite3_str_appendf(&errMsg, "index '%q'", pIdx->zName); }else{ for(j=0; jnKeyCol; j++){ char *zCol; assert( pIdx->aiColumn[j]>=0 ); - zCol = pTab->aCol[pIdx->aiColumn[j]].zName; - if( j ) sqlite3StrAccumAppend(&errMsg, ", ", 2); - sqlite3XPrintf(&errMsg, "%s.%s", pTab->zName, zCol); + zCol = pTab->aCol[pIdx->aiColumn[j]].zCnName; + if( j ) sqlite3_str_append(&errMsg, ", ", 2); + sqlite3_str_appendall(&errMsg, pTab->zName); + sqlite3_str_append(&errMsg, ".", 1); + sqlite3_str_appendall(&errMsg, zCol); } } zErr = sqlite3StrAccumFinish(&errMsg); - sqlite3HaltConstraint(pParse, - IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY + sqlite3HaltConstraint(pParse, + IsPrimaryKeyIndex(pIdx) ? SQLITE_CONSTRAINT_PRIMARYKEY : SQLITE_CONSTRAINT_UNIQUE, onError, zErr, P4_DYNAMIC, P5_ConstraintUnique); } @@ -100983,13 +127438,13 @@ SQLITE_PRIVATE void sqlite3UniqueConstraint( SQLITE_PRIVATE void sqlite3RowidConstraint( Parse *pParse, /* Parsing context */ int onError, /* Conflict resolution algorithm */ - Table *pTab /* The table with the non-unique rowid */ + Table *pTab /* The table with the non-unique rowid */ ){ char *zMsg; int rc; if( pTab->iPKey>=0 ){ zMsg = sqlite3MPrintf(pParse->db, "%s.%s", pTab->zName, - pTab->aCol[pTab->iPKey].zName); + pTab->aCol[pTab->iPKey].zCnName); rc = SQLITE_CONSTRAINT_PRIMARYKEY; }else{ zMsg = sqlite3MPrintf(pParse->db, "%s.rowid", pTab->zName); @@ -101024,13 +127479,15 @@ static int collationMatch(const char *zColl, Index *pIndex){ */ #ifndef SQLITE_OMIT_REINDEX static void reindexTable(Parse *pParse, Table *pTab, char const *zColl){ - Index *pIndex; /* An index associated with pTab */ + if( !IsVirtual(pTab) ){ + Index *pIndex; /* An index associated with pTab */ - for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ - if( zColl==0 || collationMatch(zColl, pIndex) ){ - int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - sqlite3BeginWriteOperation(pParse, 0, iDb); - sqlite3RefillIndex(pParse, pIndex, -1); + for(pIndex=pTab->pIndex; pIndex; pIndex=pIndex->pNext){ + if( zColl==0 || collationMatch(zColl, pIndex) ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3BeginWriteOperation(pParse, 0, iDb); + sqlite3RefillIndex(pParse, pIndex, -1); + } } } } @@ -101110,7 +127567,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ if( iDb<0 ) return; z = sqlite3NameFromToken(db, pObjName); if( z==0 ) return; - zDb = db->aDb[iDb].zName; + zDb = pName2->n ? db->aDb[iDb].zDbSName : 0; pTab = sqlite3FindTable(db, z, zDb); if( pTab ){ reindexTable(pParse, pTab, 0); @@ -101120,6 +127577,7 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ pIndex = sqlite3FindIndex(db, z, zDb); sqlite3DbFree(db, z); if( pIndex ){ + iDb = sqlite3SchemaToIndex(db, pIndex->pTable->pSchema); sqlite3BeginWriteOperation(pParse, 0, iDb); sqlite3RefillIndex(pParse, pIndex, -1); return; @@ -101131,10 +127589,6 @@ SQLITE_PRIVATE void sqlite3Reindex(Parse *pParse, Token *pName1, Token *pName2){ /* ** Return a KeyInfo structure that is appropriate for the given Index. ** -** The KeyInfo structure for an index is cached in the Index object. -** So there might be multiple references to the returned pointer. The -** caller should not try to modify the KeyInfo object. -** ** The caller should invoke sqlite3KeyInfoUnref() on the returned object ** when it has finished using it. */ @@ -101155,9 +127609,22 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ const char *zColl = pIdx->azColl[i]; pKey->aColl[i] = zColl==sqlite3StrBINARY ? 0 : sqlite3LocateCollSeq(pParse, zColl); - pKey->aSortOrder[i] = pIdx->aSortOrder[i]; + pKey->aSortFlags[i] = pIdx->aSortOrder[i]; + assert( 0==(pKey->aSortFlags[i] & KEYINFO_ORDER_BIGNULL) ); } if( pParse->nErr ){ + assert( pParse->rc==SQLITE_ERROR_MISSING_COLLSEQ ); + if( pIdx->bNoQuery==0 ){ + /* Deactivate the index because it contains an unknown collating + ** sequence. The only way to reactive the index is to reload the + ** schema. Adding the missing collating sequence later does not + ** reactive the index. The application had the chance to register + ** the missing index using the collation-needed callback. For + ** simplicity, SQLite will not give the application a second chance. + */ + pIdx->bNoQuery = 1; + pParse->rc = SQLITE_ERROR_RETRY; + } sqlite3KeyInfoUnref(pKey); pKey = 0; } @@ -101166,24 +127633,76 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoOfIndex(Parse *pParse, Index *pIdx){ } #ifndef SQLITE_OMIT_CTE -/* -** This routine is invoked once per CTE by the parser while parsing a -** WITH clause. +/* +** Create a new CTE object +*/ +SQLITE_PRIVATE Cte *sqlite3CteNew( + Parse *pParse, /* Parsing context */ + Token *pName, /* Name of the common-table */ + ExprList *pArglist, /* Optional column name list for the table */ + Select *pQuery, /* Query used to initialize the table */ + u8 eM10d /* The MATERIALIZED flag */ +){ + Cte *pNew; + sqlite3 *db = pParse->db; + + pNew = sqlite3DbMallocZero(db, sizeof(*pNew)); + assert( pNew!=0 || db->mallocFailed ); + + if( db->mallocFailed ){ + sqlite3ExprListDelete(db, pArglist); + sqlite3SelectDelete(db, pQuery); + }else{ + pNew->pSelect = pQuery; + pNew->pCols = pArglist; + pNew->zName = sqlite3NameFromToken(pParse->db, pName); + pNew->eM10d = eM10d; + } + return pNew; +} + +/* +** Clear information from a Cte object, but do not deallocate storage +** for the object itself. +*/ +static void cteClear(sqlite3 *db, Cte *pCte){ + assert( pCte!=0 ); + sqlite3ExprListDelete(db, pCte->pCols); + sqlite3SelectDelete(db, pCte->pSelect); + sqlite3DbFree(db, pCte->zName); +} + +/* +** Free the contents of the CTE object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3CteDelete(sqlite3 *db, Cte *pCte){ + assert( pCte!=0 ); + cteClear(db, pCte); + sqlite3DbFree(db, pCte); +} + +/* +** This routine is invoked once per CTE by the parser while parsing a +** WITH clause. The CTE described by the third argument is added to +** the WITH clause of the second argument. If the second argument is +** NULL, then a new WITH argument is created. */ SQLITE_PRIVATE With *sqlite3WithAdd( Parse *pParse, /* Parsing context */ With *pWith, /* Existing WITH clause, or NULL */ - Token *pName, /* Name of the common-table */ - ExprList *pArglist, /* Optional column name list for the table */ - Select *pQuery /* Query used to initialize the table */ + Cte *pCte /* CTE to add to the WITH clause */ ){ sqlite3 *db = pParse->db; With *pNew; char *zName; + if( pCte==0 ){ + return pWith; + } + /* Check that the CTE name is unique within this WITH clause. If ** not, store an error in the Parse structure. */ - zName = sqlite3NameFromToken(pParse->db, pName); + zName = pCte->zName; if( zName && pWith ){ int i; for(i=0; inCte; i++){ @@ -101194,7 +127713,7 @@ SQLITE_PRIVATE With *sqlite3WithAdd( } if( pWith ){ - int nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); + sqlite3_int64 nByte = sizeof(*pWith) + (sizeof(pWith->a[1]) * pWith->nCte); pNew = sqlite3DbRealloc(db, pWith, nByte); }else{ pNew = sqlite3DbMallocZero(db, sizeof(*pWith)); @@ -101202,16 +127721,11 @@ SQLITE_PRIVATE With *sqlite3WithAdd( assert( (pNew!=0 && zName!=0) || db->mallocFailed ); if( db->mallocFailed ){ - sqlite3ExprListDelete(db, pArglist); - sqlite3SelectDelete(db, pQuery); - sqlite3DbFree(db, zName); + sqlite3CteDelete(db, pCte); pNew = pWith; }else{ - pNew->a[pNew->nCte].pSelect = pQuery; - pNew->a[pNew->nCte].pCols = pArglist; - pNew->a[pNew->nCte].zName = zName; - pNew->a[pNew->nCte].zCteErr = 0; - pNew->nCte++; + pNew->a[pNew->nCte++] = *pCte; + sqlite3DbFree(db, pCte); } return pNew; @@ -101224,20 +127738,20 @@ SQLITE_PRIVATE void sqlite3WithDelete(sqlite3 *db, With *pWith){ if( pWith ){ int i; for(i=0; inCte; i++){ - struct Cte *pCte = &pWith->a[i]; - sqlite3ExprListDelete(db, pCte->pCols); - sqlite3SelectDelete(db, pCte->pSelect); - sqlite3DbFree(db, pCte->zName); + cteClear(db, &pWith->a[i]); } sqlite3DbFree(db, pWith); } } +SQLITE_PRIVATE void sqlite3WithDeleteGeneric(sqlite3 *db, void *pWith){ + sqlite3WithDelete(db, (With*)pWith); +} #endif /* !defined(SQLITE_OMIT_CTE) */ /************** End of build.c ***********************************************/ /************** Begin file callback.c ****************************************/ /* -** 2005 May 23 +** 2005 May 23 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -101303,50 +127817,6 @@ static int synthCollSeq(sqlite3 *db, CollSeq *pColl){ return SQLITE_ERROR; } -/* -** This function is responsible for invoking the collation factory callback -** or substituting a collation sequence of a different encoding when the -** requested collation sequence is not available in the desired encoding. -** -** If it is not NULL, then pColl must point to the database native encoding -** collation sequence with name zName, length nName. -** -** The return value is either the collation sequence to be used in database -** db for collation type name zName, length nName, or NULL, if no collation -** sequence can be found. If no collation is found, leave an error message. -** -** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() -*/ -SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( - Parse *pParse, /* Parsing context */ - u8 enc, /* The desired encoding for the collating sequence */ - CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ - const char *zName /* Collating sequence name */ -){ - CollSeq *p; - sqlite3 *db = pParse->db; - - p = pColl; - if( !p ){ - p = sqlite3FindCollSeq(db, enc, zName, 0); - } - if( !p || !p->xCmp ){ - /* No collation sequence of this type for this encoding is registered. - ** Call the collation factory to see if it can supply us with one. - */ - callCollNeeded(db, enc, zName); - p = sqlite3FindCollSeq(db, enc, zName, 0); - } - if( p && !p->xCmp && synthCollSeq(db, p) ){ - p = 0; - } - assert( !p || p->xCmp ); - if( p==0 ){ - sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); - } - return p; -} - /* ** This routine is called on a collation sequence before it is used to ** check that it is defined. An undefined collation sequence exists when @@ -101354,12 +127824,12 @@ SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( ** that have not been defined by sqlite3_create_collation() etc. ** ** If required, this routine calls the 'collation needed' callback to -** request a definition of the collating sequence. If this doesn't work, +** request a definition of the collating sequence. If this doesn't work, ** an equivalent collating sequence that uses a text encoding different ** from the main database is substituted, if one is available. */ SQLITE_PRIVATE int sqlite3CheckCollSeq(Parse *pParse, CollSeq *pColl){ - if( pColl ){ + if( pColl && pColl->xCmp==0 ){ const char *zName = pColl->zName; sqlite3 *db = pParse->db; CollSeq *p = sqlite3GetCollSeq(pParse, ENC(db), pColl, zName); @@ -101395,8 +127865,8 @@ static CollSeq *findCollSeqEntry( pColl = sqlite3HashFind(&db->aCollSeq, zName); if( 0==pColl && create ){ - int nName = sqlite3Strlen30(zName); - pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName + 1); + int nName = sqlite3Strlen30(zName) + 1; + pColl = sqlite3DbMallocZero(db, 3*sizeof(*pColl) + nName); if( pColl ){ CollSeq *pDel = 0; pColl[0].zName = (char*)&pColl[3]; @@ -101406,10 +127876,9 @@ static CollSeq *findCollSeqEntry( pColl[2].zName = (char*)&pColl[3]; pColl[2].enc = SQLITE_UTF16BE; memcpy(pColl[0].zName, zName, nName); - pColl[0].zName[nName] = 0; pDel = sqlite3HashInsert(&db->aCollSeq, pColl[0].zName, pColl); - /* If a malloc() failure occurred in sqlite3HashInsert(), it will + /* If a malloc() failure occurred in sqlite3HashInsert(), it will ** return the pColl pointer to be deleted (because it wasn't added ** to the hash table). */ @@ -101440,20 +127909,113 @@ static CollSeq *findCollSeqEntry( ** See also: sqlite3LocateCollSeq(), sqlite3GetCollSeq() */ SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( - sqlite3 *db, - u8 enc, - const char *zName, - int create + sqlite3 *db, /* Database connection to search */ + u8 enc, /* Desired text encoding */ + const char *zName, /* Name of the collating sequence. Might be NULL */ + int create /* True to create CollSeq if doesn't already exist */ ){ CollSeq *pColl; + assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); + assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); if( zName ){ pColl = findCollSeqEntry(db, zName, create); + if( pColl ) pColl += enc-1; }else{ pColl = db->pDfltColl; } - assert( SQLITE_UTF8==1 && SQLITE_UTF16LE==2 && SQLITE_UTF16BE==3 ); - assert( enc>=SQLITE_UTF8 && enc<=SQLITE_UTF16BE ); - if( pColl ) pColl += enc-1; + return pColl; +} + +/* +** Change the text encoding for a database connection. This means that +** the pDfltColl must change as well. +*/ +SQLITE_PRIVATE void sqlite3SetTextEncoding(sqlite3 *db, u8 enc){ + assert( enc==SQLITE_UTF8 || enc==SQLITE_UTF16LE || enc==SQLITE_UTF16BE ); + db->enc = enc; + /* EVIDENCE-OF: R-08308-17224 The default collating function for all + ** strings is BINARY. + */ + db->pDfltColl = sqlite3FindCollSeq(db, enc, sqlite3StrBINARY, 0); + sqlite3ExpirePreparedStatements(db, 1); +} + +/* +** This function is responsible for invoking the collation factory callback +** or substituting a collation sequence of a different encoding when the +** requested collation sequence is not available in the desired encoding. +** +** If it is not NULL, then pColl must point to the database native encoding +** collation sequence with name zName, length nName. +** +** The return value is either the collation sequence to be used in database +** db for collation type name zName, length nName, or NULL, if no collation +** sequence can be found. If no collation is found, leave an error message. +** +** See also: sqlite3LocateCollSeq(), sqlite3FindCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3GetCollSeq( + Parse *pParse, /* Parsing context */ + u8 enc, /* The desired encoding for the collating sequence */ + CollSeq *pColl, /* Collating sequence with native encoding, or NULL */ + const char *zName /* Collating sequence name */ +){ + CollSeq *p; + sqlite3 *db = pParse->db; + + p = pColl; + if( !p ){ + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( !p || !p->xCmp ){ + /* No collation sequence of this type for this encoding is registered. + ** Call the collation factory to see if it can supply us with one. + */ + callCollNeeded(db, enc, zName); + p = sqlite3FindCollSeq(db, enc, zName, 0); + } + if( p && !p->xCmp && synthCollSeq(db, p) ){ + p = 0; + } + assert( !p || p->xCmp ); + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such collation sequence: %s", zName); + pParse->rc = SQLITE_ERROR_MISSING_COLLSEQ; + } + return p; +} + +/* +** This function returns the collation sequence for database native text +** encoding identified by the string zName. +** +** If the requested collation sequence is not available, or not available +** in the database native encoding, the collation factory is invoked to +** request it. If the collation factory does not supply such a sequence, +** and the sequence is available in another text encoding, then that is +** returned instead. +** +** If no versions of the requested collations sequence are available, or +** another error occurs, NULL is returned and an error message written into +** pParse. +** +** This routine is a wrapper around sqlite3FindCollSeq(). This routine +** invokes the collation factory if the named collation cannot be found +** and generates an error message. +** +** See also: sqlite3FindCollSeq(), sqlite3GetCollSeq() +*/ +SQLITE_PRIVATE CollSeq *sqlite3LocateCollSeq(Parse *pParse, const char *zName){ + sqlite3 *db = pParse->db; + u8 enc = ENC(db); + u8 initbusy = db->init.busy; + CollSeq *pColl; + + pColl = sqlite3FindCollSeq(db, enc, zName, initbusy); + if( !initbusy && (!pColl || !pColl->xCmp) ){ + pColl = sqlite3GetCollSeq(pParse, enc, pColl, zName); + } + return pColl; } @@ -101467,7 +128029,7 @@ SQLITE_PRIVATE CollSeq *sqlite3FindCollSeq( ** is also -1. In other words, we are searching for a function that ** takes a variable number of arguments. ** -** If nArg is -2 that means that we are searching for any function +** If nArg is -2 that means that we are searching for any function ** regardless of the number of arguments it uses, so return a positive ** match score for any ** @@ -101492,12 +128054,13 @@ static int matchQuality( u8 enc /* Desired text encoding */ ){ int match; - - /* nArg of -2 is a special case */ - if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + assert( p->nArg>=-1 ); /* Wrong number of arguments means "no match" */ - if( p->nArg!=nArg && p->nArg>=0 ) return 0; + if( p->nArg!=nArg ){ + if( nArg==(-2) ) return (p->xSFunc==0) ? 0 : FUNC_PERFECT_MATCH; + if( p->nArg>=0 ) return 0; + } /* Give a better score to a function with a specific number of arguments ** than to function that accepts any number of arguments. */ @@ -101521,12 +128084,13 @@ static int matchQuality( ** Search a FuncDefHash for a function with the given name. Return ** a pointer to the matching FuncDef if found, or 0 if there is no match. */ -static FuncDef *functionSearch( +SQLITE_PRIVATE FuncDef *sqlite3FunctionSearch( int h, /* Hash of the name */ const char *zFunc /* Name of function */ ){ FuncDef *p; for(p=sqlite3BuiltinFunctions.a[h]; p; p=p->u.pHash){ + assert( p->funcFlags & SQLITE_FUNC_BUILTIN ); if( sqlite3StrICmp(p->zName, zFunc)==0 ){ return p; } @@ -101546,8 +128110,9 @@ SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( FuncDef *pOther; const char *zName = aDef[i].zName; int nName = sqlite3Strlen30(zName); - int h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; - pOther = functionSearch(h, zName); + int h = SQLITE_FUNC_HASH(zName[0], nName); + assert( aDef[i].funcFlags & SQLITE_FUNC_BUILTIN ); + pOther = sqlite3FunctionSearch(h, zName); if( pOther ){ assert( pOther!=&aDef[i] && pOther->pNext!=&aDef[i] ); aDef[i].pNext = pOther->pNext; @@ -101559,8 +128124,8 @@ SQLITE_PRIVATE void sqlite3InsertBuiltinFuncs( } } } - - + + /* ** Locate a user function given a name, a number of arguments and a flag @@ -101612,7 +128177,7 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( /* If no match is found, search the built-in functions. ** - ** If the SQLITE_PreferBuiltin flag is set, then search the built-in + ** If the DBFLAG_PreferBuiltin flag is set, then search the built-in ** functions even if a prior app-defined function was found. And give ** priority to built-in functions. ** @@ -101621,11 +128186,11 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** have fields overwritten with new information appropriate for the ** new function. But the FuncDefs for built-in functions are read-only. ** So we must not search for built-ins when creating a new function. - */ - if( !createFlag && (pBest==0 || (db->flags & SQLITE_PreferBuiltin)!=0) ){ + */ + if( !createFlag && (pBest==0 || (db->mDbFlags & DBFLAG_PreferBuiltin)!=0) ){ bestScore = 0; - h = (sqlite3UpperToLower[(u8)zName[0]] + nName) % SQLITE_FUNC_HASH_SZ; - p = functionSearch(h, zName); + h = SQLITE_FUNC_HASH(sqlite3UpperToLower[(u8)zName[0]], nName); + p = sqlite3FunctionSearch(h, zName); while( p ){ int score = matchQuality(p, nArg, enc); if( score>bestScore ){ @@ -101640,13 +128205,15 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( ** exact match for the name, number of arguments and encoding, then add a ** new entry to the hash table and return it. */ - if( createFlag && bestScorezName = (const char*)&pBest[1]; pBest->nArg = (u16)nArg; pBest->funcFlags = enc; memcpy((char*)&pBest[1], zName, nName+1); + for(z=(u8*)pBest->zName; *z; z++) *z = sqlite3UpperToLower[*z]; pOther = (FuncDef*)sqlite3HashInsert(&db->aFunc, pBest->zName, pBest); if( pOther==pBest ){ sqlite3DbFree(db, pBest); @@ -101665,7 +128232,7 @@ SQLITE_PRIVATE FuncDef *sqlite3FindFunction( /* ** Free all resources held by the schema structure. The void* argument points -** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the +** at a Schema struct. This function does not call sqlite3DbFree(db, ) on the ** pointer itself, it just cleans up subsidiary resources (i.e. the contents ** of the schema hash tables). ** @@ -101676,27 +128243,29 @@ SQLITE_PRIVATE void sqlite3SchemaClear(void *p){ Hash temp2; HashElem *pElem; Schema *pSchema = (Schema *)p; + sqlite3 xdb; + memset(&xdb, 0, sizeof(xdb)); temp1 = pSchema->tblHash; temp2 = pSchema->trigHash; sqlite3HashInit(&pSchema->trigHash); sqlite3HashClear(&pSchema->idxHash); for(pElem=sqliteHashFirst(&temp2); pElem; pElem=sqliteHashNext(pElem)){ - sqlite3DeleteTrigger(0, (Trigger*)sqliteHashData(pElem)); + sqlite3DeleteTrigger(&xdb, (Trigger*)sqliteHashData(pElem)); } sqlite3HashClear(&temp2); sqlite3HashInit(&pSchema->tblHash); for(pElem=sqliteHashFirst(&temp1); pElem; pElem=sqliteHashNext(pElem)){ Table *pTab = sqliteHashData(pElem); - sqlite3DeleteTable(0, pTab); + sqlite3DeleteTable(&xdb, pTab); } sqlite3HashClear(&temp1); sqlite3HashClear(&pSchema->fkeyHash); pSchema->pSeqTab = 0; if( pSchema->schemaFlags & DB_SchemaLoaded ){ pSchema->iGeneration++; - pSchema->schemaFlags &= ~DB_SchemaLoaded; } + pSchema->schemaFlags &= ~(DB_SchemaLoaded|DB_ResetWanted); } /* @@ -101745,59 +128314,112 @@ SQLITE_PRIVATE Schema *sqlite3SchemaGet(sqlite3 *db, Btree *pBt){ ** (as in the FROM clause of a SELECT statement) in this case it contains ** the name of a single table, as one might find in an INSERT, DELETE, ** or UPDATE statement. Look up that table in the symbol table and -** return a pointer. Set an error message and return NULL if the table +** return a pointer. Set an error message and return NULL if the table ** name is not found or if any other error occurs. ** ** The following fields are initialized appropriate in pSrc: ** -** pSrc->a[0].pTab Pointer to the Table object -** pSrc->a[0].pIndex Pointer to the INDEXED BY index, if there is one +** pSrc->a[0].spTab Pointer to the Table object +** pSrc->a[0].u2.pIBIndex Pointer to the INDEXED BY index, if there is one ** */ SQLITE_PRIVATE Table *sqlite3SrcListLookup(Parse *pParse, SrcList *pSrc){ - struct SrcList_item *pItem = pSrc->a; + SrcItem *pItem = pSrc->a; Table *pTab; - assert( pItem && pSrc->nSrc==1 ); + assert( pItem && pSrc->nSrc>=1 ); pTab = sqlite3LocateTableItem(pParse, 0, pItem); - sqlite3DeleteTable(pParse->db, pItem->pTab); - pItem->pTab = pTab; + if( pItem->pSTab ) sqlite3DeleteTable(pParse->db, pItem->pSTab); + pItem->pSTab = pTab; + pItem->fg.notCte = 1; if( pTab ){ - pTab->nRef++; - } - if( sqlite3IndexedByLookup(pParse, pItem) ){ - pTab = 0; + pTab->nTabRef++; + if( pItem->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pItem) ){ + pTab = 0; + } } return pTab; } -/* -** Check to make sure the given table is writable. If it is not -** writable, generate an error message and return 1. If it is -** writable return 0; +/* Generate byte-code that will report the number of rows modified +** by a DELETE, INSERT, or UPDATE statement. */ -SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, int viewOk){ - /* A table is not writable under the following circumstances: - ** - ** 1) It is a virtual table and no implementation of the xUpdate method - ** has been provided, or - ** 2) It is a system table (i.e. sqlite_master), this call is not - ** part of a nested parse and writable_schema pragma has not - ** been specified. - ** - ** In either case leave an error message in pParse and return non-zero. - */ - if( ( IsVirtual(pTab) - && sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ) - || ( (pTab->tabFlags & TF_Readonly)!=0 - && (pParse->db->flags & SQLITE_WriteSchema)==0 - && pParse->nested==0 ) - ){ - sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); +SQLITE_PRIVATE void sqlite3CodeChangeCount(Vdbe *v, int regCounter, const char *zColName){ + sqlite3VdbeAddOp0(v, OP_FkCheck); + sqlite3VdbeAddOp2(v, OP_ResultRow, regCounter, 1); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zColName, SQLITE_STATIC); +} + +/* Return true if table pTab is read-only. +** +** A table is read-only if any of the following are true: +** +** 1) It is a virtual table and no implementation of the xUpdate method +** has been provided +** +** 2) A trigger is currently being coded and the table is a virtual table +** that is SQLITE_VTAB_DIRECTONLY or if PRAGMA trusted_schema=OFF and +** the table is not SQLITE_VTAB_INNOCUOUS. +** +** 3) It is a system table (i.e. sqlite_schema), this call is not +** part of a nested parse and writable_schema pragma has not +** been specified +** +** 4) The table is a shadow table, the database connection is in +** defensive mode, and the current sqlite3_prepare() +** is for a top-level SQL statement. +*/ +static int vtabIsReadOnly(Parse *pParse, Table *pTab){ + assert( IsVirtual(pTab) ); + if( sqlite3GetVTable(pParse->db, pTab)->pMod->pModule->xUpdate==0 ){ return 1; } + /* Within triggers: + ** * Do not allow DELETE, INSERT, or UPDATE of SQLITE_VTAB_DIRECTONLY + ** virtual tables + ** * Only allow DELETE, INSERT, or UPDATE of non-SQLITE_VTAB_INNOCUOUS + ** virtual tables if PRAGMA trusted_schema=ON. + */ + if( pParse->pToplevel!=0 + && pTab->u.vtab.p->eVtabRisk > + ((pParse->db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + return 0; +} +static int tabIsReadOnly(Parse *pParse, Table *pTab){ + sqlite3 *db; + if( IsVirtual(pTab) ){ + return vtabIsReadOnly(pParse, pTab); + } + if( (pTab->tabFlags & (TF_Readonly|TF_Shadow))==0 ) return 0; + db = pParse->db; + if( (pTab->tabFlags & TF_Readonly)!=0 ){ + return sqlite3WritableSchema(db)==0 && pParse->nested==0; + } + assert( pTab->tabFlags & TF_Shadow ); + return sqlite3ReadOnlyShadowTables(db); +} + +/* +** Check to make sure the given table is writable. +** +** If pTab is not writable -> generate an error message and return 1. +** If pTab is writable but other errors have occurred -> return 1. +** If pTab is writable and no prior errors -> return 0; +*/ +SQLITE_PRIVATE int sqlite3IsReadOnly(Parse *pParse, Table *pTab, Trigger *pTrigger){ + if( tabIsReadOnly(pParse, pTab) ){ + sqlite3ErrorMsg(pParse, "table %s may not be modified", pTab->zName); + return 1; + } #ifndef SQLITE_OMIT_VIEW - if( !viewOk && pTab->pSelect ){ + if( IsView(pTab) + && (pTrigger==0 || (pTrigger->bReturning && pTrigger->pNext==0)) + ){ sqlite3ErrorMsg(pParse,"cannot modify %s because it is a view",pTab->zName); return 1; } @@ -101816,6 +128438,8 @@ SQLITE_PRIVATE void sqlite3MaterializeView( Parse *pParse, /* Parsing context */ Table *pView, /* View definition */ Expr *pWhere, /* Optional WHERE clause to be added */ + ExprList *pOrderBy, /* Optional ORDER BY clause */ + Expr *pLimit, /* Optional LIMIT clause */ int iCur /* Cursor number for ephemeral table */ ){ SelectDest dest; @@ -101824,16 +128448,17 @@ SQLITE_PRIVATE void sqlite3MaterializeView( sqlite3 *db = pParse->db; int iDb = sqlite3SchemaToIndex(db, pView->pSchema); pWhere = sqlite3ExprDup(db, pWhere, 0); - pFrom = sqlite3SrcListAppend(db, 0, 0, 0); + pFrom = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pFrom ){ assert( pFrom->nSrc==1 ); pFrom->a[0].zName = sqlite3DbStrDup(db, pView->zName); - pFrom->a[0].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); - assert( pFrom->a[0].pOn==0 ); - assert( pFrom->a[0].pUsing==0 ); + assert( pFrom->a[0].fg.fixedSchema==0 && pFrom->a[0].fg.isSubquery==0 ); + pFrom->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + assert( pFrom->a[0].fg.isUsing==0 ); + assert( pFrom->a[0].u3.pOn==0 ); } - pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, 0, - SF_IncludeHidden, 0, 0); + pSel = sqlite3SelectNew(pParse, 0, pFrom, pWhere, 0, 0, pOrderBy, + SF_IncludeHidden, pLimit); sqlite3SelectDestInit(&dest, SRT_EphemTab, iCur); sqlite3Select(pParse, pSel, &dest); sqlite3SelectDelete(db, pSel); @@ -101855,71 +128480,95 @@ SQLITE_PRIVATE Expr *sqlite3LimitWhere( Expr *pWhere, /* The WHERE clause. May be null */ ExprList *pOrderBy, /* The ORDER BY clause. May be null */ Expr *pLimit, /* The LIMIT clause. May be null */ - Expr *pOffset, /* The OFFSET clause. May be null */ char *zStmtType /* Either DELETE or UPDATE. For err msgs. */ ){ - Expr *pWhereRowid = NULL; /* WHERE rowid .. */ + sqlite3 *db = pParse->db; + Expr *pLhs = NULL; /* LHS of IN(SELECT...) operator */ Expr *pInClause = NULL; /* WHERE rowid IN ( select ) */ - Expr *pSelectRowid = NULL; /* SELECT rowid ... */ - ExprList *pEList = NULL; /* Expression list contaning only pSelectRowid */ + ExprList *pEList = NULL; /* Expression list containing only pSelectRowid*/ SrcList *pSelectSrc = NULL; /* SELECT rowid FROM x ... (dup of pSrc) */ Select *pSelect = NULL; /* Complete SELECT tree */ + Table *pTab; /* Check that there isn't an ORDER BY without a LIMIT clause. */ - if( pOrderBy && (pLimit == 0) ) { + if( pOrderBy && pLimit==0 ) { sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on %s", zStmtType); - goto limit_where_cleanup; + sqlite3ExprDelete(pParse->db, pWhere); + sqlite3ExprListDelete(pParse->db, pOrderBy); + return 0; } /* We only need to generate a select expression if there ** is a limit/offset term to enforce. */ if( pLimit == 0 ) { - /* if pLimit is null, pOffset will always be null as well. */ - assert( pOffset == 0 ); return pWhere; } - /* Generate a select expression tree to enforce the limit/offset + /* Generate a select expression tree to enforce the limit/offset ** term for the DELETE or UPDATE statement. For example: ** DELETE FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 ** becomes: - ** DELETE FROM table_a WHERE rowid IN ( + ** DELETE FROM table_a WHERE rowid IN ( ** SELECT rowid FROM table_a WHERE col1=1 ORDER BY col2 LIMIT 1 OFFSET 1 ** ); */ - pSelectRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - if( pSelectRowid == 0 ) goto limit_where_cleanup; - pEList = sqlite3ExprListAppend(pParse, 0, pSelectRowid); - if( pEList == 0 ) goto limit_where_cleanup; + pTab = pSrc->a[0].pSTab; + if( HasRowid(pTab) ){ + pLhs = sqlite3PExpr(pParse, TK_ROW, 0, 0); + pEList = sqlite3ExprListAppend( + pParse, 0, sqlite3PExpr(pParse, TK_ROW, 0, 0) + ); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol>=1 ); + if( pPk->nKeyCol==1 ){ + const char *zName; + assert( pPk->aiColumn[0]>=0 && pPk->aiColumn[0]nCol ); + zName = pTab->aCol[pPk->aiColumn[0]].zCnName; + pLhs = sqlite3Expr(db, TK_ID, zName); + pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ID, zName)); + }else{ + int i; + for(i=0; inKeyCol; i++){ + Expr *p; + assert( pPk->aiColumn[i]>=0 && pPk->aiColumn[i]nCol ); + p = sqlite3Expr(db, TK_ID, pTab->aCol[pPk->aiColumn[i]].zCnName); + pEList = sqlite3ExprListAppend(pParse, pEList, p); + } + pLhs = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( pLhs ){ + pLhs->x.pList = sqlite3ExprListDup(db, pEList, 0); + } + } + } /* duplicate the FROM clause as it is needed by both the DELETE/UPDATE tree ** and the SELECT subtree. */ - pSelectSrc = sqlite3SrcListDup(pParse->db, pSrc, 0); - if( pSelectSrc == 0 ) { - sqlite3ExprListDelete(pParse->db, pEList); - goto limit_where_cleanup; + pSrc->a[0].pSTab = 0; + pSelectSrc = sqlite3SrcListDup(db, pSrc, 0); + pSrc->a[0].pSTab = pTab; + if( pSrc->a[0].fg.isIndexedBy ){ + assert( pSrc->a[0].fg.isCte==0 ); + pSrc->a[0].u2.pIBIndex = 0; + pSrc->a[0].fg.isIndexedBy = 0; + sqlite3DbFree(db, pSrc->a[0].u1.zIndexedBy); + }else if( pSrc->a[0].fg.isCte ){ + pSrc->a[0].u2.pCteUse->nUse++; } /* generate the SELECT expression tree. */ - pSelect = sqlite3SelectNew(pParse,pEList,pSelectSrc,pWhere,0,0, - pOrderBy,0,pLimit,pOffset); - if( pSelect == 0 ) return 0; + pSelect = sqlite3SelectNew(pParse, pEList, pSelectSrc, pWhere, 0 ,0, + pOrderBy,0,pLimit + ); - /* now generate the new WHERE rowid IN clause for the DELETE/UDPATE */ - pWhereRowid = sqlite3PExpr(pParse, TK_ROW, 0, 0, 0); - pInClause = pWhereRowid ? sqlite3PExpr(pParse, TK_IN, pWhereRowid, 0, 0) : 0; + /* now generate the new WHERE rowid IN clause for the DELETE/UPDATE */ + pInClause = sqlite3PExpr(pParse, TK_IN, pLhs, 0); sqlite3PExprAddSelect(pParse, pInClause, pSelect); return pInClause; - -limit_where_cleanup: - sqlite3ExprDelete(pParse->db, pWhere); - sqlite3ExprListDelete(pParse->db, pOrderBy); - sqlite3ExprDelete(pParse->db, pLimit); - sqlite3ExprDelete(pParse->db, pOffset); - return 0; } #endif /* defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) */ /* && !defined(SQLITE_OMIT_SUBQUERY) */ @@ -101934,11 +128583,12 @@ limit_where_cleanup: SQLITE_PRIVATE void sqlite3DeleteFrom( Parse *pParse, /* The parser context */ SrcList *pTabList, /* The table from which we should delete things */ - Expr *pWhere /* The WHERE clause. May be null */ + Expr *pWhere, /* The WHERE clause. May be null */ + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit /* LIMIT clause. May be null */ ){ Vdbe *v; /* The virtual database engine */ Table *pTab; /* The table from which records will be deleted */ - const char *zDb; /* Name of database holding pTab */ int i; /* Loop counter */ WhereInfo *pWInfo; /* Information about the WHERE clause */ Index *pIdx; /* For looping over indices of the table */ @@ -101950,7 +128600,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( AuthContext sContext; /* Authorization context */ NameContext sNC; /* Name context to resolve expressions in */ int iDb; /* Database number */ - int memCnt = -1; /* Memory cell used for change counting */ + int memCnt = 0; /* Memory cell used for change counting */ int rcauth; /* Value returned by authorization callback */ int eOnePass; /* ONEPASS_OFF or _SINGLE or _MULTI */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ @@ -101967,7 +128617,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( int addrEphOpen = 0; /* Instruction to open the Ephemeral table */ int bComplex; /* True if there are triggers or FKs or ** subqueries in the WHERE clause */ - + #ifndef SQLITE_OMIT_TRIGGER int isView; /* True if attempting to delete from a view */ Trigger *pTrigger; /* List of table triggers, if required */ @@ -101975,9 +128625,11 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( memset(&sContext, 0, sizeof(sContext)); db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto delete_from_cleanup; } + assert( db->mallocFailed==0 ); assert( pTabList->nSrc==1 ); /* Locate the table which we want to delete. This table has to be @@ -101993,30 +128645,48 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( */ #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - isView = pTab->pSelect!=0; - bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); + isView = IsView(pTab); #else # define pTrigger 0 # define isView 0 #endif + bComplex = pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0); #ifdef SQLITE_OMIT_VIEW # undef isView # define isView 0 #endif +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Delete() at %s:%d", __FILE__, __LINE__); + sqlite3TreeViewDelete(pParse->pWith, pTabList, pWhere, + pOrderBy, pLimit, pTrigger); + } +#endif + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, "DELETE" + ); + pOrderBy = 0; + pLimit = 0; + } +#endif + /* If pTab is really a view, make sure it has been initialized. */ if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto delete_from_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, (pTrigger?1:0)) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto delete_from_cleanup; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - zDb = db->aDb[iDb].zName; - rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, zDb); + rcauth = sqlite3AuthCheck(pParse, SQLITE_DELETE, pTab->zName, 0, + db->aDb[iDb].zDbSName); assert( rcauth==SQLITE_OK || rcauth==SQLITE_DENY || rcauth==SQLITE_IGNORE ); if( rcauth==SQLITE_DENY ){ goto delete_from_cleanup; @@ -102044,15 +128714,19 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( goto delete_from_cleanup; } if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); - sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3BeginWriteOperation(pParse, bComplex, iDb); /* If we are trying to delete from a view, realize that view into ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iTabCur); + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, iTabCur + ); iDataCur = iIdxCur = iTabCur; + pOrderBy = 0; + pLimit = 0; } #endif @@ -102068,7 +128742,11 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( /* Initialize the counter of the number of rows deleted, if ** we are counting rows. */ - if( db->flags & SQLITE_CountRows ){ + if( (db->flags & SQLITE_CountRows)!=0 + && !pParse->nested + && !pParse->pTriggerTab + && !pParse->bReturning + ){ memCnt = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, memCnt); } @@ -102076,8 +128754,15 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( #ifndef SQLITE_OMIT_TRUNCATE_OPTIMIZATION /* Special case: A DELETE without a WHERE clause deletes everything. ** It is easier just to erase the whole table. Prior to version 3.6.5, - ** this optimization caused the row change count (the value returned by - ** API function sqlite3_count_changes) to be set incorrectly. */ + ** this optimization caused the row change count (the value returned by + ** API function sqlite3_count_changes) to be set incorrectly. + ** + ** The "rcauth==SQLITE_OK" terms is the + ** IMPLEMENTATION-OF: R-17228-37124 If the action code is SQLITE_DELETE and + ** the callback returns SQLITE_IGNORE then the DELETE operation proceeds but + ** the truncate optimization is disabled and all rows are deleted + ** individually. + */ if( rcauth==SQLITE_OK && pWhere==0 && !bComplex @@ -102089,23 +128774,27 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( assert( !isView ); sqlite3TableLock(pParse, iDb, pTab->tnum, 1, pTab->zName); if( HasRowid(pTab) ){ - sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt, + sqlite3VdbeAddOp4(v, OP_Clear, pTab->tnum, iDb, memCnt ? memCnt : -1, pTab->zName, P4_STATIC); } for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ assert( pIdx->pSchema==pTab->pSchema ); - sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Clear, pIdx->tnum, iDb, memCnt ? memCnt : -1); + }else{ + sqlite3VdbeAddOp2(v, OP_Clear, pIdx->tnum, iDb); + } } }else #endif /* SQLITE_OMIT_TRUNCATE_OPTIMIZATION */ { - u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK|WHERE_SEEK_TABLE; - if( sNC.ncFlags & NC_VarSelect ) bComplex = 1; + u16 wcf = WHERE_ONEPASS_DESIRED|WHERE_DUPLICATES_OK; + if( sNC.ncFlags & NC_Subquery ) bComplex = 1; wcf |= (bComplex ? 0 : WHERE_ONEPASS_MULTIROW); if( HasRowid(pTab) ){ /* For a rowid table, initialize the RowSet to an empty set */ pPk = 0; - nPk = 1; + assert( nPk==1 ); iRowSet = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Null, 0, iRowSet); }else{ @@ -102120,7 +128809,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( addrEphOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEphCur, nPk); sqlite3VdbeSetP4KeyInfo(pParse, pPk); } - + /* Construct a query to find the rowid or primary key for every row ** to be deleted, based on the WHERE clause. Set variable eOnePass ** to indicate the strategy used to implement this delete: @@ -102129,17 +128818,22 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( ** ONEPASS_SINGLE: One-pass approach - at most one row deleted. ** ONEPASS_MULTI: One-pass approach - any number of rows may be deleted. */ - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, wcf, iTabCur+1); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0,0,wcf,iTabCur+1); if( pWInfo==0 ) goto delete_from_cleanup; eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); assert( IsVirtual(pTab)==0 || eOnePass!=ONEPASS_MULTI ); - assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF ); - + assert( IsVirtual(pTab) || bComplex || eOnePass!=ONEPASS_OFF + || OptimizationDisabled(db, SQLITE_OnePass) ); + if( eOnePass!=ONEPASS_SINGLE ) sqlite3MultiWrite(pParse); + if( sqlite3WhereUsesDeferredSeek(pWInfo) ){ + sqlite3VdbeAddOp1(v, OP_FinishSeek, iTabCur); + } + /* Keep track of the number of rows to be deleted */ - if( db->flags & SQLITE_CountRows ){ + if( memCnt ){ sqlite3VdbeAddOp2(v, OP_AddImm, memCnt, 1); } - + /* Extract the rowid or primary key for the current row */ if( pPk ){ for(i=0; inMem + 1; - iKey = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iTabCur, iKey, 0); - if( iKey>pParse->nMem ) pParse->nMem = iKey; + iKey = ++pParse->nMem; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iTabCur, -1, iKey); } - + if( eOnePass!=ONEPASS_OFF ){ /* For ONEPASS, no need to store the rowid/primary-key. There is only ** one, so just keep it in its register(s) and fall through to the @@ -102169,6 +128862,7 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iTabCur] = 0; if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iTabCur] = 0; if( addrEphOpen ) sqlite3VdbeChangeToNoop(v, addrEphOpen); + addrBypass = sqlite3VdbeMakeLabel(pParse); }else{ if( pPk ){ /* Add the PK key for this row to the temporary table */ @@ -102176,84 +128870,82 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( nKey = 0; /* Zero tells OP_Found to use a composite key */ sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, iKey, sqlite3IndexAffinityStr(pParse->db, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEphCur, iKey); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEphCur, iKey, iPk, nPk); }else{ /* Add the rowid of the row to be deleted to the RowSet */ - nKey = 1; /* OP_Seek always uses a single rowid */ + nKey = 1; /* OP_DeferredSeek always uses a single rowid */ sqlite3VdbeAddOp2(v, OP_RowSetAdd, iRowSet, iKey); } - } - - /* If this DELETE cannot use the ONEPASS strategy, this is the - ** end of the WHERE loop */ - if( eOnePass!=ONEPASS_OFF ){ - addrBypass = sqlite3VdbeMakeLabel(v); - }else{ sqlite3WhereEnd(pWInfo); } - - /* Unless this is a view, open cursors for the table we are + + /* Unless this is a view, open cursors for the table we are ** deleting from and all its indices. If this is a view, then the - ** only effect this statement has is to fire the INSTEAD OF + ** only effect this statement has is to fire the INSTEAD OF ** triggers. */ if( !isView ){ int iAddrOnce = 0; if( eOnePass==ONEPASS_MULTI ){ - iAddrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + iAddrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } testcase( IsVirtual(pTab) ); sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, OPFLAG_FORDELETE, iTabCur, aToOpen, &iDataCur, &iIdxCur); assert( pPk || IsVirtual(pTab) || iDataCur==iTabCur ); assert( pPk || IsVirtual(pTab) || iIdxCur==iDataCur+1 ); - if( eOnePass==ONEPASS_MULTI ) sqlite3VdbeJumpHere(v, iAddrOnce); + if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeJumpHereOrPopInst(v, iAddrOnce); + } } - + /* Set up a loop over the rowids/primary-keys that were found in the ** where-clause loop above. */ if( eOnePass!=ONEPASS_OFF ){ assert( nKey==nPk ); /* OP_Found will use an unpacked key */ if( !IsVirtual(pTab) && aToOpen[iDataCur-iTabCur] ){ - assert( pPk!=0 || pTab->pSelect!=0 ); + assert( pPk!=0 || IsView(pTab) ); sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, addrBypass, iKey, nKey); VdbeCoverage(v); } }else if( pPk ){ addrLoop = sqlite3VdbeAddOp1(v, OP_Rewind, iEphCur); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowKey, iEphCur, iKey); + if( IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iEphCur, 0, iKey); + }else{ + sqlite3VdbeAddOp2(v, OP_RowData, iEphCur, iKey); + } assert( nKey==0 ); /* OP_Found will use a composite key */ }else{ addrLoop = sqlite3VdbeAddOp3(v, OP_RowSetRead, iRowSet, 0, iKey); VdbeCoverage(v); assert( nKey==1 ); - } - + } + /* Delete the row */ #ifndef SQLITE_OMIT_VIRTUALTABLE if( IsVirtual(pTab) ){ const char *pVTab = (const char *)sqlite3GetVTable(db, pTab); sqlite3VtabMakeWritable(pParse, pTab); - sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); - sqlite3VdbeChangeP5(v, OE_Abort); assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); sqlite3MayAbort(pParse); - if( eOnePass==ONEPASS_SINGLE && sqlite3IsToplevel(pParse) ){ - pParse->isMultiWrite = 0; + if( eOnePass==ONEPASS_SINGLE ){ + sqlite3VdbeAddOp1(v, OP_Close, iTabCur); + if( sqlite3IsToplevel(pParse) ){ + pParse->isMultiWrite = 0; + } } + sqlite3VdbeAddOp4(v, OP_VUpdate, 0, 1, iKey, pVTab, P4_VTAB); + sqlite3VdbeChangeP5(v, OE_Abort); }else #endif { int count = (pParse->nested==0); /* True to count changes */ - int iIdxNoSeek = -1; - if( bComplex==0 && aiCurOnePass[1]!=iDataCur ){ - iIdxNoSeek = aiCurOnePass[1]; - } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, - iKey, nKey, count, OE_Default, eOnePass, iIdxNoSeek); + iKey, nKey, count, OE_Default, eOnePass, aiCurOnePass[1]); } - + /* End of the loop over all rowids/primary-keys. */ if( eOnePass!=ONEPASS_OFF ){ sqlite3VdbeResolveLabel(v, addrBypass); @@ -102264,14 +128956,6 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( }else{ sqlite3VdbeGoto(v, addrLoop); sqlite3VdbeJumpHere(v, addrLoop); - } - - /* Close the cursors open on the table and its indexes. */ - if( !isView && !IsVirtual(pTab) ){ - if( !pPk ) sqlite3VdbeAddOp1(v, OP_Close, iDataCur); - for(i=0, pIdx=pTab->pIndex; pIdx; i++, pIdx=pIdx->pNext){ - sqlite3VdbeAddOp1(v, OP_Close, iIdxCur + i); - } } } /* End non-truncate path */ @@ -102283,21 +128967,23 @@ SQLITE_PRIVATE void sqlite3DeleteFrom( sqlite3AutoincrementEnd(pParse); } - /* Return the number of rows that were deleted. If this routine is + /* Return the number of rows that were deleted. If this routine is ** generating code because of a call to sqlite3NestedParse(), do not ** invoke the callback function. */ - if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, memCnt, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows deleted", SQLITE_STATIC); + if( memCnt ){ + sqlite3CodeChangeCount(v, memCnt, "rows deleted"); } delete_from_cleanup: sqlite3AuthContextPop(&sContext); sqlite3SrcListDelete(db, pTabList); sqlite3ExprDelete(db, pWhere); - sqlite3DbFree(db, aToOpen); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); +#endif + if( aToOpen ) sqlite3DbNNFreeNN(db, aToOpen); return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -102338,16 +129024,18 @@ delete_from_cleanup: ** and nPk before reading from it. ** ** If eMode is ONEPASS_MULTI, then this call is being made as part -** of a ONEPASS delete that affects multiple rows. In this case, if -** iIdxNoSeek is a valid cursor number (>=0), then its position should -** be preserved following the delete operation. Or, if iIdxNoSeek is not -** a valid cursor number, the position of iDataCur should be preserved -** instead. +** of a ONEPASS delete that affects multiple rows. In this case, if +** iIdxNoSeek is a valid cursor number (>=0) and is not the same as +** iDataCur, then its position should be preserved following the delete +** operation. Or, if iIdxNoSeek is not a valid cursor number, the +** position of iDataCur should be preserved instead. ** ** iIdxNoSeek: -** If iIdxNoSeek is a valid cursor number (>=0), then it identifies an -** index cursor (from within array of cursors starting at iIdxCur) that -** already points to the index entry to be deleted. +** If iIdxNoSeek is a valid cursor number (>=0) not equal to iDataCur, +** then it identifies an index cursor (from within array of cursors +** starting at iIdxCur) that already points to the index entry to be deleted. +** Except, this optimization is disabled if there are BEFORE triggers since +** the trigger body might have moved the cursor. */ SQLITE_PRIVATE void sqlite3GenerateRowDelete( Parse *pParse, /* Parsing context */ @@ -102372,17 +129060,17 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( VdbeModuleComment((v, "BEGIN: GenRowDel(%d,%d,%d,%d)", iDataCur, iIdxCur, iPk, (int)nPk)); - /* Seek cursor iCur to the row to delete. If this row no longer exists + /* Seek cursor iCur to the row to delete. If this row no longer exists ** (this can happen if a trigger program has already deleted it), do ** not attempt to delete it or fire any DELETE triggers. */ - iLabel = sqlite3VdbeMakeLabel(v); + iLabel = sqlite3VdbeMakeLabel(pParse); opSeek = HasRowid(pTab) ? OP_NotExists : OP_NotFound; if( eMode==ONEPASS_OFF ){ sqlite3VdbeAddOp4Int(v, opSeek, iDataCur, iLabel, iPk, nPk); VdbeCoverageIf(v, opSeek==OP_NotExists); VdbeCoverageIf(v, opSeek==OP_NotFound); } - + /* If there are any triggers to fire, allocate a range of registers to ** use for the old.* references in the triggers. */ if( sqlite3FkRequired(pParse, pTab, 0, 0) || pTrigger ){ @@ -102399,59 +129087,67 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( iOld = pParse->nMem+1; pParse->nMem += (1 + pTab->nCol); - /* Populate the OLD.* pseudo-table register array. These values will be + /* Populate the OLD.* pseudo-table register array. These values will be ** used by any BEFORE and AFTER triggers that exist. */ sqlite3VdbeAddOp2(v, OP_Copy, iPk, iOld); for(iCol=0; iColnCol; iCol++){ testcase( mask!=0xffffffff && iCol==31 ); testcase( mask!=0xffffffff && iCol==32 ); if( mask==0xffffffff || (iCol<=31 && (mask & MASKBIT32(iCol))!=0) ){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+iCol+1); + int kk = sqlite3TableColumnToStorage(pTab, iCol); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, iCol, iOld+kk+1); } } /* Invoke BEFORE DELETE trigger programs. */ addrStart = sqlite3VdbeCurrentAddr(v); - sqlite3CodeRowTrigger(pParse, pTrigger, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_DELETE, 0, TRIGGER_BEFORE, pTab, iOld, onconf, iLabel ); - /* If any BEFORE triggers were coded, then seek the cursor to the + /* If any BEFORE triggers were coded, then seek the cursor to the ** row to be deleted again. It may be that the BEFORE triggers moved - ** the cursor or of already deleted the row that the cursor was + ** the cursor or already deleted the row that the cursor was ** pointing to. + ** + ** Also disable the iIdxNoSeek optimization since the BEFORE trigger + ** may have moved that cursor. */ if( addrStart=0 ); + iIdxNoSeek = -1; } /* Do FK processing. This call checks that any FK constraints that - ** refer to this table (i.e. constraints attached to other tables) + ** refer to this table (i.e. constraints attached to other tables) ** are not violated by deleting this row. */ sqlite3FkCheck(pParse, pTab, iOld, 0, 0, 0); } /* Delete the index and table entries. Skip this step if pTab is really ** a view (in which case the only effect of the DELETE statement is to - ** fire the INSTEAD OF triggers). + ** fire the INSTEAD OF triggers). ** ** If variable 'count' is non-zero, then this OP_Delete instruction should ** invoke the update-hook. The pre-update-hook, on the other hand should ** be invoked unless table pTab is a system table. The difference is that - ** the update-hook is not invoked for rows removed by REPLACE, but the + ** the update-hook is not invoked for rows removed by REPLACE, but the ** pre-update-hook is. - */ - if( pTab->pSelect==0 ){ + */ + if( !IsView(pTab) ){ u8 p5 = 0; sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur,0,iIdxNoSeek); sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, (count?OPFLAG_NCHANGE:0)); - sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); + if( pParse->nested==0 || 0==sqlite3_stricmp(pTab->zName, "sqlite_stat1") ){ + sqlite3VdbeAppendP4(v, (char*)pTab, P4_TABLE); + } if( eMode!=ONEPASS_OFF ){ sqlite3VdbeChangeP5(v, OPFLAG_AUXDELETE); } - if( iIdxNoSeek>=0 ){ + if( iIdxNoSeek>=0 && iIdxNoSeek!=iDataCur ){ sqlite3VdbeAddOp1(v, OP_Delete, iIdxNoSeek); } if( eMode==ONEPASS_MULTI ) p5 |= OPFLAG_SAVEPOSITION; @@ -102460,16 +129156,18 @@ SQLITE_PRIVATE void sqlite3GenerateRowDelete( /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key - ** to the row just deleted. */ + ** to the row just deleted. */ sqlite3FkActions(pParse, pTab, 0, iOld, 0, 0); /* Invoke AFTER DELETE trigger programs. */ - sqlite3CodeRowTrigger(pParse, pTrigger, - TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel - ); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, + TK_DELETE, 0, TRIGGER_AFTER, pTab, iOld, onconf, iLabel + ); + } /* Jump here if the row had already been deleted before any BEFORE - ** trigger programs were invoked. Or if a trigger program throws a + ** trigger programs were invoked. Or if a trigger program throws a ** RAISE(IGNORE) exception. */ sqlite3VdbeResolveLabel(v, iLabel); VdbeModuleComment((v, "END: GenRowDel()")); @@ -102521,6 +129219,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( &iPartIdxLabel, pPrior, r1); sqlite3VdbeAddOp3(v, OP_IdxDelete, iIdxCur+i, r1, pIdx->uniqNotNull ? pIdx->nKeyCol : pIdx->nColumn); + sqlite3VdbeChangeP5(v, 1); /* Cause IdxDelete to error if no entry found */ sqlite3ResolvePartIdxLabel(pParse, iPartIdxLabel); pPrior = pIdx; } @@ -102553,7 +129252,7 @@ SQLITE_PRIVATE void sqlite3GenerateRowIndexDelete( ** its key into the same sequence of registers and if pPrior and pIdx share ** a column in common, then the register corresponding to that column already ** holds the correct value and the loading of that register is skipped. -** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK +** This optimization is helpful when doing a DELETE or an INTEGRITY_CHECK ** on a table with multiple indices, and especially with the ROWID or ** PRIMARY KEY columns of the index. */ @@ -102574,11 +129273,13 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( if( piPartIdxLabel ){ if( pIdx->pPartIdxWhere ){ - *piPartIdxLabel = sqlite3VdbeMakeLabel(v); - pParse->iSelfTab = iDataCur; - sqlite3ExprCachePush(pParse); - sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, + *piPartIdxLabel = sqlite3VdbeMakeLabel(pParse); + pParse->iSelfTab = iDataCur + 1; + sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, *piPartIdxLabel, SQLITE_JUMPIFNULL); + pParse->iSelfTab = 0; + pPrior = 0; /* Ticket a9efb42811fa41ee 2019-11-02; + ** pPartIdxWhere may have corrupted regPrior registers */ }else{ *piPartIdxLabel = 0; } @@ -102595,13 +129296,15 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( continue; } sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iDataCur, j, regBase+j); - /* If the column affinity is REAL but the number is an integer, then it - ** might be stored in the table as an integer (using a compact - ** representation) then converted to REAL by an OP_RealAffinity opcode. - ** But we are getting ready to store this value back into an index, where - ** it should be converted by to INTEGER again. So omit the OP_RealAffinity - ** opcode if it is present */ - sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); + if( pIdx->aiColumn[j]>=0 ){ + /* If the column affinity is REAL but the number is an integer, then it + ** might be stored in the table as an integer (using a compact + ** representation) then converted to REAL by an OP_RealAffinity opcode. + ** But we are getting ready to store this value back into an index, where + ** it should be converted by to INTEGER again. So omit the + ** OP_RealAffinity opcode if it is present */ + sqlite3VdbeDeletePriorOpcode(v, OP_RealAffinity); + } } if( regOut ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regOut); @@ -102618,7 +129321,6 @@ SQLITE_PRIVATE int sqlite3GenerateIndexKey( SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ if( iLabel ){ sqlite3VdbeResolveLabel(pParse->pVdbe, iLabel); - sqlite3ExprCachePop(pParse); } } @@ -102642,6 +129344,9 @@ SQLITE_PRIVATE void sqlite3ResolvePartIdxLabel(Parse *pParse, int iLabel){ /* #include "sqliteInt.h" */ /* #include */ /* #include */ +#ifndef SQLITE_OMIT_FLOATING_POINT +/* #include */ +#endif /* #include "vdbeInt.h" */ /* @@ -102661,6 +129366,8 @@ static CollSeq *sqlite3GetFuncCollSeq(sqlite3_context *context){ ** iteration of the aggregate loop. */ static void sqlite3SkipAccumulatorLoad(sqlite3_context *context){ + assert( context->isError<=0 ); + context->isError = -1; context->skipFlag = 1; } @@ -102702,18 +129409,34 @@ static void typeofFunc( int NotUsed, sqlite3_value **argv ){ - const char *z = 0; + static const char *azType[] = { "integer", "real", "text", "blob", "null" }; + int i = sqlite3_value_type(argv[0]) - 1; UNUSED_PARAMETER(NotUsed); - switch( sqlite3_value_type(argv[0]) ){ - case SQLITE_INTEGER: z = "integer"; break; - case SQLITE_TEXT: z = "text"; break; - case SQLITE_FLOAT: z = "real"; break; - case SQLITE_BLOB: z = "blob"; break; - default: z = "null"; break; - } - sqlite3_result_text(context, z, -1, SQLITE_STATIC); + assert( i>=0 && i=0xc0 ){ + while( (*z & 0xc0)==0x80 ){ z++; z0++; } + } + } + sqlite3_result_int(context, (int)(z-z0)); + break; + } + default: { + sqlite3_result_null(context); + break; + } + } +} + +/* +** Implementation of the octet_length() function +*/ +static void bytelengthFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + UNUSED_PARAMETER(argc); + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_BLOB: { + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + break; + } + case SQLITE_INTEGER: + case SQLITE_FLOAT: { + i64 m = sqlite3_context_db_handle(context)->enc<=SQLITE_UTF8 ? 1 : 2; + sqlite3_result_int64(context, sqlite3_value_bytes(argv[0])*m); + break; + } + case SQLITE_TEXT: { + if( sqlite3_value_encoding(argv[0])<=SQLITE_UTF8 ){ + sqlite3_result_int(context, sqlite3_value_bytes(argv[0])); + }else{ + sqlite3_result_int(context, sqlite3_value_bytes16(argv[0])); } - sqlite3_result_int(context, len); break; } default: { @@ -102756,7 +129517,7 @@ static void lengthFunc( ** Implementation of the abs() function. ** ** IMP: R-23979-26855 The abs(X) function returns the absolute value of -** the numeric argument X. +** the numeric argument X. */ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ assert( argc==1 ); @@ -102773,7 +129534,7 @@ static void absFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ return; } iVal = -iVal; - } + } sqlite3_result_int64(context, iVal); break; } @@ -102819,6 +129580,9 @@ static void instrFunc( int typeHaystack, typeNeedle; int N = 1; int isText; + unsigned char firstChar; + sqlite3_value *pC1 = 0; + sqlite3_value *pC2 = 0; UNUSED_PARAMETER(argc); typeHaystack = sqlite3_value_type(argv[0]); @@ -102826,28 +129590,51 @@ static void instrFunc( if( typeHaystack==SQLITE_NULL || typeNeedle==SQLITE_NULL ) return; nHaystack = sqlite3_value_bytes(argv[0]); nNeedle = sqlite3_value_bytes(argv[1]); - if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ - zHaystack = sqlite3_value_blob(argv[0]); - zNeedle = sqlite3_value_blob(argv[1]); - isText = 0; - }else{ - zHaystack = sqlite3_value_text(argv[0]); - zNeedle = sqlite3_value_text(argv[1]); - isText = 1; + if( nNeedle>0 ){ + if( typeHaystack==SQLITE_BLOB && typeNeedle==SQLITE_BLOB ){ + zHaystack = sqlite3_value_blob(argv[0]); + zNeedle = sqlite3_value_blob(argv[1]); + isText = 0; + }else if( typeHaystack!=SQLITE_BLOB && typeNeedle!=SQLITE_BLOB ){ + zHaystack = sqlite3_value_text(argv[0]); + zNeedle = sqlite3_value_text(argv[1]); + isText = 1; + }else{ + pC1 = sqlite3_value_dup(argv[0]); + zHaystack = sqlite3_value_text(pC1); + if( zHaystack==0 ) goto endInstrOOM; + nHaystack = sqlite3_value_bytes(pC1); + pC2 = sqlite3_value_dup(argv[1]); + zNeedle = sqlite3_value_text(pC2); + if( zNeedle==0 ) goto endInstrOOM; + nNeedle = sqlite3_value_bytes(pC2); + isText = 1; + } + if( zNeedle==0 || (nHaystack && zHaystack==0) ) goto endInstrOOM; + firstChar = zNeedle[0]; + while( nNeedle<=nHaystack + && (zHaystack[0]!=firstChar || memcmp(zHaystack, zNeedle, nNeedle)!=0) + ){ + N++; + do{ + nHaystack--; + zHaystack++; + }while( isText && (zHaystack[0]&0xc0)==0x80 ); + } + if( nNeedle>nHaystack ) N = 0; } - while( nNeedle<=nHaystack && memcmp(zHaystack, zNeedle, nNeedle)!=0 ){ - N++; - do{ - nHaystack--; - zHaystack++; - }while( isText && (zHaystack[0]&0xc0)==0x80 ); - } - if( nNeedle>nHaystack ) N = 0; sqlite3_result_int(context, N); +endInstr: + sqlite3_value_free(pC1); + sqlite3_value_free(pC2); + return; +endInstrOOM: + sqlite3_result_error_nomem(context); + goto endInstr; } /* -** Implementation of the printf() function. +** Implementation of the printf() (a.k.a. format()) SQL function. */ static void printfFunc( sqlite3_context *context, @@ -102866,7 +129653,7 @@ static void printfFunc( x.apArg = argv+1; sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); str.printfFlags = SQLITE_PRINTF_SQLFUNC; - sqlite3XPrintf(&str, zFormat, &x); + sqlite3_str_appendf(&str, zFormat, &x); n = str.nChar; sqlite3_result_text(context, sqlite3StrAccumFinish(&str), n, SQLITE_DYNAMIC); @@ -102997,12 +129784,12 @@ static void roundFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ ** handle the rounding directly, ** otherwise use printf. */ - if( n==0 && r>=0 && r+4503599627370496.0 ){ + /* The value has no fractional part so there is nothing to round */ + }else if( n==0 ){ + r = (double)((sqlite_int64)(r+(r<0?-0.5:+0.5))); }else{ - zBuf = sqlite3_mprintf("%.*f",n,r); + zBuf = sqlite3_mprintf("%!.*f",n,r); if( zBuf==0 ){ sqlite3_result_error_nomem(context); return; @@ -103092,7 +129879,7 @@ static void lowerFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ #define noopFunc versionFunc /* Substitute function - never called */ /* -** Implementation of random(). Return a random integer. +** Implementation of random(). Return a random integer. */ static void randomFunc( sqlite3_context *context, @@ -103103,11 +129890,11 @@ static void randomFunc( UNUSED_PARAMETER2(NotUsed, NotUsed2); sqlite3_randomness(sizeof(r), &r); if( r<0 ){ - /* We need to prevent a random number of 0x8000000000000000 + /* We need to prevent a random number of 0x8000000000000000 ** (or -9223372036854775808) since when you do abs() of that ** number of you get the same value back again. To do this ** in a way that is testable, mask the sign bit off of negative - ** values, resulting in a positive value. Then take the + ** values, resulting in a positive value. Then take the ** 2s complement of that positive value. The end result can ** therefore be no less than -9223372036854775807. */ @@ -103125,11 +129912,11 @@ static void randomBlob( int argc, sqlite3_value **argv ){ - int n; + sqlite3_int64 n; unsigned char *p; assert( argc==1 ); UNUSED_PARAMETER(argc); - n = sqlite3_value_int(argv[0]); + n = sqlite3_value_int64(argv[0]); if( n<1 ){ n = 1; } @@ -103145,8 +129932,8 @@ static void randomBlob( ** value is the same as the sqlite3_last_insert_rowid() API function. */ static void last_insert_rowid( - sqlite3_context *context, - int NotUsed, + sqlite3_context *context, + int NotUsed, sqlite3_value **NotUsed2 ){ sqlite3 *db = sqlite3_context_db_handle(context); @@ -103160,9 +129947,9 @@ static void last_insert_rowid( /* ** Implementation of the changes() SQL function. ** -** IMP: R-62073-11209 The changes() SQL function is a wrapper -** around the sqlite3_changes() C/C++ function and hence follows the same -** rules for counting changes. +** IMP: R-32760-32347 The changes() SQL function is a wrapper +** around the sqlite3_changes64() C/C++ function and hence follows the +** same rules for counting changes. */ static void changes( sqlite3_context *context, @@ -103171,12 +129958,12 @@ static void changes( ){ sqlite3 *db = sqlite3_context_db_handle(context); UNUSED_PARAMETER2(NotUsed, NotUsed2); - sqlite3_result_int(context, sqlite3_changes(db)); + sqlite3_result_int64(context, sqlite3_changes64(db)); } /* ** Implementation of the total_changes() SQL function. The return value is -** the same as the sqlite3_total_changes() API function. +** the same as the sqlite3_total_changes64() API function. */ static void total_changes( sqlite3_context *context, @@ -103185,9 +129972,9 @@ static void total_changes( ){ sqlite3 *db = sqlite3_context_db_handle(context); UNUSED_PARAMETER2(NotUsed, NotUsed2); - /* IMP: R-52756-41993 This function is a wrapper around the - ** sqlite3_total_changes() C/C++ interface. */ - sqlite3_result_int(context, sqlite3_total_changes(db)); + /* IMP: R-11217-42568 This function is a wrapper around the + ** sqlite3_total_changes64() C/C++ interface. */ + sqlite3_result_int64(context, sqlite3_total_changes64(db)); } /* @@ -103202,7 +129989,7 @@ struct compareInfo { /* ** For LIKE and GLOB matching on EBCDIC machines, assume that every -** character is exactly one byte in size. Also, provde the Utf8Read() +** character is exactly one byte in size. Also, provide the Utf8Read() ** macro for fast reading of the next character in the common case where ** the next character is ASCII. */ @@ -103222,9 +130009,19 @@ static const struct compareInfo likeInfoNorm = { '%', '_', 0, 1 }; static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; /* -** Compare two UTF-8 strings for equality where the first string can -** potentially be a "glob" or "like" expression. Return true (1) if they -** are the same and false (0) if they are different. +** Possible error returns from patternMatch() +*/ +#define SQLITE_MATCH 0 +#define SQLITE_NOMATCH 1 +#define SQLITE_NOWILDCARDMATCH 2 + +/* +** Compare two UTF-8 strings for equality where the first string is +** a GLOB or LIKE expression. Return values: +** +** SQLITE_MATCH: Match +** SQLITE_NOMATCH: No match +** SQLITE_NOWILDCARDMATCH: No match in spite of having * or % wildcards. ** ** Globbing rules: ** @@ -103244,7 +130041,7 @@ static const struct compareInfo likeInfoAlt = { '%', '_', 0, 0 }; ** it the last character in the list. ** ** Like matching rules: -** +** ** '%' Matches any sequence of zero or more characters ** *** '_' Matches any one character @@ -103267,75 +130064,85 @@ static int patternCompare( u32 matchAll = pInfo->matchAll; /* "*" or "%" */ u8 noCase = pInfo->noCase; /* True if uppercase==lowercase */ const u8 *zEscaped = 0; /* One past the last escaped input char */ - + while( (c = Utf8Read(zPattern))!=0 ){ if( c==matchAll ){ /* Match "*" */ /* Skip over multiple "*" characters in the pattern. If there ** are also "?" characters, skip those as well, but consume a ** single character of the input string for each "?" skipped */ - while( (c=Utf8Read(zPattern)) == matchAll || c == matchOne ){ + while( (c=Utf8Read(zPattern)) == matchAll + || (c == matchOne && matchOne!=0) ){ if( c==matchOne && sqlite3Utf8Read(&zString)==0 ){ - return 0; + return SQLITE_NOWILDCARDMATCH; } } if( c==0 ){ - return 1; /* "*" at the end of the pattern matches */ + return SQLITE_MATCH; /* "*" at the end of the pattern matches */ }else if( c==matchOther ){ if( pInfo->matchSet==0 ){ c = sqlite3Utf8Read(&zPattern); - if( c==0 ) return 0; + if( c==0 ) return SQLITE_NOWILDCARDMATCH; }else{ /* "[...]" immediately follows the "*". We have to do a slow ** recursive search in this case, but it is an unusual case. */ assert( matchOther<0x80 ); /* '[' is a single-byte character */ - while( *zString - && patternCompare(&zPattern[-1],zString,pInfo,matchOther)==0 ){ + while( *zString ){ + int bMatch = patternCompare(&zPattern[-1],zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; SQLITE_SKIP_UTF8(zString); } - return *zString!=0; + return SQLITE_NOWILDCARDMATCH; } } /* At this point variable c contains the first character of the ** pattern string past the "*". Search in the input string for the - ** first matching character and recursively contine the match from + ** first matching character and recursively continue the match from ** that point. ** ** For a case-insensitive search, set variable cx to be the same as ** c but in the other case and search the input string for either ** c or cx. */ - if( c<=0x80 ){ - u32 cx; + if( c<0x80 ){ + char zStop[3]; + int bMatch; if( noCase ){ - cx = sqlite3Toupper(c); - c = sqlite3Tolower(c); + zStop[0] = sqlite3Toupper(c); + zStop[1] = sqlite3Tolower(c); + zStop[2] = 0; }else{ - cx = c; + zStop[0] = c; + zStop[1] = 0; } - while( (c2 = *(zString++))!=0 ){ - if( c2!=c && c2!=cx ) continue; - if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + while(1){ + zString += strcspn((const char*)zString, zStop); + if( zString[0]==0 ) break; + zString++; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; } }else{ + int bMatch; while( (c2 = Utf8Read(zString))!=0 ){ if( c2!=c ) continue; - if( patternCompare(zPattern,zString,pInfo,matchOther) ) return 1; + bMatch = patternCompare(zPattern,zString,pInfo,matchOther); + if( bMatch!=SQLITE_NOMATCH ) return bMatch; } } - return 0; + return SQLITE_NOWILDCARDMATCH; } if( c==matchOther ){ if( pInfo->matchSet==0 ){ c = sqlite3Utf8Read(&zPattern); - if( c==0 ) return 0; + if( c==0 ) return SQLITE_NOMATCH; zEscaped = zPattern; }else{ u32 prior_c = 0; int seen = 0; int invert = 0; c = sqlite3Utf8Read(&zString); - if( c==0 ) return 0; + if( c==0 ) return SQLITE_NOMATCH; c2 = sqlite3Utf8Read(&zPattern); if( c2=='^' ){ invert = 1; @@ -103359,34 +130166,48 @@ static int patternCompare( c2 = sqlite3Utf8Read(&zPattern); } if( c2==0 || (seen ^ invert)==0 ){ - return 0; + return SQLITE_NOMATCH; } continue; } } c2 = Utf8Read(zString); if( c==c2 ) continue; - if( noCase && c<0x80 && c2<0x80 && sqlite3Tolower(c)==sqlite3Tolower(c2) ){ + if( noCase && sqlite3Tolower(c)==sqlite3Tolower(c2) && c<0x80 && c2<0x80 ){ continue; } if( c==matchOne && zPattern!=zEscaped && c2!=0 ) continue; - return 0; + return SQLITE_NOMATCH; } - return *zString==0; + return *zString==0 ? SQLITE_MATCH : SQLITE_NOMATCH; } /* -** The sqlite3_strglob() interface. +** The sqlite3_strglob() interface. Return 0 on a match (like strcmp()) and +** non-zero if there is no match. */ -SQLITE_API int SQLITE_STDCALL sqlite3_strglob(const char *zGlobPattern, const char *zString){ - return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '[')==0; +SQLITE_API int sqlite3_strglob(const char *zGlobPattern, const char *zString){ + if( zString==0 ){ + return zGlobPattern!=0; + }else if( zGlobPattern==0 ){ + return 1; + }else { + return patternCompare((u8*)zGlobPattern, (u8*)zString, &globInfo, '['); + } } /* -** The sqlite3_strlike() interface. +** The sqlite3_strlike() interface. Return 0 on a match and non-zero for +** a miss - like strcmp(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ - return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc)==0; +SQLITE_API int sqlite3_strlike(const char *zPattern, const char *zStr, unsigned int esc){ + if( zStr==0 ){ + return zPattern!=0; + }else if( zPattern==0 ){ + return 1; + }else{ + return patternCompare((u8*)zPattern, (u8*)zStr, &likeInfoNorm, esc); + } } /* @@ -103401,7 +130222,7 @@ SQLITE_API int sqlite3_like_count = 0; /* ** Implementation of the like() SQL function. This function implements -** the build-in LIKE operator. The first argument to the function is the +** the built-in LIKE operator. The first argument to the function is the ** pattern and the second argument is the string. So, the SQL statements: ** ** A LIKE B @@ -103412,8 +130233,8 @@ SQLITE_API int sqlite3_like_count = 0; ** the GLOB operator. */ static void likeFunc( - sqlite3_context *context, - int argc, + sqlite3_context *context, + int argc, sqlite3_value **argv ){ const unsigned char *zA, *zB; @@ -103421,6 +130242,7 @@ static void likeFunc( int nPat; sqlite3 *db = sqlite3_context_db_handle(context); struct compareInfo *pInfo = sqlite3_user_data(context); + struct compareInfo backupInfo; #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( sqlite3_value_type(argv[0])==SQLITE_BLOB @@ -103433,8 +130255,6 @@ static void likeFunc( return; } #endif - zB = sqlite3_value_text(argv[0]); - zA = sqlite3_value_text(argv[1]); /* Limit the length of the LIKE or GLOB pattern to avoid problems ** of deep recursion and N*N behavior in patternCompare(). @@ -103446,8 +130266,6 @@ static void likeFunc( sqlite3_result_error(context, "LIKE or GLOB pattern too complex", -1); return; } - assert( zB==sqlite3_value_text(argv[0]) ); /* Encoding did not change */ - if( argc==3 ){ /* The escape character string must consist of a single UTF-8 character. ** Otherwise, return an error. @@ -103455,19 +130273,28 @@ static void likeFunc( const unsigned char *zEsc = sqlite3_value_text(argv[2]); if( zEsc==0 ) return; if( sqlite3Utf8CharLen((char*)zEsc, -1)!=1 ){ - sqlite3_result_error(context, + sqlite3_result_error(context, "ESCAPE expression must be a single character", -1); return; } escape = sqlite3Utf8Read(&zEsc); + if( escape==pInfo->matchAll || escape==pInfo->matchOne ){ + memcpy(&backupInfo, pInfo, sizeof(backupInfo)); + pInfo = &backupInfo; + if( escape==pInfo->matchAll ) pInfo->matchAll = 0; + if( escape==pInfo->matchOne ) pInfo->matchOne = 0; + } }else{ escape = pInfo->matchSet; } + zB = sqlite3_value_text(argv[0]); + zA = sqlite3_value_text(argv[1]); if( zA && zB ){ #ifdef SQLITE_TEST sqlite3_like_count++; #endif - sqlite3_result_int(context, patternCompare(zB, zA, pInfo, escape)); + sqlite3_result_int(context, + patternCompare(zB, zA, pInfo, escape)==SQLITE_MATCH); } } @@ -103559,8 +130386,8 @@ static void compileoptionusedFunc( #endif /* SQLITE_OMIT_COMPILEOPTION_DIAGS */ /* -** Implementation of the sqlite_compileoption_get() function. -** The result is a string that identifies the compiler options +** Implementation of the sqlite_compileoption_get() function. +** The result is a string that identifies the compiler options ** used to build SQLite. */ #ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS @@ -103584,43 +130411,46 @@ static void compileoptiongetFunc( ** digits. */ static const char hexdigits[] = { '0', '1', '2', '3', '4', '5', '6', '7', - '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' }; /* -** Implementation of the QUOTE() function. This function takes a single -** argument. If the argument is numeric, the return value is the same as -** the argument. If the argument is NULL, the return value is the string -** "NULL". Otherwise, the argument is enclosed in single quotes with -** single-quote escapes. +** Append to pStr text that is the SQL literal representation of the +** value contained in pValue. */ -static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ - assert( argc==1 ); - UNUSED_PARAMETER(argc); - switch( sqlite3_value_type(argv[0]) ){ +SQLITE_PRIVATE void sqlite3QuoteValue(StrAccum *pStr, sqlite3_value *pValue){ + /* As currently implemented, the string must be initially empty. + ** we might relax this requirement in the future, but that will + ** require enhancements to the implementation. */ + assert( pStr!=0 && pStr->nChar==0 ); + + switch( sqlite3_value_type(pValue) ){ case SQLITE_FLOAT: { double r1, r2; - char zBuf[50]; - r1 = sqlite3_value_double(argv[0]); - sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.15g", r1); - sqlite3AtoF(zBuf, &r2, 20, SQLITE_UTF8); - if( r1!=r2 ){ - sqlite3_snprintf(sizeof(zBuf), zBuf, "%!.20e", r1); + const char *zVal; + r1 = sqlite3_value_double(pValue); + sqlite3_str_appendf(pStr, "%!0.15g", r1); + zVal = sqlite3_str_value(pStr); + if( zVal ){ + sqlite3AtoF(zVal, &r2, pStr->nChar, SQLITE_UTF8); + if( r1!=r2 ){ + sqlite3_str_reset(pStr); + sqlite3_str_appendf(pStr, "%!0.20e", r1); + } } - sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); break; } case SQLITE_INTEGER: { - sqlite3_result_value(context, argv[0]); + sqlite3_str_appendf(pStr, "%lld", sqlite3_value_int64(pValue)); break; } case SQLITE_BLOB: { - char *zText = 0; - char const *zBlob = sqlite3_value_blob(argv[0]); - int nBlob = sqlite3_value_bytes(argv[0]); - assert( zBlob==sqlite3_value_blob(argv[0]) ); /* No encoding change */ - zText = (char *)contextMalloc(context, (2*(i64)nBlob)+4); - if( zText ){ + char const *zBlob = sqlite3_value_blob(pValue); + i64 nBlob = sqlite3_value_bytes(pValue); + assert( zBlob==sqlite3_value_blob(pValue) ); /* No encoding change */ + sqlite3StrAccumEnlarge(pStr, nBlob*2 + 4); + if( pStr->accError==0 ){ + char *zText = pStr->zText; int i; for(i=0; i>4)&0x0F]; @@ -103630,45 +130460,51 @@ static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ zText[(nBlob*2)+3] = '\0'; zText[0] = 'X'; zText[1] = '\''; - sqlite3_result_text(context, zText, -1, SQLITE_TRANSIENT); - sqlite3_free(zText); + pStr->nChar = nBlob*2 + 3; } break; } case SQLITE_TEXT: { - int i,j; - u64 n; - const unsigned char *zArg = sqlite3_value_text(argv[0]); - char *z; - - if( zArg==0 ) return; - for(i=0, n=0; zArg[i]; i++){ if( zArg[i]=='\'' ) n++; } - z = contextMalloc(context, ((i64)i)+((i64)n)+3); - if( z ){ - z[0] = '\''; - for(i=0, j=1; zArg[i]; i++){ - z[j++] = zArg[i]; - if( zArg[i]=='\'' ){ - z[j++] = '\''; - } - } - z[j++] = '\''; - z[j] = 0; - sqlite3_result_text(context, z, j, sqlite3_free); - } + const unsigned char *zArg = sqlite3_value_text(pValue); + sqlite3_str_appendf(pStr, "%Q", zArg); break; } default: { - assert( sqlite3_value_type(argv[0])==SQLITE_NULL ); - sqlite3_result_text(context, "NULL", 4, SQLITE_STATIC); + assert( sqlite3_value_type(pValue)==SQLITE_NULL ); + sqlite3_str_append(pStr, "NULL", 4); break; } } } +/* +** Implementation of the QUOTE() function. +** +** The quote(X) function returns the text of an SQL literal which is the +** value of its argument suitable for inclusion into an SQL statement. +** Strings are surrounded by single-quotes with escapes on interior quotes +** as needed. BLOBs are encoded as hexadecimal literals. Strings with +** embedded NUL characters cannot be represented as string literals in SQL +** and hence the returned string literal is truncated prior to the first NUL. +*/ +static void quoteFunc(sqlite3_context *context, int argc, sqlite3_value **argv){ + sqlite3_str str; + sqlite3 *db = sqlite3_context_db_handle(context); + assert( argc==1 ); + UNUSED_PARAMETER(argc); + sqlite3StrAccumInit(&str, db, 0, 0, db->aLimit[SQLITE_LIMIT_LENGTH]); + sqlite3QuoteValue(&str,argv[0]); + sqlite3_result_text(context, sqlite3StrAccumFinish(&str), str.nChar, + SQLITE_DYNAMIC); + if( str.accError!=SQLITE_OK ){ + sqlite3_result_null(context); + sqlite3_result_error_code(context, str.accError); + } +} + /* ** The unicode() function. Return the integer unicode code-point value -** for the first character of the input string. +** for the first character of the input string. */ static void unicodeFunc( sqlite3_context *context, @@ -103719,6 +130555,7 @@ static void charFunc( *zOut++ = 0x80 + (u8)(c & 0x3F); } \ } + *zOut = 0; sqlite3_result_text64(context, (char*)z, zOut-z, sqlite3_free, SQLITE_UTF8); } @@ -103747,10 +130584,101 @@ static void hexFunc( *(z++) = hexdigits[c&0xf]; } *z = 0; - sqlite3_result_text(context, zHex, n*2, sqlite3_free); + sqlite3_result_text64(context, zHex, (u64)(z-zHex), + sqlite3_free, SQLITE_UTF8); } } +/* +** Buffer zStr contains nStr bytes of utf-8 encoded text. Return 1 if zStr +** contains character ch, or 0 if it does not. +*/ +static int strContainsChar(const u8 *zStr, int nStr, u32 ch){ + const u8 *zEnd = &zStr[nStr]; + const u8 *z = zStr; + while( zaLimit[SQLITE_LIMIT_LENGTH] ); - testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); - if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ - sqlite3_result_error_toobig(context); - sqlite3_free(zOut); - return; - } - zOld = zOut; - zOut = sqlite3_realloc64(zOut, (int)nOut); - if( zOut==0 ){ - sqlite3_result_error_nomem(context); - sqlite3_free(zOld); - return; + if( nRep>nPattern ){ + nOut += nRep - nPattern; + testcase( nOut-1==db->aLimit[SQLITE_LIMIT_LENGTH] ); + testcase( nOut-2==db->aLimit[SQLITE_LIMIT_LENGTH] ); + if( nOut-1>db->aLimit[SQLITE_LIMIT_LENGTH] ){ + sqlite3_result_error_toobig(context); + sqlite3_free(zOut); + return; + } + cntExpand++; + if( (cntExpand&(cntExpand-1))==0 ){ + /* Grow the size of the output buffer only on substitutions + ** whose index is a power of two: 1, 2, 4, 8, 16, 32, ... */ + u8 *zOld; + zOld = zOut; + zOut = sqlite3Realloc(zOut, (int)nOut + (nOut - nStr - 1)); + if( zOut==0 ){ + sqlite3_result_error_nomem(context); + sqlite3_free(zOld); + return; + } + } } memcpy(&zOut[j], zRep, nRep); j += nRep; i += nPattern-1; } } - assert( j+nStr-i+1==nOut ); + assert( j+nStr-i+1<=nOut ); memcpy(&zOut[j], &zStr[i], nStr-i); j += nStr - i; assert( j<=nOut ); @@ -103868,10 +130805,10 @@ static void trimFunc( ){ const unsigned char *zIn; /* Input string */ const unsigned char *zCharSet; /* Set of characters to trim */ - int nIn; /* Number of bytes in input */ + unsigned int nIn; /* Number of bytes in input */ int flags; /* 1: trimleft 2: trimright 3: trim */ int i; /* Loop counter */ - unsigned char *aLen = 0; /* Length of each character in zCharSet */ + unsigned int *aLen = 0; /* Length of each character in zCharSet */ unsigned char **azChar = 0; /* Individual characters in zCharSet */ int nChar; /* Number of characters in zCharSet */ @@ -103880,13 +130817,13 @@ static void trimFunc( } zIn = sqlite3_value_text(argv[0]); if( zIn==0 ) return; - nIn = sqlite3_value_bytes(argv[0]); + nIn = (unsigned)sqlite3_value_bytes(argv[0]); assert( zIn==sqlite3_value_text(argv[0]) ); if( argc==1 ){ - static const unsigned char lenOne[] = { 1 }; + static const unsigned lenOne[] = { 1 }; static unsigned char * const azOne[] = { (u8*)" " }; nChar = 1; - aLen = (u8*)lenOne; + aLen = (unsigned*)lenOne; azChar = (unsigned char **)azOne; zCharSet = 0; }else if( (zCharSet = sqlite3_value_text(argv[1]))==0 ){ @@ -103897,15 +130834,16 @@ static void trimFunc( SQLITE_SKIP_UTF8(z); } if( nChar>0 ){ - azChar = contextMalloc(context, ((i64)nChar)*(sizeof(char*)+1)); + azChar = contextMalloc(context, + ((i64)nChar)*(sizeof(char*)+sizeof(unsigned))); if( azChar==0 ){ return; } - aLen = (unsigned char*)&azChar[nChar]; + aLen = (unsigned*)&azChar[nChar]; for(z=zCharSet, nChar=0; *z; nChar++){ azChar[nChar] = (unsigned char *)z; SQLITE_SKIP_UTF8(z); - aLen[nChar] = (u8)(z - azChar[nChar]); + aLen[nChar] = (unsigned)(z - azChar[nChar]); } } } @@ -103913,7 +130851,7 @@ static void trimFunc( flags = SQLITE_PTR_TO_INT(sqlite3_user_data(context)); if( flags & 1 ){ while( nIn>0 ){ - int len = 0; + unsigned int len = 0; for(i=0; i0 ){ - int len = 0; + unsigned int len = 0; for(i=0; i0 ){ + const char *v = (const char*)sqlite3_value_text(argv[i]); + if( v!=0 ){ + if( j>0 && nSep>0 ){ + memcpy(&z[j], zSep, nSep); + j += nSep; + } + memcpy(&z[j], v, k); + j += k; + } + } + } + z[j] = 0; + assert( j<=n ); + sqlite3_result_text64(context, z, j, sqlite3_free, SQLITE_UTF8); +} + +/* +** The CONCAT(...) function. Generate a string result that is the +** concatentation of all non-null arguments. +*/ +static void concatFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + concatFuncCore(context, argc, argv, 0, ""); +} + +/* +** The CONCAT_WS(separator, ...) function. +** +** Generate a string that is the concatenation of 2nd through the Nth +** argument. Use the first argument (which must be non-NULL) as the +** separator. +*/ +static void concatwsFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int nSep = sqlite3_value_bytes(argv[0]); + const char *zSep = (const char*)sqlite3_value_text(argv[0]); + if( zSep==0 ) return; + concatFuncCore(context, argc-1, argv+1, nSep, zSep); +} + + +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION +/* +** The "unknown" function is automatically substituted in place of +** any unrecognized function name when doing an EXPLAIN or EXPLAIN QUERY PLAN +** when the SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION compile-time option is used. +** When the "sqlite3" command-line shell is built using this functionality, +** that allows an EXPLAIN or EXPLAIN QUERY PLAN for complex queries +** involving application-defined functions to be examined in a generic +** sqlite3 shell. +*/ +static void unknownFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + /* no-op */ + (void)context; + (void)argc; + (void)argv; +} +#endif /*SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION*/ + /* IMP: R-25361-16150 This function is omitted from SQLite by default. It ** is only available if the SQLITE_SOUNDEX compile-time option is used @@ -103951,7 +130987,7 @@ static void trimFunc( ** Compute the soundex encoding of a word. ** ** IMP: R-59782-00072 The soundex(X) function returns a string that is the -** soundex encoding of the string X. +** soundex encoding of the string X. */ static void soundexFunc( sqlite3_context *context, @@ -104039,13 +131075,68 @@ static void loadExt(sqlite3_context *context, int argc, sqlite3_value **argv){ */ typedef struct SumCtx SumCtx; struct SumCtx { - double rSum; /* Floating point sum */ - i64 iSum; /* Integer sum */ + double rSum; /* Running sum as as a double */ + double rErr; /* Error term for Kahan-Babushka-Neumaier summation */ + i64 iSum; /* Running sum as a signed integer */ i64 cnt; /* Number of elements summed */ - u8 overflow; /* True if integer overflow seen */ - u8 approx; /* True if non-integer value was input to the sum */ + u8 approx; /* True if any non-integer value was input to the sum */ + u8 ovrfl; /* Integer overflow seen */ }; +/* +** Do one step of the Kahan-Babushka-Neumaier summation. +** +** https://en.wikipedia.org/wiki/Kahan_summation_algorithm +** +** Variables are marked "volatile" to defeat c89 x86 floating point +** optimizations can mess up this algorithm. +*/ +static void kahanBabuskaNeumaierStep( + volatile SumCtx *pSum, + volatile double r +){ + volatile double s = pSum->rSum; + volatile double t = s + r; + if( fabs(s) > fabs(r) ){ + pSum->rErr += (s - t) + r; + }else{ + pSum->rErr += (r - t) + s; + } + pSum->rSum = t; +} + +/* +** Add a (possibly large) integer to the running sum. +*/ +static void kahanBabuskaNeumaierStepInt64(volatile SumCtx *pSum, i64 iVal){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iBig, iSm; + iSm = iVal % 16384; + iBig = iVal - iSm; + kahanBabuskaNeumaierStep(pSum, iBig); + kahanBabuskaNeumaierStep(pSum, iSm); + }else{ + kahanBabuskaNeumaierStep(pSum, (double)iVal); + } +} + +/* +** Initialize the Kahan-Babaska-Neumaier sum from a 64-bit integer +*/ +static void kahanBabuskaNeumaierInit( + volatile SumCtx *p, + i64 iVal +){ + if( iVal<=-4503599627370496LL || iVal>=+4503599627370496LL ){ + i64 iSm = iVal % 16384; + p->rSum = (double)(iVal - iSm); + p->rErr = (double)iSm; + }else{ + p->rSum = (double)iVal; + p->rErr = 0.0; + } +} + /* ** Routines used to compute the sum, average, and total. ** @@ -104065,26 +131156,75 @@ static void sumStep(sqlite3_context *context, int argc, sqlite3_value **argv){ type = sqlite3_value_numeric_type(argv[0]); if( p && type!=SQLITE_NULL ){ p->cnt++; - if( type==SQLITE_INTEGER ){ - i64 v = sqlite3_value_int64(argv[0]); - p->rSum += v; - if( (p->approx|p->overflow)==0 && sqlite3AddInt64(&p->iSum, v) ){ - p->overflow = 1; + if( p->approx==0 ){ + if( type!=SQLITE_INTEGER ){ + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + }else{ + i64 x = p->iSum; + if( sqlite3AddInt64(&x, sqlite3_value_int64(argv[0]))==0 ){ + p->iSum = x; + }else{ + p->ovrfl = 1; + kahanBabuskaNeumaierInit(p, p->iSum); + p->approx = 1; + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + } } }else{ - p->rSum += sqlite3_value_double(argv[0]); - p->approx = 1; + if( type==SQLITE_INTEGER ){ + kahanBabuskaNeumaierStepInt64(p, sqlite3_value_int64(argv[0])); + }else{ + p->ovrfl = 0; + kahanBabuskaNeumaierStep(p, sqlite3_value_double(argv[0])); + } } } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void sumInverse(sqlite3_context *context, int argc, sqlite3_value**argv){ + SumCtx *p; + int type; + assert( argc==1 ); + UNUSED_PARAMETER(argc); + p = sqlite3_aggregate_context(context, sizeof(*p)); + type = sqlite3_value_numeric_type(argv[0]); + /* p is always non-NULL because sumStep() will have been called first + ** to initialize it */ + if( ALWAYS(p) && type!=SQLITE_NULL ){ + assert( p->cnt>0 ); + p->cnt--; + if( !p->approx ){ + p->iSum -= sqlite3_value_int64(argv[0]); + }else if( type==SQLITE_INTEGER ){ + i64 iVal = sqlite3_value_int64(argv[0]); + if( iVal!=SMALLEST_INT64 ){ + kahanBabuskaNeumaierStepInt64(p, -iVal); + }else{ + kahanBabuskaNeumaierStepInt64(p, LARGEST_INT64); + kahanBabuskaNeumaierStepInt64(p, 1); + } + }else{ + kahanBabuskaNeumaierStep(p, -sqlite3_value_double(argv[0])); + } + } +} +#else +# define sumInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ static void sumFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - if( p->overflow ){ - sqlite3_result_error(context,"integer overflow",-1); - }else if( p->approx ){ - sqlite3_result_double(context, p->rSum); + if( p->approx ){ + if( p->ovrfl ){ + sqlite3_result_error(context,"integer overflow",-1); + }else if( !sqlite3IsOverflow(p->rErr) ){ + sqlite3_result_double(context, p->rSum+p->rErr); + }else{ + sqlite3_result_double(context, p->rSum); + } }else{ sqlite3_result_int64(context, p->iSum); } @@ -104094,14 +131234,29 @@ static void avgFinalize(sqlite3_context *context){ SumCtx *p; p = sqlite3_aggregate_context(context, 0); if( p && p->cnt>0 ){ - sqlite3_result_double(context, p->rSum/(double)p->cnt); + double r; + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + sqlite3_result_double(context, r/(double)p->cnt); } } static void totalFinalize(sqlite3_context *context){ SumCtx *p; + double r = 0.0; p = sqlite3_aggregate_context(context, 0); - /* (double)0 In case of SQLITE_OMIT_FLOATING_POINT... */ - sqlite3_result_double(context, p ? p->rSum : (double)0); + if( p ){ + if( p->approx ){ + r = p->rSum; + if( !sqlite3IsOverflow(p->rErr) ) r += p->rErr; + }else{ + r = (double)(p->iSum); + } + } + sqlite3_result_double(context, r); } /* @@ -104111,6 +131266,9 @@ static void totalFinalize(sqlite3_context *context){ typedef struct CountCtx CountCtx; struct CountCtx { i64 n; +#ifdef SQLITE_DEBUG + int bInverse; /* True if xInverse() ever called */ +#endif }; /* @@ -104125,25 +131283,40 @@ static void countStep(sqlite3_context *context, int argc, sqlite3_value **argv){ #ifndef SQLITE_OMIT_DEPRECATED /* The sqlite3_aggregate_count() function is deprecated. But just to make - ** sure it still operates correctly, verify that its count agrees with our + ** sure it still operates correctly, verify that its count agrees with our ** internal count when using count(*) and when the total count can be ** expressed as a 32-bit integer. */ - assert( argc==1 || p==0 || p->n>0x7fffffff + assert( argc==1 || p==0 || p->n>0x7fffffff || p->bInverse || p->n==sqlite3_aggregate_count(context) ); #endif -} +} static void countFinalize(sqlite3_context *context){ CountCtx *p; p = sqlite3_aggregate_context(context, 0); sqlite3_result_int64(context, p ? p->n : 0); } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void countInverse(sqlite3_context *ctx, int argc, sqlite3_value **argv){ + CountCtx *p; + p = sqlite3_aggregate_context(ctx, sizeof(*p)); + /* p is always non-NULL since countStep() will have been called first */ + if( (argc==0 || SQLITE_NULL!=sqlite3_value_type(argv[0])) && ALWAYS(p) ){ + p->n--; +#ifdef SQLITE_DEBUG + p->bInverse = 1; +#endif + } +} +#else +# define countInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** Routines to implement min() and max() aggregate functions. */ static void minmaxStep( - sqlite3_context *context, - int NotUsed, + sqlite3_context *context, + int NotUsed, sqlite3_value **argv ){ Mem *pArg = (Mem *)argv[0]; @@ -104153,7 +131326,7 @@ static void minmaxStep( pBest = (Mem *)sqlite3_aggregate_context(context, sizeof(*pBest)); if( !pBest ) return; - if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + if( sqlite3_value_type(pArg)==SQLITE_NULL ){ if( pBest->flags ) sqlite3SkipAccumulatorLoad(context); }else if( pBest->flags ){ int max; @@ -104179,66 +131352,203 @@ static void minmaxStep( sqlite3VdbeMemCopy(pBest, pArg); } } -static void minMaxFinalize(sqlite3_context *context){ +static void minMaxValueFinalize(sqlite3_context *context, int bValue){ sqlite3_value *pRes; pRes = (sqlite3_value *)sqlite3_aggregate_context(context, 0); if( pRes ){ if( pRes->flags ){ sqlite3_result_value(context, pRes); } - sqlite3VdbeMemRelease(pRes); + if( bValue==0 ) sqlite3VdbeMemRelease(pRes); } } +#ifndef SQLITE_OMIT_WINDOWFUNC +static void minMaxValue(sqlite3_context *context){ + minMaxValueFinalize(context, 1); +} +#else +# define minMaxValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void minMaxFinalize(sqlite3_context *context){ + minMaxValueFinalize(context, 0); +} /* ** group_concat(EXPR, ?SEPARATOR?) +** string_agg(EXPR, SEPARATOR) +** +** Content is accumulated in GroupConcatCtx.str with the SEPARATOR +** coming before the EXPR value, except for the first entry which +** omits the SEPARATOR. +** +** It is tragic that the SEPARATOR goes before the EXPR string. The +** groupConcatInverse() implementation would have been easier if the +** SEPARATOR were appended after EXPR. And the order is undocumented, +** so we could change it, in theory. But the old behavior has been +** around for so long that we dare not, for fear of breaking something. */ +typedef struct { + StrAccum str; /* The accumulated concatenation */ +#ifndef SQLITE_OMIT_WINDOWFUNC + int nAccum; /* Number of strings presently concatenated */ + int nFirstSepLength; /* Used to detect separator length change */ + /* If pnSepLengths!=0, refs an array of inter-string separator lengths, + ** stored as actually incorporated into presently accumulated result. + ** (Hence, its slots in use number nAccum-1 between method calls.) + ** If pnSepLengths==0, nFirstSepLength is the length used throughout. + */ + int *pnSepLengths; +#endif +} GroupConcatCtx; + static void groupConcatStep( sqlite3_context *context, int argc, sqlite3_value **argv ){ const char *zVal; - StrAccum *pAccum; + GroupConcatCtx *pGCC; const char *zSep; int nVal, nSep; assert( argc==1 || argc==2 ); if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; - pAccum = (StrAccum*)sqlite3_aggregate_context(context, sizeof(*pAccum)); - - if( pAccum ){ + pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); + if( pGCC ){ sqlite3 *db = sqlite3_context_db_handle(context); - int firstTerm = pAccum->mxAlloc==0; - pAccum->mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; - if( !firstTerm ){ - if( argc==2 ){ - zSep = (char*)sqlite3_value_text(argv[1]); - nSep = sqlite3_value_bytes(argv[1]); - }else{ - zSep = ","; - nSep = 1; + int firstTerm = pGCC->str.mxAlloc==0; + pGCC->str.mxAlloc = db->aLimit[SQLITE_LIMIT_LENGTH]; + if( argc==1 ){ + if( !firstTerm ){ + sqlite3_str_appendchar(&pGCC->str, 1, ','); } - if( nSep ) sqlite3StrAccumAppend(pAccum, zSep, nSep); +#ifndef SQLITE_OMIT_WINDOWFUNC + else{ + pGCC->nFirstSepLength = 1; + } +#endif + }else if( !firstTerm ){ + zSep = (char*)sqlite3_value_text(argv[1]); + nSep = sqlite3_value_bytes(argv[1]); + if( zSep ){ + sqlite3_str_append(&pGCC->str, zSep, nSep); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + else{ + nSep = 0; + } + if( nSep != pGCC->nFirstSepLength || pGCC->pnSepLengths != 0 ){ + int *pnsl = pGCC->pnSepLengths; + if( pnsl == 0 ){ + /* First separator length variation seen, start tracking them. */ + pnsl = (int*)sqlite3_malloc64((pGCC->nAccum+1) * sizeof(int)); + if( pnsl!=0 ){ + int i = 0, nA = pGCC->nAccum-1; + while( inFirstSepLength; + } + }else{ + pnsl = (int*)sqlite3_realloc64(pnsl, pGCC->nAccum * sizeof(int)); + } + if( pnsl!=0 ){ + if( ALWAYS(pGCC->nAccum>0) ){ + pnsl[pGCC->nAccum-1] = nSep; + } + pGCC->pnSepLengths = pnsl; + }else{ + sqlite3StrAccumSetError(&pGCC->str, SQLITE_NOMEM); + } + } +#endif } +#ifndef SQLITE_OMIT_WINDOWFUNC + else{ + pGCC->nFirstSepLength = sqlite3_value_bytes(argv[1]); + } + pGCC->nAccum += 1; +#endif zVal = (char*)sqlite3_value_text(argv[0]); nVal = sqlite3_value_bytes(argv[0]); - if( zVal ) sqlite3StrAccumAppend(pAccum, zVal, nVal); + if( zVal ) sqlite3_str_append(&pGCC->str, zVal, nVal); } } -static void groupConcatFinalize(sqlite3_context *context){ - StrAccum *pAccum; - pAccum = sqlite3_aggregate_context(context, 0); - if( pAccum ){ - if( pAccum->accError==STRACCUM_TOOBIG ){ - sqlite3_result_error_toobig(context); - }else if( pAccum->accError==STRACCUM_NOMEM ){ - sqlite3_result_error_nomem(context); - }else{ - sqlite3_result_text(context, sqlite3StrAccumFinish(pAccum), -1, - sqlite3_free); + +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatInverse( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GroupConcatCtx *pGCC; + assert( argc==1 || argc==2 ); + (void)argc; /* Suppress unused parameter warning */ + if( sqlite3_value_type(argv[0])==SQLITE_NULL ) return; + pGCC = (GroupConcatCtx*)sqlite3_aggregate_context(context, sizeof(*pGCC)); + /* pGCC is always non-NULL since groupConcatStep() will have always + ** run first to initialize it */ + if( ALWAYS(pGCC) ){ + int nVS; /* Number of characters to remove */ + /* Must call sqlite3_value_text() to convert the argument into text prior + ** to invoking sqlite3_value_bytes(), in case the text encoding is UTF16 */ + (void)sqlite3_value_text(argv[0]); + nVS = sqlite3_value_bytes(argv[0]); + pGCC->nAccum -= 1; + if( pGCC->pnSepLengths!=0 ){ + assert(pGCC->nAccum >= 0); + if( pGCC->nAccum>0 ){ + nVS += *pGCC->pnSepLengths; + memmove(pGCC->pnSepLengths, pGCC->pnSepLengths+1, + (pGCC->nAccum-1)*sizeof(int)); + } + }else{ + /* If removing single accumulated string, harmlessly over-do. */ + nVS += pGCC->nFirstSepLength; + } + if( nVS>=(int)pGCC->str.nChar ){ + pGCC->str.nChar = 0; + }else{ + pGCC->str.nChar -= nVS; + memmove(pGCC->str.zText, &pGCC->str.zText[nVS], pGCC->str.nChar); + } + if( pGCC->str.nChar==0 ){ + pGCC->str.mxAlloc = 0; + sqlite3_free(pGCC->pnSepLengths); + pGCC->pnSepLengths = 0; } } } +#else +# define groupConcatInverse 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ +static void groupConcatFinalize(sqlite3_context *context){ + GroupConcatCtx *pGCC + = (GroupConcatCtx*)sqlite3_aggregate_context(context, 0); + if( pGCC ){ + sqlite3ResultStrAccum(context, &pGCC->str); +#ifndef SQLITE_OMIT_WINDOWFUNC + sqlite3_free(pGCC->pnSepLengths); +#endif + } +} +#ifndef SQLITE_OMIT_WINDOWFUNC +static void groupConcatValue(sqlite3_context *context){ + GroupConcatCtx *pGCC + = (GroupConcatCtx*)sqlite3_aggregate_context(context, 0); + if( pGCC ){ + StrAccum *pAccum = &pGCC->str; + if( pAccum->accError==SQLITE_TOOBIG ){ + sqlite3_result_error_toobig(context); + }else if( pAccum->accError==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else if( pGCC->nAccum>0 && pAccum->nChar==0 ){ + sqlite3_result_text(context, "", 1, SQLITE_STATIC); + }else{ + const char *zText = sqlite3_str_value(pAccum); + sqlite3_result_text(context, zText, pAccum->nChar, SQLITE_TRANSIENT); + } + } +} +#else +# define groupConcatValue 0 +#endif /* SQLITE_OMIT_WINDOWFUNC */ /* ** This routine does per-connection function registration. Most @@ -104254,43 +131564,42 @@ SQLITE_PRIVATE void sqlite3RegisterPerConnectionBuiltinFunctions(sqlite3 *db){ } /* -** Set the LIKEOPT flag on the 2-argument function with the given name. -*/ -static void setLikeOptFlag(sqlite3 *db, const char *zName, u8 flagVal){ - FuncDef *pDef; - pDef = sqlite3FindFunction(db, zName, 2, SQLITE_UTF8, 0); - if( ALWAYS(pDef) ){ - pDef->funcFlags |= flagVal; - } -} - -/* -** Register the built-in LIKE and GLOB functions. The caseSensitive +** Re-register the built-in LIKE functions. The caseSensitive ** parameter determines whether or not the LIKE operator is case -** sensitive. GLOB is always case sensitive. +** sensitive. */ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive){ + FuncDef *pDef; struct compareInfo *pInfo; + int flags; + int nArg; if( caseSensitive ){ pInfo = (struct compareInfo*)&likeInfoAlt; + flags = SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE; }else{ pInfo = (struct compareInfo*)&likeInfoNorm; + flags = SQLITE_FUNC_LIKE; + } + for(nArg=2; nArg<=3; nArg++){ + sqlite3CreateFunc(db, "like", nArg, SQLITE_UTF8, pInfo, likeFunc, + 0, 0, 0, 0, 0); + pDef = sqlite3FindFunction(db, "like", nArg, SQLITE_UTF8, 0); + pDef->funcFlags |= flags; + pDef->funcFlags &= ~SQLITE_FUNC_UNSAFE; } - sqlite3CreateFunc(db, "like", 2, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "like", 3, SQLITE_UTF8, pInfo, likeFunc, 0, 0, 0); - sqlite3CreateFunc(db, "glob", 2, SQLITE_UTF8, - (struct compareInfo*)&globInfo, likeFunc, 0, 0, 0); - setLikeOptFlag(db, "glob", SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE); - setLikeOptFlag(db, "like", - caseSensitive ? (SQLITE_FUNC_LIKE | SQLITE_FUNC_CASE) : SQLITE_FUNC_LIKE); } /* ** pExpr points to an expression which implements a function. If ** it is appropriate to apply the LIKE optimization to that function -** then set aWc[0] through aWc[2] to the wildcard characters and -** return TRUE. If the function is not a LIKE-style function then -** return FALSE. +** then set aWc[0] through aWc[2] to the wildcard characters and the +** escape character and then return TRUE. If the function is not a +** LIKE-style function then return FALSE. +** +** The expression "a LIKE b ESCAPE c" is only considered a valid LIKE +** operator if c is a string literal that is exactly one byte in length. +** That one byte is stored in aWc[3]. aWc[3] is set to zero if there is +** no ESCAPE clause. ** ** *pIsNocase is set to true if uppercase and lowercase are equivalent for ** the function (default for LIKE). If the function makes the distinction @@ -104299,14 +131608,19 @@ SQLITE_PRIVATE void sqlite3RegisterLikeFunctions(sqlite3 *db, int caseSensitive) */ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocase, char *aWc){ FuncDef *pDef; - if( pExpr->op!=TK_FUNCTION - || !pExpr->x.pList - || pExpr->x.pList->nExpr!=2 - ){ + int nExpr; + assert( pExpr!=0 ); + assert( pExpr->op==TK_FUNCTION ); + assert( ExprUseXList(pExpr) ); + if( !pExpr->x.pList ){ return 0; } - assert( !ExprHasProperty(pExpr, EP_xIsSelect) ); - pDef = sqlite3FindFunction(db, pExpr->u.zToken, 2, SQLITE_UTF8, 0); + nExpr = pExpr->x.pList->nExpr; + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pDef = sqlite3FindFunction(db, pExpr->u.zToken, nExpr, SQLITE_UTF8, 0); +#ifdef SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION + if( pDef==0 ) return 0; +#endif if( NEVER(pDef==0) || (pDef->funcFlags & SQLITE_FUNC_LIKE)==0 ){ return 0; } @@ -104319,10 +131633,345 @@ SQLITE_PRIVATE int sqlite3IsLikeFunction(sqlite3 *db, Expr *pExpr, int *pIsNocas assert( (char*)&likeInfoAlt == (char*)&likeInfoAlt.matchAll ); assert( &((char*)&likeInfoAlt)[1] == (char*)&likeInfoAlt.matchOne ); assert( &((char*)&likeInfoAlt)[2] == (char*)&likeInfoAlt.matchSet ); + + if( nExpr<3 ){ + aWc[3] = 0; + }else{ + Expr *pEscape = pExpr->x.pList->a[2].pExpr; + char *zEscape; + if( pEscape->op!=TK_STRING ) return 0; + assert( !ExprHasProperty(pEscape, EP_IntValue) ); + zEscape = pEscape->u.zToken; + if( zEscape[0]==0 || zEscape[1]!=0 ) return 0; + if( zEscape[0]==aWc[0] ) return 0; + if( zEscape[0]==aWc[1] ) return 0; + aWc[3] = zEscape[0]; + } + *pIsNocase = (pDef->funcFlags & SQLITE_FUNC_CASE)==0; return 1; } +/* Mathematical Constants */ +#ifndef M_PI +# define M_PI 3.141592653589793238462643383279502884 +#endif +#ifndef M_LN10 +# define M_LN10 2.302585092994045684017991454684364208 +#endif +#ifndef M_LN2 +# define M_LN2 0.693147180559945309417232121458176568 +#endif + + +/* Extra math functions that require linking with -lm +*/ +#ifdef SQLITE_ENABLE_MATH_FUNCTIONS +/* +** Implementation SQL functions: +** +** ceil(X) +** ceiling(X) +** floor(X) +** +** The sqlite3_user_data() pointer is a pointer to the libm implementation +** of the underlying C function. +*/ +static void ceilingFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==1 ); + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: { + sqlite3_result_int64(context, sqlite3_value_int64(argv[0])); + break; + } + case SQLITE_FLOAT: { + double (*x)(double) = (double(*)(double))sqlite3_user_data(context); + sqlite3_result_double(context, x(sqlite3_value_double(argv[0]))); + break; + } + default: { + break; + } + } +} + +/* +** On some systems, ceil() and floor() are intrinsic function. You are +** unable to take a pointer to these functions. Hence, we here wrap them +** in our own actual functions. +*/ +static double xCeil(double x){ return ceil(x); } +static double xFloor(double x){ return floor(x); } + +/* +** Some systems do not have log2() and log10() in their standard math +** libraries. +*/ +#if defined(HAVE_LOG10) && HAVE_LOG10==0 +# define log10(X) (0.4342944819032517867*log(X)) +#endif +#if defined(HAVE_LOG2) && HAVE_LOG2==0 +# define log2(X) (1.442695040888963456*log(X)) +#endif + + +/* +** Implementation of SQL functions: +** +** ln(X) - natural logarithm +** log(X) - log X base 10 +** log10(X) - log X base 10 +** log(B,X) - log X base B +*/ +static void logFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x, b, ans; + assert( argc==1 || argc==2 ); + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + x = sqlite3_value_double(argv[0]); + if( x<=0.0 ) return; + break; + default: + return; + } + if( argc==2 ){ + switch( sqlite3_value_numeric_type(argv[0]) ){ + case SQLITE_INTEGER: + case SQLITE_FLOAT: + b = log(x); + if( b<=0.0 ) return; + x = sqlite3_value_double(argv[1]); + if( x<=0.0 ) return; + break; + default: + return; + } + ans = log(x)/b; + }else{ + switch( SQLITE_PTR_TO_INT(sqlite3_user_data(context)) ){ + case 1: + ans = log10(x); + break; + case 2: + ans = log2(x); + break; + default: + ans = log(x); + break; + } + } + sqlite3_result_double(context, ans); +} + +/* +** Functions to converts degrees to radians and radians to degrees. +*/ +static double degToRad(double x){ return x*(M_PI/180.0); } +static double radToDeg(double x){ return x*(180.0/M_PI); } + +/* +** Implementation of 1-argument SQL math functions: +** +** exp(X) - Compute e to the X-th power +*/ +static void math1Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0; + double v0, ans; + double (*x)(double); + assert( argc==1 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + v0 = sqlite3_value_double(argv[0]); + x = (double(*)(double))sqlite3_user_data(context); + ans = x(v0); + sqlite3_result_double(context, ans); +} + +/* +** Implementation of 2-argument SQL math functions: +** +** power(X,Y) - Compute X to the Y-th power +*/ +static void math2Func( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0, type1; + double v0, v1, ans; + double (*x)(double,double); + assert( argc==2 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + type1 = sqlite3_value_numeric_type(argv[1]); + if( type1!=SQLITE_INTEGER && type1!=SQLITE_FLOAT ) return; + v0 = sqlite3_value_double(argv[0]); + v1 = sqlite3_value_double(argv[1]); + x = (double(*)(double,double))sqlite3_user_data(context); + ans = x(v0, v1); + sqlite3_result_double(context, ans); +} + +/* +** Implementation of 0-argument pi() function. +*/ +static void piFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + assert( argc==0 ); + (void)argv; + sqlite3_result_double(context, M_PI); +} + +#endif /* SQLITE_ENABLE_MATH_FUNCTIONS */ + +/* +** Implementation of sign(X) function. +*/ +static void signFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + int type0; + double x; + UNUSED_PARAMETER(argc); + assert( argc==1 ); + type0 = sqlite3_value_numeric_type(argv[0]); + if( type0!=SQLITE_INTEGER && type0!=SQLITE_FLOAT ) return; + x = sqlite3_value_double(argv[0]); + sqlite3_result_int(context, x<0.0 ? -1 : x>0.0 ? +1 : 0); +} + +#ifdef SQLITE_DEBUG +/* +** Implementation of fpdecode(x,y,z) function. +** +** x is a real number that is to be decoded. y is the precision. +** z is the maximum real precision. Return a string that shows the +** results of the sqlite3FpDecode() function. +** +** Used for testing and debugging only, specifically testing and debugging +** of the sqlite3FpDecode() function. This SQL function does not appear +** in production builds. This function is not an API and is subject to +** modification or removal in future versions of SQLite. +*/ +static void fpdecodeFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + FpDecode s; + double x; + int y, z; + char zBuf[100]; + UNUSED_PARAMETER(argc); + assert( argc==3 ); + x = sqlite3_value_double(argv[0]); + y = sqlite3_value_int(argv[1]); + z = sqlite3_value_int(argv[2]); + if( z<=0 ) z = 1; + sqlite3FpDecode(&s, x, y, z); + if( s.isSpecial==2 ){ + sqlite3_snprintf(sizeof(zBuf), zBuf, "NaN"); + }else{ + sqlite3_snprintf(sizeof(zBuf), zBuf, "%c%.*s/%d", s.sign, s.n, s.z, s.iDP); + } + sqlite3_result_text(context, zBuf, -1, SQLITE_TRANSIENT); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** Implementation of parseuri(uri,flags) function. +** +** Required Arguments: +** "uri" The URI to parse. +** "flags" Bitmask of flags, as if to sqlite3_open_v2(). +** +** Additional arguments beyond the first two make calls to +** sqlite3_uri_key() for integers and sqlite3_uri_parameter for +** anything else. +** +** The result is a string showing the results of calling sqlite3ParseUri(). +** +** Used for testing and debugging only, specifically testing and debugging +** of the sqlite3ParseUri() function. This SQL function does not appear +** in production builds. This function is not an API and is subject to +** modification or removal in future versions of SQLite. +*/ +static void parseuriFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + sqlite3_str *pResult; + const char *zVfs; + const char *zUri; + unsigned int flgs; + int rc; + sqlite3_vfs *pVfs = 0; + char *zFile = 0; + char *zErr = 0; + + if( argc<2 ) return; + pVfs = sqlite3_vfs_find(0); + assert( pVfs ); + zVfs = pVfs->zName; + zUri = (const char*)sqlite3_value_text(argv[0]); + if( zUri==0 ) return; + flgs = (unsigned int)sqlite3_value_int(argv[1]); + rc = sqlite3ParseUri(zVfs, zUri, &flgs, &pVfs, &zFile, &zErr); + pResult = sqlite3_str_new(0); + if( pResult ){ + int i; + sqlite3_str_appendf(pResult, "rc=%d", rc); + sqlite3_str_appendf(pResult, ", flags=0x%x", flgs); + sqlite3_str_appendf(pResult, ", vfs=%Q", pVfs ? pVfs->zName: 0); + sqlite3_str_appendf(pResult, ", err=%Q", zErr); + sqlite3_str_appendf(pResult, ", file=%Q", zFile); + if( zFile ){ + const char *z = zFile; + z += sqlite3Strlen30(z)+1; + while( z[0] ){ + sqlite3_str_appendf(pResult, ", %Q", z); + z += sqlite3Strlen30(z)+1; + } + for(i=2; iu.pHash){ int n = sqlite3Strlen30(p->zName); int h = p->zName[0] + n; + assert( p->funcFlags & SQLITE_FUNC_BUILTIN ); printf(" %s(%d)", p->zName, h); } printf("\n"); @@ -104476,25 +132200,25 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** Foreign keys in SQLite come in two flavours: deferred and immediate. ** If an immediate foreign key constraint is violated, ** SQLITE_CONSTRAINT_FOREIGNKEY is returned and the current -** statement transaction rolled back. If a -** deferred foreign key constraint is violated, no action is taken -** immediately. However if the application attempts to commit the +** statement transaction rolled back. If a +** deferred foreign key constraint is violated, no action is taken +** immediately. However if the application attempts to commit the ** transaction before fixing the constraint violation, the attempt fails. ** ** Deferred constraints are implemented using a simple counter associated -** with the database handle. The counter is set to zero each time a -** database transaction is opened. Each time a statement is executed +** with the database handle. The counter is set to zero each time a +** database transaction is opened. Each time a statement is executed ** that causes a foreign key violation, the counter is incremented. Each ** time a statement is executed that removes an existing violation from ** the database, the counter is decremented. When the transaction is ** committed, the commit fails if the current value of the counter is ** greater than zero. This scheme has two big drawbacks: ** -** * When a commit fails due to a deferred foreign key constraint, +** * When a commit fails due to a deferred foreign key constraint, ** there is no way to tell which foreign constraint is not satisfied, ** or which row it is not satisfied for. ** -** * If the database contains foreign key violations when the +** * If the database contains foreign key violations when the ** transaction is opened, this may cause the mechanism to malfunction. ** ** Despite these problems, this approach is adopted as it seems simpler @@ -104506,26 +132230,26 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** the parent table for a match. If none is found increment the ** constraint counter. ** -** I.2) For each FK for which the table is the parent table, +** I.2) For each FK for which the table is the parent table, ** search the child table for rows that correspond to the new ** row in the parent table. Decrement the counter for each row ** found (as the constraint is now satisfied). ** ** DELETE operations: ** -** D.1) For each FK for which the table is the child table, -** search the parent table for a row that corresponds to the -** deleted row in the child table. If such a row is not found, +** D.1) For each FK for which the table is the child table, +** search the parent table for a row that corresponds to the +** deleted row in the child table. If such a row is not found, ** decrement the counter. ** -** D.2) For each FK for which the table is the parent table, search -** the child table for rows that correspond to the deleted row +** D.2) For each FK for which the table is the parent table, search +** the child table for rows that correspond to the deleted row ** in the parent table. For each found increment the counter. ** ** UPDATE operations: ** ** An UPDATE command requires that all 4 steps above are taken, but only -** for FK constraints for which the affected columns are actually +** for FK constraints for which the affected columns are actually ** modified (values must be compared at runtime). ** ** Note that I.1 and D.1 are very similar operations, as are I.2 and D.2. @@ -104534,10 +132258,10 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** For the purposes of immediate FK constraints, the OR REPLACE conflict ** resolution is considered to delete rows before the new row is inserted. ** If a delete caused by OR REPLACE violates an FK constraint, an exception -** is thrown, even if the FK constraint would be satisfied after the new +** is thrown, even if the FK constraint would be satisfied after the new ** row is inserted. ** -** Immediate constraints are usually handled similarly. The only difference +** Immediate constraints are usually handled similarly. The only difference ** is that the counter used is stored as part of each individual statement ** object (struct Vdbe). If, after the statement has run, its immediate ** constraint counter is greater than zero, @@ -104548,7 +132272,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** INSERT violates a foreign key constraint. This is necessary as such ** an INSERT does not open a statement transaction. ** -** TODO: How should dropping a table be handled? How should renaming a +** TODO: How should dropping a table be handled? How should renaming a ** table be handled? ** ** @@ -104559,7 +132283,7 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** for those two operations needs to know whether or not the operation ** requires any FK processing and, if so, which columns of the original ** row are required by the FK processing VDBE code (i.e. if FKs were -** implemented using triggers, which of the old.* columns would be +** implemented using triggers, which of the old.* columns would be ** accessed). No information is required by the code-generator before ** coding an INSERT operation. The functions used by the UPDATE/DELETE ** generation code to query for this information are: @@ -104596,13 +132320,13 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ /* ** A foreign key constraint requires that the key columns in the parent ** table are collectively subject to a UNIQUE or PRIMARY KEY constraint. -** Given that pParent is the parent table for foreign key constraint pFKey, -** search the schema for a unique index on the parent key columns. +** Given that pParent is the parent table for foreign key constraint pFKey, +** search the schema for a unique index on the parent key columns. +** +** If successful, zero is returned. If the parent key is an INTEGER PRIMARY +** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx +** is set to point to the unique index. ** -** If successful, zero is returned. If the parent key is an INTEGER PRIMARY -** KEY column, then output variable *ppIdx is set to NULL. Otherwise, *ppIdx -** is set to point to the unique index. -** ** If the parent key consists of a single column (the foreign key constraint ** is not a composite foreign key), output variable *paiCol is set to NULL. ** Otherwise, it is set to point to an allocated array of size N, where @@ -104625,8 +132349,8 @@ SQLITE_PRIVATE void sqlite3RegisterBuiltinFunctions(void){ ** PRIMARY KEY, or ** ** 4) No parent key columns were provided explicitly as part of the -** foreign key definition, and the PRIMARY KEY of the parent table -** consists of a different number of columns to the child key in +** foreign key definition, and the PRIMARY KEY of the parent table +** consists of a different number of columns to the child key in ** the child table. ** ** then non-zero is returned, and a "foreign key mismatch" error loaded @@ -104650,9 +132374,9 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( assert( !paiCol || *paiCol==0 ); assert( pParse ); - /* If this is a non-composite (single column) foreign key, check if it - ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx - ** and *paiCol set to zero and return early. + /* If this is a non-composite (single column) foreign key, check if it + ** maps to the INTEGER PRIMARY KEY of table pParent. If so, leave *ppIdx + ** and *paiCol set to zero and return early. ** ** Otherwise, for a composite foreign key (more than one column), allocate ** space for the aiCol array (returned via output parameter *paiCol). @@ -104661,14 +132385,16 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( if( nCol==1 ){ /* The FK maps to the IPK if any of the following are true: ** - ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly + ** 1) There is an INTEGER PRIMARY KEY column and the FK is implicitly ** mapped to the primary key of table pParent, or ** 2) The FK is explicitly mapped to a column declared as INTEGER ** PRIMARY KEY. */ if( pParent->iPKey>=0 ){ if( !zKey ) return 0; - if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zName, zKey) ) return 0; + if( !sqlite3StrICmp(pParent->aCol[pParent->iPKey].zCnName, zKey) ){ + return 0; + } } }else if( paiCol ){ assert( nCol>1 ); @@ -104678,14 +132404,14 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( } for(pIdx=pParent->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) ){ + if( pIdx->nKeyCol==nCol && IsUniqueIndex(pIdx) && pIdx->pPartIdxWhere==0 ){ /* pIdx is a UNIQUE index (or a PRIMARY KEY) and has the right number ** of columns. If each indexed column corresponds to a foreign key ** column of pFKey, then this index is a winner. */ if( zKey==0 ){ - /* If zKey is NULL, then this foreign key is implicitly mapped to - ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be + /* If zKey is NULL, then this foreign key is implicitly mapped to + ** the PRIMARY KEY of table pParent. The PRIMARY KEY index may be ** identified by the test. */ if( IsPrimaryKeyIndex(pIdx) ){ if( aiCol ){ @@ -104710,11 +132436,11 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( /* If the index uses a collation sequence that is different from ** the default collation sequence for the column, this index is ** unusable. Bail out early in this case. */ - zDfltColl = pParent->aCol[iCol].zColl; + zDfltColl = sqlite3ColumnColl(&pParent->aCol[iCol]); if( !zDfltColl ) zDfltColl = sqlite3StrBINARY; if( sqlite3StrICmp(pIdx->azColl[i], zDfltColl) ) break; - zIdxCol = pParent->aCol[iCol].zName; + zIdxCol = pParent->aCol[iCol].zCnName; for(j=0; jaCol[j].zCol, zIdxCol)==0 ){ if( aiCol ) aiCol[i] = pFKey->aCol[j].iFrom; @@ -104743,15 +132469,15 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( } /* -** This function is called when a row is inserted into or deleted from the -** child table of foreign key constraint pFKey. If an SQL UPDATE is executed +** This function is called when a row is inserted into or deleted from the +** child table of foreign key constraint pFKey. If an SQL UPDATE is executed ** on the child table of pFKey, this function is invoked twice for each row ** affected - once to "delete" the old row, and then again to "insert" the ** new row. ** ** Each time it is called, this function generates VDBE code to locate the -** row in the parent table that corresponds to the row being inserted into -** or deleted from the child table. If the parent row can be found, no +** row in the parent table that corresponds to the row being inserted into +** or deleted from the child table. If the parent row can be found, no ** special action is taken. Otherwise, if the parent row can *not* be ** found in the parent table: ** @@ -104765,7 +132491,7 @@ SQLITE_PRIVATE int sqlite3FkLocateIndex( ** ** DELETE deferred Decrement the "deferred constraint counter". ** -** These operations are identified in the comment at the top of this file +** These operations are identified in the comment at the top of this file ** (fkey.c) as "I.1" and "D.1". */ static void fkLookupParent( @@ -104782,21 +132508,27 @@ static void fkLookupParent( int i; /* Iterator variable */ Vdbe *v = sqlite3GetVdbe(pParse); /* Vdbe to add code to */ int iCur = pParse->nTab - 1; /* Cursor number to use */ - int iOk = sqlite3VdbeMakeLabel(v); /* jump here if parent key found */ + int iOk = sqlite3VdbeMakeLabel(pParse); /* jump here if parent key found */ + + sqlite3VdbeVerifyAbortable(v, + (!pFKey->isDeferred + && !(pParse->db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel + && !pParse->isMultiWrite) ? OE_Abort : OE_Ignore); /* If nIncr is less than zero, then check at runtime if there are any ** outstanding constraints to resolve. If there are not, there is no need ** to check if deleting this row resolves any outstanding violations. ** - ** Check if any of the key columns in the child table row are NULL. If - ** any are, then the constraint is considered satisfied. No need to + ** Check if any of the key columns in the child table row are NULL. If + ** any are, then the constraint is considered satisfied. No need to ** search for a matching row in the parent table. */ if( nIncr<0 ){ sqlite3VdbeAddOp2(v, OP_FkIfZero, pFKey->isDeferred, iOk); VdbeCoverage(v); } for(i=0; inCol; i++){ - int iReg = aiCol[i] + regData + 1; + int iReg = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + regData + 1; sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iOk); VdbeCoverage(v); } @@ -104806,16 +132538,17 @@ static void fkLookupParent( ** column of the parent table (table pTab). */ int iMustBeInt; /* Address of MustBeInt instruction */ int regTemp = sqlite3GetTempReg(pParse); - - /* Invoke MustBeInt to coerce the child key value to an integer (i.e. + + /* Invoke MustBeInt to coerce the child key value to an integer (i.e. ** apply the affinity of the parent key). If this fails, then there ** is no matching parent key. Before using MustBeInt, make a copy of ** the value. Otherwise, the value inserted into the child key column ** will have INTEGER affinity applied to it, which may not be correct. */ - sqlite3VdbeAddOp2(v, OP_SCopy, aiCol[0]+1+regData, regTemp); + sqlite3VdbeAddOp2(v, OP_SCopy, + sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[0])+1+regData, regTemp); iMustBeInt = sqlite3VdbeAddOp2(v, OP_MustBeInt, regTemp, 0); VdbeCoverage(v); - + /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not @@ -104824,7 +132557,7 @@ static void fkLookupParent( sqlite3VdbeAddOp3(v, OP_Eq, regData, iOk, regTemp); VdbeCoverage(v); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); } - + sqlite3OpenTable(pParse, iCur, iDb, pTab, OP_OpenRead); sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, regTemp); VdbeCoverage(v); sqlite3VdbeGoto(v, iOk); @@ -104834,20 +132567,21 @@ static void fkLookupParent( }else{ int nCol = pFKey->nCol; int regTemp = sqlite3GetTempRange(pParse, nCol); - int regRec = sqlite3GetTempReg(pParse); - + sqlite3VdbeAddOp3(v, OP_OpenRead, iCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); for(i=0; ipFrom, aiCol[i])+1+regData, + regTemp+i); } - + /* If the parent table is the same as the child table, and we are about ** to increment the constraint-counter (i.e. this is an INSERT operation), ** then check if the row being inserted matches itself. If so, do not - ** increment the constraint-counter. + ** increment the constraint-counter. ** - ** If any of the parent-key values are NULL, then the row cannot match + ** If any of the parent-key values are NULL, then the row cannot match ** itself. So set JUMPIFNULL to make sure we do the OP_Found if any ** of the parent-key values are NULL (at this point it is known that ** none of the child key values are). @@ -104855,8 +132589,11 @@ static void fkLookupParent( if( pTab==pFKey->pFrom && nIncr==1 ){ int iJump = sqlite3VdbeCurrentAddr(v) + nCol + 1; for(i=0; iaiColumn[i]+1+regData; + int iChild = sqlite3TableColumnToStorage(pFKey->pFrom,aiCol[i]) + +1+regData; + int iParent = 1+regData; + iParent += sqlite3TableColumnToStorage(pIdx->pTable, + pIdx->aiColumn[i]); assert( pIdx->aiColumn[i]>=0 ); assert( aiCol[i]!=pTab->iPKey ); if( pIdx->aiColumn[i]==pTab->iPKey ){ @@ -104868,19 +132605,18 @@ static void fkLookupParent( } sqlite3VdbeGoto(v, iOk); } - - sqlite3VdbeAddOp4(v, OP_MakeRecord, regTemp, nCol, regRec, + + sqlite3VdbeAddOp4(v, OP_Affinity, regTemp, nCol, 0, sqlite3IndexAffinityStr(pParse->db,pIdx), nCol); - sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regRec, 0); VdbeCoverage(v); - - sqlite3ReleaseTempReg(pParse, regRec); + sqlite3VdbeAddOp4Int(v, OP_Found, iCur, iOk, regTemp, nCol); + VdbeCoverage(v); sqlite3ReleaseTempRange(pParse, regTemp, nCol); } } if( !pFKey->isDeferred && !(pParse->db->flags & SQLITE_DeferFKs) - && !pParse->pToplevel - && !pParse->isMultiWrite + && !pParse->pToplevel + && !pParse->isMultiWrite ){ /* Special case: If this is an INSERT statement that will insert exactly ** one row into the table, raise a constraint immediately instead of @@ -104924,14 +132660,14 @@ static Expr *exprTableRegister( if( pExpr ){ if( iCol>=0 && iCol!=pTab->iPKey ){ pCol = &pTab->aCol[iCol]; - pExpr->iTable = regBase + iCol + 1; - pExpr->affinity = pCol->affinity; - zColl = pCol->zColl; + pExpr->iTable = regBase + sqlite3TableColumnToStorage(pTab,iCol) + 1; + pExpr->affExpr = pCol->affinity; + zColl = sqlite3ColumnColl(pCol); if( zColl==0 ) zColl = db->pDfltColl->zName; pExpr = sqlite3ExprAddCollateString(pParse, pExpr, zColl); }else{ pExpr->iTable = regBase; - pExpr->affinity = SQLITE_AFF_INTEGER; + pExpr->affExpr = SQLITE_AFF_INTEGER; } } return pExpr; @@ -104949,7 +132685,8 @@ static Expr *exprTableColumn( ){ Expr *pExpr = sqlite3Expr(db, TK_COLUMN, 0); if( pExpr ){ - pExpr->pTab = pTab; + assert( ExprUseYTab(pExpr) ); + pExpr->y.pTab = pTab; pExpr->iTable = iCursor; pExpr->iColumn = iCol; } @@ -104958,7 +132695,7 @@ static Expr *exprTableColumn( /* ** This function is called to generate code executed when a row is deleted -** from the parent table of foreign key constraint pFKey and, if pFKey is +** from the parent table of foreign key constraint pFKey and, if pFKey is ** deferred, when a row is inserted into the same table. When generating ** code for an SQL UPDATE operation, this function may be called twice - ** once to "delete" the old row and once to "insert" the new row. @@ -104974,18 +132711,14 @@ static Expr *exprTableColumn( ** Operation | FK type | Action taken ** -------------------------------------------------------------------------- ** DELETE immediate Increment the "immediate constraint counter". -** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "FOREIGN KEY constraint failed" exception. ** ** INSERT immediate Decrement the "immediate constraint counter". ** ** DELETE deferred Increment the "deferred constraint counter". -** Or, if the ON (UPDATE|DELETE) action is RESTRICT, -** throw a "FOREIGN KEY constraint failed" exception. ** ** INSERT deferred Decrement the "deferred constraint counter". ** -** These operations are identified in the comment at the top of this file +** These operations are identified in the comment at the top of this file ** (fkey.c) as "I.2" and "D.2". */ static void fkScanChildren( @@ -105028,17 +132761,17 @@ static void fkScanChildren( Expr *pLeft; /* Value from parent table row */ Expr *pRight; /* Column ref to child table */ Expr *pEq; /* Expression (pLeft = pRight) */ - i16 iCol; /* Index of column in child table */ + i16 iCol; /* Index of column in child table */ const char *zCol; /* Name of column in child table */ iCol = pIdx ? pIdx->aiColumn[i] : -1; pLeft = exprTableRegister(pParse, pTab, regData, iCol); iCol = aiCol ? aiCol[i] : pFKey->aCol[0].iFrom; assert( iCol>=0 ); - zCol = pFKey->pFrom->aCol[iCol].zName; + zCol = pFKey->pFrom->aCol[iCol].zCnName; pRight = sqlite3Expr(db, TK_ID, zCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); } /* If the child table is the same as the parent table, then add terms @@ -105049,8 +132782,11 @@ static void fkScanChildren( ** NOT( $current_a==a AND $current_b==b AND ... ) ** ** The first form is used for rowid tables. The second form is used - ** for WITHOUT ROWID tables. In the second form, the primary key is - ** (a,b,...) + ** for WITHOUT ROWID tables. In the second form, the *parent* key is + ** (a,b,...). Either the parent or primary key could be used to + ** uniquely identify the current row, but the parent key is more convenient + ** as the required values have already been loaded into registers + ** by the caller. */ if( pTab==pFKey->pFrom && nIncr>0 ){ Expr *pNe; /* Expression (pLeft != pRight) */ @@ -105059,22 +132795,21 @@ static void fkScanChildren( if( HasRowid(pTab) ){ pLeft = exprTableRegister(pParse, pTab, regData, -1); pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, -1); - pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight, 0); + pNe = sqlite3PExpr(pParse, TK_NE, pLeft, pRight); }else{ Expr *pEq, *pAll = 0; - Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pIdx!=0 ); - for(i=0; inKeyCol; i++){ + for(i=0; inKeyCol; i++){ i16 iCol = pIdx->aiColumn[i]; assert( iCol>=0 ); pLeft = exprTableRegister(pParse, pTab, regData, iCol); - pRight = exprTableColumn(db, pTab, pSrc->a[0].iCursor, iCol); - pEq = sqlite3PExpr(pParse, TK_EQ, pLeft, pRight, 0); - pAll = sqlite3ExprAnd(db, pAll, pEq); + pRight = sqlite3Expr(db, TK_ID, pTab->aCol[iCol].zCnName); + pEq = sqlite3PExpr(pParse, TK_IS, pLeft, pRight); + pAll = sqlite3ExprAnd(pParse, pAll, pEq); } - pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0, 0); + pNe = sqlite3PExpr(pParse, TK_NOT, pAll, 0); } - pWhere = sqlite3ExprAnd(db, pWhere, pNe); + pWhere = sqlite3ExprAnd(pParse, pWhere, pNe); } /* Resolve the references in the WHERE clause. */ @@ -105086,16 +132821,18 @@ static void fkScanChildren( /* Create VDBE to loop through the entries in pSrc that match the WHERE ** clause. For each row found, increment either the deferred or immediate ** foreign key constraint counter. */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0); - sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); - if( pWInfo ){ - sqlite3WhereEnd(pWInfo); + if( pParse->nErr==0 ){ + pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0, 0, 0, 0, 0); + sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, nIncr); + if( pWInfo ){ + sqlite3WhereEnd(pWInfo); + } } /* Clean up the WHERE clause constructed above. */ sqlite3ExprDelete(db, pWhere); if( iFkIfZero ){ - sqlite3VdbeJumpHere(v, iFkIfZero); + sqlite3VdbeJumpHereOrPopInst(v, iFkIfZero); } } @@ -105118,7 +132855,7 @@ SQLITE_PRIVATE FKey *sqlite3FkReferences(Table *pTab){ } /* -** The second argument is a Trigger structure allocated by the +** The second argument is a Trigger structure allocated by the ** fkActionTrigger() routine. This function deletes the Trigger structure ** and all of its sub-components. ** @@ -105136,6 +132873,25 @@ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ } } +/* +** Clear the apTrigger[] cache of CASCADE triggers for all foreign keys +** in a particular database. This needs to happen when the schema +** changes. +*/ +SQLITE_PRIVATE void sqlite3FkClearTriggerCache(sqlite3 *db, int iDb){ + HashElem *k; + Hash *pHash = &db->aDb[iDb].pSchema->tblHash; + for(k=sqliteHashFirst(pHash); k; k=sqliteHashNext(k)){ + Table *pTab = sqliteHashData(k); + FKey *pFKey; + if( !IsOrdinaryTable(pTab) ) continue; + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ + fkTriggerDelete(db, pFKey->apTrigger[0]); pFKey->apTrigger[0] = 0; + fkTriggerDelete(db, pFKey->apTrigger[1]); pFKey->apTrigger[1] = 0; + } + } +} + /* ** This function is called to generate code that runs when table pTab is ** being dropped from the database. The SrcList passed as the second argument @@ -105146,7 +132902,7 @@ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ ** ** (a) The table is the parent table of a FK constraint, or ** (b) The table is the child table of a deferred FK constraint and it is -** determined at runtime that there are outstanding deferred FK +** determined at runtime that there are outstanding deferred FK ** constraint violations in the database, ** ** then the equivalent of "DELETE FROM " is executed before dropping @@ -105155,40 +132911,42 @@ static void fkTriggerDelete(sqlite3 *dbMem, Trigger *p){ */ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTab){ sqlite3 *db = pParse->db; - if( (db->flags&SQLITE_ForeignKeys) && !IsVirtual(pTab) && !pTab->pSelect ){ + if( (db->flags&SQLITE_ForeignKeys) && IsOrdinaryTable(pTab) ){ int iSkip = 0; Vdbe *v = sqlite3GetVdbe(pParse); assert( v ); /* VDBE has already been allocated */ + assert( IsOrdinaryTable(pTab) ); if( sqlite3FkReferences(pTab)==0 ){ /* Search for a deferred foreign key constraint for which this table - ** is the child table. If one cannot be found, return without + ** is the child table. If one cannot be found, return without ** generating any VDBE code. If one can be found, then jump over ** the entire DELETE if there are no outstanding deferred constraints ** when this statement is run. */ FKey *p; - for(p=pTab->pFKey; p; p=p->pNextFrom){ + for(p=pTab->u.tab.pFKey; p; p=p->pNextFrom){ if( p->isDeferred || (db->flags & SQLITE_DeferFKs) ) break; } if( !p ) return; - iSkip = sqlite3VdbeMakeLabel(v); + iSkip = sqlite3VdbeMakeLabel(pParse); sqlite3VdbeAddOp2(v, OP_FkIfZero, 1, iSkip); VdbeCoverage(v); } pParse->disableTriggers = 1; - sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0); + sqlite3DeleteFrom(pParse, sqlite3SrcListDup(db, pName, 0), 0, 0, 0); pParse->disableTriggers = 0; - /* If the DELETE has generated immediate foreign key constraint + /* If the DELETE has generated immediate foreign key constraint ** violations, halt the VDBE and return an error at this point, before ** any modifications to the schema are made. This is because statement - ** transactions are not able to rollback schema changes. + ** transactions are not able to rollback schema changes. ** ** If the SQLITE_DeferFKs flag is set, then this is not required, as ** the statement transaction will not be rolled back even if FK ** constraints are violated. */ if( (db->flags & SQLITE_DeferFKs)==0 ){ + sqlite3VdbeVerifyAbortable(v, OE_Abort); sqlite3VdbeAddOp2(v, OP_FkIfZero, 0, sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_FOREIGNKEY, @@ -105205,7 +132963,7 @@ SQLITE_PRIVATE void sqlite3FkDropTable(Parse *pParse, SrcList *pName, Table *pTa /* ** The second argument points to an FKey object representing a foreign key ** for which pTab is the child table. An UPDATE statement against pTab -** is currently being processed. For each column of the table that is +** is currently being processed. For each column of the table that is ** actually updated, the corresponding element in the aChange[] array ** is zero or greater (if a column is unmodified the corresponding element ** is set to -1). If the rowid column is modified by the UPDATE statement @@ -105232,7 +132990,7 @@ static int fkChildIsModified( /* ** The second argument points to an FKey object representing a foreign key ** for which pTab is the parent table. An UPDATE statement against pTab -** is currently being processed. For each column of the table that is +** is currently being processed. For each column of the table that is ** actually updated, the corresponding element in the aChange[] array ** is zero or greater (if a column is unmodified the corresponding element ** is set to -1). If the rowid column is modified by the UPDATE statement @@ -105242,9 +133000,9 @@ static int fkChildIsModified( ** parent key for FK constraint *p are modified. */ static int fkParentIsModified( - Table *pTab, - FKey *p, - int *aChange, + Table *pTab, + FKey *p, + int *aChange, int bChngRowid ){ int i; @@ -105255,7 +133013,7 @@ static int fkParentIsModified( if( aChange[iKey]>=0 || (iKey==pTab->iPKey && bChngRowid) ){ Column *pCol = &pTab->aCol[iKey]; if( zKey ){ - if( 0==sqlite3StrICmp(pCol->zName, zKey) ) return 1; + if( 0==sqlite3StrICmp(pCol->zCnName, zKey) ) return 1; }else if( pCol->colFlags & COLFLAG_PRIMKEY ){ return 1; } @@ -105277,6 +133035,7 @@ static int isSetNullAction(Parse *pParse, FKey *pFKey){ if( (p==pFKey->apTrigger[0] && pFKey->aAction[0]==OE_SetNull) || (p==pFKey->apTrigger[1] && pFKey->aAction[1]==OE_SetNull) ){ + assert( (pTop->db->flags & SQLITE_FkNoAction)==0 ); return 1; } } @@ -105285,7 +133044,7 @@ static int isSetNullAction(Parse *pParse, FKey *pFKey){ /* ** This function is called when inserting, deleting or updating a row of -** table pTab to generate VDBE code to perform foreign key constraint +** table pTab to generate VDBE code to perform foreign key constraint ** processing for the operation. ** ** For a DELETE operation, parameter regOld is passed the index of the @@ -105301,11 +133060,11 @@ static int isSetNullAction(Parse *pParse, FKey *pFKey){ ** For an UPDATE operation, this function is called twice. Once before ** the original record is deleted from the table using the calling convention ** described for DELETE. Then again after the original record is deleted -** but before the new record is inserted using the INSERT convention. +** but before the new record is inserted using the INSERT convention. */ SQLITE_PRIVATE void sqlite3FkCheck( Parse *pParse, /* Parse context */ - Table *pTab, /* Row is being deleted from this table */ + Table *pTab, /* Row is being deleted from this table */ int regOld, /* Previous row data is stored here */ int regNew, /* New row data is stored here */ int *aChange, /* Array indicating UPDATEd columns (or 0) */ @@ -105322,13 +133081,14 @@ SQLITE_PRIVATE void sqlite3FkCheck( /* If foreign-keys are disabled, this function is a no-op. */ if( (db->flags&SQLITE_ForeignKeys)==0 ) return; + if( !IsOrdinaryTable(pTab) ) return; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zDb = db->aDb[iDb].zName; + zDb = db->aDb[iDb].zDbSName; /* Loop through all the foreign key constraints for which pTab is the ** child table (the table that the foreign key definition is part of). */ - for(pFKey=pTab->pFKey; pFKey; pFKey=pFKey->pNextFrom){ + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pFKey->pNextFrom){ Table *pTo; /* Parent table of foreign key pFKey */ Index *pIdx = 0; /* Index on key columns in pTo */ int *aiFree = 0; @@ -105337,16 +133097,16 @@ SQLITE_PRIVATE void sqlite3FkCheck( int i; int bIgnore = 0; - if( aChange + if( aChange && sqlite3_stricmp(pTab->zName, pFKey->zTo)!=0 - && fkChildIsModified(pTab, pFKey, aChange, bChngRowid)==0 + && fkChildIsModified(pTab, pFKey, aChange, bChngRowid)==0 ){ continue; } - /* Find the parent table of this foreign key. Also find a unique index - ** on the parent key columns in the parent table. If either of these - ** schema items cannot be located, set an error in pParse and return + /* Find the parent table of this foreign key. Also find a unique index + ** on the parent key columns in the parent table. If either of these + ** schema items cannot be located, set an error in pParse and return ** early. */ if( pParse->disableTriggers ){ pTo = sqlite3FindTable(db, pFKey->zTo, zDb); @@ -105367,7 +133127,9 @@ SQLITE_PRIVATE void sqlite3FkCheck( Vdbe *v = sqlite3GetVdbe(pParse); int iJump = sqlite3VdbeCurrentAddr(v) + pFKey->nCol + 1; for(i=0; inCol; i++){ - int iReg = pFKey->aCol[i].iFrom + regOld + 1; + int iFromCol, iReg; + iFromCol = pFKey->aCol[i].iFrom; + iReg = sqlite3TableColumnToStorage(pFKey->pFrom,iFromCol) + regOld+1; sqlite3VdbeAddOp2(v, OP_IsNull, iReg, iJump); VdbeCoverage(v); } sqlite3VdbeAddOp2(v, OP_FkCounter, pFKey->isDeferred, -1); @@ -105388,36 +133150,36 @@ SQLITE_PRIVATE void sqlite3FkCheck( } assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); #ifndef SQLITE_OMIT_AUTHORIZATION - /* Request permission to read the parent key columns. If the + /* Request permission to read the parent key columns. If the ** authorization callback returns SQLITE_IGNORE, behave as if any ** values read from the parent table are NULL. */ if( db->xAuth ){ int rcauth; - char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zName; + char *zCol = pTo->aCol[pIdx ? pIdx->aiColumn[i] : pTo->iPKey].zCnName; rcauth = sqlite3AuthReadCol(pParse, pTo->zName, zCol, iDb); bIgnore = (rcauth==SQLITE_IGNORE); } #endif } - /* Take a shared-cache advisory read-lock on the parent table. Allocate - ** a cursor to use to search the unique index on the parent key columns + /* Take a shared-cache advisory read-lock on the parent table. Allocate + ** a cursor to use to search the unique index on the parent key columns ** in the parent table. */ sqlite3TableLock(pParse, iDb, pTo->tnum, 0, pTo->zName); pParse->nTab++; if( regOld!=0 ){ /* A row is being removed from the child table. Search for the parent. - ** If the parent does not exist, removing the child row resolves an + ** If the parent does not exist, removing the child row resolves an ** outstanding foreign key constraint violation. */ fkLookupParent(pParse, iDb, pTo, pIdx, pFKey, aiCol, regOld, -1, bIgnore); } if( regNew!=0 && !isSetNullAction(pParse, pFKey) ){ /* A row is being added to the child table. If a parent row cannot - ** be found, adding the child row has violated the FK constraint. + ** be found, adding the child row has violated the FK constraint. ** ** If this operation is being performed as part of a trigger program - ** that is actually a "SET NULL" action belonging to this very + ** that is actually a "SET NULL" action belonging to this very ** foreign key, then omit this scan altogether. As all child key ** values are guaranteed to be NULL, it is not possible for adding ** this row to cause an FK violation. */ @@ -105438,8 +133200,8 @@ SQLITE_PRIVATE void sqlite3FkCheck( continue; } - if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) - && !pParse->pToplevel && !pParse->isMultiWrite + if( !pFKey->isDeferred && !(db->flags & SQLITE_DeferFKs) + && !pParse->pToplevel && !pParse->isMultiWrite ){ assert( regOld==0 && regNew!=0 ); /* Inserting a single row into a parent table cannot cause (or fix) @@ -105455,19 +133217,21 @@ SQLITE_PRIVATE void sqlite3FkCheck( /* Create a SrcList structure containing the child table. We need the ** child table as a SrcList for sqlite3WhereBegin() */ - pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); if( pSrc ){ - struct SrcList_item *pItem = pSrc->a; - pItem->pTab = pFKey->pFrom; + SrcItem *pItem = pSrc->a; + pItem->pSTab = pFKey->pFrom; pItem->zName = pFKey->pFrom->zName; - pItem->pTab->nRef++; + pItem->pSTab->nTabRef++; pItem->iCursor = pParse->nTab++; - + if( regNew!=0 ){ fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regNew, -1); } if( regOld!=0 ){ int eAction = pFKey->aAction[aChange!=0]; + if( (db->flags & SQLITE_FkNoAction) ) eAction = OE_None; + fkScanChildren(pParse, pSrc, pTab, pIdx, pFKey, aiCol, regOld, 1); /* If this is a deferred FK constraint, or a CASCADE or SET NULL ** action applies, then any foreign key violations caused by @@ -105481,10 +133245,10 @@ SQLITE_PRIVATE void sqlite3FkCheck( ** ** Note 2: At first glance it may seem like SQLite could simply omit ** all OP_FkCounter related scans when either CASCADE or SET NULL - ** applies. The trouble starts if the CASCADE or SET NULL action - ** trigger causes other triggers or action rules attached to the + ** applies. The trouble starts if the CASCADE or SET NULL action + ** trigger causes other triggers or action rules attached to the ** child table to fire. In these cases the fk constraint counters - ** might be set incorrectly if any OP_FkCounter related scans are + ** might be set incorrectly if any OP_FkCounter related scans are ** omitted. */ if( !pFKey->isDeferred && eAction!=OE_Cascade && eAction!=OE_SetNull ){ sqlite3MayAbort(pParse); @@ -105500,7 +133264,7 @@ SQLITE_PRIVATE void sqlite3FkCheck( #define COLUMN_MASK(x) (((x)>31) ? 0xffffffff : ((u32)1<<(x))) /* -** This function is called before generating code to update or delete a +** This function is called before generating code to update or delete a ** row contained in table pTab. */ SQLITE_PRIVATE u32 sqlite3FkOldmask( @@ -105508,10 +133272,10 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( Table *pTab /* Table being modified */ ){ u32 mask = 0; - if( pParse->db->flags&SQLITE_ForeignKeys ){ + if( pParse->db->flags&SQLITE_ForeignKeys && IsOrdinaryTable(pTab) ){ FKey *p; int i; - for(p=pTab->pFKey; p; p=p->pNextFrom){ + for(p=pTab->u.tab.pFKey; p; p=p->pNextFrom){ for(i=0; inCol; i++) mask |= COLUMN_MASK(p->aCol[i].iFrom); } for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ @@ -105530,18 +133294,28 @@ SQLITE_PRIVATE u32 sqlite3FkOldmask( /* -** This function is called before generating code to update or delete a +** This function is called before generating code to update or delete a ** row contained in table pTab. If the operation is a DELETE, then ** parameter aChange is passed a NULL value. For an UPDATE, aChange points ** to an array of size N, where N is the number of columns in table pTab. -** If the i'th column is not modified by the UPDATE, then the corresponding +** If the i'th column is not modified by the UPDATE, then the corresponding ** entry in the aChange[] array is set to -1. If the column is modified, ** the value is 0 or greater. Parameter chngRowid is set to true if the ** UPDATE statement modifies the rowid fields of the table. ** ** If any foreign key processing will be required, this function returns -** true. If there is no foreign key related processing, this function -** returns false. +** non-zero. If there is no foreign key related processing, this function +** returns zero. +** +** For an UPDATE, this function returns 2 if: +** +** * There are any FKs for which pTab is the child and the parent table +** and any FK processing at all is required (even of a different FK), or +** +** * the UPDATE modifies one or more parent keys for which the action is +** not "NO ACTION" (i.e. is CASCADE, SET DEFAULT or SET NULL). +** +** Or, assuming some other foreign key processing is required, 1. */ SQLITE_PRIVATE int sqlite3FkRequired( Parse *pParse, /* Parse context */ @@ -105549,33 +133323,45 @@ SQLITE_PRIVATE int sqlite3FkRequired( int *aChange, /* Non-NULL for UPDATE operations */ int chngRowid /* True for UPDATE that affects rowid */ ){ - if( pParse->db->flags&SQLITE_ForeignKeys ){ + int eRet = 1; /* Value to return if bHaveFK is true */ + int bHaveFK = 0; /* If FK processing is required */ + if( pParse->db->flags&SQLITE_ForeignKeys && IsOrdinaryTable(pTab) ){ if( !aChange ){ - /* A DELETE operation. Foreign key processing is required if the - ** table in question is either the child or parent table for any + /* A DELETE operation. Foreign key processing is required if the + ** table in question is either the child or parent table for any ** foreign key constraint. */ - return (sqlite3FkReferences(pTab) || pTab->pFKey); + bHaveFK = (sqlite3FkReferences(pTab) || pTab->u.tab.pFKey); }else{ /* This is an UPDATE. Foreign key processing is only required if the ** operation modifies one or more child or parent key columns. */ FKey *p; /* Check if any child key columns are being modified. */ - for(p=pTab->pFKey; p; p=p->pNextFrom){ - if( fkChildIsModified(pTab, p, aChange, chngRowid) ) return 1; + for(p=pTab->u.tab.pFKey; p; p=p->pNextFrom){ + if( fkChildIsModified(pTab, p, aChange, chngRowid) ){ + if( 0==sqlite3_stricmp(pTab->zName, p->zTo) ) eRet = 2; + bHaveFK = 1; + } } /* Check if any parent key columns are being modified. */ for(p=sqlite3FkReferences(pTab); p; p=p->pNextTo){ - if( fkParentIsModified(pTab, p, aChange, chngRowid) ) return 1; + if( fkParentIsModified(pTab, p, aChange, chngRowid) ){ + if( (pParse->db->flags & SQLITE_FkNoAction)==0 + && p->aAction[1]!=OE_None + ){ + return 2; + } + bHaveFK = 1; + } } } } - return 0; + return bHaveFK ? eRet : 0; } /* -** This function is called when an UPDATE or DELETE operation is being +** This function is called when an UPDATE or DELETE operation is being ** compiled on table pTab, which is the parent table of foreign-key pFKey. ** If the current operation is an UPDATE, then the pChanges parameter is ** passed a pointer to the list of columns being modified. If it is a @@ -105583,11 +133369,11 @@ SQLITE_PRIVATE int sqlite3FkRequired( ** ** It returns a pointer to a Trigger structure containing a trigger ** equivalent to the ON UPDATE or ON DELETE action specified by pFKey. -** If the action is "NO ACTION" or "RESTRICT", then a NULL pointer is -** returned (these actions require no special handling by the triggers -** sub-system, code for them is created by fkScanChildren()). +** If the action is "NO ACTION" then a NULL pointer is returned (these actions +** require no special handling by the triggers sub-system, code for them is +** created by fkScanChildren()). ** -** For example, if pFKey is the foreign key and pTab is table "p" in +** For example, if pFKey is the foreign key and pTab is table "p" in ** the following schema: ** ** CREATE TABLE p(pk PRIMARY KEY); @@ -105600,7 +133386,7 @@ SQLITE_PRIVATE int sqlite3FkRequired( ** END; ** ** The returned pointer is cached as part of the foreign key object. It -** is eventually freed along with the rest of the foreign key object by +** is eventually freed along with the rest of the foreign key object by ** sqlite3FkDelete(). */ static Trigger *fkActionTrigger( @@ -105615,6 +133401,7 @@ static Trigger *fkActionTrigger( int iAction = (pChanges!=0); /* 1 for UPDATE, 0 for DELETE */ action = pFKey->aAction[iAction]; + if( (db->flags & SQLITE_FkNoAction) ) action = OE_None; if( action==OE_Restrict && (db->flags & SQLITE_DeferFKs) ){ return 0; } @@ -105648,21 +133435,20 @@ static Trigger *fkActionTrigger( assert( pIdx!=0 || (pTab->iPKey>=0 && pTab->iPKeynCol) ); assert( pIdx==0 || pIdx->aiColumn[i]>=0 ); sqlite3TokenInit(&tToCol, - pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zName); - sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zName); + pTab->aCol[pIdx ? pIdx->aiColumn[i] : pTab->iPKey].zCnName); + sqlite3TokenInit(&tFromCol, pFKey->pFrom->aCol[iFromCol].zCnName); /* Create the expression "OLD.zToCol = zFromCol". It is important ** that the "OLD.zToCol" term is on the LHS of the = operator, so ** that the affinity and collation sequence associated with the ** parent table are used for the comparison. */ pEq = sqlite3PExpr(pParse, TK_EQ, - sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tOld, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) - , 0), + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), sqlite3ExprAlloc(db, TK_ID, &tFromCol, 0) - , 0); - pWhere = sqlite3ExprAnd(db, pWhere, pEq); + ); + pWhere = sqlite3ExprAnd(pParse, pWhere, pEq); /* For ON UPDATE, construct the next term of the WHEN clause. ** The final WHEN clause will be like this: @@ -105671,34 +133457,39 @@ static Trigger *fkActionTrigger( */ if( pChanges ){ pEq = sqlite3PExpr(pParse, TK_IS, - sqlite3PExpr(pParse, TK_DOT, + sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tOld, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), - 0), - sqlite3PExpr(pParse, TK_DOT, + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)), + sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tNew, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0), - 0), - 0); - pWhen = sqlite3ExprAnd(db, pWhen, pEq); + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)) + ); + pWhen = sqlite3ExprAnd(pParse, pWhen, pEq); } - + if( action!=OE_Restrict && (action!=OE_Cascade || pChanges) ){ Expr *pNew; if( action==OE_Cascade ){ - pNew = sqlite3PExpr(pParse, TK_DOT, + pNew = sqlite3PExpr(pParse, TK_DOT, sqlite3ExprAlloc(db, TK_ID, &tNew, 0), - sqlite3ExprAlloc(db, TK_ID, &tToCol, 0) - , 0); + sqlite3ExprAlloc(db, TK_ID, &tToCol, 0)); }else if( action==OE_SetDflt ){ - Expr *pDflt = pFKey->pFrom->aCol[iFromCol].pDflt; + Column *pCol = pFKey->pFrom->aCol + iFromCol; + Expr *pDflt; + if( pCol->colFlags & COLFLAG_GENERATED ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + pDflt = 0; + }else{ + pDflt = sqlite3ColumnExpr(pFKey->pFrom, pCol); + } if( pDflt ){ pNew = sqlite3ExprDup(db, pDflt, 0); }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } }else{ - pNew = sqlite3PExpr(pParse, TK_NULL, 0, 0, 0); + pNew = sqlite3ExprAlloc(db, TK_NULL, 0, 0); } pList = sqlite3ExprListAppend(pParse, pList, pNew); sqlite3ExprListSetName(pParse, pList, &tFromCol, 0); @@ -105710,28 +133501,35 @@ static Trigger *fkActionTrigger( nFrom = sqlite3Strlen30(zFrom); if( action==OE_Restrict ){ - Token tFrom; - Expr *pRaise; + int iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + SrcList *pSrc; + Expr *pRaise; - tFrom.z = zFrom; - tFrom.n = nFrom; - pRaise = sqlite3Expr(db, TK_RAISE, "FOREIGN KEY constraint failed"); + pRaise = sqlite3Expr(db, TK_STRING, "FOREIGN KEY constraint failed"), + pRaise = sqlite3PExpr(pParse, TK_RAISE, pRaise, 0); if( pRaise ){ - pRaise->affinity = OE_Abort; + pRaise->affExpr = OE_Abort; } - pSelect = sqlite3SelectNew(pParse, + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + if( pSrc ){ + assert( pSrc->nSrc==1 ); + pSrc->a[0].zName = sqlite3DbStrDup(db, zFrom); + assert( pSrc->a[0].fg.fixedSchema==0 && pSrc->a[0].fg.isSubquery==0 ); + pSrc->a[0].u4.zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zDbSName); + } + pSelect = sqlite3SelectNew(pParse, sqlite3ExprListAppend(pParse, 0, pRaise), - sqlite3SrcListAppend(db, 0, &tFrom, 0), + pSrc, pWhere, - 0, 0, 0, 0, 0, 0 + 0, 0, 0, 0, 0 ); pWhere = 0; } /* Disable lookaside memory allocation */ - db->lookaside.bDisable++; + DisableLookaside; - pTrigger = (Trigger *)sqlite3DbMallocZero(db, + pTrigger = (Trigger *)sqlite3DbMallocZero(db, sizeof(Trigger) + /* struct Trigger */ sizeof(TriggerStep) + /* Single step in trigger program */ nFrom + 1 /* Space for pStep->zTarget */ @@ -105740,18 +133538,18 @@ static Trigger *fkActionTrigger( pStep = pTrigger->step_list = (TriggerStep *)&pTrigger[1]; pStep->zTarget = (char *)&pStep[1]; memcpy((char *)pStep->zTarget, zFrom, nFrom); - + pStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); pStep->pExprList = sqlite3ExprListDup(db, pList, EXPRDUP_REDUCE); pStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); if( pWhen ){ - pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0, 0); + pWhen = sqlite3PExpr(pParse, TK_NOT, pWhen, 0); pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); } } /* Re-enable the lookaside buffer, if it was disabled earlier. */ - db->lookaside.bDisable--; + EnableLookaside; sqlite3ExprDelete(db, pWhere); sqlite3ExprDelete(db, pWhen); @@ -105762,16 +133560,18 @@ static Trigger *fkActionTrigger( return 0; } assert( pStep!=0 ); + assert( pTrigger!=0 ); switch( action ){ case OE_Restrict: - pStep->op = TK_SELECT; + pStep->op = TK_SELECT; break; - case OE_Cascade: - if( !pChanges ){ - pStep->op = TK_DELETE; - break; + case OE_Cascade: + if( !pChanges ){ + pStep->op = TK_DELETE; + break; } + /* no break */ deliberate_fall_through default: pStep->op = TK_UPDATE; } @@ -105797,9 +133597,9 @@ SQLITE_PRIVATE void sqlite3FkActions( int *aChange, /* Array indicating UPDATEd columns (or 0) */ int bChngRowid /* True if rowid is UPDATEd */ ){ - /* If foreign-key support is enabled, iterate through all FKs that - ** refer to table pTab. If there is an action associated with the FK - ** for this operation (either update or delete), invoke the associated + /* If foreign-key support is enabled, iterate through all FKs that + ** refer to table pTab. If there is an action associated with the FK + ** for this operation (either update or delete), invoke the associated ** trigger sub-program. */ if( pParse->db->flags&SQLITE_ForeignKeys ){ FKey *pFKey; /* Iterator variable */ @@ -105825,17 +133625,18 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ FKey *pFKey; /* Iterator variable */ FKey *pNext; /* Copy of pFKey->pNextFrom */ - assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); - for(pFKey=pTab->pFKey; pFKey; pFKey=pNext){ + assert( IsOrdinaryTable(pTab) ); + assert( db!=0 ); + for(pFKey=pTab->u.tab.pFKey; pFKey; pFKey=pNext){ + assert( db==0 || sqlite3SchemaMutexHeld(db, 0, pTab->pSchema) ); /* Remove the FK from the fkeyHash hash table. */ - if( !db || db->pnBytesFreed==0 ){ + if( db->pnBytesFreed==0 ){ if( pFKey->pPrevTo ){ pFKey->pPrevTo->pNextTo = pFKey->pNextTo; }else{ - void *p = (void *)pFKey->pNextTo; - const char *z = (p ? pFKey->pNextTo->zTo : pFKey->zTo); - sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, p); + const char *z = (pFKey->pNextTo ? pFKey->pNextTo->zTo : pFKey->zTo); + sqlite3HashInsert(&pTab->pSchema->fkeyHash, z, pFKey->pNextTo); } if( pFKey->pNextTo ){ pFKey->pNextTo->pPrevTo = pFKey->pPrevTo; @@ -105878,7 +133679,7 @@ SQLITE_PRIVATE void sqlite3FkDelete(sqlite3 *db, Table *pTab){ /* #include "sqliteInt.h" */ /* -** Generate code that will +** Generate code that will ** ** (1) acquire a lock for table pTab then ** (2) open pTab as cursor iCur. @@ -105895,17 +133696,20 @@ SQLITE_PRIVATE void sqlite3OpenTable( ){ Vdbe *v; assert( !IsVirtual(pTab) ); - v = sqlite3GetVdbe(pParse); + assert( pParse->pVdbe!=0 ); + v = pParse->pVdbe; assert( opcode==OP_OpenWrite || opcode==OP_OpenRead ); - sqlite3TableLock(pParse, iDb, pTab->tnum, - (opcode==OP_OpenWrite)?1:0, pTab->zName); + if( !pParse->db->noSharedCache ){ + sqlite3TableLock(pParse, iDb, pTab->tnum, + (opcode==OP_OpenWrite)?1:0, pTab->zName); + } if( HasRowid(pTab) ){ - sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nCol); + sqlite3VdbeAddOp4Int(v, opcode, iCur, pTab->tnum, iDb, pTab->nNVCol); VdbeComment((v, "%s", pTab->zName)); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); assert( pPk!=0 ); - assert( pPk->tnum==pTab->tnum ); + assert( pPk->tnum==pTab->tnum || CORRUPT_DB ); sqlite3VdbeAddOp3(v, opcode, iCur, pPk->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pPk); VdbeComment((v, "%s", pTab->zName)); @@ -105914,7 +133718,7 @@ SQLITE_PRIVATE void sqlite3OpenTable( /* ** Return a pointer to the column affinity string associated with index -** pIdx. A column affinity string has one character for each column in +** pIdx. A column affinity string has one character for each column in ** the table, according to the affinity of the column: ** ** Character Column affinity @@ -105932,88 +133736,149 @@ SQLITE_PRIVATE void sqlite3OpenTable( ** is managed along with the rest of the Index structure. It will be ** released when sqlite3DeleteIndex() is called. */ -SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ +static SQLITE_NOINLINE const char *computeIndexAffStr(sqlite3 *db, Index *pIdx){ + /* The first time a column affinity string for a particular index is + ** required, it is allocated and populated here. It is then stored as + ** a member of the Index structure for subsequent use. + ** + ** The column affinity string will eventually be deleted by + ** sqliteDeleteIndex() when the Index structure itself is cleaned + ** up. + */ + int n; + Table *pTab = pIdx->pTable; + pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); if( !pIdx->zColAff ){ - /* The first time a column affinity string for a particular index is - ** required, it is allocated and populated here. It is then stored as - ** a member of the Index structure for subsequent use. - ** - ** The column affinity string will eventually be deleted by - ** sqliteDeleteIndex() when the Index structure itself is cleaned - ** up. - */ - int n; - Table *pTab = pIdx->pTable; - pIdx->zColAff = (char *)sqlite3DbMallocRaw(0, pIdx->nColumn+1); - if( !pIdx->zColAff ){ - sqlite3OomFault(db); - return 0; - } - for(n=0; nnColumn; n++){ - i16 x = pIdx->aiColumn[n]; - if( x>=0 ){ - pIdx->zColAff[n] = pTab->aCol[x].affinity; - }else if( x==XN_ROWID ){ - pIdx->zColAff[n] = SQLITE_AFF_INTEGER; - }else{ - char aff; - assert( x==XN_EXPR ); - assert( pIdx->aColExpr!=0 ); - aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); - if( aff==0 ) aff = SQLITE_AFF_BLOB; - pIdx->zColAff[n] = aff; - } - } - pIdx->zColAff[n] = 0; + sqlite3OomFault(db); + return 0; } - + for(n=0; nnColumn; n++){ + i16 x = pIdx->aiColumn[n]; + char aff; + if( x>=0 ){ + aff = pTab->aCol[x].affinity; + }else if( x==XN_ROWID ){ + aff = SQLITE_AFF_INTEGER; + }else{ + assert( x==XN_EXPR ); + assert( pIdx->bHasExpr ); + assert( pIdx->aColExpr!=0 ); + aff = sqlite3ExprAffinity(pIdx->aColExpr->a[n].pExpr); + } + if( affSQLITE_AFF_NUMERIC) aff = SQLITE_AFF_NUMERIC; + pIdx->zColAff[n] = aff; + } + pIdx->zColAff[n] = 0; + return pIdx->zColAff; +} +SQLITE_PRIVATE const char *sqlite3IndexAffinityStr(sqlite3 *db, Index *pIdx){ + if( !pIdx->zColAff ) return computeIndexAffStr(db, pIdx); return pIdx->zColAff; } + /* +** Compute an affinity string for a table. Space is obtained +** from sqlite3DbMalloc(). The caller is responsible for freeing +** the space when done. +*/ +SQLITE_PRIVATE char *sqlite3TableAffinityStr(sqlite3 *db, const Table *pTab){ + char *zColAff; + zColAff = (char *)sqlite3DbMallocRaw(db, pTab->nCol+1); + if( zColAff ){ + int i, j; + for(i=j=0; inCol; i++){ + if( (pTab->aCol[i].colFlags & COLFLAG_VIRTUAL)==0 ){ + zColAff[j++] = pTab->aCol[i].affinity; + } + } + do{ + zColAff[j--] = 0; + }while( j>=0 && zColAff[j]<=SQLITE_AFF_BLOB ); + } + return zColAff; +} + +/* +** Make changes to the evolving bytecode to do affinity transformations +** of values that are about to be gathered into a row for table pTab. +** +** For ordinary (legacy, non-strict) tables: +** ----------------------------------------- +** ** Compute the affinity string for table pTab, if it has not already been ** computed. As an optimization, omit trailing SQLITE_AFF_BLOB affinities. ** -** If the affinity exists (if it is no entirely SQLITE_AFF_BLOB values) and -** if iReg>0 then code an OP_Affinity opcode that will set the affinities -** for register iReg and following. Or if affinities exists and iReg==0, +** If the affinity string is empty (because it was all SQLITE_AFF_BLOB entries +** which were then optimized out) then this routine becomes a no-op. +** +** Otherwise if iReg>0 then code an OP_Affinity opcode that will set the +** affinities for register iReg and following. Or if iReg==0, ** then just set the P4 operand of the previous opcode (which should be ** an OP_MakeRecord) to the affinity string. ** ** A column affinity string has one character per column: ** -** Character Column affinity -** ------------------------------ -** 'A' BLOB -** 'B' TEXT -** 'C' NUMERIC -** 'D' INTEGER -** 'E' REAL +** Character Column affinity +** --------- --------------- +** 'A' BLOB +** 'B' TEXT +** 'C' NUMERIC +** 'D' INTEGER +** 'E' REAL +** +** For STRICT tables: +** ------------------ +** +** Generate an appropriate OP_TypeCheck opcode that will verify the +** datatypes against the column definitions in pTab. If iReg==0, that +** means an OP_MakeRecord opcode has already been generated and should be +** the last opcode generated. The new OP_TypeCheck needs to be inserted +** before the OP_MakeRecord. The new OP_TypeCheck should use the same +** register set as the OP_MakeRecord. If iReg>0 then register iReg is +** the first of a series of registers that will form the new record. +** Apply the type checking to that array of registers. */ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ int i; - char *zColAff = pTab->zColAff; + char *zColAff; + if( pTab->tabFlags & TF_Strict ){ + if( iReg==0 ){ + /* Move the previous opcode (which should be OP_MakeRecord) forward + ** by one slot and insert a new OP_TypeCheck where the current + ** OP_MakeRecord is found */ + VdbeOp *pPrev; + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + pPrev = sqlite3VdbeGetLastOp(v); + assert( pPrev!=0 ); + assert( pPrev->opcode==OP_MakeRecord || sqlite3VdbeDb(v)->mallocFailed ); + pPrev->opcode = OP_TypeCheck; + sqlite3VdbeAddOp3(v, OP_MakeRecord, pPrev->p1, pPrev->p2, pPrev->p3); + }else{ + /* Insert an isolated OP_Typecheck */ + sqlite3VdbeAddOp2(v, OP_TypeCheck, iReg, pTab->nNVCol); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); + } + return; + } + zColAff = pTab->zColAff; if( zColAff==0 ){ - sqlite3 *db = sqlite3VdbeDb(v); - zColAff = (char *)sqlite3DbMallocRaw(0, pTab->nCol+1); + zColAff = sqlite3TableAffinityStr(0, pTab); if( !zColAff ){ - sqlite3OomFault(db); + sqlite3OomFault(sqlite3VdbeDb(v)); return; } - - for(i=0; inCol; i++){ - zColAff[i] = pTab->aCol[i].affinity; - } - do{ - zColAff[i--] = 0; - }while( i>=0 && zColAff[i]==SQLITE_AFF_BLOB ); pTab->zColAff = zColAff; } - i = sqlite3Strlen30(zColAff); + assert( zColAff!=0 ); + i = sqlite3Strlen30NN(zColAff); if( i ){ if( iReg ){ sqlite3VdbeAddOp4(v, OP_Affinity, iReg, i, 0, zColAff, i); }else{ + assert( sqlite3VdbeGetLastOp(v)->opcode==OP_MakeRecord + || sqlite3VdbeDb(v)->mallocFailed ); sqlite3VdbeChangeP4(v, -1, zColAff, i); } } @@ -106021,9 +133886,9 @@ SQLITE_PRIVATE void sqlite3TableAffinity(Vdbe *v, Table *pTab, int iReg){ /* ** Return non-zero if the table pTab in database iDb or any of its indices -** have been opened at any point in the VDBE program. This is used to see if -** a statement of the form "INSERT INTO SELECT ..." can -** run without using a temporary table for the results of the SELECT. +** have been opened at any point in the VDBE program. This is used to see if +** a statement of the form "INSERT INTO SELECT ..." can +** run without using a temporary table for the results of the SELECT. */ static int readsTable(Parse *p, int iDb, Table *pTab){ Vdbe *v = sqlite3GetVdbe(p); @@ -106038,7 +133903,7 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ assert( pOp!=0 ); if( pOp->opcode==OP_OpenRead && pOp->p3==iDb ){ Index *pIndex; - int tnum = pOp->p2; + Pgno tnum = pOp->p2; if( tnum==pTab->tnum ){ return 1; } @@ -106059,11 +133924,132 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ return 0; } +/* This walker callback will compute the union of colFlags flags for all +** referenced columns in a CHECK constraint or generated column expression. +*/ +static int exprColumnFlagUnion(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_COLUMN && pExpr->iColumn>=0 ){ + assert( pExpr->iColumn < pWalker->u.pTab->nCol ); + pWalker->eCode |= pWalker->u.pTab->aCol[pExpr->iColumn].colFlags; + } + return WRC_Continue; +} + +#ifndef SQLITE_OMIT_GENERATED_COLUMNS +/* +** All regular columns for table pTab have been puts into registers +** starting with iRegStore. The registers that correspond to STORED +** or VIRTUAL columns have not yet been initialized. This routine goes +** back and computes the values for those columns based on the previously +** computed normal columns. +*/ +SQLITE_PRIVATE void sqlite3ComputeGeneratedColumns( + Parse *pParse, /* Parsing context */ + int iRegStore, /* Register holding the first column */ + Table *pTab /* The table */ +){ + int i; + Walker w; + Column *pRedo; + int eProgress; + VdbeOp *pOp; + + assert( pTab->tabFlags & TF_HasGenerated ); + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + + /* Before computing generated columns, first go through and make sure + ** that appropriate affinity has been applied to the regular columns + */ + sqlite3TableAffinity(pParse->pVdbe, pTab, iRegStore); + if( (pTab->tabFlags & TF_HasStored)!=0 ){ + pOp = sqlite3VdbeGetLastOp(pParse->pVdbe); + if( pOp->opcode==OP_Affinity ){ + /* Change the OP_Affinity argument to '@' (NONE) for all stored + ** columns. '@' is the no-op affinity and those columns have not + ** yet been computed. */ + int ii, jj; + char *zP4 = pOp->p4.z; + assert( zP4!=0 ); + assert( pOp->p4type==P4_DYNAMIC ); + for(ii=jj=0; zP4[jj]; ii++){ + if( pTab->aCol[ii].colFlags & COLFLAG_VIRTUAL ){ + continue; + } + if( pTab->aCol[ii].colFlags & COLFLAG_STORED ){ + zP4[jj] = SQLITE_AFF_NONE; + } + jj++; + } + }else if( pOp->opcode==OP_TypeCheck ){ + /* If an OP_TypeCheck was generated because the table is STRICT, + ** then set the P3 operand to indicate that generated columns should + ** not be checked */ + pOp->p3 = 1; + } + } + + /* Because there can be multiple generated columns that refer to one another, + ** this is a two-pass algorithm. On the first pass, mark all generated + ** columns as "not available". + */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + pTab->aCol[i].colFlags |= COLFLAG_NOTAVAIL; + } + } + + w.u.pTab = pTab; + w.xExprCallback = exprColumnFlagUnion; + w.xSelectCallback = 0; + w.xSelectCallback2 = 0; + + /* On the second pass, compute the value of each NOT-AVAILABLE column. + ** Companion code in the TK_COLUMN case of sqlite3ExprCodeTarget() will + ** compute dependencies and mark remove the COLSPAN_NOTAVAIL mark, as + ** they are needed. + */ + pParse->iSelfTab = -iRegStore; + do{ + eProgress = 0; + pRedo = 0; + for(i=0; inCol; i++){ + Column *pCol = pTab->aCol + i; + if( (pCol->colFlags & COLFLAG_NOTAVAIL)!=0 ){ + int x; + pCol->colFlags |= COLFLAG_BUSY; + w.eCode = 0; + sqlite3WalkExpr(&w, sqlite3ColumnExpr(pTab, pCol)); + pCol->colFlags &= ~COLFLAG_BUSY; + if( w.eCode & COLFLAG_NOTAVAIL ){ + pRedo = pCol; + continue; + } + eProgress = 1; + assert( pCol->colFlags & COLFLAG_GENERATED ); + x = sqlite3TableColumnToStorage(pTab, i) + iRegStore; + sqlite3ExprCodeGeneratedColumn(pParse, pTab, pCol, x); + pCol->colFlags &= ~COLFLAG_NOTAVAIL; + } + } + }while( pRedo && eProgress ); + if( pRedo ){ + sqlite3ErrorMsg(pParse, "generated column loop on \"%s\"", pRedo->zCnName); + } + pParse->iSelfTab = 0; +} +#endif /* SQLITE_OMIT_GENERATED_COLUMNS */ + + #ifndef SQLITE_OMIT_AUTOINCREMENT /* ** Locate or create an AutoincInfo structure associated with table pTab ** which is in database iDb. Return the register number for the register -** that holds the maximum rowid. +** that holds the maximum rowid. Return zero if pTab is not an AUTOINCREMENT +** table. (Also return zero when doing a VACUUM since we do not want to +** update the AUTOINCREMENT counters during a VACUUM.) ** ** There is at most one AutoincInfo structure per table even if the ** same table is autoincremented multiple times due to inserts within @@ -106071,11 +134057,12 @@ static int readsTable(Parse *p, int iDb, Table *pTab){ ** first use of table pTab. On 2nd and subsequent uses, the original ** AutoincInfo structure is used. ** -** Three memory locations are allocated: +** Four consecutive registers are allocated: ** -** (1) Register to hold the name of the pTab table. -** (2) Register to hold the maximum ROWID of pTab. -** (3) Register to hold the rowid in sqlite_sequence of pTab +** (1) The name of the pTab table. +** (2) The maximum ROWID of pTab. +** (3) The rowid in sqlite_sequence of pTab +** (4) The original value of the max ROWID in pTab, or NULL if none ** ** The 2nd register is the one that is returned. That is all the ** insert routine needs to know about. @@ -106086,22 +134073,41 @@ static int autoIncBegin( Table *pTab /* The table we are writing to */ ){ int memId = 0; /* Register holding maximum rowid */ - if( pTab->tabFlags & TF_Autoincrement ){ + assert( pParse->db->aDb[iDb].pSchema!=0 ); + if( (pTab->tabFlags & TF_Autoincrement)!=0 + && (pParse->db->mDbFlags & DBFLAG_Vacuum)==0 + ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); AutoincInfo *pInfo; + Table *pSeqTab = pParse->db->aDb[iDb].pSchema->pSeqTab; + + /* Verify that the sqlite_sequence table exists and is an ordinary + ** rowid table with exactly two columns. + ** Ticket d8dc2b3a58cd5dc2918a1d4acb 2018-05-23 */ + if( pSeqTab==0 + || !HasRowid(pSeqTab) + || NEVER(IsVirtual(pSeqTab)) + || pSeqTab->nCol!=2 + ){ + pParse->nErr++; + pParse->rc = SQLITE_CORRUPT_SEQUENCE; + return 0; + } pInfo = pToplevel->pAinc; while( pInfo && pInfo->pTab!=pTab ){ pInfo = pInfo->pNext; } if( pInfo==0 ){ pInfo = sqlite3DbMallocRawNN(pParse->db, sizeof(*pInfo)); - if( pInfo==0 ) return 0; + sqlite3ParserAddCleanup(pToplevel, sqlite3DbFree, pInfo); + testcase( pParse->earlyCleanup ); + if( pParse->db->mallocFailed ) return 0; pInfo->pNext = pToplevel->pAinc; pToplevel->pAinc = pInfo; pInfo->pTab = pTab; pInfo->iDb = iDb; pToplevel->nMem++; /* Register to hold name of table */ pInfo->regCtr = ++pToplevel->nMem; /* Max rowid register */ - pToplevel->nMem++; /* Rowid in sqlite_sequence */ + pToplevel->nMem +=2; /* Rowid in sqlite_sequence + orig max val */ } memId = pInfo->regCtr; } @@ -106110,7 +134116,7 @@ static int autoIncBegin( /* ** This routine generates code that will initialize all of the -** register used by the autoincrement tracker. +** register used by the autoincrement tracker. */ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ AutoincInfo *p; /* Information about an AUTOINCREMENT */ @@ -106129,15 +134135,17 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList autoInc[] = { /* 0 */ {OP_Null, 0, 0, 0}, - /* 1 */ {OP_Rewind, 0, 9, 0}, + /* 1 */ {OP_Rewind, 0, 10, 0}, /* 2 */ {OP_Column, 0, 0, 0}, - /* 3 */ {OP_Ne, 0, 7, 0}, + /* 3 */ {OP_Ne, 0, 9, 0}, /* 4 */ {OP_Rowid, 0, 0, 0}, /* 5 */ {OP_Column, 0, 1, 0}, - /* 6 */ {OP_Goto, 0, 9, 0}, - /* 7 */ {OP_Next, 0, 2, 0}, - /* 8 */ {OP_Integer, 0, 0, 0}, - /* 9 */ {OP_Close, 0, 0, 0} + /* 6 */ {OP_AddImm, 0, 0, 0}, + /* 7 */ {OP_Copy, 0, 0, 0}, + /* 8 */ {OP_Goto, 0, 11, 0}, + /* 9 */ {OP_Next, 0, 2, 0}, + /* 10 */ {OP_Integer, 0, 0, 0}, + /* 11 */ {OP_Close, 0, 0, 0} }; VdbeOp *aOp; pDb = &db->aDb[p->iDb]; @@ -106148,14 +134156,18 @@ SQLITE_PRIVATE void sqlite3AutoincrementBegin(Parse *pParse){ aOp = sqlite3VdbeAddOpList(v, ArraySize(autoInc), autoInc, iLn); if( aOp==0 ) break; aOp[0].p2 = memId; - aOp[0].p3 = memId+1; + aOp[0].p3 = memId+2; aOp[2].p3 = memId; aOp[3].p1 = memId-1; aOp[3].p3 = memId; aOp[3].p5 = SQLITE_JUMPIFNULL; aOp[4].p2 = memId+1; aOp[5].p3 = memId; - aOp[8].p2 = memId; + aOp[6].p1 = memId; + aOp[7].p2 = memId+2; + aOp[7].p1 = memId; + aOp[10].p2 = memId; + if( pParse->nTab==0 ) pParse->nTab = 1; } } @@ -106202,6 +134214,8 @@ static SQLITE_NOINLINE void autoIncrementEnd(Parse *pParse){ iRec = sqlite3GetTempReg(pParse); assert( sqlite3SchemaMutexHeld(db, 0, pDb->pSchema) ); + sqlite3VdbeAddOp3(v, OP_Le, memId+2, sqlite3VdbeCurrentAddr(v)+7, memId); + VdbeCoverage(v); sqlite3OpenTable(pParse, 0, p->iDb, pDb->pSchema->pSeqTab, OP_OpenWrite); aOp = sqlite3VdbeAddOpList(v, ArraySize(autoIncEnd), autoIncEnd, iLn); if( aOp==0 ) break; @@ -106227,6 +134241,210 @@ SQLITE_PRIVATE void sqlite3AutoincrementEnd(Parse *pParse){ # define autoIncStep(A,B,C) #endif /* SQLITE_OMIT_AUTOINCREMENT */ +/* +** If argument pVal is a Select object returned by an sqlite3MultiValues() +** that was able to use the co-routine optimization, finish coding the +** co-routine. +*/ +SQLITE_PRIVATE void sqlite3MultiValuesEnd(Parse *pParse, Select *pVal){ + if( ALWAYS(pVal) && pVal->pSrc->nSrc>0 ){ + SrcItem *pItem = &pVal->pSrc->a[0]; + assert( (pItem->fg.isSubquery && pItem->u4.pSubq!=0) || pParse->nErr ); + if( pItem->fg.isSubquery ){ + sqlite3VdbeEndCoroutine(pParse->pVdbe, pItem->u4.pSubq->regReturn); + sqlite3VdbeJumpHere(pParse->pVdbe, pItem->u4.pSubq->addrFillSub - 1); + } + } +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are constant. +*/ +static int exprListIsConstant(Parse *pParse, ExprList *pRow){ + int ii; + for(ii=0; iinExpr; ii++){ + if( 0==sqlite3ExprIsConstant(pParse, pRow->a[ii].pExpr) ) return 0; + } + return 1; +} + +/* +** Return true if all expressions in the expression-list passed as the +** only argument are both constant and have no affinity. +*/ +static int exprListIsNoAffinity(Parse *pParse, ExprList *pRow){ + int ii; + if( exprListIsConstant(pParse,pRow)==0 ) return 0; + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pRow->a[ii].pExpr; + assert( pExpr->op!=TK_RAISE ); + assert( pExpr->affExpr==0 ); + if( 0!=sqlite3ExprAffinity(pExpr) ) return 0; + } + return 1; + +} + +/* +** This function is called by the parser for the second and subsequent +** rows of a multi-row VALUES clause. Argument pLeft is the part of +** the VALUES clause already parsed, argument pRow is the vector of values +** for the new row. The Select object returned represents the complete +** VALUES clause, including the new row. +** +** There are two ways in which this may be achieved - by incremental +** coding of a co-routine (the "co-routine" method) or by returning a +** Select object equivalent to the following (the "UNION ALL" method): +** +** "pLeft UNION ALL SELECT pRow" +** +** If the VALUES clause contains a lot of rows, this compound Select +** object may consume a lot of memory. +** +** When the co-routine method is used, each row that will be returned +** by the VALUES clause is coded into part of a co-routine as it is +** passed to this function. The returned Select object is equivalent to: +** +** SELECT * FROM ( +** Select object to read co-routine +** ) +** +** The co-routine method is used in most cases. Exceptions are: +** +** a) If the current statement has a WITH clause. This is to avoid +** statements like: +** +** WITH cte AS ( VALUES('x'), ('y') ... ) +** SELECT * FROM cte AS a, cte AS b; +** +** This will not work, as the co-routine uses a hard-coded register +** for its OP_Yield instructions, and so it is not possible for two +** cursors to iterate through it concurrently. +** +** b) The schema is currently being parsed (i.e. the VALUES clause is part +** of a schema item like a VIEW or TRIGGER). In this case there is no VM +** being generated when parsing is taking place, and so generating +** a co-routine is not possible. +** +** c) There are non-constant expressions in the VALUES clause (e.g. +** the VALUES clause is part of a correlated sub-query). +** +** d) One or more of the values in the first row of the VALUES clause +** has an affinity (i.e. is a CAST expression). This causes problems +** because the complex rules SQLite uses (see function +** sqlite3SubqueryColumnTypes() in select.c) to determine the effective +** affinity of such a column for all rows require access to all values in +** the column simultaneously. +*/ +SQLITE_PRIVATE Select *sqlite3MultiValues(Parse *pParse, Select *pLeft, ExprList *pRow){ + + if( pParse->bHasWith /* condition (a) above */ + || pParse->db->init.busy /* condition (b) above */ + || exprListIsConstant(pParse,pRow)==0 /* condition (c) above */ + || (pLeft->pSrc->nSrc==0 && + exprListIsNoAffinity(pParse,pLeft->pEList)==0) /* condition (d) above */ + || IN_SPECIAL_PARSE + ){ + /* The co-routine method cannot be used. Fall back to UNION ALL. */ + Select *pSelect = 0; + int f = SF_Values | SF_MultiValue; + if( pLeft->pSrc->nSrc ){ + sqlite3MultiValuesEnd(pParse, pLeft); + f = SF_Values; + }else if( pLeft->pPrior ){ + /* In this case set the SF_MultiValue flag only if it was set on pLeft */ + f = (f & pLeft->selFlags); + } + pSelect = sqlite3SelectNew(pParse, pRow, 0, 0, 0, 0, 0, f, 0); + pLeft->selFlags &= ~SF_MultiValue; + if( pSelect ){ + pSelect->op = TK_ALL; + pSelect->pPrior = pLeft; + pLeft = pSelect; + } + }else{ + SrcItem *p = 0; /* SrcItem that reads from co-routine */ + + if( pLeft->pSrc->nSrc==0 ){ + /* Co-routine has not yet been started and the special Select object + ** that accesses the co-routine has not yet been created. This block + ** does both those things. */ + Vdbe *v = sqlite3GetVdbe(pParse); + Select *pRet = sqlite3SelectNew(pParse, 0, 0, 0, 0, 0, 0, 0, 0); + + /* Ensure the database schema has been read. This is to ensure we have + ** the correct text encoding. */ + if( (pParse->db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ){ + sqlite3ReadSchema(pParse); + } + + if( pRet ){ + SelectDest dest; + Subquery *pSubq; + pRet->pSrc->nSrc = 1; + pRet->pPrior = pLeft->pPrior; + pRet->op = pLeft->op; + if( pRet->pPrior ) pRet->selFlags |= SF_Values; + pLeft->pPrior = 0; + pLeft->op = TK_SELECT; + assert( pLeft->pNext==0 ); + assert( pRet->pNext==0 ); + p = &pRet->pSrc->a[0]; + p->fg.viaCoroutine = 1; + p->iCursor = -1; + assert( !p->fg.isIndexedBy && !p->fg.isTabFunc ); + p->u1.nRow = 2; + if( sqlite3SrcItemAttachSubquery(pParse, p, pLeft, 0) ){ + pSubq = p->u4.pSubq; + pSubq->addrFillSub = sqlite3VdbeCurrentAddr(v) + 1; + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, + pSubq->regReturn, 0, pSubq->addrFillSub); + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); + + /* Allocate registers for the output of the co-routine. Do so so + ** that there are two unused registers immediately before those + ** used by the co-routine. This allows the code in sqlite3Insert() + ** to use these registers directly, instead of copying the output + ** of the co-routine to a separate array for processing. */ + dest.iSdst = pParse->nMem + 3; + dest.nSdst = pLeft->pEList->nExpr; + pParse->nMem += 2 + dest.nSdst; + + pLeft->selFlags |= SF_MultiValue; + sqlite3Select(pParse, pLeft, &dest); + pSubq->regResult = dest.iSdst; + assert( pParse->nErr || dest.iSdst>0 ); + } + pLeft = pRet; + } + }else{ + p = &pLeft->pSrc->a[0]; + assert( !p->fg.isTabFunc && !p->fg.isIndexedBy ); + p->u1.nRow++; + } + + if( pParse->nErr==0 ){ + Subquery *pSubq; + assert( p!=0 ); + assert( p->fg.isSubquery ); + pSubq = p->u4.pSubq; + assert( pSubq!=0 ); + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + if( pSubq->pSelect->pEList->nExpr!=pRow->nExpr ){ + sqlite3SelectWrongNumTermsError(pParse, pSubq->pSelect); + }else{ + sqlite3ExprCodeExprList(pParse, pRow, pSubq->regResult, 0, 0); + sqlite3VdbeAddOp1(pParse->pVdbe, OP_Yield, pSubq->regReturn); + } + } + sqlite3ExprListDelete(pParse->db, pRow); + } + + return pLeft; +} /* Forward declaration */ static int xferOptimization( @@ -106338,14 +134556,13 @@ SQLITE_PRIVATE void sqlite3Insert( Parse *pParse, /* Parser context */ SrcList *pTabList, /* Name of table into which we are inserting */ Select *pSelect, /* A SELECT statement to use as the data source */ - IdList *pColumn, /* Column names corresponding to IDLIST. */ - int onError /* How to handle constraint errors */ + IdList *pColumn, /* Column names corresponding to IDLIST, or NULL. */ + int onError, /* How to handle constraint errors */ + Upsert *pUpsert /* ON CONFLICT clauses for upsert, or NULL */ ){ sqlite3 *db; /* The main database structure */ Table *pTab; /* The table to insert into. aka TABLE */ - char *zTab; /* Name of the table into which we are inserting */ - const char *zDb; /* Name of the database holding this table */ - int i, j, idx; /* Loop counters */ + int i, j; /* Loop counters */ Vdbe *v; /* Generate code into this virtual machine */ Index *pIdx; /* For looping over indices of the table */ int nColumn; /* Number of columns in the data */ @@ -106359,12 +134576,12 @@ SQLITE_PRIVATE void sqlite3Insert( int addrCont = 0; /* Top of insert loop. Label "C" in templates 3 and 4 */ SelectDest dest; /* Destination for SELECT on rhs of INSERT */ int iDb; /* Index of database holding TABLE */ - Db *pDb; /* The database containing table being inserted into */ u8 useTempTable = 0; /* Store SELECT results in intermediate table */ u8 appendFlag = 0; /* True if the insert is likely to be an append */ u8 withoutRowid; /* 0 for normal table. 1 for WITHOUT ROWID table */ u8 bIdListInOrder; /* True if IDLIST is in table order */ ExprList *pList = 0; /* List of VALUES() to be inserted */ + int iRegStore; /* Register in which to store next column */ /* Register allocations */ int regFromSelect = 0;/* Base register for data coming from SELECT */ @@ -106382,10 +134599,12 @@ SQLITE_PRIVATE void sqlite3Insert( #endif db = pParse->db; - memset(&dest, 0, sizeof(dest)); - if( pParse->nErr || db->mallocFailed ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto insert_cleanup; } + assert( db->mallocFailed==0 ); + dest.iSDParm = 0; /* Suppress a harmless compiler warning */ /* If the Select object is really just a simple VALUES() list with a ** single row (the common case) then keep that one row of values @@ -106401,17 +134620,14 @@ SQLITE_PRIVATE void sqlite3Insert( /* Locate the table into which we will be inserting new information. */ assert( pTabList->nSrc==1 ); - zTab = pTabList->a[0].zName; - if( NEVER(zTab==0) ) goto insert_cleanup; pTab = sqlite3SrcListLookup(pParse, pTabList); if( pTab==0 ){ goto insert_cleanup; } iDb = sqlite3SchemaToIndex(db, pTab->pSchema); assert( iDbnDb ); - pDb = &db->aDb[iDb]; - zDb = pDb->zName; - if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, zDb) ){ + if( sqlite3AuthCheck(pParse, SQLITE_INSERT, pTab->zName, 0, + db->aDb[iDb].zDbSName) ){ goto insert_cleanup; } withoutRowid = !HasRowid(pTab); @@ -106421,7 +134637,7 @@ SQLITE_PRIVATE void sqlite3Insert( */ #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_INSERT, 0, &tmask); - isView = pTab->pSelect!=0; + isView = IsView(pTab); #else # define pTrigger 0 # define tmask 0 @@ -106433,6 +134649,14 @@ SQLITE_PRIVATE void sqlite3Insert( #endif assert( (pTrigger && tmask) || (pTrigger==0 && tmask==0) ); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Insert() at %s:%d", __FILE__, __LINE__); + sqlite3TreeViewInsert(pParse->pWith, pTabList, pColumn, pSelect, pList, + onError, pUpsert, pTrigger); + } +#endif + /* If pTab is really a view, make sure it has been initialized. ** ViewGetColumnNames() is a no-op if pTab is not a view. */ @@ -106442,7 +134666,7 @@ SQLITE_PRIVATE void sqlite3Insert( /* Cannot insert into a read-only table. */ - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto insert_cleanup; } @@ -106463,7 +134687,11 @@ SQLITE_PRIVATE void sqlite3Insert( ** ** This is the 2nd template. */ - if( pColumn==0 && xferOptimization(pParse, pTab, pSelect, onError, iDb) ){ + if( pColumn==0 + && pSelect!=0 + && pTrigger==0 + && xferOptimization(pParse, pTab, pSelect, onError, iDb) + ){ assert( !pTrigger ); assert( pList==0 ); goto insert_end; @@ -106475,8 +134703,8 @@ SQLITE_PRIVATE void sqlite3Insert( */ regAutoinc = autoIncBegin(pParse, iDb, pTab); - /* Allocate registers for holding the rowid of the new row, - ** the content of the new row, and the assembled row record. + /* Allocate a block registers to hold the rowid and the values + ** for all columns of the new row. */ regRowid = regIns = pParse->nMem+1; pParse->nMem += pTab->nCol + 1; @@ -106487,7 +134715,7 @@ SQLITE_PRIVATE void sqlite3Insert( regData = regRowid+1; /* If the INSERT statement included an IDLIST term, then make sure - ** all elements of the IDLIST really are columns of the table and + ** all elements of the IDLIST really are columns of the table and ** remember the column indices. ** ** If the table has an INTEGER PRIMARY KEY column and that column @@ -106495,21 +134723,39 @@ SQLITE_PRIVATE void sqlite3Insert( ** the index into IDLIST of the primary key column. ipkColumn is ** the index of the primary key as it appears in IDLIST, not as ** is appears in the original table. (The index of the INTEGER - ** PRIMARY KEY in the original table is pTab->iPKey.) + ** PRIMARY KEY in the original table is pTab->iPKey.) After this + ** loop, if ipkColumn==(-1), that means that integer primary key + ** is unspecified, and hence the table is either WITHOUT ROWID or + ** it will automatically generated an integer primary key. + ** + ** bIdListInOrder is true if the columns in IDLIST are in storage + ** order. This enables an optimization that avoids shuffling the + ** columns into storage order. False negatives are harmless, + ** but false positives will cause database corruption. */ - bIdListInOrder = (pTab->tabFlags & TF_OOOHidden)==0; + bIdListInOrder = (pTab->tabFlags & (TF_OOOHidden|TF_HasStored))==0; if( pColumn ){ + assert( pColumn->eU4!=EU4_EXPR ); + pColumn->eU4 = EU4_IDX; for(i=0; inId; i++){ - pColumn->a[i].idx = -1; + pColumn->a[i].u4.idx = -1; } for(i=0; inId; i++){ for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zName)==0 ){ - pColumn->a[i].idx = j; + if( sqlite3StrICmp(pColumn->a[i].zName, pTab->aCol[j].zCnName)==0 ){ + pColumn->a[i].u4.idx = j; if( i!=j ) bIdListInOrder = 0; if( j==pTab->iPKey ){ ipkColumn = i; assert( !withoutRowid ); } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->aCol[j].colFlags & (COLFLAG_STORED|COLFLAG_VIRTUAL) ){ + sqlite3ErrorMsg(pParse, + "cannot INSERT into generated column \"%s\"", + pTab->aCol[j].zCnName); + goto insert_cleanup; + } +#endif break; } } @@ -106519,7 +134765,7 @@ SQLITE_PRIVATE void sqlite3Insert( bIdListInOrder = 0; }else{ sqlite3ErrorMsg(pParse, "table %S has no column named %s", - pTabList, 0, pColumn->a[i].zName); + pTabList->a, pColumn->a[i].zName); pParse->checkSchema = 1; goto insert_cleanup; } @@ -106535,23 +134781,45 @@ SQLITE_PRIVATE void sqlite3Insert( if( pSelect ){ /* Data is coming from a SELECT or from a multi-row VALUES clause. ** Generate a co-routine to run the SELECT. */ - int regYield; /* Register holding co-routine entry-point */ - int addrTop; /* Top of the co-routine */ int rc; /* Result code */ - regYield = ++pParse->nMem; - addrTop = sqlite3VdbeCurrentAddr(v) + 1; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); - sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); - dest.iSdst = bIdListInOrder ? regData : 0; - dest.nSdst = pTab->nCol; - rc = sqlite3Select(pParse, pSelect, &dest); - regFromSelect = dest.iSdst; - if( rc || db->mallocFailed || pParse->nErr ) goto insert_cleanup; - sqlite3VdbeEndCoroutine(v, regYield); - sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ - assert( pSelect->pEList ); - nColumn = pSelect->pEList->nExpr; + if( pSelect->pSrc->nSrc==1 + && pSelect->pSrc->a[0].fg.viaCoroutine + && pSelect->pPrior==0 + ){ + SrcItem *pItem = &pSelect->pSrc->a[0]; + Subquery *pSubq; + assert( pItem->fg.isSubquery ); + pSubq = pItem->u4.pSubq; + dest.iSDParm = pSubq->regReturn; + regFromSelect = pSubq->regResult; + assert( pSubq->pSelect!=0 ); + assert( pSubq->pSelect->pEList!=0 ); + nColumn = pSubq->pSelect->pEList->nExpr; + ExplainQueryPlan((pParse, 0, "SCAN %S", pItem)); + if( bIdListInOrder && nColumn==pTab->nCol ){ + regData = regFromSelect; + regRowid = regData - 1; + regIns = regRowid - (IsVirtual(pTab) ? 1 : 0); + } + }else{ + int addrTop; /* Top of the co-routine */ + int regYield = ++pParse->nMem; + addrTop = sqlite3VdbeCurrentAddr(v) + 1; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, addrTop); + sqlite3SelectDestInit(&dest, SRT_Coroutine, regYield); + dest.iSdst = bIdListInOrder ? regData : 0; + dest.nSdst = pTab->nCol; + rc = sqlite3Select(pParse, pSelect, &dest); + regFromSelect = dest.iSdst; + assert( db->pParse==pParse ); + if( rc || pParse->nErr ) goto insert_cleanup; + assert( db->mallocFailed==0 ); + sqlite3VdbeEndCoroutine(v, regYield); + sqlite3VdbeJumpHere(v, addrTop - 1); /* label B: */ + assert( pSelect->pEList ); + nColumn = pSelect->pEList->nExpr; + } /* Set useTempTable to TRUE if the result of the SELECT statement ** should be written into a temporary table (template 4). Set to @@ -106559,7 +134827,7 @@ SQLITE_PRIVATE void sqlite3Insert( ** the destination table (template 3). ** ** A temp table must be used if the table being updated is also one - ** of the tables being read by the SELECT statement. Also use a + ** of the tables being read by the SELECT statement. Also use a ** temp table in the case of row triggers. */ if( pTrigger || readsTable(pParse, iDb, pTab) ){ @@ -106595,7 +134863,7 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3ReleaseTempReg(pParse, regTempRowid); } }else{ - /* This is the case if the data for the INSERT is coming from a + /* This is the case if the data for the INSERT is coming from a ** single-row VALUES clause */ NameContext sNC; @@ -106614,33 +134882,55 @@ SQLITE_PRIVATE void sqlite3Insert( } /* If there is no IDLIST term but the table has an integer primary - ** key, the set the ipkColumn variable to the integer primary key + ** key, the set the ipkColumn variable to the integer primary key ** column index in the original table definition. */ if( pColumn==0 && nColumn>0 ){ ipkColumn = pTab->iPKey; - } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( ipkColumn>=0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + for(i=ipkColumn-1; i>=0; i--){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[i].colFlags & COLFLAG_STORED ); + ipkColumn--; + } + } + } +#endif - /* Make sure the number of columns in the source data matches the number - ** of columns to be inserted into the table. - */ - for(i=0; inCol; i++){ - nHidden += (IsHiddenColumn(&pTab->aCol[i]) ? 1 : 0); - } - if( pColumn==0 && nColumn && nColumn!=(pTab->nCol-nHidden) ){ - sqlite3ErrorMsg(pParse, - "table %S has %d columns but %d values were supplied", - pTabList, 0, pTab->nCol-nHidden, nColumn); - goto insert_cleanup; + /* Make sure the number of columns in the source data matches the number + ** of columns to be inserted into the table. + */ + assert( TF_HasHidden==COLFLAG_HIDDEN ); + assert( TF_HasGenerated==COLFLAG_GENERATED ); + assert( COLFLAG_NOINSERT==(COLFLAG_GENERATED|COLFLAG_HIDDEN) ); + if( (pTab->tabFlags & (TF_HasGenerated|TF_HasHidden))!=0 ){ + for(i=0; inCol; i++){ + if( pTab->aCol[i].colFlags & COLFLAG_NOINSERT ) nHidden++; + } + } + if( nColumn!=(pTab->nCol-nHidden) ){ + sqlite3ErrorMsg(pParse, + "table %S has %d columns but %d values were supplied", + pTabList->a, pTab->nCol-nHidden, nColumn); + goto insert_cleanup; + } } if( pColumn!=0 && nColumn!=pColumn->nId ){ sqlite3ErrorMsg(pParse, "%d values for %d columns", nColumn, pColumn->nId); goto insert_cleanup; } - + /* Initialize the count of rows to be inserted */ - if( db->flags & SQLITE_CountRows ){ + if( (db->flags & SQLITE_CountRows)!=0 + && !pParse->nested + && !pParse->pTriggerTab + && !pParse->bReturning + ){ regRowCount = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); } @@ -106650,14 +134940,49 @@ SQLITE_PRIVATE void sqlite3Insert( int nIdx; nIdx = sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, -1, 0, &iDataCur, &iIdxCur); - aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+1)); + aRegIdx = sqlite3DbMallocRawNN(db, sizeof(int)*(nIdx+2)); if( aRegIdx==0 ){ goto insert_cleanup; } - for(i=0; ipIndex; ipNext, i++){ + assert( pIdx ); aRegIdx[i] = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; } + aRegIdx[i] = ++pParse->nMem; /* Register to store the table record */ } +#ifndef SQLITE_OMIT_UPSERT + if( pUpsert ){ + Upsert *pNx; + if( IsVirtual(pTab) ){ + sqlite3ErrorMsg(pParse, "UPSERT not implemented for virtual table \"%s\"", + pTab->zName); + goto insert_cleanup; + } + if( IsView(pTab) ){ + sqlite3ErrorMsg(pParse, "cannot UPSERT a view"); + goto insert_cleanup; + } + if( sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget) ){ + goto insert_cleanup; + } + pTabList->a[0].iCursor = iDataCur; + pNx = pUpsert; + do{ + pNx->pUpsertSrc = pTabList; + pNx->regData = regData; + pNx->iDataCur = iDataCur; + pNx->iIdxCur = iIdxCur; + if( pNx->pUpsertTarget ){ + if( sqlite3UpsertAnalyzeTarget(pParse, pTabList, pNx, pUpsert) ){ + goto insert_cleanup; + } + } + pNx = pNx->pNextUpsert; + }while( pNx!=0 ); + } +#endif + /* This is the top of the main insertion loop */ if( useTempTable ){ @@ -106681,13 +135006,106 @@ SQLITE_PRIVATE void sqlite3Insert( ** goto C ** D: ... */ + sqlite3VdbeReleaseRegisters(pParse, regData, pTab->nCol, 0, 0); addrInsTop = addrCont = sqlite3VdbeAddOp1(v, OP_Yield, dest.iSDParm); VdbeCoverage(v); + if( ipkColumn>=0 ){ + /* tag-20191021-001: If the INTEGER PRIMARY KEY is being generated by the + ** SELECT, go ahead and copy the value into the rowid slot now, so that + ** the value does not get overwritten by a NULL at tag-20191021-002. */ + sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + } } + /* Compute data for ordinary columns of the new entry. Values + ** are written in storage order into registers starting with regData. + ** Only ordinary columns are computed in this loop. The rowid + ** (if there is one) is computed later and generated columns are + ** computed after the rowid since they might depend on the value + ** of the rowid. + */ + nHidden = 0; + iRegStore = regData; assert( regData==regRowid+1 ); + for(i=0; inCol; i++, iRegStore++){ + int k; + u32 colFlags; + assert( i>=nHidden ); + if( i==pTab->iPKey ){ + /* tag-20191021-002: References to the INTEGER PRIMARY KEY are filled + ** using the rowid. So put a NULL in the IPK slot of the record to avoid + ** using excess space. The file format definition requires this extra + ** NULL - we cannot optimize further by skipping the column completely */ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + continue; + } + if( ((colFlags = pTab->aCol[i].colFlags) & COLFLAG_NOINSERT)!=0 ){ + nHidden++; + if( (colFlags & COLFLAG_VIRTUAL)!=0 ){ + /* Virtual columns do not participate in OP_MakeRecord. So back up + ** iRegStore by one slot to compensate for the iRegStore++ in the + ** outer for() loop */ + iRegStore--; + continue; + }else if( (colFlags & COLFLAG_STORED)!=0 ){ + /* Stored columns are computed later. But if there are BEFORE + ** triggers, the slots used for stored columns will be OP_Copy-ed + ** to a second block of registers, so the register needs to be + ** initialized to NULL to avoid an uninitialized register read */ + if( tmask & TRIGGER_BEFORE ){ + sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); + } + continue; + }else if( pColumn==0 ){ + /* Hidden columns that are not explicitly named in the INSERT + ** get there default value */ + sqlite3ExprCodeFactorable(pParse, + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + iRegStore); + continue; + } + } + if( pColumn ){ + assert( pColumn->eU4==EU4_IDX ); + for(j=0; jnId && pColumn->a[j].u4.idx!=i; j++){} + if( j>=pColumn->nId ){ + /* A column not named in the insert column list gets its + ** default value */ + sqlite3ExprCodeFactorable(pParse, + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + iRegStore); + continue; + } + k = j; + }else if( nColumn==0 ){ + /* This is INSERT INTO ... DEFAULT VALUES. Load the default value. */ + sqlite3ExprCodeFactorable(pParse, + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + iRegStore); + continue; + }else{ + k = i - nHidden; + } + + if( useTempTable ){ + sqlite3VdbeAddOp3(v, OP_Column, srcTab, k, iRegStore); + }else if( pSelect ){ + if( regFromSelect!=regData ){ + sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+k, iRegStore); + } + }else{ + Expr *pX = pList->a[k].pExpr; + int y = sqlite3ExprCodeTarget(pParse, pX, iRegStore); + if( y!=iRegStore ){ + sqlite3VdbeAddOp2(v, + ExprHasProperty(pX, EP_Subquery) ? OP_Copy : OP_SCopy, y, iRegStore); + } + } + } + + /* Run the BEFORE and INSTEAD OF triggers, if there are any */ - endOfLoop = sqlite3VdbeMakeLabel(v); + endOfLoop = sqlite3VdbeMakeLabel(pParse); if( tmask & TRIGGER_BEFORE ){ int regCols = sqlite3GetTempRange(pParse, pTab->nCol+1); @@ -106714,30 +135132,21 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp1(v, OP_MustBeInt, regCols); VdbeCoverage(v); } - /* Cannot have triggers on a virtual table. If it were possible, - ** this block would have to account for hidden column. - */ - assert( !IsVirtual(pTab) ); + /* Copy the new data already generated. */ + assert( pTab->nNVCol>0 || pParse->nErr>0 ); + sqlite3VdbeAddOp3(v, OP_Copy, regRowid+1, regCols+1, pTab->nNVCol-1); - /* Create the new column data - */ - for(i=j=0; inCol; i++){ - if( pColumn ){ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( (!useTempTable && !pList) || (pColumn && j>=pColumn->nId) - || (pColumn==0 && IsOrdinaryHiddenColumn(&pTab->aCol[i])) ){ - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regCols+i+1); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, regCols+i+1); - }else{ - assert( pSelect==0 ); /* Otherwise useTempTable is true */ - sqlite3ExprCodeAndCache(pParse, pList->a[j].pExpr, regCols+i+1); - } - if( pColumn==0 && !IsOrdinaryHiddenColumn(&pTab->aCol[i]) ) j++; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** refers to the ROWID. */ + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regCols+1, pTab); } +#endif /* If this is an INSERT on a view with an INSTEAD OF INSERT trigger, ** do not attempt any conversions before assembling the record. @@ -106749,35 +135158,30 @@ SQLITE_PRIVATE void sqlite3Insert( } /* Fire BEFORE or INSTEAD OF triggers */ - sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_BEFORE, pTab, regCols-pTab->nCol-1, onError, endOfLoop); sqlite3ReleaseTempRange(pParse, regCols, pTab->nCol+1); } - /* Compute the content of the next row to insert into a range of - ** registers beginning at regIns. - */ if( !isView ){ if( IsVirtual(pTab) ){ /* The row that the VUpdate opcode will delete: none */ sqlite3VdbeAddOp2(v, OP_Null, 0, regIns); } if( ipkColumn>=0 ){ + /* Compute the new rowid */ if( useTempTable ){ sqlite3VdbeAddOp3(v, OP_Column, srcTab, ipkColumn, regRowid); }else if( pSelect ){ - sqlite3VdbeAddOp2(v, OP_Copy, regFromSelect+ipkColumn, regRowid); + /* Rowid already initialized at tag-20191021-001 */ }else{ - VdbeOp *pOp; - sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); - pOp = sqlite3VdbeGetOp(v, -1); - if( ALWAYS(pOp) && pOp->opcode==OP_Null && !IsVirtual(pTab) ){ + Expr *pIpk = pList->a[ipkColumn].pExpr; + if( pIpk->op==TK_NULL && !IsVirtual(pTab) ){ + sqlite3VdbeAddOp3(v, OP_NewRowid, iDataCur, regRowid, regAutoinc); appendFlag = 1; - pOp->opcode = OP_NewRowid; - pOp->p1 = iDataCur; - pOp->p2 = regRowid; - pOp->p3 = regAutoinc; + }else{ + sqlite3ExprCode(pParse, pList->a[ipkColumn].pExpr, regRowid); } } /* If the PRIMARY KEY expression is NULL, then use OP_NewRowid @@ -106803,45 +135207,15 @@ SQLITE_PRIVATE void sqlite3Insert( } autoIncStep(pParse, regAutoinc, regRowid); - /* Compute data for all columns of the new entry, beginning - ** with the first column. - */ - nHidden = 0; - for(i=0; inCol; i++){ - int iRegStore = regRowid+1+i; - if( i==pTab->iPKey ){ - /* The value of the INTEGER PRIMARY KEY column is always a NULL. - ** Whenever this column is read, the rowid will be substituted - ** in its place. Hence, fill this column with a NULL to avoid - ** taking up data space with information that will never be used. - ** As there may be shallow copies of this value, make it a soft-NULL */ - sqlite3VdbeAddOp1(v, OP_SoftNull, iRegStore); - continue; - } - if( pColumn==0 ){ - if( IsHiddenColumn(&pTab->aCol[i]) ){ - j = -1; - nHidden++; - }else{ - j = i - nHidden; - } - }else{ - for(j=0; jnId; j++){ - if( pColumn->a[j].idx==i ) break; - } - } - if( j<0 || nColumn==0 || (pColumn && j>=pColumn->nId) ){ - sqlite3ExprCodeFactorable(pParse, pTab->aCol[i].pDflt, iRegStore); - }else if( useTempTable ){ - sqlite3VdbeAddOp3(v, OP_Column, srcTab, j, iRegStore); - }else if( pSelect ){ - if( regFromSelect!=regData ){ - sqlite3VdbeAddOp2(v, OP_SCopy, regFromSelect+j, iRegStore); - } - }else{ - sqlite3ExprCode(pParse, pList->a[j].pExpr, iRegStore); - } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Compute the new value for generated columns after all other + ** columns have already been computed. This must be done after + ** computing the ROWID in case one of the generated columns + ** is derived from the INTEGER PRIMARY KEY. */ + if( pTab->tabFlags & TF_HasGenerated ){ + sqlite3ComputeGeneratedColumns(pParse, regRowid+1, pTab); } +#endif /* Generate code to check constraints and generate index keys and ** do the insertion. @@ -106856,25 +135230,46 @@ SQLITE_PRIVATE void sqlite3Insert( }else #endif { - int isReplace; /* Set to true if constraints may cause a replace */ + int isReplace = 0;/* Set to true if constraints may cause a replace */ + int bUseSeek; /* True to use OPFLAG_SEEKRESULT */ sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, - regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0 + regIns, 0, ipkColumn>=0, onError, endOfLoop, &isReplace, 0, pUpsert ); - sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + if( db->flags & SQLITE_ForeignKeys ){ + sqlite3FkCheck(pParse, pTab, 0, regIns, 0, 0); + } + + /* Set the OPFLAG_USESEEKRESULT flag if either (a) there are no REPLACE + ** constraints or (b) there are no triggers and this table is not a + ** parent table in a foreign key constraint. It is safe to set the + ** flag in the second case as if any REPLACE constraint is hit, an + ** OP_Delete or OP_IdxDelete instruction will be executed on each + ** cursor that is disturbed. And these instructions both clear the + ** VdbeCursor.seekResult variable, disabling the OPFLAG_USESEEKRESULT + ** functionality. */ + bUseSeek = (isReplace==0 || !sqlite3VdbeHasSubProgram(v)); sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, - regIns, aRegIdx, 0, appendFlag, isReplace==0); + regIns, aRegIdx, 0, appendFlag, bUseSeek + ); } +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + }else if( pParse->bReturning ){ + /* If there is a RETURNING clause, populate the rowid register with + ** constant value -1, in case one or more of the returned expressions + ** refer to the "rowid" of the view. */ + sqlite3VdbeAddOp2(v, OP_Integer, -1, regRowid); +#endif } /* Update the count of rows that are inserted */ - if( (db->flags & SQLITE_CountRows)!=0 ){ + if( regRowCount ){ sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); } if( pTrigger ){ /* Code AFTER triggers */ - sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_INSERT, 0, TRIGGER_AFTER, pTab, regData-2-pTab->nCol, onError, endOfLoop); } @@ -106888,18 +135283,21 @@ SQLITE_PRIVATE void sqlite3Insert( sqlite3VdbeAddOp1(v, OP_Close, srcTab); }else if( pSelect ){ sqlite3VdbeGoto(v, addrCont); +#ifdef SQLITE_DEBUG + /* If we are jumping back to an OP_Yield that is preceded by an + ** OP_ReleaseReg, set the p5 flag on the OP_Goto so that the + ** OP_ReleaseReg will be included in the loop. */ + if( sqlite3VdbeGetOp(v, addrCont-1)->opcode==OP_ReleaseReg ){ + assert( sqlite3VdbeGetOp(v, addrCont)->opcode==OP_Yield ); + sqlite3VdbeChangeP5(v, 1); + } +#endif sqlite3VdbeJumpHere(v, addrInsTop); } - if( !IsVirtual(pTab) && !isView ){ - /* Close all tables opened */ - if( iDataCurpIndex; pIdx; pIdx=pIdx->pNext, idx++){ - sqlite3VdbeAddOp1(v, OP_Close, idx+iIdxCur); - } - } - +#ifndef SQLITE_OMIT_XFER_OPT insert_end: +#endif /* SQLITE_OMIT_XFER_OPT */ /* Update the sqlite_sequence table by storing the content of the ** maximum rowid counter values recorded while inserting into ** autoincrement tables. @@ -106909,22 +135307,21 @@ insert_end: } /* - ** Return the number of rows inserted. If this routine is + ** Return the number of rows inserted. If this routine is ** generating code because of a call to sqlite3NestedParse(), do not ** invoke the callback function. */ - if( (db->flags&SQLITE_CountRows) && !pParse->nested && !pParse->pTriggerTab ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows inserted", SQLITE_STATIC); + if( regRowCount ){ + sqlite3CodeChangeCount(v, regRowCount, "rows inserted"); } insert_cleanup: sqlite3SrcListDelete(db, pTabList); sqlite3ExprListDelete(db, pList); + sqlite3UpsertDelete(db, pUpsert); sqlite3SelectDelete(db, pSelect); sqlite3IdListDelete(db, pColumn); - sqlite3DbFree(db, aRegIdx); + if( aRegIdx ) sqlite3DbNNFreeNN(db, aRegIdx); } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -106941,15 +135338,16 @@ insert_cleanup: #endif /* -** Meanings of bits in of pWalker->eCode for checkConstraintUnchanged() +** Meanings of bits in of pWalker->eCode for +** sqlite3ExprReferencesUpdatedColumn() */ #define CKCNSTRNT_COLUMN 0x01 /* CHECK constraint uses a changing column */ #define CKCNSTRNT_ROWID 0x02 /* CHECK constraint references the ROWID */ -/* This is the Walker callback from checkConstraintUnchanged(). Set -** bit 0x01 of pWalker->eCode if -** pWalker->eCode to 0 if this expression node references any of the -** columns that are being modifed by an UPDATE statement. +/* This is the Walker callback from sqlite3ExprReferencesUpdatedColumn(). +* Set bit 0x01 of pWalker->eCode if pWalker->eCode to 0 and if this +** expression node references any of the +** columns that are being modified by an UPDATE statement. */ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ if( pExpr->op==TK_COLUMN ){ @@ -106970,12 +135368,21 @@ static int checkConstraintExprNode(Walker *pWalker, Expr *pExpr){ ** only columns that are modified by the UPDATE are those for which ** aiChng[i]>=0, and also the ROWID is modified if chngRowid is true. ** -** Return true if CHECK constraint pExpr does not use any of the +** Return true if CHECK constraint pExpr uses any of the ** changing columns (or the rowid if it is changing). In other words, -** return true if this CHECK constraint can be skipped when validating +** return true if this CHECK constraint must be validated for ** the new row in the UPDATE statement. +** +** 2018-09-15: pExpr might also be an expression for an index-on-expressions. +** The operation of this routine is the same - return true if an only if +** the expression uses one or more of columns identified by the second and +** third arguments. */ -static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ +SQLITE_PRIVATE int sqlite3ExprReferencesUpdatedColumn( + Expr *pExpr, /* The expression to be checked */ + int *aiChng, /* aiChng[x]>=0 if column x changed by the UPDATE */ + int chngRowid /* True if UPDATE changes the rowid */ +){ Walker w; memset(&w, 0, sizeof(w)); w.eCode = 0; @@ -106990,7 +135397,71 @@ static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ testcase( w.eCode==CKCNSTRNT_COLUMN ); testcase( w.eCode==CKCNSTRNT_ROWID ); testcase( w.eCode==(CKCNSTRNT_ROWID|CKCNSTRNT_COLUMN) ); - return !w.eCode; + return w.eCode!=0; +} + +/* +** The sqlite3GenerateConstraintChecks() routine usually wants to visit +** the indexes of a table in the order provided in the Table->pIndex list. +** However, sometimes (rarely - when there is an upsert) it wants to visit +** the indexes in a different order. The following data structures accomplish +** this. +** +** The IndexIterator object is used to walk through all of the indexes +** of a table in either Index.pNext order, or in some other order established +** by an array of IndexListTerm objects. +*/ +typedef struct IndexListTerm IndexListTerm; +typedef struct IndexIterator IndexIterator; +struct IndexIterator { + int eType; /* 0 for Index.pNext list. 1 for an array of IndexListTerm */ + int i; /* Index of the current item from the list */ + union { + struct { /* Use this object for eType==0: A Index.pNext list */ + Index *pIdx; /* The current Index */ + } lx; + struct { /* Use this object for eType==1; Array of IndexListTerm */ + int nIdx; /* Size of the array */ + IndexListTerm *aIdx; /* Array of IndexListTerms */ + } ax; + } u; +}; + +/* When IndexIterator.eType==1, then each index is an array of instances +** of the following object +*/ +struct IndexListTerm { + Index *p; /* The index */ + int ix; /* Which entry in the original Table.pIndex list is this index*/ +}; + +/* Return the first index on the list */ +static Index *indexIteratorFirst(IndexIterator *pIter, int *pIx){ + assert( pIter->i==0 ); + if( pIter->eType ){ + *pIx = pIter->u.ax.aIdx[0].ix; + return pIter->u.ax.aIdx[0].p; + }else{ + *pIx = 0; + return pIter->u.lx.pIdx; + } +} + +/* Return the next index from the list. Return NULL when out of indexes */ +static Index *indexIteratorNext(IndexIterator *pIter, int *pIx){ + if( pIter->eType ){ + int i = ++pIter->i; + if( i>=pIter->u.ax.nIdx ){ + *pIx = i; + return 0; + } + *pIx = pIter->u.ax.aIdx[i].ix; + return pIter->u.ax.aIdx[i].p; + }else{ + ++(*pIx); + pIter->u.lx.pIdx = pIter->u.lx.pIdx->pNext; + return pIter->u.lx.pIdx; + } } /* @@ -107028,6 +135499,14 @@ static int checkConstraintUnchanged(Expr *pExpr, int *aiChng, int chngRowid){ ** the same as the order of indices on the linked list of indices ** at pTab->pIndex. ** +** (2019-05-07) The generated code also creates a new record for the +** main table, if pTab is a rowid table, and stores that record in the +** register identified by aRegIdx[nIdx] - in other words in the first +** entry of aRegIdx[] past the last index. It is important that the +** record be generated during constraint checks to avoid affinity changes +** to the register content that occur after constraint checks but before +** the new record is inserted. +** ** The caller must have already opened writeable cursors on the main ** table and all applicable indices (that is to say, all indices for which ** aRegIdx[] is not zero). iDataCur is the cursor for the main table when @@ -107088,34 +135567,44 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( u8 overrideError, /* Override onError to this if not OE_Default */ int ignoreDest, /* Jump to this label on an OE_Ignore resolution */ int *pbMayReplace, /* OUT: Set to true if constraint may cause a replace */ - int *aiChng /* column i is unchanged if aiChng[i]<0 */ + int *aiChng, /* column i is unchanged if aiChng[i]<0 */ + Upsert *pUpsert /* ON CONFLICT clauses, if any. NULL otherwise */ ){ - Vdbe *v; /* VDBE under constrution */ + Vdbe *v; /* VDBE under construction */ Index *pIdx; /* Pointer to one of the indices */ - Index *pPk = 0; /* The PRIMARY KEY index */ + Index *pPk = 0; /* The PRIMARY KEY index for WITHOUT ROWID tables */ sqlite3 *db; /* Database connection */ int i; /* loop counter */ int ix; /* Index loop counter */ int nCol; /* Number of columns */ int onError; /* Conflict resolution strategy */ - int addr1; /* Address of jump instruction */ int seenReplace = 0; /* True if REPLACE is used to resolve INT PK conflict */ int nPkField; /* Number of fields in PRIMARY KEY. 1 for ROWID tables */ - int ipkTop = 0; /* Top of the rowid change constraint check */ - int ipkBottom = 0; /* Bottom of the rowid change constraint check */ - u8 isUpdate; /* True if this is an UPDATE operation */ + Upsert *pUpsertClause = 0; /* The specific ON CONFLICT clause for pIdx */ + u8 isUpdate; /* True if this is an UPDATE operation */ u8 bAffinityDone = 0; /* True if the OP_Affinity operation has been run */ - int regRowid = -1; /* Register holding ROWID value */ + int upsertIpkReturn = 0; /* Address of Goto at end of IPK uniqueness check */ + int upsertIpkDelay = 0; /* Address of Goto to bypass initial IPK check */ + int ipkTop = 0; /* Top of the IPK uniqueness check */ + int ipkBottom = 0; /* OP_Goto at the end of the IPK uniqueness check */ + /* Variables associated with retesting uniqueness constraints after + ** replace triggers fire have run */ + int regTrigCnt; /* Register used to count replace trigger invocations */ + int addrRecheck = 0; /* Jump here to recheck all uniqueness constraints */ + int lblRecheckOk = 0; /* Each recheck jumps to this label if it passes */ + Trigger *pTrigger; /* List of DELETE triggers on the table pTab */ + int nReplaceTrig = 0; /* Number of replace triggers coded */ + IndexIterator sIdxIter; /* Index iterator */ isUpdate = regOldData!=0; db = pParse->db; - v = sqlite3GetVdbe(pParse); + v = pParse->pVdbe; assert( v!=0 ); - assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + assert( !IsView(pTab) ); /* This table is not a VIEW */ nCol = pTab->nCol; - + /* pPk is the PRIMARY KEY index for WITHOUT ROWID tables and NULL for - ** normal rowid tables. nPkField is the number of key fields in the + ** normal rowid tables. nPkField is the number of key fields in the ** pPk index or 1 for a rowid table. In other words, nPkField is the ** number of fields in the true primary key of the table. */ if( HasRowid(pTab) ){ @@ -107132,89 +135621,288 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Test all NOT NULL constraints. */ - for(i=0; iiPKey ){ - continue; /* ROWID is never NULL */ - } - if( aiChng && aiChng[i]<0 ){ - /* Don't bother checking for NOT NULL on columns that do not change */ - continue; - } - onError = pTab->aCol[i].notNull; - if( onError==OE_None ) continue; /* This column is allowed to be NULL */ - if( overrideError!=OE_Default ){ - onError = overrideError; - }else if( onError==OE_Default ){ - onError = OE_Abort; - } - if( onError==OE_Replace && pTab->aCol[i].pDflt==0 ){ - onError = OE_Abort; - } - assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail - || onError==OE_Ignore || onError==OE_Replace ); - switch( onError ){ - case OE_Abort: - sqlite3MayAbort(pParse); - /* Fall through */ - case OE_Rollback: - case OE_Fail: { - char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, - pTab->aCol[i].zName); - sqlite3VdbeAddOp4(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, onError, - regNewData+1+i, zMsg, P4_DYNAMIC); - sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); - VdbeCoverage(v); + if( pTab->tabFlags & TF_HasNotNull ){ + int b2ndPass = 0; /* True if currently running 2nd pass */ + int nSeenReplace = 0; /* Number of ON CONFLICT REPLACE operations */ + int nGenerated = 0; /* Number of generated columns with NOT NULL */ + while(1){ /* Make 2 passes over columns. Exit loop via "break" */ + for(i=0; iaCol[i]; /* The column to check for NOT NULL */ + int isGenerated; /* non-zero if column is generated */ + onError = pCol->notNull; + if( onError==OE_None ) continue; /* No NOT NULL on this column */ + if( i==pTab->iPKey ){ + continue; /* ROWID is never NULL */ + } + isGenerated = pCol->colFlags & COLFLAG_GENERATED; + if( isGenerated && !b2ndPass ){ + nGenerated++; + continue; /* Generated columns processed on 2nd pass */ + } + if( aiChng && aiChng[i]<0 && !isGenerated ){ + /* Do not check NOT NULL on columns that do not change */ + continue; + } + if( overrideError!=OE_Default ){ + onError = overrideError; + }else if( onError==OE_Default ){ + onError = OE_Abort; + } + if( onError==OE_Replace ){ + if( b2ndPass /* REPLACE becomes ABORT on the 2nd pass */ + || pCol->iDflt==0 /* REPLACE is ABORT if no DEFAULT value */ + ){ + testcase( pCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pCol->colFlags & COLFLAG_STORED ); + testcase( pCol->colFlags & COLFLAG_GENERATED ); + onError = OE_Abort; + }else{ + assert( !isGenerated ); + } + }else if( b2ndPass && !isGenerated ){ + continue; + } + assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail + || onError==OE_Ignore || onError==OE_Replace ); + testcase( i!=sqlite3TableColumnToStorage(pTab, i) ); + iReg = sqlite3TableColumnToStorage(pTab, i) + regNewData + 1; + switch( onError ){ + case OE_Replace: { + int addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, iReg); + VdbeCoverage(v); + assert( (pCol->colFlags & COLFLAG_GENERATED)==0 ); + nSeenReplace++; + sqlite3ExprCodeCopy(pParse, + sqlite3ColumnExpr(pTab, pCol), iReg); + sqlite3VdbeJumpHere(v, addr1); + break; + } + case OE_Abort: + sqlite3MayAbort(pParse); + /* no break */ deliberate_fall_through + case OE_Rollback: + case OE_Fail: { + char *zMsg = sqlite3MPrintf(db, "%s.%s", pTab->zName, + pCol->zCnName); + testcase( zMsg==0 && db->mallocFailed==0 ); + sqlite3VdbeAddOp3(v, OP_HaltIfNull, SQLITE_CONSTRAINT_NOTNULL, + onError, iReg); + sqlite3VdbeAppendP4(v, zMsg, P4_DYNAMIC); + sqlite3VdbeChangeP5(v, P5_ConstraintNotNull); + VdbeCoverage(v); + break; + } + default: { + assert( onError==OE_Ignore ); + sqlite3VdbeAddOp2(v, OP_IsNull, iReg, ignoreDest); + VdbeCoverage(v); + break; + } + } /* end switch(onError) */ + } /* end loop i over columns */ + if( nGenerated==0 && nSeenReplace==0 ){ + /* If there are no generated columns with NOT NULL constraints + ** and no NOT NULL ON CONFLICT REPLACE constraints, then a single + ** pass is sufficient */ break; } - case OE_Ignore: { - sqlite3VdbeAddOp2(v, OP_IsNull, regNewData+1+i, ignoreDest); - VdbeCoverage(v); - break; + if( b2ndPass ) break; /* Never need more than 2 passes */ + b2ndPass = 1; +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( nSeenReplace>0 && (pTab->tabFlags & TF_HasGenerated)!=0 ){ + /* If any NOT NULL ON CONFLICT REPLACE constraints fired on the + ** first pass, recomputed values for all generated columns, as + ** those values might depend on columns affected by the REPLACE. + */ + sqlite3ComputeGeneratedColumns(pParse, regNewData+1, pTab); } - default: { - assert( onError==OE_Replace ); - addr1 = sqlite3VdbeAddOp1(v, OP_NotNull, regNewData+1+i); - VdbeCoverage(v); - sqlite3ExprCode(pParse, pTab->aCol[i].pDflt, regNewData+1+i); - sqlite3VdbeJumpHere(v, addr1); - break; - } - } - } +#endif + } /* end of 2-pass loop */ + } /* end if( has-not-null-constraints ) */ /* Test all CHECK constraints */ #ifndef SQLITE_OMIT_CHECK if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ ExprList *pCheck = pTab->pCheck; - pParse->ckBase = regNewData+1; + pParse->iSelfTab = -(regNewData+1); onError = overrideError!=OE_Default ? overrideError : OE_Abort; for(i=0; inExpr; i++){ int allOk; + Expr *pCopy; Expr *pExpr = pCheck->a[i].pExpr; - if( aiChng && checkConstraintUnchanged(pExpr, aiChng, pkChng) ) continue; - allOk = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfTrue(pParse, pExpr, allOk, SQLITE_JUMPIFNULL); + if( aiChng + && !sqlite3ExprReferencesUpdatedColumn(pExpr, aiChng, pkChng) + ){ + /* The check constraints do not reference any of the columns being + ** updated so there is no point it verifying the check constraint */ + continue; + } + if( bAffinityDone==0 ){ + sqlite3TableAffinity(v, pTab, regNewData+1); + bAffinityDone = 1; + } + allOk = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeVerifyAbortable(v, onError); + pCopy = sqlite3ExprDup(db, pExpr, 0); + if( !db->mallocFailed ){ + sqlite3ExprIfTrue(pParse, pCopy, allOk, SQLITE_JUMPIFNULL); + } + sqlite3ExprDelete(db, pCopy); if( onError==OE_Ignore ){ sqlite3VdbeGoto(v, ignoreDest); }else{ - char *zName = pCheck->a[i].zName; - if( zName==0 ) zName = pTab->zName; - if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-15569-63625 */ + char *zName = pCheck->a[i].zEName; + assert( zName!=0 || pParse->db->mallocFailed ); + if( onError==OE_Replace ) onError = OE_Abort; /* IMP: R-26383-51744 */ sqlite3HaltConstraint(pParse, SQLITE_CONSTRAINT_CHECK, onError, zName, P4_TRANSIENT, P5_ConstraintCheck); } sqlite3VdbeResolveLabel(v, allOk); } + pParse->iSelfTab = 0; } #endif /* !defined(SQLITE_OMIT_CHECK) */ + /* UNIQUE and PRIMARY KEY constraints should be handled in the following + ** order: + ** + ** (1) OE_Update + ** (2) OE_Abort, OE_Fail, OE_Rollback, OE_Ignore + ** (3) OE_Replace + ** + ** OE_Fail and OE_Ignore must happen before any changes are made. + ** OE_Update guarantees that only a single row will change, so it + ** must happen before OE_Replace. Technically, OE_Abort and OE_Rollback + ** could happen in any order, but they are grouped up front for + ** convenience. + ** + ** 2018-08-14: Ticket https://www.sqlite.org/src/info/908f001483982c43 + ** The order of constraints used to have OE_Update as (2) and OE_Abort + ** and so forth as (1). But apparently PostgreSQL checks the OE_Update + ** constraint before any others, so it had to be moved. + ** + ** Constraint checking code is generated in this order: + ** (A) The rowid constraint + ** (B) Unique index constraints that do not have OE_Replace as their + ** default conflict resolution strategy + ** (C) Unique index that do use OE_Replace by default. + ** + ** The ordering of (2) and (3) is accomplished by making sure the linked + ** list of indexes attached to a table puts all OE_Replace indexes last + ** in the list. See sqlite3CreateIndex() for where that happens. + */ + sIdxIter.eType = 0; + sIdxIter.i = 0; + sIdxIter.u.ax.aIdx = 0; /* Silence harmless compiler warning */ + sIdxIter.u.lx.pIdx = pTab->pIndex; + if( pUpsert ){ + if( pUpsert->pUpsertTarget==0 ){ + /* There is just on ON CONFLICT clause and it has no constraint-target */ + assert( pUpsert->pNextUpsert==0 ); + if( pUpsert->isDoUpdate==0 ){ + /* A single ON CONFLICT DO NOTHING clause, without a constraint-target. + ** Make all unique constraint resolution be OE_Ignore */ + overrideError = OE_Ignore; + pUpsert = 0; + }else{ + /* A single ON CONFLICT DO UPDATE. Make all resolutions OE_Update */ + overrideError = OE_Update; + } + }else if( pTab->pIndex!=0 ){ + /* Otherwise, we'll need to run the IndexListTerm array version of the + ** iterator to ensure that all of the ON CONFLICT conditions are + ** checked first and in order. */ + int nIdx, jj; + u64 nByte; + Upsert *pTerm; + u8 *bUsed; + for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ + assert( aRegIdx[nIdx]>0 ); + } + sIdxIter.eType = 1; + sIdxIter.u.ax.nIdx = nIdx; + nByte = (sizeof(IndexListTerm)+1)*nIdx + nIdx; + sIdxIter.u.ax.aIdx = sqlite3DbMallocZero(db, nByte); + if( sIdxIter.u.ax.aIdx==0 ) return; /* OOM */ + bUsed = (u8*)&sIdxIter.u.ax.aIdx[nIdx]; + pUpsert->pToFree = sIdxIter.u.ax.aIdx; + for(i=0, pTerm=pUpsert; pTerm; pTerm=pTerm->pNextUpsert){ + if( pTerm->pUpsertTarget==0 ) break; + if( pTerm->pUpsertIdx==0 ) continue; /* Skip ON CONFLICT for the IPK */ + jj = 0; + pIdx = pTab->pIndex; + while( ALWAYS(pIdx!=0) && pIdx!=pTerm->pUpsertIdx ){ + pIdx = pIdx->pNext; + jj++; + } + if( bUsed[jj] ) continue; /* Duplicate ON CONFLICT clause ignored */ + bUsed[jj] = 1; + sIdxIter.u.ax.aIdx[i].p = pIdx; + sIdxIter.u.ax.aIdx[i].ix = jj; + i++; + } + for(jj=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, jj++){ + if( bUsed[jj] ) continue; + sIdxIter.u.ax.aIdx[i].p = pIdx; + sIdxIter.u.ax.aIdx[i].ix = jj; + i++; + } + assert( i==nIdx ); + } + } + + /* Determine if it is possible that triggers (either explicitly coded + ** triggers or FK resolution actions) might run as a result of deletes + ** that happen when OE_Replace conflict resolution occurs. (Call these + ** "replace triggers".) If any replace triggers run, we will need to + ** recheck all of the uniqueness constraints after they have all run. + ** But on the recheck, the resolution is OE_Abort instead of OE_Replace. + ** + ** If replace triggers are a possibility, then + ** + ** (1) Allocate register regTrigCnt and initialize it to zero. + ** That register will count the number of replace triggers that + ** fire. Constraint recheck only occurs if the number is positive. + ** (2) Initialize pTrigger to the list of all DELETE triggers on pTab. + ** (3) Initialize addrRecheck and lblRecheckOk + ** + ** The uniqueness rechecking code will create a series of tests to run + ** in a second pass. The addrRecheck and lblRecheckOk variables are + ** used to link together these tests which are separated from each other + ** in the generate bytecode. + */ + if( (db->flags & (SQLITE_RecTriggers|SQLITE_ForeignKeys))==0 ){ + /* There are not DELETE triggers nor FK constraints. No constraint + ** rechecks are needed. */ + pTrigger = 0; + regTrigCnt = 0; + }else{ + if( db->flags&SQLITE_RecTriggers ){ + pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + regTrigCnt = pTrigger!=0 || sqlite3FkRequired(pParse, pTab, 0, 0); + }else{ + pTrigger = 0; + regTrigCnt = sqlite3FkRequired(pParse, pTab, 0, 0); + } + if( regTrigCnt ){ + /* Replace triggers might exist. Allocate the counter and + ** initialize it to zero. */ + regTrigCnt = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regTrigCnt); + VdbeComment((v, "trigger count")); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + addrRecheck = lblRecheckOk; + } + } + /* If rowid is changing, make sure the new rowid does not previously ** exist in the table. */ if( pkChng && pPk==0 ){ - int addrRowidOk = sqlite3VdbeMakeLabel(v); + int addrRowidOk = sqlite3VdbeMakeLabel(pParse); /* Figure out what action to take in case of a rowid collision */ onError = pTab->keyConf; @@ -107224,13 +135912,22 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( onError = OE_Abort; } - if( isUpdate ){ - /* pkChng!=0 does not mean that the rowid has change, only that - ** it might have changed. Skip the conflict logic below if the rowid - ** is unchanged. */ - sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - VdbeCoverage(v); + /* figure out whether or not upsert applies in this case */ + if( pUpsert ){ + pUpsertClause = sqlite3UpsertOfIndex(pUpsert,0); + if( pUpsertClause!=0 ){ + if( pUpsertClause->isDoUpdate==0 ){ + onError = OE_Ignore; /* DO NOTHING is the same as INSERT OR IGNORE */ + }else{ + onError = OE_Update; /* DO UPDATE */ + } + } + if( pUpsertClause!=pUpsert ){ + /* The first ON CONFLICT clause has a conflict target other than + ** the IPK. We have to jump ahead to that first ON CONFLICT clause + ** and then come back here and deal with the IPK afterwards */ + upsertIpkDelay = sqlite3VdbeAddOp0(v, OP_Goto); + } } /* If the response to a rowid conflict is REPLACE but the response @@ -107238,29 +135935,42 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** to defer the running of the rowid conflict checking until after ** the UNIQUE constraints have run. */ - if( onError==OE_Replace && overrideError!=OE_Replace ){ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_Ignore || pIdx->onError==OE_Fail ){ - ipkTop = sqlite3VdbeAddOp0(v, OP_Goto); - break; - } - } + if( onError==OE_Replace /* IPK rule is REPLACE */ + && onError!=overrideError /* Rules for other constraints are different */ + && pTab->pIndex /* There exist other constraints */ + && !upsertIpkDelay /* IPK check already deferred by UPSERT */ + ){ + ipkTop = sqlite3VdbeAddOp0(v, OP_Goto)+1; + VdbeComment((v, "defer IPK REPLACE until last")); + } + + if( isUpdate ){ + /* pkChng!=0 does not mean that the rowid has changed, only that + ** it might have changed. Skip the conflict logic below if the rowid + ** is unchanged. */ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRowidOk, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); } /* Check to see if the new rowid already exists in the table. Skip ** the following conflict logic if it does not. */ + VdbeNoopComment((v, "uniqueness check for ROWID")); + sqlite3VdbeVerifyAbortable(v, onError); sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRowidOk, regNewData); VdbeCoverage(v); - /* Generate code that deals with a rowid collision */ switch( onError ){ default: { onError = OE_Abort; - /* Fall thru into the next case */ + /* no break */ deliberate_fall_through } case OE_Rollback: case OE_Abort: case OE_Fail: { + testcase( onError==OE_Rollback ); + testcase( onError==OE_Abort ); + testcase( onError==OE_Fail ); sqlite3RowidConstraint(pParse, onError, pTab); break; } @@ -107271,10 +135981,10 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** the triggers and remove both the table and index b-tree entries. ** ** Otherwise, if there are no triggers or the recursive-triggers - ** flag is not set, but the table has one or more indexes, call - ** GenerateRowIndexDelete(). This removes the index b-tree entries - ** only. The table b-tree entry will be replaced by the new entry - ** when it is inserted. + ** flag is not set, but the table has one or more indexes, call + ** GenerateRowIndexDelete(). This removes the index b-tree entries + ** only. The table b-tree entry will be replaced by the new entry + ** when it is inserted. ** ** If either GenerateRowDelete() or GenerateRowIndexDelete() is called, ** also invoke MultiWrite() to indicate that this VDBE may require @@ -107287,24 +135997,21 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** to run without a statement journal if there are no indexes on the ** table. */ - Trigger *pTrigger = 0; - if( db->flags&SQLITE_RecTriggers ){ - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); - } - if( pTrigger || sqlite3FkRequired(pParse, pTab, 0, 0) ){ + if( regTrigCnt ){ sqlite3MultiWrite(pParse); sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regNewData, 1, 0, OE_Replace, 1, -1); + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + nReplaceTrig++; }else{ #ifdef SQLITE_ENABLE_PREUPDATE_HOOK - if( HasRowid(pTab) ){ - /* This OP_Delete opcode fires the pre-update-hook only. It does - ** not modify the b-tree. It is more efficient to let the coming - ** OP_Insert replace the existing entry than it is to delete the - ** existing entry and then insert a new one. */ - sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); - sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); - } + assert( HasRowid(pTab) ); + /* This OP_Delete opcode fires the pre-update-hook only. It does + ** not modify the b-tree. It is more efficient to let the coming + ** OP_Insert replace the existing entry than it is to delete the + ** existing entry and then insert a new one. */ + sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, OPFLAG_ISNOOP); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ if( pTab->pIndex ){ sqlite3MultiWrite(pParse); @@ -107314,16 +136021,24 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( seenReplace = 1; break; } +#ifndef SQLITE_OMIT_UPSERT + case OE_Update: { + sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, 0, iDataCur); + /* no break */ deliberate_fall_through + } +#endif case OE_Ignore: { - /*assert( seenReplace==0 );*/ + testcase( onError==OE_Ignore ); sqlite3VdbeGoto(v, ignoreDest); break; } } sqlite3VdbeResolveLabel(v, addrRowidOk); - if( ipkTop ){ + if( pUpsert && pUpsertClause!=pUpsert ){ + upsertIpkReturn = sqlite3VdbeAddOp0(v, OP_Goto); + }else if( ipkTop ){ ipkBottom = sqlite3VdbeAddOp0(v, OP_Goto); - sqlite3VdbeJumpHere(v, ipkTop); + sqlite3VdbeJumpHere(v, ipkTop-1); } } @@ -107334,58 +136049,74 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( ** This loop also handles the case of the PRIMARY KEY index for a ** WITHOUT ROWID table. */ - for(ix=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, ix++){ - int regIdx; /* Range of registers hold conent for pIdx */ + for(pIdx = indexIteratorFirst(&sIdxIter, &ix); + pIdx; + pIdx = indexIteratorNext(&sIdxIter, &ix) + ){ + int regIdx; /* Range of registers holding content for pIdx */ int regR; /* Range of registers holding conflicting PK */ int iThisCur; /* Cursor for this UNIQUE index */ int addrUniqueOk; /* Jump here if the UNIQUE constraint is satisfied */ + int addrConflictCk; /* First opcode in the conflict check logic */ if( aRegIdx[ix]==0 ) continue; /* Skip indices that do not change */ + if( pUpsert ){ + pUpsertClause = sqlite3UpsertOfIndex(pUpsert, pIdx); + if( upsertIpkDelay && pUpsertClause==pUpsert ){ + sqlite3VdbeJumpHere(v, upsertIpkDelay); + } + } + addrUniqueOk = sqlite3VdbeMakeLabel(pParse); if( bAffinityDone==0 ){ sqlite3TableAffinity(v, pTab, regNewData+1); bAffinityDone = 1; } + VdbeNoopComment((v, "prep index %s", pIdx->zName)); iThisCur = iIdxCur+ix; - addrUniqueOk = sqlite3VdbeMakeLabel(v); + /* Skip partial indices for which the WHERE clause is not true */ if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_Null, 0, aRegIdx[ix]); - pParse->ckBase = regNewData+1; + pParse->iSelfTab = -(regNewData+1); sqlite3ExprIfFalseDup(pParse, pIdx->pPartIdxWhere, addrUniqueOk, SQLITE_JUMPIFNULL); - pParse->ckBase = 0; + pParse->iSelfTab = 0; } /* Create a record for this index entry as it should appear after ** the insert or update. Store that record in the aRegIdx[ix] register */ - regIdx = sqlite3GetTempRange(pParse, pIdx->nColumn); + regIdx = aRegIdx[ix]+1; for(i=0; inColumn; i++){ int iField = pIdx->aiColumn[i]; int x; if( iField==XN_EXPR ){ - pParse->ckBase = regNewData+1; + pParse->iSelfTab = -(regNewData+1); sqlite3ExprCodeCopy(pParse, pIdx->aColExpr->a[i].pExpr, regIdx+i); - pParse->ckBase = 0; + pParse->iSelfTab = 0; VdbeComment((v, "%s column %d", pIdx->zName, i)); + }else if( iField==XN_ROWID || iField==pTab->iPKey ){ + x = regNewData; + sqlite3VdbeAddOp2(v, OP_IntCopy, x, regIdx+i); + VdbeComment((v, "rowid")); }else{ - if( iField==XN_ROWID || iField==pTab->iPKey ){ - if( regRowid==regIdx+i ) continue; /* ROWID already in regIdx+i */ - x = regNewData; - regRowid = pIdx->pPartIdxWhere ? -1 : regIdx+i; - }else{ - x = iField + regNewData + 1; - } - sqlite3VdbeAddOp2(v, iField<0 ? OP_IntCopy : OP_SCopy, x, regIdx+i); - VdbeComment((v, "%s", iField<0 ? "rowid" : pTab->aCol[iField].zName)); + testcase( sqlite3TableColumnToStorage(pTab, iField)!=iField ); + x = sqlite3TableColumnToStorage(pTab, iField) + regNewData + 1; + sqlite3VdbeAddOp2(v, OP_SCopy, x, regIdx+i); + VdbeComment((v, "%s", pTab->aCol[iField].zCnName)); } } sqlite3VdbeAddOp3(v, OP_MakeRecord, regIdx, pIdx->nColumn, aRegIdx[ix]); VdbeComment((v, "for %s", pIdx->zName)); - sqlite3ExprCacheAffinityChange(pParse, regIdx, pIdx->nColumn); +#ifdef SQLITE_ENABLE_NULL_TRIM + if( pIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ + sqlite3SetMakeRecordP5(v, pIdx->pTable); + } +#endif + sqlite3VdbeReleaseRegisters(pParse, regIdx, pIdx->nColumn, 0, 0); - /* In an UPDATE operation, if this index is the PRIMARY KEY index + /* In an UPDATE operation, if this index is the PRIMARY KEY index ** of a WITHOUT ROWID table and there has been no change the ** primary key, then no collision is possible. The collision detection ** logic below can all be skipped. */ @@ -107396,8 +136127,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Find out what action to take in case there is a uniqueness conflict */ onError = pIdx->onError; - if( onError==OE_None ){ - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); + if( onError==OE_None ){ sqlite3VdbeResolveLabel(v, addrUniqueOk); continue; /* pIdx is not a UNIQUE index */ } @@ -107406,13 +136136,49 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( }else if( onError==OE_Default ){ onError = OE_Abort; } - + + /* Figure out if the upsert clause applies to this index */ + if( pUpsertClause ){ + if( pUpsertClause->isDoUpdate==0 ){ + onError = OE_Ignore; /* DO NOTHING is the same as INSERT OR IGNORE */ + }else{ + onError = OE_Update; /* DO UPDATE */ + } + } + + /* Collision detection may be omitted if all of the following are true: + ** (1) The conflict resolution algorithm is REPLACE + ** (2) The table is a WITHOUT ROWID table + ** (3) There are no secondary indexes on the table + ** (4) No delete triggers need to be fired if there is a conflict + ** (5) No FK constraint counters need to be updated if a conflict occurs. + ** + ** This is not possible for ENABLE_PREUPDATE_HOOK builds, as the row + ** must be explicitly deleted in order to ensure any pre-update hook + ** is invoked. */ + assert( IsOrdinaryTable(pTab) ); +#ifndef SQLITE_ENABLE_PREUPDATE_HOOK + if( (ix==0 && pIdx->pNext==0) /* Condition 3 */ + && pPk==pIdx /* Condition 2 */ + && onError==OE_Replace /* Condition 1 */ + && ( 0==(db->flags&SQLITE_RecTriggers) || /* Condition 4 */ + 0==sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0)) + && ( 0==(db->flags&SQLITE_ForeignKeys) || /* Condition 5 */ + (0==pTab->u.tab.pFKey && 0==sqlite3FkReferences(pTab))) + ){ + sqlite3VdbeResolveLabel(v, addrUniqueOk); + continue; + } +#endif /* ifndef SQLITE_ENABLE_PREUPDATE_HOOK */ + /* Check to see if the new index entry will be unique */ - sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, - regIdx, pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeVerifyAbortable(v, onError); + addrConflictCk = + sqlite3VdbeAddOp4Int(v, OP_NoConflict, iThisCur, addrUniqueOk, + regIdx, pIdx->nKeyCol); VdbeCoverage(v); /* Generate code to handle collisions */ - regR = (pIdx==pPk) ? regIdx : sqlite3GetTempRange(pParse, nPkField); + regR = pIdx==pPk ? regIdx : sqlite3GetTempRange(pParse, nPkField); if( isUpdate || onError==OE_Replace ){ if( HasRowid(pTab) ){ sqlite3VdbeAddOp2(v, OP_IdxRowid, iThisCur, regR); @@ -107430,16 +136196,16 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( if( pIdx!=pPk ){ for(i=0; inKeyCol; i++){ assert( pPk->aiColumn[i]>=0 ); - x = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[i]); + x = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); sqlite3VdbeAddOp3(v, OP_Column, iThisCur, x, regR+i); VdbeComment((v, "%s.%s", pTab->zName, - pTab->aCol[pPk->aiColumn[i]].zName)); + pTab->aCol[pPk->aiColumn[i]].zCnName)); } } if( isUpdate ){ - /* If currently processing the PRIMARY KEY of a WITHOUT ROWID + /* If currently processing the PRIMARY KEY of a WITHOUT ROWID ** table, only conflict if the new PRIMARY KEY values are actually - ** different from the old. + ** different from the old. See TH3 withoutrowid04.test. ** ** For a UNIQUE index, only conflict if the PRIMARY KEY values ** of the matched index row are different from the original PRIMARY @@ -107447,7 +136213,7 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( int addrJump = sqlite3VdbeCurrentAddr(v)+pPk->nKeyCol; int op = OP_Ne; int regCmp = (IsPrimaryKeyIndex(pIdx) ? regIdx : regR); - + for(i=0; inKeyCol; i++){ char *p4 = (char*)sqlite3LocateCollSeq(pParse, pPk->azColl[i]); x = pPk->aiColumn[i]; @@ -107456,7 +136222,8 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( addrJump = addrUniqueOk; op = OP_Eq; } - sqlite3VdbeAddOp4(v, op, + x = sqlite3TableColumnToStorage(pTab, x); + sqlite3VdbeAddOp4(v, op, regOldData+1+x, addrJump, regCmp+i, p4, P4_COLLSEQ ); sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); @@ -107469,45 +136236,203 @@ SQLITE_PRIVATE void sqlite3GenerateConstraintChecks( /* Generate code that executes if the new index entry is not unique */ assert( onError==OE_Rollback || onError==OE_Abort || onError==OE_Fail - || onError==OE_Ignore || onError==OE_Replace ); + || onError==OE_Ignore || onError==OE_Replace || onError==OE_Update ); switch( onError ){ case OE_Rollback: case OE_Abort: case OE_Fail: { + testcase( onError==OE_Rollback ); + testcase( onError==OE_Abort ); + testcase( onError==OE_Fail ); sqlite3UniqueConstraint(pParse, onError, pIdx); break; } +#ifndef SQLITE_OMIT_UPSERT + case OE_Update: { + sqlite3UpsertDoUpdate(pParse, pUpsert, pTab, pIdx, iIdxCur+ix); + /* no break */ deliberate_fall_through + } +#endif case OE_Ignore: { + testcase( onError==OE_Ignore ); sqlite3VdbeGoto(v, ignoreDest); break; } default: { - Trigger *pTrigger = 0; + int nConflictCk; /* Number of opcodes in conflict check logic */ + assert( onError==OE_Replace ); - sqlite3MultiWrite(pParse); - if( db->flags&SQLITE_RecTriggers ){ - pTrigger = sqlite3TriggersExist(pParse, pTab, TK_DELETE, 0, 0); + nConflictCk = sqlite3VdbeCurrentAddr(v) - addrConflictCk; + assert( nConflictCk>0 || db->mallocFailed ); + testcase( nConflictCk<=0 ); + testcase( nConflictCk>1 ); + if( regTrigCnt ){ + sqlite3MultiWrite(pParse); + nReplaceTrig++; + } + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorLock, iDataCur); } sqlite3GenerateRowDelete(pParse, pTab, pTrigger, iDataCur, iIdxCur, regR, nPkField, 0, OE_Replace, - (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), -1); + (pIdx==pPk ? ONEPASS_SINGLE : ONEPASS_OFF), iThisCur); + if( pTrigger && isUpdate ){ + sqlite3VdbeAddOp1(v, OP_CursorUnlock, iDataCur); + } + if( regTrigCnt ){ + int addrBypass; /* Jump destination to bypass recheck logic */ + + sqlite3VdbeAddOp2(v, OP_AddImm, regTrigCnt, 1); /* incr trigger cnt */ + addrBypass = sqlite3VdbeAddOp0(v, OP_Goto); /* Bypass recheck */ + VdbeComment((v, "bypass recheck")); + + /* Here we insert code that will be invoked after all constraint + ** checks have run, if and only if one or more replace triggers + ** fired. */ + sqlite3VdbeResolveLabel(v, lblRecheckOk); + lblRecheckOk = sqlite3VdbeMakeLabel(pParse); + if( pIdx->pPartIdxWhere ){ + /* Bypass the recheck if this partial index is not defined + ** for the current row */ + sqlite3VdbeAddOp2(v, OP_IsNull, regIdx-1, lblRecheckOk); + VdbeCoverage(v); + } + /* Copy the constraint check code from above, except change + ** the constraint-ok jump destination to be the address of + ** the next retest block */ + while( nConflictCk>0 ){ + VdbeOp x; /* Conflict check opcode to copy */ + /* The sqlite3VdbeAddOp4() call might reallocate the opcode array. + ** Hence, make a complete copy of the opcode, rather than using + ** a pointer to the opcode. */ + x = *sqlite3VdbeGetOp(v, addrConflictCk); + if( x.opcode!=OP_IdxRowid ){ + int p2; /* New P2 value for copied conflict check opcode */ + const char *zP4; + if( sqlite3OpcodeProperty[x.opcode]&OPFLG_JUMP ){ + p2 = lblRecheckOk; + }else{ + p2 = x.p2; + } + zP4 = x.p4type==P4_INT32 ? SQLITE_INT_TO_PTR(x.p4.i) : x.p4.z; + sqlite3VdbeAddOp4(v, x.opcode, x.p1, p2, x.p3, zP4, x.p4type); + sqlite3VdbeChangeP5(v, x.p5); + VdbeCoverageIf(v, p2!=x.p2); + } + nConflictCk--; + addrConflictCk++; + } + /* If the retest fails, issue an abort */ + sqlite3UniqueConstraint(pParse, OE_Abort, pIdx); + + sqlite3VdbeJumpHere(v, addrBypass); /* Terminate the recheck bypass */ + } seenReplace = 1; break; } } sqlite3VdbeResolveLabel(v, addrUniqueOk); - sqlite3ReleaseTempRange(pParse, regIdx, pIdx->nColumn); if( regR!=regIdx ) sqlite3ReleaseTempRange(pParse, regR, nPkField); + if( pUpsertClause + && upsertIpkReturn + && sqlite3UpsertNextIsIPK(pUpsertClause) + ){ + sqlite3VdbeGoto(v, upsertIpkDelay+1); + sqlite3VdbeJumpHere(v, upsertIpkReturn); + upsertIpkReturn = 0; + } } + + /* If the IPK constraint is a REPLACE, run it last */ if( ipkTop ){ - sqlite3VdbeGoto(v, ipkTop+1); + sqlite3VdbeGoto(v, ipkTop); + VdbeComment((v, "Do IPK REPLACE")); + assert( ipkBottom>0 ); sqlite3VdbeJumpHere(v, ipkBottom); } - + + /* Recheck all uniqueness constraints after replace triggers have run */ + testcase( regTrigCnt!=0 && nReplaceTrig==0 ); + assert( regTrigCnt!=0 || nReplaceTrig==0 ); + if( nReplaceTrig ){ + sqlite3VdbeAddOp2(v, OP_IfNot, regTrigCnt, lblRecheckOk);VdbeCoverage(v); + if( !pPk ){ + if( isUpdate ){ + sqlite3VdbeAddOp3(v, OP_Eq, regNewData, addrRecheck, regOldData); + sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); + VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, addrRecheck, regNewData); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, OE_Abort, pTab); + }else{ + sqlite3VdbeGoto(v, addrRecheck); + } + sqlite3VdbeResolveLabel(v, lblRecheckOk); + } + + /* Generate the table record */ + if( HasRowid(pTab) ){ + int regRec = aRegIdx[ix]; + sqlite3VdbeAddOp3(v, OP_MakeRecord, regNewData+1, pTab->nNVCol, regRec); + sqlite3SetMakeRecordP5(v, pTab); + if( !bAffinityDone ){ + sqlite3TableAffinity(v, pTab, 0); + } + } + *pbMayReplace = seenReplace; VdbeModuleComment((v, "END: GenCnstCks(%d)", seenReplace)); } +#ifdef SQLITE_ENABLE_NULL_TRIM +/* +** Change the P5 operand on the last opcode (which should be an OP_MakeRecord) +** to be the number of columns in table pTab that must not be NULL-trimmed. +** +** Or if no columns of pTab may be NULL-trimmed, leave P5 at zero. +*/ +SQLITE_PRIVATE void sqlite3SetMakeRecordP5(Vdbe *v, Table *pTab){ + u16 i; + + /* Records with omitted columns are only allowed for schema format + ** version 2 and later (SQLite version 3.1.4, 2005-02-20). */ + if( pTab->pSchema->file_format<2 ) return; + + for(i=pTab->nCol-1; i>0; i--){ + if( pTab->aCol[i].iDflt!=0 ) break; + if( pTab->aCol[i].colFlags & COLFLAG_PRIMKEY ) break; + } + sqlite3VdbeChangeP5(v, i+1); +} +#endif + +/* +** Table pTab is a WITHOUT ROWID table that is being written to. The cursor +** number is iCur, and register regData contains the new record for the +** PK index. This function adds code to invoke the pre-update hook, +** if one is registered. +*/ +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK +static void codeWithoutRowidPreupdate( + Parse *pParse, /* Parse context */ + Table *pTab, /* Table being updated */ + int iCur, /* Cursor number for table */ + int regData /* Data containing new record */ +){ + Vdbe *v = pParse->pVdbe; + int r = sqlite3GetTempReg(pParse); + assert( !HasRowid(pTab) ); + assert( 0==(pParse->db->mDbFlags & DBFLAG_Vacuum) || CORRUPT_DB ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, r); + sqlite3VdbeAddOp4(v, OP_Insert, iCur, regData, r, (char*)pTab, P4_TABLE); + sqlite3VdbeChangeP5(v, OPFLAG_ISNOOP); + sqlite3ReleaseTempReg(pParse, r); +} +#else +# define codeWithoutRowidPreupdate(a,b,c,d) +#endif + /* ** This routine generates code to finish the INSERT or UPDATE operation ** that was started by a prior call to sqlite3GenerateConstraintChecks. @@ -107524,48 +136449,52 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( int iIdxCur, /* First index cursor */ int regNewData, /* Range of content */ int *aRegIdx, /* Register used by each index. 0 for unused indices */ - int isUpdate, /* True for UPDATE, False for INSERT */ + int update_flags, /* True for UPDATE, False for INSERT */ int appendBias, /* True if this is likely to be an append */ int useSeekResult /* True to set the USESEEKRESULT flag on OP_[Idx]Insert */ ){ Vdbe *v; /* Prepared statements under construction */ Index *pIdx; /* An index being inserted or updated */ u8 pik_flags; /* flag values passed to the btree insert */ - int regData; /* Content registers (after the rowid) */ - int regRec; /* Register holding assembled record for the table */ int i; /* Loop counter */ - u8 bAffinityDone = 0; /* True if OP_Affinity has been run already */ - v = sqlite3GetVdbe(pParse); + assert( update_flags==0 + || update_flags==OPFLAG_ISUPDATE + || update_flags==(OPFLAG_ISUPDATE|OPFLAG_SAVEPOSITION) + ); + + v = pParse->pVdbe; assert( v!=0 ); - assert( pTab->pSelect==0 ); /* This table is not a VIEW */ + assert( !IsView(pTab) ); /* This table is not a VIEW */ for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ + /* All REPLACE indexes are at the end of the list */ + assert( pIdx->onError!=OE_Replace + || pIdx->pNext==0 + || pIdx->pNext->onError==OE_Replace ); if( aRegIdx[i]==0 ) continue; - bAffinityDone = 1; if( pIdx->pPartIdxWhere ){ sqlite3VdbeAddOp2(v, OP_IsNull, aRegIdx[i], sqlite3VdbeCurrentAddr(v)+2); VdbeCoverage(v); } - sqlite3VdbeAddOp2(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i]); - pik_flags = 0; - if( useSeekResult ) pik_flags = OPFLAG_USESEEKRESULT; + pik_flags = (useSeekResult ? OPFLAG_USESEEKRESULT : 0); if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - assert( pParse->nested==0 ); pik_flags |= OPFLAG_NCHANGE; + pik_flags |= (update_flags & OPFLAG_SAVEPOSITION); + if( update_flags==0 ){ + codeWithoutRowidPreupdate(pParse, pTab, iIdxCur+i, aRegIdx[i]); + } } + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iIdxCur+i, aRegIdx[i], + aRegIdx[i]+1, + pIdx->uniqNotNull ? pIdx->nKeyCol: pIdx->nColumn); sqlite3VdbeChangeP5(v, pik_flags); } if( !HasRowid(pTab) ) return; - regData = regNewData + 1; - regRec = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp3(v, OP_MakeRecord, regData, pTab->nCol, regRec); - if( !bAffinityDone ) sqlite3TableAffinity(v, pTab, 0); - sqlite3ExprCacheAffinityChange(pParse, regData, pTab->nCol); if( pParse->nested ){ pik_flags = 0; }else{ pik_flags = OPFLAG_NCHANGE; - pik_flags |= (isUpdate?OPFLAG_ISUPDATE:OPFLAG_LASTROWID); + pik_flags |= (update_flags?update_flags:OPFLAG_LASTROWID); } if( appendBias ){ pik_flags |= OPFLAG_APPEND; @@ -107573,9 +136502,9 @@ SQLITE_PRIVATE void sqlite3CompleteInsertion( if( useSeekResult ){ pik_flags |= OPFLAG_USESEEKRESULT; } - sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, regRec, regNewData); + sqlite3VdbeAddOp3(v, OP_Insert, iDataCur, aRegIdx[i], regNewData); if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, (char *)pTab, P4_TABLE); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } sqlite3VdbeChangeP5(v, pik_flags); } @@ -107619,36 +136548,39 @@ SQLITE_PRIVATE int sqlite3OpenTableAndIndices( assert( op==OP_OpenRead || op==OP_OpenWrite ); assert( op==OP_OpenWrite || p5==0 ); + assert( piDataCur!=0 ); + assert( piIdxCur!=0 ); if( IsVirtual(pTab) ){ /* This routine is a no-op for virtual tables. Leave the output - ** variables *piDataCur and *piIdxCur uninitialized so that valgrind - ** can detect if they are used by mistake in the caller. */ + ** variables *piDataCur and *piIdxCur set to illegal cursor numbers + ** for improved error detection. */ + *piDataCur = *piIdxCur = -999; return 0; } iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); - v = sqlite3GetVdbe(pParse); + v = pParse->pVdbe; assert( v!=0 ); if( iBase<0 ) iBase = pParse->nTab; iDataCur = iBase++; - if( piDataCur ) *piDataCur = iDataCur; + *piDataCur = iDataCur; if( HasRowid(pTab) && (aToOpen==0 || aToOpen[0]) ){ sqlite3OpenTable(pParse, iDataCur, iDb, pTab, op); - }else{ + }else if( pParse->db->noSharedCache==0 ){ sqlite3TableLock(pParse, iDb, pTab->tnum, op==OP_OpenWrite, pTab->zName); } - if( piIdxCur ) *piIdxCur = iBase; + *piIdxCur = iBase; for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ int iIdxCur = iBase++; assert( pIdx->pSchema==pTab->pSchema ); + if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ + *piDataCur = iIdxCur; + p5 = 0; + } if( aToOpen==0 || aToOpen[i+1] ){ sqlite3VdbeAddOp3(v, op, iIdxCur, pIdx->tnum, iDb); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); - VdbeComment((v, "%s", pIdx->zName)); - } - if( IsPrimaryKeyIndex(pIdx) && !HasRowid(pTab) ){ - if( piDataCur ) *piDataCur = iIdxCur; - }else{ sqlite3VdbeChangeP5(v, p5); + VdbeComment((v, "%s", pIdx->zName)); } } if( iBase>pParse->nTab ) pParse->nTab = iBase; @@ -107683,7 +136615,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ int i; assert( pDest && pSrc ); assert( pDest->pTable!=pSrc->pTable ); - if( pDest->nKeyCol!=pSrc->nKeyCol ){ + if( pDest->nKeyCol!=pSrc->nKeyCol || pDest->nColumn!=pSrc->nColumn ){ return 0; /* Different number of columns */ } if( pDest->onError!=pSrc->onError ){ @@ -107695,7 +136627,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ } if( pSrc->aiColumn[i]==XN_EXPR ){ assert( pSrc->aColExpr!=0 && pDest->aColExpr!=0 ); - if( sqlite3ExprCompare(pSrc->aColExpr->a[i].pExpr, + if( sqlite3ExprCompare(0, pSrc->aColExpr->a[i].pExpr, pDest->aColExpr->a[i].pExpr, -1)!=0 ){ return 0; /* Different expressions in the index */ } @@ -107707,7 +136639,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ return 0; /* Different collating sequences */ } } - if( sqlite3ExprCompare(pSrc->pPartIdxWhere, pDest->pPartIdxWhere, -1) ){ + if( sqlite3ExprCompare(0, pSrc->pPartIdxWhere, pDest->pPartIdxWhere, -1) ){ return 0; /* Different WHERE clauses */ } @@ -107720,7 +136652,7 @@ static int xferCompatibleIndex(Index *pDest, Index *pSrc){ ** ** INSERT INTO tab1 SELECT * FROM tab2; ** -** The xfer optimization transfers raw records from tab2 over to tab1. +** The xfer optimization transfers raw records from tab2 over to tab1. ** Columns are not decoded and reassembled, which greatly improves ** performance. Raw index records are transferred in the same way. ** @@ -107751,7 +136683,7 @@ static int xferOptimization( ExprList *pEList; /* The result set of the SELECT */ Table *pSrc; /* The table in the FROM clause of SELECT */ Index *pSrcIdx, *pDestIdx; /* Source and destination indices */ - struct SrcList_item *pItem; /* An element of pSelect->pSrc */ + SrcItem *pItem; /* An element of pSelect->pSrc */ int i; /* Loop counter */ int iDbSrc; /* The database of pSrc */ int iSrc, iDest; /* Cursors from source and destination */ @@ -107763,20 +136695,15 @@ static int xferOptimization( int destHasUniqueIdx = 0; /* True if pDest has a UNIQUE index */ int regData, regRowid; /* Registers holding data and rowid */ - if( pSelect==0 ){ - return 0; /* Must be of the form INSERT INTO ... SELECT ... */ - } + assert( pSelect!=0 ); if( pParse->pWith || pSelect->pWith ){ /* Do not attempt to process this query if there are an WITH clauses ** attached to it. Proceeding may generate a false "no such table: xxx" ** error if pSelect reads from a CTE named "xxx". */ return 0; } - if( sqlite3TriggerList(pParse, pDest) ){ - return 0; /* tab1 must not have triggers */ - } #ifndef SQLITE_OMIT_VIRTUALTABLE - if( pDest->tabFlags & TF_Virtual ){ + if( IsVirtual(pDest) ){ return 0; /* tab1 must not be a virtual table */ } #endif @@ -107788,7 +136715,7 @@ static int xferOptimization( if( pSelect->pSrc->nSrc!=1 ){ return 0; /* FROM clause must have exactly one term */ } - if( pSelect->pSrc->a[0].pSelect ){ + if( pSelect->pSrc->a[0].fg.isSubquery ){ return 0; /* FROM clause cannot contain a subquery */ } if( pSelect->pWhere ){ @@ -107805,7 +136732,6 @@ static int xferOptimization( if( pSelect->pLimit ){ return 0; /* SELECT may not have a LIMIT clause */ } - assert( pSelect->pOffset==0 ); /* Must be so if pLimit==0 */ if( pSelect->pPrior ){ return 0; /* SELECT may not be a compound query */ } @@ -107831,19 +136757,15 @@ static int xferOptimization( if( pSrc==0 ){ return 0; /* FROM clause does not contain a real table */ } - if( pSrc==pDest ){ + if( pSrc->tnum==pDest->tnum && pSrc->pSchema==pDest->pSchema ){ + testcase( pSrc!=pDest ); /* Possible due to bad sqlite_schema.rootpage */ return 0; /* tab1 and tab2 may not be the same table */ } if( HasRowid(pDest)!=HasRowid(pSrc) ){ return 0; /* source and destination must both be WITHOUT ROWID or not */ } -#ifndef SQLITE_OMIT_VIRTUALTABLE - if( pSrc->tabFlags & TF_Virtual ){ - return 0; /* tab2 must not be a virtual table */ - } -#endif - if( pSrc->pSelect ){ - return 0; /* tab2 may not be a view */ + if( !IsOrdinaryTable(pSrc) ){ + return 0; /* tab2 may not be a view or virtual table */ } if( pDest->nCol!=pSrc->nCol ){ return 0; /* Number of columns must be the same in tab1 and tab2 */ @@ -107851,32 +136773,75 @@ static int xferOptimization( if( pDest->iPKey!=pSrc->iPKey ){ return 0; /* Both tables must have the same INTEGER PRIMARY KEY */ } + if( (pDest->tabFlags & TF_Strict)!=0 && (pSrc->tabFlags & TF_Strict)==0 ){ + return 0; /* Cannot feed from a non-strict into a strict table */ + } for(i=0; inCol; i++){ Column *pDestCol = &pDest->aCol[i]; Column *pSrcCol = &pSrc->aCol[i]; #ifdef SQLITE_ENABLE_HIDDEN_COLUMNS - if( (db->flags & SQLITE_Vacuum)==0 - && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && (pDestCol->colFlags | pSrcCol->colFlags) & COLFLAG_HIDDEN ){ return 0; /* Neither table may have __hidden__ columns */ } +#endif +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Even if tables t1 and t2 have identical schemas, if they contain + ** generated columns, then this statement is semantically incorrect: + ** + ** INSERT INTO t2 SELECT * FROM t1; + ** + ** The reason is that generated column values are returned by the + ** the SELECT statement on the right but the INSERT statement on the + ** left wants them to be omitted. + ** + ** Nevertheless, this is a useful notational shorthand to tell SQLite + ** to do a bulk transfer all of the content from t1 over to t2. + ** + ** We could, in theory, disable this (except for internal use by the + ** VACUUM command where it is actually needed). But why do that? It + ** seems harmless enough, and provides a useful service. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED) != + (pSrcCol->colFlags & COLFLAG_GENERATED) ){ + return 0; /* Both columns have the same generated-column type */ + } + /* But the transfer is only allowed if both the source and destination + ** tables have the exact same expressions for generated columns. + ** This requirement could be relaxed for VIRTUAL columns, I suppose. + */ + if( (pDestCol->colFlags & COLFLAG_GENERATED)!=0 ){ + if( sqlite3ExprCompare(0, + sqlite3ColumnExpr(pSrc, pSrcCol), + sqlite3ColumnExpr(pDest, pDestCol), -1)!=0 ){ + testcase( pDestCol->colFlags & COLFLAG_VIRTUAL ); + testcase( pDestCol->colFlags & COLFLAG_STORED ); + return 0; /* Different generator expressions */ + } + } #endif if( pDestCol->affinity!=pSrcCol->affinity ){ return 0; /* Affinity must be the same on all columns */ } - if( sqlite3_stricmp(pDestCol->zColl, pSrcCol->zColl)!=0 ){ + if( sqlite3_stricmp(sqlite3ColumnColl(pDestCol), + sqlite3ColumnColl(pSrcCol))!=0 ){ return 0; /* Collating sequence must be the same on all columns */ } if( pDestCol->notNull && !pSrcCol->notNull ){ return 0; /* tab2 must be NOT NULL if tab1 is */ } /* Default values for second and subsequent columns need to match. */ - if( i>0 ){ - assert( pDestCol->pDflt==0 || pDestCol->pDflt->op==TK_SPAN ); - assert( pSrcCol->pDflt==0 || pSrcCol->pDflt->op==TK_SPAN ); - if( (pDestCol->pDflt==0)!=(pSrcCol->pDflt==0) - || (pDestCol->pDflt && strcmp(pDestCol->pDflt->u.zToken, - pSrcCol->pDflt->u.zToken)!=0) + if( (pDestCol->colFlags & COLFLAG_GENERATED)==0 && i>0 ){ + Expr *pDestExpr = sqlite3ColumnExpr(pDest, pDestCol); + Expr *pSrcExpr = sqlite3ColumnExpr(pSrc, pSrcCol); + assert( pDestExpr==0 || pDestExpr->op==TK_SPAN ); + assert( pDestExpr==0 || !ExprHasProperty(pDestExpr, EP_IntValue) ); + assert( pSrcExpr==0 || pSrcExpr->op==TK_SPAN ); + assert( pSrcExpr==0 || !ExprHasProperty(pSrcExpr, EP_IntValue) ); + if( (pDestExpr==0)!=(pSrcExpr==0) + || (pDestExpr!=0 && strcmp(pDestExpr->u.zToken, + pSrcExpr->u.zToken)!=0) ){ return 0; /* Default values must be the same for all columns */ } @@ -107892,21 +136857,32 @@ static int xferOptimization( if( pSrcIdx==0 ){ return 0; /* pDestIdx has no corresponding index in pSrc */ } + if( pSrcIdx->tnum==pDestIdx->tnum && pSrc->pSchema==pDest->pSchema + && sqlite3FaultSim(411)==SQLITE_OK ){ + /* The sqlite3FaultSim() call allows this corruption test to be + ** bypassed during testing, in order to exercise other corruption tests + ** further downstream. */ + return 0; /* Corrupt schema - two indexes on the same btree */ + } } #ifndef SQLITE_OMIT_CHECK - if( pDest->pCheck && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) ){ + if( pDest->pCheck + && (db->mDbFlags & DBFLAG_Vacuum)==0 + && sqlite3ExprListCompare(pSrc->pCheck,pDest->pCheck,-1) + ){ return 0; /* Tables have different CHECK constraints. Ticket #2252 */ } #endif #ifndef SQLITE_OMIT_FOREIGN_KEY - /* Disallow the transfer optimization if the destination table constains + /* Disallow the transfer optimization if the destination table contains ** any foreign key constraints. This is more restrictive than necessary. - ** But the main beneficiary of the transfer optimization is the VACUUM + ** But the main beneficiary of the transfer optimization is the VACUUM ** command, and the VACUUM command disables foreign key constraints. So ** the extra complication to make this rule less restrictive is probably ** not worth the effort. Ticket [6284df89debdfa61db8073e062908af0c9b6118e] */ - if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->pFKey!=0 ){ + assert( IsOrdinaryTable(pDest) ); + if( (db->flags & SQLITE_ForeignKeys)!=0 && pDest->u.tab.pFKey!=0 ){ return 0; } #endif @@ -107928,18 +136904,19 @@ static int xferOptimization( iDest = pParse->nTab++; regAutoinc = autoIncBegin(pParse, iDbDest, pDest); regData = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, regData); regRowid = sqlite3GetTempReg(pParse); sqlite3OpenTable(pParse, iDest, iDbDest, pDest, OP_OpenWrite); assert( HasRowid(pDest) || destHasUniqueIdx ); - if( (db->flags & SQLITE_Vacuum)==0 && ( + if( (db->mDbFlags & DBFLAG_Vacuum)==0 && ( (pDest->iPKey<0 && pDest->pIndex!=0) /* (1) */ || destHasUniqueIdx /* (2) */ || (onError!=OE_Abort && onError!=OE_Rollback) /* (3) */ )){ /* In some circumstances, we are able to run the xfer optimization ** only if the destination table is initially empty. Unless the - ** SQLITE_Vacuum flag is set, this block generates code to make - ** that determination. If SQLITE_Vacuum is set, then the destination + ** DBFLAG_Vacuum flag is set, this block generates code to make + ** that determination. If DBFLAG_Vacuum is set, then the destination ** table is always empty. ** ** Conditions under which the destination must be empty: @@ -107948,7 +136925,7 @@ static int xferOptimization( ** (If the destination is not initially empty, the rowid fields ** of index entries might need to change.) ** - ** (2) The destination has a unique index. (The xfer optimization + ** (2) The destination has a unique index. (The xfer optimization ** is unable to test uniqueness.) ** ** (3) onError is something other than OE_Abort and OE_Rollback. @@ -107958,25 +136935,47 @@ static int xferOptimization( sqlite3VdbeJumpHere(v, addr1); } if( HasRowid(pSrc) ){ + u8 insFlags; sqlite3OpenTable(pParse, iSrc, iDbSrc, pSrc, OP_OpenRead); emptySrcTest = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); if( pDest->iPKey>=0 ){ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); - addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); - VdbeCoverage(v); - sqlite3RowidConstraint(pParse, onError, pDest); - sqlite3VdbeJumpHere(v, addr2); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeVerifyAbortable(v, onError); + addr2 = sqlite3VdbeAddOp3(v, OP_NotExists, iDest, 0, regRowid); + VdbeCoverage(v); + sqlite3RowidConstraint(pParse, onError, pDest); + sqlite3VdbeJumpHere(v, addr2); + } autoIncStep(pParse, regAutoinc, regRowid); - }else if( pDest->pIndex==0 ){ + }else if( pDest->pIndex==0 && !(db->mDbFlags & DBFLAG_VacuumInto) ){ addr1 = sqlite3VdbeAddOp2(v, OP_NewRowid, iDest, regRowid); }else{ addr1 = sqlite3VdbeAddOp2(v, OP_Rowid, iSrc, regRowid); assert( (pDest->tabFlags & TF_Autoincrement)==0 ); } - sqlite3VdbeAddOp2(v, OP_RowData, iSrc, regData); - sqlite3VdbeAddOp4(v, OP_Insert, iDest, regData, regRowid, - (char*)pDest, P4_TABLE); - sqlite3VdbeChangeP5(v, OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND); + + if( db->mDbFlags & DBFLAG_Vacuum ){ + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); + insFlags = OPFLAG_APPEND|OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; + }else{ + insFlags = OPFLAG_NCHANGE|OPFLAG_LASTROWID|OPFLAG_APPEND|OPFLAG_PREFORMAT; + } +#ifdef SQLITE_ENABLE_PREUPDATE_HOOK + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + insFlags &= ~OPFLAG_PREFORMAT; + }else +#endif + { + sqlite3VdbeAddOp3(v, OP_RowCell, iDest, iSrc, regRowid); + } + sqlite3VdbeAddOp3(v, OP_Insert, iDest, regData, regRowid); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 ){ + sqlite3VdbeChangeP4(v, -1, (char*)pDest, P4_TABLE); + } + sqlite3VdbeChangeP5(v, insFlags); + sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1); VdbeCoverage(v); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -107998,38 +136997,44 @@ static int xferOptimization( sqlite3VdbeChangeP5(v, OPFLAG_BULKCSR); VdbeComment((v, "%s", pDestIdx->zName)); addr1 = sqlite3VdbeAddOp2(v, OP_Rewind, iSrc, 0); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_RowKey, iSrc, regData); - if( db->flags & SQLITE_Vacuum ){ + if( db->mDbFlags & DBFLAG_Vacuum ){ /* This INSERT command is part of a VACUUM operation, which guarantees ** that the destination table is empty. If all indexed columns use ** collation sequence BINARY, then it can also be assumed that the - ** index will be populated by inserting keys in strictly sorted + ** index will be populated by inserting keys in strictly sorted ** order. In this case, instead of seeking within the b-tree as part - ** of every OP_IdxInsert opcode, an OP_Last is added before the - ** OP_IdxInsert to seek to the point within the b-tree where each key + ** of every OP_IdxInsert opcode, an OP_SeekEnd is added before the + ** OP_IdxInsert to seek to the point within the b-tree where each key ** should be inserted. This is faster. ** ** If any of the indexed columns use a collation sequence other than - ** BINARY, this optimization is disabled. This is because the user + ** BINARY, this optimization is disabled. This is because the user ** might change the definition of a collation sequence and then run ** a VACUUM command. In that case keys may not be written in strictly ** sorted order. */ for(i=0; inColumn; i++){ const char *zColl = pSrcIdx->azColl[i]; - assert( sqlite3_stricmp(sqlite3StrBINARY, zColl)!=0 - || sqlite3StrBINARY==zColl ); if( sqlite3_stricmp(sqlite3StrBINARY, zColl) ) break; } if( i==pSrcIdx->nColumn ){ - idxInsFlags = OPFLAG_USESEEKRESULT; - sqlite3VdbeAddOp3(v, OP_Last, iDest, 0, -1); + idxInsFlags = OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT; + sqlite3VdbeAddOp1(v, OP_SeekEnd, iDest); + sqlite3VdbeAddOp2(v, OP_RowCell, iDest, iSrc); } - } - if( !HasRowid(pSrc) && pDestIdx->idxType==2 ){ + }else if( !HasRowid(pSrc) && pDestIdx->idxType==SQLITE_IDXTYPE_PRIMARYKEY ){ idxInsFlags |= OPFLAG_NCHANGE; } - sqlite3VdbeAddOp3(v, OP_IdxInsert, iDest, regData, 1); - sqlite3VdbeChangeP5(v, idxInsFlags); + if( idxInsFlags!=(OPFLAG_USESEEKRESULT|OPFLAG_PREFORMAT) ){ + sqlite3VdbeAddOp3(v, OP_RowData, iSrc, regData, 1); + if( (db->mDbFlags & DBFLAG_Vacuum)==0 + && !HasRowid(pDest) + && IsPrimaryKeyIndex(pDestIdx) + ){ + codeWithoutRowidPreupdate(pParse, pDest, iDest, regData); + } + } + sqlite3VdbeAddOp2(v, OP_IdxInsert, iDest, regData); + sqlite3VdbeChangeP5(v, idxInsFlags|OPFLAG_APPEND); sqlite3VdbeAddOp2(v, OP_Next, iSrc, addr1+1); VdbeCoverage(v); sqlite3VdbeJumpHere(v, addr1); sqlite3VdbeAddOp2(v, OP_Close, iSrc, 0); @@ -108039,6 +137044,7 @@ static int xferOptimization( sqlite3ReleaseTempReg(pParse, regRowid); sqlite3ReleaseTempReg(pParse, regData); if( emptyDestTest ){ + sqlite3AutoincrementEnd(pParse); sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_OK, 0); sqlite3VdbeJumpHere(v, emptyDestTest); sqlite3VdbeAddOp2(v, OP_Close, iDest, 0); @@ -108080,7 +137086,7 @@ static int xferOptimization( ** argument to xCallback(). If xCallback=NULL then no callback ** is invoked, even for queries. */ -SQLITE_API int SQLITE_STDCALL sqlite3_exec( +SQLITE_API int sqlite3_exec( sqlite3 *db, /* The database on which the SQL executes */ const char *zSql, /* The SQL to be executed */ sqlite3_callback xCallback, /* Invoke this callback routine */ @@ -108099,7 +137105,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( sqlite3_mutex_enter(db->mutex); sqlite3Error(db, SQLITE_OK); while( rc==SQLITE_OK && zSql[0] ){ - int nCol; + int nCol = 0; char **azVals = 0; pStmt = 0; @@ -108113,20 +137119,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( zSql = zLeftover; continue; } - callbackIsInit = 0; - nCol = sqlite3_column_count(pStmt); while( 1 ){ int i; rc = sqlite3_step(pStmt); /* Invoke the callback function if required */ - if( xCallback && (SQLITE_ROW==rc || + if( xCallback && (SQLITE_ROW==rc || (SQLITE_DONE==rc && !callbackIsInit && db->flags&SQLITE_NullCallback)) ){ if( !callbackIsInit ){ - azCols = sqlite3DbMallocZero(db, 2*nCol*sizeof(const char*) + 1); + nCol = sqlite3_column_count(pStmt); + azCols = sqlite3DbMallocRaw(db, (2*nCol+1)*sizeof(const char*)); if( azCols==0 ){ goto exec_out; } @@ -108147,6 +137152,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_exec( goto exec_out; } } + azVals[i] = 0; } if( xCallback(pArg, nCol, azVals, azCols) ){ /* EVIDENCE-OF: R-38229-40159 If the callback function to @@ -108179,11 +137185,8 @@ exec_out: rc = sqlite3ApiExit(db, rc); if( rc!=SQLITE_OK && pzErrMsg ){ - int nErrMsg = 1 + sqlite3Strlen30(sqlite3_errmsg(db)); - *pzErrMsg = sqlite3Malloc(nErrMsg); - if( *pzErrMsg ){ - memcpy(*pzErrMsg, sqlite3_errmsg(db), nErrMsg); - }else{ + *pzErrMsg = sqlite3DbStrDup(0, sqlite3_errmsg(db)); + if( *pzErrMsg==0 ){ rc = SQLITE_NOMEM_BKPT; sqlite3Error(db, SQLITE_NOMEM); } @@ -108232,15 +137235,13 @@ exec_out: ** This header file defines the SQLite interface for use by ** shared libraries that want to be imported as extensions into ** an SQLite instance. Shared libraries that intend to be loaded -** as extensions by SQLite should #include this file instead of +** as extensions by SQLite should #include this file instead of ** sqlite3.h. */ -#ifndef _SQLITE3EXT_H_ -#define _SQLITE3EXT_H_ +#ifndef SQLITE3EXT_H +#define SQLITE3EXT_H /* #include "sqlite3.h" */ -typedef struct sqlite3_api_routines sqlite3_api_routines; - /* ** The following structure holds pointers to all of the SQLite API ** routines. @@ -108356,7 +137357,7 @@ struct sqlite3_api_routines { int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*, const char*,const char*),void*); void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*)); - char * (*snprintf)(int,char*,const char*,...); + char * (*xsnprintf)(int,char*,const char*,...); int (*step)(sqlite3_stmt*); int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*, char const**,char const**,int*,int*,int*); @@ -108468,7 +137469,7 @@ struct sqlite3_api_routines { int (*uri_boolean)(const char*,const char*,int); sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64); const char *(*uri_parameter)(const char*,const char*); - char *(*vsnprintf)(int,char*,const char*,va_list); + char *(*xvsnprintf)(int,char*,const char*,va_list); int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*); /* Version 3.8.7 and later */ int (*auto_extension)(void(*)(void)); @@ -108501,8 +137502,105 @@ struct sqlite3_api_routines { int (*db_cacheflush)(sqlite3*); /* Version 3.12.0 and later */ int (*system_errno)(sqlite3*); + /* Version 3.14.0 and later */ + int (*trace_v2)(sqlite3*,unsigned,int(*)(unsigned,void*,void*,void*),void*); + char *(*expanded_sql)(sqlite3_stmt*); + /* Version 3.18.0 and later */ + void (*set_last_insert_rowid)(sqlite3*,sqlite3_int64); + /* Version 3.20.0 and later */ + int (*prepare_v3)(sqlite3*,const char*,int,unsigned int, + sqlite3_stmt**,const char**); + int (*prepare16_v3)(sqlite3*,const void*,int,unsigned int, + sqlite3_stmt**,const void**); + int (*bind_pointer)(sqlite3_stmt*,int,void*,const char*,void(*)(void*)); + void (*result_pointer)(sqlite3_context*,void*,const char*,void(*)(void*)); + void *(*value_pointer)(sqlite3_value*,const char*); + int (*vtab_nochange)(sqlite3_context*); + int (*value_nochange)(sqlite3_value*); + const char *(*vtab_collation)(sqlite3_index_info*,int); + /* Version 3.24.0 and later */ + int (*keyword_count)(void); + int (*keyword_name)(int,const char**,int*); + int (*keyword_check)(const char*,int); + sqlite3_str *(*str_new)(sqlite3*); + char *(*str_finish)(sqlite3_str*); + void (*str_appendf)(sqlite3_str*, const char *zFormat, ...); + void (*str_vappendf)(sqlite3_str*, const char *zFormat, va_list); + void (*str_append)(sqlite3_str*, const char *zIn, int N); + void (*str_appendall)(sqlite3_str*, const char *zIn); + void (*str_appendchar)(sqlite3_str*, int N, char C); + void (*str_reset)(sqlite3_str*); + int (*str_errcode)(sqlite3_str*); + int (*str_length)(sqlite3_str*); + char *(*str_value)(sqlite3_str*); + /* Version 3.25.0 and later */ + int (*create_window_function)(sqlite3*,const char*,int,int,void*, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInv)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*)); + /* Version 3.26.0 and later */ + const char *(*normalized_sql)(sqlite3_stmt*); + /* Version 3.28.0 and later */ + int (*stmt_isexplain)(sqlite3_stmt*); + int (*value_frombind)(sqlite3_value*); + /* Version 3.30.0 and later */ + int (*drop_modules)(sqlite3*,const char**); + /* Version 3.31.0 and later */ + sqlite3_int64 (*hard_heap_limit64)(sqlite3_int64); + const char *(*uri_key)(const char*,int); + const char *(*filename_database)(const char*); + const char *(*filename_journal)(const char*); + const char *(*filename_wal)(const char*); + /* Version 3.32.0 and later */ + const char *(*create_filename)(const char*,const char*,const char*, + int,const char**); + void (*free_filename)(const char*); + sqlite3_file *(*database_file_object)(const char*); + /* Version 3.34.0 and later */ + int (*txn_state)(sqlite3*,const char*); + /* Version 3.36.1 and later */ + sqlite3_int64 (*changes64)(sqlite3*); + sqlite3_int64 (*total_changes64)(sqlite3*); + /* Version 3.37.0 and later */ + int (*autovacuum_pages)(sqlite3*, + unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int), + void*, void(*)(void*)); + /* Version 3.38.0 and later */ + int (*error_offset)(sqlite3*); + int (*vtab_rhs_value)(sqlite3_index_info*,int,sqlite3_value**); + int (*vtab_distinct)(sqlite3_index_info*); + int (*vtab_in)(sqlite3_index_info*,int,int); + int (*vtab_in_first)(sqlite3_value*,sqlite3_value**); + int (*vtab_in_next)(sqlite3_value*,sqlite3_value**); + /* Version 3.39.0 and later */ + int (*deserialize)(sqlite3*,const char*,unsigned char*, + sqlite3_int64,sqlite3_int64,unsigned); + unsigned char *(*serialize)(sqlite3*,const char *,sqlite3_int64*, + unsigned int); + const char *(*db_name)(sqlite3*,int); + /* Version 3.40.0 and later */ + int (*value_encoding)(sqlite3_value*); + /* Version 3.41.0 and later */ + int (*is_interrupted)(sqlite3*); + /* Version 3.43.0 and later */ + int (*stmt_explain)(sqlite3_stmt*,int); + /* Version 3.44.0 and later */ + void *(*get_clientdata)(sqlite3*,const char*); + int (*set_clientdata)(sqlite3*, const char*, void*, void(*)(void*)); }; +/* +** This is the function signature used for all extension entry points. It +** is also defined in the file "loadext.c". +*/ +typedef int (*sqlite3_loadext_entry)( + sqlite3 *db, /* Handle to the database. */ + char **pzErrMsg, /* Used to set error string on failure. */ + const sqlite3_api_routines *pThunk /* Extension API function pointers. */ +); + /* ** The following macros redefine the API routines so that they are ** redirected through the global sqlite3_api structure. @@ -108617,7 +137715,7 @@ struct sqlite3_api_routines { #define sqlite3_rollback_hook sqlite3_api->rollback_hook #define sqlite3_set_authorizer sqlite3_api->set_authorizer #define sqlite3_set_auxdata sqlite3_api->set_auxdata -#define sqlite3_snprintf sqlite3_api->snprintf +#define sqlite3_snprintf sqlite3_api->xsnprintf #define sqlite3_step sqlite3_api->step #define sqlite3_table_column_metadata sqlite3_api->table_column_metadata #define sqlite3_thread_cleanup sqlite3_api->thread_cleanup @@ -108641,7 +137739,7 @@ struct sqlite3_api_routines { #define sqlite3_value_text16le sqlite3_api->value_text16le #define sqlite3_value_type sqlite3_api->value_type #define sqlite3_vmprintf sqlite3_api->vmprintf -#define sqlite3_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_vsnprintf sqlite3_api->xvsnprintf #define sqlite3_overload_function sqlite3_api->overload_function #define sqlite3_prepare_v2 sqlite3_api->prepare_v2 #define sqlite3_prepare16_v2 sqlite3_api->prepare16_v2 @@ -108717,7 +137815,7 @@ struct sqlite3_api_routines { #define sqlite3_uri_boolean sqlite3_api->uri_boolean #define sqlite3_uri_int64 sqlite3_api->uri_int64 #define sqlite3_uri_parameter sqlite3_api->uri_parameter -#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf +#define sqlite3_uri_vsnprintf sqlite3_api->xvsnprintf #define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2 /* Version 3.8.7 and later */ #define sqlite3_auto_extension sqlite3_api->auto_extension @@ -108746,32 +137844,108 @@ struct sqlite3_api_routines { #define sqlite3_db_cacheflush sqlite3_api->db_cacheflush /* Version 3.12.0 and later */ #define sqlite3_system_errno sqlite3_api->system_errno +/* Version 3.14.0 and later */ +#define sqlite3_trace_v2 sqlite3_api->trace_v2 +#define sqlite3_expanded_sql sqlite3_api->expanded_sql +/* Version 3.18.0 and later */ +#define sqlite3_set_last_insert_rowid sqlite3_api->set_last_insert_rowid +/* Version 3.20.0 and later */ +#define sqlite3_prepare_v3 sqlite3_api->prepare_v3 +#define sqlite3_prepare16_v3 sqlite3_api->prepare16_v3 +#define sqlite3_bind_pointer sqlite3_api->bind_pointer +#define sqlite3_result_pointer sqlite3_api->result_pointer +#define sqlite3_value_pointer sqlite3_api->value_pointer +/* Version 3.22.0 and later */ +#define sqlite3_vtab_nochange sqlite3_api->vtab_nochange +#define sqlite3_value_nochange sqlite3_api->value_nochange +#define sqlite3_vtab_collation sqlite3_api->vtab_collation +/* Version 3.24.0 and later */ +#define sqlite3_keyword_count sqlite3_api->keyword_count +#define sqlite3_keyword_name sqlite3_api->keyword_name +#define sqlite3_keyword_check sqlite3_api->keyword_check +#define sqlite3_str_new sqlite3_api->str_new +#define sqlite3_str_finish sqlite3_api->str_finish +#define sqlite3_str_appendf sqlite3_api->str_appendf +#define sqlite3_str_vappendf sqlite3_api->str_vappendf +#define sqlite3_str_append sqlite3_api->str_append +#define sqlite3_str_appendall sqlite3_api->str_appendall +#define sqlite3_str_appendchar sqlite3_api->str_appendchar +#define sqlite3_str_reset sqlite3_api->str_reset +#define sqlite3_str_errcode sqlite3_api->str_errcode +#define sqlite3_str_length sqlite3_api->str_length +#define sqlite3_str_value sqlite3_api->str_value +/* Version 3.25.0 and later */ +#define sqlite3_create_window_function sqlite3_api->create_window_function +/* Version 3.26.0 and later */ +#define sqlite3_normalized_sql sqlite3_api->normalized_sql +/* Version 3.28.0 and later */ +#define sqlite3_stmt_isexplain sqlite3_api->stmt_isexplain +#define sqlite3_value_frombind sqlite3_api->value_frombind +/* Version 3.30.0 and later */ +#define sqlite3_drop_modules sqlite3_api->drop_modules +/* Version 3.31.0 and later */ +#define sqlite3_hard_heap_limit64 sqlite3_api->hard_heap_limit64 +#define sqlite3_uri_key sqlite3_api->uri_key +#define sqlite3_filename_database sqlite3_api->filename_database +#define sqlite3_filename_journal sqlite3_api->filename_journal +#define sqlite3_filename_wal sqlite3_api->filename_wal +/* Version 3.32.0 and later */ +#define sqlite3_create_filename sqlite3_api->create_filename +#define sqlite3_free_filename sqlite3_api->free_filename +#define sqlite3_database_file_object sqlite3_api->database_file_object +/* Version 3.34.0 and later */ +#define sqlite3_txn_state sqlite3_api->txn_state +/* Version 3.36.1 and later */ +#define sqlite3_changes64 sqlite3_api->changes64 +#define sqlite3_total_changes64 sqlite3_api->total_changes64 +/* Version 3.37.0 and later */ +#define sqlite3_autovacuum_pages sqlite3_api->autovacuum_pages +/* Version 3.38.0 and later */ +#define sqlite3_error_offset sqlite3_api->error_offset +#define sqlite3_vtab_rhs_value sqlite3_api->vtab_rhs_value +#define sqlite3_vtab_distinct sqlite3_api->vtab_distinct +#define sqlite3_vtab_in sqlite3_api->vtab_in +#define sqlite3_vtab_in_first sqlite3_api->vtab_in_first +#define sqlite3_vtab_in_next sqlite3_api->vtab_in_next +/* Version 3.39.0 and later */ +#ifndef SQLITE_OMIT_DESERIALIZE +#define sqlite3_deserialize sqlite3_api->deserialize +#define sqlite3_serialize sqlite3_api->serialize +#endif +#define sqlite3_db_name sqlite3_api->db_name +/* Version 3.40.0 and later */ +#define sqlite3_value_encoding sqlite3_api->value_encoding +/* Version 3.41.0 and later */ +#define sqlite3_is_interrupted sqlite3_api->is_interrupted +/* Version 3.43.0 and later */ +#define sqlite3_stmt_explain sqlite3_api->stmt_explain +/* Version 3.44.0 and later */ +#define sqlite3_get_clientdata sqlite3_api->get_clientdata +#define sqlite3_set_clientdata sqlite3_api->set_clientdata #endif /* !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) */ #if !defined(SQLITE_CORE) && !defined(SQLITE_OMIT_LOAD_EXTENSION) - /* This case when the file really is being compiled as a loadable + /* This case when the file really is being compiled as a loadable ** extension */ # define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0; # define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v; # define SQLITE_EXTENSION_INIT3 \ extern const sqlite3_api_routines *sqlite3_api; #else - /* This case when the file is being statically linked into the + /* This case when the file is being statically linked into the ** application */ # define SQLITE_EXTENSION_INIT1 /*no-op*/ # define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */ # define SQLITE_EXTENSION_INIT3 /*no-op*/ #endif -#endif /* _SQLITE3EXT_H_ */ +#endif /* SQLITE3EXT_H */ /************** End of sqlite3ext.h ******************************************/ /************** Continuing where we left off in loadext.c ********************/ /* #include "sqliteInt.h" */ -/* #include */ #ifndef SQLITE_OMIT_LOAD_EXTENSION - /* ** Some API routines are omitted when various features are ** excluded from a build of SQLite. Substitute a NULL pointer @@ -108803,6 +137977,7 @@ struct sqlite3_api_routines { # define sqlite3_open16 0 # define sqlite3_prepare16 0 # define sqlite3_prepare16_v2 0 +# define sqlite3_prepare16_v3 0 # define sqlite3_result_error16 0 # define sqlite3_result_text16 0 # define sqlite3_result_text16be 0 @@ -108835,13 +138010,14 @@ struct sqlite3_api_routines { # define sqlite3_declare_vtab 0 # define sqlite3_vtab_config 0 # define sqlite3_vtab_on_conflict 0 +# define sqlite3_vtab_collation 0 #endif #ifdef SQLITE_OMIT_SHARED_CACHE # define sqlite3_enable_shared_cache 0 #endif -#ifdef SQLITE_OMIT_TRACE +#if defined(SQLITE_OMIT_TRACE) || defined(SQLITE_OMIT_DEPRECATED) # define sqlite3_profile 0 # define sqlite3_trace 0 #endif @@ -108861,6 +138037,10 @@ struct sqlite3_api_routines { #define sqlite3_blob_reopen 0 #endif +#if defined(SQLITE_OMIT_TRACE) +# define sqlite3_trace_v2 0 +#endif + /* ** The following structure contains pointers to all SQLite API routines. ** A pointer to this structure is passed into extensions when they are @@ -109044,8 +138224,8 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_memory_highwater, sqlite3_memory_used, #ifdef SQLITE_MUTEX_OMIT - 0, - 0, + 0, + 0, 0, 0, 0, @@ -109166,9 +138346,114 @@ static const sqlite3_api_routines sqlite3Apis = { sqlite3_strlike, sqlite3_db_cacheflush, /* Version 3.12.0 and later */ - sqlite3_system_errno + sqlite3_system_errno, + /* Version 3.14.0 and later */ + sqlite3_trace_v2, + sqlite3_expanded_sql, + /* Version 3.18.0 and later */ + sqlite3_set_last_insert_rowid, + /* Version 3.20.0 and later */ + sqlite3_prepare_v3, + sqlite3_prepare16_v3, + sqlite3_bind_pointer, + sqlite3_result_pointer, + sqlite3_value_pointer, + /* Version 3.22.0 and later */ + sqlite3_vtab_nochange, + sqlite3_value_nochange, + sqlite3_vtab_collation, + /* Version 3.24.0 and later */ + sqlite3_keyword_count, + sqlite3_keyword_name, + sqlite3_keyword_check, + sqlite3_str_new, + sqlite3_str_finish, + sqlite3_str_appendf, + sqlite3_str_vappendf, + sqlite3_str_append, + sqlite3_str_appendall, + sqlite3_str_appendchar, + sqlite3_str_reset, + sqlite3_str_errcode, + sqlite3_str_length, + sqlite3_str_value, + /* Version 3.25.0 and later */ + sqlite3_create_window_function, + /* Version 3.26.0 and later */ +#ifdef SQLITE_ENABLE_NORMALIZE + sqlite3_normalized_sql, +#else + 0, +#endif + /* Version 3.28.0 and later */ + sqlite3_stmt_isexplain, + sqlite3_value_frombind, + /* Version 3.30.0 and later */ +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_drop_modules, +#else + 0, +#endif + /* Version 3.31.0 and later */ + sqlite3_hard_heap_limit64, + sqlite3_uri_key, + sqlite3_filename_database, + sqlite3_filename_journal, + sqlite3_filename_wal, + /* Version 3.32.0 and later */ + sqlite3_create_filename, + sqlite3_free_filename, + sqlite3_database_file_object, + /* Version 3.34.0 and later */ + sqlite3_txn_state, + /* Version 3.36.1 and later */ + sqlite3_changes64, + sqlite3_total_changes64, + /* Version 3.37.0 and later */ + sqlite3_autovacuum_pages, + /* Version 3.38.0 and later */ + sqlite3_error_offset, +#ifndef SQLITE_OMIT_VIRTUALTABLE + sqlite3_vtab_rhs_value, + sqlite3_vtab_distinct, + sqlite3_vtab_in, + sqlite3_vtab_in_first, + sqlite3_vtab_in_next, +#else + 0, + 0, + 0, + 0, + 0, +#endif + /* Version 3.39.0 and later */ +#ifndef SQLITE_OMIT_DESERIALIZE + sqlite3_deserialize, + sqlite3_serialize, +#else + 0, + 0, +#endif + sqlite3_db_name, + /* Version 3.40.0 and later */ + sqlite3_value_encoding, + /* Version 3.41.0 and later */ + sqlite3_is_interrupted, + /* Version 3.43.0 and later */ + sqlite3_stmt_explain, + /* Version 3.44.0 and later */ + sqlite3_get_clientdata, + sqlite3_set_clientdata }; +/* True if x is the directory separator character +*/ +#if SQLITE_OS_WIN +# define DirSep(X) ((X)=='/'||(X)=='\\') +#else +# define DirSep(X) ((X)=='/') +#endif + /* ** Attempt to load an SQLite extension library contained in the file ** zFile. The entry point is zProc. zProc may be 0 in which case a @@ -109177,7 +138462,7 @@ static const sqlite3_api_routines sqlite3Apis = { ** ** Return SQLITE_OK on success and SQLITE_ERROR if something goes wrong. ** -** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with +** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with ** error message text. The calling function should free this memory ** by calling sqlite3DbFree(db, ). */ @@ -109189,18 +138474,19 @@ static int sqlite3LoadExtension( ){ sqlite3_vfs *pVfs = db->pVfs; void *handle; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; char *zErrmsg = 0; const char *zEntry; char *zAltEntry = 0; void **aHandle; - u64 nMsg = 300 + sqlite3Strlen30(zFile); + u64 nMsg = strlen(zFile); int ii; + int rc; /* Shared library endings to try if zFile cannot be loaded as written */ static const char *azEndings[] = { #if SQLITE_OS_WIN - "dll" + "dll" #elif defined(__APPLE__) "dylib" #else @@ -109227,35 +138513,40 @@ static int sqlite3LoadExtension( zEntry = zProc ? zProc : "sqlite3_extension_init"; + /* tag-20210611-1. Some dlopen() implementations will segfault if given + ** an oversize filename. Most filesystems have a pathname limit of 4K, + ** so limit the extension filename length to about twice that. + ** https://sqlite.org/forum/forumpost/08a0d6d9bf + ** + ** Later (2023-03-25): Save an extra 6 bytes for the filename suffix. + ** See https://sqlite.org/forum/forumpost/24083b579d. + */ + if( nMsg>SQLITE_MAX_PATHLEN ) goto extension_not_found; + + /* Do not allow sqlite3_load_extension() to link to a copy of the + ** running application, by passing in an empty filename. */ + if( nMsg==0 ) goto extension_not_found; + handle = sqlite3OsDlOpen(pVfs, zFile); #if SQLITE_OS_UNIX || SQLITE_OS_WIN for(ii=0; ii sqlite3_example_init @@ -109270,7 +138561,7 @@ static int sqlite3LoadExtension( return SQLITE_NOMEM_BKPT; } memcpy(zAltEntry, "sqlite3_", 8); - for(iFile=ncFile-1; iFile>=0 && zFile[iFile]!='/'; iFile--){} + for(iFile=ncFile-1; iFile>=0 && !DirSep(zFile[iFile]); iFile--){} iFile++; if( sqlite3_strnicmp(zFile+iFile, "lib", 3)==0 ) iFile += 3; for(iEntry=8; (c = zFile[iFile])!=0 && c!='.'; iFile++){ @@ -109280,15 +138571,15 @@ static int sqlite3LoadExtension( } memcpy(zAltEntry+iEntry, "_init", 6); zEntry = zAltEntry; - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - sqlite3OsDlSym(pVfs, handle, zEntry); + xInit = (sqlite3_loadext_entry)sqlite3OsDlSym(pVfs, handle, zEntry); } if( xInit==0 ){ if( pzErrMsg ){ - nMsg += sqlite3Strlen30(zEntry); + nMsg += strlen(zEntry) + 300; *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); if( zErrmsg ){ - sqlite3_snprintf(nMsg, zErrmsg, + assert( nMsg<0x7fffffff ); /* zErrmsg would be NULL if not so */ + sqlite3_snprintf((int)nMsg, zErrmsg, "no entry point [%s] in shared library [%s]", zEntry, zFile); sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); } @@ -109298,7 +138589,9 @@ static int sqlite3LoadExtension( return SQLITE_ERROR; } sqlite3_free(zAltEntry); - if( xInit(db, &zErrmsg, &sqlite3Apis) ){ + rc = xInit(db, &zErrmsg, &sqlite3Apis); + if( rc ){ + if( rc==SQLITE_OK_LOAD_PERMANENTLY ) return SQLITE_OK; if( pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("error during initialization: %s", zErrmsg); } @@ -109320,8 +138613,21 @@ static int sqlite3LoadExtension( db->aExtension[db->nExtension++] = handle; return SQLITE_OK; + +extension_not_found: + if( pzErrMsg ){ + nMsg += 300; + *pzErrMsg = zErrmsg = sqlite3_malloc64(nMsg); + if( zErrmsg ){ + assert( nMsg<0x7fffffff ); /* zErrmsg would be NULL if not so */ + sqlite3_snprintf((int)nMsg, zErrmsg, + "unable to open shared library [%.*s]", SQLITE_MAX_PATHLEN, zFile); + sqlite3OsDlError(pVfs, nMsg-1, zErrmsg); + } + } + return SQLITE_ERROR; } -SQLITE_API int SQLITE_STDCALL sqlite3_load_extension( +SQLITE_API int sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Use "sqlite3_extension_init" if 0 */ @@ -109352,40 +138658,32 @@ SQLITE_PRIVATE void sqlite3CloseExtensions(sqlite3 *db){ ** Enable or disable extension loading. Extension loading is disabled by ** default so as not to open security holes in older applications. */ -SQLITE_API int SQLITE_STDCALL sqlite3_enable_load_extension(sqlite3 *db, int onoff){ +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); if( onoff ){ db->flags |= SQLITE_LoadExtension|SQLITE_LoadExtFunc; }else{ - db->flags &= ~(SQLITE_LoadExtension|SQLITE_LoadExtFunc); + db->flags &= ~(u64)(SQLITE_LoadExtension|SQLITE_LoadExtFunc); } sqlite3_mutex_leave(db->mutex); return SQLITE_OK; } -#endif /* SQLITE_OMIT_LOAD_EXTENSION */ - -/* -** The auto-extension code added regardless of whether or not extension -** loading is supported. We need a dummy sqlite3Apis pointer for that -** code if regular extension loading is not available. This is that -** dummy pointer. -*/ -#ifdef SQLITE_OMIT_LOAD_EXTENSION -static const sqlite3_api_routines sqlite3Apis = { 0 }; -#endif - +#endif /* !defined(SQLITE_OMIT_LOAD_EXTENSION) */ /* ** The following object holds the list of automatically loaded ** extensions. ** -** This list is shared across threads. The SQLITE_MUTEX_STATIC_MASTER +** This list is shared across threads. The SQLITE_MUTEX_STATIC_MAIN ** mutex must be held while accessing this list. */ typedef struct sqlite3AutoExtList sqlite3AutoExtList; static SQLITE_WSD struct sqlite3AutoExtList { - u32 nExt; /* Number of entries in aExt[] */ + u32 nExt; /* Number of entries in aExt[] */ void (**aExt)(void); /* Pointers to the extension init functions */ } sqlite3Autoext = { 0, 0 }; @@ -109409,8 +138707,13 @@ static SQLITE_WSD struct sqlite3AutoExtList { ** Register a statically linked extension that is automatically ** loaded by every new database connection. */ -SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xInit)(void)){ +SQLITE_API int sqlite3_auto_extension( + void (*xInit)(void) +){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return SQLITE_MISUSE_BKPT; +#endif #ifndef SQLITE_OMIT_AUTOINIT rc = sqlite3_initialize(); if( rc ){ @@ -109420,7 +138723,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xInit)(void)){ { u32 i; #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); #endif wsdAutoextInit; sqlite3_mutex_enter(mutex); @@ -109454,13 +138757,18 @@ SQLITE_API int SQLITE_STDCALL sqlite3_auto_extension(void (*xInit)(void)){ ** Return 1 if xInit was found on the list and removed. Return 0 if xInit ** was not on the list. */ -SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xInit)(void)){ +SQLITE_API int sqlite3_cancel_auto_extension( + void (*xInit)(void) +){ #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); #endif int i; int n = 0; wsdAutoextInit; +#ifdef SQLITE_ENABLE_API_ARMOR + if( xInit==0 ) return 0; +#endif sqlite3_mutex_enter(mutex); for(i=(int)wsdAutoext.nExt-1; i>=0; i--){ if( wsdAutoext.aExt[i]==xInit ){ @@ -109477,13 +138785,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_cancel_auto_extension(void (*xInit)(void)) /* ** Reset the automatic extension loading mechanism. */ -SQLITE_API void SQLITE_STDCALL sqlite3_reset_auto_extension(void){ +SQLITE_API void sqlite3_reset_auto_extension(void){ #ifndef SQLITE_OMIT_AUTOINIT if( sqlite3_initialize()==SQLITE_OK ) #endif { #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); #endif wsdAutoextInit; sqlite3_mutex_enter(mutex); @@ -109503,7 +138811,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ u32 i; int go = 1; int rc; - int (*xInit)(sqlite3*,char**,const sqlite3_api_routines*); + sqlite3_loadext_entry xInit; wsdAutoextInit; if( wsdAutoext.nExt==0 ){ @@ -109513,19 +138821,23 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ for(i=0; go; i++){ char *zErrmsg; #if SQLITE_THREADSAFE - sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); + sqlite3_mutex *mutex = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); +#endif +#ifdef SQLITE_OMIT_LOAD_EXTENSION + const sqlite3_api_routines *pThunk = 0; +#else + const sqlite3_api_routines *pThunk = &sqlite3Apis; #endif sqlite3_mutex_enter(mutex); if( i>=wsdAutoext.nExt ){ xInit = 0; go = 0; }else{ - xInit = (int(*)(sqlite3*,char**,const sqlite3_api_routines*)) - wsdAutoext.aExt[i]; + xInit = (sqlite3_loadext_entry)wsdAutoext.aExt[i]; } sqlite3_mutex_leave(mutex); zErrmsg = 0; - if( xInit && (rc = xInit(db, &zErrmsg, &sqlite3Apis))!=0 ){ + if( xInit && (rc = xInit(db, &zErrmsg, pThunk))!=0 ){ sqlite3ErrorWithMsg(db, rc, "automatic extension loading failed: %s", zErrmsg); go = 0; @@ -109564,7 +138876,7 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ** that includes the PragType_XXXX macro definitions and the aPragmaName[] ** object. This ensures that the aPragmaName[] table is arranged in ** lexicographical order to facility a binary search of the pragma name. -** Do not edit pragma.h directly. Edit and rerun the script in at +** Do not edit pragma.h directly. Edit and rerun the script in at ** ../tool/mkpragmatab.tcl. */ /************** Include pragma.h in the middle of pragma.c *******************/ /************** Begin file pragma.h ******************************************/ @@ -109573,472 +138885,696 @@ SQLITE_PRIVATE void sqlite3AutoLoadExtensions(sqlite3 *db){ ** ../tool/mkpragmatab.tcl. To update the set of pragmas, edit ** that script and rerun it. */ -#define PragTyp_HEADER_VALUE 0 -#define PragTyp_AUTO_VACUUM 1 -#define PragTyp_FLAG 2 -#define PragTyp_BUSY_TIMEOUT 3 -#define PragTyp_CACHE_SIZE 4 -#define PragTyp_CACHE_SPILL 5 -#define PragTyp_CASE_SENSITIVE_LIKE 6 -#define PragTyp_COLLATION_LIST 7 -#define PragTyp_COMPILE_OPTIONS 8 -#define PragTyp_DATA_STORE_DIRECTORY 9 -#define PragTyp_DATABASE_LIST 10 -#define PragTyp_DEFAULT_CACHE_SIZE 11 -#define PragTyp_ENCODING 12 -#define PragTyp_FOREIGN_KEY_CHECK 13 -#define PragTyp_FOREIGN_KEY_LIST 14 -#define PragTyp_INCREMENTAL_VACUUM 15 -#define PragTyp_INDEX_INFO 16 -#define PragTyp_INDEX_LIST 17 -#define PragTyp_INTEGRITY_CHECK 18 -#define PragTyp_JOURNAL_MODE 19 -#define PragTyp_JOURNAL_SIZE_LIMIT 20 -#define PragTyp_LOCK_PROXY_FILE 21 -#define PragTyp_LOCKING_MODE 22 -#define PragTyp_PAGE_COUNT 23 -#define PragTyp_MMAP_SIZE 24 -#define PragTyp_PAGE_SIZE 25 -#define PragTyp_SECURE_DELETE 26 -#define PragTyp_SHRINK_MEMORY 27 -#define PragTyp_SOFT_HEAP_LIMIT 28 -#define PragTyp_STATS 29 -#define PragTyp_SYNCHRONOUS 30 -#define PragTyp_TABLE_INFO 31 -#define PragTyp_TEMP_STORE 32 -#define PragTyp_TEMP_STORE_DIRECTORY 33 -#define PragTyp_THREADS 34 -#define PragTyp_WAL_AUTOCHECKPOINT 35 -#define PragTyp_WAL_CHECKPOINT 36 -#define PragTyp_ACTIVATE_EXTENSIONS 37 -#define PragTyp_HEXKEY 38 -#define PragTyp_KEY 39 -#define PragTyp_REKEY 40 -#define PragTyp_LOCK_STATUS 41 -#define PragTyp_PARSER_TRACE 42 -#define PragFlag_NeedSchema 0x01 -#define PragFlag_ReadOnly 0x02 -static const struct sPragmaNames { - const char *const zName; /* Name of pragma */ - u8 ePragTyp; /* PragTyp_XXX value */ - u8 mPragFlag; /* Zero or more PragFlag_XXX values */ - u32 iArg; /* Extra argument */ -} aPragmaNames[] = { -#if defined(SQLITE_HAS_CODEC) || defined(SQLITE_ENABLE_CEROD) - { /* zName: */ "activate_extensions", - /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + +/* The various pragma types */ +#define PragTyp_ACTIVATE_EXTENSIONS 0 +#define PragTyp_ANALYSIS_LIMIT 1 +#define PragTyp_HEADER_VALUE 2 +#define PragTyp_AUTO_VACUUM 3 +#define PragTyp_FLAG 4 +#define PragTyp_BUSY_TIMEOUT 5 +#define PragTyp_CACHE_SIZE 6 +#define PragTyp_CACHE_SPILL 7 +#define PragTyp_CASE_SENSITIVE_LIKE 8 +#define PragTyp_COLLATION_LIST 9 +#define PragTyp_COMPILE_OPTIONS 10 +#define PragTyp_DATA_STORE_DIRECTORY 11 +#define PragTyp_DATABASE_LIST 12 +#define PragTyp_DEFAULT_CACHE_SIZE 13 +#define PragTyp_ENCODING 14 +#define PragTyp_FOREIGN_KEY_CHECK 15 +#define PragTyp_FOREIGN_KEY_LIST 16 +#define PragTyp_FUNCTION_LIST 17 +#define PragTyp_HARD_HEAP_LIMIT 18 +#define PragTyp_INCREMENTAL_VACUUM 19 +#define PragTyp_INDEX_INFO 20 +#define PragTyp_INDEX_LIST 21 +#define PragTyp_INTEGRITY_CHECK 22 +#define PragTyp_JOURNAL_MODE 23 +#define PragTyp_JOURNAL_SIZE_LIMIT 24 +#define PragTyp_LOCK_PROXY_FILE 25 +#define PragTyp_LOCKING_MODE 26 +#define PragTyp_PAGE_COUNT 27 +#define PragTyp_MMAP_SIZE 28 +#define PragTyp_MODULE_LIST 29 +#define PragTyp_OPTIMIZE 30 +#define PragTyp_PAGE_SIZE 31 +#define PragTyp_PRAGMA_LIST 32 +#define PragTyp_SECURE_DELETE 33 +#define PragTyp_SHRINK_MEMORY 34 +#define PragTyp_SOFT_HEAP_LIMIT 35 +#define PragTyp_SYNCHRONOUS 36 +#define PragTyp_TABLE_INFO 37 +#define PragTyp_TABLE_LIST 38 +#define PragTyp_TEMP_STORE 39 +#define PragTyp_TEMP_STORE_DIRECTORY 40 +#define PragTyp_THREADS 41 +#define PragTyp_WAL_AUTOCHECKPOINT 42 +#define PragTyp_WAL_CHECKPOINT 43 +#define PragTyp_LOCK_STATUS 44 +#define PragTyp_STATS 45 + +/* Property flags associated with various pragma. */ +#define PragFlg_NeedSchema 0x01 /* Force schema load before running */ +#define PragFlg_NoColumns 0x02 /* OP_ResultRow called with zero columns */ +#define PragFlg_NoColumns1 0x04 /* zero columns if RHS argument is present */ +#define PragFlg_ReadOnly 0x08 /* Read-only HEADER_VALUE */ +#define PragFlg_Result0 0x10 /* Acts as query when no argument */ +#define PragFlg_Result1 0x20 /* Acts as query when has one argument */ +#define PragFlg_SchemaOpt 0x40 /* Schema restricts name search if present */ +#define PragFlg_SchemaReq 0x80 /* Schema required - "main" is default */ + +/* Names of columns for pragmas that return multi-column result +** or that return single-column results where the name of the +** result column is different from the name of the pragma +*/ +static const char *const pragCName[] = { + /* 0 */ "id", /* Used by: foreign_key_list */ + /* 1 */ "seq", + /* 2 */ "table", + /* 3 */ "from", + /* 4 */ "to", + /* 5 */ "on_update", + /* 6 */ "on_delete", + /* 7 */ "match", + /* 8 */ "cid", /* Used by: table_xinfo */ + /* 9 */ "name", + /* 10 */ "type", + /* 11 */ "notnull", + /* 12 */ "dflt_value", + /* 13 */ "pk", + /* 14 */ "hidden", + /* table_info reuses 8 */ + /* 15 */ "schema", /* Used by: table_list */ + /* 16 */ "name", + /* 17 */ "type", + /* 18 */ "ncol", + /* 19 */ "wr", + /* 20 */ "strict", + /* 21 */ "seqno", /* Used by: index_xinfo */ + /* 22 */ "cid", + /* 23 */ "name", + /* 24 */ "desc", + /* 25 */ "coll", + /* 26 */ "key", + /* 27 */ "name", /* Used by: function_list */ + /* 28 */ "builtin", + /* 29 */ "type", + /* 30 */ "enc", + /* 31 */ "narg", + /* 32 */ "flags", + /* 33 */ "tbl", /* Used by: stats */ + /* 34 */ "idx", + /* 35 */ "wdth", + /* 36 */ "hght", + /* 37 */ "flgs", + /* 38 */ "seq", /* Used by: index_list */ + /* 39 */ "name", + /* 40 */ "unique", + /* 41 */ "origin", + /* 42 */ "partial", + /* 43 */ "table", /* Used by: foreign_key_check */ + /* 44 */ "rowid", + /* 45 */ "parent", + /* 46 */ "fkid", + /* index_info reuses 21 */ + /* 47 */ "seq", /* Used by: database_list */ + /* 48 */ "name", + /* 49 */ "file", + /* 50 */ "busy", /* Used by: wal_checkpoint */ + /* 51 */ "log", + /* 52 */ "checkpointed", + /* collation_list reuses 38 */ + /* 53 */ "database", /* Used by: lock_status */ + /* 54 */ "status", + /* 55 */ "cache_size", /* Used by: default_cache_size */ + /* module_list pragma_list reuses 9 */ + /* 56 */ "timeout", /* Used by: busy_timeout */ +}; + +/* Definitions of all built-in pragmas */ +typedef struct PragmaName { + const char *const zName; /* Name of pragma */ + u8 ePragTyp; /* PragTyp_XXX value */ + u8 mPragFlg; /* Zero or more PragFlg_XXX values */ + u8 iPragCName; /* Start of column names in pragCName[] */ + u8 nPragCName; /* Num of col names. 0 means use pragma name */ + u64 iArg; /* Extra argument */ +} PragmaName; +static const PragmaName aPragmaName[] = { +#if defined(SQLITE_ENABLE_CEROD) + {/* zName: */ "activate_extensions", + /* ePragTyp: */ PragTyp_ACTIVATE_EXTENSIONS, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif + {/* zName: */ "analysis_limit", + /* ePragTyp: */ PragTyp_ANALYSIS_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "application_id", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ BTREE_APPLICATION_ID }, + {/* zName: */ "application_id", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_APPLICATION_ID }, #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) - { /* zName: */ "auto_vacuum", - /* ePragTyp: */ PragTyp_AUTO_VACUUM, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "auto_vacuum", + /* ePragTyp: */ PragTyp_AUTO_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_AUTOMATIC_INDEX) - { /* zName: */ "automatic_index", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_AutoIndex }, + {/* zName: */ "automatic_index", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_AutoIndex }, #endif #endif - { /* zName: */ "busy_timeout", - /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "busy_timeout", + /* ePragTyp: */ PragTyp_BUSY_TIMEOUT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 56, 1, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "cache_size", - /* ePragTyp: */ PragTyp_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "cache_size", + /* ePragTyp: */ PragTyp_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "cache_spill", - /* ePragTyp: */ PragTyp_CACHE_SPILL, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "cache_spill", + /* ePragTyp: */ PragTyp_CACHE_SPILL, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif - { /* zName: */ "case_sensitive_like", - /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "cell_size_check", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CellSizeCk }, +#if !defined(SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA) + {/* zName: */ "case_sensitive_like", + /* ePragTyp: */ PragTyp_CASE_SENSITIVE_LIKE, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "cell_size_check", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CellSizeCk }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "checkpoint_fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CkptFullFSync }, + {/* zName: */ "checkpoint_fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CkptFullFSync }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "collation_list", - /* ePragTyp: */ PragTyp_COLLATION_LIST, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "collation_list", + /* ePragTyp: */ PragTyp_COLLATION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 38, 2, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_COMPILEOPTION_DIAGS) - { /* zName: */ "compile_options", - /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "compile_options", + /* ePragTyp: */ PragTyp_COMPILE_OPTIONS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "count_changes", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_CountRows }, + {/* zName: */ "count_changes", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_CountRows }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_OS_WIN - { /* zName: */ "data_store_directory", - /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "data_store_directory", + /* ePragTyp: */ PragTyp_DATA_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "data_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ PragFlag_ReadOnly, - /* iArg: */ BTREE_DATA_VERSION }, + {/* zName: */ "data_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_DATA_VERSION }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "database_list", - /* ePragTyp: */ PragTyp_DATABASE_LIST, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "database_list", + /* ePragTyp: */ PragTyp_DATABASE_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 47, 3, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) - { /* zName: */ "default_cache_size", - /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "default_cache_size", + /* ePragTyp: */ PragTyp_DEFAULT_CACHE_SIZE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 55, 1, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { /* zName: */ "defer_foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_DeferFKs }, + {/* zName: */ "defer_foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_DeferFKs }, #endif #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "empty_result_callbacks", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_NullCallback }, + {/* zName: */ "empty_result_callbacks", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_NullCallback }, #endif #if !defined(SQLITE_OMIT_UTF16) - { /* zName: */ "encoding", - /* ePragTyp: */ PragTyp_ENCODING, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "encoding", + /* ePragTyp: */ PragTyp_ENCODING, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { /* zName: */ "foreign_key_check", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "foreign_key_check", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 43, 4, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FOREIGN_KEY) - { /* zName: */ "foreign_key_list", - /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "foreign_key_list", + /* ePragTyp: */ PragTyp_FOREIGN_KEY_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 8, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_FOREIGN_KEY) && !defined(SQLITE_OMIT_TRIGGER) - { /* zName: */ "foreign_keys", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ForeignKeys }, + {/* zName: */ "foreign_keys", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ForeignKeys }, #endif #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "freelist_count", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ PragFlag_ReadOnly, - /* iArg: */ BTREE_FREE_PAGE_COUNT }, + {/* zName: */ "freelist_count", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_ReadOnly|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_FREE_PAGE_COUNT }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "full_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_FullColNames }, - { /* zName: */ "fullfsync", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_FullFSync }, + {/* zName: */ "full_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullColNames }, + {/* zName: */ "fullfsync", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_FullFSync }, #endif -#if defined(SQLITE_HAS_CODEC) - { /* zName: */ "hexkey", - /* ePragTyp: */ PragTyp_HEXKEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "hexrekey", - /* ePragTyp: */ PragTyp_HEXKEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "function_list", + /* ePragTyp: */ PragTyp_FUNCTION_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 27, 6, + /* iArg: */ 0 }, #endif +#endif + {/* zName: */ "hard_heap_limit", + /* ePragTyp: */ PragTyp_HARD_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if !defined(SQLITE_OMIT_CHECK) - { /* zName: */ "ignore_check_constraints", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_IgnoreChecks }, + {/* zName: */ "ignore_check_constraints", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_IgnoreChecks }, #endif #endif #if !defined(SQLITE_OMIT_AUTOVACUUM) - { /* zName: */ "incremental_vacuum", - /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "incremental_vacuum", + /* ePragTyp: */ PragTyp_INCREMENTAL_VACUUM, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "index_info", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "index_list", - /* ePragTyp: */ PragTyp_INDEX_LIST, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "index_xinfo", - /* ePragTyp: */ PragTyp_INDEX_INFO, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 1 }, + {/* zName: */ "index_info", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 21, 3, + /* iArg: */ 0 }, + {/* zName: */ "index_list", + /* ePragTyp: */ PragTyp_INDEX_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 38, 5, + /* iArg: */ 0 }, + {/* zName: */ "index_xinfo", + /* ePragTyp: */ PragTyp_INDEX_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 21, 6, + /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - { /* zName: */ "integrity_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "integrity_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "journal_mode", - /* ePragTyp: */ PragTyp_JOURNAL_MODE, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "journal_size_limit", - /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, -#endif -#if defined(SQLITE_HAS_CODEC) - { /* zName: */ "key", - /* ePragTyp: */ PragTyp_KEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "journal_mode", + /* ePragTyp: */ PragTyp_JOURNAL_MODE, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "journal_size_limit", + /* ePragTyp: */ PragTyp_JOURNAL_SIZE_LIMIT, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "legacy_file_format", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_LegacyFileFmt }, + {/* zName: */ "legacy_alter_table", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_LegacyAlter }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && SQLITE_ENABLE_LOCKING_STYLE - { /* zName: */ "lock_proxy_file", - /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "lock_proxy_file", + /* ePragTyp: */ PragTyp_LOCK_PROXY_FILE, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) - { /* zName: */ "lock_status", - /* ePragTyp: */ PragTyp_LOCK_STATUS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "lock_status", + /* ePragTyp: */ PragTyp_LOCK_STATUS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 53, 2, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "locking_mode", - /* ePragTyp: */ PragTyp_LOCKING_MODE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "max_page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "mmap_size", - /* ePragTyp: */ PragTyp_MMAP_SIZE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "page_count", - /* ePragTyp: */ PragTyp_PAGE_COUNT, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, - { /* zName: */ "page_size", - /* ePragTyp: */ PragTyp_PAGE_SIZE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "locking_mode", + /* ePragTyp: */ PragTyp_LOCKING_MODE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "max_page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "mmap_size", + /* ePragTyp: */ PragTyp_MMAP_SIZE, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif -#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_PARSER_TRACE) - { /* zName: */ "parser_trace", - /* ePragTyp: */ PragTyp_PARSER_TRACE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) +#if !defined(SQLITE_OMIT_VIRTUALTABLE) +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "module_list", + /* ePragTyp: */ PragTyp_MODULE_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 9, 1, + /* iArg: */ 0 }, +#endif +#endif +#endif + {/* zName: */ "optimize", + /* ePragTyp: */ PragTyp_OPTIMIZE, + /* ePragFlg: */ PragFlg_Result1|PragFlg_NeedSchema, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_PAGER_PRAGMAS) + {/* zName: */ "page_count", + /* ePragTyp: */ PragTyp_PAGE_COUNT, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "page_size", + /* ePragTyp: */ PragTyp_PAGE_SIZE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "query_only", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_QueryOnly }, +#if defined(SQLITE_DEBUG) + {/* zName: */ "parser_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ParserTrace }, +#endif +#endif +#if !defined(SQLITE_OMIT_INTROSPECTION_PRAGMAS) + {/* zName: */ "pragma_list", + /* ePragTyp: */ PragTyp_PRAGMA_LIST, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 9, 1, + /* iArg: */ 0 }, +#endif +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "query_only", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_QueryOnly }, #endif #if !defined(SQLITE_OMIT_INTEGRITY_CHECK) - { /* zName: */ "quick_check", - /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "quick_check", + /* ePragTyp: */ PragTyp_INTEGRITY_CHECK, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "read_uncommitted", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ReadUncommitted }, - { /* zName: */ "recursive_triggers", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_RecTriggers }, -#endif -#if defined(SQLITE_HAS_CODEC) - { /* zName: */ "rekey", - /* ePragTyp: */ PragTyp_REKEY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, -#endif -#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "reverse_unordered_selects", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ReverseOrder }, + {/* zName: */ "read_uncommitted", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReadUncommit }, + {/* zName: */ "recursive_triggers", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_RecTriggers }, + {/* zName: */ "reverse_unordered_selects", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ReverseOrder }, #endif #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "schema_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ BTREE_SCHEMA_VERSION }, + {/* zName: */ "schema_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_SCHEMA_VERSION }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "secure_delete", - /* ePragTyp: */ PragTyp_SECURE_DELETE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "secure_delete", + /* ePragTyp: */ PragTyp_SECURE_DELETE, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "short_column_names", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_ShortColNames }, + {/* zName: */ "short_column_names", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_ShortColNames }, #endif - { /* zName: */ "shrink_memory", - /* ePragTyp: */ PragTyp_SHRINK_MEMORY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "soft_heap_limit", - /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "shrink_memory", + /* ePragTyp: */ PragTyp_SHRINK_MEMORY, + /* ePragFlg: */ PragFlg_NoColumns, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "soft_heap_limit", + /* ePragTyp: */ PragTyp_SOFT_HEAP_LIMIT, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) - { /* zName: */ "sql_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_SqlTrace }, + {/* zName: */ "sql_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace }, #endif #endif -#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "stats", - /* ePragTyp: */ PragTyp_STATS, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) && defined(SQLITE_DEBUG) + {/* zName: */ "stats", + /* ePragTyp: */ PragTyp_STATS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq, + /* ColNames: */ 33, 5, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "synchronous", - /* ePragTyp: */ PragTyp_SYNCHRONOUS, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "synchronous", + /* ePragTyp: */ PragTyp_SYNCHRONOUS, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result0|PragFlg_SchemaReq|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_SCHEMA_PRAGMAS) - { /* zName: */ "table_info", - /* ePragTyp: */ PragTyp_TABLE_INFO, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "table_info", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 6, + /* iArg: */ 0 }, + {/* zName: */ "table_list", + /* ePragTyp: */ PragTyp_TABLE_LIST, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1, + /* ColNames: */ 15, 6, + /* iArg: */ 0 }, + {/* zName: */ "table_xinfo", + /* ePragTyp: */ PragTyp_TABLE_INFO, + /* ePragFlg: */ PragFlg_NeedSchema|PragFlg_Result1|PragFlg_SchemaOpt, + /* ColNames: */ 8, 7, + /* iArg: */ 1 }, #endif #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) - { /* zName: */ "temp_store", - /* ePragTyp: */ PragTyp_TEMP_STORE, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "temp_store_directory", - /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, + {/* zName: */ "temp_store", + /* ePragTyp: */ PragTyp_TEMP_STORE, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "temp_store_directory", + /* ePragTyp: */ PragTyp_TEMP_STORE_DIRECTORY, + /* ePragFlg: */ PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#endif + {/* zName: */ "threads", + /* ePragTyp: */ PragTyp_THREADS, + /* ePragFlg: */ PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, +#if !defined(SQLITE_OMIT_FLAG_PRAGMAS) + {/* zName: */ "trusted_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_TrustedSchema }, #endif - { /* zName: */ "threads", - /* ePragTyp: */ PragTyp_THREADS, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, #if !defined(SQLITE_OMIT_SCHEMA_VERSION_PRAGMAS) - { /* zName: */ "user_version", - /* ePragTyp: */ PragTyp_HEADER_VALUE, - /* ePragFlag: */ 0, - /* iArg: */ BTREE_USER_VERSION }, + {/* zName: */ "user_version", + /* ePragTyp: */ PragTyp_HEADER_VALUE, + /* ePragFlg: */ PragFlg_NoColumns1|PragFlg_Result0, + /* ColNames: */ 0, 0, + /* iArg: */ BTREE_USER_VERSION }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) #if defined(SQLITE_DEBUG) - { /* zName: */ "vdbe_addoptrace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeAddopTrace }, - { /* zName: */ "vdbe_debug", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, - { /* zName: */ "vdbe_eqp", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeEQP }, - { /* zName: */ "vdbe_listing", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeListing }, - { /* zName: */ "vdbe_trace", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_VdbeTrace }, + {/* zName: */ "vdbe_addoptrace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeAddopTrace }, + {/* zName: */ "vdbe_debug", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_SqlTrace|SQLITE_VdbeListing|SQLITE_VdbeTrace }, + {/* zName: */ "vdbe_eqp", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeEQP }, + {/* zName: */ "vdbe_listing", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeListing }, + {/* zName: */ "vdbe_trace", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_VdbeTrace }, #endif #endif #if !defined(SQLITE_OMIT_WAL) - { /* zName: */ "wal_autocheckpoint", - /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, - /* ePragFlag: */ 0, - /* iArg: */ 0 }, - { /* zName: */ "wal_checkpoint", - /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, - /* ePragFlag: */ PragFlag_NeedSchema, - /* iArg: */ 0 }, + {/* zName: */ "wal_autocheckpoint", + /* ePragTyp: */ PragTyp_WAL_AUTOCHECKPOINT, + /* ePragFlg: */ 0, + /* ColNames: */ 0, 0, + /* iArg: */ 0 }, + {/* zName: */ "wal_checkpoint", + /* ePragTyp: */ PragTyp_WAL_CHECKPOINT, + /* ePragFlg: */ PragFlg_NeedSchema, + /* ColNames: */ 50, 3, + /* iArg: */ 0 }, #endif #if !defined(SQLITE_OMIT_FLAG_PRAGMAS) - { /* zName: */ "writable_schema", - /* ePragTyp: */ PragTyp_FLAG, - /* ePragFlag: */ 0, - /* iArg: */ SQLITE_WriteSchema|SQLITE_RecoveryMode }, + {/* zName: */ "writable_schema", + /* ePragTyp: */ PragTyp_FLAG, + /* ePragFlg: */ PragFlg_Result0|PragFlg_NoColumns1, + /* ColNames: */ 0, 0, + /* iArg: */ SQLITE_WriteSchema|SQLITE_NoSchemaError }, #endif }; -/* Number of pragmas: 60 on by default, 73 total. */ +/* Number of pragmas: 68 on by default, 78 total. */ /************** End of pragma.h **********************************************/ /************** Continuing where we left off in pragma.c *********************/ +/* +** When the 0x10 bit of PRAGMA optimize is set, any ANALYZE commands +** will be run with an analysis_limit set to the lessor of the value of +** the following macro or to the actual analysis_limit if it is non-zero, +** in order to prevent PRAGMA optimize from running for too long. +** +** The value of 2000 is chosen emperically so that the worst-case run-time +** for PRAGMA optimize does not exceed 100 milliseconds against a variety +** of test databases on a RaspberryPI-4 compiled using -Os and without +** -DSQLITE_DEBUG. Of course, your mileage may vary. For the purpose of +** this paragraph, "worst-case" means that ANALYZE ends up being +** run on every table in the database. The worst case typically only +** happens if PRAGMA optimize is run on a database file for which ANALYZE +** has not been previously run and the 0x10000 flag is included so that +** all tables are analyzed. The usual case for PRAGMA optimize is that +** no ANALYZE commands will be run at all, or if any ANALYZE happens it +** will be against a single table, so that expected timing for PRAGMA +** optimize on a PI-4 is more like 1 millisecond or less with the 0x10000 +** flag or less than 100 microseconds without the 0x10000 flag. +** +** An analysis limit of 2000 is almost always sufficient for the query +** planner to fully characterize an index. The additional accuracy from +** a larger analysis is not usually helpful. +*/ +#ifndef SQLITE_DEFAULT_OPTIMIZE_LIMIT +# define SQLITE_DEFAULT_OPTIMIZE_LIMIT 2000 +#endif + /* ** Interpret the given string as a safety level. Return 0 for OFF, -** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or +** 1 for ON or NORMAL, 2 for FULL, and 3 for EXTRA. Return 1 for an empty or ** unrecognized string argument. The FULL and EXTRA option is disallowed ** if the omitFull parameter it 1. ** @@ -110097,7 +139633,7 @@ static int getLockingMode(const char *z){ /* ** Interpret the given string as an auto-vacuum mode value. ** -** The following strings, "none", "full" and "incremental" are +** The following strings, "none", "full" and "incremental" are ** acceptable, as are their numeric equivalents: 0, 1 and 2 respectively. */ static int getAutoVacuum(const char *z){ @@ -110137,7 +139673,9 @@ static int getTempStore(const char *z){ static int invalidateTempStorage(Parse *pParse){ sqlite3 *db = pParse->db; if( db->aDb[1].pBt!=0 ){ - if( !db->autoCommit || sqlite3BtreeIsInReadTrans(db->aDb[1].pBt) ){ + if( !db->autoCommit + || sqlite3BtreeTxnState(db->aDb[1].pBt)!=SQLITE_TXN_NONE + ){ sqlite3ErrorMsg(pParse, "temporary storage cannot be changed " "from within a transaction"); return SQLITE_ERROR; @@ -110169,29 +139707,29 @@ static int changeTempStorage(Parse *pParse, const char *zStorageType){ #endif /* SQLITE_PAGER_PRAGMAS */ /* -** Set the names of the first N columns to the values in azCol[] +** Set result column names for a pragma. */ -static void setAllColumnNames( - Vdbe *v, /* The query under construction */ - int N, /* Number of columns */ - const char **azCol /* Names of columns */ +static void setPragmaResultColumnNames( + Vdbe *v, /* The query under construction */ + const PragmaName *pPragma /* The pragma */ ){ - int i; - sqlite3VdbeSetNumCols(v, N); - for(i=0; inPragCName; + sqlite3VdbeSetNumCols(v, n==0 ? 1 : n); + if( n==0 ){ + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, pPragma->zName, SQLITE_STATIC); + }else{ + int i, j; + for(i=0, j=pPragma->iPragCName; iupr ? 0 : &aPragmaName[mid]; +} + +/* +** Create zero or more entries in the output for the SQL functions +** defined by FuncDef p. +*/ +static void pragmaFunclistLine( + Vdbe *v, /* The prepared statement being created */ + FuncDef *p, /* A particular function definition */ + int isBuiltin, /* True if this is a built-in function */ + int showInternFuncs /* True if showing internal functions */ +){ + u32 mask = + SQLITE_DETERMINISTIC | + SQLITE_DIRECTONLY | + SQLITE_SUBTYPE | + SQLITE_INNOCUOUS | + SQLITE_FUNC_INTERNAL + ; + if( showInternFuncs ) mask = 0xffffffff; + for(; p; p=p->pNext){ + const char *zType; + static const char *azEnc[] = { 0, "utf8", "utf16le", "utf16be" }; + + assert( SQLITE_FUNC_ENCMASK==0x3 ); + assert( strcmp(azEnc[SQLITE_UTF8],"utf8")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16LE],"utf16le")==0 ); + assert( strcmp(azEnc[SQLITE_UTF16BE],"utf16be")==0 ); + + if( p->xSFunc==0 ) continue; + if( (p->funcFlags & SQLITE_FUNC_INTERNAL)!=0 + && showInternFuncs==0 + ){ + continue; + } + if( p->xValue!=0 ){ + zType = "w"; + }else if( p->xFinalize!=0 ){ + zType = "a"; + }else{ + zType = "s"; + } + sqlite3VdbeMultiLoad(v, 1, "sissii", + p->zName, isBuiltin, + zType, azEnc[p->funcFlags&SQLITE_FUNC_ENCMASK], + p->nArg, + (p->funcFlags & mask) ^ SQLITE_INNOCUOUS + ); + } +} + + +/* +** Helper subroutine for PRAGMA integrity_check: +** +** Generate code to output a single-column result row with a value of the +** string held in register 3. Decrement the result count in register 1 +** and halt if the maximum number of result rows have been issued. +*/ +static int integrityCheckResultRow(Vdbe *v){ + int addr; + sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); + addr = sqlite3VdbeAddOp3(v, OP_IfPos, 1, sqlite3VdbeCurrentAddr(v)+2, 1); + VdbeCoverage(v); + sqlite3VdbeAddOp0(v, OP_Halt); + return addr; +} + +/* +** Process a pragma statement. ** ** Pragmas are of this form: ** @@ -110299,7 +139921,7 @@ SQLITE_PRIVATE const char *sqlite3JournalModename(int eMode){ ** id and pId2 is any empty string. */ SQLITE_PRIVATE void sqlite3Pragma( - Parse *pParse, + Parse *pParse, Token *pId1, /* First part of [schema.]id field */ Token *pId2, /* Second part of [schema.]id field, or NULL */ Token *pValue, /* Token for , or NULL */ @@ -110311,12 +139933,11 @@ SQLITE_PRIVATE void sqlite3Pragma( Token *pId; /* Pointer to token */ char *aFcntl[4]; /* Argument to SQLITE_FCNTL_PRAGMA */ int iDb; /* Database index for */ - int lwr, upr, mid = 0; /* Binary search bounds */ int rc; /* return value form SQLITE_FCNTL_PRAGMA */ sqlite3 *db = pParse->db; /* The database connection */ Db *pDb; /* The specific database being pragmaed */ Vdbe *v = sqlite3GetVdbe(pParse); /* Prepared statement */ - const struct sPragmaNames *pPragma; + const PragmaName *pPragma; /* The pragma */ if( v==0 ) return; sqlite3VdbeRunOnlyOnce(v); @@ -110328,8 +139949,8 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iDb<0 ) return; pDb = &db->aDb[iDb]; - /* If the temp database has been explicitly named as part of the - ** pragma, make sure it is open. + /* If the temp database has been explicitly named as part of the + ** pragma, make sure it is open. */ if( iDb==1 && sqlite3OpenTempDatabase(pParse) ){ return; @@ -110344,7 +139965,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } assert( pId2 ); - zDb = pId2->n>0 ? pDb->zName : 0; + zDb = pId2->n>0 ? pDb->zDbSName : 0; if( sqlite3AuthCheck(pParse, SQLITE_PRAGMA, zLeft, zRight, zDb) ){ goto pragma_out; } @@ -110371,7 +139992,9 @@ SQLITE_PRIVATE void sqlite3Pragma( db->busyHandler.nBusy = 0; rc = sqlite3_file_control(db, zDb, SQLITE_FCNTL_PRAGMA, (void*)aFcntl); if( rc==SQLITE_OK ){ - returnSingleText(v, "result", aFcntl[0]); + sqlite3VdbeSetNumCols(v, 1); + sqlite3VdbeSetColName(v, 0, COLNAME_NAME, aFcntl[0], SQLITE_TRANSIENT); + returnSingleText(v, aFcntl[0]); sqlite3_free(aFcntl[0]); goto pragma_out; } @@ -110386,29 +140009,28 @@ SQLITE_PRIVATE void sqlite3Pragma( } /* Locate the pragma in the lookup table */ - lwr = 0; - upr = ArraySize(aPragmaNames)-1; - while( lwr<=upr ){ - mid = (lwr+upr)/2; - rc = sqlite3_stricmp(zLeft, aPragmaNames[mid].zName); - if( rc==0 ) break; - if( rc<0 ){ - upr = mid - 1; - }else{ - lwr = mid + 1; - } + pPragma = pragmaLocate(zLeft); + if( pPragma==0 ){ + /* IMP: R-43042-22504 No error messages are generated if an + ** unknown pragma is issued. */ + goto pragma_out; } - if( lwr>upr ) goto pragma_out; - pPragma = &aPragmaNames[mid]; /* Make sure the database schema is loaded if the pragma requires that */ - if( (pPragma->mPragFlag & PragFlag_NeedSchema)!=0 ){ + if( (pPragma->mPragFlg & PragFlg_NeedSchema)!=0 ){ if( sqlite3ReadSchema(pParse) ) goto pragma_out; } + /* Register the result column names for pragmas that return results */ + if( (pPragma->mPragFlg & PragFlg_NoColumns)==0 + && ((pPragma->mPragFlg & PragFlg_NoColumns1)==0 || zRight==0) + ){ + setPragmaResultColumnNames(v, pPragma); + } + /* Jump to the appropriate pragma handler */ switch( pPragma->ePragTyp ){ - + #if !defined(SQLITE_OMIT_PAGER_PRAGMAS) && !defined(SQLITE_OMIT_DEPRECATED) /* ** PRAGMA [schema.]default_cache_size @@ -110442,7 +140064,6 @@ SQLITE_PRIVATE void sqlite3Pragma( VdbeOp *aOp; sqlite3VdbeUsesBtree(v, iDb); if( !zRight ){ - setOneColumnName(v, "cache_size"); pParse->nMem += 2; sqlite3VdbeVerifyNoMallocRequired(v, ArraySize(getCacheSize)); aOp = sqlite3VdbeAddOpList(v, ArraySize(getCacheSize), getCacheSize, iLn); @@ -110477,13 +140098,13 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( pBt!=0 ); if( !zRight ){ int size = ALWAYS(pBt) ? sqlite3BtreeGetPageSize(pBt) : 0; - returnSingleInt(v, "page_size", size); + returnSingleInt(v, size); }else{ /* Malloc may fail when setting the page-size, as there is an internal ** buffer that the pager module resizes using sqlite3_realloc(). */ db->nextPagesize = sqlite3Atoi(zRight); - if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,-1,0) ){ + if( SQLITE_NOMEM==sqlite3BtreeSetPageSize(pBt, db->nextPagesize,0,0) ){ sqlite3OomFault(db); } } @@ -110492,18 +140113,22 @@ SQLITE_PRIVATE void sqlite3Pragma( /* ** PRAGMA [schema.]secure_delete - ** PRAGMA [schema.]secure_delete=ON/OFF + ** PRAGMA [schema.]secure_delete=ON/OFF/FAST ** ** The first form reports the current setting for the ** secure_delete flag. The second form changes the secure_delete - ** flag setting and reports thenew value. + ** flag setting and reports the new value. */ case PragTyp_SECURE_DELETE: { Btree *pBt = pDb->pBt; int b = -1; assert( pBt!=0 ); if( zRight ){ - b = sqlite3GetBoolean(zRight, 0); + if( sqlite3_stricmp(zRight, "fast")==0 ){ + b = 2; + }else{ + b = sqlite3GetBoolean(zRight, 0); + } } if( pId2->n==0 && b>=0 ){ int ii; @@ -110512,7 +140137,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } } b = sqlite3BtreeSecureDelete(pBt, b); - returnSingleInt(v, "secure_delete", b); + returnSingleInt(v, b); break; } @@ -110521,7 +140146,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** PRAGMA [schema.]max_page_count=N ** ** The first form reports the current setting for the - ** maximum number of pages in the database file. The + ** maximum number of pages in the database file. The ** second form attempts to change this setting. Both ** forms return the current setting. ** @@ -110535,17 +140160,21 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_PAGE_COUNT: { int iReg; + i64 x = 0; sqlite3CodeVerifySchema(pParse, iDb); iReg = ++pParse->nMem; if( sqlite3Tolower(zLeft[0])=='p' ){ sqlite3VdbeAddOp2(v, OP_Pagecount, iDb, iReg); }else{ - sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, - sqlite3AbsInt32(sqlite3Atoi(zRight))); + if( zRight && sqlite3DecOrHexToI64(zRight,&x)==0 ){ + if( x<0 ) x = 0; + else if( x>0xfffffffe ) x = 0xfffffffe; + }else{ + x = 0; + } + sqlite3VdbeAddOp3(v, OP_MaxPgcnt, iDb, iReg, (int)x); } sqlite3VdbeAddOp2(v, OP_ResultRow, iReg, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); break; } @@ -110591,7 +140220,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( eMode==PAGER_LOCKINGMODE_EXCLUSIVE ){ zRet = "exclusive"; } - returnSingleText(v, "locking_mode", zRet); + returnSingleText(v, zRet); break; } @@ -110604,7 +140233,6 @@ SQLITE_PRIVATE void sqlite3Pragma( int eMode; /* One of the PAGER_JOURNALMODE_XXX symbols */ int ii; /* Loop counter */ - setOneColumnName(v, "journal_mode"); if( zRight==0 ){ /* If there is no "=MODE" part of the pragma, do a query for the ** current mode */ @@ -110620,6 +140248,11 @@ SQLITE_PRIVATE void sqlite3Pragma( ** then do a query */ eMode = PAGER_JOURNALMODE_QUERY; } + if( eMode==PAGER_JOURNALMODE_OFF && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow journal-mode "OFF" in defensive since the database + ** can become corrupted using ordinary SQL when the journal is off */ + eMode = PAGER_JOURNALMODE_QUERY; + } } if( eMode==PAGER_JOURNALMODE_QUERY && pId2->n==0 ){ /* Convert "PRAGMA journal_mode" into "PRAGMA main.journal_mode" */ @@ -110650,7 +140283,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( iLimit<-1 ) iLimit = -1; } iLimit = sqlite3PagerJournalSizeLimit(pPager, iLimit); - returnSingleInt(v, "journal_size_limit", iLimit); + returnSingleInt(v, iLimit); break; } @@ -110668,7 +140301,7 @@ SQLITE_PRIVATE void sqlite3Pragma( Btree *pBt = pDb->pBt; assert( pBt!=0 ); if( !zRight ){ - returnSingleInt(v, "auto_vacuum", sqlite3BtreeGetAutoVacuum(pBt)); + returnSingleInt(v, sqlite3BtreeGetAutoVacuum(pBt)); }else{ int eAuto = getAutoVacuum(zRight); assert( eAuto>=0 && eAuto<=2 ); @@ -110680,7 +140313,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ rc = sqlite3BtreeSetAutoVacuum(pBt, eAuto); if( rc==SQLITE_OK && (eAuto==1 || eAuto==2) ){ - /* When setting the auto_vacuum mode to either "full" or + /* When setting the auto_vacuum mode to either "full" or ** "incremental", write the value of meta[6] in the database ** file. Before writing to meta[6], check that meta[3] indicates ** that this really is an auto-vacuum capable database. @@ -110717,7 +140350,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ #ifndef SQLITE_OMIT_AUTOVACUUM case PragTyp_INCREMENTAL_VACUUM: { - int iLimit, addr; + int iLimit = 0, addr; if( zRight==0 || !sqlite3GetInt32(zRight, &iLimit) || iLimit<=0 ){ iLimit = 0x7fffffff; } @@ -110747,7 +140380,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SIZE: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(v, "cache_size", pDb->pSchema->cache_size); + returnSingleInt(v, pDb->pSchema->cache_size); }else{ int size = sqlite3Atoi(zRight); pDb->pSchema->cache_size = size; @@ -110763,7 +140396,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** ** The first form reports the current local setting for the ** page cache spill size. The second form turns cache spill on - ** or off. When turnning cache spill on, the size is set to the + ** or off. When turning cache spill on, the size is set to the ** current cache_size. The third form sets a spill size that ** may be different form the cache size. ** If N is positive then that is the @@ -110781,8 +140414,8 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_CACHE_SPILL: { assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( !zRight ){ - returnSingleInt(v, "cache_spill", - (db->flags & SQLITE_CacheSpill)==0 ? 0 : + returnSingleInt(v, + (db->flags & SQLITE_CacheSpill)==0 ? 0 : sqlite3BtreeSetSpillSize(pDb->pBt,0)); }else{ int size = 1; @@ -110792,7 +140425,7 @@ SQLITE_PRIVATE void sqlite3Pragma( if( sqlite3GetBoolean(zRight, size!=0) ){ db->flags |= SQLITE_CacheSpill; }else{ - db->flags &= ~SQLITE_CacheSpill; + db->flags &= ~(u64)SQLITE_CacheSpill; } setAllPagerFlags(db); } @@ -110835,7 +140468,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = SQLITE_OK; #endif if( rc==SQLITE_OK ){ - returnSingleInt(v, "mmap_size", sz); + returnSingleInt(v, sz); }else if( rc!=SQLITE_NOTFOUND ){ pParse->nErr++; pParse->rc = rc; @@ -110856,7 +140489,7 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_TEMP_STORE: { if( !zRight ){ - returnSingleInt(v, "temp_store", db->temp_store); + returnSingleInt(v, db->temp_store); }else{ changeTempStorage(pParse, zRight); } @@ -110874,8 +140507,9 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_TEMP_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ - returnSingleText(v, "temp_store_directory", sqlite3_temp_directory); + returnSingleText(v, sqlite3_temp_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -110883,6 +140517,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -110900,6 +140535,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } @@ -110918,8 +140554,9 @@ SQLITE_PRIVATE void sqlite3Pragma( ** */ case PragTyp_DATA_STORE_DIRECTORY: { + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); if( !zRight ){ - returnSingleText(v, "data_store_directory", sqlite3_data_directory); + returnSingleText(v, sqlite3_data_directory); }else{ #ifndef SQLITE_OMIT_WSD if( zRight[0] ){ @@ -110927,6 +140564,7 @@ SQLITE_PRIVATE void sqlite3Pragma( rc = sqlite3OsAccess(db->pVfs, zRight, SQLITE_ACCESS_READWRITE, &res); if( rc!=SQLITE_OK || res==0 ){ sqlite3ErrorMsg(pParse, "not a writable directory"); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); goto pragma_out; } } @@ -110938,6 +140576,7 @@ SQLITE_PRIVATE void sqlite3Pragma( } #endif /* SQLITE_OMIT_WSD */ } + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_TEMPDIR)); break; } #endif @@ -110956,18 +140595,18 @@ SQLITE_PRIVATE void sqlite3Pragma( Pager *pPager = sqlite3BtreePager(pDb->pBt); char *proxy_file_path = NULL; sqlite3_file *pFile = sqlite3PagerFile(pPager); - sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, + sqlite3OsFileControlHint(pFile, SQLITE_GET_LOCKPROXYFILE, &proxy_file_path); - returnSingleText(v, "lock_proxy_file", proxy_file_path); + returnSingleText(v, proxy_file_path); }else{ Pager *pPager = sqlite3BtreePager(pDb->pBt); sqlite3_file *pFile = sqlite3PagerFile(pPager); int res; if( zRight[0] ){ - res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, zRight); } else { - res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, + res=sqlite3OsFileControl(pFile, SQLITE_SET_LOCKPROXYFILE, NULL); } if( res!=SQLITE_OK ){ @@ -110977,8 +140616,8 @@ SQLITE_PRIVATE void sqlite3Pragma( } break; } -#endif /* SQLITE_ENABLE_LOCKING_STYLE */ - +#endif /* SQLITE_ENABLE_LOCKING_STYLE */ + /* ** PRAGMA [schema.]synchronous ** PRAGMA [schema.]synchronous=OFF|ON|NORMAL|FULL|EXTRA @@ -110990,12 +140629,12 @@ SQLITE_PRIVATE void sqlite3Pragma( */ case PragTyp_SYNCHRONOUS: { if( !zRight ){ - returnSingleInt(v, "synchronous", pDb->safety_level-1); + returnSingleInt(v, pDb->safety_level-1); }else{ if( !db->autoCommit ){ - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "Safety level may not be changed inside a transaction"); - }else{ + }else if( iDb!=1 ){ int iLevel = (getSafetyLevel(zRight,0,1)+1) & PAGER_SYNCHRONOUS_MASK; if( iLevel==0 ) iLevel = 1; pDb->safety_level = iLevel; @@ -111010,9 +140649,10 @@ SQLITE_PRIVATE void sqlite3Pragma( #ifndef SQLITE_OMIT_FLAG_PRAGMAS case PragTyp_FLAG: { if( zRight==0 ){ - returnSingleInt(v, pPragma->zName, (db->flags & pPragma->iArg)!=0 ); + setPragmaResultColumnNames(v, pPragma); + returnSingleInt(v, (db->flags & pPragma->iArg)!=0 ); }else{ - int mask = pPragma->iArg; /* Mask of bits to set or clear. */ + u64 mask = pPragma->iArg; /* Mask of bits to set or clear. */ if( db->autoCommit==0 ){ /* Foreign key support may not be enabled or disabled while not ** in auto-commit mode. */ @@ -111026,17 +140666,29 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif if( sqlite3GetBoolean(zRight, 0) ){ - db->flags |= mask; + if( (mask & SQLITE_WriteSchema)==0 + || (db->flags & SQLITE_Defensive)==0 + ){ + db->flags |= mask; + } }else{ db->flags &= ~mask; if( mask==SQLITE_DeferFKs ) db->nDeferredImmCons = 0; + if( (mask & SQLITE_WriteSchema)!=0 + && sqlite3_stricmp(zRight, "reset")==0 + ){ + /* IMP: R-60817-01178 If the argument is "RESET" then schema + ** writing is disabled (as with "PRAGMA writable_schema=OFF") and, + ** in addition, the schema is reloaded. */ + sqlite3ResetAllSchemasOfConnection(db); + } } - /* Many of the flag-pragmas modify the code generated by the SQL + /* Many of the flag-pragmas modify the code generated by the SQL ** compiler (eg. count_changes). So add an opcode to expire all ** compiled SQL statements after modifying a pragma value. */ - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); + sqlite3VdbeAddOp0(v, OP_Expire); setAllPagerFlags(db); } break; @@ -111055,26 +140707,34 @@ SQLITE_PRIVATE void sqlite3Pragma( ** type: Column declaration type. ** notnull: True if 'NOT NULL' is part of column declaration ** dflt_value: The default value for the column, if any. + ** pk: Non-zero for PK fields. */ case PragTyp_TABLE_INFO: if( zRight ){ Table *pTab; - pTab = sqlite3FindTable(db, zRight, zDb); + sqlite3CodeVerifyNamedSchema(pParse, zDb); + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); if( pTab ){ - static const char *azCol[] = { - "cid", "name", "type", "notnull", "dflt_value", "pk" - }; int i, k; int nHidden = 0; Column *pCol; Index *pPk = sqlite3PrimaryKeyIndex(pTab); - pParse->nMem = 6; - sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 6, azCol); assert( 6==ArraySize(azCol) ); + pParse->nMem = 7; sqlite3ViewGetColumnNames(pParse, pTab); for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ - if( IsHiddenColumn(pCol) ){ - nHidden++; - continue; + int isHidden = 0; + const Expr *pColExpr; + if( pCol->colFlags & COLFLAG_NOINSERT ){ + if( pPragma->iArg==0 ){ + nHidden++; + continue; + } + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + isHidden = 2; /* GENERATED ALWAYS AS ... VIRTUAL */ + }else if( pCol->colFlags & COLFLAG_STORED ){ + isHidden = 3; /* GENERATED ALWAYS AS ... STORED */ + }else{ assert( pCol->colFlags & COLFLAG_HIDDEN ); + isHidden = 1; /* HIDDEN */ + } } if( (pCol->colFlags & COLFLAG_PRIMKEY)==0 ){ k = 0; @@ -111083,55 +140743,144 @@ SQLITE_PRIVATE void sqlite3Pragma( }else{ for(k=1; k<=pTab->nCol && pPk->aiColumn[k-1]!=i; k++){} } - assert( pCol->pDflt==0 || pCol->pDflt->op==TK_SPAN ); - sqlite3VdbeMultiLoad(v, 1, "issisi", + pColExpr = sqlite3ColumnExpr(pTab,pCol); + assert( pColExpr==0 || pColExpr->op==TK_SPAN || isHidden>=2 ); + assert( pColExpr==0 || !ExprHasProperty(pColExpr, EP_IntValue) + || isHidden>=2 ); + sqlite3VdbeMultiLoad(v, 1, pPragma->iArg ? "issisii" : "issisi", i-nHidden, - pCol->zName, + pCol->zCnName, sqlite3ColumnType(pCol,""), pCol->notNull ? 1 : 0, - pCol->pDflt ? pCol->pDflt->u.zToken : 0, - k); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 6); + (isHidden>=2 || pColExpr==0) ? 0 : pColExpr->u.zToken, + k, + isHidden); } } } break; - case PragTyp_STATS: { - static const char *azCol[] = { "table", "index", "width", "height" }; - Index *pIdx; - HashElem *i; - v = sqlite3GetVdbe(pParse); - pParse->nMem = 4; - sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); - for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ - Table *pTab = sqliteHashData(i); - sqlite3VdbeMultiLoad(v, 1, "ssii", - pTab->zName, - 0, - pTab->szTabRow, - pTab->nRowLogEst); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - sqlite3VdbeMultiLoad(v, 2, "sii", - pIdx->zName, - pIdx->szIdxRow, - pIdx->aiRowLogEst[0]); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 4); + /* + ** PRAGMA table_list + ** + ** Return a single row for each table, virtual table, or view in the + ** entire schema. + ** + ** schema: Name of attached database hold this table + ** name: Name of the table itself + ** type: "table", "view", "virtual", "shadow" + ** ncol: Number of columns + ** wr: True for a WITHOUT ROWID table + ** strict: True for a STRICT table + */ + case PragTyp_TABLE_LIST: { + int ii; + pParse->nMem = 6; + sqlite3CodeVerifyNamedSchema(pParse, zDb); + for(ii=0; iinDb; ii++){ + HashElem *k; + Hash *pHash; + int initNCol; + if( zDb && sqlite3_stricmp(zDb, db->aDb[ii].zDbSName)!=0 ) continue; + + /* Ensure that the Table.nCol field is initialized for all views + ** and virtual tables. Each time we initialize a Table.nCol value + ** for a table, that can potentially disrupt the hash table, so restart + ** the initialization scan. + */ + pHash = &db->aDb[ii].pSchema->tblHash; + initNCol = sqliteHashCount(pHash); + while( initNCol-- ){ + for(k=sqliteHashFirst(pHash); 1; k=sqliteHashNext(k) ){ + Table *pTab; + if( k==0 ){ initNCol = 0; break; } + pTab = sqliteHashData(k); + if( pTab->nCol==0 ){ + char *zSql = sqlite3MPrintf(db, "SELECT*FROM\"%w\"", pTab->zName); + if( zSql ){ + sqlite3_stmt *pDummy = 0; + (void)sqlite3_prepare(db, zSql, -1, &pDummy, 0); + (void)sqlite3_finalize(pDummy); + sqlite3DbFree(db, zSql); + } + if( db->mallocFailed ){ + sqlite3ErrorMsg(db->pParse, "out of memory"); + db->pParse->rc = SQLITE_NOMEM_BKPT; + } + pHash = &db->aDb[ii].pSchema->tblHash; + break; + } + } + } + + for(k=sqliteHashFirst(pHash); k; k=sqliteHashNext(k) ){ + Table *pTab = sqliteHashData(k); + const char *zType; + if( zRight && sqlite3_stricmp(zRight, pTab->zName)!=0 ) continue; + if( IsView(pTab) ){ + zType = "view"; + }else if( IsVirtual(pTab) ){ + zType = "virtual"; + }else if( pTab->tabFlags & TF_Shadow ){ + zType = "shadow"; + }else{ + zType = "table"; + } + sqlite3VdbeMultiLoad(v, 1, "sssiii", + db->aDb[ii].zDbSName, + sqlite3PreferredTableName(pTab->zName), + zType, + pTab->nCol, + (pTab->tabFlags & TF_WithoutRowid)!=0, + (pTab->tabFlags & TF_Strict)!=0 + ); } } } break; +#ifdef SQLITE_DEBUG + case PragTyp_STATS: { + Index *pIdx; + HashElem *i; + pParse->nMem = 5; + sqlite3CodeVerifySchema(pParse, iDb); + for(i=sqliteHashFirst(&pDb->pSchema->tblHash); i; i=sqliteHashNext(i)){ + Table *pTab = sqliteHashData(i); + sqlite3VdbeMultiLoad(v, 1, "ssiii", + sqlite3PreferredTableName(pTab->zName), + 0, + pTab->szTabRow, + pTab->nRowLogEst, + pTab->tabFlags); + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + sqlite3VdbeMultiLoad(v, 2, "siiiX", + pIdx->zName, + pIdx->szIdxRow, + pIdx->aiRowLogEst[0], + pIdx->hasStat1); + sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); + } + } + } + break; +#endif + case PragTyp_INDEX_INFO: if( zRight ){ Index *pIdx; Table *pTab; pIdx = sqlite3FindIndex(db, zRight, zDb); + if( pIdx==0 ){ + /* If there is no index named zRight, check to see if there is a + ** WITHOUT ROWID table named zRight, and if there is, show the + ** structure of the PRIMARY KEY index for that table. */ + pTab = sqlite3LocateTable(pParse, LOCATE_NOERR, zRight, zDb); + if( pTab && !HasRowid(pTab) ){ + pIdx = sqlite3PrimaryKeyIndex(pTab); + } + } if( pIdx ){ - static const char *azCol[] = { - "seqno", "cid", "name", "desc", "coll", "key" - }; + int iIdxDb = sqlite3SchemaToIndex(db, pIdx->pSchema); int i; int mx; if( pPragma->iArg ){ @@ -111144,15 +140893,14 @@ SQLITE_PRIVATE void sqlite3Pragma( pParse->nMem = 3; } pTab = pIdx->pTable; - sqlite3CodeVerifySchema(pParse, iDb); - assert( pParse->nMem<=ArraySize(azCol) ); - setAllColumnNames(v, pParse->nMem, azCol); + sqlite3CodeVerifySchema(pParse, iIdxDb); + assert( pParse->nMem<=pPragma->nPragCName ); for(i=0; iaiColumn[i]; - sqlite3VdbeMultiLoad(v, 1, "iis", i, cnum, - cnum<0 ? 0 : pTab->aCol[cnum].zName); + sqlite3VdbeMultiLoad(v, 1, "iisX", i, cnum, + cnum<0 ? 0 : pTab->aCol[cnum].zCnName); if( pPragma->iArg ){ - sqlite3VdbeMultiLoad(v, 4, "isi", + sqlite3VdbeMultiLoad(v, 4, "isiX", pIdx->aSortOrder[i], pIdx->azColl[i], inKeyCol); @@ -111169,13 +140917,9 @@ SQLITE_PRIVATE void sqlite3Pragma( int i; pTab = sqlite3FindTable(db, zRight, zDb); if( pTab ){ - static const char *azCol[] = { - "seq", "name", "unique", "origin", "partial" - }; - v = sqlite3GetVdbe(pParse); + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); pParse->nMem = 5; - sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 5, azCol); assert( 5==ArraySize(azCol) ); + sqlite3CodeVerifySchema(pParse, iTabDb); for(pIdx=pTab->pIndex, i=0; pIdx; pIdx=pIdx->pNext, i++){ const char *azOrigin[] = { "c", "u", "pk" }; sqlite3VdbeMultiLoad(v, 1, "isisi", @@ -111184,42 +140928,78 @@ SQLITE_PRIVATE void sqlite3Pragma( IsUniqueIndex(pIdx), azOrigin[pIdx->idxType], pIdx->pPartIdxWhere!=0); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 5); } } } break; case PragTyp_DATABASE_LIST: { - static const char *azCol[] = { "seq", "name", "file" }; int i; pParse->nMem = 3; - setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); for(i=0; inDb; i++){ if( db->aDb[i].pBt==0 ) continue; - assert( db->aDb[i].zName!=0 ); + assert( db->aDb[i].zDbSName!=0 ); sqlite3VdbeMultiLoad(v, 1, "iss", i, - db->aDb[i].zName, + db->aDb[i].zDbSName, sqlite3BtreeGetFilename(db->aDb[i].pBt)); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); } } break; case PragTyp_COLLATION_LIST: { - static const char *azCol[] = { "seq", "name" }; int i = 0; HashElem *p; pParse->nMem = 2; - setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); for(p=sqliteHashFirst(&db->aCollSeq); p; p=sqliteHashNext(p)){ CollSeq *pColl = (CollSeq *)sqliteHashData(p); sqlite3VdbeMultiLoad(v, 1, "is", i++, pColl->zName); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); } } break; + +#ifndef SQLITE_OMIT_INTROSPECTION_PRAGMAS + case PragTyp_FUNCTION_LIST: { + int i; + HashElem *j; + FuncDef *p; + int showInternFunc = (db->mDbFlags & DBFLAG_InternalFunc)!=0; + pParse->nMem = 6; + for(i=0; iu.pHash ){ + assert( p->funcFlags & SQLITE_FUNC_BUILTIN ); + pragmaFunclistLine(v, p, 1, showInternFunc); + } + } + for(j=sqliteHashFirst(&db->aFunc); j; j=sqliteHashNext(j)){ + p = (FuncDef*)sqliteHashData(j); + assert( (p->funcFlags & SQLITE_FUNC_BUILTIN)==0 ); + pragmaFunclistLine(v, p, 0, showInternFunc); + } + } + break; + +#ifndef SQLITE_OMIT_VIRTUALTABLE + case PragTyp_MODULE_LIST: { + HashElem *j; + pParse->nMem = 1; + for(j=sqliteHashFirst(&db->aModule); j; j=sqliteHashNext(j)){ + Module *pMod = (Module*)sqliteHashData(j); + sqlite3VdbeMultiLoad(v, 1, "s", pMod->zName); + } + } + break; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + + case PragTyp_PRAGMA_LIST: { + int i; + for(i=0; ipFKey; + if( pTab && IsOrdinaryTable(pTab) ){ + pFK = pTab->u.tab.pFKey; if( pFK ){ - static const char *azCol[] = { - "id", "seq", "table", "from", "to", "on_update", "on_delete", - "match" - }; - int i = 0; + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); + int i = 0; pParse->nMem = 8; - sqlite3CodeVerifySchema(pParse, iDb); - setAllColumnNames(v, 8, azCol); assert( 8==ArraySize(azCol) ); + sqlite3CodeVerifySchema(pParse, iTabDb); while(pFK){ int j; for(j=0; jnCol; j++){ @@ -111246,12 +141021,11 @@ SQLITE_PRIVATE void sqlite3Pragma( i, j, pFK->zTo, - pTab->aCol[pFK->aCol[j].iFrom].zName, + pTab->aCol[pFK->aCol[j].iFrom].zCnName, pFK->aCol[j].zCol, actionName(pFK->aAction[1]), /* ON UPDATE */ actionName(pFK->aAction[0]), /* ON DELETE */ "NONE"); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 8); } ++i; pFK = pFK->pNextFrom; @@ -111274,20 +141048,14 @@ SQLITE_PRIVATE void sqlite3Pragma( HashElem *k; /* Loop counter: Next table in schema */ int x; /* result variable */ int regResult; /* 3 registers to hold a result row */ - int regKey; /* Register to hold key for checking the FK */ int regRow; /* Registers to hold a row from pTab */ int addrTop; /* Top of a loop checking foreign keys */ int addrOk; /* Jump here if the key is OK */ int *aiCols; /* child to parent column mapping */ - static const char *azCol[] = { "table", "rowid", "parent", "fkid" }; regResult = pParse->nMem+1; pParse->nMem += 4; - regKey = ++pParse->nMem; regRow = ++pParse->nMem; - v = sqlite3GetVdbe(pParse); - setAllColumnNames(v, 4, azCol); assert( 4==ArraySize(azCol) ); - sqlite3CodeVerifySchema(pParse, iDb); k = sqliteHashFirst(&db->aDb[iDb].pSchema->tblHash); while( k ){ if( zRight ){ @@ -111297,12 +141065,16 @@ SQLITE_PRIVATE void sqlite3Pragma( pTab = (Table*)sqliteHashData(k); k = sqliteHashNext(k); } - if( pTab==0 || pTab->pFKey==0 ) continue; + if( pTab==0 || !IsOrdinaryTable(pTab) || pTab->u.tab.pFKey==0 ) continue; + iDb = sqlite3SchemaToIndex(db, pTab->pSchema); + zDb = db->aDb[iDb].zDbSName; + sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); - if( pTab->nCol+regRow>pParse->nMem ) pParse->nMem = pTab->nCol + regRow; + sqlite3TouchRegister(pParse, pTab->nCol+regRow); sqlite3OpenTable(pParse, 0, iDb, pTab, OP_OpenRead); sqlite3VdbeLoadString(v, regResult, pTab->zName); - for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + assert( IsOrdinaryTable(pTab) ); + for(i=1, pFK=pTab->u.tab.pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); if( pParent==0 ) continue; pIdx = 0; @@ -111324,45 +141096,49 @@ SQLITE_PRIVATE void sqlite3Pragma( if( pFK ) break; if( pParse->nTabnTab = i; addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, 0); VdbeCoverage(v); - for(i=1, pFK=pTab->pFKey; pFK; i++, pFK=pFK->pNextFrom){ + assert( IsOrdinaryTable(pTab) ); + for(i=1, pFK=pTab->u.tab.pFKey; pFK; i++, pFK=pFK->pNextFrom){ pParent = sqlite3FindTable(db, pFK->zTo, zDb); pIdx = 0; aiCols = 0; if( pParent ){ x = sqlite3FkLocateIndex(pParse, pParent, pFK, &pIdx, &aiCols); - assert( x==0 ); + assert( x==0 || db->mallocFailed ); } - addrOk = sqlite3VdbeMakeLabel(v); - if( pParent && pIdx==0 ){ - int iKey = pFK->aCol[0].iFrom; - assert( iKey>=0 && iKeynCol ); - if( iKey!=pTab->iPKey ){ - sqlite3VdbeAddOp3(v, OP_Column, 0, iKey, regRow); - sqlite3ColumnDefault(v, pTab, iKey, regRow); - sqlite3VdbeAddOp2(v, OP_IsNull, regRow, addrOk); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_MustBeInt, regRow, - sqlite3VdbeCurrentAddr(v)+3); VdbeCoverage(v); - }else{ - sqlite3VdbeAddOp2(v, OP_Rowid, 0, regRow); - } - sqlite3VdbeAddOp3(v, OP_NotExists, i, 0, regRow); VdbeCoverage(v); + addrOk = sqlite3VdbeMakeLabel(pParse); + + /* Generate code to read the child key values into registers + ** regRow..regRow+n. If any of the child key values are NULL, this + ** row cannot cause an FK violation. Jump directly to addrOk in + ** this case. */ + sqlite3TouchRegister(pParse, regRow + pFK->nCol); + for(j=0; jnCol; j++){ + int iCol = aiCols ? aiCols[j] : pFK->aCol[j].iFrom; + sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, iCol, regRow+j); + sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); + } + + /* Generate code to query the parent index for a matching parent + ** key. If a match is found, jump to addrOk. */ + if( pIdx ){ + sqlite3VdbeAddOp4(v, OP_Affinity, regRow, pFK->nCol, 0, + sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); + sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regRow, pFK->nCol); + VdbeCoverage(v); + }else if( pParent ){ + int jmp = sqlite3VdbeCurrentAddr(v)+2; + sqlite3VdbeAddOp3(v, OP_SeekRowid, i, jmp, regRow); VdbeCoverage(v); sqlite3VdbeGoto(v, addrOk); - sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); - }else{ - for(j=0; jnCol; j++){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, 0, - aiCols ? aiCols[j] : pFK->aCol[j].iFrom, regRow+j); - sqlite3VdbeAddOp2(v, OP_IsNull, regRow+j, addrOk); VdbeCoverage(v); - } - if( pParent ){ - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, pFK->nCol, regKey, - sqlite3IndexAffinityStr(db,pIdx), pFK->nCol); - sqlite3VdbeAddOp4Int(v, OP_Found, i, addrOk, regKey, 0); - VdbeCoverage(v); - } + assert( pFK->nCol==1 || db->mallocFailed ); } - sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); - sqlite3VdbeMultiLoad(v, regResult+2, "si", pFK->zTo, i-1); + + /* Generate code to report an FK violation to the caller. */ + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, 0, regResult+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Null, 0, regResult+1); + } + sqlite3VdbeMultiLoad(v, regResult+2, "siX", pFK->zTo, i-1); sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, 4); sqlite3VdbeResolveLabel(v, addrOk); sqlite3DbFree(db, aiCols); @@ -111375,19 +141151,7 @@ SQLITE_PRIVATE void sqlite3Pragma( #endif /* !defined(SQLITE_OMIT_TRIGGER) */ #endif /* !defined(SQLITE_OMIT_FOREIGN_KEY) */ -#ifndef NDEBUG - case PragTyp_PARSER_TRACE: { - if( zRight ){ - if( sqlite3GetBoolean(zRight, 0) ){ - sqlite3ParserTrace(stdout, "parser: "); - }else{ - sqlite3ParserTrace(0, 0); - } - } - } - break; -#endif - +#ifndef SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA /* Reinstall the LIKE and GLOB functions. The variant of LIKE ** used will be case sensitive or not depending on the RHS. */ @@ -111397,18 +141161,40 @@ SQLITE_PRIVATE void sqlite3Pragma( } } break; +#endif /* SQLITE_OMIT_CASE_SENSITIVE_LIKE_PRAGMA */ #ifndef SQLITE_INTEGRITY_CHECK_ERROR_MAX # define SQLITE_INTEGRITY_CHECK_ERROR_MAX 100 #endif #ifndef SQLITE_OMIT_INTEGRITY_CHECK - /* Pragma "quick_check" is reduced version of + /* PRAGMA integrity_check + ** PRAGMA integrity_check(N) + ** PRAGMA quick_check + ** PRAGMA quick_check(N) + ** + ** Verify the integrity of the database. + ** + ** The "quick_check" is reduced version of ** integrity_check designed to detect most database corruption - ** without most of the overhead of a full integrity-check. + ** without the overhead of cross-checking indexes. Quick_check + ** is linear time whereas integrity_check is O(NlogN). + ** + ** The maximum number of errors is 100 by default. A different default + ** can be specified using a numeric parameter N. + ** + ** Or, the parameter N can be the name of a table. In that case, only + ** the one table named is verified. The freelist is only verified if + ** the named table is "sqlite_schema" (or one of its aliases). + ** + ** All schemas are checked by default. To check just a single + ** schema, use the form: + ** + ** PRAGMA schema.integrity_check; */ case PragTyp_INTEGRITY_CHECK: { int i, j, addr, mxErr; + Table *pObjTab = 0; /* Check only this one table, if not NULL */ int isQuick = (sqlite3Tolower(zLeft[0])=='q'); @@ -111427,35 +141213,33 @@ SQLITE_PRIVATE void sqlite3Pragma( /* Initialize the VDBE program */ pParse->nMem = 6; - setOneColumnName(v, "integrity_check"); /* Set the maximum error count */ mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; if( zRight ){ - sqlite3GetInt32(zRight, &mxErr); - if( mxErr<=0 ){ - mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + if( sqlite3GetInt32(pValue->z, &mxErr) ){ + if( mxErr<=0 ){ + mxErr = SQLITE_INTEGRITY_CHECK_ERROR_MAX; + } + }else{ + pObjTab = sqlite3LocateTable(pParse, 0, zRight, + iDb>=0 ? db->aDb[iDb].zDbSName : 0); } } - sqlite3VdbeAddOp2(v, OP_Integer, mxErr, 1); /* reg[1] holds errors left */ + sqlite3VdbeAddOp2(v, OP_Integer, mxErr-1, 1); /* reg[1] holds errors left */ /* Do an integrity check on each database file */ for(i=0; inDb; i++){ - HashElem *x; - Hash *pTbls; - int *aRoot; - int cnt = 0; - int mxIdx = 0; - int nIdx; + HashElem *x; /* For looping over tables in the schema */ + Hash *pTbls; /* Set of all tables in the schema */ + int *aRoot; /* Array of root page numbers of all btrees */ + int cnt = 0; /* Number of entries in aRoot[] */ if( OMIT_TEMPDB && i==1 ) continue; if( iDb>=0 && i!=iDb ) continue; sqlite3CodeVerifySchema(pParse, i); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Halt if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); + pParse->okConstFactor = 0; /* tag-20230327-1 */ /* Do an integrity check of the B-Tree ** @@ -111465,58 +141249,104 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3SchemaMutexHeld(db, i, 0) ); pTbls = &db->aDb[i].pSchema->tblHash; for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ - Table *pTab = sqliteHashData(x); - Index *pIdx; + Table *pTab = sqliteHashData(x); /* Current table */ + Index *pIdx; /* An index on pTab */ + int nIdx; /* Number of indexes on pTab */ + if( pObjTab && pObjTab!=pTab ) continue; if( HasRowid(pTab) ) cnt++; for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ cnt++; } - if( nIdx>mxIdx ) mxIdx = nIdx; } + if( cnt==0 ) continue; + if( pObjTab ) cnt++; aRoot = sqlite3DbMallocRawNN(db, sizeof(int)*(cnt+1)); if( aRoot==0 ) break; - for(cnt=0, x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + cnt = 0; + if( pObjTab ) aRoot[++cnt] = 0; + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx; - if( HasRowid(pTab) ) aRoot[cnt++] = pTab->tnum; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ) aRoot[++cnt] = pTab->tnum; for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - aRoot[cnt++] = pIdx->tnum; + aRoot[++cnt] = pIdx->tnum; } } - aRoot[cnt] = 0; + aRoot[0] = cnt; /* Make sure sufficient number of registers have been allocated */ - pParse->nMem = MAX( pParse->nMem, 8+mxIdx ); + sqlite3TouchRegister(pParse, 8+cnt); + sqlite3VdbeAddOp3(v, OP_Null, 0, 8, 8+cnt); + sqlite3ClearTempRegCache(pParse); /* Do the b-tree integrity checks */ - sqlite3VdbeAddOp4(v, OP_IntegrityCk, 2, cnt, 1, (char*)aRoot,P4_INTARRAY); + sqlite3VdbeAddOp4(v, OP_IntegrityCk, 1, cnt, 8, (char*)aRoot,P4_INTARRAY); sqlite3VdbeChangeP5(v, (u8)i); addr = sqlite3VdbeAddOp1(v, OP_IsNull, 2); VdbeCoverage(v); sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, - sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zName), + sqlite3MPrintf(db, "*** in database %s ***\n", db->aDb[i].zDbSName), P4_DYNAMIC); - sqlite3VdbeAddOp3(v, OP_Move, 2, 4, 1); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 2); - sqlite3VdbeAddOp2(v, OP_ResultRow, 2, 1); + sqlite3VdbeAddOp3(v, OP_Concat, 2, 3, 3); + integrityCheckResultRow(v); sqlite3VdbeJumpHere(v, addr); + /* Check that the indexes all have the right number of rows */ + cnt = pObjTab ? 1 : 0; + sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + int iTab = 0; + Table *pTab = sqliteHashData(x); + Index *pIdx; + if( pObjTab && pObjTab!=pTab ) continue; + if( HasRowid(pTab) ){ + iTab = cnt++; + }else{ + iTab = cnt; + for(pIdx=pTab->pIndex; ALWAYS(pIdx); pIdx=pIdx->pNext){ + if( IsPrimaryKeyIndex(pIdx) ) break; + iTab++; + } + } + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->pPartIdxWhere==0 ){ + addr = sqlite3VdbeAddOp3(v, OP_Eq, 8+cnt, 0, 8+iTab); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 2, 3); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, addr); + } + cnt++; + } + } + /* Make sure all the indices are constructed correctly. */ - for(x=sqliteHashFirst(pTbls); x && !isQuick; x=sqliteHashNext(x)){ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ Table *pTab = sqliteHashData(x); Index *pIdx, *pPk; - Index *pPrior = 0; + Index *pPrior = 0; /* Previous index */ int loopTop; int iDataCur, iIdxCur; int r1 = -1; + int bStrict; /* True for a STRICT table */ + int r2; /* Previous key for WITHOUT ROWID tables */ + int mxCol; /* Maximum non-virtual column number */ - if( pTab->pIndex==0 ) continue; - pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); - addr = sqlite3VdbeAddOp1(v, OP_IfPos, 1); /* Stop if out of errors */ - VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeJumpHere(v, addr); - sqlite3ExprCacheClear(pParse); + if( pObjTab && pObjTab!=pTab ) continue; + if( !IsOrdinaryTable(pTab) ) continue; + if( isQuick || HasRowid(pTab) ){ + pPk = 0; + r2 = 0; + }else{ + pPk = sqlite3PrimaryKeyIndex(pTab); + r2 = sqlite3GetTempRange(pParse, pPk->nKeyCol); + sqlite3VdbeAddOp3(v, OP_Null, 1, r2, r2+pPk->nKeyCol-1); + } sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenRead, 0, 1, 0, &iDataCur, &iIdxCur); + /* reg[7] counts the number of entries in the table. + ** reg[8+i] counts the number of entries in the i-th index + */ sqlite3VdbeAddOp2(v, OP_Integer, 0, 7); for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ sqlite3VdbeAddOp2(v, OP_Integer, 0, 8+j); /* index entries counter */ @@ -111525,111 +141355,353 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( sqlite3NoTempsInRange(pParse,1,7+j) ); sqlite3VdbeAddOp2(v, OP_Rewind, iDataCur, 0); VdbeCoverage(v); loopTop = sqlite3VdbeAddOp2(v, OP_AddImm, 7, 1); - /* Verify that all NOT NULL columns really are NOT NULL */ + + /* Fetch the right-most column from the table. This will cause + ** the entire record header to be parsed and sanity checked. It + ** will also prepopulate the cursor column cache that is used + ** by the OP_IsType code, so it is a required step. + */ + assert( !IsVirtual(pTab) ); + if( HasRowid(pTab) ){ + mxCol = -1; + for(j=0; jnCol; j++){ + if( (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)==0 ) mxCol++; + } + if( mxCol==pTab->iPKey ) mxCol--; + }else{ + /* COLFLAG_VIRTUAL columns are not included in the WITHOUT ROWID + ** PK index column-count, so there is no need to account for them + ** in this case. */ + mxCol = sqlite3PrimaryKeyIndex(pTab)->nColumn-1; + } + if( mxCol>=0 ){ + sqlite3VdbeAddOp3(v, OP_Column, iDataCur, mxCol, 3); + sqlite3VdbeTypeofColumn(v, 3); + } + + if( !isQuick ){ + if( pPk ){ + /* Verify WITHOUT ROWID keys are in ascending order */ + int a1; + char *zErr; + a1 = sqlite3VdbeAddOp4Int(v, OP_IdxGT, iDataCur, 0,r2,pPk->nKeyCol); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_IsNull, r2); VdbeCoverage(v); + zErr = sqlite3MPrintf(db, + "row not in PRIMARY KEY order for %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + sqlite3VdbeJumpHere(v, a1+1); + for(j=0; jnKeyCol; j++){ + sqlite3ExprCodeLoadIndexColumn(pParse, pPk, iDataCur, j, r2+j); + } + } + } + /* Verify datatypes for all columns: + ** + ** (1) NOT NULL columns may not contain a NULL + ** (2) Datatype must be exact for non-ANY columns in STRICT tables + ** (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. + ** (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be losslessly converted to numeric. + */ + bStrict = (pTab->tabFlags & TF_Strict)!=0; for(j=0; jnCol; j++){ char *zErr; - int jmp2, jmp3; + Column *pCol = pTab->aCol + j; /* The column to be checked */ + int labelError; /* Jump here to report an error */ + int labelOk; /* Jump here if all looks ok */ + int p1, p3, p4; /* Operands to the OP_IsType opcode */ + int doTypeCheck; /* Check datatypes (besides NOT NULL) */ + if( j==pTab->iPKey ) continue; - if( pTab->aCol[j].notNull==0 ) continue; - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); - sqlite3VdbeChangeP5(v, OPFLAG_TYPEOFARG); - jmp2 = sqlite3VdbeAddOp1(v, OP_NotNull, 3); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, - pTab->aCol[j].zName); - sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp3 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); - sqlite3VdbeJumpHere(v, jmp2); - sqlite3VdbeJumpHere(v, jmp3); - } - /* Validate index entries for the current row */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - int jmp2, jmp3, jmp4, jmp5; - int ckUniq = sqlite3VdbeMakeLabel(v); - if( pPk==pIdx ) continue; - r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, - pPrior, r1); - pPrior = pIdx; - sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1); /* increment entry count */ - /* Verify that an index entry exists for the current table row */ - jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, - pIdx->nColumn); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeLoadString(v, 3, "row "); - sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); - sqlite3VdbeLoadString(v, 4, " missing from index "); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); - sqlite3VdbeAddOp2(v, OP_ResultRow, 3, 1); - jmp4 = sqlite3VdbeAddOp1(v, OP_IfPos, 1); VdbeCoverage(v); - sqlite3VdbeAddOp0(v, OP_Halt); - sqlite3VdbeJumpHere(v, jmp2); - /* For UNIQUE indexes, verify that only one entry exists with the - ** current key. The entry is unique if (1) any column is NULL - ** or (2) the next entry has a different key */ - if( IsUniqueIndex(pIdx) ){ - int uniqOk = sqlite3VdbeMakeLabel(v); - int jmp6; - int kk; - for(kk=0; kknKeyCol; kk++){ - int iCol = pIdx->aiColumn[kk]; - assert( iCol!=XN_ROWID && iColnCol ); - if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; - sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + if( bStrict ){ + doTypeCheck = pCol->eCType>COLTYPE_ANY; + }else{ + doTypeCheck = pCol->affinity>SQLITE_AFF_BLOB; + } + if( pCol->notNull==0 && !doTypeCheck ) continue; + + /* Compute the operands that will be needed for OP_IsType */ + p4 = SQLITE_NULL; + if( pCol->colFlags & COLFLAG_VIRTUAL ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + p1 = -1; + p3 = 3; + }else{ + if( pCol->iDflt ){ + sqlite3_value *pDfltValue = 0; + sqlite3ValueFromExpr(db, sqlite3ColumnExpr(pTab,pCol), ENC(db), + pCol->affinity, &pDfltValue); + if( pDfltValue ){ + p4 = sqlite3_value_type(pDfltValue); + sqlite3ValueFree(pDfltValue); + } + } + p1 = iDataCur; + if( !HasRowid(pTab) ){ + testcase( j!=sqlite3TableColumnToStorage(pTab, j) ); + p3 = sqlite3TableColumnToIndex(sqlite3PrimaryKeyIndex(pTab), j); + }else{ + p3 = sqlite3TableColumnToStorage(pTab,j); + testcase( p3!=j); + } + } + + labelError = sqlite3VdbeMakeLabel(pParse); + labelOk = sqlite3VdbeMakeLabel(pParse); + if( pCol->notNull ){ + /* (1) NOT NULL columns may not contain a NULL */ + int jmp3; + int jmp2 = sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + VdbeCoverage(v); + if( p1<0 ){ + sqlite3VdbeChangeP5(v, 0x0f); /* INT, REAL, TEXT, or BLOB */ + jmp3 = jmp2; + }else{ + sqlite3VdbeChangeP5(v, 0x0d); /* INT, TEXT, or BLOB */ + /* OP_IsType does not detect NaN values in the database file + ** which should be treated as a NULL. So if the header type + ** is REAL, we have to load the actual data using OP_Column + ** to reliably determine if the value is a NULL. */ + sqlite3VdbeAddOp3(v, OP_Column, p1, p3, 3); + sqlite3ColumnDefault(v, pTab, j, 3); + jmp3 = sqlite3VdbeAddOp2(v, OP_NotNull, 3, labelOk); VdbeCoverage(v); } - jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); - sqlite3VdbeGoto(v, uniqOk); - sqlite3VdbeJumpHere(v, jmp6); - sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, - pIdx->nKeyCol); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); /* Decrement error limit */ - sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); - sqlite3VdbeGoto(v, jmp5); - sqlite3VdbeResolveLabel(v, uniqOk); + zErr = sqlite3MPrintf(db, "NULL value in %s.%s", pTab->zName, + pCol->zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + if( doTypeCheck ){ + sqlite3VdbeGoto(v, labelError); + sqlite3VdbeJumpHere(v, jmp2); + sqlite3VdbeJumpHere(v, jmp3); + }else{ + /* VDBE byte code will fall thru */ + } + } + if( bStrict && doTypeCheck ){ + /* (2) Datatype must be exact for non-ANY columns in STRICT tables*/ + static unsigned char aStdTypeMask[] = { + 0x1f, /* ANY */ + 0x18, /* BLOB */ + 0x11, /* INT */ + 0x11, /* INTEGER */ + 0x13, /* REAL */ + 0x14 /* TEXT */ + }; + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + assert( pCol->eCType>=1 && pCol->eCType<=sizeof(aStdTypeMask) ); + sqlite3VdbeChangeP5(v, aStdTypeMask[pCol->eCType-1]); + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "non-%s value in %s.%s", + sqlite3StdType[pCol->eCType-1], + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + }else if( !bStrict && pCol->affinity==SQLITE_AFF_TEXT ){ + /* (3) Datatype for TEXT columns in non-STRICT tables must be + ** NULL, TEXT, or BLOB. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "NUMERIC value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + }else if( !bStrict && pCol->affinity>=SQLITE_AFF_NUMERIC ){ + /* (4) Datatype for numeric columns in non-STRICT tables must not + ** be a TEXT value that can be converted to numeric. */ + sqlite3VdbeAddOp4Int(v, OP_IsType, p1, labelOk, p3, p4); + sqlite3VdbeChangeP5(v, 0x1b); /* NULL, INT, FLOAT, or BLOB */ + VdbeCoverage(v); + if( p1>=0 ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, j, 3); + } + sqlite3VdbeAddOp4(v, OP_Affinity, 3, 1, 0, "C", P4_STATIC); + sqlite3VdbeAddOp4Int(v, OP_IsType, -1, labelOk, 3, p4); + sqlite3VdbeChangeP5(v, 0x1c); /* NULL, TEXT, or BLOB */ + VdbeCoverage(v); + zErr = sqlite3MPrintf(db, "TEXT value in %s.%s", + pTab->zName, pTab->aCol[j].zCnName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + } + sqlite3VdbeResolveLabel(v, labelError); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, labelOk); + } + /* Verify CHECK constraints */ + if( pTab->pCheck && (db->flags & SQLITE_IgnoreChecks)==0 ){ + ExprList *pCheck = sqlite3ExprListDup(db, pTab->pCheck, 0); + if( db->mallocFailed==0 ){ + int addrCkFault = sqlite3VdbeMakeLabel(pParse); + int addrCkOk = sqlite3VdbeMakeLabel(pParse); + char *zErr; + int k; + pParse->iSelfTab = iDataCur + 1; + for(k=pCheck->nExpr-1; k>0; k--){ + sqlite3ExprIfFalse(pParse, pCheck->a[k].pExpr, addrCkFault, 0); + } + sqlite3ExprIfTrue(pParse, pCheck->a[0].pExpr, addrCkOk, + SQLITE_JUMPIFNULL); + sqlite3VdbeResolveLabel(v, addrCkFault); + pParse->iSelfTab = 0; + zErr = sqlite3MPrintf(db, "CHECK constraint failed in %s", + pTab->zName); + sqlite3VdbeAddOp4(v, OP_String8, 0, 3, 0, zErr, P4_DYNAMIC); + integrityCheckResultRow(v); + sqlite3VdbeResolveLabel(v, addrCkOk); + } + sqlite3ExprListDelete(db, pCheck); + } + if( !isQuick ){ /* Omit the remaining tests for quick_check */ + /* Validate index entries for the current row */ + for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + int jmp2, jmp3, jmp4, jmp5, label6; + int kk; + int ckUniq = sqlite3VdbeMakeLabel(pParse); + if( pPk==pIdx ) continue; + r1 = sqlite3GenerateIndexKey(pParse, pIdx, iDataCur, 0, 0, &jmp3, + pPrior, r1); + pPrior = pIdx; + sqlite3VdbeAddOp2(v, OP_AddImm, 8+j, 1);/* increment entry count */ + /* Verify that an index entry exists for the current table row */ + jmp2 = sqlite3VdbeAddOp4Int(v, OP_Found, iIdxCur+j, ckUniq, r1, + pIdx->nColumn); VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " missing from index "); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp5 = sqlite3VdbeLoadString(v, 4, pIdx->zName); + sqlite3VdbeAddOp3(v, OP_Concat, 4, 3, 3); + jmp4 = integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, jmp2); + + /* The OP_IdxRowid opcode is an optimized version of OP_Column + ** that extracts the rowid off the end of the index record. + ** But it only works correctly if index record does not have + ** any extra bytes at the end. Verify that this is the case. */ + if( HasRowid(pTab) ){ + int jmp7; + sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur+j, 3); + jmp7 = sqlite3VdbeAddOp3(v, OP_Eq, 3, 0, r1+pIdx->nColumn-1); + VdbeCoverageNeverNull(v); + sqlite3VdbeLoadString(v, 3, + "rowid not at end-of-record for row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " of index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp7); + } + + /* Any indexed columns with non-BINARY collations must still hold + ** the exact same text value as the table. */ + label6 = 0; + for(kk=0; kknKeyCol; kk++){ + if( pIdx->azColl[kk]==sqlite3StrBINARY ) continue; + if( label6==0 ) label6 = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iIdxCur+j, kk, 3); + sqlite3VdbeAddOp3(v, OP_Ne, 3, label6, r1+kk); VdbeCoverage(v); + } + if( label6 ){ + int jmp6 = sqlite3VdbeAddOp0(v, OP_Goto); + sqlite3VdbeResolveLabel(v, label6); + sqlite3VdbeLoadString(v, 3, "row "); + sqlite3VdbeAddOp3(v, OP_Concat, 7, 3, 3); + sqlite3VdbeLoadString(v, 4, " values differ from index "); + sqlite3VdbeGoto(v, jmp5-1); + sqlite3VdbeJumpHere(v, jmp6); + } + + /* For UNIQUE indexes, verify that only one entry exists with the + ** current key. The entry is unique if (1) any column is NULL + ** or (2) the next entry has a different key */ + if( IsUniqueIndex(pIdx) ){ + int uniqOk = sqlite3VdbeMakeLabel(pParse); + int jmp6; + for(kk=0; kknKeyCol; kk++){ + int iCol = pIdx->aiColumn[kk]; + assert( iCol!=XN_ROWID && iColnCol ); + if( iCol>=0 && pTab->aCol[iCol].notNull ) continue; + sqlite3VdbeAddOp2(v, OP_IsNull, r1+kk, uniqOk); + VdbeCoverage(v); + } + jmp6 = sqlite3VdbeAddOp1(v, OP_Next, iIdxCur+j); VdbeCoverage(v); + sqlite3VdbeGoto(v, uniqOk); + sqlite3VdbeJumpHere(v, jmp6); + sqlite3VdbeAddOp4Int(v, OP_IdxGT, iIdxCur+j, uniqOk, r1, + pIdx->nKeyCol); VdbeCoverage(v); + sqlite3VdbeLoadString(v, 3, "non-unique entry in index "); + sqlite3VdbeGoto(v, jmp5); + sqlite3VdbeResolveLabel(v, uniqOk); + } + sqlite3VdbeJumpHere(v, jmp4); + sqlite3ResolvePartIdxLabel(pParse, jmp3); } - sqlite3VdbeJumpHere(v, jmp4); - sqlite3ResolvePartIdxLabel(pParse, jmp3); } sqlite3VdbeAddOp2(v, OP_Next, iDataCur, loopTop); VdbeCoverage(v); sqlite3VdbeJumpHere(v, loopTop-1); -#ifndef SQLITE_OMIT_BTREECOUNT - sqlite3VdbeLoadString(v, 2, "wrong # of entries in index "); - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ - if( pPk==pIdx ) continue; - addr = sqlite3VdbeCurrentAddr(v); - sqlite3VdbeAddOp2(v, OP_IfPos, 1, addr+2); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_Halt, 0, 0); - sqlite3VdbeAddOp2(v, OP_Count, iIdxCur+j, 3); - sqlite3VdbeAddOp3(v, OP_Eq, 8+j, addr+8, 3); VdbeCoverage(v); - sqlite3VdbeChangeP5(v, SQLITE_NOTNULL); - sqlite3VdbeAddOp2(v, OP_AddImm, 1, -1); - sqlite3VdbeLoadString(v, 3, pIdx->zName); - sqlite3VdbeAddOp3(v, OP_Concat, 3, 2, 7); - sqlite3VdbeAddOp2(v, OP_ResultRow, 7, 1); + if( pPk ){ + assert( !isQuick ); + sqlite3ReleaseTempRange(pParse, r2, pPk->nKeyCol); } -#endif /* SQLITE_OMIT_BTREECOUNT */ - } + } + +#ifndef SQLITE_OMIT_VIRTUALTABLE + /* Second pass to invoke the xIntegrity method on all virtual + ** tables. + */ + for(x=sqliteHashFirst(pTbls); x; x=sqliteHashNext(x)){ + Table *pTab = sqliteHashData(x); + sqlite3_vtab *pVTab; + int a1; + if( pObjTab && pObjTab!=pTab ) continue; + if( IsOrdinaryTable(pTab) ) continue; + if( !IsVirtual(pTab) ) continue; + if( pTab->nCol<=0 ){ + const char *zMod = pTab->u.vtab.azArg[0]; + if( sqlite3HashFind(&db->aModule, zMod)==0 ) continue; + } + sqlite3ViewGetColumnNames(pParse, pTab); + if( pTab->u.vtab.p==0 ) continue; + pVTab = pTab->u.vtab.p->pVtab; + if( NEVER(pVTab==0) ) continue; + if( NEVER(pVTab->pModule==0) ) continue; + if( pVTab->pModule->iVersion<4 ) continue; + if( pVTab->pModule->xIntegrity==0 ) continue; + sqlite3VdbeAddOp3(v, OP_VCheck, i, 3, isQuick); + pTab->nTabRef++; + sqlite3VdbeAppendP4(v, pTab, P4_TABLEREF); + a1 = sqlite3VdbeAddOp1(v, OP_IsNull, 3); VdbeCoverage(v); + integrityCheckResultRow(v); + sqlite3VdbeJumpHere(v, a1); + continue; + } +#endif } { static const int iLn = VDBE_OFFSET_LINENO(2); static const VdbeOpList endCode[] = { { OP_AddImm, 1, 0, 0}, /* 0 */ - { OP_If, 1, 4, 0}, /* 1 */ + { OP_IfNotZero, 1, 4, 0}, /* 1 */ { OP_String8, 0, 3, 0}, /* 2 */ { OP_ResultRow, 3, 1, 0}, /* 3 */ + { OP_Halt, 0, 0, 0}, /* 4 */ + { OP_String8, 0, 3, 0}, /* 5 */ + { OP_Goto, 0, 3, 0}, /* 6 */ }; VdbeOp *aOp; aOp = sqlite3VdbeAddOpList(v, ArraySize(endCode), endCode, iLn); if( aOp ){ - aOp[0].p2 = -mxErr; + aOp[0].p2 = 1-mxErr; aOp[2].p4type = P4_STATIC; aOp[2].p4.z = "ok"; + aOp[5].p4type = P4_STATIC; + aOp[5].p4.z = (char*)sqlite3ErrStr(SQLITE_CORRUPT); } + sqlite3VdbeChangeP3(v, 0, sqlite3VdbeCurrentAddr(v)-2); } } break; @@ -111648,7 +141720,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** encoding that will be used for the main database file if a new file ** is created. If an existing main database file is opened, then the ** default text encoding for the existing database is used. - ** + ** ** In all cases new databases created using the ATTACH command are ** created to use the same default text encoding as the main database. If ** the main database has not been initialized and/or created when ATTACH @@ -111679,21 +141751,19 @@ SQLITE_PRIVATE void sqlite3Pragma( assert( encnames[SQLITE_UTF8].enc==SQLITE_UTF8 ); assert( encnames[SQLITE_UTF16LE].enc==SQLITE_UTF16LE ); assert( encnames[SQLITE_UTF16BE].enc==SQLITE_UTF16BE ); - returnSingleText(v, "encoding", encnames[ENC(pParse->db)].zName); + returnSingleText(v, encnames[ENC(pParse->db)].zName); }else{ /* "PRAGMA encoding = XXX" */ /* Only change the value of sqlite.enc if the database handle is not ** initialized. If the main database exists, the new sqlite.enc value ** will be overwritten when the schema is next loaded. If it does not ** already exists, it will be created to use the new encoding value. */ - if( - !(DbHasProperty(db, 0, DB_SchemaLoaded)) || - DbHasProperty(db, 0, DB_Empty) - ){ + if( (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ for(pEnc=&encnames[0]; pEnc->zName; pEnc++){ if( 0==sqlite3StrICmp(zRight, pEnc->zName) ){ - SCHEMA_ENC(db) = ENC(db) = - pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + u8 enc = pEnc->enc ? pEnc->enc : SQLITE_UTF16NATIVE; + SCHEMA_ENC(db) = enc; + sqlite3SetTextEncoding(db, enc); break; } } @@ -111742,7 +141812,7 @@ SQLITE_PRIVATE void sqlite3Pragma( case PragTyp_HEADER_VALUE: { int iCookie = pPragma->iArg; /* Which cookie to read or write */ sqlite3VdbeUsesBtree(v, iDb); - if( zRight && (pPragma->mPragFlag & PragFlag_ReadOnly)==0 ){ + if( zRight && (pPragma->mPragFlg & PragFlg_ReadOnly)==0 ){ /* Write the specified cookie value */ static const VdbeOpList setCookie[] = { { OP_Transaction, 0, 1, 0}, /* 0 */ @@ -111756,6 +141826,12 @@ SQLITE_PRIVATE void sqlite3Pragma( aOp[1].p1 = iDb; aOp[1].p2 = iCookie; aOp[1].p3 = sqlite3Atoi(zRight); + aOp[1].p5 = 1; + if( iCookie==BTREE_SCHEMA_VERSION && (db->flags & SQLITE_Defensive)!=0 ){ + /* Do not allow the use of PRAGMA schema_version=VALUE in defensive + ** mode. Change the OP_SetCookie opcode into a no-op. */ + aOp[1].opcode = OP_Noop; + } }else{ /* Read the specified cookie value */ static const VdbeOpList readCookie[] = { @@ -111770,8 +141846,6 @@ SQLITE_PRIVATE void sqlite3Pragma( aOp[0].p1 = iDb; aOp[1].p1 = iDb; aOp[1].p3 = iCookie; - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, zLeft, SQLITE_TRANSIENT); sqlite3VdbeReusable(v); } } @@ -111789,7 +141863,6 @@ SQLITE_PRIVATE void sqlite3Pragma( int i = 0; const char *zOpt; pParse->nMem = 1; - setOneColumnName(v, "compile_option"); while( (zOpt = sqlite3_compileoption_get(i++))!=0 ){ sqlite3VdbeLoadString(v, 1, zOpt); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 1); @@ -111806,8 +141879,7 @@ SQLITE_PRIVATE void sqlite3Pragma( ** Checkpoint the database. */ case PragTyp_WAL_CHECKPOINT: { - static const char *azCol[] = { "busy", "log", "checkpointed" }; - int iBt = (pId2->z?iDb:SQLITE_MAX_ATTACHED); + int iBt = (pId2->z?iDb:SQLITE_MAX_DB); int eMode = SQLITE_CHECKPOINT_PASSIVE; if( zRight ){ if( sqlite3StrICmp(zRight, "full")==0 ){ @@ -111818,7 +141890,6 @@ SQLITE_PRIVATE void sqlite3Pragma( eMode = SQLITE_CHECKPOINT_TRUNCATE; } } - setAllColumnNames(v, 3, azCol); assert( 3==ArraySize(azCol) ); pParse->nMem = 3; sqlite3VdbeAddOp3(v, OP_Checkpoint, iBt, eMode, 1); sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 3); @@ -111837,8 +141908,8 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight ){ sqlite3_wal_autocheckpoint(db, sqlite3Atoi(zRight)); } - returnSingleInt(v, "wal_autocheckpoint", - db->xWalCallback==sqlite3WalDefaultHook ? + returnSingleInt(v, + db->xWalCallback==sqlite3WalDefaultHook ? SQLITE_PTR_TO_INT(db->pWalArg) : 0); } break; @@ -111856,6 +141927,204 @@ SQLITE_PRIVATE void sqlite3Pragma( break; } + /* + ** PRAGMA optimize + ** PRAGMA optimize(MASK) + ** PRAGMA schema.optimize + ** PRAGMA schema.optimize(MASK) + ** + ** Attempt to optimize the database. All schemas are optimized in the first + ** two forms, and only the specified schema is optimized in the latter two. + ** + ** The details of optimizations performed by this pragma are expected + ** to change and improve over time. Applications should anticipate that + ** this pragma will perform new optimizations in future releases. + ** + ** The optional argument is a bitmask of optimizations to perform: + ** + ** 0x00001 Debugging mode. Do not actually perform any optimizations + ** but instead return one line of text for each optimization + ** that would have been done. Off by default. + ** + ** 0x00002 Run ANALYZE on tables that might benefit. On by default. + ** See below for additional information. + ** + ** 0x00010 Run all ANALYZE operations using an analysis_limit that + ** is the lessor of the current analysis_limit and the + ** SQLITE_DEFAULT_OPTIMIZE_LIMIT compile-time option. + ** The default value of SQLITE_DEFAULT_OPTIMIZE_LIMIT is + ** currently (2024-02-19) set to 2000, which is such that + ** the worst case run-time for PRAGMA optimize on a 100MB + ** database will usually be less than 100 milliseconds on + ** a RaspberryPI-4 class machine. On by default. + ** + ** 0x10000 Look at tables to see if they need to be reanalyzed + ** due to growth or shrinkage even if they have not been + ** queried during the current connection. Off by default. + ** + ** The default MASK is and always shall be 0x0fffe. In the current + ** implementation, the default mask only covers the 0x00002 optimization, + ** though additional optimizations that are covered by 0x0fffe might be + ** added in the future. Optimizations that are off by default and must + ** be explicitly requested have masks of 0x10000 or greater. + ** + ** DETERMINATION OF WHEN TO RUN ANALYZE + ** + ** In the current implementation, a table is analyzed if only if all of + ** the following are true: + ** + ** (1) MASK bit 0x00002 is set. + ** + ** (2) The table is an ordinary table, not a virtual table or view. + ** + ** (3) The table name does not begin with "sqlite_". + ** + ** (4) One or more of the following is true: + ** (4a) The 0x10000 MASK bit is set. + ** (4b) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. + ** (4c) The query planner used sqlite_stat1-style statistics for one + ** or more indexes of the table at some point during the lifetime + ** of the current connection. + ** + ** (5) One or more of the following is true: + ** (5a) One or more indexes on the table lacks an entry + ** in the sqlite_stat1 table. (Same as 4a) + ** (5b) The number of rows in the table has increased or decreased by + ** 10-fold. In other words, the current size of the table is + ** 10 times larger than the size in sqlite_stat1 or else the + ** current size is less than 1/10th the size in sqlite_stat1. + ** + ** The rules for when tables are analyzed are likely to change in + ** future releases. Future versions of SQLite might accept a string + ** literal argument to this pragma that contains a mnemonic description + ** of the options rather than a bitmap. + */ + case PragTyp_OPTIMIZE: { + int iDbLast; /* Loop termination point for the schema loop */ + int iTabCur; /* Cursor for a table whose size needs checking */ + HashElem *k; /* Loop over tables of a schema */ + Schema *pSchema; /* The current schema */ + Table *pTab; /* A table in the schema */ + Index *pIdx; /* An index of the table */ + LogEst szThreshold; /* Size threshold above which reanalysis needed */ + char *zSubSql; /* SQL statement for the OP_SqlExec opcode */ + u32 opMask; /* Mask of operations to perform */ + int nLimit; /* Analysis limit to use */ + int nCheck = 0; /* Number of tables to be optimized */ + int nBtree = 0; /* Number of btrees to scan */ + int nIndex; /* Number of indexes on the current table */ + + if( zRight ){ + opMask = (u32)sqlite3Atoi(zRight); + if( (opMask & 0x02)==0 ) break; + }else{ + opMask = 0xfffe; + } + if( (opMask & 0x10)==0 ){ + nLimit = 0; + }else if( db->nAnalysisLimit>0 + && db->nAnalysisLimitnTab++; + for(iDbLast = zDb?iDb:db->nDb-1; iDb<=iDbLast; iDb++){ + if( iDb==1 ) continue; + sqlite3CodeVerifySchema(pParse, iDb); + pSchema = db->aDb[iDb].pSchema; + for(k=sqliteHashFirst(&pSchema->tblHash); k; k=sqliteHashNext(k)){ + pTab = (Table*)sqliteHashData(k); + + /* This only works for ordinary tables */ + if( !IsOrdinaryTable(pTab) ) continue; + + /* Do not scan system tables */ + if( 0==sqlite3StrNICmp(pTab->zName, "sqlite_", 7) ) continue; + + /* Find the size of the table as last recorded in sqlite_stat1. + ** If any index is unanalyzed, then the threshold is -1 to + ** indicate a new, unanalyzed index + */ + szThreshold = pTab->nRowLogEst; + nIndex = 0; + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + nIndex++; + if( !pIdx->hasStat1 ){ + szThreshold = -1; /* Always analyze if any index lacks statistics */ + } + } + + /* If table pTab has not been used in a way that would benefit from + ** having analysis statistics during the current session, then skip it, + ** unless the 0x10000 MASK bit is set. */ + if( (pTab->tabFlags & TF_MaybeReanalyze)!=0 ){ + /* Check for size change if stat1 has been used for a query */ + }else if( opMask & 0x10000 ){ + /* Check for size change if 0x10000 is set */ + }else if( pTab->pIndex!=0 && szThreshold<0 ){ + /* Do analysis if unanalyzed indexes exists */ + }else{ + /* Otherwise, we can skip this table */ + continue; + } + + nCheck++; + if( nCheck==2 ){ + /* If ANALYZE might be invoked two or more times, hold a write + ** transaction for efficiency */ + sqlite3BeginWriteOperation(pParse, 0, iDb); + } + nBtree += nIndex+1; + + /* Reanalyze if the table is 10 times larger or smaller than + ** the last analysis. Unconditional reanalysis if there are + ** unanalyzed indexes. */ + sqlite3OpenTable(pParse, iTabCur, iDb, pTab, OP_OpenRead); + if( szThreshold>=0 ){ + const LogEst iRange = 33; /* 10x size change */ + sqlite3VdbeAddOp4Int(v, OP_IfSizeBetween, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1), + szThreshold>=iRange ? szThreshold-iRange : -1, + szThreshold+iRange); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Rewind, iTabCur, + sqlite3VdbeCurrentAddr(v)+2+(opMask&1)); + VdbeCoverage(v); + } + zSubSql = sqlite3MPrintf(db, "ANALYZE \"%w\".\"%w\"", + db->aDb[iDb].zDbSName, pTab->zName); + if( opMask & 0x01 ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, r1, 0, zSubSql, P4_DYNAMIC); + sqlite3VdbeAddOp2(v, OP_ResultRow, r1, 1); + }else{ + sqlite3VdbeAddOp4(v, OP_SqlExec, nLimit ? 0x02 : 00, nLimit, 0, + zSubSql, P4_DYNAMIC); + } + } + } + sqlite3VdbeAddOp0(v, OP_Expire); + + /* In a schema with a large number of tables and indexes, scale back + ** the analysis_limit to avoid excess run-time in the worst case. + */ + if( !db->mallocFailed && nLimit>0 && nBtree>100 ){ + int iAddr, iEnd; + VdbeOp *aOp; + nLimit = 100*nLimit/nBtree; + if( nLimit<100 ) nLimit = 100; + aOp = sqlite3VdbeGetOp(v, 0); + iEnd = sqlite3VdbeCurrentAddr(v); + for(iAddr=0; iAddrbusyTimeout); + returnSingleInt(v, db->busyTimeout); break; } @@ -111890,7 +142159,28 @@ SQLITE_PRIVATE void sqlite3Pragma( if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ sqlite3_soft_heap_limit64(N); } - returnSingleInt(v, "soft_heap_limit", sqlite3_soft_heap_limit64(-1)); + returnSingleInt(v, sqlite3_soft_heap_limit64(-1)); + break; + } + + /* + ** PRAGMA hard_heap_limit + ** PRAGMA hard_heap_limit = N + ** + ** Invoke sqlite3_hard_heap_limit64() to query or set the hard heap + ** limit. The hard heap limit can be activated or lowered by this + ** pragma, but not raised or deactivated. Only the + ** sqlite3_hard_heap_limit64() C-language API can raise or deactivate + ** the hard heap limit. This allows an application to set a heap limit + ** constraint that cannot be relaxed by an untrusted SQL script. + */ + case PragTyp_HARD_HEAP_LIMIT: { + sqlite3_int64 N; + if( zRight && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK ){ + sqlite3_int64 iPrior = sqlite3_hard_heap_limit64(-1); + if( N>0 && (iPrior==0 || iPrior>N) ) sqlite3_hard_heap_limit64(N); + } + returnSingleInt(v, sqlite3_hard_heap_limit64(-1)); break; } @@ -111909,8 +142199,26 @@ SQLITE_PRIVATE void sqlite3Pragma( ){ sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, (int)(N&0x7fffffff)); } - returnSingleInt(v, "threads", - sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + returnSingleInt(v, sqlite3_limit(db, SQLITE_LIMIT_WORKER_THREADS, -1)); + break; + } + + /* + ** PRAGMA analysis_limit + ** PRAGMA analysis_limit = N + ** + ** Configure the maximum number of rows that ANALYZE will examine + ** in each index that it looks at. Return the new limit. + */ + case PragTyp_ANALYSIS_LIMIT: { + sqlite3_int64 N; + if( zRight + && sqlite3DecOrHexToI64(zRight, &N)==SQLITE_OK /* IMP: R-40975-20399 */ + && N>=0 + ){ + db->nAnalysisLimit = (int)(N&0x7fffffff); + } + returnSingleInt(v, db->nAnalysisLimit); /* IMP: R-57594-65522 */ break; } @@ -111922,78 +142230,363 @@ SQLITE_PRIVATE void sqlite3Pragma( static const char *const azLockName[] = { "unlocked", "shared", "reserved", "pending", "exclusive" }; - static const char *azCol[] = { "database", "status" }; int i; - setAllColumnNames(v, 2, azCol); assert( 2==ArraySize(azCol) ); pParse->nMem = 2; for(i=0; inDb; i++){ Btree *pBt; const char *zState = "unknown"; int j; - if( db->aDb[i].zName==0 ) continue; + if( db->aDb[i].zDbSName==0 ) continue; pBt = db->aDb[i].pBt; if( pBt==0 || sqlite3BtreePager(pBt)==0 ){ zState = "closed"; - }else if( sqlite3_file_control(db, i ? db->aDb[i].zName : 0, + }else if( sqlite3_file_control(db, i ? db->aDb[i].zDbSName : 0, SQLITE_FCNTL_LOCKSTATE, &j)==SQLITE_OK ){ zState = azLockName[j]; } - sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zName, zState); - sqlite3VdbeAddOp2(v, OP_ResultRow, 1, 2); + sqlite3VdbeMultiLoad(v, 1, "ss", db->aDb[i].zDbSName, zState); } break; } #endif -#ifdef SQLITE_HAS_CODEC - case PragTyp_KEY: { - if( zRight ) sqlite3_key_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); - break; - } - case PragTyp_REKEY: { - if( zRight ) sqlite3_rekey_v2(db, zDb, zRight, sqlite3Strlen30(zRight)); - break; - } - case PragTyp_HEXKEY: { - if( zRight ){ - u8 iByte; - int i; - char zKey[40]; - for(i=0, iByte=0; imPragFlg & PragFlg_NoColumns1) && zRight ){ + sqlite3VdbeVerifyNoResultRow(v); + } + pragma_out: sqlite3DbFree(db, zLeft); sqlite3DbFree(db, zRight); } +#ifndef SQLITE_OMIT_VIRTUALTABLE +/***************************************************************************** +** Implementation of an eponymous virtual table that runs a pragma. +** +*/ +typedef struct PragmaVtab PragmaVtab; +typedef struct PragmaVtabCursor PragmaVtabCursor; +struct PragmaVtab { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database connection to which it belongs */ + const PragmaName *pName; /* Name of the pragma */ + u8 nHidden; /* Number of hidden columns */ + u8 iHidden; /* Index of the first hidden column */ +}; +struct PragmaVtabCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + sqlite3_stmt *pPragma; /* The pragma statement to run */ + sqlite_int64 iRowid; /* Current rowid */ + char *azArg[2]; /* Value of the argument and schema */ +}; + +/* +** Pragma virtual table module xConnect method. +*/ +static int pragmaVtabConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + const PragmaName *pPragma = (const PragmaName*)pAux; + PragmaVtab *pTab = 0; + int rc; + int i, j; + char cSep = '('; + StrAccum acc; + char zBuf[200]; + + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + sqlite3StrAccumInit(&acc, 0, zBuf, sizeof(zBuf), 0); + sqlite3_str_appendall(&acc, "CREATE TABLE x"); + for(i=0, j=pPragma->iPragCName; inPragCName; i++, j++){ + sqlite3_str_appendf(&acc, "%c\"%s\"", cSep, pragCName[j]); + cSep = ','; + } + if( i==0 ){ + sqlite3_str_appendf(&acc, "(\"%s\"", pPragma->zName); + i++; + } + j = 0; + if( pPragma->mPragFlg & PragFlg_Result1 ){ + sqlite3_str_appendall(&acc, ",arg HIDDEN"); + j++; + } + if( pPragma->mPragFlg & (PragFlg_SchemaOpt|PragFlg_SchemaReq) ){ + sqlite3_str_appendall(&acc, ",schema HIDDEN"); + j++; + } + sqlite3_str_append(&acc, ")", 1); + sqlite3StrAccumFinish(&acc); + assert( strlen(zBuf) < sizeof(zBuf)-1 ); + rc = sqlite3_declare_vtab(db, zBuf); + if( rc==SQLITE_OK ){ + pTab = (PragmaVtab*)sqlite3_malloc(sizeof(PragmaVtab)); + if( pTab==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pTab, 0, sizeof(PragmaVtab)); + pTab->pName = pPragma; + pTab->db = db; + pTab->iHidden = i; + pTab->nHidden = j; + } + }else{ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Pragma virtual table module xDisconnect method. +*/ +static int pragmaVtabDisconnect(sqlite3_vtab *pVtab){ + PragmaVtab *pTab = (PragmaVtab*)pVtab; + sqlite3_free(pTab); + return SQLITE_OK; +} + +/* Figure out the best index to use to search a pragma virtual table. +** +** There are not really any index choices. But we want to encourage the +** query planner to give == constraints on as many hidden parameters as +** possible, and especially on the first hidden parameter. So return a +** high cost if hidden parameters are unconstrained. +*/ +static int pragmaVtabBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + PragmaVtab *pTab = (PragmaVtab*)tab; + const struct sqlite3_index_constraint *pConstraint; + int i, j; + int seen[2]; + + pIdxInfo->estimatedCost = (double)1; + if( pTab->nHidden==0 ){ return SQLITE_OK; } + pConstraint = pIdxInfo->aConstraint; + seen[0] = 0; + seen[1] = 0; + for(i=0; inConstraint; i++, pConstraint++){ + if( pConstraint->iColumn < pTab->iHidden ) continue; + if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( pConstraint->usable==0 ) return SQLITE_CONSTRAINT; + j = pConstraint->iColumn - pTab->iHidden; + assert( j < 2 ); + seen[j] = i+1; + } + if( seen[0]==0 ){ + pIdxInfo->estimatedCost = (double)2147483647; + pIdxInfo->estimatedRows = 2147483647; + return SQLITE_OK; + } + j = seen[0]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 1; + pIdxInfo->aConstraintUsage[j].omit = 1; + pIdxInfo->estimatedCost = (double)20; + pIdxInfo->estimatedRows = 20; + if( seen[1] ){ + j = seen[1]-1; + pIdxInfo->aConstraintUsage[j].argvIndex = 2; + pIdxInfo->aConstraintUsage[j].omit = 1; + } + return SQLITE_OK; +} + +/* Create a new cursor for the pragma virtual table */ +static int pragmaVtabOpen(sqlite3_vtab *pVtab, sqlite3_vtab_cursor **ppCursor){ + PragmaVtabCursor *pCsr; + pCsr = (PragmaVtabCursor*)sqlite3_malloc(sizeof(*pCsr)); + if( pCsr==0 ) return SQLITE_NOMEM; + memset(pCsr, 0, sizeof(PragmaVtabCursor)); + pCsr->base.pVtab = pVtab; + *ppCursor = &pCsr->base; + return SQLITE_OK; +} + +/* Clear all content from pragma virtual table cursor. */ +static void pragmaVtabCursorClear(PragmaVtabCursor *pCsr){ + int i; + sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + pCsr->iRowid = 0; + for(i=0; iazArg); i++){ + sqlite3_free(pCsr->azArg[i]); + pCsr->azArg[i] = 0; + } +} + +/* Close a pragma virtual table cursor */ +static int pragmaVtabClose(sqlite3_vtab_cursor *cur){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)cur; + pragmaVtabCursorClear(pCsr); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* Advance the pragma virtual table cursor to the next row */ +static int pragmaVtabNext(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + int rc = SQLITE_OK; + + /* Increment the xRowid value */ + pCsr->iRowid++; + assert( pCsr->pPragma ); + if( SQLITE_ROW!=sqlite3_step(pCsr->pPragma) ){ + rc = sqlite3_finalize(pCsr->pPragma); + pCsr->pPragma = 0; + pragmaVtabCursorClear(pCsr); + } + return rc; +} + +/* +** Pragma virtual table module xFilter method. +*/ +static int pragmaVtabFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + int rc; + int i, j; + StrAccum acc; + char *zSql; + + UNUSED_PARAMETER(idxNum); + UNUSED_PARAMETER(idxStr); + pragmaVtabCursorClear(pCsr); + j = (pTab->pName->mPragFlg & PragFlg_Result1)!=0 ? 0 : 1; + for(i=0; iazArg) ); + assert( pCsr->azArg[j]==0 ); + if( zText ){ + pCsr->azArg[j] = sqlite3_mprintf("%s", zText); + if( pCsr->azArg[j]==0 ){ + return SQLITE_NOMEM; + } + } + } + sqlite3StrAccumInit(&acc, 0, 0, 0, pTab->db->aLimit[SQLITE_LIMIT_SQL_LENGTH]); + sqlite3_str_appendall(&acc, "PRAGMA "); + if( pCsr->azArg[1] ){ + sqlite3_str_appendf(&acc, "%Q.", pCsr->azArg[1]); + } + sqlite3_str_appendall(&acc, pTab->pName->zName); + if( pCsr->azArg[0] ){ + sqlite3_str_appendf(&acc, "=%Q", pCsr->azArg[0]); + } + zSql = sqlite3StrAccumFinish(&acc); + if( zSql==0 ) return SQLITE_NOMEM; + rc = sqlite3_prepare_v2(pTab->db, zSql, -1, &pCsr->pPragma, 0); + sqlite3_free(zSql); + if( rc!=SQLITE_OK ){ + pTab->base.zErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pTab->db)); + return rc; + } + return pragmaVtabNext(pVtabCursor); +} + +/* +** Pragma virtual table module xEof method. +*/ +static int pragmaVtabEof(sqlite3_vtab_cursor *pVtabCursor){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + return (pCsr->pPragma==0); +} + +/* The xColumn method simply returns the corresponding column from +** the PRAGMA. +*/ +static int pragmaVtabColumn( + sqlite3_vtab_cursor *pVtabCursor, + sqlite3_context *ctx, + int i +){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + PragmaVtab *pTab = (PragmaVtab*)(pVtabCursor->pVtab); + if( iiHidden ){ + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pPragma, i)); + }else{ + sqlite3_result_text(ctx, pCsr->azArg[i-pTab->iHidden],-1,SQLITE_TRANSIENT); + } + return SQLITE_OK; +} + +/* +** Pragma virtual table module xRowid method. +*/ +static int pragmaVtabRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *p){ + PragmaVtabCursor *pCsr = (PragmaVtabCursor*)pVtabCursor; + *p = pCsr->iRowid; + return SQLITE_OK; +} + +/* The pragma virtual table object */ +static const sqlite3_module pragmaVtabModule = { + 0, /* iVersion */ + 0, /* xCreate - create a table */ + pragmaVtabConnect, /* xConnect - connect to an existing table */ + pragmaVtabBestIndex, /* xBestIndex - Determine search strategy */ + pragmaVtabDisconnect, /* xDisconnect - Disconnect from a table */ + 0, /* xDestroy - Drop a table */ + pragmaVtabOpen, /* xOpen - open a cursor */ + pragmaVtabClose, /* xClose - close a cursor */ + pragmaVtabFilter, /* xFilter - configure scan constraints */ + pragmaVtabNext, /* xNext - advance a cursor */ + pragmaVtabEof, /* xEof */ + pragmaVtabColumn, /* xColumn - read data */ + pragmaVtabRowid, /* xRowid - read data */ + 0, /* xUpdate - write data */ + 0, /* xBegin - begin transaction */ + 0, /* xSync - sync transaction */ + 0, /* xCommit - commit transaction */ + 0, /* xRollback - rollback transaction */ + 0, /* xFindFunction - function overloading */ + 0, /* xRename - rename the table */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +/* +** Check to see if zTabName is really the name of a pragma. If it is, +** then register an eponymous virtual table for that pragma and return +** a pointer to the Module object for the new virtual table. +*/ +SQLITE_PRIVATE Module *sqlite3PragmaVtabRegister(sqlite3 *db, const char *zName){ + const PragmaName *pName; + assert( sqlite3_strnicmp(zName, "pragma_", 7)==0 ); + pName = pragmaLocate(zName+7); + if( pName==0 ) return 0; + if( (pName->mPragFlg & (PragFlg_Result0|PragFlg_Result1))==0 ) return 0; + assert( sqlite3HashFind(&db->aModule, zName)==0 ); + return sqlite3VtabCreateModule(db, zName, &pragmaVtabModule, (void*)pName, 0); +} + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ #endif /* SQLITE_OMIT_PRAGMA */ @@ -112022,21 +142615,63 @@ pragma_out: */ static void corruptSchema( InitData *pData, /* Initialization context */ - const char *zObj, /* Object being parsed at the point of error */ + char **azObj, /* Type and name of object being parsed */ const char *zExtra /* Error information */ ){ sqlite3 *db = pData->db; - if( !db->mallocFailed && (db->flags & SQLITE_RecoveryMode)==0 ){ + if( db->mallocFailed ){ + pData->rc = SQLITE_NOMEM_BKPT; + }else if( pData->pzErrMsg[0]!=0 ){ + /* A error message has already been generated. Do not overwrite it */ + }else if( pData->mInitFlags & (INITFLAG_AlterMask) ){ + static const char *azAlterType[] = { + "rename", + "drop column", + "add column" + }; + *pData->pzErrMsg = sqlite3MPrintf(db, + "error in %s %s after %s: %s", azObj[0], azObj[1], + azAlterType[(pData->mInitFlags&INITFLAG_AlterMask)-1], + zExtra + ); + pData->rc = SQLITE_ERROR; + }else if( db->flags & SQLITE_WriteSchema ){ + pData->rc = SQLITE_CORRUPT_BKPT; + }else{ char *z; - if( zObj==0 ) zObj = "?"; + const char *zObj = azObj[1] ? azObj[1] : "?"; z = sqlite3MPrintf(db, "malformed database schema (%s)", zObj); - if( zExtra ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); - sqlite3DbFree(db, *pData->pzErrMsg); + if( zExtra && zExtra[0] ) z = sqlite3MPrintf(db, "%z - %s", z, zExtra); *pData->pzErrMsg = z; + pData->rc = SQLITE_CORRUPT_BKPT; } - pData->rc = db->mallocFailed ? SQLITE_NOMEM_BKPT : SQLITE_CORRUPT_BKPT; } +/* +** Check to see if any sibling index (another index on the same table) +** of pIndex has the same root page number, and if it does, return true. +** This would indicate a corrupt schema. +*/ +SQLITE_PRIVATE int sqlite3IndexHasDuplicateRootPage(Index *pIndex){ + Index *p; + for(p=pIndex->pTable->pIndex; p; p=p->pNext){ + if( p->tnum==pIndex->tnum && p!=pIndex ) return 1; + } + return 0; +} + +/* forward declaration */ +static int sqlite3Prepare( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + Vdbe *pReprepare, /* VM being reprepared */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +); + + /* ** This is the callback routine for the code that initializes the ** database. See sqlite3Init() below for additional information. @@ -112044,9 +142679,11 @@ static void corruptSchema( ** ** Each callback contains the following information: ** -** argv[0] = name of thing being created -** argv[1] = root page number for table or index. 0 for trigger or view. -** argv[2] = SQL text for the CREATE statement. +** argv[0] = type of object: "table", "index", "trigger", or "view". +** argv[1] = name of thing being created +** argv[2] = associated table if an index or trigger +** argv[3] = root page number for table or index. 0 for trigger or view. +** argv[4] = SQL text for the CREATE statement. ** */ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char **NotUsed){ @@ -112054,52 +142691,71 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char sqlite3 *db = pData->db; int iDb = pData->iDb; - assert( argc==3 ); + assert( argc==5 ); UNUSED_PARAMETER2(NotUsed, argc); assert( sqlite3_mutex_held(db->mutex) ); - DbClearProperty(db, iDb, DB_Empty); + db->mDbFlags |= DBFLAG_EncodingFixed; + if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ + pData->nInitRow++; if( db->mallocFailed ){ - corruptSchema(pData, argv[0], 0); + corruptSchema(pData, argv, 0); return 1; } assert( iDb>=0 && iDbnDb ); - if( argv==0 ) return 0; /* Might happen if EMPTY_RESULT_CALLBACKS are on */ - if( argv[1]==0 ){ - corruptSchema(pData, argv[0], 0); - }else if( sqlite3_strnicmp(argv[2],"create ",7)==0 ){ + if( argv[3]==0 ){ + corruptSchema(pData, argv, 0); + }else if( argv[4] + && 'c'==sqlite3UpperToLower[(unsigned char)argv[4][0]] + && 'r'==sqlite3UpperToLower[(unsigned char)argv[4][1]] ){ /* Call the parser to process a CREATE TABLE, INDEX or VIEW. ** But because db->init.busy is set to 1, no VDBE code is generated ** or executed. All the parser does is build the internal data ** structures that describe the table, index, or view. + ** + ** No other valid SQL statement, other than the variable CREATE statements, + ** can begin with the letters "C" and "R". Thus, it is not possible run + ** any other kind of statement while parsing the schema, even a corrupt + ** schema. */ int rc; + u8 saved_iDb = db->init.iDb; sqlite3_stmt *pStmt; TESTONLY(int rcp); /* Return code from sqlite3_prepare() */ assert( db->init.busy ); db->init.iDb = iDb; - db->init.newTnum = sqlite3Atoi(argv[1]); + if( sqlite3GetUInt32(argv[3], &db->init.newTnum)==0 + || (db->init.newTnum>pData->mxPage && pData->mxPage>0) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + corruptSchema(pData, argv, "invalid rootpage"); + } + } db->init.orphanTrigger = 0; - TESTONLY(rcp = ) sqlite3_prepare(db, argv[2], -1, &pStmt, 0); + db->init.azInit = (const char**)argv; + pStmt = 0; + TESTONLY(rcp = ) sqlite3Prepare(db, argv[4], -1, 0, 0, &pStmt, 0); rc = db->errCode; assert( (rc&0xFF)==(rcp&0xFF) ); - db->init.iDb = 0; + db->init.iDb = saved_iDb; + /* assert( saved_iDb==0 || (db->mDbFlags & DBFLAG_Vacuum)!=0 ); */ if( SQLITE_OK!=rc ){ if( db->init.orphanTrigger ){ assert( iDb==1 ); }else{ - pData->rc = rc; + if( rc > pData->rc ) pData->rc = rc; if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); }else if( rc!=SQLITE_INTERRUPT && (rc&0xFF)!=SQLITE_LOCKED ){ - corruptSchema(pData, argv[0], sqlite3_errmsg(db)); + corruptSchema(pData, argv, sqlite3_errmsg(db)); } } } + db->init.azInit = sqlite3StdType; /* Any array of string ptrs will do */ sqlite3_finalize(pStmt); - }else if( argv[0]==0 || (argv[2]!=0 && argv[2][0]!=0) ){ - corruptSchema(pData, argv[0], 0); + }else if( argv[1]==0 || (argv[4]!=0 && argv[4][0]!=0) ){ + corruptSchema(pData, argv, 0); }else{ /* If the SQL column is blank it means this is an index that ** was created to be the PRIMARY KEY or to fulfill a UNIQUE @@ -112108,16 +142764,18 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** to do here is record the root page number for that index. */ Index *pIndex; - pIndex = sqlite3FindIndex(db, argv[0], db->aDb[iDb].zName); + pIndex = sqlite3FindIndex(db, argv[1], db->aDb[iDb].zDbSName); if( pIndex==0 ){ - /* This can occur if there exists an index on a TEMP table which - ** has the same name as another index on a permanent index. Since - ** the permanent table is hidden by the TEMP table, we can also - ** safely ignore the index on the permanent table. - */ - /* Do Nothing */; - }else if( sqlite3GetInt32(argv[1], &pIndex->tnum)==0 ){ - corruptSchema(pData, argv[0], "invalid rootpage"); + corruptSchema(pData, argv, "orphan index"); + }else + if( sqlite3GetUInt32(argv[3],&pIndex->tnum)==0 + || pIndex->tnum<2 + || pIndex->tnum>pData->mxPage + || sqlite3IndexHasDuplicateRootPage(pIndex) + ){ + if( sqlite3Config.bExtraSchemaChecks ){ + corruptSchema(pData, argv, "invalid rootpage"); + } } } return 0; @@ -112131,39 +142789,49 @@ SQLITE_PRIVATE int sqlite3InitCallback(void *pInit, int argc, char **argv, char ** auxiliary databases. Return one of the SQLITE_ error codes to ** indicate success or failure. */ -static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ +SQLITE_PRIVATE int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg, u32 mFlags){ int rc; int i; #ifndef SQLITE_OMIT_DEPRECATED int size; #endif Db *pDb; - char const *azArg[4]; + char const *azArg[6]; int meta[5]; InitData initData; - const char *zMasterName; + const char *zSchemaTabName; int openedTransaction = 0; + int mask = ((db->mDbFlags & DBFLAG_EncodingFixed) | ~DBFLAG_EncodingFixed); + assert( (db->mDbFlags & DBFLAG_SchemaKnownOk)==0 ); assert( iDb>=0 && iDbnDb ); assert( db->aDb[iDb].pSchema ); assert( sqlite3_mutex_held(db->mutex) ); assert( iDb==1 || sqlite3BtreeHoldsMutex(db->aDb[iDb].pBt) ); - /* Construct the in-memory representation schema tables (sqlite_master or - ** sqlite_temp_master) by invoking the parser directly. The appropriate + db->init.busy = 1; + + /* Construct the in-memory representation schema tables (sqlite_schema or + ** sqlite_temp_schema) by invoking the parser directly. The appropriate ** table name will be inserted automatically by the parser so we can just ** use the abbreviation "x" here. The parser will also automatically tag ** the schema table as read-only. */ - azArg[0] = zMasterName = SCHEMA_TABLE(iDb); - azArg[1] = "1"; - azArg[2] = "CREATE TABLE x(type text,name text,tbl_name text," - "rootpage integer,sql text)"; - azArg[3] = 0; + azArg[0] = "table"; + azArg[1] = zSchemaTabName = SCHEMA_TABLE(iDb); + azArg[2] = azArg[1]; + azArg[3] = "1"; + azArg[4] = "CREATE TABLE x(type text,name text,tbl_name text," + "rootpage int,sql text)"; + azArg[5] = 0; initData.db = db; initData.iDb = iDb; initData.rc = SQLITE_OK; initData.pzErrMsg = pzErrMsg; - sqlite3InitCallback(&initData, 3, (char **)azArg, 0); + initData.mInitFlags = mFlags; + initData.nInitRow = 0; + initData.mxPage = 0; + sqlite3InitCallback(&initData, 5, (char **)azArg, 0); + db->mDbFlags &= mask; if( initData.rc ){ rc = initData.rc; goto error_out; @@ -112173,18 +142841,18 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ */ pDb = &db->aDb[iDb]; if( pDb->pBt==0 ){ - if( !OMIT_TEMPDB && ALWAYS(iDb==1) ){ - DbSetProperty(db, 1, DB_SchemaLoaded); - } - return SQLITE_OK; + assert( iDb==1 ); + DbSetProperty(db, 1, DB_SchemaLoaded); + rc = SQLITE_OK; + goto error_out; } /* If there is not already a read-only (or read-write) transaction opened - ** on the b-tree database, open one now. If a transaction is opened, it + ** on the b-tree database, open one now. If a transaction is opened, it ** will be closed before this function returns. */ sqlite3BtreeEnter(pDb->pBt); - if( !sqlite3BtreeIsInReadTrans(pDb->pBt) ){ - rc = sqlite3BtreeBeginTrans(pDb->pBt, 0); + if( sqlite3BtreeTxnState(pDb->pBt)==SQLITE_TXN_NONE ){ + rc = sqlite3BtreeBeginTrans(pDb->pBt, 0, 0); if( rc!=SQLITE_OK ){ sqlite3SetString(pzErrMsg, db, sqlite3ErrStr(rc)); goto initone_error_out; @@ -112212,6 +142880,9 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ for(i=0; ipBt, i+1, (u32 *)&meta[i]); } + if( (db->flags & SQLITE_ResetDatabase)!=0 ){ + memset(meta, 0, sizeof(meta)); + } pDb->pSchema->schema_cookie = meta[BTREE_SCHEMA_VERSION-1]; /* If opening a non-empty database, check the text encoding. For the @@ -112220,27 +142891,32 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** as sqlite3.enc. */ if( meta[BTREE_TEXT_ENCODING-1] ){ /* text encoding */ - if( iDb==0 ){ -#ifndef SQLITE_OMIT_UTF16 + if( iDb==0 && (db->mDbFlags & DBFLAG_EncodingFixed)==0 ){ u8 encoding; +#ifndef SQLITE_OMIT_UTF16 /* If opening the main database, set ENC(db). */ encoding = (u8)meta[BTREE_TEXT_ENCODING-1] & 3; if( encoding==0 ) encoding = SQLITE_UTF8; - ENC(db) = encoding; #else - ENC(db) = SQLITE_UTF8; + encoding = SQLITE_UTF8; #endif + if( db->nVdbeActive>0 && encoding!=ENC(db) + && (db->mDbFlags & DBFLAG_Vacuum)==0 + ){ + rc = SQLITE_LOCKED; + goto initone_error_out; + }else{ + sqlite3SetTextEncoding(db, encoding); + } }else{ /* If opening an attached database, the encoding much match ENC(db) */ - if( meta[BTREE_TEXT_ENCODING-1]!=ENC(db) ){ + if( (meta[BTREE_TEXT_ENCODING-1] & 3)!=ENC(db) ){ sqlite3SetString(pzErrMsg, db, "attached databases must use the same" " text encoding as main database"); rc = SQLITE_ERROR; goto initone_error_out; } } - }else{ - DbSetProperty(db, iDb, DB_Empty); } pDb->pSchema->enc = ENC(db); @@ -112277,17 +142953,18 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ ** indices that the user might have created. */ if( iDb==0 && meta[BTREE_FILE_FORMAT-1]>=4 ){ - db->flags &= ~SQLITE_LegacyFileFmt; + db->flags &= ~(u64)SQLITE_LegacyFileFmt; } /* Read the schema information out of the schema tables */ assert( db->init.busy ); + initData.mxPage = sqlite3BtreeLastPage(pDb->pBt); { char *zSql; - zSql = sqlite3MPrintf(db, - "SELECT name, rootpage, sql FROM \"%w\".%s ORDER BY rowid", - db->aDb[iDb].zName, zMasterName); + zSql = sqlite3MPrintf(db, + "SELECT*FROM\"%w\".%s ORDER BY rowid", + db->aDb[iDb].zDbSName, zSchemaTabName); #ifndef SQLITE_OMIT_AUTHORIZATION { sqlite3_xauth xAuth; @@ -112307,18 +142984,22 @@ static int sqlite3InitOne(sqlite3 *db, int iDb, char **pzErrMsg){ } #endif } + assert( pDb == &(db->aDb[iDb]) ); if( db->mallocFailed ){ rc = SQLITE_NOMEM_BKPT; sqlite3ResetAllSchemasOfConnection(db); - } - if( rc==SQLITE_OK || (db->flags&SQLITE_RecoveryMode)){ - /* Black magic: If the SQLITE_RecoveryMode flag is set, then consider - ** the schema loaded, even if errors occurred. In this situation the - ** current sqlite3_prepare() operation will fail, but the following one - ** will attempt to compile the supplied statement against whatever subset - ** of the schema was loaded before the error occurred. The primary - ** purpose of this is to allow access to the sqlite_master table - ** even when its contents have been corrupted. + pDb = &db->aDb[iDb]; + }else + if( rc==SQLITE_OK || ((db->flags&SQLITE_NoSchemaError) && rc!=SQLITE_NOMEM)){ + /* Hack: If the SQLITE_NoSchemaError flag is set, then consider + ** the schema loaded, even if errors (other than OOM) occurred. In + ** this situation the current sqlite3_prepare() operation will fail, + ** but the following one will attempt to compile the supplied statement + ** against whatever subset of the schema was loaded before the error + ** occurred. + ** + ** The primary purpose of this is to allow access to the sqlite_schema + ** table even when its contents have been corrupted. */ DbSetProperty(db, iDb, DB_SchemaLoaded); rc = SQLITE_OK; @@ -112335,9 +143016,13 @@ initone_error_out: sqlite3BtreeLeave(pDb->pBt); error_out: - if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ - sqlite3OomFault(db); + if( rc ){ + if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ + sqlite3OomFault(db); + } + sqlite3ResetOneSchema(db, iDb); } + db->init.busy = 0; return rc; } @@ -112348,47 +143033,34 @@ error_out: ** error occurs, write an error message into *pzErrMsg. ** ** After a database is initialized, the DB_SchemaLoaded bit is set -** bit is set in the flags field of the Db structure. If the database -** file was of zero-length, then the DB_Empty flag is also set. +** bit is set in the flags field of the Db structure. */ SQLITE_PRIVATE int sqlite3Init(sqlite3 *db, char **pzErrMsg){ int i, rc; - int commit_internal = !(db->flags&SQLITE_InternChanges); - + int commit_internal = !(db->mDbFlags&DBFLAG_SchemaChange); + assert( sqlite3_mutex_held(db->mutex) ); assert( sqlite3BtreeHoldsMutex(db->aDb[0].pBt) ); assert( db->init.busy==0 ); - rc = SQLITE_OK; - db->init.busy = 1; ENC(db) = SCHEMA_ENC(db); - for(i=0; rc==SQLITE_OK && inDb; i++){ - if( DbHasProperty(db, i, DB_SchemaLoaded) || i==1 ) continue; - rc = sqlite3InitOne(db, i, pzErrMsg); - if( rc ){ - sqlite3ResetOneSchema(db, i); + assert( db->nDb>0 ); + /* Do the main schema first */ + if( !DbHasProperty(db, 0, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, 0, pzErrMsg, 0); + if( rc ) return rc; + } + /* All other schemas after the main schema. The "temp" schema must be last */ + for(i=db->nDb-1; i>0; i--){ + assert( i==1 || sqlite3BtreeHoldsMutex(db->aDb[i].pBt) ); + if( !DbHasProperty(db, i, DB_SchemaLoaded) ){ + rc = sqlite3InitOne(db, i, pzErrMsg, 0); + if( rc ) return rc; } } - - /* Once all the other databases have been initialized, load the schema - ** for the TEMP database. This is loaded last, as the TEMP database - ** schema may contain references to objects in other databases. - */ -#ifndef SQLITE_OMIT_TEMPDB - assert( db->nDb>1 ); - if( rc==SQLITE_OK && !DbHasProperty(db, 1, DB_SchemaLoaded) ){ - rc = sqlite3InitOne(db, 1, pzErrMsg); - if( rc ){ - sqlite3ResetOneSchema(db, 1); - } - } -#endif - - db->init.busy = 0; - if( rc==SQLITE_OK && commit_internal ){ + if( commit_internal ){ sqlite3CommitInternalChanges(db); } - - return rc; + return SQLITE_OK; } /* @@ -112401,10 +143073,12 @@ SQLITE_PRIVATE int sqlite3ReadSchema(Parse *pParse){ assert( sqlite3_mutex_held(db->mutex) ); if( !db->init.busy ){ rc = sqlite3Init(db, &pParse->zErrMsg); - } - if( rc!=SQLITE_OK ){ - pParse->rc = rc; - pParse->nErr++; + if( rc!=SQLITE_OK ){ + pParse->rc = rc; + pParse->nErr++; + }else if( db->noSharedCache ){ + db->mDbFlags |= DBFLAG_SchemaKnownOk; + } } return rc; } @@ -112429,25 +143103,26 @@ static void schemaIsValid(Parse *pParse){ if( pBt==0 ) continue; /* If there is not already a read-only (or read-write) transaction opened - ** on the b-tree database, open one now. If a transaction is opened, it + ** on the b-tree database, open one now. If a transaction is opened, it ** will be closed immediately after reading the meta-value. */ - if( !sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); + if( sqlite3BtreeTxnState(pBt)==SQLITE_TXN_NONE ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); if( rc==SQLITE_NOMEM || rc==SQLITE_IOERR_NOMEM ){ sqlite3OomFault(db); + pParse->rc = SQLITE_NOMEM; } if( rc!=SQLITE_OK ) return; openedTransaction = 1; } - /* Read the schema cookie from the database. If it does not match the + /* Read the schema cookie from the database. If it does not match the ** value stored as part of the in-memory schema representation, ** set Parse.rc to SQLITE_SCHEMA. */ sqlite3BtreeGetMeta(pBt, BTREE_SCHEMA_VERSION, (u32 *)&cookie); assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); if( cookie!=db->aDb[iDb].pSchema->schema_cookie ){ + if( DbHasProperty(db, iDb, DB_SchemaLoaded) ) pParse->rc = SQLITE_SCHEMA; sqlite3ResetOneSchema(db, iDb); - pParse->rc = SQLITE_SCHEMA; } /* Close the transaction, if one was opened. */ @@ -112465,21 +143140,23 @@ static void schemaIsValid(Parse *pParse){ ** attached database is returned. */ SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ - int i = -1000000; + int i = -32768; - /* If pSchema is NULL, then return -1000000. This happens when code in + /* If pSchema is NULL, then return -32768. This happens when code in ** expr.c is trying to resolve a reference to a transient table (i.e. one - ** created by a sub-select). In this case the return value of this + ** created by a sub-select). In this case the return value of this ** function should never be used. ** - ** We return -1000000 instead of the more usual -1 simply because using - ** -1000000 as the incorrect index into db->aDb[] is much + ** We return -32768 instead of the more usual -1 simply because using + ** -32768 as the incorrect index into db->aDb[] is much ** more likely to cause a segfault than -1 (of course there are assert() - ** statements too, but it never hurts to play the odds). + ** statements too, but it never hurts to play the odds) and + ** -32768 will still fit into a 16-bit signed integer. */ assert( sqlite3_mutex_held(db->mutex) ); if( pSchema ){ - for(i=0; ALWAYS(inDb); i++){ + for(i=0; 1; i++){ + assert( inDb ); if( db->aDb[i].pSchema==pSchema ){ break; } @@ -112492,19 +143169,113 @@ SQLITE_PRIVATE int sqlite3SchemaToIndex(sqlite3 *db, Schema *pSchema){ /* ** Free all memory allocations in the pParse object */ -SQLITE_PRIVATE void sqlite3ParserReset(Parse *pParse){ - if( pParse ){ - sqlite3 *db = pParse->db; - sqlite3DbFree(db, pParse->aLabel); - sqlite3ExprListDelete(db, pParse->pConstExpr); - if( db ){ - assert( db->lookaside.bDisable >= pParse->disableLookaside ); - db->lookaside.bDisable -= pParse->disableLookaside; - } - pParse->disableLookaside = 0; +SQLITE_PRIVATE void sqlite3ParseObjectReset(Parse *pParse){ + sqlite3 *db = pParse->db; + assert( db!=0 ); + assert( db->pParse==pParse ); + assert( pParse->nested==0 ); +#ifndef SQLITE_OMIT_SHARED_CACHE + if( pParse->aTableLock ) sqlite3DbNNFreeNN(db, pParse->aTableLock); +#endif + while( pParse->pCleanup ){ + ParseCleanup *pCleanup = pParse->pCleanup; + pParse->pCleanup = pCleanup->pNext; + pCleanup->xCleanup(db, pCleanup->pPtr); + sqlite3DbNNFreeNN(db, pCleanup); } + if( pParse->aLabel ) sqlite3DbNNFreeNN(db, pParse->aLabel); + if( pParse->pConstExpr ){ + sqlite3ExprListDelete(db, pParse->pConstExpr); + } + assert( db->lookaside.bDisable >= pParse->disableLookaside ); + db->lookaside.bDisable -= pParse->disableLookaside; + db->lookaside.sz = db->lookaside.bDisable ? 0 : db->lookaside.szTrue; + assert( pParse->db->pParse==pParse ); + db->pParse = pParse->pOuterParse; } +/* +** Add a new cleanup operation to a Parser. The cleanup should happen when +** the parser object is destroyed. But, beware: the cleanup might happen +** immediately. +** +** Use this mechanism for uncommon cleanups. There is a higher setup +** cost for this mechanism (an extra malloc), so it should not be used +** for common cleanups that happen on most calls. But for less +** common cleanups, we save a single NULL-pointer comparison in +** sqlite3ParseObjectReset(), which reduces the total CPU cycle count. +** +** If a memory allocation error occurs, then the cleanup happens immediately. +** When either SQLITE_DEBUG or SQLITE_COVERAGE_TEST are defined, the +** pParse->earlyCleanup flag is set in that case. Calling code show verify +** that test cases exist for which this happens, to guard against possible +** use-after-free errors following an OOM. The preferred way to do this is +** to immediately follow the call to this routine with: +** +** testcase( pParse->earlyCleanup ); +** +** This routine returns a copy of its pPtr input (the third parameter) +** except if an early cleanup occurs, in which case it returns NULL. So +** another way to check for early cleanup is to check the return value. +** Or, stop using the pPtr parameter with this call and use only its +** return value thereafter. Something like this: +** +** pObj = sqlite3ParserAddCleanup(pParse, destructor, pObj); +*/ +SQLITE_PRIVATE void *sqlite3ParserAddCleanup( + Parse *pParse, /* Destroy when this Parser finishes */ + void (*xCleanup)(sqlite3*,void*), /* The cleanup routine */ + void *pPtr /* Pointer to object to be cleaned up */ +){ + ParseCleanup *pCleanup; + if( sqlite3FaultSim(300) ){ + pCleanup = 0; + sqlite3OomFault(pParse->db); + }else{ + pCleanup = sqlite3DbMallocRaw(pParse->db, sizeof(*pCleanup)); + } + if( pCleanup ){ + pCleanup->pNext = pParse->pCleanup; + pParse->pCleanup = pCleanup; + pCleanup->pPtr = pPtr; + pCleanup->xCleanup = xCleanup; + }else{ + xCleanup(pParse->db, pPtr); + pPtr = 0; +#if defined(SQLITE_DEBUG) || defined(SQLITE_COVERAGE_TEST) + pParse->earlyCleanup = 1; +#endif + } + return pPtr; +} + +/* +** Turn bulk memory into a valid Parse object and link that Parse object +** into database connection db. +** +** Call sqlite3ParseObjectReset() to undo this operation. +** +** Caution: Do not confuse this routine with sqlite3ParseObjectInit() which +** is generated by Lemon. +*/ +SQLITE_PRIVATE void sqlite3ParseObjectInit(Parse *pParse, sqlite3 *db){ + memset(PARSE_HDR(pParse), 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(pParse), 0, PARSE_TAIL_SZ); + assert( db->pParse!=pParse ); + pParse->pOuterParse = db->pParse; + db->pParse = pParse; + pParse->db = db; + if( db->mallocFailed ) sqlite3ErrorMsg(pParse, "out of memory"); +} + +/* +** Maximum number of times that we will try again to prepare a statement +** that returns SQLITE_ERROR_RETRY. +*/ +#ifndef SQLITE_MAX_PREPARE_RETRY +# define SQLITE_MAX_PREPARE_RETRY 25 +#endif + /* ** Compile the UTF-8 encoded SQL statement zSql into a statement handle. */ @@ -112512,27 +143283,44 @@ static int sqlite3Prepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ - int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ Vdbe *pReprepare, /* VM being reprepared */ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ - Parse *pParse; /* Parsing context */ - char *zErrMsg = 0; /* Error message */ int rc = SQLITE_OK; /* Result code */ int i; /* Loop counter */ + Parse sParse; /* Parsing context */ - /* Allocate the parsing context */ - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM_BKPT; + /* sqlite3ParseObjectInit(&sParse, db); // inlined for performance */ + memset(PARSE_HDR(&sParse), 0, PARSE_HDR_SZ); + memset(PARSE_TAIL(&sParse), 0, PARSE_TAIL_SZ); + sParse.pOuterParse = db->pParse; + db->pParse = &sParse; + sParse.db = db; + if( pReprepare ){ + sParse.pReprepare = pReprepare; + sParse.explain = sqlite3_stmt_isexplain((sqlite3_stmt*)pReprepare); + }else{ + assert( sParse.pReprepare==0 ); + } + assert( ppStmt && *ppStmt==0 ); + if( db->mallocFailed ){ + sqlite3ErrorMsg(&sParse, "out of memory"); + db->errCode = rc = SQLITE_NOMEM; goto end_prepare; } - pParse->pReprepare = pReprepare; - assert( ppStmt && *ppStmt==0 ); - /* assert( !db->mallocFailed ); // not true with SQLITE_USE_ALLOCA */ assert( sqlite3_mutex_held(db->mutex) ); + /* For a long-term use prepared statement avoid the use of + ** lookaside memory. + */ + if( prepFlags & SQLITE_PREPARE_PERSISTENT ){ + sParse.disableLookaside++; + DisableLookaside; + } + sParse.prepFlags = prepFlags & 0xff; + /* Check to verify that it is possible to get a read lock on all ** database schemas. The inability to get a read lock indicates that ** some other database connection is holding a write-lock, which in @@ -112548,32 +143336,34 @@ static int sqlite3Prepare( ** This thread is currently holding mutexes on all Btrees (because ** of the sqlite3BtreeEnterAll() in sqlite3LockAndPrepare()) so it ** is not possible for another thread to start a new schema change - ** while this routine is running. Hence, we do not need to hold - ** locks on the schema, we just need to make sure nobody else is + ** while this routine is running. Hence, we do not need to hold + ** locks on the schema, we just need to make sure nobody else is ** holding them. ** ** Note that setting READ_UNCOMMITTED overrides most lock detection, ** but it does *not* override schema lock detection, so this all still ** works even if READ_UNCOMMITTED is set. */ - for(i=0; inDb; i++) { - Btree *pBt = db->aDb[i].pBt; - if( pBt ){ - assert( sqlite3BtreeHoldsMutex(pBt) ); - rc = sqlite3BtreeSchemaLocked(pBt); - if( rc ){ - const char *zDb = db->aDb[i].zName; - sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); - testcase( db->flags & SQLITE_ReadUncommitted ); - goto end_prepare; + if( !db->noSharedCache ){ + for(i=0; inDb; i++) { + Btree *pBt = db->aDb[i].pBt; + if( pBt ){ + assert( sqlite3BtreeHoldsMutex(pBt) ); + rc = sqlite3BtreeSchemaLocked(pBt); + if( rc ){ + const char *zDb = db->aDb[i].zDbSName; + sqlite3ErrorWithMsg(db, rc, "database schema is locked: %s", zDb); + testcase( db->flags & SQLITE_ReadUncommit ); + goto end_prepare; + } } } } - sqlite3VtabUnlockList(db); +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( db->pDisconnect ) sqlite3VtabUnlockList(db); +#endif - pParse->db = db; - pParse->nQueryLoop = 0; /* Logarithmic, so 0 really means 1 */ if( nBytes>=0 && (nBytes==0 || zSql[nBytes-1]!=0) ){ char *zSqlCopy; int mxLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; @@ -112586,95 +143376,74 @@ static int sqlite3Prepare( } zSqlCopy = sqlite3DbStrNDup(db, zSql, nBytes); if( zSqlCopy ){ - sqlite3RunParser(pParse, zSqlCopy, &zErrMsg); - pParse->zTail = &zSql[pParse->zTail-zSqlCopy]; + sqlite3RunParser(&sParse, zSqlCopy); + sParse.zTail = &zSql[sParse.zTail-zSqlCopy]; sqlite3DbFree(db, zSqlCopy); }else{ - pParse->zTail = &zSql[nBytes]; + sParse.zTail = &zSql[nBytes]; } }else{ - sqlite3RunParser(pParse, zSql, &zErrMsg); + sqlite3RunParser(&sParse, zSql); } - assert( 0==pParse->nQueryLoop ); + assert( 0==sParse.nQueryLoop ); - if( pParse->rc==SQLITE_DONE ) pParse->rc = SQLITE_OK; - if( pParse->checkSchema ){ - schemaIsValid(pParse); - } - if( db->mallocFailed ){ - pParse->rc = SQLITE_NOMEM_BKPT; - } if( pzTail ){ - *pzTail = pParse->zTail; + *pzTail = sParse.zTail; } - rc = pParse->rc; - -#ifndef SQLITE_OMIT_EXPLAIN - if( rc==SQLITE_OK && pParse->pVdbe && pParse->explain ){ - static const char * const azColName[] = { - "addr", "opcode", "p1", "p2", "p3", "p4", "p5", "comment", - "selectid", "order", "from", "detail" - }; - int iFirst, mx; - if( pParse->explain==2 ){ - sqlite3VdbeSetNumCols(pParse->pVdbe, 4); - iFirst = 8; - mx = 12; - }else{ - sqlite3VdbeSetNumCols(pParse->pVdbe, 8); - iFirst = 0; - mx = 8; - } - for(i=iFirst; ipVdbe, i-iFirst, COLNAME_NAME, - azColName[i], SQLITE_STATIC); - } - } -#endif if( db->init.busy==0 ){ - Vdbe *pVdbe = pParse->pVdbe; - sqlite3VdbeSetSql(pVdbe, zSql, (int)(pParse->zTail-zSql), saveSqlFlag); + sqlite3VdbeSetSql(sParse.pVdbe, zSql, (int)(sParse.zTail-zSql), prepFlags); } - if( pParse->pVdbe && (rc!=SQLITE_OK || db->mallocFailed) ){ - sqlite3VdbeFinalize(pParse->pVdbe); - assert(!(*ppStmt)); + if( db->mallocFailed ){ + sParse.rc = SQLITE_NOMEM_BKPT; + sParse.checkSchema = 0; + } + if( sParse.rc!=SQLITE_OK && sParse.rc!=SQLITE_DONE ){ + if( sParse.checkSchema && db->init.busy==0 ){ + schemaIsValid(&sParse); + } + if( sParse.pVdbe ){ + sqlite3VdbeFinalize(sParse.pVdbe); + } + assert( 0==(*ppStmt) ); + rc = sParse.rc; + if( sParse.zErrMsg ){ + sqlite3ErrorWithMsg(db, rc, "%s", sParse.zErrMsg); + sqlite3DbFree(db, sParse.zErrMsg); + }else{ + sqlite3Error(db, rc); + } }else{ - *ppStmt = (sqlite3_stmt*)pParse->pVdbe; + assert( sParse.zErrMsg==0 ); + *ppStmt = (sqlite3_stmt*)sParse.pVdbe; + rc = SQLITE_OK; + sqlite3ErrorClear(db); } - if( zErrMsg ){ - sqlite3ErrorWithMsg(db, rc, "%s", zErrMsg); - sqlite3DbFree(db, zErrMsg); - }else{ - sqlite3Error(db, rc); - } /* Delete any TriggerPrg structures allocated while parsing this statement. */ - while( pParse->pTriggerPrg ){ - TriggerPrg *pT = pParse->pTriggerPrg; - pParse->pTriggerPrg = pT->pNext; + while( sParse.pTriggerPrg ){ + TriggerPrg *pT = sParse.pTriggerPrg; + sParse.pTriggerPrg = pT->pNext; sqlite3DbFree(db, pT); } end_prepare: - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); - rc = sqlite3ApiExit(db, rc); - assert( (rc&db->errMask)==rc ); + sqlite3ParseObjectReset(&sParse); return rc; } static int sqlite3LockAndPrepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ - int saveSqlFlag, /* True to copy SQL text into the sqlite3_stmt */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ Vdbe *pOld, /* VM being reprepared */ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const char **pzTail /* OUT: End of parsed string */ ){ int rc; + int cnt = 0; #ifdef SQLITE_ENABLE_API_ARMOR if( ppStmt==0 ) return SQLITE_MISUSE_BKPT; @@ -112685,23 +143454,31 @@ static int sqlite3LockAndPrepare( } sqlite3_mutex_enter(db->mutex); sqlite3BtreeEnterAll(db); - rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); - if( rc==SQLITE_SCHEMA ){ - sqlite3_finalize(*ppStmt); - rc = sqlite3Prepare(db, zSql, nBytes, saveSqlFlag, pOld, ppStmt, pzTail); - } + do{ + /* Make multiple attempts to compile the SQL, until it either succeeds + ** or encounters a permanent error. A schema problem after one schema + ** reset is considered a permanent error. */ + rc = sqlite3Prepare(db, zSql, nBytes, prepFlags, pOld, ppStmt, pzTail); + assert( rc==SQLITE_OK || *ppStmt==0 ); + if( rc==SQLITE_OK || db->mallocFailed ) break; + }while( (rc==SQLITE_ERROR_RETRY && (cnt++)errMask)==rc ); + db->busyHandler.nBusy = 0; sqlite3_mutex_leave(db->mutex); - assert( rc==SQLITE_OK || *ppStmt==0 ); + assert( rc==SQLITE_OK || (*ppStmt)==0 ); return rc; } + /* ** Rerun the compilation of a statement after a schema change. ** ** If the statement is successfully recompiled, return SQLITE_OK. Otherwise, ** if the statement cannot be recompiled because another connection has -** locked the sqlite3_master table, return SQLITE_LOCKED. If any other error +** locked the sqlite3_schema table, return SQLITE_LOCKED. If any other error ** occurs, return SQLITE_SCHEMA. */ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ @@ -112709,13 +143486,15 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ sqlite3_stmt *pNew; const char *zSql; sqlite3 *db; + u8 prepFlags; assert( sqlite3_mutex_held(sqlite3VdbeDb(p)->mutex) ); zSql = sqlite3_sql((sqlite3_stmt *)p); assert( zSql!=0 ); /* Reprepare only called for prepare_v2() statements */ db = sqlite3VdbeDb(p); assert( sqlite3_mutex_held(db->mutex) ); - rc = sqlite3LockAndPrepare(db, zSql, -1, 0, p, &pNew, 0); + prepFlags = sqlite3VdbePrepareFlags(p); + rc = sqlite3LockAndPrepare(db, zSql, -1, prepFlags, p, &pNew, 0); if( rc ){ if( rc==SQLITE_NOMEM ){ sqlite3OomFault(db); @@ -112741,7 +143520,7 @@ SQLITE_PRIVATE int sqlite3Reprepare(Vdbe *p){ ** and the statement is automatically recompiled if an schema change ** occurs. */ -SQLITE_API int SQLITE_STDCALL sqlite3_prepare( +SQLITE_API int sqlite3_prepare( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -112753,7 +143532,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare( assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( +SQLITE_API int sqlite3_prepare_v2( sqlite3 *db, /* Database handle. */ const char *zSql, /* UTF-8 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ @@ -112761,8 +143540,36 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( const char **pzTail /* OUT: End of parsed string */ ){ int rc; - rc = sqlite3LockAndPrepare(db,zSql,nBytes,1,0,ppStmt,pzTail); - assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + /* EVIDENCE-OF: R-37923-12173 The sqlite3_prepare_v2() interface works + ** exactly the same as sqlite3_prepare_v3() with a zero prepFlags + ** parameter. + ** + ** Proof in that the 5th parameter to sqlite3LockAndPrepare is 0 */ + rc = sqlite3LockAndPrepare(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,0, + ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); + return rc; +} +SQLITE_API int sqlite3_prepare_v3( + sqlite3 *db, /* Database handle. */ + const char *zSql, /* UTF-8 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const char **pzTail /* OUT: End of parsed string */ +){ + int rc; + /* EVIDENCE-OF: R-56861-42673 sqlite3_prepare_v3() differs from + ** sqlite3_prepare_v2() only in having the extra prepFlags parameter, + ** which is a bit array consisting of zero or more of the + ** SQLITE_PREPARE_* flags. + ** + ** Proof by comparison to the implementation of sqlite3_prepare_v2() + ** directly above. */ + rc = sqlite3LockAndPrepare(db,zSql,nBytes, + SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), + 0,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); return rc; } @@ -112772,10 +143579,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare_v2( ** Compile the UTF-16 encoded SQL statement zSql into a statement handle. */ static int sqlite3Prepare16( - sqlite3 *db, /* Database handle. */ + sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ - int saveSqlFlag, /* True to save SQL text into the sqlite3_stmt */ + u32 prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const void **pzTail /* OUT: End of parsed string */ ){ @@ -112794,16 +143601,28 @@ static int sqlite3Prepare16( if( !sqlite3SafetyCheckOk(db)||zSql==0 ){ return SQLITE_MISUSE_BKPT; } + + /* Make sure nBytes is non-negative and correct. It should be the + ** number of bytes until the end of the input buffer or until the first + ** U+0000 character. If the input nBytes is odd, convert it into + ** an even number. If the input nBytes is negative, then the input + ** must be terminated by at least one U+0000 character */ if( nBytes>=0 ){ int sz; const char *z = (const char*)zSql; for(sz=0; szmutex); zSql8 = sqlite3Utf16to8(db, zSql, nBytes, SQLITE_UTF16NATIVE); if( zSql8 ){ - rc = sqlite3LockAndPrepare(db, zSql8, -1, saveSqlFlag, 0, ppStmt, &zTail8); + rc = sqlite3LockAndPrepare(db, zSql8, -1, prepFlags, 0, ppStmt, &zTail8); } if( zTail8 && pzTail ){ @@ -112813,9 +143632,9 @@ static int sqlite3Prepare16( ** the same number of characters into the UTF-16 string. */ int chars_parsed = sqlite3Utf8CharLen(zSql8, (int)(zTail8-zSql8)); - *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, chars_parsed); + *pzTail = (u8 *)zSql + sqlite3Utf16ByteLen(zSql, nBytes, chars_parsed); } - sqlite3DbFree(db, zSql8); + sqlite3DbFree(db, zSql8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); return rc; @@ -112829,8 +143648,8 @@ static int sqlite3Prepare16( ** and the statement is automatically recompiled if an schema change ** occurs. */ -SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( - sqlite3 *db, /* Database handle. */ +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ @@ -112841,15 +143660,30 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare16( assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( - sqlite3 *db, /* Database handle. */ +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle. */ const void *zSql, /* UTF-16 encoded SQL statement. */ int nBytes, /* Length of zSql in bytes. */ sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ const void **pzTail /* OUT: End of parsed string */ ){ int rc; - rc = sqlite3Prepare16(db,zSql,nBytes,1,ppStmt,pzTail); + rc = sqlite3Prepare16(db,zSql,nBytes,SQLITE_PREPARE_SAVESQL,ppStmt,pzTail); + assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ + return rc; +} +SQLITE_API int sqlite3_prepare16_v3( + sqlite3 *db, /* Database handle. */ + const void *zSql, /* UTF-16 encoded SQL statement. */ + int nBytes, /* Length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_* flags */ + sqlite3_stmt **ppStmt, /* OUT: A pointer to the prepared statement */ + const void **pzTail /* OUT: End of parsed string */ +){ + int rc; + rc = sqlite3Prepare16(db,zSql,nBytes, + SQLITE_PREPARE_SAVESQL|(prepFlags&SQLITE_PREPARE_MASK), + ppStmt,pzTail); assert( rc==SQLITE_OK || ppStmt==0 || *ppStmt==0 ); /* VERIFY: F13021 */ return rc; } @@ -112874,21 +143708,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( */ /* #include "sqliteInt.h" */ -/* -** Trace output macros -*/ -#if SELECTTRACE_ENABLED -/***/ int sqlite3SelectTrace = 0; -# define SELECTTRACE(K,P,S,X) \ - if(sqlite3SelectTrace&(K)) \ - sqlite3DebugPrintf("%*s%s.%p: ",(P)->nSelectIndent*2-2,"",\ - (S)->zSelName,(S)),\ - sqlite3DebugPrintf X -#else -# define SELECTTRACE(K,P,S,X) -#endif - - /* ** An instance of the following object is used to record information about ** how to process the DISTINCT keyword, to simplify passing that information @@ -112896,7 +143715,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_prepare16_v2( */ typedef struct DistinctCtx DistinctCtx; struct DistinctCtx { - u8 isTnct; /* True if the DISTINCT keyword is present */ + u8 isTnct; /* 0: Not distinct. 1: DISTICT 2: DISTINCT and ORDER BY */ u8 eTnctType; /* One of the WHERE_DISTINCT_* operators */ int tabTnct; /* Ephemeral table used for DISTINCT processing */ int addrTnct; /* Address of OP_OpenEphemeral opcode for tabTnct */ @@ -112905,6 +143724,20 @@ struct DistinctCtx { /* ** An instance of the following object is used to record information about ** the ORDER BY (or GROUP BY) clause of query is being coded. +** +** The aDefer[] array is used by the sorter-references optimization. For +** example, assuming there is no index that can be used for the ORDER BY, +** for the query: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10; +** +** it may be more efficient to add just the "a" values to the sorter, and +** retrieve the associated "bigblob" values directly from table t1 as the +** 10 smallest "a" values are extracted from the sorter. +** +** When the sorter-reference optimization is used, there is one entry in the +** aDefer[] array for each database table that may be read as values are +** extracted from the sorter. */ typedef struct SortCtx SortCtx; struct SortCtx { @@ -112915,15 +143748,33 @@ struct SortCtx { int labelBkOut; /* Start label for the block-output subroutine */ int addrSortIndex; /* Address of the OP_SorterOpen or OP_OpenEphemeral */ int labelDone; /* Jump here when done, ex: LIMIT reached */ + int labelOBLopt; /* Jump here when sorter is full */ u8 sortFlags; /* Zero or more SORTFLAG_* bits */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + u8 nDefer; /* Number of valid entries in aDefer[] */ + struct DeferredCsr { + Table *pTab; /* Table definition */ + int iCsr; /* Cursor number for table */ + int nKey; /* Number of PK columns for table pTab (>=1) */ + } aDefer[4]; +#endif + struct RowLoadInfo *pDeferredRowLoad; /* Deferred row loading info or NULL */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrPush; /* First instruction to push data into sorter */ + int addrPushEnd; /* Last instruction that pushes data into sorter */ +#endif }; #define SORTFLAG_UseSorter 0x01 /* Use SorterOpen instead of OpenEphemeral */ /* ** Delete all the content of a Select structure. Deallocate the structure -** itself only if bFree is true. +** itself depending on the value of bFree +** +** If bFree==1, call sqlite3DbFree() on the p object. +** If bFree==0, Leave the first Select object unfreed */ static void clearSelect(sqlite3 *db, Select *p, int bFree){ + assert( db!=0 ); while( p ){ Select *pPrior = p->pPrior; sqlite3ExprListDelete(db, p->pEList); @@ -112933,9 +143784,17 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ sqlite3ExprDelete(db, p->pHaving); sqlite3ExprListDelete(db, p->pOrderBy); sqlite3ExprDelete(db, p->pLimit); - sqlite3ExprDelete(db, p->pOffset); - if( p->pWith ) sqlite3WithDelete(db, p->pWith); - if( bFree ) sqlite3DbFree(db, p); + if( OK_IF_ALWAYS_TRUE(p->pWith) ) sqlite3WithDelete(db, p->pWith); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( OK_IF_ALWAYS_TRUE(p->pWinDefn) ){ + sqlite3WindowListDelete(db, p->pWinDefn); + } + while( p->pWin ){ + assert( p->pWin->ppThis==&p->pWin ); + sqlite3WindowUnlinkFromSelect(p->pWin); + } +#endif + if( bFree ) sqlite3DbNNFreeNN(db, p); p = pPrior; bFree = 1; } @@ -112947,7 +143806,8 @@ static void clearSelect(sqlite3 *db, Select *p, int bFree){ SQLITE_PRIVATE void sqlite3SelectDestInit(SelectDest *pDest, int eDest, int iParm){ pDest->eDest = (u8)eDest; pDest->iSDParm = iParm; - pDest->affSdst = 0; + pDest->iSDParm2 = 0; + pDest->zAffSdst = 0; pDest->iSdst = 0; pDest->nSdst = 0; } @@ -112966,32 +143826,29 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( Expr *pHaving, /* the HAVING clause */ ExprList *pOrderBy, /* the ORDER BY clause */ u32 selFlags, /* Flag parameters, such as SF_Distinct */ - Expr *pLimit, /* LIMIT value. NULL means not used */ - Expr *pOffset /* OFFSET value. NULL means no offset */ + Expr *pLimit /* LIMIT value. NULL means not used */ ){ - Select *pNew; + Select *pNew, *pAllocated; Select standin; - sqlite3 *db = pParse->db; - pNew = sqlite3DbMallocRawNN(db, sizeof(*pNew) ); + pAllocated = pNew = sqlite3DbMallocRawNN(pParse->db, sizeof(*pNew) ); if( pNew==0 ){ - assert( db->mallocFailed ); + assert( pParse->db->mallocFailed ); pNew = &standin; } if( pEList==0 ){ - pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db,TK_ASTERISK,0)); + pEList = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(pParse->db,TK_ASTERISK,0)); } pNew->pEList = pEList; pNew->op = TK_SELECT; pNew->selFlags = selFlags; pNew->iLimit = 0; pNew->iOffset = 0; -#if SELECTTRACE_ENABLED - pNew->zSelName[0] = 0; -#endif + pNew->selId = ++pParse->nSelect; pNew->addrOpenEphm[0] = -1; pNew->addrOpenEphm[1] = -1; pNew->nSelectRow = 0; - if( pSrc==0 ) pSrc = sqlite3DbMallocZero(db, sizeof(*pSrc)); + if( pSrc==0 ) pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*pSrc)); pNew->pSrc = pSrc; pNew->pWhere = pWhere; pNew->pGroupBy = pGroupBy; @@ -113000,36 +143857,29 @@ SQLITE_PRIVATE Select *sqlite3SelectNew( pNew->pPrior = 0; pNew->pNext = 0; pNew->pLimit = pLimit; - pNew->pOffset = pOffset; pNew->pWith = 0; - assert( pOffset==0 || pLimit!=0 || pParse->nErr>0 || db->mallocFailed!=0 ); - if( db->mallocFailed ) { - clearSelect(db, pNew, pNew!=&standin); - pNew = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + pNew->pWin = 0; + pNew->pWinDefn = 0; +#endif + if( pParse->db->mallocFailed ) { + clearSelect(pParse->db, pNew, pNew!=&standin); + pAllocated = 0; }else{ assert( pNew->pSrc!=0 || pParse->nErr>0 ); } - assert( pNew!=&standin ); - return pNew; + return pAllocated; } -#if SELECTTRACE_ENABLED -/* -** Set the name of a Select object -*/ -SQLITE_PRIVATE void sqlite3SelectSetName(Select *p, const char *zName){ - if( p && zName ){ - sqlite3_snprintf(sizeof(p->zSelName), p->zSelName, "%s", zName); - } -} -#endif - /* ** Delete the given Select structure and all of its substructures. */ SQLITE_PRIVATE void sqlite3SelectDelete(sqlite3 *db, Select *p){ - if( p ) clearSelect(db, p, 1); + if( OK_IF_ALWAYS_TRUE(p) ) clearSelect(db, p, 1); +} +SQLITE_PRIVATE void sqlite3SelectDeleteGeneric(sqlite3 *db, void *p){ + if( ALWAYS(p) ) clearSelect(db, (Select*)p, 1); } /* @@ -113056,6 +143906,52 @@ static Select *findRightmost(Select *p){ ** ** If an illegal or unsupported join type is seen, then still return ** a join type, but put an error in the pParse structure. +** +** These are the valid join types: +** +** +** pA pB pC Return Value +** ------- ----- ----- ------------ +** CROSS - - JT_CROSS +** INNER - - JT_INNER +** LEFT - - JT_LEFT|JT_OUTER +** LEFT OUTER - JT_LEFT|JT_OUTER +** RIGHT - - JT_RIGHT|JT_OUTER +** RIGHT OUTER - JT_RIGHT|JT_OUTER +** FULL - - JT_LEFT|JT_RIGHT|JT_OUTER +** FULL OUTER - JT_LEFT|JT_RIGHT|JT_OUTER +** NATURAL INNER - JT_NATURAL|JT_INNER +** NATURAL LEFT - JT_NATURAL|JT_LEFT|JT_OUTER +** NATURAL LEFT OUTER JT_NATURAL|JT_LEFT|JT_OUTER +** NATURAL RIGHT - JT_NATURAL|JT_RIGHT|JT_OUTER +** NATURAL RIGHT OUTER JT_NATURAL|JT_RIGHT|JT_OUTER +** NATURAL FULL - JT_NATURAL|JT_LEFT|JT_RIGHT +** NATURAL FULL OUTER JT_NATRUAL|JT_LEFT|JT_RIGHT +** +** To preserve historical compatibly, SQLite also accepts a variety +** of other non-standard and in many cases nonsensical join types. +** This routine makes as much sense at it can from the nonsense join +** type and returns a result. Examples of accepted nonsense join types +** include but are not limited to: +** +** INNER CROSS JOIN -> same as JOIN +** NATURAL CROSS JOIN -> same as NATURAL JOIN +** OUTER LEFT JOIN -> same as LEFT JOIN +** LEFT NATURAL JOIN -> same as NATURAL LEFT JOIN +** LEFT RIGHT JOIN -> same as FULL JOIN +** RIGHT OUTER FULL JOIN -> same as FULL JOIN +** CROSS CROSS CROSS JOIN -> same as JOIN +** +** The only restrictions on the join type name are: +** +** * "INNER" cannot appear together with "OUTER", "LEFT", "RIGHT", +** or "FULL". +** +** * "CROSS" cannot appear together with "OUTER", "LEFT", "RIGHT, +** or "FULL". +** +** * If "OUTER" is present then there must also be one of +** "LEFT", "RIGHT", or "FULL" */ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *pC){ int jointype = 0; @@ -113068,13 +143964,13 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p u8 nChar; /* Length of the keyword in characters */ u8 code; /* Join type mask */ } aKeyword[] = { - /* natural */ { 0, 7, JT_NATURAL }, - /* left */ { 6, 4, JT_LEFT|JT_OUTER }, - /* outer */ { 10, 5, JT_OUTER }, - /* right */ { 14, 5, JT_RIGHT|JT_OUTER }, - /* full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, - /* inner */ { 23, 5, JT_INNER }, - /* cross */ { 28, 5, JT_INNER|JT_CROSS }, + /* (0) natural */ { 0, 7, JT_NATURAL }, + /* (1) left */ { 6, 4, JT_LEFT|JT_OUTER }, + /* (2) outer */ { 10, 5, JT_OUTER }, + /* (3) right */ { 14, 5, JT_RIGHT|JT_OUTER }, + /* (4) full */ { 19, 4, JT_LEFT|JT_RIGHT|JT_OUTER }, + /* (5) inner */ { 23, 5, JT_INNER }, + /* (6) cross */ { 28, 5, JT_INNER|JT_CROSS }, }; int i, j; apAll[0] = pA; @@ -113083,7 +143979,7 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p for(i=0; i<3 && apAll[i]; i++){ p = apAll[i]; for(j=0; jn==aKeyword[j].nChar + if( p->n==aKeyword[j].nChar && sqlite3StrNICmp((char*)p->z, &zKeyText[aKeyword[j].i], p->n)==0 ){ jointype |= aKeyword[j].code; break; @@ -113097,18 +143993,15 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p } if( (jointype & (JT_INNER|JT_OUTER))==(JT_INNER|JT_OUTER) || - (jointype & JT_ERROR)!=0 + (jointype & JT_ERROR)!=0 || + (jointype & (JT_OUTER|JT_LEFT|JT_RIGHT))==JT_OUTER ){ - const char *zSp = " "; - assert( pB!=0 ); - if( pC==0 ){ zSp++; } - sqlite3ErrorMsg(pParse, "unknown or unsupported join type: " - "%T %T%s%T", pA, pB, zSp, pC); - jointype = JT_INNER; - }else if( (jointype & JT_OUTER)!=0 - && (jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ){ - sqlite3ErrorMsg(pParse, - "RIGHT and FULL OUTER JOINs are not currently supported"); + const char *zSp1 = " "; + const char *zSp2 = " "; + if( pB==0 ){ zSp1++; } + if( pC==0 ){ zSp2++; } + sqlite3ErrorMsg(pParse, "unknown join type: " + "%T%s%T%s%T", pA, zSp1, pB, zSp2, pC); jointype = JT_INNER; } return jointype; @@ -113118,17 +144011,38 @@ SQLITE_PRIVATE int sqlite3JoinType(Parse *pParse, Token *pA, Token *pB, Token *p ** Return the index of a column in a table. Return -1 if the column ** is not contained in the table. */ -static int columnIndex(Table *pTab, const char *zCol){ +SQLITE_PRIVATE int sqlite3ColumnIndex(Table *pTab, const char *zCol){ int i; - for(i=0; inCol; i++){ - if( sqlite3StrICmp(pTab->aCol[i].zName, zCol)==0 ) return i; + u8 h = sqlite3StrIHash(zCol); + Column *pCol; + for(pCol=pTab->aCol, i=0; inCol; pCol++, i++){ + if( pCol->hName==h && sqlite3StrICmp(pCol->zCnName, zCol)==0 ) return i; } return -1; } /* -** Search the first N tables in pSrc, from left to right, looking for a -** table that has a column named zCol. +** Mark a subquery result column as having been used. +*/ +SQLITE_PRIVATE void sqlite3SrcItemColumnUsed(SrcItem *pItem, int iCol){ + assert( pItem!=0 ); + assert( (int)pItem->fg.isNestedFrom == IsNestedFrom(pItem) ); + if( pItem->fg.isNestedFrom ){ + ExprList *pResults; + assert( pItem->fg.isSubquery ); + assert( pItem->u4.pSubq!=0 ); + assert( pItem->u4.pSubq->pSelect!=0 ); + pResults = pItem->u4.pSubq->pSelect->pEList; + assert( pResults!=0 ); + assert( iCol>=0 && iColnExpr ); + pResults->a[iCol].fg.bUsed = 1; + } +} + +/* +** Search the tables iStart..iEnd (inclusive) in pSrc, looking for a +** table that has a column named zCol. The search is left-to-right. +** The first match found is returned. ** ** When found, set *piTab and *piCol to the table index and column index ** of the matching column and return TRUE. @@ -113137,19 +144051,27 @@ static int columnIndex(Table *pTab, const char *zCol){ */ static int tableAndColumnIndex( SrcList *pSrc, /* Array of tables to search */ - int N, /* Number of tables in pSrc->a[] to search */ + int iStart, /* First member of pSrc->a[] to check */ + int iEnd, /* Last member of pSrc->a[] to check */ const char *zCol, /* Name of the column we are looking for */ int *piTab, /* Write index of pSrc->a[] here */ - int *piCol /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int *piCol, /* Write index of pSrc->a[*piTab].pTab->aCol[] here */ + int bIgnoreHidden /* Ignore hidden columns */ ){ int i; /* For looping over tables in pSrc */ int iCol; /* Index of column matching zCol */ + assert( iEndnSrc ); + assert( iStart>=0 ); assert( (piTab==0)==(piCol==0) ); /* Both or neither are NULL */ - for(i=0; ia[i].pTab, zCol); - if( iCol>=0 ){ + + for(i=iStart; i<=iEnd; i++){ + iCol = sqlite3ColumnIndex(pSrc->a[i].pSTab, zCol); + if( iCol>=0 + && (bIgnoreHidden==0 || IsHiddenColumn(&pSrc->a[i].pSTab->aCol[iCol])==0) + ){ if( piTab ){ + sqlite3SrcItemColumnUsed(&pSrc->a[i], iCol); *piTab = i; *piCol = iCol; } @@ -113160,63 +144082,19 @@ static int tableAndColumnIndex( } /* -** This function is used to add terms implied by JOIN syntax to the -** WHERE clause expression of a SELECT statement. The new term, which -** is ANDed with the existing WHERE clause, is of the form: -** -** (tab1.col1 = tab2.col2) -** -** where tab1 is the iSrc'th table in SrcList pSrc and tab2 is the -** (iSrc+1)'th. Column col1 is column iColLeft of tab1, and col2 is -** column iColRight of tab2. -*/ -static void addWhereTerm( - Parse *pParse, /* Parsing context */ - SrcList *pSrc, /* List of tables in FROM clause */ - int iLeft, /* Index of first table to join in pSrc */ - int iColLeft, /* Index of column in first table */ - int iRight, /* Index of second table in pSrc */ - int iColRight, /* Index of column in second table */ - int isOuterJoin, /* True if this is an OUTER join */ - Expr **ppWhere /* IN/OUT: The WHERE clause to add to */ -){ - sqlite3 *db = pParse->db; - Expr *pE1; - Expr *pE2; - Expr *pEq; - - assert( iLeftnSrc>iRight ); - assert( pSrc->a[iLeft].pTab ); - assert( pSrc->a[iRight].pTab ); - - pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iColLeft); - pE2 = sqlite3CreateColumnExpr(db, pSrc, iRight, iColRight); - - pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2, 0); - if( pEq && isOuterJoin ){ - ExprSetProperty(pEq, EP_FromJoin); - assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); - ExprSetVVAProperty(pEq, EP_NoReduce); - pEq->iRightJoinTable = (i16)pE2->iTable; - } - *ppWhere = sqlite3ExprAnd(db, *ppWhere, pEq); -} - -/* -** Set the EP_FromJoin property on all terms of the given expression. -** And set the Expr.iRightJoinTable to iTable for every term in the +** Set the EP_OuterON property on all terms of the given expression. +** And set the Expr.w.iJoin to iTable for every term in the ** expression. ** -** The EP_FromJoin property is used on terms of an expression to tell -** the LEFT OUTER JOIN processing logic that this term is part of the +** The EP_OuterON property is used on terms of an expression to tell +** the OUTER JOIN processing logic that this term is part of the ** join restriction specified in the ON or USING clause and not a part ** of the more general WHERE clause. These terms are moved over to the ** WHERE clause during join processing but we need to remember that they ** originated in the ON or USING clause. ** -** The Expr.iRightJoinTable tells the WHERE clause processing that the -** expression depends on table iRightJoinTable even if that table is not +** The Expr.w.iJoin tells the WHERE clause processing that the +** expression depends on table w.iJoin even if that table is not ** explicitly mentioned in the expression. That information is needed ** for cases like this: ** @@ -113229,132 +144107,282 @@ static void addWhereTerm( ** after the t1 loop and rows with t1.x!=5 will never appear in ** the output, which is incorrect. */ -static void setJoinExpr(Expr *p, int iTable){ +SQLITE_PRIVATE void sqlite3SetJoinExpr(Expr *p, int iTable, u32 joinFlag){ + assert( joinFlag==EP_OuterON || joinFlag==EP_InnerON ); while( p ){ - ExprSetProperty(p, EP_FromJoin); + ExprSetProperty(p, joinFlag); assert( !ExprHasProperty(p, EP_TokenOnly|EP_Reduced) ); ExprSetVVAProperty(p, EP_NoReduce); - p->iRightJoinTable = (i16)iTable; - if( p->op==TK_FUNCTION && p->x.pList ){ - int i; - for(i=0; ix.pList->nExpr; i++){ - setJoinExpr(p->x.pList->a[i].pExpr, iTable); + p->w.iJoin = iTable; + if( p->op==TK_FUNCTION ){ + assert( ExprUseXList(p) ); + if( p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + sqlite3SetJoinExpr(p->x.pList->a[i].pExpr, iTable, joinFlag); + } } } - setJoinExpr(p->pLeft, iTable); + sqlite3SetJoinExpr(p->pLeft, iTable, joinFlag); p = p->pRight; - } + } +} + +/* Undo the work of sqlite3SetJoinExpr(). This is used when a LEFT JOIN +** is simplified into an ordinary JOIN, and when an ON expression is +** "pushed down" into the WHERE clause of a subquery. +** +** Convert every term that is marked with EP_OuterON and w.iJoin==iTable into +** an ordinary term that omits the EP_OuterON mark. Or if iTable<0, then +** just clear every EP_OuterON and EP_InnerON mark from the expression tree. +** +** If nullable is true, that means that Expr p might evaluate to NULL even +** if it is a reference to a NOT NULL column. This can happen, for example, +** if the table that p references is on the left side of a RIGHT JOIN. +** If nullable is true, then take care to not remove the EP_CanBeNull bit. +** See forum thread https://sqlite.org/forum/forumpost/b40696f50145d21c +*/ +static void unsetJoinExpr(Expr *p, int iTable, int nullable){ + while( p ){ + if( iTable<0 || (ExprHasProperty(p, EP_OuterON) && p->w.iJoin==iTable) ){ + ExprClearProperty(p, EP_OuterON|EP_InnerON); + if( iTable>=0 ) ExprSetProperty(p, EP_InnerON); + } + if( p->op==TK_COLUMN && p->iTable==iTable && !nullable ){ + ExprClearProperty(p, EP_CanBeNull); + } + if( p->op==TK_FUNCTION ){ + assert( ExprUseXList(p) ); + assert( p->pLeft==0 ); + if( p->x.pList ){ + int i; + for(i=0; ix.pList->nExpr; i++){ + unsetJoinExpr(p->x.pList->a[i].pExpr, iTable, nullable); + } + } + } + unsetJoinExpr(p->pLeft, iTable, nullable); + p = p->pRight; + } } /* ** This routine processes the join information for a SELECT statement. -** ON and USING clauses are converted into extra terms of the WHERE clause. -** NATURAL joins also create extra WHERE clause terms. +** +** * A NATURAL join is converted into a USING join. After that, we +** do not need to be concerned with NATURAL joins and we only have +** think about USING joins. +** +** * ON and USING clauses result in extra terms being added to the +** WHERE clause to enforce the specified constraints. The extra +** WHERE clause terms will be tagged with EP_OuterON or +** EP_InnerON so that we know that they originated in ON/USING. ** ** The terms of a FROM clause are contained in the Select.pSrc structure. ** The left most table is the first entry in Select.pSrc. The right-most ** table is the last entry. The join operator is held in the entry to -** the left. Thus entry 0 contains the join operator for the join between +** the right. Thus entry 1 contains the join operator for the join between ** entries 0 and 1. Any ON or USING clauses associated with the join are -** also attached to the left entry. +** also attached to the right entry. ** ** This routine returns the number of errors encountered. */ -static int sqliteProcessJoin(Parse *pParse, Select *p){ +static int sqlite3ProcessJoin(Parse *pParse, Select *p){ SrcList *pSrc; /* All tables in the FROM clause */ int i, j; /* Loop counters */ - struct SrcList_item *pLeft; /* Left table being joined */ - struct SrcList_item *pRight; /* Right table being joined */ + SrcItem *pLeft; /* Left table being joined */ + SrcItem *pRight; /* Right table being joined */ pSrc = p->pSrc; pLeft = &pSrc->a[0]; pRight = &pLeft[1]; for(i=0; inSrc-1; i++, pRight++, pLeft++){ - Table *pLeftTab = pLeft->pTab; - Table *pRightTab = pRight->pTab; - int isOuter; + Table *pRightTab = pRight->pSTab; + u32 joinType; - if( NEVER(pLeftTab==0 || pRightTab==0) ) continue; - isOuter = (pRight->fg.jointype & JT_OUTER)!=0; + if( NEVER(pLeft->pSTab==0 || pRightTab==0) ) continue; + joinType = (pRight->fg.jointype & JT_OUTER)!=0 ? EP_OuterON : EP_InnerON; - /* When the NATURAL keyword is present, add WHERE clause terms for - ** every column that the two tables have in common. + /* If this is a NATURAL join, synthesize an appropriate USING clause + ** to specify which columns should be joined. */ if( pRight->fg.jointype & JT_NATURAL ){ - if( pRight->pOn || pRight->pUsing ){ + IdList *pUsing = 0; + if( pRight->fg.isUsing || pRight->u3.pOn ){ sqlite3ErrorMsg(pParse, "a NATURAL join may not have " "an ON or USING clause", 0); return 1; } for(j=0; jnCol; j++){ char *zName; /* Name of column in the right table */ - int iLeft; /* Matching left table */ - int iLeftCol; /* Matching column in the left table */ - zName = pRightTab->aCol[j].zName; - if( tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) ){ - addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, j, - isOuter, &p->pWhere); + if( IsHiddenColumn(&pRightTab->aCol[j]) ) continue; + zName = pRightTab->aCol[j].zCnName; + if( tableAndColumnIndex(pSrc, 0, i, zName, 0, 0, 1) ){ + pUsing = sqlite3IdListAppend(pParse, pUsing, 0); + if( pUsing ){ + assert( pUsing->nId>0 ); + assert( pUsing->a[pUsing->nId-1].zName==0 ); + pUsing->a[pUsing->nId-1].zName = sqlite3DbStrDup(pParse->db, zName); + } } } - } - - /* Disallow both ON and USING clauses in the same join - */ - if( pRight->pOn && pRight->pUsing ){ - sqlite3ErrorMsg(pParse, "cannot have both ON and USING " - "clauses in the same join"); - return 1; - } - - /* Add the ON clause to the end of the WHERE clause, connected by - ** an AND operator. - */ - if( pRight->pOn ){ - if( isOuter ) setJoinExpr(pRight->pOn, pRight->iCursor); - p->pWhere = sqlite3ExprAnd(pParse->db, p->pWhere, pRight->pOn); - pRight->pOn = 0; + if( pUsing ){ + pRight->fg.isUsing = 1; + pRight->fg.isSynthUsing = 1; + pRight->u3.pUsing = pUsing; + } + if( pParse->nErr ) return 1; } /* Create extra terms on the WHERE clause for each column named - ** in the USING clause. Example: If the two tables to be joined are + ** in the USING clause. Example: If the two tables to be joined are ** A and B and the USING clause names X, Y, and Z, then add this ** to the WHERE clause: A.X=B.X AND A.Y=B.Y AND A.Z=B.Z ** Report an error if any column mentioned in the USING clause is ** not contained in both tables to be joined. */ - if( pRight->pUsing ){ - IdList *pList = pRight->pUsing; + if( pRight->fg.isUsing ){ + IdList *pList = pRight->u3.pUsing; + sqlite3 *db = pParse->db; + assert( pList!=0 ); for(j=0; jnId; j++){ char *zName; /* Name of the term in the USING clause */ int iLeft; /* Table on the left with matching column name */ int iLeftCol; /* Column number of matching column on the left */ int iRightCol; /* Column number of matching column on the right */ + Expr *pE1; /* Reference to the column on the LEFT of the join */ + Expr *pE2; /* Reference to the column on the RIGHT of the join */ + Expr *pEq; /* Equality constraint. pE1 == pE2 */ zName = pList->a[j].zName; - iRightCol = columnIndex(pRightTab, zName); + iRightCol = sqlite3ColumnIndex(pRightTab, zName); if( iRightCol<0 - || !tableAndColumnIndex(pSrc, i+1, zName, &iLeft, &iLeftCol) + || tableAndColumnIndex(pSrc, 0, i, zName, &iLeft, &iLeftCol, + pRight->fg.isSynthUsing)==0 ){ sqlite3ErrorMsg(pParse, "cannot join using column %s - column " "not present in both tables", zName); return 1; } - addWhereTerm(pParse, pSrc, iLeft, iLeftCol, i+1, iRightCol, - isOuter, &p->pWhere); + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); + sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); + if( (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + /* This branch runs if the query contains one or more RIGHT or FULL + ** JOINs. If only a single table on the left side of this join + ** contains the zName column, then this branch is a no-op. + ** But if there are two or more tables on the left side + ** of the join, construct a coalesce() function that gathers all + ** such tables. Raise an error if more than one of those references + ** to zName is not also within a prior USING clause. + ** + ** We really ought to raise an error if there are two or more + ** non-USING references to zName on the left of an INNER or LEFT + ** JOIN. But older versions of SQLite do not do that, so we avoid + ** adding a new error so as to not break legacy applications. + */ + ExprList *pFuncArgs = 0; /* Arguments to the coalesce() */ + static const Token tkCoalesce = { "coalesce", 8 }; + while( tableAndColumnIndex(pSrc, iLeft+1, i, zName, &iLeft, &iLeftCol, + pRight->fg.isSynthUsing)!=0 ){ + if( pSrc->a[iLeft].fg.isUsing==0 + || sqlite3IdListIndex(pSrc->a[iLeft].u3.pUsing, zName)<0 + ){ + sqlite3ErrorMsg(pParse, "ambiguous reference to %s in USING()", + zName); + break; + } + pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); + pE1 = sqlite3CreateColumnExpr(db, pSrc, iLeft, iLeftCol); + sqlite3SrcItemColumnUsed(&pSrc->a[iLeft], iLeftCol); + } + if( pFuncArgs ){ + pFuncArgs = sqlite3ExprListAppend(pParse, pFuncArgs, pE1); + pE1 = sqlite3ExprFunction(pParse, pFuncArgs, &tkCoalesce, 0); + } + } + pE2 = sqlite3CreateColumnExpr(db, pSrc, i+1, iRightCol); + sqlite3SrcItemColumnUsed(pRight, iRightCol); + pEq = sqlite3PExpr(pParse, TK_EQ, pE1, pE2); + assert( pE2!=0 || pEq==0 ); + if( pEq ){ + ExprSetProperty(pEq, joinType); + assert( !ExprHasProperty(pEq, EP_TokenOnly|EP_Reduced) ); + ExprSetVVAProperty(pEq, EP_NoReduce); + pEq->w.iJoin = pE2->iTable; + } + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pEq); } } + + /* Add the ON clause to the end of the WHERE clause, connected by + ** an AND operator. + */ + else if( pRight->u3.pOn ){ + sqlite3SetJoinExpr(pRight->u3.pOn, pRight->iCursor, joinType); + p->pWhere = sqlite3ExprAnd(pParse, p->pWhere, pRight->u3.pOn); + pRight->u3.pOn = 0; + pRight->fg.isOn = 1; + } } return 0; } -/* Forward reference */ -static KeyInfo *keyInfoFromExprList( - Parse *pParse, /* Parsing context */ - ExprList *pList, /* Form the KeyInfo object from this ExprList */ - int iStart, /* Begin with this column of pList */ - int nExtra /* Add this many extra columns to the end */ -); +/* +** An instance of this object holds information (beyond pParse and pSelect) +** needed to load the next result row that is to be added to the sorter. +*/ +typedef struct RowLoadInfo RowLoadInfo; +struct RowLoadInfo { + int regResult; /* Store results in array of registers here */ + u8 ecelFlags; /* Flag argument to ExprCodeExprList() */ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra; /* Extra columns needed by sorter refs */ + int regExtraResult; /* Where to load the extra columns */ +#endif +}; + +/* +** This routine does the work of loading query data into an array of +** registers so that it can be added to the sorter. +*/ +static void innerLoopLoadRow( + Parse *pParse, /* Statement under construction */ + Select *pSelect, /* The query being coded */ + RowLoadInfo *pInfo /* Info needed to complete the row load */ +){ + sqlite3ExprCodeExprList(pParse, pSelect->pEList, pInfo->regResult, + 0, pInfo->ecelFlags); +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( pInfo->pExtra ){ + sqlite3ExprCodeExprList(pParse, pInfo->pExtra, pInfo->regExtraResult, 0, 0); + sqlite3ExprListDelete(pParse->db, pInfo->pExtra); + } +#endif +} + +/* +** Code the OP_MakeRecord instruction that generates the entry to be +** added into the sorter. +** +** Return the register in which the result is stored. +*/ +static int makeSorterRecord( + Parse *pParse, + SortCtx *pSort, + Select *pSelect, + int regBase, + int nBase +){ + int nOBSat = pSort->nOBSat; + Vdbe *v = pParse->pVdbe; + int regOut = ++pParse->nMem; + if( pSort->pDeferredRowLoad ){ + innerLoopLoadRow(pParse, pSelect, pSort->pDeferredRowLoad); + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regOut); + return regOut; +} /* ** Generate code that will push the record in registers regData @@ -113366,7 +144394,7 @@ static void pushOntoSorter( Select *pSelect, /* The whole SELECT statement */ int regData, /* First register holding data to be sorted */ int regOrigData, /* First register holding data before packing */ - int nData, /* Number of elements in the data array */ + int nData, /* Number of elements in the regData data array */ int nPrefixReg /* No. of reg prior to regData available for use */ ){ Vdbe *v = pParse->pVdbe; /* Stmt under construction */ @@ -113374,32 +144402,51 @@ static void pushOntoSorter( int nExpr = pSort->pOrderBy->nExpr; /* No. of ORDER BY terms */ int nBase = nExpr + bSeq + nData; /* Fields in sorter record */ int regBase; /* Regs for sorter record */ - int regRecord = ++pParse->nMem; /* Assembled sorter record */ + int regRecord = 0; /* Assembled sorter record */ int nOBSat = pSort->nOBSat; /* ORDER BY terms to skip */ int op; /* Opcode to add sorter record to sorter */ int iLimit; /* LIMIT counter */ + int iSkip = 0; /* End of the sorter insert loop */ assert( bSeq==0 || bSeq==1 ); - assert( nData==1 || regData==regOrigData ); + + /* Three cases: + ** (1) The data to be sorted has already been packed into a Record + ** by a prior OP_MakeRecord. In this case nData==1 and regData + ** will be completely unrelated to regOrigData. + ** (2) All output columns are included in the sort record. In that + ** case regData==regOrigData. + ** (3) Some output columns are omitted from the sort record due to + ** the SQLITE_ENABLE_SORTER_REFERENCES optimization, or due to the + ** SQLITE_ECEL_OMITREF optimization, or due to the + ** SortCtx.pDeferredRowLoad optimization. In any of these cases + ** regOrigData is 0 to prevent this routine from trying to copy + ** values that might not yet exist. + */ + assert( nData==1 || regData==regOrigData || regOrigData==0 ); + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPush = sqlite3VdbeCurrentAddr(v); +#endif + if( nPrefixReg ){ assert( nPrefixReg==nExpr+bSeq ); - regBase = regData - nExpr - bSeq; + regBase = regData - nPrefixReg; }else{ regBase = pParse->nMem + 1; pParse->nMem += nBase; } assert( pSelect->iOffset==0 || pSelect->iLimit!=0 ); iLimit = pSelect->iOffset ? pSelect->iOffset+1 : pSelect->iLimit; - pSort->labelDone = sqlite3VdbeMakeLabel(v); + pSort->labelDone = sqlite3VdbeMakeLabel(pParse); sqlite3ExprCodeExprList(pParse, pSort->pOrderBy, regBase, regOrigData, - SQLITE_ECEL_DUP|SQLITE_ECEL_REF); + SQLITE_ECEL_DUP | (regOrigData? SQLITE_ECEL_REF : 0)); if( bSeq ){ sqlite3VdbeAddOp2(v, OP_Sequence, pSort->iECursor, regBase+nExpr); } - if( nPrefixReg==0 ){ + if( nPrefixReg==0 && nData>0 ){ sqlite3ExprCodeMove(pParse, regData, regBase+nExpr+bSeq, nData); } - sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase+nOBSat, nBase-nOBSat, regRecord); if( nOBSat>0 ){ int regPrevKey; /* The first nOBSat columns of the previous row */ int addrFirst; /* Address of the OP_IfNot opcode */ @@ -113408,11 +144455,12 @@ static void pushOntoSorter( int nKey; /* Number of sorting key columns, including OP_Sequence */ KeyInfo *pKI; /* Original KeyInfo on the sorter table */ + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); regPrevKey = pParse->nMem+1; pParse->nMem += pSort->nOBSat; nKey = nExpr - pSort->nOBSat + bSeq; if( bSeq ){ - addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); + addrFirst = sqlite3VdbeAddOp1(v, OP_IfNot, regBase+nExpr); }else{ addrFirst = sqlite3VdbeAddOp1(v, OP_SequenceTest, pSort->iECursor); } @@ -113422,14 +144470,15 @@ static void pushOntoSorter( if( pParse->db->mallocFailed ) return; pOp->p2 = nKey + nData; pKI = pOp->p4.pKeyInfo; - memset(pKI->aSortOrder, 0, pKI->nField); /* Makes OP_Jump below testable */ + memset(pKI->aSortFlags, 0, pKI->nKeyField); /* Makes OP_Jump testable */ sqlite3VdbeChangeP4(v, -1, (char*)pKI, P4_KEYINFO); - testcase( pKI->nXField>2 ); - pOp->p4.pKeyInfo = keyInfoFromExprList(pParse, pSort->pOrderBy, nOBSat, - pKI->nXField-1); + testcase( pKI->nAllField > pKI->nKeyField+2 ); + pOp->p4.pKeyInfo = sqlite3KeyInfoFromExprList(pParse,pSort->pOrderBy,nOBSat, + pKI->nAllField-pKI->nKeyField-1); + pOp = 0; /* Ensure pOp not used after sqlite3VdbeAddOp3() */ addrJmp = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp3(v, OP_Jump, addrJmp+1, 0, addrJmp+1); VdbeCoverage(v); - pSort->labelBkOut = sqlite3VdbeMakeLabel(v); + pSort->labelBkOut = sqlite3VdbeMakeLabel(pParse); pSort->regReturn = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeAddOp1(v, OP_ResetSorter, pSort->iECursor); @@ -113441,19 +144490,48 @@ static void pushOntoSorter( sqlite3ExprCodeMove(pParse, regBase, regPrevKey, pSort->nOBSat); sqlite3VdbeJumpHere(v, addrJmp); } + if( iLimit ){ + /* At this point the values for the new sorter entry are stored + ** in an array of registers. They need to be composed into a record + ** and inserted into the sorter if either (a) there are currently + ** less than LIMIT+OFFSET items or (b) the new record is smaller than + ** the largest record currently in the sorter. If (b) is true and there + ** are already LIMIT+OFFSET items in the sorter, delete the largest + ** entry before inserting the new one. This way there are never more + ** than LIMIT+OFFSET items in the sorter. + ** + ** If the new record does not need to be inserted into the sorter, + ** jump to the next iteration of the loop. If the pSort->labelOBLopt + ** value is not zero, then it is a label of where to jump. Otherwise, + ** just bypass the row insert logic. See the header comment on the + ** sqlite3WhereOrderByLimitOptLabel() function for additional info. + */ + int iCsr = pSort->iECursor; + sqlite3VdbeAddOp2(v, OP_IfNotZero, iLimit, sqlite3VdbeCurrentAddr(v)+4); + VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Last, iCsr, 0); + iSkip = sqlite3VdbeAddOp4Int(v, OP_IdxLE, + iCsr, 0, regBase+nOBSat, nExpr-nOBSat); + VdbeCoverage(v); + sqlite3VdbeAddOp1(v, OP_Delete, iCsr); + } + if( regRecord==0 ){ + regRecord = makeSorterRecord(pParse, pSort, pSelect, regBase, nBase); + } if( pSort->sortFlags & SORTFLAG_UseSorter ){ op = OP_SorterInsert; }else{ op = OP_IdxInsert; } - sqlite3VdbeAddOp2(v, op, pSort->iECursor, regRecord); - if( iLimit ){ - int addr; - addr = sqlite3VdbeAddOp3(v, OP_IfNotZero, iLimit, 0, 1); VdbeCoverage(v); - sqlite3VdbeAddOp1(v, OP_Last, pSort->iECursor); - sqlite3VdbeAddOp1(v, OP_Delete, pSort->iECursor); - sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp4Int(v, op, pSort->iECursor, regRecord, + regBase+nOBSat, nBase-nOBSat); + if( iSkip ){ + sqlite3VdbeChangeP2(v, iSkip, + pSort->labelOBLopt ? pSort->labelOBLopt : sqlite3VdbeCurrentAddr(v)); } +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + pSort->addrPushEnd = sqlite3VdbeCurrentAddr(v)-1; +#endif } /* @@ -113471,70 +144549,258 @@ static void codeOffset( } /* -** Add code that will check to make sure the N registers starting at iMem -** form a distinct entry. iTab is a sorting index that holds previously -** seen combinations of the N values. A new entry is made in iTab -** if the current N values are new. +** Add code that will check to make sure the array of registers starting at +** iMem form a distinct entry. This is used by both "SELECT DISTINCT ..." and +** distinct aggregates ("SELECT count(DISTINCT ) ..."). Three strategies +** are available. Which is used depends on the value of parameter eTnctType, +** as follows: ** -** A jump to addrRepeat is made and the N+1 values are popped from the -** stack if the top N elements are not distinct. +** WHERE_DISTINCT_UNORDERED/WHERE_DISTINCT_NOOP: +** Build an ephemeral table that contains all entries seen before and +** skip entries which have been seen before. +** +** Parameter iTab is the cursor number of an ephemeral table that must +** be opened before the VM code generated by this routine is executed. +** The ephemeral cursor table is queried for a record identical to the +** record formed by the current array of registers. If one is found, +** jump to VM address addrRepeat. Otherwise, insert a new record into +** the ephemeral cursor and proceed. +** +** The returned value in this case is a copy of parameter iTab. +** +** WHERE_DISTINCT_ORDERED: +** In this case rows are being delivered sorted order. The ephemeral +** table is not required. Instead, the current set of values +** is compared against previous row. If they match, the new row +** is not distinct and control jumps to VM address addrRepeat. Otherwise, +** the VM program proceeds with processing the new row. +** +** The returned value in this case is the register number of the first +** in an array of registers used to store the previous result row so that +** it can be compared to the next. The caller must ensure that this +** register is initialized to NULL. (The fixDistinctOpenEph() routine +** will take care of this initialization.) +** +** WHERE_DISTINCT_UNIQUE: +** In this case it has already been determined that the rows are distinct. +** No special action is required. The return value is zero. +** +** Parameter pEList is the list of expressions used to generated the +** contents of each row. It is used by this routine to determine (a) +** how many elements there are in the array of registers and (b) the +** collation sequences that should be used for the comparisons if +** eTnctType is WHERE_DISTINCT_ORDERED. */ -static void codeDistinct( +static int codeDistinct( Parse *pParse, /* Parsing and code generating context */ + int eTnctType, /* WHERE_DISTINCT_* value */ int iTab, /* A sorting index used to test for distinctness */ int addrRepeat, /* Jump to here if not distinct */ - int N, /* Number of elements */ - int iMem /* First element */ + ExprList *pEList, /* Expression for each element */ + int regElem /* First element */ ){ - Vdbe *v; - int r1; + int iRet = 0; + int nResultCol = pEList->nExpr; + Vdbe *v = pParse->pVdbe; - v = pParse->pVdbe; - r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, iMem, N); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_MakeRecord, iMem, N, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iTab, r1); - sqlite3ReleaseTempReg(pParse, r1); + switch( eTnctType ){ + case WHERE_DISTINCT_ORDERED: { + int i; + int iJump; /* Jump destination */ + int regPrev; /* Previous row content */ + + /* Allocate space for the previous row */ + iRet = regPrev = pParse->nMem+1; + pParse->nMem += nResultCol; + + iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; + for(i=0; ia[i].pExpr); + if( idb->mallocFailed ); + sqlite3VdbeAddOp3(v, OP_Copy, regElem, regPrev, nResultCol-1); + break; + } + + case WHERE_DISTINCT_UNIQUE: { + /* nothing to do */ + break; + } + + default: { + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4Int(v, OP_Found, iTab, addrRepeat, regElem, nResultCol); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regElem, nResultCol, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iTab, r1, regElem, nResultCol); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3ReleaseTempReg(pParse, r1); + iRet = iTab; + break; + } + } + + return iRet; } -#ifndef SQLITE_OMIT_SUBQUERY /* -** Generate an error message when a SELECT is used within a subexpression -** (example: "a IN (SELECT * FROM table)") but it has more than 1 result -** column. We do this in a subroutine because the error used to occur -** in multiple places. (The error only occurs in one place now, but we -** retain the subroutine to minimize code disruption.) +** This routine runs after codeDistinct(). It makes necessary +** adjustments to the OP_OpenEphemeral opcode that the codeDistinct() +** routine made use of. This processing must be done separately since +** sometimes codeDistinct is called before the OP_OpenEphemeral is actually +** laid down. +** +** WHERE_DISTINCT_NOOP: +** WHERE_DISTINCT_UNORDERED: +** +** No adjustments necessary. This function is a no-op. +** +** WHERE_DISTINCT_UNIQUE: +** +** The ephemeral table is not needed. So change the +** OP_OpenEphemeral opcode into an OP_Noop. +** +** WHERE_DISTINCT_ORDERED: +** +** The ephemeral table is not needed. But we do need register +** iVal to be initialized to NULL. So change the OP_OpenEphemeral +** into an OP_Null on the iVal register. */ -static int checkForMultiColumnSelectError( - Parse *pParse, /* Parse context. */ - SelectDest *pDest, /* Destination of SELECT results */ - int nExpr /* Number of result columns returned by SELECT */ +static void fixDistinctOpenEph( + Parse *pParse, /* Parsing and code generating context */ + int eTnctType, /* WHERE_DISTINCT_* value */ + int iVal, /* Value returned by codeDistinct() */ + int iOpenEphAddr /* Address of OP_OpenEphemeral instruction for iTab */ ){ - int eDest = pDest->eDest; - if( nExpr>1 && (eDest==SRT_Mem || eDest==SRT_Set) ){ - sqlite3ErrorMsg(pParse, "only a single result allowed for " - "a SELECT that is part of an expression"); - return 1; - }else{ - return 0; + if( pParse->nErr==0 + && (eTnctType==WHERE_DISTINCT_UNIQUE || eTnctType==WHERE_DISTINCT_ORDERED) + ){ + Vdbe *v = pParse->pVdbe; + sqlite3VdbeChangeToNoop(v, iOpenEphAddr); + if( sqlite3VdbeGetOp(v, iOpenEphAddr+1)->opcode==OP_Explain ){ + sqlite3VdbeChangeToNoop(v, iOpenEphAddr+1); + } + if( eTnctType==WHERE_DISTINCT_ORDERED ){ + /* Change the OP_OpenEphemeral to an OP_Null that sets the MEM_Cleared + ** bit on the first register of the previous value. This will cause the + ** OP_Ne added in codeDistinct() to always fail on the first iteration of + ** the loop even if the first row is all NULLs. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, iOpenEphAddr); + pOp->opcode = OP_Null; + pOp->p1 = 1; + pOp->p2 = iVal; + } } } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES +/* +** This function is called as part of inner-loop generation for a SELECT +** statement with an ORDER BY that is not optimized by an index. It +** determines the expressions, if any, that the sorter-reference +** optimization should be used for. The sorter-reference optimization +** is used for SELECT queries like: +** +** SELECT a, bigblob FROM t1 ORDER BY a LIMIT 10 +** +** If the optimization is used for expression "bigblob", then instead of +** storing values read from that column in the sorter records, the PK of +** the row from table t1 is stored instead. Then, as records are extracted from +** the sorter to return to the user, the required value of bigblob is +** retrieved directly from table t1. If the values are very large, this +** can be more efficient than storing them directly in the sorter records. +** +** The ExprList_item.fg.bSorterRef flag is set for each expression in pEList +** for which the sorter-reference optimization should be enabled. +** Additionally, the pSort->aDefer[] array is populated with entries +** for all cursors required to evaluate all selected expressions. Finally. +** output variable (*ppExtra) is set to an expression list containing +** expressions for all extra PK values that should be stored in the +** sorter records. +*/ +static void selectExprDefer( + Parse *pParse, /* Leave any error here */ + SortCtx *pSort, /* Sorter context */ + ExprList *pEList, /* Expressions destined for sorter */ + ExprList **ppExtra /* Expressions to append to sorter record */ +){ + int i; + int nDefer = 0; + ExprList *pExtra = 0; + for(i=0; inExpr; i++){ + struct ExprList_item *pItem = &pEList->a[i]; + if( pItem->u.x.iOrderByCol==0 ){ + Expr *pExpr = pItem->pExpr; + Table *pTab; + if( pExpr->op==TK_COLUMN + && pExpr->iColumn>=0 + && ALWAYS( ExprUseYTab(pExpr) ) + && (pTab = pExpr->y.pTab)!=0 + && IsOrdinaryTable(pTab) + && (pTab->aCol[pExpr->iColumn].colFlags & COLFLAG_SORTERREF)!=0 + ){ + int j; + for(j=0; jaDefer[j].iCsr==pExpr->iTable ) break; + } + if( j==nDefer ){ + if( nDefer==ArraySize(pSort->aDefer) ){ + continue; + }else{ + int nKey = 1; + int k; + Index *pPk = 0; + if( !HasRowid(pTab) ){ + pPk = sqlite3PrimaryKeyIndex(pTab); + nKey = pPk->nKeyCol; + } + for(k=0; kiTable = pExpr->iTable; + assert( ExprUseYTab(pNew) ); + pNew->y.pTab = pExpr->y.pTab; + pNew->iColumn = pPk ? pPk->aiColumn[k] : -1; + pExtra = sqlite3ExprListAppend(pParse, pExtra, pNew); + } + } + pSort->aDefer[nDefer].pTab = pExpr->y.pTab; + pSort->aDefer[nDefer].iCsr = pExpr->iTable; + pSort->aDefer[nDefer].nKey = nKey; + nDefer++; + } + } + pItem->fg.bSorterRef = 1; + } + } + } + pSort->nDefer = (u8)nDefer; + *ppExtra = pExtra; +} #endif /* ** This routine generates the code for the inside of the inner loop ** of a SELECT. ** -** If srcTab is negative, then the pEList expressions +** If srcTab is negative, then the p->pEList expressions ** are evaluated in order to get the data for this row. If srcTab is -** zero or more, then data is pulled from srcTab and pEList is used only -** to get number columns and the datatype for each column. +** zero or more, then data is pulled from srcTab and p->pEList is used only +** to get the number of columns and the collation sequence for each column. */ static void selectInnerLoop( Parse *pParse, /* The parser context */ Select *p, /* The complete select statement being coded */ - ExprList *pEList, /* List of values being extracted */ - int srcTab, /* Pull data from this table */ + int srcTab, /* Pull data from this table if non-negative */ SortCtx *pSort, /* If not NULL, info on how to process ORDER BY */ DistinctCtx *pDistinct, /* If not NULL, info on how to process DISTINCT */ SelectDest *pDest, /* How to dispose of the results */ @@ -113543,15 +144809,23 @@ static void selectInnerLoop( ){ Vdbe *v = pParse->pVdbe; int i; - int hasDistinct; /* True if the DISTINCT keyword is present */ - int regResult; /* Start of memory holding result set */ + int hasDistinct; /* True if the DISTINCT keyword is present */ int eDest = pDest->eDest; /* How to dispose of results */ int iParm = pDest->iSDParm; /* First argument to disposal method */ int nResultCol; /* Number of result columns */ int nPrefixReg = 0; /* Number of extra registers before regResult */ + RowLoadInfo sRowLoadInfo; /* Info for deferred row loading */ + + /* Usually, regResult is the first cell in an array of memory cells + ** containing the current result row. In this case regOrig is set to the + ** same value. However, if the results are being sent to the sorter, the + ** values for any expressions that are also part of the sort-key are omitted + ** from this array. In this case regOrig is set to zero. */ + int regResult; /* Start of memory holding current results */ + int regOrig; /* Start of memory holding full result (or 0) */ assert( v ); - assert( pEList!=0 ); + assert( p->pEList!=0 ); hasDistinct = pDistinct ? pDistinct->eTnctType : WHERE_DISTINCT_NOOP; if( pSort && pSort->pOrderBy==0 ) pSort = 0; if( pSort==0 && !hasDistinct ){ @@ -113561,7 +144835,7 @@ static void selectInnerLoop( /* Pull the requested columns. */ - nResultCol = pEList->nExpr; + nResultCol = p->pEList->nExpr; if( pDest->iSdst==0 ){ if( pSort ){ @@ -113580,23 +144854,97 @@ static void selectInnerLoop( pParse->nMem += nResultCol; } pDest->nSdst = nResultCol; - regResult = pDest->iSdst; + regOrig = regResult = pDest->iSdst; if( srcTab>=0 ){ for(i=0; ia[i].zName)); + VdbeComment((v, "%s", p->pEList->a[i].zEName)); } }else if( eDest!=SRT_Exists ){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + ExprList *pExtra = 0; +#endif /* If the destination is an EXISTS(...) expression, the actual ** values returned by the SELECT are not required. */ - u8 ecelFlags; + u8 ecelFlags; /* "ecel" is an abbreviation of "ExprCodeExprList" */ + ExprList *pEList; if( eDest==SRT_Mem || eDest==SRT_Output || eDest==SRT_Coroutine ){ ecelFlags = SQLITE_ECEL_DUP; }else{ ecelFlags = 0; } - sqlite3ExprCodeExprList(pParse, pEList, regResult, 0, ecelFlags); + if( pSort && hasDistinct==0 && eDest!=SRT_EphemTab && eDest!=SRT_Table ){ + /* For each expression in p->pEList that is a copy of an expression in + ** the ORDER BY clause (pSort->pOrderBy), set the associated + ** iOrderByCol value to one more than the index of the ORDER BY + ** expression within the sort-key that pushOntoSorter() will generate. + ** This allows the p->pEList field to be omitted from the sorted record, + ** saving space and CPU cycles. */ + ecelFlags |= (SQLITE_ECEL_OMITREF|SQLITE_ECEL_REF); + + for(i=pSort->nOBSat; ipOrderBy->nExpr; i++){ + int j; + if( (j = pSort->pOrderBy->a[i].u.x.iOrderByCol)>0 ){ + p->pEList->a[j-1].u.x.iOrderByCol = i+1-pSort->nOBSat; + } + } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + selectExprDefer(pParse, pSort, p->pEList, &pExtra); + if( pExtra && pParse->db->mallocFailed==0 ){ + /* If there are any extra PK columns to add to the sorter records, + ** allocate extra memory cells and adjust the OpenEphemeral + ** instruction to account for the larger records. This is only + ** required if there are one or more WITHOUT ROWID tables with + ** composite primary keys in the SortCtx.aDefer[] array. */ + VdbeOp *pOp = sqlite3VdbeGetOp(v, pSort->addrSortIndex); + pOp->p2 += (pExtra->nExpr - pSort->nDefer); + pOp->p4.pKeyInfo->nAllField += (pExtra->nExpr - pSort->nDefer); + pParse->nMem += pExtra->nExpr; + } +#endif + + /* Adjust nResultCol to account for columns that are omitted + ** from the sorter by the optimizations in this branch */ + pEList = p->pEList; + for(i=0; inExpr; i++){ + if( pEList->a[i].u.x.iOrderByCol>0 +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + || pEList->a[i].fg.bSorterRef +#endif + ){ + nResultCol--; + regOrig = 0; + } + } + + testcase( regOrig ); + testcase( eDest==SRT_Set ); + testcase( eDest==SRT_Mem ); + testcase( eDest==SRT_Coroutine ); + testcase( eDest==SRT_Output ); + assert( eDest==SRT_Set || eDest==SRT_Mem + || eDest==SRT_Coroutine || eDest==SRT_Output + || eDest==SRT_Upfrom ); + } + sRowLoadInfo.regResult = regResult; + sRowLoadInfo.ecelFlags = ecelFlags; +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + sRowLoadInfo.pExtra = pExtra; + sRowLoadInfo.regExtraResult = regResult + nResultCol; + if( pExtra ) nResultCol += pExtra->nExpr; +#endif + if( p->iLimit + && (ecelFlags & SQLITE_ECEL_OMITREF)!=0 + && nPrefixReg>0 + ){ + assert( pSort!=0 ); + assert( hasDistinct==0 ); + pSort->pDeferredRowLoad = &sRowLoadInfo; + regOrig = 0; + }else{ + innerLoopLoadRow(pParse, p, &sRowLoadInfo); + } } /* If the DISTINCT keyword was present on the SELECT statement @@ -113604,58 +144952,11 @@ static void selectInnerLoop( ** part of the result. */ if( hasDistinct ){ - switch( pDistinct->eTnctType ){ - case WHERE_DISTINCT_ORDERED: { - VdbeOp *pOp; /* No longer required OpenEphemeral instr. */ - int iJump; /* Jump destination */ - int regPrev; /* Previous row content */ - - /* Allocate space for the previous row */ - regPrev = pParse->nMem+1; - pParse->nMem += nResultCol; - - /* Change the OP_OpenEphemeral coded earlier to an OP_Null - ** sets the MEM_Cleared bit on the first register of the - ** previous value. This will cause the OP_Ne below to always - ** fail on the first iteration of the loop even if the first - ** row is all NULLs. - */ - sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); - pOp = sqlite3VdbeGetOp(v, pDistinct->addrTnct); - pOp->opcode = OP_Null; - pOp->p1 = 1; - pOp->p2 = regPrev; - - iJump = sqlite3VdbeCurrentAddr(v) + nResultCol; - for(i=0; ia[i].pExpr); - if( idb->mallocFailed ); - sqlite3VdbeAddOp3(v, OP_Copy, regResult, regPrev, nResultCol-1); - break; - } - - case WHERE_DISTINCT_UNIQUE: { - sqlite3VdbeChangeToNoop(v, pDistinct->addrTnct); - break; - } - - default: { - assert( pDistinct->eTnctType==WHERE_DISTINCT_UNORDERED ); - codeDistinct(pParse, pDistinct->tabTnct, iContinue, nResultCol, - regResult); - break; - } - } + int eType = pDistinct->eTnctType; + int iTab = pDistinct->tabTnct; + assert( nResultCol==p->pEList->nExpr ); + iTab = codeDistinct(pParse, eType, iTab, iContinue, p->pEList, regResult); + fixDistinctOpenEph(pParse, eType, iTab, pDistinct->addrTnct); if( pSort==0 ){ codeOffset(v, p->iOffset, iContinue); } @@ -113670,7 +144971,7 @@ static void selectInnerLoop( int r1; r1 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); sqlite3ReleaseTempReg(pParse, r1); break; } @@ -113697,6 +144998,16 @@ static void selectInnerLoop( testcase( eDest==SRT_Fifo ); testcase( eDest==SRT_DistFifo ); sqlite3VdbeAddOp3(v, OP_MakeRecord, regResult, nResultCol, r1+nPrefixReg); +#if !defined(SQLITE_ENABLE_NULL_TRIM) && defined(SQLITE_DEBUG) + /* A destination of SRT_Table and a non-zero iSDParm2 parameter means + ** that this is an "UPDATE ... FROM" on a virtual table or view. In this + ** case set the p5 parameter of the OP_MakeRecord to OPFLAG_NOCHNG_MAGIC. + ** This does not affect operation in any way - it just allows MakeRecord + ** to process OPFLAG_NOCHANGE values without an assert() failing. */ + if( eDest==SRT_Table && pDest->iSDParm2 ){ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); + } +#endif #ifndef SQLITE_OMIT_CTE if( eDest==SRT_DistFifo ){ /* If the destination is DistFifo, then cursor (iParm+1) is open @@ -113707,12 +145018,13 @@ static void selectInnerLoop( int addr = sqlite3VdbeCurrentAddr(v) + 4; sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, addr, r1, 0); VdbeCoverage(v); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm+1, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm+1, r1,regResult,nResultCol); assert( pSort==0 ); } #endif if( pSort ){ - pushOntoSorter(pParse, pSort, p, r1+nPrefixReg,regResult,1,nPrefixReg); + assert( regResult==regOrig ); + pushOntoSorter(pParse, pSort, p, r1+nPrefixReg, regOrig, 1, nPrefixReg); }else{ int r2 = sqlite3GetTempReg(pParse); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, r2); @@ -113724,31 +145036,61 @@ static void selectInnerLoop( break; } + case SRT_Upfrom: { + if( pSort ){ + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + }else{ + int i2 = pDest->iSDParm2; + int r1 = sqlite3GetTempReg(pParse); + + /* If the UPDATE FROM join is an aggregate that matches no rows, it + ** might still be trying to return one row, because that is what + ** aggregates do. Don't record that empty row in the output table. */ + sqlite3VdbeAddOp2(v, OP_IsNull, regResult, iBreak); VdbeCoverage(v); + + sqlite3VdbeAddOp3(v, OP_MakeRecord, + regResult+(i2<0), nResultCol-(i2<0), r1); + if( i2<0 ){ + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regResult); + }else{ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, i2); + } + } + break; + } + #ifndef SQLITE_OMIT_SUBQUERY /* If we are creating a set for an "expr IN (SELECT ...)" construct, ** then there should be a single item on the stack. Write this ** item into the set table with bogus data. */ case SRT_Set: { - assert( nResultCol==1 ); - pDest->affSdst = - sqlite3CompareAffinity(pEList->a[0].pExpr, pDest->affSdst); if( pSort ){ /* At first glance you would think we could optimize out the ** ORDER BY in this case since the order of entries in the set ** does not matter. But there might be a LIMIT clause, in which ** case the order does matter */ - pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); + pDest->iSDParm2 = 0; /* Signal that any Bloom filter is unpopulated */ }else{ int r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult,1,r1, &pDest->affSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regResult, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + assert( sqlite3Strlen30(pDest->zAffSdst)==nResultCol ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regResult, nResultCol, + r1, pDest->zAffSdst, nResultCol); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regResult, nResultCol); + if( pDest->iSDParm2 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + regResult, nResultCol); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); } break; } + /* If any row exist in the result set, record that fact and abort. */ case SRT_Exists: { @@ -113758,14 +145100,16 @@ static void selectInnerLoop( } /* If this is a scalar select that is part of an expression, then - ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** store the results in the appropriate memory cell or array of + ** memory cells and break out of the scan loop. */ case SRT_Mem: { - assert( nResultCol==1 ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, 1, nPrefixReg); + assert( nResultCol<=pDest->nSdst ); + pushOntoSorter( + pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else{ + assert( nResultCol==pDest->nSdst ); assert( regResult==iParm ); /* The LIMIT clause will jump out of the loop for us */ } @@ -113778,13 +145122,12 @@ static void selectInnerLoop( testcase( eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); if( pSort ){ - pushOntoSorter(pParse, pSort, p, regResult, regResult, nResultCol, + pushOntoSorter(pParse, pSort, p, regResult, regOrig, nResultCol, nPrefixReg); }else if( eDest==SRT_Coroutine ){ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); }else{ sqlite3VdbeAddOp2(v, OP_ResultRow, regResult, nResultCol); - sqlite3ExprCacheAffinityChange(pParse, regResult, nResultCol); } break; } @@ -113812,7 +145155,7 @@ static void selectInnerLoop( /* If the destination is DistQueue, then cursor (iParm+1) is open ** on a second ephemeral index that holds all values every previously ** added to the queue. */ - addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, + addrTest = sqlite3VdbeAddOp4Int(v, OP_Found, iParm+1, 0, regResult, nResultCol); VdbeCoverage(v); } @@ -113828,7 +145171,7 @@ static void selectInnerLoop( } sqlite3VdbeAddOp2(v, OP_Sequence, iParm, r2+nKey); sqlite3VdbeAddOp3(v, OP_MakeRecord, r2, nKey+2, r1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, r1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, r2, nKey+2); if( addrTest ) sqlite3VdbeJumpHere(v, addrTest); sqlite3ReleaseTempReg(pParse, r1); sqlite3ReleaseTempRange(pParse, r2, nKey+2); @@ -113865,18 +145208,18 @@ static void selectInnerLoop( ** X extra columns. */ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ - int nExtra = (N+X)*(sizeof(CollSeq*)+1); - KeyInfo *p = sqlite3Malloc(sizeof(KeyInfo) + nExtra); + int nExtra = (N+X)*(sizeof(CollSeq*)+1) - sizeof(CollSeq*); + KeyInfo *p = sqlite3DbMallocRawNN(db, sizeof(KeyInfo) + nExtra); if( p ){ - p->aSortOrder = (u8*)&p->aColl[N+X]; - p->nField = (u16)N; - p->nXField = (u16)X; + p->aSortFlags = (u8*)&p->aColl[N+X]; + p->nKeyField = (u16)N; + p->nAllField = (u16)(N+X); p->enc = ENC(db); p->db = db; p->nRef = 1; memset(&p[1], 0, nExtra); }else{ - sqlite3OomFault(db); + return (KeyInfo*)sqlite3OomFault(db); } return p; } @@ -113886,9 +145229,10 @@ SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoAlloc(sqlite3 *db, int N, int X){ */ SQLITE_PRIVATE void sqlite3KeyInfoUnref(KeyInfo *p){ if( p ){ + assert( p->db!=0 ); assert( p->nRef>0 ); p->nRef--; - if( p->nRef==0 ) sqlite3DbFree(0, p); + if( p->nRef==0 ) sqlite3DbNNFreeNN(p->db, p); } } @@ -113927,7 +145271,7 @@ SQLITE_PRIVATE int sqlite3KeyInfoIsWriteable(KeyInfo *p){ return p->nRef==1; } ** function is responsible for seeing that this structure is eventually ** freed. */ -static KeyInfo *keyInfoFromExprList( +SQLITE_PRIVATE KeyInfo *sqlite3KeyInfoFromExprList( Parse *pParse, /* Parsing context */ ExprList *pList, /* Form the KeyInfo object from this ExprList */ int iStart, /* Begin with this column of pList */ @@ -113944,11 +145288,8 @@ static KeyInfo *keyInfoFromExprList( if( pInfo ){ assert( sqlite3KeyInfoIsWriteable(pInfo) ); for(i=iStart, pItem=pList->a+iStart; ipExpr); - if( !pColl ) pColl = db->pDfltColl; - pInfo->aColl[i-iStart] = pColl; - pInfo->aSortOrder[i-iStart] = pItem->sortOrder; + pInfo->aColl[i-iStart] = sqlite3ExprNNCollSeq(pParse, pItem->pExpr); + pInfo->aSortFlags[i-iStart] = pItem->fg.sortFlags; } } return pInfo; @@ -113957,7 +145298,7 @@ static KeyInfo *keyInfoFromExprList( /* ** Name of the connection operator, used for error messages. */ -static const char *selectOpName(int id){ +SQLITE_PRIVATE const char *sqlite3SelectOpName(int id){ char *z; switch( id ){ case TK_ALL: z = "UNION ALL"; break; @@ -113980,11 +145321,7 @@ static const char *selectOpName(int id){ ** is determined by the zUsage argument. */ static void explainTempTable(Parse *pParse, const char *zUsage){ - if( pParse->explain==2 ){ - Vdbe *v = pParse->pVdbe; - char *zMsg = sqlite3MPrintf(pParse->db, "USE TEMP B-TREE FOR %s", zUsage); - sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); - } + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s", zUsage)); } /* @@ -114002,42 +145339,6 @@ static void explainTempTable(Parse *pParse, const char *zUsage){ # define explainSetInteger(y,z) #endif -#if !defined(SQLITE_OMIT_EXPLAIN) && !defined(SQLITE_OMIT_COMPOUND_SELECT) -/* -** Unless an "EXPLAIN QUERY PLAN" command is being processed, this function -** is a no-op. Otherwise, it adds a single row of output to the EQP result, -** where the caption is of one of the two forms: -** -** "COMPOSITE SUBQUERIES iSub1 and iSub2 (op)" -** "COMPOSITE SUBQUERIES iSub1 and iSub2 USING TEMP B-TREE (op)" -** -** where iSub1 and iSub2 are the integers passed as the corresponding -** function parameters, and op is the text representation of the parameter -** of the same name. The parameter "op" must be one of TK_UNION, TK_EXCEPT, -** TK_INTERSECT or TK_ALL. The first form is used if argument bUseTmp is -** false, or the second form if it is true. -*/ -static void explainComposite( - Parse *pParse, /* Parse context */ - int op, /* One of TK_UNION, TK_EXCEPT etc. */ - int iSub1, /* Subquery id 1 */ - int iSub2, /* Subquery id 2 */ - int bUseTmp /* True if a temp table was used */ -){ - assert( op==TK_UNION || op==TK_EXCEPT || op==TK_INTERSECT || op==TK_ALL ); - if( pParse->explain==2 ){ - Vdbe *v = pParse->pVdbe; - char *zMsg = sqlite3MPrintf( - pParse->db, "COMPOUND SUBQUERIES %d AND %d %s(%s)", iSub1, iSub2, - bUseTmp?"USING TEMP B-TREE ":"", selectOpName(op) - ); - sqlite3VdbeAddOp4(v, OP_Explain, pParse->iSelectId, 0, 0, zMsg, P4_DYNAMIC); - } -} -#else -/* No-op versions of the explainXXX() functions and macros. */ -# define explainComposite(v,w,x,y,z) -#endif /* ** If the inner loop was generated using a non-null pOrderBy argument, @@ -114054,8 +145355,8 @@ static void generateSortTail( ){ Vdbe *v = pParse->pVdbe; /* The prepared statement */ int addrBreak = pSort->labelDone; /* Jump here to exit loop */ - int addrContinue = sqlite3VdbeMakeLabel(v); /* Jump here for next cycle */ - int addr; + int addrContinue = sqlite3VdbeMakeLabel(pParse);/* Jump here for next cycle */ + int addr; /* Top of output loop. Jump for Next. */ int addrOnce = 0; int iTab; ExprList *pOrderBy = pSort->pOrderBy; @@ -114063,43 +145364,76 @@ static void generateSortTail( int iParm = pDest->iSDParm; int regRow; int regRowid; - int nKey; + int iCol; + int nKey; /* Number of key columns in sorter record */ int iSortTab; /* Sorter cursor to read from */ - int nSortData; /* Trailing values to read from sorter */ int i; int bSeq; /* True if sorter record includes seq. no. */ -#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + int nRefKey = 0; struct ExprList_item *aOutEx = p->pEList->a; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; /* Address of OP_Explain instruction */ #endif + nKey = pOrderBy->nExpr - pSort->nOBSat; + if( pSort->nOBSat==0 || nKey==1 ){ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR %sORDER BY", pSort->nOBSat?"LAST TERM OF ":"" + )); + }else{ + ExplainQueryPlan2(addrExplain, (pParse, 0, + "USE TEMP B-TREE FOR LAST %d TERMS OF ORDER BY", nKey + )); + } + sqlite3VdbeScanStatusRange(v, addrExplain,pSort->addrPush,pSort->addrPushEnd); + sqlite3VdbeScanStatusCounters(v, addrExplain, addrExplain, pSort->addrPush); + + assert( addrBreak<0 ); if( pSort->labelBkOut ){ sqlite3VdbeAddOp2(v, OP_Gosub, pSort->regReturn, pSort->labelBkOut); sqlite3VdbeGoto(v, addrBreak); sqlite3VdbeResolveLabel(v, pSort->labelBkOut); } + +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + /* Open any cursors needed for sorter-reference expressions */ + for(i=0; inDefer; i++){ + Table *pTab = pSort->aDefer[i].pTab; + int iDb = sqlite3SchemaToIndex(pParse->db, pTab->pSchema); + sqlite3OpenTable(pParse, pSort->aDefer[i].iCsr, iDb, pTab, OP_OpenRead); + nRefKey = MAX(nRefKey, pSort->aDefer[i].nKey); + } +#endif + iTab = pSort->iECursor; - if( eDest==SRT_Output || eDest==SRT_Coroutine ){ + if( eDest==SRT_Output || eDest==SRT_Coroutine || eDest==SRT_Mem ){ + if( eDest==SRT_Mem && p->iOffset ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pDest->iSdst); + } regRowid = 0; regRow = pDest->iSdst; - nSortData = nColumn; }else{ regRowid = sqlite3GetTempReg(pParse); - regRow = sqlite3GetTempReg(pParse); - nSortData = 1; + if( eDest==SRT_EphemTab || eDest==SRT_Table ){ + regRow = sqlite3GetTempReg(pParse); + nColumn = 0; + }else{ + regRow = sqlite3GetTempRange(pParse, nColumn); + } } - nKey = pOrderBy->nExpr - pSort->nOBSat; if( pSort->sortFlags & SORTFLAG_UseSorter ){ int regSortOut = ++pParse->nMem; iSortTab = pParse->nTab++; if( pSort->labelBkOut ){ - addrOnce = sqlite3CodeOnce(pParse); VdbeCoverage(v); + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); } - sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, nKey+1+nSortData); + sqlite3VdbeAddOp3(v, OP_OpenPseudo, iSortTab, regSortOut, + nKey+1+nColumn+nRefKey); if( addrOnce ) sqlite3VdbeJumpHere(v, addrOnce); addr = 1 + sqlite3VdbeAddOp2(v, OP_SorterSort, iTab, addrBreak); VdbeCoverage(v); - codeOffset(v, p->iOffset, addrContinue); + assert( p->iLimit==0 && p->iOffset==0 ); sqlite3VdbeAddOp3(v, OP_SorterData, iTab, regSortOut, iSortTab); bSeq = 0; }else{ @@ -114107,13 +145441,69 @@ static void generateSortTail( codeOffset(v, p->iOffset, addrContinue); iSortTab = iTab; bSeq = 1; + if( p->iOffset>0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, p->iLimit, -1); + } } - for(i=0; inDefer ){ + int iKey = iCol+1; + int regKey = sqlite3GetTempRange(pParse, nRefKey); + + for(i=0; inDefer; i++){ + int iCsr = pSort->aDefer[i].iCsr; + Table *pTab = pSort->aDefer[i].pTab; + int nKey = pSort->aDefer[i].nKey; + + sqlite3VdbeAddOp1(v, OP_NullRow, iCsr); + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iKey++, regKey); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCsr, + sqlite3VdbeCurrentAddr(v)+1, regKey); + }else{ + int k; + int iJmp; + assert( sqlite3PrimaryKeyIndex(pTab)->nKeyCol==nKey ); + for(k=0; k=0; i--){ +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + if( aOutEx[i].fg.bSorterRef ){ + sqlite3ExprCode(pParse, aOutEx[i].pExpr, regRow+i); + }else +#endif + { + int iRead; + if( aOutEx[i].u.x.iOrderByCol ){ + iRead = aOutEx[i].u.x.iOrderByCol-1; + }else{ + iRead = iCol--; + } + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, iRead, regRow+i); + VdbeComment((v, "%s", aOutEx[i].zEName)); + } + } + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); switch( eDest ){ + case SRT_Table: case SRT_EphemTab: { + sqlite3VdbeAddOp3(v, OP_Column, iSortTab, nKey+bSeq, regRow); sqlite3VdbeAddOp2(v, OP_NewRowid, iParm, regRowid); sqlite3VdbeAddOp3(v, OP_Insert, iParm, regRow, regRowid); sqlite3VdbeChangeP5(v, OPFLAG_APPEND); @@ -114121,27 +145511,34 @@ static void generateSortTail( } #ifndef SQLITE_OMIT_SUBQUERY case SRT_Set: { - assert( nColumn==1 ); - sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, 1, regRowid, - &pDest->affSdst, 1); - sqlite3ExprCacheAffinityChange(pParse, regRow, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iParm, regRowid); + assert( nColumn==sqlite3Strlen30(pDest->zAffSdst) ); + sqlite3VdbeAddOp4(v, OP_MakeRecord, regRow, nColumn, regRowid, + pDest->zAffSdst, nColumn); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, regRowid, regRow, nColumn); break; } case SRT_Mem: { - assert( nColumn==1 ); - sqlite3ExprCodeMove(pParse, regRow, iParm, 1); /* The LIMIT clause will terminate the loop for us */ break; } #endif + case SRT_Upfrom: { + int i2 = pDest->iSDParm2; + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord,regRow+(i2<0),nColumn-(i2<0),r1); + if( i2<0 ){ + sqlite3VdbeAddOp3(v, OP_Insert, iParm, r1, regRow); + }else{ + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iParm, r1, regRow, i2); + } + break; + } default: { - assert( eDest==SRT_Output || eDest==SRT_Coroutine ); + assert( eDest==SRT_Output || eDest==SRT_Coroutine ); testcase( eDest==SRT_Output ); testcase( eDest==SRT_Coroutine ); if( eDest==SRT_Output ){ sqlite3VdbeAddOp2(v, OP_ResultRow, pDest->iSdst, nColumn); - sqlite3ExprCacheAffinityChange(pParse, pDest->iSdst, nColumn); }else{ sqlite3VdbeAddOp1(v, OP_Yield, pDest->iSDParm); } @@ -114149,7 +145546,11 @@ static void generateSortTail( } } if( regRowid ){ - sqlite3ReleaseTempReg(pParse, regRow); + if( eDest==SRT_Set ){ + sqlite3ReleaseTempRange(pParse, regRow, nColumn); + }else{ + sqlite3ReleaseTempReg(pParse, regRow); + } sqlite3ReleaseTempReg(pParse, regRowid); } /* The bottom of the loop @@ -114160,6 +145561,7 @@ static void generateSortTail( }else{ sqlite3VdbeAddOp2(v, OP_Next, iTab, addr); VdbeCoverage(v); } + sqlite3VdbeScanStatusRange(v, addrExplain, sqlite3VdbeCurrentAddr(v)-1, -1); if( pSort->regReturn ) sqlite3VdbeAddOp1(v, OP_Return, pSort->regReturn); sqlite3VdbeResolveLabel(v, addrBreak); } @@ -114168,44 +145570,41 @@ static void generateSortTail( ** Return a pointer to a string containing the 'declaration type' of the ** expression pExpr. The string may be treated as static by the caller. ** -** Also try to estimate the size of the returned value and return that -** result in *pEstWidth. -** ** The declaration type is the exact datatype definition extracted from the ** original CREATE TABLE statement if the expression is a column. The ** declaration type for a ROWID field is INTEGER. Exactly when an expression ** is considered a column can be complex in the presence of subqueries. The -** result-set expression in all of the following SELECT statements is +** result-set expression in all of the following SELECT statements is ** considered a column by this function. ** ** SELECT col FROM tbl; ** SELECT (SELECT col FROM tbl; ** SELECT (SELECT col FROM tbl); ** SELECT abc FROM (SELECT col AS abc FROM tbl); -** +** ** The declaration type for any expression other than a column is NULL. ** ** This routine has either 3 or 6 parameters depending on whether or not ** the SQLITE_ENABLE_COLUMN_METADATA compile-time option is used. */ #ifdef SQLITE_ENABLE_COLUMN_METADATA -# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,C,D,E,F) +# define columnType(A,B,C,D,E) columnTypeImpl(A,B,C,D,E) #else /* if !defined(SQLITE_ENABLE_COLUMN_METADATA) */ -# define columnType(A,B,C,D,E,F) columnTypeImpl(A,B,F) +# define columnType(A,B,C,D,E) columnTypeImpl(A,B) #endif static const char *columnTypeImpl( - NameContext *pNC, + NameContext *pNC, +#ifndef SQLITE_ENABLE_COLUMN_METADATA + Expr *pExpr +#else Expr *pExpr, -#ifdef SQLITE_ENABLE_COLUMN_METADATA const char **pzOrigDb, const char **pzOrigTab, - const char **pzOrigCol, + const char **pzOrigCol #endif - u8 *pEstWidth ){ char const *zType = 0; int j; - u8 estWidth = 1; #ifdef SQLITE_ENABLE_COLUMN_METADATA char const *zOrigDb = 0; char const *zOrigTab = 0; @@ -114215,7 +145614,6 @@ static const char *columnTypeImpl( assert( pExpr!=0 ); assert( pNC->pSrcList!=0 ); switch( pExpr->op ){ - case TK_AGG_COLUMN: case TK_COLUMN: { /* The expression is a column. Locate the table the column is being ** extracted from in NameContext.pSrcList. This table may be real @@ -114224,14 +145622,16 @@ static const char *columnTypeImpl( Table *pTab = 0; /* Table structure column is extracted from */ Select *pS = 0; /* Select the column is extracted from */ int iCol = pExpr->iColumn; /* Index of column in pTab */ - testcase( pExpr->op==TK_AGG_COLUMN ); - testcase( pExpr->op==TK_COLUMN ); while( pNC && !pTab ){ SrcList *pTabList = pNC->pSrcList; for(j=0;jnSrc && pTabList->a[j].iCursor!=pExpr->iTable;j++); if( jnSrc ){ - pTab = pTabList->a[j].pTab; - pS = pTabList->a[j].pSelect; + pTab = pTabList->a[j].pSTab; + if( pTabList->a[j].fg.isSubquery ){ + pS = pTabList->a[j].u4.pSubq->pSelect; + }else{ + pS = 0; + } }else{ pNC = pNC->pNext; } @@ -114240,70 +145640,68 @@ static const char *columnTypeImpl( if( pTab==0 ){ /* At one time, code such as "SELECT new.x" within a trigger would ** cause this condition to run. Since then, we have restructured how - ** trigger code is generated and so this condition is no longer + ** trigger code is generated and so this condition is no longer ** possible. However, it can still be true for statements like ** the following: ** ** CREATE TABLE t1(col INTEGER); ** SELECT (SELECT t1.col) FROM FROM t1; ** - ** when columnType() is called on the expression "t1.col" in the + ** when columnType() is called on the expression "t1.col" in the ** sub-select. In this case, set the column type to NULL, even ** though it should really be "INTEGER". ** ** This is not a problem, as the column type of "t1.col" is never - ** used. When columnType() is called on the expression + ** used. When columnType() is called on the expression ** "(SELECT t1.col)", the correct type is returned (see the TK_SELECT ** branch below. */ break; } - assert( pTab && pExpr->pTab==pTab ); + assert( pTab && ExprUseYTab(pExpr) && pExpr->y.pTab==pTab ); if( pS ){ /* The "table" is actually a sub-select or a view in the FROM clause ** of the SELECT statement. Return the declaration type and origin ** data for the result-set column of the sub-select. */ - if( iCol>=0 && ALWAYS(iColpEList->nExpr) ){ + if( iColpEList->nExpr + && (!ViewCanHaveRowid || iCol>=0) + ){ /* If iCol is less than zero, then the expression requests the - ** rowid of the sub-select or view. This expression is legal (see + ** rowid of the sub-select or view. This expression is legal (see ** test case misc2.2.2) - it always evaluates to NULL. - ** - ** The ALWAYS() is because iCol>=pS->pEList->nExpr will have been - ** caught already by name resolution. */ NameContext sNC; Expr *p = pS->pEList->a[iCol].pExpr; sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol, &estWidth); + zType = columnType(&sNC, p,&zOrigDb,&zOrigTab,&zOrigCol); } - }else if( pTab->pSchema ){ - /* A real table */ + }else{ + /* A real table or a CTE table */ assert( !pS ); - if( iCol<0 ) iCol = pTab->iPKey; - assert( iCol==-1 || (iCol>=0 && iColnCol) ); #ifdef SQLITE_ENABLE_COLUMN_METADATA + if( iCol<0 ) iCol = pTab->iPKey; + assert( iCol==XN_ROWID || (iCol>=0 && iColnCol) ); if( iCol<0 ){ zType = "INTEGER"; zOrigCol = "rowid"; }else{ - zOrigCol = pTab->aCol[iCol].zName; + zOrigCol = pTab->aCol[iCol].zCnName; zType = sqlite3ColumnType(&pTab->aCol[iCol],0); - estWidth = pTab->aCol[iCol].szEst; } zOrigTab = pTab->zName; - if( pNC->pParse ){ + if( pNC->pParse && pTab->pSchema ){ int iDb = sqlite3SchemaToIndex(pNC->pParse->db, pTab->pSchema); - zOrigDb = pNC->pParse->db->aDb[iDb].zName; + zOrigDb = pNC->pParse->db->aDb[iDb].zDbSName; } #else + assert( iCol==XN_ROWID || (iCol>=0 && iColnCol) ); if( iCol<0 ){ zType = "INTEGER"; }else{ zType = sqlite3ColumnType(&pTab->aCol[iCol],0); - estWidth = pTab->aCol[iCol].szEst; } #endif } @@ -114316,19 +145714,21 @@ static const char *columnTypeImpl( ** statement. */ NameContext sNC; - Select *pS = pExpr->x.pSelect; - Expr *p = pS->pEList->a[0].pExpr; - assert( ExprHasProperty(pExpr, EP_xIsSelect) ); + Select *pS; + Expr *p; + assert( ExprUseXSelect(pExpr) ); + pS = pExpr->x.pSelect; + p = pS->pEList->a[0].pExpr; sNC.pSrcList = pS->pSrc; sNC.pNext = pNC; sNC.pParse = pNC->pParse; - zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, &estWidth); + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); break; } #endif } -#ifdef SQLITE_ENABLE_COLUMN_METADATA +#ifdef SQLITE_ENABLE_COLUMN_METADATA if( pzOrigDb ){ assert( pzOrigTab && pzOrigCol ); *pzOrigDb = zOrigDb; @@ -114336,7 +145736,6 @@ static const char *columnTypeImpl( *pzOrigCol = zOrigCol; } #endif - if( pEstWidth ) *pEstWidth = estWidth; return zType; } @@ -114355,6 +145754,7 @@ static void generateColumnTypes( NameContext sNC; sNC.pSrcList = pTabList; sNC.pParse = pParse; + sNC.pNext = 0; for(i=0; inExpr; i++){ Expr *p = pEList->a[i].pExpr; const char *zType; @@ -114362,9 +145762,9 @@ static void generateColumnTypes( const char *zOrigDb = 0; const char *zOrigTab = 0; const char *zOrigCol = 0; - zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol, 0); + zType = columnType(&sNC, p, &zOrigDb, &zOrigTab, &zOrigCol); - /* The vdbe must make its own copy of the column-type and other + /* The vdbe must make its own copy of the column-type and other ** column specific strings, in case the schema is reset before this ** virtual machine is deleted. */ @@ -114372,69 +145772,93 @@ static void generateColumnTypes( sqlite3VdbeSetColName(v, i, COLNAME_TABLE, zOrigTab, SQLITE_TRANSIENT); sqlite3VdbeSetColName(v, i, COLNAME_COLUMN, zOrigCol, SQLITE_TRANSIENT); #else - zType = columnType(&sNC, p, 0, 0, 0, 0); + zType = columnType(&sNC, p, 0, 0, 0); #endif sqlite3VdbeSetColName(v, i, COLNAME_DECLTYPE, zType, SQLITE_TRANSIENT); } #endif /* !defined(SQLITE_OMIT_DECLTYPE) */ } + /* -** Generate code that will tell the VDBE the names of columns -** in the result set. This information is used to provide the -** azCol[] values in the callback. +** Compute the column names for a SELECT statement. +** +** The only guarantee that SQLite makes about column names is that if the +** column has an AS clause assigning it a name, that will be the name used. +** That is the only documented guarantee. However, countless applications +** developed over the years have made baseless assumptions about column names +** and will break if those assumptions changes. Hence, use extreme caution +** when modifying this routine to avoid breaking legacy. +** +** See Also: sqlite3ColumnsFromExprList() +** +** The PRAGMA short_column_names and PRAGMA full_column_names settings are +** deprecated. The default setting is short=ON, full=OFF. 99.9% of all +** applications should operate this way. Nevertheless, we need to support the +** other modes for legacy: +** +** short=OFF, full=OFF: Column name is the text of the expression has it +** originally appears in the SELECT statement. In +** other words, the zSpan of the result expression. +** +** short=ON, full=OFF: (This is the default setting). If the result +** refers directly to a table column, then the +** result column name is just the table column +** name: COLUMN. Otherwise use zSpan. +** +** full=ON, short=ANY: If the result refers directly to a table column, +** then the result column name with the table name +** prefix, ex: TABLE.COLUMN. Otherwise use zSpan. */ -static void generateColumnNames( +SQLITE_PRIVATE void sqlite3GenerateColumnNames( Parse *pParse, /* Parser context */ - SrcList *pTabList, /* List of tables */ - ExprList *pEList /* Expressions defining the result set */ + Select *pSelect /* Generate column names for this SELECT statement */ ){ Vdbe *v = pParse->pVdbe; - int i, j; + int i; + Table *pTab; + SrcList *pTabList; + ExprList *pEList; sqlite3 *db = pParse->db; - int fullNames, shortNames; + int fullName; /* TABLE.COLUMN if no AS clause and is a direct table ref */ + int srcName; /* COLUMN or TABLE.COLUMN if no AS clause and is direct */ -#ifndef SQLITE_OMIT_EXPLAIN - /* If this is an EXPLAIN, skip this step */ - if( pParse->explain ){ - return; - } -#endif - - if( pParse->colNamesSet || db->mallocFailed ) return; + if( pParse->colNamesSet ) return; + /* Column names are determined by the left-most term of a compound select */ + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + TREETRACE(0x80,pParse,pSelect,("generating column names\n")); + pTabList = pSelect->pSrc; + pEList = pSelect->pEList; assert( v!=0 ); assert( pTabList!=0 ); pParse->colNamesSet = 1; - fullNames = (db->flags & SQLITE_FullColNames)!=0; - shortNames = (db->flags & SQLITE_ShortColNames)!=0; + fullName = (db->flags & SQLITE_FullColNames)!=0; + srcName = (db->flags & SQLITE_ShortColNames)!=0 || fullName; sqlite3VdbeSetNumCols(v, pEList->nExpr); for(i=0; inExpr; i++){ - Expr *p; - p = pEList->a[i].pExpr; - if( NEVER(p==0) ) continue; - if( pEList->a[i].zName ){ - char *zName = pEList->a[i].zName; + Expr *p = pEList->a[i].pExpr; + + assert( p!=0 ); + assert( p->op!=TK_AGG_COLUMN ); /* Agg processing has not run yet */ + assert( p->op!=TK_COLUMN + || (ExprUseYTab(p) && p->y.pTab!=0) ); /* Covering idx not yet coded */ + if( pEList->a[i].zEName && pEList->a[i].fg.eEName==ENAME_NAME ){ + /* An AS clause always takes first priority */ + char *zName = pEList->a[i].zEName; sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_TRANSIENT); - }else if( p->op==TK_COLUMN || p->op==TK_AGG_COLUMN ){ - Table *pTab; + }else if( srcName && p->op==TK_COLUMN ){ char *zCol; int iCol = p->iColumn; - for(j=0; ALWAYS(jnSrc); j++){ - if( pTabList->a[j].iCursor==p->iTable ) break; - } - assert( jnSrc ); - pTab = pTabList->a[j].pTab; + pTab = p->y.pTab; + assert( pTab!=0 ); if( iCol<0 ) iCol = pTab->iPKey; assert( iCol==-1 || (iCol>=0 && iColnCol) ); if( iCol<0 ){ zCol = "rowid"; }else{ - zCol = pTab->aCol[iCol].zName; + zCol = pTab->aCol[iCol].zCnName; } - if( !shortNames && !fullNames ){ - sqlite3VdbeSetColName(v, i, COLNAME_NAME, - sqlite3DbStrDup(db, pEList->a[i].zSpan), SQLITE_DYNAMIC); - }else if( fullNames ){ + if( fullName ){ char *zName = 0; zName = sqlite3MPrintf(db, "%s.%s", pTab->zName, zCol); sqlite3VdbeSetColName(v, i, COLNAME_NAME, zName, SQLITE_DYNAMIC); @@ -114442,7 +145866,7 @@ static void generateColumnNames( sqlite3VdbeSetColName(v, i, COLNAME_NAME, zCol, SQLITE_TRANSIENT); } }else{ - const char *z = pEList->a[i].zSpan; + const char *z = pEList->a[i].zEName; z = z==0 ? sqlite3MPrintf(db, "column%d", i+1) : sqlite3DbStrDup(db, z); sqlite3VdbeSetColName(v, i, COLNAME_NAME, z, SQLITE_DYNAMIC); } @@ -114462,6 +145886,15 @@ static void generateColumnNames( ** ** Return SQLITE_OK on success. If a memory allocation error occurs, ** store NULL in *paCol and 0 in *pnCol and return SQLITE_NOMEM. +** +** The only guarantee that SQLite makes about column names is that if the +** column has an AS clause assigning it a name, that will be the name used. +** That is the only documented guarantee. However, countless applications +** developed over the years have made baseless assumptions about column names +** and will break if those assumptions changes. Hence, use extreme caution +** when modifying this routine to avoid breaking legacy. +** +** See Also: sqlite3GenerateColumnNames() */ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( Parse *pParse, /* Parsing context */ @@ -114474,16 +145907,17 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( u32 cnt; /* Index added to make the name unique */ Column *aCol, *pCol; /* For looping over result columns */ int nCol; /* Number of columns in the result set */ - Expr *p; /* Expression for a single result column */ char *zName; /* Column name */ int nName; /* Size of name in zName[] */ Hash ht; /* Hash table of column names */ + Table *pTab; sqlite3HashInit(&ht); if( pEList ){ nCol = pEList->nExpr; aCol = sqlite3DbMallocZero(db, sizeof(aCol[0])*nCol); testcase( aCol==0 ); + if( NEVER(nCol>32767) ) nCol = 32767; }else{ nCol = 0; aCol = 0; @@ -114492,151 +145926,209 @@ SQLITE_PRIVATE int sqlite3ColumnsFromExprList( *pnCol = nCol; *paCol = aCol; - for(i=0, pCol=aCol; imallocFailed; i++, pCol++){ + for(i=0, pCol=aCol; inErr; i++, pCol++){ + struct ExprList_item *pX = &pEList->a[i]; + struct ExprList_item *pCollide; /* Get an appropriate name for the column */ - p = sqlite3ExprSkipCollate(pEList->a[i].pExpr); - if( (zName = pEList->a[i].zName)!=0 ){ + if( (zName = pX->zEName)!=0 && pX->fg.eEName==ENAME_NAME ){ /* If the column contains an "AS " phrase, use as the name */ }else{ - Expr *pColExpr = p; /* The expression that is the result column name */ - Table *pTab; /* Table associated with this expression */ - while( pColExpr->op==TK_DOT ){ + Expr *pColExpr = sqlite3ExprSkipCollateAndLikely(pX->pExpr); + while( ALWAYS(pColExpr!=0) && pColExpr->op==TK_DOT ){ pColExpr = pColExpr->pRight; assert( pColExpr!=0 ); } - if( pColExpr->op==TK_COLUMN && ALWAYS(pColExpr->pTab!=0) ){ + if( pColExpr->op==TK_COLUMN + && ALWAYS( ExprUseYTab(pColExpr) ) + && ALWAYS( pColExpr->y.pTab!=0 ) + ){ /* For columns use the column name name */ int iCol = pColExpr->iColumn; - pTab = pColExpr->pTab; + pTab = pColExpr->y.pTab; if( iCol<0 ) iCol = pTab->iPKey; - zName = iCol>=0 ? pTab->aCol[iCol].zName : "rowid"; + zName = iCol>=0 ? pTab->aCol[iCol].zCnName : "rowid"; }else if( pColExpr->op==TK_ID ){ assert( !ExprHasProperty(pColExpr, EP_IntValue) ); zName = pColExpr->u.zToken; }else{ /* Use the original text of the column expression as its name */ - zName = pEList->a[i].zSpan; + assert( zName==pX->zEName ); /* pointer comparison intended */ } } - zName = sqlite3MPrintf(db, "%s", zName); + if( zName && !sqlite3IsTrueOrFalse(zName) ){ + zName = sqlite3DbStrDup(db, zName); + }else{ + zName = sqlite3MPrintf(db,"column%d",i+1); + } /* Make sure the column name is unique. If the name is not unique, ** append an integer to the name so that it becomes unique. */ cnt = 0; - while( zName && sqlite3HashFind(&ht, zName)!=0 ){ + while( zName && (pCollide = sqlite3HashFind(&ht, zName))!=0 ){ + if( pCollide->fg.bUsingTerm ){ + pCol->colFlags |= COLFLAG_NOEXPAND; + } nName = sqlite3Strlen30(zName); if( nName>0 ){ for(j=nName-1; j>0 && sqlite3Isdigit(zName[j]); j--){} if( zName[j]==':' ) nName = j; } zName = sqlite3MPrintf(db, "%.*z:%u", nName, zName, ++cnt); - if( cnt>3 ) sqlite3_randomness(sizeof(cnt), &cnt); + sqlite3ProgressCheck(pParse); + if( cnt>3 ){ + sqlite3_randomness(sizeof(cnt), &cnt); + } + } + pCol->zCnName = zName; + pCol->hName = sqlite3StrIHash(zName); + if( pX->fg.bNoExpand ){ + pCol->colFlags |= COLFLAG_NOEXPAND; } - pCol->zName = zName; sqlite3ColumnPropertiesFromName(0, pCol); - if( zName && sqlite3HashInsert(&ht, zName, pCol)==pCol ){ + if( zName && sqlite3HashInsert(&ht, zName, pX)==pX ){ sqlite3OomFault(db); } } sqlite3HashClear(&ht); - if( db->mallocFailed ){ + if( pParse->nErr ){ for(j=0; jrc; } return SQLITE_OK; } /* -** Add type and collation information to a column list based on -** a SELECT statement. -** -** The column list presumably came from selectColumnNamesFromExprList(). -** The column list has only names, not types or collations. This -** routine goes through and adds the types and collations. +** pTab is a transient Table object that represents a subquery of some +** kind (maybe a parenthesized subquery in the FROM clause of a larger +** query, or a VIEW, or a CTE). This routine computes type information +** for that Table object based on the Select object that implements the +** subquery. For the purposes of this routine, "type information" means: ** -** This routine requires that all identifiers in the SELECT -** statement be resolved. +** * The datatype name, as it might appear in a CREATE TABLE statement +** * Which collating sequence to use for the column +** * The affinity of the column */ -SQLITE_PRIVATE void sqlite3SelectAddColumnTypeAndCollation( - Parse *pParse, /* Parsing contexts */ - Table *pTab, /* Add column type information to this table */ - Select *pSelect /* SELECT used to determine types and collations */ +SQLITE_PRIVATE void sqlite3SubqueryColumnTypes( + Parse *pParse, /* Parsing contexts */ + Table *pTab, /* Add column type information to this table */ + Select *pSelect, /* SELECT used to determine types and collations */ + char aff /* Default affinity. */ ){ sqlite3 *db = pParse->db; - NameContext sNC; Column *pCol; CollSeq *pColl; - int i; + int i,j; Expr *p; struct ExprList_item *a; - u64 szAll = 0; + NameContext sNC; assert( pSelect!=0 ); assert( (pSelect->selFlags & SF_Resolved)!=0 ); - assert( pTab->nCol==pSelect->pEList->nExpr || db->mallocFailed ); - if( db->mallocFailed ) return; + assert( pTab->nCol==pSelect->pEList->nExpr || pParse->nErr>0 ); + assert( aff==SQLITE_AFF_NONE || aff==SQLITE_AFF_BLOB ); + if( db->mallocFailed || IN_RENAME_OBJECT ) return; + while( pSelect->pPrior ) pSelect = pSelect->pPrior; + a = pSelect->pEList->a; memset(&sNC, 0, sizeof(sNC)); sNC.pSrcList = pSelect->pSrc; - a = pSelect->pEList->a; for(i=0, pCol=pTab->aCol; inCol; i++, pCol++){ const char *zType; - int n, m; + i64 n; + int m = 0; + Select *pS2 = pSelect; + pTab->tabFlags |= (pCol->colFlags & COLFLAG_NOINSERT); p = a[i].pExpr; - zType = columnType(&sNC, p, 0, 0, 0, &pCol->szEst); - szAll += pCol->szEst; + /* pCol->szEst = ... // Column size est for SELECT tables never used */ pCol->affinity = sqlite3ExprAffinity(p); - if( zType && (m = sqlite3Strlen30(zType))>0 ){ - n = sqlite3Strlen30(pCol->zName); - pCol->zName = sqlite3DbReallocOrFree(db, pCol->zName, n+m+2); - if( pCol->zName ){ - memcpy(&pCol->zName[n+1], zType, m+1); + while( pCol->affinity<=SQLITE_AFF_NONE && pS2->pNext!=0 ){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + pS2 = pS2->pNext; + pCol->affinity = sqlite3ExprAffinity(pS2->pEList->a[i].pExpr); + } + if( pCol->affinity<=SQLITE_AFF_NONE ){ + pCol->affinity = aff; + } + if( pCol->affinity>=SQLITE_AFF_TEXT && (pS2->pNext || pS2!=pSelect) ){ + for(pS2=pS2->pNext; pS2; pS2=pS2->pNext){ + m |= sqlite3ExprDataType(pS2->pEList->a[i].pExpr); + } + if( pCol->affinity==SQLITE_AFF_TEXT && (m&0x01)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + }else + if( pCol->affinity>=SQLITE_AFF_NUMERIC && (m&0x02)!=0 ){ + pCol->affinity = SQLITE_AFF_BLOB; + } + if( pCol->affinity>=SQLITE_AFF_NUMERIC && p->op==TK_CAST ){ + pCol->affinity = SQLITE_AFF_FLEXNUM; + } + } + zType = columnType(&sNC, p, 0, 0, 0); + if( zType==0 || pCol->affinity!=sqlite3AffinityType(zType, 0) ){ + if( pCol->affinity==SQLITE_AFF_NUMERIC + || pCol->affinity==SQLITE_AFF_FLEXNUM + ){ + zType = "NUM"; + }else{ + zType = 0; + for(j=1; jaffinity ){ + zType = sqlite3StdType[j]; + break; + } + } + } + } + if( zType ){ + const i64 k = sqlite3Strlen30(zType); + n = sqlite3Strlen30(pCol->zCnName); + pCol->zCnName = sqlite3DbReallocOrFree(db, pCol->zCnName, n+k+2); + pCol->colFlags &= ~(COLFLAG_HASTYPE|COLFLAG_HASCOLL); + if( pCol->zCnName ){ + memcpy(&pCol->zCnName[n+1], zType, k+1); pCol->colFlags |= COLFLAG_HASTYPE; } } - if( pCol->affinity==0 ) pCol->affinity = SQLITE_AFF_BLOB; pColl = sqlite3ExprCollSeq(pParse, p); - if( pColl && pCol->zColl==0 ){ - pCol->zColl = sqlite3DbStrDup(db, pColl->zName); + if( pColl ){ + assert( pTab->pIndex==0 ); + sqlite3ColumnSetColl(db, pCol, pColl->zName); } } - pTab->szTabRow = sqlite3LogEst(szAll*4); + pTab->szTabRow = 1; /* Any non-zero value works */ } /* ** Given a SELECT statement, generate a Table structure that describes ** the result set of that SELECT. */ -SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ +SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect, char aff){ Table *pTab; sqlite3 *db = pParse->db; - int savedFlags; + u64 savedFlags; savedFlags = db->flags; - db->flags &= ~SQLITE_FullColNames; + db->flags &= ~(u64)SQLITE_FullColNames; db->flags |= SQLITE_ShortColNames; sqlite3SelectPrep(pParse, pSelect, 0); + db->flags = savedFlags; if( pParse->nErr ) return 0; while( pSelect->pPrior ) pSelect = pSelect->pPrior; - db->flags = savedFlags; pTab = sqlite3DbMallocZero(db, sizeof(Table) ); if( pTab==0 ){ return 0; } - /* The sqlite3ResultSetOfSelect() is only used n contexts where lookaside - ** is disabled */ - assert( db->lookaside.bDisable ); - pTab->nRef = 1; + pTab->nTabRef = 1; pTab->zName = 0; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); sqlite3ColumnsFromExprList(pParse, pSelect->pEList, &pTab->nCol, &pTab->aCol); - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSelect); + sqlite3SubqueryColumnTypes(pParse, pTab, pSelect, aff); pTab->iPKey = -1; if( db->mallocFailed ){ sqlite3DeleteTable(db, pTab); @@ -114649,41 +146141,38 @@ SQLITE_PRIVATE Table *sqlite3ResultSetOfSelect(Parse *pParse, Select *pSelect){ ** Get a VDBE for the given parser context. Create a new one if necessary. ** If an error occurs, return NULL and leave a message in pParse. */ -static SQLITE_NOINLINE Vdbe *allocVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe = sqlite3VdbeCreate(pParse); - if( v ) sqlite3VdbeAddOp0(v, OP_Init); +SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ + if( pParse->pVdbe ){ + return pParse->pVdbe; + } if( pParse->pToplevel==0 && OptimizationEnabled(pParse->db,SQLITE_FactorOutConst) ){ pParse->okConstFactor = 1; } - return v; -} -SQLITE_PRIVATE Vdbe *sqlite3GetVdbe(Parse *pParse){ - Vdbe *v = pParse->pVdbe; - return v ? v : allocVdbe(pParse); + return sqlite3VdbeCreate(pParse); } /* ** Compute the iLimit and iOffset fields of the SELECT based on the -** pLimit and pOffset expressions. pLimit and pOffset hold the expressions +** pLimit expressions. pLimit->pLeft and pLimit->pRight hold the expressions ** that appear in the original SQL statement after the LIMIT and OFFSET -** keywords. Or NULL if those keywords are omitted. iLimit and iOffset -** are the integer memory register numbers for counters used to compute -** the limit and offset. If there is no limit and/or offset, then +** keywords. Or NULL if those keywords are omitted. iLimit and iOffset +** are the integer memory register numbers for counters used to compute +** the limit and offset. If there is no limit and/or offset, then ** iLimit and iOffset are negative. ** ** This routine changes the values of iLimit and iOffset only if -** a limit or offset is defined by pLimit and pOffset. iLimit and -** iOffset should have been preset to appropriate default values (zero) +** a limit or offset is defined by pLimit->pLeft and pLimit->pRight. iLimit +** and iOffset should have been preset to appropriate default values (zero) ** prior to calling this routine. ** ** The iOffset register (if it exists) is initialized to the value ** of the OFFSET. The iLimit register is initialized to LIMIT. Register ** iOffset+1 is initialized to LIMIT+OFFSET. ** -** Only if pLimit!=0 or pOffset!=0 do the limit registers get +** Only if pLimit->pLeft!=0 do the limit registers get ** redefined. The UNION ALL operator uses this property to force ** the reuse of the same limit and offset registers across multiple ** SELECT statements. @@ -114693,21 +146182,23 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ int iLimit = 0; int iOffset; int n; + Expr *pLimit = p->pLimit; + if( p->iLimit ) return; - /* + /* ** "LIMIT -1" always shows all rows. There is some ** controversy about what the correct behavior should be. ** The current implementation interprets "LIMIT 0" to mean ** no rows. */ - sqlite3ExprCacheClear(pParse); - assert( p->pOffset==0 || p->pLimit!=0 ); - if( p->pLimit ){ + if( pLimit ){ + assert( pLimit->op==TK_LIMIT ); + assert( pLimit->pLeft!=0 ); p->iLimit = iLimit = ++pParse->nMem; v = sqlite3GetVdbe(pParse); assert( v!=0 ); - if( sqlite3ExprIsInteger(p->pLimit, &n) ){ + if( sqlite3ExprIsInteger(pLimit->pLeft, &n, pParse) ){ sqlite3VdbeAddOp2(v, OP_Integer, n, iLimit); VdbeComment((v, "LIMIT counter")); if( n==0 ){ @@ -114717,15 +146208,15 @@ static void computeLimitRegisters(Parse *pParse, Select *p, int iBreak){ p->selFlags |= SF_FixedLimit; } }else{ - sqlite3ExprCode(pParse, p->pLimit, iLimit); + sqlite3ExprCode(pParse, pLimit->pLeft, iLimit); sqlite3VdbeAddOp1(v, OP_MustBeInt, iLimit); VdbeCoverage(v); VdbeComment((v, "LIMIT counter")); sqlite3VdbeAddOp2(v, OP_IfNot, iLimit, iBreak); VdbeCoverage(v); } - if( p->pOffset ){ + if( pLimit->pRight ){ p->iOffset = iOffset = ++pParse->nMem; pParse->nMem++; /* Allocate an extra register for limit+offset */ - sqlite3ExprCode(pParse, p->pOffset, iOffset); + sqlite3ExprCode(pParse, pLimit->pRight, iOffset); sqlite3VdbeAddOp1(v, OP_MustBeInt, iOffset); VdbeCoverage(v); VdbeComment((v, "OFFSET counter")); sqlite3VdbeAddOp3(v, OP_OffsetLimit, iLimit, iOffset+1, iOffset); @@ -114771,7 +146262,7 @@ static CollSeq *multiSelectCollSeq(Parse *pParse, Select *p, int iCol){ */ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ExprList *pOrderBy = p->pOrderBy; - int nOrderBy = p->pOrderBy->nExpr; + int nOrderBy = ALWAYS(pOrderBy!=0) ? pOrderBy->nExpr : 0; sqlite3 *db = pParse->db; KeyInfo *pRet = sqlite3KeyInfoAlloc(db, nOrderBy+nExtra, 1); if( pRet ){ @@ -114791,7 +146282,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ } assert( sqlite3KeyInfoIsWriteable(pRet) ); pRet->aColl[i] = pColl; - pRet->aSortOrder[i] = pOrderBy->a[i].sortOrder; + pRet->aSortFlags[i] = pOrderBy->a[i].fg.sortFlags; } } @@ -114823,7 +146314,7 @@ static KeyInfo *multiSelectOrderByKeyInfo(Parse *pParse, Select *p, int nExtra){ ** inserted into the Queue table. The iDistinct table keeps a copy of all rows ** that have ever been inserted into Queue and causes duplicates to be ** discarded. If the operator is UNION ALL, then duplicates are allowed. -** +** ** If the query has an ORDER BY, then entries in the Queue table are kept in ** ORDER BY order and the first entry is extracted for each cycle. Without ** an ORDER BY, the Queue table is just a FIFO. @@ -114843,7 +146334,8 @@ static void generateWithRecursiveQuery( SrcList *pSrc = p->pSrc; /* The FROM clause of the recursive query */ int nCol = p->pEList->nExpr; /* Number of columns in the recursive table */ Vdbe *v = pParse->pVdbe; /* The prepared statement under construction */ - Select *pSetup = p->pPrior; /* The setup query */ + Select *pSetup; /* The setup query */ + Select *pFirstRec; /* Left-most recursive term */ int addrTop; /* Top of the loop */ int addrCont, addrBreak; /* CONTINUE and BREAK addresses */ int iCurrent = 0; /* The Current table */ @@ -114851,24 +146343,31 @@ static void generateWithRecursiveQuery( int iQueue; /* The Queue table */ int iDistinct = 0; /* To ensure unique results if UNION */ int eDest = SRT_Fifo; /* How to write to Queue */ - SelectDest destQueue; /* SelectDest targetting the Queue table */ + SelectDest destQueue; /* SelectDest targeting the Queue table */ int i; /* Loop counter */ int rc; /* Result code */ ExprList *pOrderBy; /* The ORDER BY clause */ - Expr *pLimit, *pOffset; /* Saved LIMIT and OFFSET */ + Expr *pLimit; /* Saved LIMIT and OFFSET */ int regLimit, regOffset; /* Registers used by LIMIT and OFFSET */ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ){ + sqlite3ErrorMsg(pParse, "cannot use window functions in recursive queries"); + return; + } +#endif + /* Obtain authorization to do a recursive query */ if( sqlite3AuthCheck(pParse, SQLITE_RECURSIVE, 0, 0, 0) ) return; /* Process the LIMIT and OFFSET clauses, if they exist */ - addrBreak = sqlite3VdbeMakeLabel(v); + addrBreak = sqlite3VdbeMakeLabel(pParse); + p->nSelectRow = 320; /* 4 billion rows */ computeLimitRegisters(pParse, p, addrBreak); pLimit = p->pLimit; - pOffset = p->pOffset; regLimit = p->iLimit; regOffset = p->iOffset; - p->pLimit = p->pOffset = 0; + p->pLimit = 0; p->iLimit = p->iOffset = 0; pOrderBy = p->pOrderBy; @@ -114912,8 +146411,26 @@ static void generateWithRecursiveQuery( /* Detach the ORDER BY clause from the compound SELECT */ p->pOrderBy = 0; + /* Figure out how many elements of the compound SELECT are part of the + ** recursive query. Make sure no recursive elements use aggregate + ** functions. Mark the recursive elements as UNION ALL even if they + ** are really UNION because the distinctness will be enforced by the + ** iDistinct table. pFirstRec is left pointing to the left-most + ** recursive term of the CTE. + */ + for(pFirstRec=p; ALWAYS(pFirstRec!=0); pFirstRec=pFirstRec->pPrior){ + if( pFirstRec->selFlags & SF_Aggregate ){ + sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); + goto end_of_recursive_query; + } + pFirstRec->op = TK_ALL; + if( (pFirstRec->pPrior->selFlags & SF_Recursive)==0 ) break; + } + /* Store the results of the setup-query in Queue. */ + pSetup = pFirstRec->pPrior; pSetup->pNext = 0; + ExplainQueryPlan((pParse, 1, "SETUP")); rc = sqlite3Select(pParse, pSetup, &destQueue); pSetup->pNext = p; if( rc ) goto end_of_recursive_query; @@ -114931,9 +146448,9 @@ static void generateWithRecursiveQuery( sqlite3VdbeAddOp1(v, OP_Delete, iQueue); /* Output the single row in Current */ - addrCont = sqlite3VdbeMakeLabel(v); + addrCont = sqlite3VdbeMakeLabel(pParse); codeOffset(v, regOffset, addrCont); - selectInnerLoop(pParse, p, p->pEList, iCurrent, + selectInnerLoop(pParse, p, iCurrent, 0, 0, pDest, addrCont, addrBreak); if( regLimit ){ sqlite3VdbeAddOp2(v, OP_DecrJumpZero, regLimit, addrBreak); @@ -114944,14 +146461,11 @@ static void generateWithRecursiveQuery( /* Execute the recursive SELECT taking the single row in Current as ** the value for the recursive-table. Store the results in the Queue. */ - if( p->selFlags & SF_Aggregate ){ - sqlite3ErrorMsg(pParse, "recursive aggregate queries not supported"); - }else{ - p->pPrior = 0; - sqlite3Select(pParse, p, &destQueue); - assert( p->pPrior==0 ); - p->pPrior = pSetup; - } + pFirstRec->pPrior = 0; + ExplainQueryPlan((pParse, 1, "RECURSIVE STEP")); + sqlite3Select(pParse, p, &destQueue); + assert( pFirstRec->pPrior==0 ); + pFirstRec->pPrior = pSetup; /* Keep running the loop until the Queue is empty */ sqlite3VdbeGoto(v, addrTop); @@ -114961,7 +146475,6 @@ end_of_recursive_query: sqlite3ExprListDelete(pParse->db, p->pOrderBy); p->pOrderBy = pOrderBy; p->pLimit = pLimit; - p->pOffset = pOffset; return; } #endif /* SQLITE_OMIT_CTE */ @@ -114980,42 +146493,57 @@ static int multiSelectOrderBy( ** on a VALUES clause. ** ** Because the Select object originates from a VALUES clause: -** (1) It has no LIMIT or OFFSET +** (1) There is no LIMIT or OFFSET or else there is a LIMIT of exactly 1 ** (2) All terms are UNION ALL ** (3) There is no ORDER BY clause +** +** The "LIMIT of exactly 1" case of condition (1) comes about when a VALUES +** clause occurs within scalar expression (ex: "SELECT (VALUES(1),(2),(3))"). +** The sqlite3CodeSubselect will have added the LIMIT 1 clause in tht case. +** Since the limit is exactly 1, we only need to evaluate the left-most VALUES. */ static int multiSelectValues( Parse *pParse, /* Parsing context */ Select *p, /* The right-most of SELECTs to be coded */ SelectDest *pDest /* What to do with query results */ ){ - Select *pPrior; int nRow = 1; int rc = 0; + int bShowAll = p->pLimit==0; assert( p->selFlags & SF_MultiValue ); do{ assert( p->selFlags & SF_Values ); assert( p->op==TK_ALL || (p->op==TK_SELECT && p->pPrior==0) ); - assert( p->pLimit==0 ); - assert( p->pOffset==0 ); assert( p->pNext==0 || p->pEList->nExpr==p->pNext->pEList->nExpr ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin ) return -1; +#endif if( p->pPrior==0 ) break; assert( p->pPrior->pNext==p ); p = p->pPrior; - nRow++; + nRow += bShowAll; }while(1); + ExplainQueryPlan((pParse, 0, "SCAN %d CONSTANT ROW%s", nRow, + nRow==1 ? "" : "S")); while( p ){ - pPrior = p->pPrior; - p->pPrior = 0; - rc = sqlite3Select(pParse, p, pDest); - p->pPrior = pPrior; - if( rc ) break; + selectInnerLoop(pParse, p, -1, 0, 0, pDest, 1, 1); + if( !bShowAll ) break; p->nSelectRow = nRow; p = p->pNext; } return rc; } +/* +** Return true if the SELECT statement which is known to be the recursive +** part of a recursive CTE still has its anchor terms attached. If the +** anchor terms have already been removed, then return false. +*/ +static int hasAnchor(Select *p){ + while( p && (p->selFlags & SF_Recursive)!=0 ){ p = p->pPrior; } + return p!=0; +} + /* ** This routine is called to process a compound query form from ** two or more separate queries using UNION, UNION ALL, EXCEPT, or @@ -115023,7 +146551,7 @@ static int multiSelectValues( ** ** "p" points to the right-most of the two queries. the query on the ** left is p->pPrior. The left query could also be a compound query -** in which case this routine will be called recursively. +** in which case this routine will be called recursively. ** ** The results of the total query are to be written into a destination ** of type eDest with parameter iParm. @@ -115058,31 +146586,18 @@ static int multiSelect( SelectDest dest; /* Alternative data destination */ Select *pDelete = 0; /* Chain of simple selects to delete */ sqlite3 *db; /* Database connection */ -#ifndef SQLITE_OMIT_EXPLAIN - int iSub1 = 0; /* EQP id of left-hand query */ - int iSub2 = 0; /* EQP id of right-hand query */ -#endif /* Make sure there is no ORDER BY or LIMIT clause on prior SELECTs. Only ** the last (right-most) SELECT in the series may have an ORDER BY or LIMIT. */ assert( p && p->pPrior ); /* Calling function guarantees this much */ assert( (p->selFlags & SF_Recursive)==0 || p->op==TK_ALL || p->op==TK_UNION ); + assert( p->selFlags & SF_Compound ); db = pParse->db; pPrior = p->pPrior; dest = *pDest; - if( pPrior->pOrderBy ){ - sqlite3ErrorMsg(pParse,"ORDER BY clause should come after %s not before", - selectOpName(p->op)); - rc = 1; - goto multi_select_end; - } - if( pPrior->pLimit ){ - sqlite3ErrorMsg(pParse,"LIMIT clause should come after %s not before", - selectOpName(p->op)); - rc = 1; - goto multi_select_end; - } + assert( pPrior->pOrderBy==0 ); + assert( pPrior->pLimit==0 ); v = sqlite3GetVdbe(pParse); assert( v!=0 ); /* The VDBE already created by calling function */ @@ -115099,7 +146614,8 @@ static int multiSelect( */ if( p->selFlags & SF_MultiValue ){ rc = multiSelectValues(pParse, p, &dest); - goto multi_select_end; + if( rc>=0 ) goto multi_select_end; + rc = SQLITE_OK; } /* Make sure all SELECTs in the statement have the same number of elements @@ -115109,7 +146625,7 @@ static int multiSelect( assert( p->pEList->nExpr==pPrior->pEList->nExpr ); #ifndef SQLITE_OMIT_CTE - if( p->selFlags & SF_Recursive ){ + if( (p->selFlags & SF_Recursive)!=0 && hasAnchor(p) ){ generateWithRecursiveQuery(pParse, p, &dest); }else #endif @@ -115118,237 +146634,239 @@ static int multiSelect( */ if( p->pOrderBy ){ return multiSelectOrderBy(pParse, p, pDest); - }else + }else{ - /* Generate code for the left and right SELECT statements. - */ - switch( p->op ){ - case TK_ALL: { - int addr = 0; - int nLimit; - assert( !pPrior->pLimit ); - pPrior->iLimit = p->iLimit; - pPrior->iOffset = p->iOffset; - pPrior->pLimit = p->pLimit; - pPrior->pOffset = p->pOffset; - explainSetInteger(iSub1, pParse->iNextSelectId); - rc = sqlite3Select(pParse, pPrior, &dest); - p->pLimit = 0; - p->pOffset = 0; - if( rc ){ - goto multi_select_end; - } - p->pPrior = 0; - p->iLimit = pPrior->iLimit; - p->iOffset = pPrior->iOffset; - if( p->iLimit ){ - addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); - VdbeComment((v, "Jump ahead if LIMIT reached")); - if( p->iOffset ){ - sqlite3VdbeAddOp3(v, OP_OffsetLimit, - p->iLimit, p->iOffset+1, p->iOffset); - } - } - explainSetInteger(iSub2, pParse->iNextSelectId); - rc = sqlite3Select(pParse, p, &dest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - if( pPrior->pLimit - && sqlite3ExprIsInteger(pPrior->pLimit, &nLimit) - && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) - ){ - p->nSelectRow = sqlite3LogEst((u64)nLimit); - } - if( addr ){ - sqlite3VdbeJumpHere(v, addr); - } - break; +#ifndef SQLITE_OMIT_EXPLAIN + if( pPrior->pPrior==0 ){ + ExplainQueryPlan((pParse, 1, "COMPOUND QUERY")); + ExplainQueryPlan((pParse, 1, "LEFT-MOST SUBQUERY")); } - case TK_EXCEPT: - case TK_UNION: { - int unionTab; /* Cursor number of the temporary table holding result */ - u8 op = 0; /* One of the SRT_ operations to apply to self */ - int priorOp; /* The SRT_ operation to apply to prior selects */ - Expr *pLimit, *pOffset; /* Saved values of p->nLimit and p->nOffset */ - int addr; - SelectDest uniondest; +#endif - testcase( p->op==TK_EXCEPT ); - testcase( p->op==TK_UNION ); - priorOp = SRT_Union; - if( dest.eDest==priorOp ){ - /* We can reuse a temporary table generated by a SELECT to our - ** right. + /* Generate code for the left and right SELECT statements. + */ + switch( p->op ){ + case TK_ALL: { + int addr = 0; + int nLimit = 0; /* Initialize to suppress harmless compiler warning */ + assert( !pPrior->pLimit ); + pPrior->iLimit = p->iLimit; + pPrior->iOffset = p->iOffset; + pPrior->pLimit = p->pLimit; + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL left...\n")); + rc = sqlite3Select(pParse, pPrior, &dest); + pPrior->pLimit = 0; + if( rc ){ + goto multi_select_end; + } + p->pPrior = 0; + p->iLimit = pPrior->iLimit; + p->iOffset = pPrior->iOffset; + if( p->iLimit ){ + addr = sqlite3VdbeAddOp1(v, OP_IfNot, p->iLimit); VdbeCoverage(v); + VdbeComment((v, "Jump ahead if LIMIT reached")); + if( p->iOffset ){ + sqlite3VdbeAddOp3(v, OP_OffsetLimit, + p->iLimit, p->iOffset+1, p->iOffset); + } + } + ExplainQueryPlan((pParse, 1, "UNION ALL")); + TREETRACE(0x200, pParse, p, ("multiSelect UNION ALL right...\n")); + rc = sqlite3Select(pParse, p, &dest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + if( p->pLimit + && sqlite3ExprIsInteger(p->pLimit->pLeft, &nLimit, pParse) + && nLimit>0 && p->nSelectRow > sqlite3LogEst((u64)nLimit) + ){ + p->nSelectRow = sqlite3LogEst((u64)nLimit); + } + if( addr ){ + sqlite3VdbeJumpHere(v, addr); + } + break; + } + case TK_EXCEPT: + case TK_UNION: { + int unionTab; /* Cursor number of the temp table holding result */ + u8 op = 0; /* One of the SRT_ operations to apply to self */ + int priorOp; /* The SRT_ operation to apply to prior selects */ + Expr *pLimit; /* Saved values of p->nLimit */ + int addr; + SelectDest uniondest; + + testcase( p->op==TK_EXCEPT ); + testcase( p->op==TK_UNION ); + priorOp = SRT_Union; + if( dest.eDest==priorOp ){ + /* We can reuse a temporary table generated by a SELECT to our + ** right. + */ + assert( p->pLimit==0 ); /* Not allowed on leftward elements */ + unionTab = dest.iSDParm; + }else{ + /* We will need to create our own temporary table to hold the + ** intermediate results. + */ + unionTab = pParse->nTab++; + assert( p->pOrderBy==0 ); + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + assert( p->addrOpenEphm[0] == -1 ); + p->addrOpenEphm[0] = addr; + findRightmost(p)->selFlags |= SF_UsesEphemeral; + assert( p->pEList ); + } + + + /* Code the SELECT statements to our left */ - assert( p->pLimit==0 ); /* Not allowed on leftward elements */ - assert( p->pOffset==0 ); /* Not allowed on leftward elements */ - unionTab = dest.iSDParm; - }else{ - /* We will need to create our own temporary table to hold the - ** intermediate results. + assert( !pPrior->pOrderBy ); + sqlite3SelectDestInit(&uniondest, priorOp, unionTab); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION left...\n")); + rc = sqlite3Select(pParse, pPrior, &uniondest); + if( rc ){ + goto multi_select_end; + } + + /* Code the current SELECT statement */ - unionTab = pParse->nTab++; + if( p->op==TK_EXCEPT ){ + op = SRT_Except; + }else{ + assert( p->op==TK_UNION ); + op = SRT_Union; + } + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + uniondest.eDest = op; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + TREETRACE(0x200, pParse, p, ("multiSelect EXCEPT/UNION right...\n")); + rc = sqlite3Select(pParse, p, &uniondest); + testcase( rc!=SQLITE_OK ); assert( p->pOrderBy==0 ); - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, unionTab, 0); + pDelete = p->pPrior; + p->pPrior = pPrior; + p->pOrderBy = 0; + if( p->op==TK_UNION ){ + p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + p->iLimit = 0; + p->iOffset = 0; + + /* Convert the data in the temporary table into whatever form + ** it is that we currently need. + */ + assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); + assert( p->pEList || db->mallocFailed ); + if( dest.eDest!=priorOp && db->mallocFailed==0 ){ + int iCont, iBreak, iStart; + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); + computeLimitRegisters(pParse, p, iBreak); + sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); + iStart = sqlite3VdbeCurrentAddr(v); + selectInnerLoop(pParse, p, unionTab, + 0, 0, &dest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); + sqlite3VdbeResolveLabel(v, iBreak); + sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + } + break; + } + default: assert( p->op==TK_INTERSECT ); { + int tab1, tab2; + int iCont, iBreak, iStart; + Expr *pLimit; + int addr; + SelectDest intersectdest; + int r1; + + /* INTERSECT is different from the others since it requires + ** two temporary tables. Hence it has its own case. Begin + ** by allocating the tables we will need. + */ + tab1 = pParse->nTab++; + tab2 = pParse->nTab++; + assert( p->pOrderBy==0 ); + + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); assert( p->addrOpenEphm[0] == -1 ); p->addrOpenEphm[0] = addr; findRightmost(p)->selFlags |= SF_UsesEphemeral; assert( p->pEList ); - } - /* Code the SELECT statements to our left - */ - assert( !pPrior->pOrderBy ); - sqlite3SelectDestInit(&uniondest, priorOp, unionTab); - explainSetInteger(iSub1, pParse->iNextSelectId); - rc = sqlite3Select(pParse, pPrior, &uniondest); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT statement - */ - if( p->op==TK_EXCEPT ){ - op = SRT_Except; - }else{ - assert( p->op==TK_UNION ); - op = SRT_Union; - } - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; - uniondest.eDest = op; - explainSetInteger(iSub2, pParse->iNextSelectId); - rc = sqlite3Select(pParse, p, &uniondest); - testcase( rc!=SQLITE_OK ); - /* Query flattening in sqlite3Select() might refill p->pOrderBy. - ** Be sure to delete p->pOrderBy, therefore, to avoid a memory leak. */ - sqlite3ExprListDelete(db, p->pOrderBy); - pDelete = p->pPrior; - p->pPrior = pPrior; - p->pOrderBy = 0; - if( p->op==TK_UNION ){ - p->nSelectRow = sqlite3LogEstAdd(p->nSelectRow, pPrior->nSelectRow); - } - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->pOffset = pOffset; - p->iLimit = 0; - p->iOffset = 0; - - /* Convert the data in the temporary table into whatever form - ** it is that we currently need. - */ - assert( unionTab==dest.iSDParm || dest.eDest!=priorOp ); - if( dest.eDest!=priorOp ){ - int iCont, iBreak, iStart; - assert( p->pEList ); - if( dest.eDest==SRT_Output ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); + /* Code the SELECTs to our left into temporary table "tab1". + */ + sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT left...\n")); + rc = sqlite3Select(pParse, pPrior, &intersectdest); + if( rc ){ + goto multi_select_end; } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); + + /* Code the current SELECT into temporary table "tab2" + */ + addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); + assert( p->addrOpenEphm[1] == -1 ); + p->addrOpenEphm[1] = addr; + p->pPrior = 0; + pLimit = p->pLimit; + p->pLimit = 0; + intersectdest.iSDParm = tab2; + ExplainQueryPlan((pParse, 1, "%s USING TEMP B-TREE", + sqlite3SelectOpName(p->op))); + TREETRACE(0x400, pParse, p, ("multiSelect INTERSECT right...\n")); + rc = sqlite3Select(pParse, p, &intersectdest); + testcase( rc!=SQLITE_OK ); + pDelete = p->pPrior; + p->pPrior = pPrior; + if( p->nSelectRow>pPrior->nSelectRow ){ + p->nSelectRow = pPrior->nSelectRow; + } + sqlite3ExprDelete(db, p->pLimit); + p->pLimit = pLimit; + + /* Generate code to take the intersection of the two temporary + ** tables. + */ + if( rc ) break; + assert( p->pEList ); + iBreak = sqlite3VdbeMakeLabel(pParse); + iCont = sqlite3VdbeMakeLabel(pParse); computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, unionTab, iBreak); VdbeCoverage(v); - iStart = sqlite3VdbeCurrentAddr(v); - selectInnerLoop(pParse, p, p->pEList, unionTab, + sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); + r1 = sqlite3GetTempReg(pParse); + iStart = sqlite3VdbeAddOp2(v, OP_RowData, tab1, r1); + sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, r1); + selectInnerLoop(pParse, p, tab1, 0, 0, &dest, iCont, iBreak); sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, unionTab, iStart); VdbeCoverage(v); + sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, unionTab, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); + sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); + break; } - break; } - default: assert( p->op==TK_INTERSECT ); { - int tab1, tab2; - int iCont, iBreak, iStart; - Expr *pLimit, *pOffset; - int addr; - SelectDest intersectdest; - int r1; - /* INTERSECT is different from the others since it requires - ** two temporary tables. Hence it has its own case. Begin - ** by allocating the tables we will need. - */ - tab1 = pParse->nTab++; - tab2 = pParse->nTab++; - assert( p->pOrderBy==0 ); - - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab1, 0); - assert( p->addrOpenEphm[0] == -1 ); - p->addrOpenEphm[0] = addr; - findRightmost(p)->selFlags |= SF_UsesEphemeral; - assert( p->pEList ); - - /* Code the SELECTs to our left into temporary table "tab1". - */ - sqlite3SelectDestInit(&intersectdest, SRT_Union, tab1); - explainSetInteger(iSub1, pParse->iNextSelectId); - rc = sqlite3Select(pParse, pPrior, &intersectdest); - if( rc ){ - goto multi_select_end; - } - - /* Code the current SELECT into temporary table "tab2" - */ - addr = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, tab2, 0); - assert( p->addrOpenEphm[1] == -1 ); - p->addrOpenEphm[1] = addr; - p->pPrior = 0; - pLimit = p->pLimit; - p->pLimit = 0; - pOffset = p->pOffset; - p->pOffset = 0; - intersectdest.iSDParm = tab2; - explainSetInteger(iSub2, pParse->iNextSelectId); - rc = sqlite3Select(pParse, p, &intersectdest); - testcase( rc!=SQLITE_OK ); - pDelete = p->pPrior; - p->pPrior = pPrior; - if( p->nSelectRow>pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; - sqlite3ExprDelete(db, p->pLimit); - p->pLimit = pLimit; - p->pOffset = pOffset; - - /* Generate code to take the intersection of the two temporary - ** tables. - */ - assert( p->pEList ); - if( dest.eDest==SRT_Output ){ - Select *pFirst = p; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); - } - iBreak = sqlite3VdbeMakeLabel(v); - iCont = sqlite3VdbeMakeLabel(v); - computeLimitRegisters(pParse, p, iBreak); - sqlite3VdbeAddOp2(v, OP_Rewind, tab1, iBreak); VdbeCoverage(v); - r1 = sqlite3GetTempReg(pParse); - iStart = sqlite3VdbeAddOp2(v, OP_RowKey, tab1, r1); - sqlite3VdbeAddOp4Int(v, OP_NotFound, tab2, iCont, r1, 0); VdbeCoverage(v); - sqlite3ReleaseTempReg(pParse, r1); - selectInnerLoop(pParse, p, p->pEList, tab1, - 0, 0, &dest, iCont, iBreak); - sqlite3VdbeResolveLabel(v, iCont); - sqlite3VdbeAddOp2(v, OP_Next, tab1, iStart); VdbeCoverage(v); - sqlite3VdbeResolveLabel(v, iBreak); - sqlite3VdbeAddOp2(v, OP_Close, tab2, 0); - sqlite3VdbeAddOp2(v, OP_Close, tab1, 0); - break; + #ifndef SQLITE_OMIT_EXPLAIN + if( p->pNext==0 ){ + ExplainQueryPlanPop(pParse); } + #endif } + if( pParse->nErr ) goto multi_select_end; - explainComposite(pParse, p->op, iSub1, iSub2, p->op!=TK_ALL); - - /* Compute collating sequences used by + /* Compute collating sequences used by ** temporary tables needed to implement the compound select. ** Attach the KeyInfo structure to all temporary tables. ** @@ -115365,6 +146883,7 @@ static int multiSelect( int nCol; /* Number of columns in result set */ assert( p->pNext==0 ); + assert( p->pEList!=0 ); nCol = p->pEList->nExpr; pKeyInfo = sqlite3KeyInfoAlloc(db, nCol, 1); if( !pKeyInfo ){ @@ -115399,7 +146918,9 @@ static int multiSelect( multi_select_end: pDest->iSdst = dest.iSdst; pDest->nSdst = dest.nSdst; - sqlite3SelectDelete(db, pDelete); + if( pDelete ){ + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pDelete); + } return rc; } #endif /* SQLITE_OMIT_COMPOUND_SELECT */ @@ -115413,13 +146934,14 @@ SQLITE_PRIVATE void sqlite3SelectWrongNumTermsError(Parse *pParse, Select *p){ sqlite3ErrorMsg(pParse, "all VALUES must have the same number of terms"); }else{ sqlite3ErrorMsg(pParse, "SELECTs to the left and right of %s" - " do not have the same number of result columns", selectOpName(p->op)); + " do not have the same number of result columns", + sqlite3SelectOpName(p->op)); } } /* ** Code an output subroutine for a coroutine implementation of a -** SELECT statment. +** SELECT statement. ** ** The data to be output is contained in pIn->iSdst. There are ** pIn->nSdst columns to be output. pDest is where the output should @@ -115452,9 +146974,9 @@ static int generateOutputSubroutine( int addr; addr = sqlite3VdbeCurrentAddr(v); - iContinue = sqlite3VdbeMakeLabel(v); + iContinue = sqlite3VdbeMakeLabel(pParse); - /* Suppress duplicates for UNION, EXCEPT, and INTERSECT + /* Suppress duplicates for UNION, EXCEPT, and INTERSECT */ if( regPrev ){ int addr1, addr2; @@ -115490,30 +147012,33 @@ static int generateOutputSubroutine( } #ifndef SQLITE_OMIT_SUBQUERY - /* If we are creating a set for an "expr IN (SELECT ...)" construct, - ** then there should be a single item on the stack. Write this - ** item into the set table with bogus data. + /* If we are creating a set for an "expr IN (SELECT ...)". */ case SRT_Set: { int r1; - assert( pIn->nSdst==1 || pParse->nErr>0 ); - pDest->affSdst = - sqlite3CompareAffinity(p->pEList->a[0].pExpr, pDest->affSdst); + testcase( pIn->nSdst>1 ); r1 = sqlite3GetTempReg(pParse); - sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, 1, r1, &pDest->affSdst,1); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, 1); - sqlite3VdbeAddOp2(v, OP_IdxInsert, pDest->iSDParm, r1); + sqlite3VdbeAddOp4(v, OP_MakeRecord, pIn->iSdst, pIn->nSdst, + r1, pDest->zAffSdst, pIn->nSdst); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pDest->iSDParm, r1, + pIn->iSdst, pIn->nSdst); + if( pDest->iSDParm2>0 ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pDest->iSDParm2, 0, + pIn->iSdst, pIn->nSdst); + ExplainQueryPlan((pParse, 0, "CREATE BLOOM FILTER")); + } sqlite3ReleaseTempReg(pParse, r1); break; } /* If this is a scalar select that is part of an expression, then ** store the results in the appropriate memory cell and break out - ** of the scan loop. + ** of the scan loop. Note that the select might return multiple columns + ** if it is the RHS of a row-value IN operator. */ case SRT_Mem: { - assert( pIn->nSdst==1 || pParse->nErr>0 ); testcase( pIn->nSdst!=1 ); - sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, 1); + testcase( pIn->nSdst>1 ); + sqlite3ExprCodeMove(pParse, pIn->iSdst, pDest->iSDParm, pIn->nSdst); /* The LIMIT clause will jump out of the loop for us */ break; } @@ -115536,14 +147061,13 @@ static int generateOutputSubroutine( ** SRT_Output. This routine is never called with any other ** destination other than the ones handled above or SRT_Output. ** - ** For SRT_Output, results are stored in a sequence of registers. + ** For SRT_Output, results are stored in a sequence of registers. ** Then the OP_ResultRow opcode is used to cause sqlite3_step() to ** return the next row of result. */ default: { assert( pDest->eDest==SRT_Output ); sqlite3VdbeAddOp2(v, OP_ResultRow, pIn->iSdst, pIn->nSdst); - sqlite3ExprCacheAffinityChange(pParse, pIn->iSdst, pIn->nSdst); break; } } @@ -115594,7 +147118,7 @@ static int generateOutputSubroutine( ** ** EofB: Called when data is exhausted from selectB. ** -** The implementation of the latter five subroutines depend on which +** The implementation of the latter five subroutines depend on which ** is used: ** ** @@ -115644,7 +147168,7 @@ static int generateOutputSubroutine( ** ** We call AltB, AeqB, AgtB, EofA, and EofB "subroutines" but they are not ** actually called using Gosub and they do not Return. EofA and EofB loop -** until all data is exhausted then jump to the "end" labe. AltB, AeqB, +** until all data is exhausted then jump to the "end" label. AltB, AeqB, ** and AgtB jump to either L2 or to one of EofA or EofB. */ #ifndef SQLITE_OMIT_COMPOUND_SELECT @@ -115655,6 +147179,8 @@ static int multiSelectOrderBy( ){ int i, j; /* Loop counters */ Select *pPrior; /* Another SELECT immediately to our left */ + Select *pSplit; /* Left-most SELECT in the right-hand group */ + int nSelect; /* Number of SELECT statements in the compound */ Vdbe *v; /* Generate code to this VDBE */ SelectDest destA; /* Destination for coroutine A */ SelectDest destB; /* Destination for coroutine B */ @@ -115679,33 +147205,28 @@ static int multiSelectOrderBy( int savedOffset; /* Saved value of p->iOffset */ int labelCmpr; /* Label for the start of the merge algorithm */ int labelEnd; /* Label for the end of the overall SELECT stmt */ - int addr1; /* Jump instructions that get retargetted */ + int addr1; /* Jump instructions that get retargeted */ int op; /* One of TK_ALL, TK_UNION, TK_EXCEPT, TK_INTERSECT */ KeyInfo *pKeyDup = 0; /* Comparison information for duplicate removal */ KeyInfo *pKeyMerge; /* Comparison information for merging rows */ sqlite3 *db; /* Database connection */ ExprList *pOrderBy; /* The ORDER BY clause */ int nOrderBy; /* Number of terms in the ORDER BY clause */ - int *aPermute; /* Mapping from ORDER BY terms to result set columns */ -#ifndef SQLITE_OMIT_EXPLAIN - int iSub1; /* EQP id of left-hand query */ - int iSub2; /* EQP id of right-hand query */ -#endif + u32 *aPermute; /* Mapping from ORDER BY terms to result set columns */ assert( p->pOrderBy!=0 ); assert( pKeyDup==0 ); /* "Managed" code needs this. Ticket #3382. */ db = pParse->db; v = pParse->pVdbe; assert( v!=0 ); /* Already thrown the error if VDBE alloc failed */ - labelEnd = sqlite3VdbeMakeLabel(v); - labelCmpr = sqlite3VdbeMakeLabel(v); + labelEnd = sqlite3VdbeMakeLabel(pParse); + labelCmpr = sqlite3VdbeMakeLabel(pParse); /* Patch up the ORDER BY clause */ - op = p->op; - pPrior = p->pPrior; - assert( pPrior->pOrderBy==0 ); + op = p->op; + assert( p->pPrior->pOrderBy==0 ); pOrderBy = p->pOrderBy; assert( pOrderBy ); nOrderBy = pOrderBy->nExpr; @@ -115718,6 +147239,7 @@ static int multiSelectOrderBy( for(i=1; db->mallocFailed==0 && i<=p->pEList->nExpr; i++){ struct ExprList_item *pItem; for(j=0, pItem=pOrderBy->a; ju.x.iOrderByCol>0 ); if( pItem->u.x.iOrderByCol==i ) break; } @@ -115726,7 +147248,7 @@ static int multiSelectOrderBy( if( pNew==0 ) return SQLITE_NOMEM_BKPT; pNew->flags |= EP_IntValue; pNew->u.iValue = i; - pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); + p->pOrderBy = pOrderBy = sqlite3ExprListAppend(pParse, pOrderBy, pNew); if( pOrderBy ) pOrderBy->a[nOrderBy++].u.x.iOrderByCol = (u16)i; } } @@ -115739,11 +147261,12 @@ static int multiSelectOrderBy( ** to the right and the left are evaluated, they use the correct ** collation. */ - aPermute = sqlite3DbMallocRawNN(db, sizeof(int)*(nOrderBy + 1)); + aPermute = sqlite3DbMallocRawNN(db, sizeof(u32)*(nOrderBy + 1)); if( aPermute ){ struct ExprList_item *pItem; aPermute[0] = nOrderBy; for(i=1, pItem=pOrderBy->a; i<=nOrderBy; i++, pItem++){ + assert( pItem!=0 ); assert( pItem->u.x.iOrderByCol>0 ); assert( pItem->u.x.iOrderByCol<=p->pEList->nExpr ); aPermute[i] = pItem->u.x.iOrderByCol - 1; @@ -115753,11 +147276,6 @@ static int multiSelectOrderBy( pKeyMerge = 0; } - /* Reattach the ORDER BY clause to the query. - */ - p->pOrderBy = pOrderBy; - pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); - /* Allocate a range of temporary registers and the KeyInfo needed ** for the logic that removes duplicate result rows when the ** operator is UNION, EXCEPT, or INTERSECT (but not UNION ALL). @@ -115775,19 +147293,37 @@ static int multiSelectOrderBy( assert( sqlite3KeyInfoIsWriteable(pKeyDup) ); for(i=0; iaColl[i] = multiSelectCollSeq(pParse, p, i); - pKeyDup->aSortOrder[i] = 0; + pKeyDup->aSortFlags[i] = 0; } } } - + /* Separate the left and the right query from one another */ - p->pPrior = 0; - pPrior->pNext = 0; - sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); - if( pPrior->pPrior==0 ){ - sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); + nSelect = 1; + if( (op==TK_ALL || op==TK_UNION) + && OptimizationEnabled(db, SQLITE_BalancedMerge) + ){ + for(pSplit=p; pSplit->pPrior!=0 && pSplit->op==op; pSplit=pSplit->pPrior){ + nSelect++; + assert( pSplit->pPrior->pNext==pSplit ); + } } + if( nSelect<=3 ){ + pSplit = p; + }else{ + pSplit = p; + for(i=2; ipPrior; } + } + pPrior = pSplit->pPrior; + assert( pPrior!=0 ); + pSplit->pPrior = 0; + pPrior->pNext = 0; + assert( p->pOrderBy == pOrderBy ); + assert( pOrderBy!=0 || db->mallocFailed ); + pPrior->pOrderBy = sqlite3ExprListDup(pParse->db, pOrderBy, 0); + sqlite3ResolveOrderGroupBy(pParse, p, p->pOrderBy, "ORDER"); + sqlite3ResolveOrderGroupBy(pParse, pPrior, pPrior->pOrderBy, "ORDER"); /* Compute the limit registers */ computeLimitRegisters(pParse, p, labelEnd); @@ -115802,8 +147338,6 @@ static int multiSelectOrderBy( } sqlite3ExprDelete(db, p->pLimit); p->pLimit = 0; - sqlite3ExprDelete(db, p->pOffset); - p->pOffset = 0; regAddrA = ++pParse->nMem; regAddrB = ++pParse->nMem; @@ -115812,6 +147346,8 @@ static int multiSelectOrderBy( sqlite3SelectDestInit(&destA, SRT_Coroutine, regAddrA); sqlite3SelectDestInit(&destB, SRT_Coroutine, regAddrB); + ExplainQueryPlan((pParse, 1, "MERGE (%s)", sqlite3SelectOpName(p->op))); + /* Generate a coroutine to evaluate the SELECT statement to the ** left of the compound operator - the "A" select. */ @@ -115819,12 +147355,12 @@ static int multiSelectOrderBy( addr1 = sqlite3VdbeAddOp3(v, OP_InitCoroutine, regAddrA, 0, addrSelectA); VdbeComment((v, "left SELECT")); pPrior->iLimit = regLimitA; - explainSetInteger(iSub1, pParse->iNextSelectId); + ExplainQueryPlan((pParse, 1, "LEFT")); sqlite3Select(pParse, pPrior, &destA); sqlite3VdbeEndCoroutine(v, regAddrA); sqlite3VdbeJumpHere(v, addr1); - /* Generate a coroutine to evaluate the SELECT statement on + /* Generate a coroutine to evaluate the SELECT statement on ** the right - the "B" select */ addrSelectB = sqlite3VdbeCurrentAddr(v) + 1; @@ -115833,8 +147369,8 @@ static int multiSelectOrderBy( savedLimit = p->iLimit; savedOffset = p->iOffset; p->iLimit = regLimitB; - p->iOffset = 0; - explainSetInteger(iSub2, pParse->iNextSelectId); + p->iOffset = 0; + ExplainQueryPlan((pParse, 1, "RIGHT")); sqlite3Select(pParse, p, &destB); p->iLimit = savedLimit; p->iOffset = savedOffset; @@ -115847,7 +147383,7 @@ static int multiSelectOrderBy( addrOutA = generateOutputSubroutine(pParse, p, &destA, pDest, regOutA, regPrev, pKeyDup, labelEnd); - + /* Generate a subroutine that outputs the current row of the B ** select as the next output row of the compound select. */ @@ -115864,7 +147400,7 @@ static int multiSelectOrderBy( */ if( op==TK_EXCEPT || op==TK_INTERSECT ){ addrEofA_noB = addrEofA = labelEnd; - }else{ + }else{ VdbeNoopComment((v, "eof-A subroutine")); addrEofA = sqlite3VdbeAddOp2(v, OP_Gosub, regOutB, addrOutB); addrEofA_noB = sqlite3VdbeAddOp2(v, OP_Yield, regAddrB, labelEnd); @@ -115879,7 +147415,7 @@ static int multiSelectOrderBy( if( op==TK_INTERSECT ){ addrEofB = addrEofA; if( p->nSelectRow > pPrior->nSelectRow ) p->nSelectRow = pPrior->nSelectRow; - }else{ + }else{ VdbeNoopComment((v, "eof-B subroutine")); addrEofB = sqlite3VdbeAddOp2(v, OP_Gosub, regOutA, addrOutA); sqlite3VdbeAddOp2(v, OP_Yield, regAddrA, labelEnd); VdbeCoverage(v); @@ -115936,117 +147472,399 @@ static int multiSelectOrderBy( */ sqlite3VdbeResolveLabel(v, labelEnd); - /* Set the number of output columns - */ - if( pDest->eDest==SRT_Output ){ - Select *pFirst = pPrior; - while( pFirst->pPrior ) pFirst = pFirst->pPrior; - generateColumnNames(pParse, pFirst->pSrc, pFirst->pEList); + /* Make arrangements to free the 2nd and subsequent arms of the compound + ** after the parse has finished */ + if( pSplit->pPrior ){ + sqlite3ParserAddCleanup(pParse, sqlite3SelectDeleteGeneric, pSplit->pPrior); } - - /* Reassembly the compound query so that it will be freed correctly - ** by the calling function */ - if( p->pPrior ){ - sqlite3SelectDelete(db, p->pPrior); - } - p->pPrior = pPrior; - pPrior->pNext = p; + pSplit->pPrior = pPrior; + pPrior->pNext = pSplit; + sqlite3ExprListDelete(db, pPrior->pOrderBy); + pPrior->pOrderBy = 0; /*** TBD: Insert subroutine calls to close cursors on incomplete **** subqueries ****/ - explainComposite(pParse, p->op, iSub1, iSub2, 0); + ExplainQueryPlanPop(pParse); return pParse->nErr!=0; } #endif #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + +/* An instance of the SubstContext object describes an substitution edit +** to be performed on a parse tree. +** +** All references to columns in table iTable are to be replaced by corresponding +** expressions in pEList. +** +** ## About "isOuterJoin": +** +** The isOuterJoin column indicates that the replacement will occur into a +** position in the parent that NULL-able due to an OUTER JOIN. Either the +** target slot in the parent is the right operand of a LEFT JOIN, or one of +** the left operands of a RIGHT JOIN. In either case, we need to potentially +** bypass the substituted expression with OP_IfNullRow. +** +** Suppose the original expression is an integer constant. Even though the table +** has the nullRow flag set, because the expression is an integer constant, +** it will not be NULLed out. So instead, we insert an OP_IfNullRow opcode +** that checks to see if the nullRow flag is set on the table. If the nullRow +** flag is set, then the value in the register is set to NULL and the original +** expression is bypassed. If the nullRow flag is not set, then the original +** expression runs to populate the register. +** +** Example where this is needed: +** +** CREATE TABLE t1(a INTEGER PRIMARY KEY, b INT); +** CREATE TABLE t2(x INT UNIQUE); +** +** SELECT a,b,m,x FROM t1 LEFT JOIN (SELECT 59 AS m,x FROM t2) ON b=x; +** +** When the subquery on the right side of the LEFT JOIN is flattened, we +** have to add OP_IfNullRow in front of the OP_Integer that implements the +** "m" value of the subquery so that a NULL will be loaded instead of 59 +** when processing a non-matched row of the left. +*/ +typedef struct SubstContext { + Parse *pParse; /* The parsing context */ + int iTable; /* Replace references to this table */ + int iNewTable; /* New table number */ + int isOuterJoin; /* Add TK_IF_NULL_ROW opcodes on each replacement */ + ExprList *pEList; /* Replacement expressions */ + ExprList *pCList; /* Collation sequences for replacement expr */ +} SubstContext; + /* Forward Declarations */ -static void substExprList(sqlite3*, ExprList*, int, ExprList*); -static void substSelect(sqlite3*, Select *, int, ExprList*, int); +static void substExprList(SubstContext*, ExprList*); +static void substSelect(SubstContext*, Select*, int); /* ** Scan through the expression pExpr. Replace every reference to ** a column in table number iTable with a copy of the iColumn-th -** entry in pEList. (But leave references to the ROWID column +** entry in pEList. (But leave references to the ROWID column ** unchanged.) ** ** This routine is part of the flattening procedure. A subquery ** whose result set is defined by pEList appears as entry in the ** FROM clause of a SELECT such that the VDBE cursor assigned to that -** FORM clause entry is iTable. This routine make the necessary +** FORM clause entry is iTable. This routine makes the necessary ** changes to pExpr so that it refers directly to the source table ** of the subquery rather the result set of the subquery. */ static Expr *substExpr( - sqlite3 *db, /* Report malloc errors to this connection */ - Expr *pExpr, /* Expr in which substitution occurs */ - int iTable, /* Table to be substituted */ - ExprList *pEList /* Substitute expressions */ + SubstContext *pSubst, /* Description of the substitution */ + Expr *pExpr /* Expr in which substitution occurs */ ){ if( pExpr==0 ) return 0; - if( pExpr->op==TK_COLUMN && pExpr->iTable==iTable ){ + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) + && pExpr->w.iJoin==pSubst->iTable + ){ + testcase( ExprHasProperty(pExpr, EP_InnerON) ); + pExpr->w.iJoin = pSubst->iNewTable; + } + if( pExpr->op==TK_COLUMN + && pExpr->iTable==pSubst->iTable + && !ExprHasProperty(pExpr, EP_FixedCol) + ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW if( pExpr->iColumn<0 ){ pExpr->op = TK_NULL; - }else{ + }else +#endif + { Expr *pNew; - assert( pEList!=0 && pExpr->iColumnnExpr ); - assert( pExpr->pLeft==0 && pExpr->pRight==0 ); - pNew = sqlite3ExprDup(db, pEList->a[pExpr->iColumn].pExpr, 0); - sqlite3ExprDelete(db, pExpr); - pExpr = pNew; + int iColumn; + Expr *pCopy; + Expr ifNullRow; + iColumn = pExpr->iColumn; + assert( iColumn>=0 ); + assert( pSubst->pEList!=0 && iColumnpEList->nExpr ); + assert( pExpr->pRight==0 ); + pCopy = pSubst->pEList->a[iColumn].pExpr; + if( sqlite3ExprIsVector(pCopy) ){ + sqlite3VectorErrorMsg(pSubst->pParse, pCopy); + }else{ + sqlite3 *db = pSubst->pParse->db; + if( pSubst->isOuterJoin + && (pCopy->op!=TK_COLUMN || pCopy->iTable!=pSubst->iNewTable) + ){ + memset(&ifNullRow, 0, sizeof(ifNullRow)); + ifNullRow.op = TK_IF_NULL_ROW; + ifNullRow.pLeft = pCopy; + ifNullRow.iTable = pSubst->iNewTable; + ifNullRow.iColumn = -99; + ifNullRow.flags = EP_IfNullRow; + pCopy = &ifNullRow; + } + testcase( ExprHasProperty(pCopy, EP_Subquery) ); + pNew = sqlite3ExprDup(db, pCopy, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pNew); + return pExpr; + } + if( pSubst->isOuterJoin ){ + ExprSetProperty(pNew, EP_CanBeNull); + } + if( ExprHasProperty(pExpr,EP_OuterON|EP_InnerON) ){ + sqlite3SetJoinExpr(pNew, pExpr->w.iJoin, + pExpr->flags & (EP_OuterON|EP_InnerON)); + } + sqlite3ExprDelete(db, pExpr); + pExpr = pNew; + if( pExpr->op==TK_TRUEFALSE ){ + pExpr->u.iValue = sqlite3ExprTruthValue(pExpr); + pExpr->op = TK_INTEGER; + ExprSetProperty(pExpr, EP_IntValue); + } + + /* Ensure that the expression now has an implicit collation sequence, + ** just as it did when it was a column of a view or sub-query. */ + { + CollSeq *pNat = sqlite3ExprCollSeq(pSubst->pParse, pExpr); + CollSeq *pColl = sqlite3ExprCollSeq(pSubst->pParse, + pSubst->pCList->a[iColumn].pExpr + ); + if( pNat!=pColl || (pExpr->op!=TK_COLUMN && pExpr->op!=TK_COLLATE) ){ + pExpr = sqlite3ExprAddCollateString(pSubst->pParse, pExpr, + (pColl ? pColl->zName : "BINARY") + ); + } + } + ExprClearProperty(pExpr, EP_Collate); + } } }else{ - pExpr->pLeft = substExpr(db, pExpr->pLeft, iTable, pEList); - pExpr->pRight = substExpr(db, pExpr->pRight, iTable, pEList); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ - substSelect(db, pExpr->x.pSelect, iTable, pEList, 1); - }else{ - substExprList(db, pExpr->x.pList, iTable, pEList); + if( pExpr->op==TK_IF_NULL_ROW && pExpr->iTable==pSubst->iTable ){ + pExpr->iTable = pSubst->iNewTable; } + pExpr->pLeft = substExpr(pSubst, pExpr->pLeft); + pExpr->pRight = substExpr(pSubst, pExpr->pRight); + if( ExprUseXSelect(pExpr) ){ + substSelect(pSubst, pExpr->x.pSelect, 1); + }else{ + substExprList(pSubst, pExpr->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + Window *pWin = pExpr->y.pWin; + pWin->pFilter = substExpr(pSubst, pWin->pFilter); + substExprList(pSubst, pWin->pPartition); + substExprList(pSubst, pWin->pOrderBy); + } +#endif } return pExpr; } static void substExprList( - sqlite3 *db, /* Report malloc errors here */ - ExprList *pList, /* List to scan and in which to make substitutes */ - int iTable, /* Table to be substituted */ - ExprList *pEList /* Substitute values */ + SubstContext *pSubst, /* Description of the substitution */ + ExprList *pList /* List to scan and in which to make substitutes */ ){ int i; if( pList==0 ) return; for(i=0; inExpr; i++){ - pList->a[i].pExpr = substExpr(db, pList->a[i].pExpr, iTable, pEList); + pList->a[i].pExpr = substExpr(pSubst, pList->a[i].pExpr); } } static void substSelect( - sqlite3 *db, /* Report malloc errors here */ - Select *p, /* SELECT statement in which to make substitutions */ - int iTable, /* Table to be replaced */ - ExprList *pEList, /* Substitute values */ - int doPrior /* Do substitutes on p->pPrior too */ + SubstContext *pSubst, /* Description of the substitution */ + Select *p, /* SELECT statement in which to make substitutions */ + int doPrior /* Do substitutes on p->pPrior too */ ){ SrcList *pSrc; - struct SrcList_item *pItem; + SrcItem *pItem; int i; if( !p ) return; do{ - substExprList(db, p->pEList, iTable, pEList); - substExprList(db, p->pGroupBy, iTable, pEList); - substExprList(db, p->pOrderBy, iTable, pEList); - p->pHaving = substExpr(db, p->pHaving, iTable, pEList); - p->pWhere = substExpr(db, p->pWhere, iTable, pEList); + substExprList(pSubst, p->pEList); + substExprList(pSubst, p->pGroupBy); + substExprList(pSubst, p->pOrderBy); + p->pHaving = substExpr(pSubst, p->pHaving); + p->pWhere = substExpr(pSubst, p->pWhere); pSrc = p->pSrc; assert( pSrc!=0 ); for(i=pSrc->nSrc, pItem=pSrc->a; i>0; i--, pItem++){ - substSelect(db, pItem->pSelect, iTable, pEList, 1); + if( pItem->fg.isSubquery ){ + substSelect(pSubst, pItem->u4.pSubq->pSelect, 1); + } if( pItem->fg.isTabFunc ){ - substExprList(db, pItem->u1.pFuncArg, iTable, pEList); + substExprList(pSubst, pItem->u1.pFuncArg); } } }while( doPrior && (p = p->pPrior)!=0 ); } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** pSelect is a SELECT statement and pSrcItem is one item in the FROM +** clause of that SELECT. +** +** This routine scans the entire SELECT statement and recomputes the +** pSrcItem->colUsed mask. +*/ +static int recomputeColumnsUsedExpr(Walker *pWalker, Expr *pExpr){ + SrcItem *pItem; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + pItem = pWalker->u.pSrcItem; + if( pItem->iCursor!=pExpr->iTable ) return WRC_Continue; + if( pExpr->iColumn<0 ) return WRC_Continue; + pItem->colUsed |= sqlite3ExprColUsed(pExpr); + return WRC_Continue; +} +static void recomputeColumnsUsed( + Select *pSelect, /* The complete SELECT statement */ + SrcItem *pSrcItem /* Which FROM clause item to recompute */ +){ + Walker w; + if( NEVER(pSrcItem->pSTab==0) ) return; + memset(&w, 0, sizeof(w)); + w.xExprCallback = recomputeColumnsUsedExpr; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pSrcItem = pSrcItem; + pSrcItem->colUsed = 0; + sqlite3WalkSelect(&w, pSelect); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +/* +** Assign new cursor numbers to each of the items in pSrc. For each +** new cursor number assigned, set an entry in the aCsrMap[] array +** to map the old cursor number to the new: +** +** aCsrMap[iOld+1] = iNew; +** +** The array is guaranteed by the caller to be large enough for all +** existing cursor numbers in pSrc. aCsrMap[0] is the array size. +** +** If pSrc contains any sub-selects, call this routine recursively +** on the FROM clause of each such sub-select, with iExcept set to -1. +*/ +static void srclistRenumberCursors( + Parse *pParse, /* Parse context */ + int *aCsrMap, /* Array to store cursor mappings in */ + SrcList *pSrc, /* FROM clause to renumber */ + int iExcept /* FROM clause item to skip */ +){ + int i; + SrcItem *pItem; + for(i=0, pItem=pSrc->a; inSrc; i++, pItem++){ + if( i!=iExcept ){ + Select *p; + assert( pItem->iCursor < aCsrMap[0] ); + if( !pItem->fg.isRecursive || aCsrMap[pItem->iCursor+1]==0 ){ + aCsrMap[pItem->iCursor+1] = pParse->nTab++; + } + pItem->iCursor = aCsrMap[pItem->iCursor+1]; + if( pItem->fg.isSubquery ){ + for(p=pItem->u4.pSubq->pSelect; p; p=p->pPrior){ + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, -1); + } + } + } + } +} + +/* +** *piCursor is a cursor number. Change it if it needs to be mapped. +*/ +static void renumberCursorDoMapping(Walker *pWalker, int *piCursor){ + int *aCsrMap = pWalker->u.aiCol; + int iCsr = *piCursor; + if( iCsr < aCsrMap[0] && aCsrMap[iCsr+1]>0 ){ + *piCursor = aCsrMap[iCsr+1]; + } +} + +/* +** Expression walker callback used by renumberCursors() to update +** Expr objects to match newly assigned cursor numbers. +*/ +static int renumberCursorsCb(Walker *pWalker, Expr *pExpr){ + int op = pExpr->op; + if( op==TK_COLUMN || op==TK_IF_NULL_ROW ){ + renumberCursorDoMapping(pWalker, &pExpr->iTable); + } + if( ExprHasProperty(pExpr, EP_OuterON) ){ + renumberCursorDoMapping(pWalker, &pExpr->w.iJoin); + } + return WRC_Continue; +} + +/* +** Assign a new cursor number to each cursor in the FROM clause (Select.pSrc) +** of the SELECT statement passed as the second argument, and to each +** cursor in the FROM clause of any FROM clause sub-selects, recursively. +** Except, do not assign a new cursor number to the iExcept'th element in +** the FROM clause of (*p). Update all expressions and other references +** to refer to the new cursor numbers. +** +** Argument aCsrMap is an array that may be used for temporary working +** space. Two guarantees are made by the caller: +** +** * the array is larger than the largest cursor number used within the +** select statement passed as an argument, and +** +** * the array entries for all cursor numbers that do *not* appear in +** FROM clauses of the select statement as described above are +** initialized to zero. +*/ +static void renumberCursors( + Parse *pParse, /* Parse context */ + Select *p, /* Select to renumber cursors within */ + int iExcept, /* FROM clause item to skip */ + int *aCsrMap /* Working space */ +){ + Walker w; + srclistRenumberCursors(pParse, aCsrMap, p->pSrc, iExcept); + memset(&w, 0, sizeof(w)); + w.u.aiCol = aCsrMap; + w.xExprCallback = renumberCursorsCb; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkSelect(&w, p); +} +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ + +/* +** If pSel is not part of a compound SELECT, return a pointer to its +** expression list. Otherwise, return a pointer to the expression list +** of the leftmost SELECT in the compound. +*/ +static ExprList *findLeftmostExprlist(Select *pSel){ + while( pSel->pPrior ){ + pSel = pSel->pPrior; + } + return pSel->pEList; +} + +/* +** Return true if any of the result-set columns in the compound query +** have incompatible affinities on one or more arms of the compound. +*/ +static int compoundHasDifferentAffinities(Select *p){ + int ii; + ExprList *pList; + assert( p!=0 ); + assert( p->pEList!=0 ); + assert( p->pPrior!=0 ); + pList = p->pEList; + for(ii=0; iinExpr; ii++){ + char aff; + Select *pSub1; + assert( pList->a[ii].pExpr!=0 ); + aff = sqlite3ExprAffinity(pList->a[ii].pExpr); + for(pSub1=p->pPrior; pSub1; pSub1=pSub1->pPrior){ + assert( pSub1->pEList!=0 ); + assert( pSub1->pEList->nExpr>ii ); + assert( pSub1->pEList->a[ii].pExpr!=0 ); + if( sqlite3ExprAffinity(pSub1->pEList->a[ii].pExpr)!=aff ){ + return 1; + } + } + } + return 0; +} + #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* ** This routine attempts to flatten subqueries as a performance optimization. @@ -116070,70 +147888,87 @@ static void substSelect( ** SELECT x+y AS a FROM t1 WHERE z<100 AND a>5 ** ** The code generated for this simplification gives the same result -** but only has to scan the data once. And because indices might +** but only has to scan the data once. And because indices might ** exist on the table t1, a complete scan of the data might be ** avoided. ** -** Flattening is only attempted if all of the following are true: +** Flattening is subject to the following constraints: ** -** (1) The subquery and the outer query do not both use aggregates. +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery and the outer query cannot both be aggregates. ** -** (2) The subquery is not an aggregate or (2a) the outer query is not a join -** and (2b) the outer query does not use subqueries other than the one -** FROM-clause subquery that is a candidate for flattening. (2b is -** due to ticket [2f7170d73bf9abf80] from 2015-02-09.) +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** (2) If the subquery is an aggregate then +** (2a) the outer query must not be a join and +** (2b) the outer query must not use subqueries +** other than the one FROM-clause subquery that is a candidate +** for flattening. (This is due to ticket [2f7170d73bf9abf80] +** from 2015-02-09.) ** -** (3) The subquery is not the right operand of a left outer join -** (Originally ticket #306. Strengthened by ticket #3300) +** (3) If the subquery is the right operand of a LEFT JOIN then +** (3a) the subquery may not be a join and +** (3b) the FROM clause of the subquery may not contain a virtual +** table and +** (**) Was: "The outer query may not have a GROUP BY." This case +** is now managed correctly +** (3d) the outer query may not be DISTINCT. +** See also (26) for restrictions on RIGHT JOIN. ** -** (4) The subquery is not DISTINCT. +** (4) The subquery can not be DISTINCT. ** ** (**) At one point restrictions (4) and (5) defined a subset of DISTINCT -** sub-queries that were excluded from this optimization. Restriction +** sub-queries that were excluded from this optimization. Restriction ** (4) has since been expanded to exclude all DISTINCT subqueries. ** -** (6) The subquery does not use aggregates or the outer query is not -** DISTINCT. +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** If the subquery is aggregate, the outer query may not be DISTINCT. ** -** (7) The subquery has a FROM clause. TODO: For subqueries without -** A FROM clause, consider adding a FROM close with the special +** (7) The subquery must have a FROM clause. TODO: For subqueries without +** A FROM clause, consider adding a FROM clause with the special ** table sqlite_once that consists of a single row containing a ** single NULL. ** -** (8) The subquery does not use LIMIT or the outer query is not a join. +** (8) If the subquery uses LIMIT then the outer query may not be a join. ** -** (9) The subquery does not use LIMIT or the outer query does not use -** aggregates. +** (9) If the subquery uses LIMIT then the outer query may not be aggregate. ** ** (**) Restriction (10) was removed from the code on 2005-02-05 but we -** accidently carried the comment forward until 2014-09-15. Original -** text: "The subquery does not use aggregates or the outer query -** does not use LIMIT." +** accidentally carried the comment forward until 2014-09-15. Original +** constraint: "If the subquery is aggregate then the outer query +** may not use LIMIT." ** -** (11) The subquery and the outer query do not both have ORDER BY clauses. +** (11) The subquery and the outer query may not both have ORDER BY clauses. ** ** (**) Not implemented. Subsumed into restriction (3). Was previously ** a separate restriction deriving from ticket #350. ** -** (13) The subquery and outer query do not both use LIMIT. +** (13) The subquery and outer query may not both use LIMIT. ** -** (14) The subquery does not use OFFSET. +** (14) The subquery may not use OFFSET. ** -** (15) The outer query is not part of a compound select or the -** subquery does not have a LIMIT clause. +** (15) If the outer query is part of a compound select, then the +** subquery may not use LIMIT. ** (See ticket #2339 and ticket [02a8e81d44]). ** -** (16) The outer query is not an aggregate or the subquery does -** not contain ORDER BY. (Ticket #2942) This used to not matter -** until we introduced the group_concat() function. +** (16) If the outer query is aggregate, then the subquery may not +** use ORDER BY. (Ticket #2942) This used to not matter +** until we introduced the group_concat() function. ** -** (17) The sub-query is not a compound select, or it is a UNION ALL -** compound clause made up entirely of non-aggregate queries, and -** the parent query: -** -** * is not itself part of a compound select, -** * is not an aggregate or DISTINCT query, and -** * is not a join +** (17) If the subquery is a compound select, then +** (17a) all compound operators must be a UNION ALL, and +** (17b) no terms within the subquery compound may be aggregate +** or DISTINCT, and +** (17c) every term within the subquery compound must have a FROM clause +** (17d) the outer query may not be +** (17d1) aggregate, or +** (17d2) DISTINCT +** (17e) the subquery may not contain window functions, and +** (17f) the subquery must not be the RHS of a LEFT JOIN. +** (17g) either the subquery is the first element of the outer +** query or there are no RIGHT or FULL JOINs in any arm +** of the subquery. (This is a duplicate of condition (27b).) +** (17h) The corresponding result set expressions in all arms of the +** compound must have the same affinity. ** ** The parent and sub-query may contain WHERE clauses. Subject to ** rules (11), (13) and (14), they may also contain ORDER BY, @@ -116149,10 +147984,10 @@ static void substSelect( ** syntax error and return a detailed message. ** ** (18) If the sub-query is a compound select, then all terms of the -** ORDER by clause of the parent must be simple references to -** columns of the sub-query. +** ORDER BY clause of the parent must be copies of a term returned +** by the parent query. ** -** (19) The subquery does not use LIMIT or the outer query does not +** (19) If the subquery uses LIMIT then the outer query may not ** have a WHERE clause. ** ** (20) If the sub-query is a compound select, then it must not use @@ -116161,25 +147996,42 @@ static void substSelect( ** appear as unmodified result columns in the outer query. But we ** have other optimizations in mind to deal with that case. ** -** (21) The subquery does not use LIMIT or the outer query is not +** (21) If the subquery uses LIMIT then the outer query may not be ** DISTINCT. (See ticket [752e1646fc]). ** -** (22) The subquery is not a recursive CTE. +** (22) The subquery may not be a recursive CTE. ** -** (23) The parent is not a recursive CTE, or the sub-query is not a -** compound query. This restriction is because transforming the +** (23) If the outer query is a recursive CTE, then the sub-query may not be +** a compound query. This restriction is because transforming the ** parent to a compound query confuses the code that handles ** recursive queries in multiSelect(). ** -** (24) The subquery is not an aggregate that uses the built-in min() or +** (**) We no longer attempt to flatten aggregate subqueries. Was: +** The subquery may not be an aggregate that uses the built-in min() or ** or max() functions. (Without this restriction, a query like: ** "SELECT x FROM (SELECT max(y), x FROM t1)" would not necessarily ** return the value X for which Y was maximal.) ** +** (25) If either the subquery or the parent query contains a window +** function in the select list or ORDER BY clause, flattening +** is not attempted. +** +** (26) The subquery may not be the right operand of a RIGHT JOIN. +** See also (3) for restrictions on LEFT JOIN. +** +** (27) The subquery may not contain a FULL or RIGHT JOIN unless it +** is the first element of the parent query. Two subcases: +** (27a) the subquery is not a compound query. +** (27b) the subquery is a compound query and the RIGHT JOIN occurs +** in any arm of the compound query. (See also (17g).) +** +** (28) The subquery is not a MATERIALIZED CTE. (This is handled +** in the caller before ever reaching this routine.) +** ** ** In this routine, the "p" parameter is a pointer to the outer query. ** The subquery is p->pSrc->a[iFrom]. isAgg is true if the outer query -** uses aggregates and subqueryIsAgg is true if the subquery uses aggregates. +** uses aggregates. ** ** If flattening is not attempted, this routine is a no-op and returns 0. ** If flattening is attempted this routine returns 1. @@ -116191,8 +148043,7 @@ static int flattenSubquery( Parse *pParse, /* Parsing context */ Select *p, /* The parent or outer SELECT statement */ int iFrom, /* Index in p->pSrc->a[] of the inner subquery */ - int isAgg, /* True if outer SELECT uses aggregate functions */ - int subqueryIsAgg /* True if the subquery uses aggregate functions */ + int isAgg /* True if outer SELECT uses aggregate functions */ ){ const char *zSavedAuthContext = pParse->zAuthContext; Select *pParent; /* Current UNION ALL term of the other query */ @@ -116200,35 +148051,33 @@ static int flattenSubquery( Select *pSub1; /* Pointer to the rightmost select in sub-query */ SrcList *pSrc; /* The FROM clause of the outer query */ SrcList *pSubSrc; /* The FROM clause of the subquery */ - ExprList *pList; /* The result set of the outer query */ int iParent; /* VDBE cursor number of the pSub result set temp table */ + int iNewParent = -1;/* Replacement table for iParent */ + int isOuterJoin = 0; /* True if pSub is the right side of a LEFT JOIN */ int i; /* Loop counter */ Expr *pWhere; /* The WHERE clause */ - struct SrcList_item *pSubitem; /* The subquery */ + SrcItem *pSubitem; /* The subquery */ sqlite3 *db = pParse->db; + Walker w; /* Walker to persist agginfo data */ + int *aCsrMap = 0; /* Check to see if flattening is permitted. Return 0 if not. */ assert( p!=0 ); - assert( p->pPrior==0 ); /* Unable to flatten compound queries */ + assert( p->pPrior==0 ); if( OptimizationDisabled(db, SQLITE_QueryFlattener) ) return 0; pSrc = p->pSrc; assert( pSrc && iFrom>=0 && iFromnSrc ); pSubitem = &pSrc->a[iFrom]; iParent = pSubitem->iCursor; - pSub = pSubitem->pSelect; + assert( pSubitem->fg.isSubquery ); + pSub = pSubitem->u4.pSubq->pSelect; assert( pSub!=0 ); - if( subqueryIsAgg ){ - if( isAgg ) return 0; /* Restriction (1) */ - if( pSrc->nSrc>1 ) return 0; /* Restriction (2a) */ - if( (p->pWhere && ExprHasProperty(p->pWhere,EP_Subquery)) - || (sqlite3ExprListFlags(p->pEList) & EP_Subquery)!=0 - || (sqlite3ExprListFlags(p->pOrderBy) & EP_Subquery)!=0 - ){ - return 0; /* Restriction (2b) */ - } - } - + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( p->pWin || pSub->pWin ) return 0; /* Restriction (25) */ +#endif + pSubSrc = pSub->pSrc; assert( pSubSrc ); /* Prior to version 3.1.2, when LIMIT and OFFSET had to be simple constants, @@ -116237,18 +148086,15 @@ static int flattenSubquery( ** became arbitrary expressions, we were forced to add restrictions (13) ** and (14). */ if( pSub->pLimit && p->pLimit ) return 0; /* Restriction (13) */ - if( pSub->pOffset ) return 0; /* Restriction (14) */ + if( pSub->pLimit && pSub->pLimit->pRight ) return 0; /* Restriction (14) */ if( (p->selFlags & SF_Compound)!=0 && pSub->pLimit ){ return 0; /* Restriction (15) */ } if( pSubSrc->nSrc==0 ) return 0; /* Restriction (7) */ - if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (5) */ + if( pSub->selFlags & SF_Distinct ) return 0; /* Restriction (4) */ if( pSub->pLimit && (pSrc->nSrc>1 || isAgg) ){ return 0; /* Restrictions (8)(9) */ } - if( (p->selFlags & SF_Distinct)!=0 && subqueryIsAgg ){ - return 0; /* Restriction (6) */ - } if( p->pOrderBy && pSub->pOrderBy ){ return 0; /* Restriction (11) */ } @@ -116257,19 +148103,14 @@ static int flattenSubquery( if( pSub->pLimit && (p->selFlags & SF_Distinct)!=0 ){ return 0; /* Restriction (21) */ } - testcase( pSub->selFlags & SF_Recursive ); - testcase( pSub->selFlags & SF_MinMaxAgg ); - if( pSub->selFlags & (SF_Recursive|SF_MinMaxAgg) ){ - return 0; /* Restrictions (22) and (24) */ - } - if( (p->selFlags & SF_Recursive) && pSub->pPrior ){ - return 0; /* Restriction (23) */ + if( pSub->selFlags & (SF_Recursive) ){ + return 0; /* Restrictions (22) */ } - /* OBSOLETE COMMENT 1: - ** Restriction 3: If the subquery is a join, make sure the subquery is - ** not used as the right operand of an outer join. Examples of why this - ** is not allowed: + /* + ** If the subquery is the right operand of a LEFT JOIN, then the + ** subquery may not be a join itself (3a). Example of why this is not + ** allowed: ** ** t1 LEFT OUTER JOIN (t2 JOIN t3) ** @@ -116279,67 +148120,88 @@ static int flattenSubquery( ** ** which is not at all the same thing. ** - ** OBSOLETE COMMENT 2: - ** Restriction 12: If the subquery is the right operand of a left outer - ** join, make sure the subquery has no WHERE clause. - ** An examples of why this is not allowed: - ** - ** t1 LEFT OUTER JOIN (SELECT * FROM t2 WHERE t2.x>0) - ** - ** If we flatten the above, we would get - ** - ** (t1 LEFT OUTER JOIN t2) WHERE t2.x>0 - ** - ** But the t2.x>0 test will always fail on a NULL row of t2, which - ** effectively converts the OUTER JOIN into an INNER JOIN. - ** - ** THIS OVERRIDES OBSOLETE COMMENTS 1 AND 2 ABOVE: - ** Ticket #3300 shows that flattening the right term of a LEFT JOIN - ** is fraught with danger. Best to avoid the whole thing. If the - ** subquery is the right term of a LEFT JOIN, then do not flatten. + ** See also tickets #306, #350, and #3300. */ - if( (pSubitem->fg.jointype & JT_OUTER)!=0 ){ - return 0; + if( (pSubitem->fg.jointype & (JT_OUTER|JT_LTORJ))!=0 ){ + if( pSubSrc->nSrc>1 /* (3a) */ + || IsVirtual(pSubSrc->a[0].pSTab) /* (3b) */ + || (p->selFlags & SF_Distinct)!=0 /* (3d) */ + || (pSubitem->fg.jointype & JT_RIGHT)!=0 /* (26) */ + ){ + return 0; + } + isOuterJoin = 1; } - /* Restriction 17: If the sub-query is a compound SELECT, then it must + assert( pSubSrc->nSrc>0 ); /* True by restriction (7) */ + if( iFrom>0 && (pSubSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + return 0; /* Restriction (27a) */ + } + + /* Condition (28) is blocked by the caller */ + assert( !pSubitem->fg.isCte || pSubitem->u2.pCteUse->eM10d!=M10d_Yes ); + + /* Restriction (17): If the sub-query is a compound SELECT, then it must ** use only the UNION ALL operator. And none of the simple select queries ** that make up the compound SELECT are allowed to be aggregate or distinct ** queries. */ if( pSub->pPrior ){ + int ii; if( pSub->pOrderBy ){ - return 0; /* Restriction 20 */ + return 0; /* Restriction (20) */ } - if( isAgg || (p->selFlags & SF_Distinct)!=0 || pSrc->nSrc!=1 ){ - return 0; + if( isAgg || (p->selFlags & SF_Distinct)!=0 || isOuterJoin>0 ){ + return 0; /* (17d1), (17d2), or (17f) */ } for(pSub1=pSub; pSub1; pSub1=pSub1->pPrior){ testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct ); testcase( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))==SF_Aggregate ); assert( pSub->pSrc!=0 ); + assert( (pSub->selFlags & SF_Recursive)==0 ); assert( pSub->pEList->nExpr==pSub1->pEList->nExpr ); - if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 - || (pSub1->pPrior && pSub1->op!=TK_ALL) - || pSub1->pSrc->nSrc<1 + if( (pSub1->selFlags & (SF_Distinct|SF_Aggregate))!=0 /* (17b) */ + || (pSub1->pPrior && pSub1->op!=TK_ALL) /* (17a) */ + || pSub1->pSrc->nSrc<1 /* (17c) */ +#ifndef SQLITE_OMIT_WINDOWFUNC + || pSub1->pWin /* (17e) */ +#endif ){ return 0; } + if( iFrom>0 && (pSub1->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + /* Without this restriction, the JT_LTORJ flag would end up being + ** omitted on left-hand tables of the right join that is being + ** flattened. */ + return 0; /* Restrictions (17g), (27b) */ + } testcase( pSub1->pSrc->nSrc>1 ); } - /* Restriction 18. */ + /* Restriction (18). */ if( p->pOrderBy ){ - int ii; for(ii=0; iipOrderBy->nExpr; ii++){ if( p->pOrderBy->a[ii].u.x.iOrderByCol==0 ) return 0; } } + + /* Restriction (23) */ + if( (p->selFlags & SF_Recursive) ) return 0; + + /* Restriction (17h) */ + if( compoundHasDifferentAffinities(pSub) ) return 0; + + if( pSrc->nSrc>1 ){ + if( pParse->nSelect>500 ) return 0; + if( OptimizationDisabled(db, SQLITE_FlttnUnionAll) ) return 0; + aCsrMap = sqlite3DbMallocZero(db, ((i64)pParse->nTab+1)*sizeof(int)); + if( aCsrMap ) aCsrMap[0] = pParse->nTab; + } } /***** If we reach this point, flattening is permitted. *****/ - SELECTTRACE(1,pParse,p,("flatten %s.%p from term %d\n", - pSub->zSelName, pSub, iFrom)); + TREETRACE(0x4,pParse,p,("flatten %u.%p from term %d\n", + pSub->selId, pSub, iFrom)); /* Authorize the subquery */ pParse->zAuthContext = pSubitem->zName; @@ -116347,14 +148209,29 @@ static int flattenSubquery( testcase( i==SQLITE_DENY ); pParse->zAuthContext = zSavedAuthContext; + /* Delete the transient structures associated with the subquery */ + + if( ALWAYS(pSubitem->fg.isSubquery) ){ + pSub1 = sqlite3SubqueryDetach(db, pSubitem); + }else{ + pSub1 = 0; + } + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->fg.fixedSchema==0 ); + sqlite3DbFree(db, pSubitem->zName); + sqlite3DbFree(db, pSubitem->zAlias); + pSubitem->zName = 0; + pSubitem->zAlias = 0; + assert( pSubitem->fg.isUsing!=0 || pSubitem->u3.pOn==0 ); + /* If the sub-query is a compound SELECT statement, then (by restrictions - ** 17 and 18 above) it must be a UNION ALL and the parent query must + ** 17 and 18 above) it must be a UNION ALL and the parent query must ** be of the form: ** - ** SELECT FROM () + ** SELECT FROM () ** ** followed by any ORDER BY, LIMIT and/or OFFSET clauses. This block - ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or + ** creates N-1 copies of the parent query without any ORDER BY, LIMIT or ** OFFSET clauses and joins them to the left-hand-side of the original ** using UNION ALL operators. In this case N is the number of simple ** select statements in the compound sub-query. @@ -116384,49 +148261,41 @@ static int flattenSubquery( Select *pNew; ExprList *pOrderBy = p->pOrderBy; Expr *pLimit = p->pLimit; - Expr *pOffset = p->pOffset; Select *pPrior = p->pPrior; + Table *pItemTab = pSubitem->pSTab; + pSubitem->pSTab = 0; p->pOrderBy = 0; - p->pSrc = 0; p->pPrior = 0; p->pLimit = 0; - p->pOffset = 0; pNew = sqlite3SelectDup(db, p, 0); - sqlite3SelectSetName(pNew, pSub->zSelName); - p->pOffset = pOffset; p->pLimit = pLimit; p->pOrderBy = pOrderBy; - p->pSrc = pSrc; p->op = TK_ALL; + pSubitem->pSTab = pItemTab; if( pNew==0 ){ p->pPrior = pPrior; }else{ + pNew->selId = ++pParse->nSelect; + if( aCsrMap && ALWAYS(db->mallocFailed==0) ){ + renumberCursors(pParse, pNew, iFrom, aCsrMap); + } pNew->pPrior = pPrior; if( pPrior ) pPrior->pNext = pNew; pNew->pNext = p; p->pPrior = pNew; - SELECTTRACE(2,pParse,p, - ("compound-subquery flattener creates %s.%p as peer\n", - pNew->zSelName, pNew)); + TREETRACE(0x4,pParse,p,("compound-subquery flattener" + " creates %u as peer\n",pNew->selId)); } - if( db->mallocFailed ) return 1; + assert( pSubitem->fg.isSubquery==0 ); + } + sqlite3DbFree(db, aCsrMap); + if( db->mallocFailed ){ + assert( pSubitem->fg.fixedSchema==0 ); + assert( pSubitem->fg.isSubquery==0 ); + assert( pSubitem->u4.zDatabase==0 ); + sqlite3SrcItemAttachSubquery(pParse, pSubitem, pSub1, 0); + return 1; } - - /* Begin flattening the iFrom-th entry of the FROM clause - ** in the outer query. - */ - pSub = pSub1 = pSubitem->pSelect; - - /* Delete the transient table structure associated with the - ** subquery - */ - sqlite3DbFree(db, pSubitem->zDatabase); - sqlite3DbFree(db, pSubitem->zName); - sqlite3DbFree(db, pSubitem->zAlias); - pSubitem->zDatabase = 0; - pSubitem->zName = 0; - pSubitem->zAlias = 0; - pSubitem->pSelect = 0; /* Defer deleting the Table object associated with the ** subquery until code generation is @@ -116435,16 +148304,16 @@ static int flattenSubquery( ** ** pSubitem->pTab is always non-NULL by test restrictions and tests above. */ - if( ALWAYS(pSubitem->pTab!=0) ){ - Table *pTabToDel = pSubitem->pTab; - if( pTabToDel->nRef==1 ){ + if( ALWAYS(pSubitem->pSTab!=0) ){ + Table *pTabToDel = pSubitem->pSTab; + if( pTabToDel->nTabRef==1 ){ Parse *pToplevel = sqlite3ParseToplevel(pParse); - pTabToDel->pNextZombie = pToplevel->pZombieTab; - pToplevel->pZombieTab = pTabToDel; + sqlite3ParserAddCleanup(pToplevel, sqlite3DeleteTableGeneric, pTabToDel); + testcase( pToplevel->earlyCleanup ); }else{ - pTabToDel->nRef--; + pTabToDel->nTabRef--; } - pSubitem->pTab = 0; + pSubitem->pSTab = 0; } /* The following loop runs once for each term in a compound-subquery @@ -116460,23 +148329,18 @@ static int flattenSubquery( ** those references with expressions that resolve to the subquery FROM ** elements we are now copying in. */ + pSub = pSub1; for(pParent=p; pParent; pParent=pParent->pPrior, pSub=pSub->pPrior){ int nSubSrc; u8 jointype = 0; + u8 ltorj = pSrc->a[iFrom].fg.jointype & JT_LTORJ; + assert( pSub!=0 ); pSubSrc = pSub->pSrc; /* FROM clause of subquery */ nSubSrc = pSubSrc->nSrc; /* Number of terms in subquery FROM clause */ pSrc = pParent->pSrc; /* FROM clause of the outer query */ - if( pSrc ){ - assert( pParent==p ); /* First time through the loop */ - jointype = pSubitem->fg.jointype; - }else{ - assert( pParent!=p ); /* 2nd and subsequent times through the loop */ - pSrc = pParent->pSrc = sqlite3SrcListAppend(db, 0, 0, 0); - if( pSrc==0 ){ - assert( db->mallocFailed ); - break; - } + if( pParent==p ){ + jointype = pSubitem->fg.jointype; /* First time through the loop */ } /* The subquery uses a single slot of the FROM clause of the outer @@ -116495,26 +148359,32 @@ static int flattenSubquery( ** for the two elements in the FROM clause of the subquery. */ if( nSubSrc>1 ){ - pParent->pSrc = pSrc = sqlite3SrcListEnlarge(db, pSrc, nSubSrc-1,iFrom+1); - if( db->mallocFailed ){ - break; - } + pSrc = sqlite3SrcListEnlarge(pParse, pSrc, nSubSrc-1,iFrom+1); + if( pSrc==0 ) break; + pParent->pSrc = pSrc; } /* Transfer the FROM clause terms from the subquery into the ** outer query. */ for(i=0; ia[i+iFrom].pUsing); - assert( pSrc->a[i+iFrom].fg.isTabFunc==0 ); - pSrc->a[i+iFrom] = pSubSrc->a[i]; + SrcItem *pItem = &pSrc->a[i+iFrom]; + assert( pItem->fg.isTabFunc==0 ); + assert( pItem->fg.isSubquery + || pItem->fg.fixedSchema + || pItem->u4.zDatabase==0 ); + if( pItem->fg.isUsing ) sqlite3IdListDelete(db, pItem->u3.pUsing); + *pItem = pSubSrc->a[i]; + pItem->fg.jointype |= ltorj; + iNewParent = pSubSrc->a[i].iCursor; memset(&pSubSrc->a[i], 0, sizeof(pSubSrc->a[i])); } - pSrc->a[iFrom].fg.jointype = jointype; - - /* Now begin substituting subquery result set expressions for + pSrc->a[iFrom].fg.jointype &= JT_LTORJ; + pSrc->a[iFrom].fg.jointype |= jointype | ltorj; + + /* Now begin substituting subquery result set expressions for ** references to the iParent in the outer query. - ** + ** ** Example: ** ** SELECT a+5, b*10 FROM (SELECT x*3 AS a, y+10 AS b FROM t1) WHERE a>b; @@ -116524,20 +148394,12 @@ static int flattenSubquery( ** We look at every expression in the outer query and every place we see ** "a" we substitute "x*3" and every place we see "b" we substitute "y+10". */ - pList = pParent->pEList; - for(i=0; inExpr; i++){ - if( pList->a[i].zName==0 ){ - char *zName = sqlite3DbStrDup(db, pList->a[i].zSpan); - sqlite3Dequote(zName); - pList->a[i].zName = zName; - } - } - if( pSub->pOrderBy ){ + if( pSub->pOrderBy && (pParent->selFlags & SF_NoopOrderBy)==0 ){ /* At this point, any non-zero iOrderByCol values indicate that the ** ORDER BY column expression is identical to the iOrderByCol'th ** expression returned by SELECT statement pSub. Since these values ** do not necessarily correspond to columns in SELECT statement pParent, - ** zero them before transfering the ORDER BY clause. + ** zero them before transferring the ORDER BY clause. ** ** Not doing this may cause an error if a subsequent call to this ** function attempts to flatten a compound sub-query into pParent @@ -116548,29 +148410,37 @@ static int flattenSubquery( pOrderBy->a[i].u.x.iOrderByCol = 0; } assert( pParent->pOrderBy==0 ); - assert( pSub->pPrior==0 ); pParent->pOrderBy = pOrderBy; pSub->pOrderBy = 0; } - pWhere = sqlite3ExprDup(db, pSub->pWhere, 0); - if( subqueryIsAgg ){ - assert( pParent->pHaving==0 ); - pParent->pHaving = pParent->pWhere; - pParent->pWhere = pWhere; - pParent->pHaving = sqlite3ExprAnd(db, pParent->pHaving, - sqlite3ExprDup(db, pSub->pHaving, 0)); - assert( pParent->pGroupBy==0 ); - pParent->pGroupBy = sqlite3ExprListDup(db, pSub->pGroupBy, 0); - }else{ - pParent->pWhere = sqlite3ExprAnd(db, pParent->pWhere, pWhere); + pWhere = pSub->pWhere; + pSub->pWhere = 0; + if( isOuterJoin>0 ){ + sqlite3SetJoinExpr(pWhere, iNewParent, EP_OuterON); } - substSelect(db, pParent, iParent, pSub->pEList, 0); - - /* The flattened query is distinct if either the inner or the - ** outer query is distinct. - */ - pParent->selFlags |= pSub->selFlags & SF_Distinct; - + if( pWhere ){ + if( pParent->pWhere ){ + pParent->pWhere = sqlite3PExpr(pParse, TK_AND, pWhere, pParent->pWhere); + }else{ + pParent->pWhere = pWhere; + } + } + if( db->mallocFailed==0 ){ + SubstContext x; + x.pParse = pParse; + x.iTable = iParent; + x.iNewTable = iNewParent; + x.isOuterJoin = isOuterJoin; + x.pEList = pSub->pEList; + x.pCList = findLeftmostExprlist(pSub); + substSelect(&x, pParent, 0); + } + + /* The flattened query is a compound if either the inner or the + ** outer query is a compound. */ + pParent->selFlags |= pSub->selFlags & SF_Compound; + assert( (pSub->selFlags & SF_Distinct)==0 ); /* restriction (17b) */ + /* ** SELECT ... FROM (SELECT ... LIMIT a OFFSET b) LIMIT x OFFSET y; ** @@ -116581,16 +148451,23 @@ static int flattenSubquery( pParent->pLimit = pSub->pLimit; pSub->pLimit = 0; } + + /* Recompute the SrcItem.colUsed masks for the flattened + ** tables. */ + for(i=0; ia[i+iFrom]); + } } - /* Finially, delete what is left of the subquery and return - ** success. + /* Finally, delete what is left of the subquery and return success. */ + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w,pSub1); sqlite3SelectDelete(db, pSub1); -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After flattening:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4 ){ + TREETRACE(0x4,pParse,p,("After flattening:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -116599,7 +148476,304 @@ static int flattenSubquery( } #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ +/* +** A structure to keep track of all of the column values that are fixed to +** a known value due to WHERE clause constraints of the form COLUMN=VALUE. +*/ +typedef struct WhereConst WhereConst; +struct WhereConst { + Parse *pParse; /* Parsing context */ + u8 *pOomFault; /* Pointer to pParse->db->mallocFailed */ + int nConst; /* Number for COLUMN=CONSTANT terms */ + int nChng; /* Number of times a constant is propagated */ + int bHasAffBlob; /* At least one column in apExpr[] as affinity BLOB */ + u32 mExcludeOn; /* Which ON expressions to exclude from considertion. + ** Either EP_OuterON or EP_InnerON|EP_OuterON */ + Expr **apExpr; /* [i*2] is COLUMN and [i*2+1] is VALUE */ +}; +/* +** Add a new entry to the pConst object. Except, do not add duplicate +** pColumn entries. Also, do not add if doing so would not be appropriate. +** +** The caller guarantees the pColumn is a column and pValue is a constant. +** This routine has to do some additional checks before completing the +** insert. +*/ +static void constInsert( + WhereConst *pConst, /* The WhereConst into which we are inserting */ + Expr *pColumn, /* The COLUMN part of the constraint */ + Expr *pValue, /* The VALUE part of the constraint */ + Expr *pExpr /* Overall expression: COLUMN=VALUE or VALUE=COLUMN */ +){ + int i; + assert( pColumn->op==TK_COLUMN ); + assert( sqlite3ExprIsConstant(pConst->pParse, pValue) ); + + if( ExprHasProperty(pColumn, EP_FixedCol) ) return; + if( sqlite3ExprAffinity(pValue)!=0 ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pConst->pParse,pExpr)) ){ + return; + } + + /* 2018-10-25 ticket [cf5ed20f] + ** Make sure the same pColumn is not inserted more than once */ + for(i=0; inConst; i++){ + const Expr *pE2 = pConst->apExpr[i*2]; + assert( pE2->op==TK_COLUMN ); + if( pE2->iTable==pColumn->iTable + && pE2->iColumn==pColumn->iColumn + ){ + return; /* Already present. Return without doing anything. */ + } + } + if( sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + pConst->bHasAffBlob = 1; + } + + pConst->nConst++; + pConst->apExpr = sqlite3DbReallocOrFree(pConst->pParse->db, pConst->apExpr, + pConst->nConst*2*sizeof(Expr*)); + if( pConst->apExpr==0 ){ + pConst->nConst = 0; + }else{ + pConst->apExpr[pConst->nConst*2-2] = pColumn; + pConst->apExpr[pConst->nConst*2-1] = pValue; + } +} + +/* +** Find all terms of COLUMN=VALUE or VALUE=COLUMN in pExpr where VALUE +** is a constant expression and where the term must be true because it +** is part of the AND-connected terms of the expression. For each term +** found, add it to the pConst structure. +*/ +static void findConstInWhere(WhereConst *pConst, Expr *pExpr){ + Expr *pRight, *pLeft; + if( NEVER(pExpr==0) ) return; + if( ExprHasProperty(pExpr, pConst->mExcludeOn) ){ + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + testcase( ExprHasProperty(pExpr, EP_InnerON) ); + return; + } + if( pExpr->op==TK_AND ){ + findConstInWhere(pConst, pExpr->pRight); + findConstInWhere(pConst, pExpr->pLeft); + return; + } + if( pExpr->op!=TK_EQ ) return; + pRight = pExpr->pRight; + pLeft = pExpr->pLeft; + assert( pRight!=0 ); + assert( pLeft!=0 ); + if( pRight->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pLeft) ){ + constInsert(pConst,pRight,pLeft,pExpr); + } + if( pLeft->op==TK_COLUMN && sqlite3ExprIsConstant(pConst->pParse, pRight) ){ + constInsert(pConst,pLeft,pRight,pExpr); + } +} + +/* +** This is a helper function for Walker callback propagateConstantExprRewrite(). +** +** Argument pExpr is a candidate expression to be replaced by a value. If +** pExpr is equivalent to one of the columns named in pWalker->u.pConst, +** then overwrite it with the corresponding value. Except, do not do so +** if argument bIgnoreAffBlob is non-zero and the affinity of pExpr +** is SQLITE_AFF_BLOB. +*/ +static int propagateConstantExprRewriteOne( + WhereConst *pConst, + Expr *pExpr, + int bIgnoreAffBlob +){ + int i; + if( pConst->pOomFault[0] ) return WRC_Prune; + if( pExpr->op!=TK_COLUMN ) return WRC_Continue; + if( ExprHasProperty(pExpr, EP_FixedCol|pConst->mExcludeOn) ){ + testcase( ExprHasProperty(pExpr, EP_FixedCol) ); + testcase( ExprHasProperty(pExpr, EP_OuterON) ); + testcase( ExprHasProperty(pExpr, EP_InnerON) ); + return WRC_Continue; + } + for(i=0; inConst; i++){ + Expr *pColumn = pConst->apExpr[i*2]; + if( pColumn==pExpr ) continue; + if( pColumn->iTable!=pExpr->iTable ) continue; + if( pColumn->iColumn!=pExpr->iColumn ) continue; + if( bIgnoreAffBlob && sqlite3ExprAffinity(pColumn)==SQLITE_AFF_BLOB ){ + break; + } + /* A match is found. Add the EP_FixedCol property */ + pConst->nChng++; + ExprClearProperty(pExpr, EP_Leaf); + ExprSetProperty(pExpr, EP_FixedCol); + assert( pExpr->pLeft==0 ); + pExpr->pLeft = sqlite3ExprDup(pConst->pParse->db, pConst->apExpr[i*2+1], 0); + if( pConst->pParse->db->mallocFailed ) return WRC_Prune; + break; + } + return WRC_Prune; +} + +/* +** This is a Walker expression callback. pExpr is a node from the WHERE +** clause of a SELECT statement. This function examines pExpr to see if +** any substitutions based on the contents of pWalker->u.pConst should +** be made to pExpr or its immediate children. +** +** A substitution is made if: +** +** + pExpr is a column with an affinity other than BLOB that matches +** one of the columns in pWalker->u.pConst, or +** +** + pExpr is a binary comparison operator (=, <=, >=, <, >) that +** uses an affinity other than TEXT and one of its immediate +** children is a column that matches one of the columns in +** pWalker->u.pConst. +*/ +static int propagateConstantExprRewrite(Walker *pWalker, Expr *pExpr){ + WhereConst *pConst = pWalker->u.pConst; + assert( TK_GT==TK_EQ+1 ); + assert( TK_LE==TK_EQ+2 ); + assert( TK_LT==TK_EQ+3 ); + assert( TK_GE==TK_EQ+4 ); + if( pConst->bHasAffBlob ){ + if( (pExpr->op>=TK_EQ && pExpr->op<=TK_GE) + || pExpr->op==TK_IS + ){ + propagateConstantExprRewriteOne(pConst, pExpr->pLeft, 0); + if( pConst->pOomFault[0] ) return WRC_Prune; + if( sqlite3ExprAffinity(pExpr->pLeft)!=SQLITE_AFF_TEXT ){ + propagateConstantExprRewriteOne(pConst, pExpr->pRight, 0); + } + } + } + return propagateConstantExprRewriteOne(pConst, pExpr, pConst->bHasAffBlob); +} + +/* +** The WHERE-clause constant propagation optimization. +** +** If the WHERE clause contains terms of the form COLUMN=CONSTANT or +** CONSTANT=COLUMN that are top-level AND-connected terms that are not +** part of a ON clause from a LEFT JOIN, then throughout the query +** replace all other occurrences of COLUMN with CONSTANT. +** +** For example, the query: +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=t1.a AND t3.c=t2.b +** +** Is transformed into +** +** SELECT * FROM t1, t2, t3 WHERE t1.a=39 AND t2.b=39 AND t3.c=39 +** +** Return true if any transformations where made and false if not. +** +** Implementation note: Constant propagation is tricky due to affinity +** and collating sequence interactions. Consider this example: +** +** CREATE TABLE t1(a INT,b TEXT); +** INSERT INTO t1 VALUES(123,'0123'); +** SELECT * FROM t1 WHERE a=123 AND b=a; +** SELECT * FROM t1 WHERE a=123 AND b=123; +** +** The two SELECT statements above should return different answers. b=a +** is always true because the comparison uses numeric affinity, but b=123 +** is false because it uses text affinity and '0123' is not the same as '123'. +** To work around this, the expression tree is not actually changed from +** "b=a" to "b=123" but rather the "a" in "b=a" is tagged with EP_FixedCol +** and the "123" value is hung off of the pLeft pointer. Code generator +** routines know to generate the constant "123" instead of looking up the +** column value. Also, to avoid collation problems, this optimization is +** only attempted if the "a=123" term uses the default BINARY collation. +** +** 2021-05-25 forum post 6a06202608: Another troublesome case is... +** +** CREATE TABLE t1(x); +** INSERT INTO t1 VALUES(10.0); +** SELECT 1 FROM t1 WHERE x=10 AND x LIKE 10; +** +** The query should return no rows, because the t1.x value is '10.0' not '10' +** and '10.0' is not LIKE '10'. But if we are not careful, the first WHERE +** term "x=10" will cause the second WHERE term to become "10 LIKE 10", +** resulting in a false positive. To avoid this, constant propagation for +** columns with BLOB affinity is only allowed if the constant is used with +** operators ==, <=, <, >=, >, or IS in a way that will cause the correct +** type conversions to occur. See logic associated with the bHasAffBlob flag +** for details. +*/ +static int propagateConstants( + Parse *pParse, /* The parsing context */ + Select *p /* The query in which to propagate constants */ +){ + WhereConst x; + Walker w; + int nChng = 0; + x.pParse = pParse; + x.pOomFault = &pParse->db->mallocFailed; + do{ + x.nConst = 0; + x.nChng = 0; + x.apExpr = 0; + x.bHasAffBlob = 0; + if( ALWAYS(p->pSrc!=0) + && p->pSrc->nSrc>0 + && (p->pSrc->a[0].fg.jointype & JT_LTORJ)!=0 + ){ + /* Do not propagate constants on any ON clause if there is a + ** RIGHT JOIN anywhere in the query */ + x.mExcludeOn = EP_InnerON | EP_OuterON; + }else{ + /* Do not propagate constants through the ON clause of a LEFT JOIN */ + x.mExcludeOn = EP_OuterON; + } + findConstInWhere(&x, p->pWhere); + if( x.nConst ){ + memset(&w, 0, sizeof(w)); + w.pParse = pParse; + w.xExprCallback = propagateConstantExprRewrite; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.xSelectCallback2 = 0; + w.walkerDepth = 0; + w.u.pConst = &x; + sqlite3WalkExpr(&w, p->pWhere); + sqlite3DbFree(x.pParse->db, x.apExpr); + nChng += x.nChng; + } + }while( x.nChng ); + return nChng; +} + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) +# if !defined(SQLITE_OMIT_WINDOWFUNC) +/* +** This function is called to determine whether or not it is safe to +** push WHERE clause expression pExpr down to FROM clause sub-query +** pSubq, which contains at least one window function. Return 1 +** if it is safe and the expression should be pushed down, or 0 +** otherwise. +** +** It is only safe to push the expression down if it consists only +** of constants and copies of expressions that appear in the PARTITION +** BY clause of all window function used by the sub-query. It is safe +** to filter out entire partitions, but not rows within partitions, as +** this may change the results of the window functions. +** +** At the time this function is called it is guaranteed that +** +** * the sub-query uses only one distinct window frame, and +** * that the window frame has a PARTITION BY clause. +*/ +static int pushDownWindowCheck(Parse *pParse, Select *pSubq, Expr *pExpr){ + assert( pSubq->pWin->pPartition ); + assert( (pSubq->selFlags & SF_MultiPart)==0 ); + assert( pSubq->pPrior==0 ); + return sqlite3ExprIsConstantOrGroupBy(pParse, pExpr, pSubq->pWin->pPartition); +} +# endif /* SQLITE_OMIT_WINDOWFUNC */ +#endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) /* @@ -116616,59 +148790,246 @@ static int flattenSubquery( ** The hope is that the terms added to the inner query will make it more ** efficient. ** +** NAME AMBIGUITY +** +** This optimization is called the "WHERE-clause push-down optimization" +** or sometimes the "predicate push-down optimization". +** +** Do not confuse this optimization with another unrelated optimization +** with a similar name: The "MySQL push-down optimization" causes WHERE +** clause terms that can be evaluated using only the index and without +** reference to the table are run first, so that if they are false, +** unnecessary table seeks are avoided. +** +** RULES +** ** Do not attempt this optimization if: ** -** (1) The inner query is an aggregate. (In that case, we'd really want -** to copy the outer WHERE-clause terms onto the HAVING clause of the -** inner query. But they probably won't help there so do not bother.) +** (1) (** This restriction was removed on 2017-09-29. We used to +** disallow this optimization for aggregate subqueries, but now +** it is allowed by putting the extra terms on the HAVING clause. +** The added HAVING clause is pointless if the subquery lacks +** a GROUP BY clause. But such a HAVING clause is also harmless +** so there does not appear to be any reason to add extra logic +** to suppress it. **) ** ** (2) The inner query is the recursive part of a common table expression. ** ** (3) The inner query has a LIMIT clause (since the changes to the WHERE -** close would change the meaning of the LIMIT). +** clause would change the meaning of the LIMIT). ** -** (4) The inner query is the right operand of a LEFT JOIN. (The caller -** enforces this restriction since this routine does not have enough -** information to know.) +** (4) The inner query is the right operand of a LEFT JOIN and the +** expression to be pushed down does not come from the ON clause +** on that LEFT JOIN. ** ** (5) The WHERE clause expression originates in the ON or USING clause -** of a LEFT JOIN. +** of a LEFT JOIN where iCursor is not the right-hand table of that +** left join. An example: +** +** SELECT * +** FROM (SELECT 1 AS a1 UNION ALL SELECT 2) AS aa +** JOIN (SELECT 1 AS b2 UNION ALL SELECT 2) AS bb ON (a1=b2) +** LEFT JOIN (SELECT 8 AS c3 UNION ALL SELECT 9) AS cc ON (b2=2); +** +** The correct answer is three rows: (1,1,NULL),(2,2,8),(2,2,9). +** But if the (b2=2) term were to be pushed down into the bb subquery, +** then the (1,1,NULL) row would be suppressed. +** +** (6) Window functions make things tricky as changes to the WHERE clause +** of the inner query could change the window over which window +** functions are calculated. Therefore, do not attempt the optimization +** if: +** +** (6a) The inner query uses multiple incompatible window partitions. +** +** (6b) The inner query is a compound and uses window-functions. +** +** (6c) The WHERE clause does not consist entirely of constants and +** copies of expressions found in the PARTITION BY clause of +** all window-functions used by the sub-query. It is safe to +** filter out entire partitions, as this does not change the +** window over which any window-function is calculated. +** +** (7) The inner query is a Common Table Expression (CTE) that should +** be materialized. (This restriction is implemented in the calling +** routine.) +** +** (8) If the subquery is a compound that uses UNION, INTERSECT, +** or EXCEPT, then all of the result set columns for all arms of +** the compound must use the BINARY collating sequence. +** +** (9) All three of the following are true: +** +** (9a) The WHERE clause expression originates in the ON or USING clause +** of a join (either an INNER or an OUTER join), and +** +** (9b) The subquery is to the right of the ON/USING clause +** +** (9c) There is a RIGHT JOIN (or FULL JOIN) in between the ON/USING +** clause and the subquery. +** +** Without this restriction, the WHERE-clause push-down optimization +** might move the ON/USING filter expression from the left side of a +** RIGHT JOIN over to the right side, which leads to incorrect answers. +** See also restriction (6) in sqlite3ExprIsSingleTableConstraint(). +** +** (10) The inner query is not the right-hand table of a RIGHT JOIN. +** +** (11) The subquery is not a VALUES clause +** +** (12) The WHERE clause is not "rowid ISNULL" or the equivalent. This +** case only comes up if SQLite is compiled using +** SQLITE_ALLOW_ROWID_IN_VIEW. ** ** Return 0 if no changes are made and non-zero if one or more WHERE clause ** terms are duplicated into the subquery. */ static int pushDownWhereTerms( - sqlite3 *db, /* The database connection (for malloc()) */ + Parse *pParse, /* Parse context (for malloc() and error reporting) */ Select *pSubq, /* The subquery whose WHERE clause is to be augmented */ Expr *pWhere, /* The WHERE clause of the outer query */ - int iCursor /* Cursor number of the subquery */ + SrcList *pSrcList, /* The complete from clause of the outer query */ + int iSrc /* Which FROM clause term to try to push into */ ){ Expr *pNew; + SrcItem *pSrc; /* The subquery FROM term into which WHERE is pushed */ int nChng = 0; - Select *pX; /* For looping over compound SELECTs in pSubq */ + pSrc = &pSrcList->a[iSrc]; if( pWhere==0 ) return 0; - for(pX=pSubq; pX; pX=pX->pPrior){ - if( (pX->selFlags & (SF_Aggregate|SF_Recursive))!=0 ){ - testcase( pX->selFlags & SF_Aggregate ); - testcase( pX->selFlags & SF_Recursive ); - testcase( pX!=pSubq ); - return 0; /* restrictions (1) and (2) */ + if( pSubq->selFlags & (SF_Recursive|SF_MultiPart) ){ + return 0; /* restrictions (2) and (11) */ + } + if( pSrc->fg.jointype & (JT_LTORJ|JT_RIGHT) ){ + return 0; /* restrictions (10) */ + } + + if( pSubq->pPrior ){ + Select *pSel; + int notUnionAll = 0; + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + u8 op = pSel->op; + assert( op==TK_ALL || op==TK_SELECT + || op==TK_UNION || op==TK_INTERSECT || op==TK_EXCEPT ); + if( op!=TK_ALL && op!=TK_SELECT ){ + notUnionAll = 1; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSel->pWin ) return 0; /* restriction (6b) */ +#endif + } + if( notUnionAll ){ + /* If any of the compound arms are connected using UNION, INTERSECT, + ** or EXCEPT, then we must ensure that none of the columns use a + ** non-BINARY collating sequence. */ + for(pSel=pSubq; pSel; pSel=pSel->pPrior){ + int ii; + const ExprList *pList = pSel->pEList; + assert( pList!=0 ); + for(ii=0; iinExpr; ii++){ + CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[ii].pExpr); + if( !sqlite3IsBinary(pColl) ){ + return 0; /* Restriction (8) */ + } + } + } + } + }else{ +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pWin && pSubq->pWin->pPartition==0 ) return 0; +#endif + } + +#ifdef SQLITE_DEBUG + /* Only the first term of a compound can have a WITH clause. But make + ** sure no other terms are marked SF_Recursive in case something changes + ** in the future. + */ + { + Select *pX; + for(pX=pSubq; pX; pX=pX->pPrior){ + assert( (pX->selFlags & (SF_Recursive))==0 ); } } +#endif + if( pSubq->pLimit!=0 ){ return 0; /* restriction (3) */ } while( pWhere->op==TK_AND ){ - nChng += pushDownWhereTerms(db, pSubq, pWhere->pRight, iCursor); + nChng += pushDownWhereTerms(pParse, pSubq, pWhere->pRight, pSrcList, iSrc); pWhere = pWhere->pLeft; } - if( ExprHasProperty(pWhere,EP_FromJoin) ) return 0; /* restriction 5 */ - if( sqlite3ExprIsTableConstant(pWhere, iCursor) ){ + +#if 0 /* These checks now done by sqlite3ExprIsSingleTableConstraint() */ + if( ExprHasProperty(pWhere, EP_OuterON|EP_InnerON) /* (9a) */ + && (pSrcList->a[0].fg.jointype & JT_LTORJ)!=0 /* Fast pre-test of (9c) */ + ){ + int jj; + for(jj=0; jjw.iJoin==pSrcList->a[jj].iCursor ){ + /* If we reach this point, both (9a) and (9b) are satisfied. + ** The following loop checks (9c): + */ + for(jj++; jja[jj].fg.jointype & JT_RIGHT)!=0 ){ + return 0; /* restriction (9) */ + } + } + } + } + } + if( isLeftJoin + && (ExprHasProperty(pWhere,EP_OuterON)==0 + || pWhere->w.iJoin!=iCursor) + ){ + return 0; /* restriction (4) */ + } + if( ExprHasProperty(pWhere,EP_OuterON) + && pWhere->w.iJoin!=iCursor + ){ + return 0; /* restriction (5) */ + } +#endif + +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( ViewCanHaveRowid && (pWhere->op==TK_ISNULL || pWhere->op==TK_NOTNULL) ){ + Expr *pLeft = pWhere->pLeft; + if( ALWAYS(pLeft) + && pLeft->op==TK_COLUMN + && pLeft->iColumn < 0 + ){ + return 0; /* Restriction (12) */ + } + } +#endif + + if( sqlite3ExprIsSingleTableConstraint(pWhere, pSrcList, iSrc, 1) ){ nChng++; + pSubq->selFlags |= SF_PushDown; while( pSubq ){ - pNew = sqlite3ExprDup(db, pWhere, 0); - pNew = substExpr(db, pNew, iCursor, pSubq->pEList); - pSubq->pWhere = sqlite3ExprAnd(db, pSubq->pWhere, pNew); + SubstContext x; + pNew = sqlite3ExprDup(pParse->db, pWhere, 0); + unsetJoinExpr(pNew, -1, 1); + x.pParse = pParse; + x.iTable = pSrc->iCursor; + x.iNewTable = pSrc->iCursor; + x.isOuterJoin = 0; + x.pEList = pSubq->pEList; + x.pCList = findLeftmostExprlist(pSubq); + pNew = substExpr(&x, pNew); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pSubq->pWin && 0==pushDownWindowCheck(pParse, pSubq, pNew) ){ + /* Restriction 6c has prevented push-down in this case */ + sqlite3ExprDelete(pParse->db, pNew); + nChng--; + break; + } +#endif + if( pSubq->selFlags & SF_Aggregate ){ + pSubq->pHaving = sqlite3ExprAnd(pParse, pSubq->pHaving, pNew); + }else{ + pSubq->pWhere = sqlite3ExprAnd(pParse, pSubq->pWhere, pNew); + } pSubq = pSubq->pPrior; } } @@ -116677,55 +149038,147 @@ static int pushDownWhereTerms( #endif /* !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) */ /* -** Based on the contents of the AggInfo structure indicated by the first -** argument, this function checks if the following are true: +** Check to see if a subquery contains result-set columns that are +** never used. If it does, change the value of those result-set columns +** to NULL so that they do not cause unnecessary work to compute. ** -** * the query contains just a single aggregate function, -** * the aggregate function is either min() or max(), and -** * the argument to the aggregate function is a column value. -** -** If all of the above are true, then WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX -** is returned as appropriate. Also, *ppMinMax is set to point to the -** list of arguments passed to the aggregate before returning. -** -** Or, if the conditions above are not met, *ppMinMax is set to 0 and -** WHERE_ORDERBY_NORMAL is returned. +** Return the number of column that were changed to NULL. */ -static u8 minMaxQuery(AggInfo *pAggInfo, ExprList **ppMinMax){ - int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ +static int disableUnusedSubqueryResultColumns(SrcItem *pItem){ + int nCol; + Select *pSub; /* The subquery to be simplified */ + Select *pX; /* For looping over compound elements of pSub */ + Table *pTab; /* The table that describes the subquery */ + int j; /* Column number */ + int nChng = 0; /* Number of columns converted to NULL */ + Bitmask colUsed; /* Columns that may not be NULLed out */ - *ppMinMax = 0; - if( pAggInfo->nFunc==1 ){ - Expr *pExpr = pAggInfo->aFunc[0].pExpr; /* Aggregate function */ - ExprList *pEList = pExpr->x.pList; /* Arguments to agg function */ - - assert( pExpr->op==TK_AGG_FUNCTION ); - if( pEList && pEList->nExpr==1 && pEList->a[0].pExpr->op==TK_AGG_COLUMN ){ - const char *zFunc = pExpr->u.zToken; - if( sqlite3StrICmp(zFunc, "min")==0 ){ - eRet = WHERE_ORDERBY_MIN; - *ppMinMax = pEList; - }else if( sqlite3StrICmp(zFunc, "max")==0 ){ - eRet = WHERE_ORDERBY_MAX; - *ppMinMax = pEList; + assert( pItem!=0 ); + if( pItem->fg.isCorrelated || pItem->fg.isCte ){ + return 0; + } + assert( pItem->pSTab!=0 ); + pTab = pItem->pSTab; + assert( pItem->fg.isSubquery ); + pSub = pItem->u4.pSubq->pSelect; + assert( pSub->pEList->nExpr==pTab->nCol ); + for(pX=pSub; pX; pX=pX->pPrior){ + if( (pX->selFlags & (SF_Distinct|SF_Aggregate))!=0 ){ + testcase( pX->selFlags & SF_Distinct ); + testcase( pX->selFlags & SF_Aggregate ); + return 0; + } + if( pX->pPrior && pX->op!=TK_ALL ){ + /* This optimization does not work for compound subqueries that + ** use UNION, INTERSECT, or EXCEPT. Only UNION ALL is allowed. */ + return 0; + } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pX->pWin ){ + /* This optimization does not work for subqueries that use window + ** functions. */ + return 0; + } +#endif + } + colUsed = pItem->colUsed; + if( pSub->pOrderBy ){ + ExprList *pList = pSub->pOrderBy; + for(j=0; jnExpr; j++){ + u16 iCol = pList->a[j].u.x.iOrderByCol; + if( iCol>0 ){ + iCol--; + colUsed |= ((Bitmask)1)<<(iCol>=BMS ? BMS-1 : iCol); } } } + nCol = pTab->nCol; + for(j=0; jpPrior) { + Expr *pY = pX->pEList->a[j].pExpr; + if( pY->op==TK_NULL ) continue; + pY->op = TK_NULL; + ExprClearProperty(pY, EP_Skip|EP_Unlikely); + pX->selFlags |= SF_PushDown; + nChng++; + } + } + return nChng; +} - assert( *ppMinMax==0 || (*ppMinMax)->nExpr==1 ); + +/* +** The pFunc is the only aggregate function in the query. Check to see +** if the query is a candidate for the min/max optimization. +** +** If the query is a candidate for the min/max optimization, then set +** *ppMinMax to be an ORDER BY clause to be used for the optimization +** and return either WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX depending on +** whether pFunc is a min() or max() function. +** +** If the query is not a candidate for the min/max optimization, return +** WHERE_ORDERBY_NORMAL (which must be zero). +** +** This routine must be called after aggregate functions have been +** located but before their arguments have been subjected to aggregate +** analysis. +*/ +static u8 minMaxQuery(sqlite3 *db, Expr *pFunc, ExprList **ppMinMax){ + int eRet = WHERE_ORDERBY_NORMAL; /* Return value */ + ExprList *pEList; /* Arguments to agg function */ + const char *zFunc; /* Name of aggregate function pFunc */ + ExprList *pOrderBy; + u8 sortFlags = 0; + + assert( *ppMinMax==0 ); + assert( pFunc->op==TK_AGG_FUNCTION ); + assert( !IsWindowFunc(pFunc) ); + assert( ExprUseXList(pFunc) ); + pEList = pFunc->x.pList; + if( pEList==0 + || pEList->nExpr!=1 + || ExprHasProperty(pFunc, EP_WinFunc) + || OptimizationDisabled(db, SQLITE_MinMaxOpt) + ){ + return eRet; + } + assert( !ExprHasProperty(pFunc, EP_IntValue) ); + zFunc = pFunc->u.zToken; + if( sqlite3StrICmp(zFunc, "min")==0 ){ + eRet = WHERE_ORDERBY_MIN; + if( sqlite3ExprCanBeNull(pEList->a[0].pExpr) ){ + sortFlags = KEYINFO_ORDER_BIGNULL; + } + }else if( sqlite3StrICmp(zFunc, "max")==0 ){ + eRet = WHERE_ORDERBY_MAX; + sortFlags = KEYINFO_ORDER_DESC; + }else{ + return eRet; + } + *ppMinMax = pOrderBy = sqlite3ExprListDup(db, pEList, 0); + assert( pOrderBy!=0 || db->mallocFailed ); + if( pOrderBy ) pOrderBy->a[0].fg.sortFlags = sortFlags; return eRet; } /* ** The select statement passed as the first argument is an aggregate query. -** The second argument is the associated aggregate-info object. This +** The second argument is the associated aggregate-info object. This ** function tests if the SELECT is of the form: ** ** SELECT count(*) FROM ** ** where table is a database table, not a sub-select or view. If the query ** does match this pattern, then a pointer to the Table object representing -** is returned. Otherwise, 0 is returned. +** is returned. Otherwise, NULL is returned. +** +** This routine checks to see if it is safe to use the count optimization. +** A correct answer is still obtained (though perhaps more slowly) if +** this routine returns NULL when it could have returned a table pointer. +** But returning the pointer when NULL should have been returned can +** result in incorrect answers and/or crashes. So, when in doubt, return NULL. */ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ Table *pTab; @@ -116733,20 +149186,28 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ assert( !p->pGroupBy ); - if( p->pWhere || p->pEList->nExpr!=1 - || p->pSrc->nSrc!=1 || p->pSrc->a[0].pSelect + if( p->pWhere + || p->pEList->nExpr!=1 + || p->pSrc->nSrc!=1 + || p->pSrc->a[0].fg.isSubquery + || pAggInfo->nFunc!=1 + || p->pHaving ){ return 0; } - pTab = p->pSrc->a[0].pTab; + pTab = p->pSrc->a[0].pSTab; + assert( pTab!=0 ); + assert( !IsView(pTab) ); + if( !IsOrdinaryTable(pTab) ) return 0; pExpr = p->pEList->a[0].pExpr; - assert( pTab && !pTab->pSelect && pExpr ); - - if( IsVirtual(pTab) ) return 0; + assert( pExpr!=0 ); if( pExpr->op!=TK_AGG_FUNCTION ) return 0; - if( NEVER(pAggInfo->nFunc==0) ) return 0; + if( pExpr->pAggInfo!=pAggInfo ) return 0; if( (pAggInfo->aFunc[0].pFunc->funcFlags&SQLITE_FUNC_COUNT)==0 ) return 0; - if( pExpr->flags&EP_Distinct ) return 0; + assert( pAggInfo->aFunc[0].pFExpr==pExpr ); + testcase( ExprHasProperty(pExpr, EP_Distinct) ); + testcase( ExprHasProperty(pExpr, EP_WinFunc) ); + if( ExprHasProperty(pExpr, EP_Distinct|EP_WinFunc) ) return 0; return pTab; } @@ -116754,30 +149215,33 @@ static Table *isSimpleCount(Select *p, AggInfo *pAggInfo){ /* ** If the source-list item passed as an argument was augmented with an ** INDEXED BY clause, then try to locate the specified index. If there -** was such a clause and the named index cannot be found, return -** SQLITE_ERROR and leave an error in pParse. Otherwise, populate +** was such a clause and the named index cannot be found, return +** SQLITE_ERROR and leave an error in pParse. Otherwise, populate ** pFrom->pIndex and return SQLITE_OK. */ -SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, struct SrcList_item *pFrom){ - if( pFrom->pTab && pFrom->fg.isIndexedBy ){ - Table *pTab = pFrom->pTab; - char *zIndexedBy = pFrom->u1.zIndexedBy; - Index *pIdx; - for(pIdx=pTab->pIndex; - pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); - pIdx=pIdx->pNext - ); - if( !pIdx ){ - sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); - pParse->checkSchema = 1; - return SQLITE_ERROR; - } - pFrom->pIBIndex = pIdx; +SQLITE_PRIVATE int sqlite3IndexedByLookup(Parse *pParse, SrcItem *pFrom){ + Table *pTab = pFrom->pSTab; + char *zIndexedBy = pFrom->u1.zIndexedBy; + Index *pIdx; + assert( pTab!=0 ); + assert( pFrom->fg.isIndexedBy!=0 ); + + for(pIdx=pTab->pIndex; + pIdx && sqlite3StrICmp(pIdx->zName, zIndexedBy); + pIdx=pIdx->pNext + ); + if( !pIdx ){ + sqlite3ErrorMsg(pParse, "no such index: %s", zIndexedBy, 0); + pParse->checkSchema = 1; + return SQLITE_ERROR; } + assert( pFrom->fg.isCte==0 ); + pFrom->u2.pIBIndex = pIdx; return SQLITE_OK; } + /* -** Detect compound SELECT statements that use an ORDER BY clause with +** Detect compound SELECT statements that use an ORDER BY clause with ** an alternative collating sequence. ** ** SELECT ... FROM t1 EXCEPT SELECT ... FROM t2 ORDER BY .. COLLATE ... @@ -116812,6 +149276,14 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ for(pX=p; pX && (pX->op==TK_ALL || pX->op==TK_SELECT); pX=pX->pPrior){} if( pX==0 ) return WRC_Continue; a = p->pOrderBy->a; +#ifndef SQLITE_OMIT_WINDOWFUNC + /* If iOrderByCol is already non-zero, then it has already been matched + ** to a result column of the SELECT statement. This occurs when the + ** SELECT is rewritten for window-functions processing and then passed + ** to sqlite3SelectPrep() and similar a second time. The rewriting done + ** by this function is not required in this case. */ + if( a[0].u.x.iOrderByCol ) return WRC_Continue; +#endif for(i=p->pOrderBy->nExpr-1; i>=0; i--){ if( a[i].pExpr->flags & EP_Collate ) break; } @@ -116824,8 +149296,12 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ pNew = sqlite3DbMallocZero(db, sizeof(*pNew) ); if( pNew==0 ) return WRC_Abort; memset(&dummy, 0, sizeof(dummy)); - pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0,0); - if( pNewSrc==0 ) return WRC_Abort; + pNewSrc = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&dummy,pNew,0); + assert( pNewSrc!=0 || pParse->nErr ); + if( pParse->nErr ){ + sqlite3SrcListDelete(db, pNewSrc); + return WRC_Abort; + } *pNew = *p; p->pSrc = pNewSrc; p->pEList = sqlite3ExprListAppend(pParse, 0, sqlite3Expr(db, TK_ASTERISK, 0)); @@ -116837,13 +149313,15 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ p->pPrior = 0; p->pNext = 0; p->pWith = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + p->pWinDefn = 0; +#endif p->selFlags &= ~SF_Compound; assert( (p->selFlags & SF_Converted)==0 ); p->selFlags |= SF_Converted; assert( pNew->pPrior!=0 ); pNew->pPrior->pNext = pNew; pNew->pLimit = 0; - pNew->pOffset = 0; return WRC_Continue; } @@ -116852,7 +149330,7 @@ static int convertCompoundSelectToSubquery(Walker *pWalker, Select *p){ ** arguments. If it does, leave an error message in pParse and return ** non-zero, since pFrom is not allowed to be a table-valued function. */ -static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ +static int cannotBeFunction(Parse *pParse, SrcItem *pFrom){ if( pFrom->fg.isTabFunc ){ sqlite3ErrorMsg(pParse, "'%s' is not a function", pFrom->zName); return 1; @@ -116862,9 +149340,9 @@ static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ #ifndef SQLITE_OMIT_CTE /* -** Argument pWith (which may be NULL) points to a linked list of nested -** WITH contexts, from inner to outermost. If the table identified by -** FROM clause element pItem is really a common-table-expression (CTE) +** Argument pWith (which may be NULL) points to a linked list of nested +** WITH contexts, from inner to outermost. If the table identified by +** FROM clause element pItem is really a common-table-expression (CTE) ** then return a pointer to the CTE definition for that table. Otherwise ** return NULL. ** @@ -116873,21 +149351,22 @@ static int cannotBeFunction(Parse *pParse, struct SrcList_item *pFrom){ */ static struct Cte *searchWith( With *pWith, /* Current innermost WITH clause */ - struct SrcList_item *pItem, /* FROM clause element to resolve */ + SrcItem *pItem, /* FROM clause element to resolve */ With **ppContext /* OUT: WITH clause return value belongs to */ ){ - const char *zName; - if( pItem->zDatabase==0 && (zName = pItem->zName)!=0 ){ - With *p; - for(p=pWith; p; p=p->pOuter){ - int i; - for(i=0; inCte; i++){ - if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ - *ppContext = p; - return &p->a[i]; - } + const char *zName = pItem->zName; + With *p; + assert( pItem->fg.fixedSchema || pItem->u4.zDatabase==0 ); + assert( zName!=0 ); + for(p=pWith; p; p=p->pOuter){ + int i; + for(i=0; inCte; i++){ + if( sqlite3StrICmp(zName, p->a[i].zName)==0 ){ + *ppContext = p; + return &p->a[i]; } } + if( p->bView ) break; } return 0; } @@ -116897,55 +149376,92 @@ static struct Cte *searchWith( ** ** This routine pushes the WITH clause passed as the second argument ** onto the top of the stack. If argument bFree is true, then this -** WITH clause will never be popped from the stack. In this case it -** should be freed along with the Parse object. In other cases, when -** bFree==0, the With object will be freed along with the SELECT +** WITH clause will never be popped from the stack but should instead +** be freed along with the Parse object. In other cases, when +** bFree==0, the With object will be freed along with the SELECT ** statement with which it is associated. +** +** This routine returns a copy of pWith. Or, if bFree is true and +** the pWith object is destroyed immediately due to an OOM condition, +** then this routine return NULL. +** +** If bFree is true, do not continue to use the pWith pointer after +** calling this routine, Instead, use only the return value. */ -SQLITE_PRIVATE void sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ - assert( bFree==0 || (pParse->pWith==0 && pParse->pWithToFree==0) ); +SQLITE_PRIVATE With *sqlite3WithPush(Parse *pParse, With *pWith, u8 bFree){ if( pWith ){ - assert( pParse->pWith!=pWith ); - pWith->pOuter = pParse->pWith; - pParse->pWith = pWith; - if( bFree ) pParse->pWithToFree = pWith; + if( bFree ){ + pWith = (With*)sqlite3ParserAddCleanup(pParse, sqlite3WithDeleteGeneric, + pWith); + if( pWith==0 ) return 0; + } + if( pParse->nErr==0 ){ + assert( pParse->pWith!=pWith ); + pWith->pOuter = pParse->pWith; + pParse->pWith = pWith; + } } + return pWith; } /* -** This function checks if argument pFrom refers to a CTE declared by -** a WITH clause on the stack currently maintained by the parser. And, -** if currently processing a CTE expression, if it is a recursive -** reference to the current CTE. +** This function checks if argument pFrom refers to a CTE declared by +** a WITH clause on the stack currently maintained by the parser (on the +** pParse->pWith linked list). And if currently processing a CTE +** CTE expression, through routine checks to see if the reference is +** a recursive reference to the CTE. ** -** If pFrom falls into either of the two categories above, pFrom->pTab -** and other fields are populated accordingly. The caller should check -** (pFrom->pTab!=0) to determine whether or not a successful match -** was found. +** If pFrom matches a CTE according to either of these two above, pFrom->pTab +** and other fields are populated accordingly. ** -** Whether or not a match is found, SQLITE_OK is returned if no error -** occurs. If an error does occur, an error message is stored in the -** parser and some error code other than SQLITE_OK returned. +** Return 0 if no match is found. +** Return 1 if a match is found. +** Return 2 if an error condition is detected. */ -static int withExpand( - Walker *pWalker, - struct SrcList_item *pFrom +static int resolveFromTermToCte( + Parse *pParse, /* The parsing context */ + Walker *pWalker, /* Current tree walker */ + SrcItem *pFrom /* The FROM clause term to check */ ){ - Parse *pParse = pWalker->pParse; - sqlite3 *db = pParse->db; - struct Cte *pCte; /* Matched CTE (or NULL if no match) */ - With *pWith; /* WITH clause that pCte belongs to */ - - assert( pFrom->pTab==0 ); + Cte *pCte; /* Matched CTE (or NULL if no match) */ + With *pWith; /* The matching WITH */ + assert( pFrom->pSTab==0 ); + if( pParse->pWith==0 ){ + /* There are no WITH clauses in the stack. No match is possible */ + return 0; + } + if( pParse->nErr ){ + /* Prior errors might have left pParse->pWith in a goofy state, so + ** go no further. */ + return 0; + } + assert( pFrom->fg.hadSchema==0 || pFrom->fg.notCte!=0 ); + if( pFrom->fg.fixedSchema==0 && pFrom->u4.zDatabase!=0 ){ + /* The FROM term contains a schema qualifier (ex: main.t1) and so + ** it cannot possibly be a CTE reference. */ + return 0; + } + if( pFrom->fg.notCte ){ + /* The FROM term is specifically excluded from matching a CTE. + ** (1) It is part of a trigger that used to have zDatabase but had + ** zDatabase removed by sqlite3FixTriggerStep(). + ** (2) This is the first term in the FROM clause of an UPDATE. + */ + return 0; + } pCte = searchWith(pParse->pWith, pFrom, &pWith); if( pCte ){ + sqlite3 *db = pParse->db; Table *pTab; ExprList *pEList; Select *pSel; Select *pLeft; /* Left-most SELECT statement */ + Select *pRecTerm; /* Left-most recursive term */ int bMayRecursive; /* True if compound joined by UNION [ALL] */ With *pSavedWith; /* Initial value of pParse->pWith */ + int iRecTab = -1; /* Cursor for recursive table */ + CteUse *pCteUse; /* If pCte->zCteErr is non-NULL at this point, then this is an illegal ** recursive reference to CTE pCte. Leave an error in pParse and return @@ -116953,55 +149469,101 @@ static int withExpand( ** In this case, proceed. */ if( pCte->zCteErr ){ sqlite3ErrorMsg(pParse, pCte->zCteErr, pCte->zName); - return SQLITE_ERROR; + return 2; } - if( cannotBeFunction(pParse, pFrom) ) return SQLITE_ERROR; + if( cannotBeFunction(pParse, pFrom) ) return 2; - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTab==0 ) return WRC_Abort; - pTab->nRef = 1; + assert( pFrom->pSTab==0 ); + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ) return 2; + pCteUse = pCte->pUse; + if( pCteUse==0 ){ + pCte->pUse = pCteUse = sqlite3DbMallocZero(db, sizeof(pCteUse[0])); + if( pCteUse==0 + || sqlite3ParserAddCleanup(pParse,sqlite3DbFree,pCteUse)==0 + ){ + sqlite3DbFree(db, pTab); + return 2; + } + pCteUse->eM10d = pCte->eM10d; + } + pFrom->pSTab = pTab; + pTab->nTabRef = 1; pTab->zName = sqlite3DbStrDup(db, pCte->zName); pTab->iPKey = -1; pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; - pFrom->pSelect = sqlite3SelectDup(db, pCte->pSelect, 0); - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; - assert( pFrom->pSelect ); + sqlite3SrcItemAttachSubquery(pParse, pFrom, pCte->pSelect, 1); + if( db->mallocFailed ) return 2; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + pSel = pFrom->u4.pSubq->pSelect; + assert( pSel!=0 ); + pSel->selFlags |= SF_CopyCte; + if( pFrom->fg.isIndexedBy ){ + sqlite3ErrorMsg(pParse, "no such index: \"%s\"", pFrom->u1.zIndexedBy); + return 2; + } + assert( !pFrom->fg.isIndexedBy ); + pFrom->fg.isCte = 1; + pFrom->u2.pCteUse = pCteUse; + pCteUse->nUse++; /* Check if this is a recursive CTE. */ - pSel = pFrom->pSelect; + pRecTerm = pSel; bMayRecursive = ( pSel->op==TK_ALL || pSel->op==TK_UNION ); - if( bMayRecursive ){ + while( bMayRecursive && pRecTerm->op==pSel->op ){ int i; - SrcList *pSrc = pFrom->pSelect->pSrc; + SrcList *pSrc = pRecTerm->pSrc; + assert( pRecTerm->pPrior!=0 ); for(i=0; inSrc; i++){ - struct SrcList_item *pItem = &pSrc->a[i]; - if( pItem->zDatabase==0 - && pItem->zName!=0 + SrcItem *pItem = &pSrc->a[i]; + if( pItem->zName!=0 + && !pItem->fg.hadSchema + && ALWAYS( !pItem->fg.isSubquery ) + && (pItem->fg.fixedSchema || pItem->u4.zDatabase==0) && 0==sqlite3StrICmp(pItem->zName, pCte->zName) - ){ - pItem->pTab = pTab; + ){ + pItem->pSTab = pTab; + pTab->nTabRef++; pItem->fg.isRecursive = 1; - pTab->nRef++; - pSel->selFlags |= SF_Recursive; + if( pRecTerm->selFlags & SF_Recursive ){ + sqlite3ErrorMsg(pParse, + "multiple references to recursive table: %s", pCte->zName + ); + return 2; + } + pRecTerm->selFlags |= SF_Recursive; + if( iRecTab<0 ) iRecTab = pParse->nTab++; + pItem->iCursor = iRecTab; } } + if( (pRecTerm->selFlags & SF_Recursive)==0 ) break; + pRecTerm = pRecTerm->pPrior; } - /* Only one recursive reference is permitted. */ - if( pTab->nRef>2 ){ - sqlite3ErrorMsg( - pParse, "multiple references to recursive table: %s", pCte->zName - ); - return SQLITE_ERROR; - } - assert( pTab->nRef==1 || ((pSel->selFlags&SF_Recursive) && pTab->nRef==2 )); - pCte->zCteErr = "circular reference: %s"; pSavedWith = pParse->pWith; pParse->pWith = pWith; - sqlite3WalkSelect(pWalker, bMayRecursive ? pSel->pPrior : pSel); + if( pSel->selFlags & SF_Recursive ){ + int rc; + assert( pRecTerm!=0 ); + assert( (pRecTerm->selFlags & SF_Recursive)==0 ); + assert( pRecTerm->pNext!=0 ); + assert( (pRecTerm->pNext->selFlags & SF_Recursive)!=0 ); + assert( pRecTerm->pWith==0 ); + pRecTerm->pWith = pSel->pWith; + rc = sqlite3WalkSelect(pWalker, pRecTerm); + pRecTerm->pWith = 0; + if( rc ){ + pParse->pWith = pSavedWith; + return 2; + } + }else{ + if( sqlite3WalkSelect(pWalker, pSel) ){ + pParse->pWith = pSavedWith; + return 2; + } + } pParse->pWith = pWith; for(pLeft=pSel; pLeft->pPrior; pLeft=pLeft->pPrior); @@ -117012,7 +149574,7 @@ static int withExpand( pCte->zName, pEList->nExpr, pCte->pCols->nExpr ); pParse->pWith = pSavedWith; - return SQLITE_ERROR; + return 2; } pEList = pCte->pCols; } @@ -117028,33 +149590,97 @@ static int withExpand( } pCte->zCteErr = 0; pParse->pWith = pSavedWith; + return 1; /* Success */ } - - return SQLITE_OK; + return 0; /* No match */ } #endif #ifndef SQLITE_OMIT_CTE /* -** If the SELECT passed as the second argument has an associated WITH +** If the SELECT passed as the second argument has an associated WITH ** clause, pop it from the stack stored as part of the Parse object. ** ** This function is used as the xSelectCallback2() callback by ** sqlite3SelectExpand() when walking a SELECT tree to resolve table -** names and other FROM clause elements. +** names and other FROM clause elements. */ -static void selectPopWith(Walker *pWalker, Select *p){ +SQLITE_PRIVATE void sqlite3SelectPopWith(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; - With *pWith = findRightmost(p)->pWith; - if( pWith!=0 ){ - assert( pParse->pWith==pWith ); - pParse->pWith = pWith->pOuter; + if( OK_IF_ALWAYS_TRUE(pParse->pWith) && p->pPrior==0 ){ + With *pWith = findRightmost(p)->pWith; + if( pWith!=0 ){ + assert( pParse->pWith==pWith || pParse->nErr ); + pParse->pWith = pWith->pOuter; + } } } -#else -#define selectPopWith 0 #endif +/* +** The SrcItem structure passed as the second argument represents a +** sub-query in the FROM clause of a SELECT statement. This function +** allocates and populates the SrcItem.pTab object. If successful, +** SQLITE_OK is returned. Otherwise, if an OOM error is encountered, +** SQLITE_NOMEM. +*/ +SQLITE_PRIVATE int sqlite3ExpandSubquery(Parse *pParse, SrcItem *pFrom){ + Select *pSel; + Table *pTab; + + assert( pFrom->fg.isSubquery ); + assert( pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; + assert( pSel ); + pFrom->pSTab = pTab = sqlite3DbMallocZero(pParse->db, sizeof(Table)); + if( pTab==0 ) return SQLITE_NOMEM; + pTab->nTabRef = 1; + if( pFrom->zAlias ){ + pTab->zName = sqlite3DbStrDup(pParse->db, pFrom->zAlias); + }else{ + pTab->zName = sqlite3MPrintf(pParse->db, "%!S", pFrom); + } + while( pSel->pPrior ){ pSel = pSel->pPrior; } + sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); + pTab->iPKey = -1; + pTab->eTabType = TABTYP_VIEW; + pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); +#ifndef SQLITE_ALLOW_ROWID_IN_VIEW + /* The usual case - do not allow ROWID on a subquery */ + pTab->tabFlags |= TF_Ephemeral | TF_NoVisibleRowid; +#else + /* Legacy compatibility mode */ + pTab->tabFlags |= TF_Ephemeral | sqlite3Config.mNoVisibleRowid; +#endif + return pParse->nErr ? SQLITE_ERROR : SQLITE_OK; +} + + +/* +** Check the N SrcItem objects to the right of pBase. (N might be zero!) +** If any of those SrcItem objects have a USING clause containing zName +** then return true. +** +** If N is zero, or none of the N SrcItem objects to the right of pBase +** contains a USING clause, or if none of the USING clauses contain zName, +** then return false. +*/ +static int inAnyUsingClause( + const char *zName, /* Name we are looking for */ + SrcItem *pBase, /* The base SrcItem. Looking at pBase[1] and following */ + int N /* How many SrcItems to check */ +){ + while( N>0 ){ + N--; + pBase++; + if( pBase->fg.isUsing==0 ) continue; + if( NEVER(pBase->u3.pUsing==0) ) continue; + if( sqlite3IdListIndex(pBase->u3.pUsing, zName)>=0 ) return 1; + } + return 0; +} + + /* ** This routine is a Walker callback for "expanding" a SELECT statement. ** "Expanding" means to do the following: @@ -117062,7 +149688,7 @@ static void selectPopWith(Walker *pWalker, Select *p){ ** (1) Make sure VDBE cursor numbers have been assigned to every ** element of the FROM clause. ** -** (2) Fill in the pTabList->a[].pTab fields in the SrcList that +** (2) Fill in the pTabList->a[].pTab fields in the SrcList that ** defines FROM clause. When views appear in the FROM clause, ** fill pTabList->a[].pSelect with a copy of the SELECT statement ** that implements the view. A copy is made of the view's SELECT @@ -117081,26 +149707,39 @@ static void selectPopWith(Walker *pWalker, Select *p){ */ static int selectExpander(Walker *pWalker, Select *p){ Parse *pParse = pWalker->pParse; - int i, j, k; + int i, j, k, rc; SrcList *pTabList; ExprList *pEList; - struct SrcList_item *pFrom; + SrcItem *pFrom; sqlite3 *db = pParse->db; Expr *pE, *pRight, *pExpr; u16 selFlags = p->selFlags; + u32 elistFlags = 0; p->selFlags |= SF_Expanded; if( db->mallocFailed ){ return WRC_Abort; } - if( NEVER(p->pSrc==0) || (selFlags & SF_Expanded)!=0 ){ + assert( p->pSrc!=0 ); + if( (selFlags & SF_Expanded)!=0 ){ return WRC_Prune; } + if( pWalker->eCode ){ + /* Renumber selId because it has been copied from a view */ + p->selId = ++pParse->nSelect; + } pTabList = p->pSrc; pEList = p->pEList; - if( pWalker->xSelectCallback2==selectPopWith ){ - sqlite3WithPush(pParse, findRightmost(p)->pWith, 0); + if( pParse->pWith && (p->selFlags & SF_View) ){ + if( p->pWith==0 ){ + p->pWith = (With*)sqlite3DbMallocZero(db, sizeof(With)); + if( p->pWith==0 ){ + return WRC_Abort; + } + } + p->pWith->bView = 1; } + sqlite3WithPush(pParse, p->pWith, 0); /* Make sure cursor numbers have been assigned to all entries in ** the FROM clause of the SELECT statement. @@ -117113,69 +149752,89 @@ static int selectExpander(Walker *pWalker, Select *p){ */ for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ Table *pTab; - assert( pFrom->fg.isRecursive==0 || pFrom->pTab!=0 ); - if( pFrom->fg.isRecursive ) continue; - assert( pFrom->pTab==0 ); -#ifndef SQLITE_OMIT_CTE - if( withExpand(pWalker, pFrom) ) return WRC_Abort; - if( pFrom->pTab ) {} else -#endif + assert( pFrom->fg.isRecursive==0 || pFrom->pSTab!=0 ); + if( pFrom->pSTab ) continue; + assert( pFrom->fg.isRecursive==0 ); if( pFrom->zName==0 ){ #ifndef SQLITE_OMIT_SUBQUERY - Select *pSel = pFrom->pSelect; + Select *pSel; + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq!=0 ); + pSel = pFrom->u4.pSubq->pSelect; /* A sub-query in the FROM clause of a SELECT */ assert( pSel!=0 ); - assert( pFrom->pTab==0 ); + assert( pFrom->pSTab==0 ); if( sqlite3WalkSelect(pWalker, pSel) ) return WRC_Abort; - pFrom->pTab = pTab = sqlite3DbMallocZero(db, sizeof(Table)); - if( pTab==0 ) return WRC_Abort; - pTab->nRef = 1; - pTab->zName = sqlite3MPrintf(db, "sqlite_sq_%p", (void*)pTab); - while( pSel->pPrior ){ pSel = pSel->pPrior; } - sqlite3ColumnsFromExprList(pParse, pSel->pEList,&pTab->nCol,&pTab->aCol); - pTab->iPKey = -1; - pTab->nRowLogEst = 200; assert( 200==sqlite3LogEst(1048576) ); - pTab->tabFlags |= TF_Ephemeral; + if( sqlite3ExpandSubquery(pParse, pFrom) ) return WRC_Abort; +#endif +#ifndef SQLITE_OMIT_CTE + }else if( (rc = resolveFromTermToCte(pParse, pWalker, pFrom))!=0 ){ + if( rc>1 ) return WRC_Abort; + pTab = pFrom->pSTab; + assert( pTab!=0 ); #endif }else{ /* An ordinary table or view name in the FROM clause */ - assert( pFrom->pTab==0 ); - pFrom->pTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); + assert( pFrom->pSTab==0 ); + pFrom->pSTab = pTab = sqlite3LocateTableItem(pParse, 0, pFrom); if( pTab==0 ) return WRC_Abort; - if( pTab->nRef==0xffff ){ + if( pTab->nTabRef>=0xffff ){ sqlite3ErrorMsg(pParse, "too many references to \"%s\": max 65535", pTab->zName); - pFrom->pTab = 0; + pFrom->pSTab = 0; return WRC_Abort; } - pTab->nRef++; + pTab->nTabRef++; if( !IsVirtual(pTab) && cannotBeFunction(pParse, pFrom) ){ return WRC_Abort; } -#if !defined(SQLITE_OMIT_VIEW) || !defined (SQLITE_OMIT_VIRTUALTABLE) - if( IsVirtual(pTab) || pTab->pSelect ){ +#if !defined(SQLITE_OMIT_VIEW) || !defined(SQLITE_OMIT_VIRTUALTABLE) + if( !IsOrdinaryTable(pTab) ){ i16 nCol; + u8 eCodeOrig = pWalker->eCode; if( sqlite3ViewGetColumnNames(pParse, pTab) ) return WRC_Abort; - assert( pFrom->pSelect==0 ); - pFrom->pSelect = sqlite3SelectDup(db, pTab->pSelect, 0); - sqlite3SelectSetName(pFrom->pSelect, pTab->zName); + assert( pFrom->fg.isSubquery==0 ); + if( IsView(pTab) ){ + if( (db->flags & SQLITE_EnableView)==0 + && pTab->pSchema!=db->aDb[1].pSchema + ){ + sqlite3ErrorMsg(pParse, "access to view \"%s\" prohibited", + pTab->zName); + } + sqlite3SrcItemAttachSubquery(pParse, pFrom, pTab->u.view.pSelect, 1); + } +#ifndef SQLITE_OMIT_VIRTUALTABLE + else if( ALWAYS(IsVirtual(pTab)) + && pFrom->fg.fromDDL + && ALWAYS(pTab->u.vtab.p!=0) + && pTab->u.vtab.p->eVtabRisk > ((db->flags & SQLITE_TrustedSchema)!=0) + ){ + sqlite3ErrorMsg(pParse, "unsafe use of virtual table \"%s\"", + pTab->zName); + } + assert( SQLITE_VTABRISK_Normal==1 && SQLITE_VTABRISK_High==2 ); +#endif nCol = pTab->nCol; pTab->nCol = -1; - sqlite3WalkSelect(pWalker, pFrom->pSelect); + pWalker->eCode = 1; /* Turn on Select.selId renumbering */ + if( pFrom->fg.isSubquery ){ + sqlite3WalkSelect(pWalker, pFrom->u4.pSubq->pSelect); + } + pWalker->eCode = eCodeOrig; pTab->nCol = nCol; } #endif } /* Locate the index named by the INDEXED BY clause, if any. */ - if( sqlite3IndexedByLookup(pParse, pFrom) ){ + if( pFrom->fg.isIndexedBy && sqlite3IndexedByLookup(pParse, pFrom) ){ return WRC_Abort; } } /* Process NATURAL keywords, and ON and USING clauses of joins. */ - if( db->mallocFailed || sqliteProcessJoin(pParse, p) ){ + assert( db->mallocFailed==0 || pParse->nErr!=0 ); + if( pParse->nErr || sqlite3ProcessJoin(pParse, p) ){ return WRC_Abort; } @@ -117196,6 +149855,7 @@ static int selectExpander(Walker *pWalker, Select *p){ assert( pE->op!=TK_DOT || pE->pRight!=0 ); assert( pE->op!=TK_DOT || (pE->pLeft!=0 && pE->pLeft->op==TK_ID) ); if( pE->op==TK_DOT && pE->pRight->op==TK_ASTERISK ) break; + elistFlags |= pE->flags; } if( knExpr ){ /* @@ -117211,6 +149871,7 @@ static int selectExpander(Walker *pWalker, Select *p){ for(k=0; knExpr; k++){ pE = a[k].pExpr; + elistFlags |= pE->flags; pRight = pE->pRight; assert( pE->op!=TK_DOT || pRight!=0 ); if( pE->op!=TK_ASTERISK @@ -117220,10 +149881,9 @@ static int selectExpander(Walker *pWalker, Select *p){ */ pNew = sqlite3ExprListAppend(pParse, pNew, a[k].pExpr); if( pNew ){ - pNew->a[pNew->nExpr-1].zName = a[k].zName; - pNew->a[pNew->nExpr-1].zSpan = a[k].zSpan; - a[k].zName = 0; - a[k].zSpan = 0; + pNew->a[pNew->nExpr-1].zEName = a[k].zEName; + pNew->a[pNew->nExpr-1].fg.eEName = a[k].fg.eEName; + a[k].zEName = 0; } a[k].pExpr = 0; }else{ @@ -117231,101 +149891,177 @@ static int selectExpander(Walker *pWalker, Select *p){ ** expanded. */ int tableSeen = 0; /* Set to 1 when TABLE matches */ char *zTName = 0; /* text of name of TABLE */ + int iErrOfst; if( pE->op==TK_DOT ){ + assert( (selFlags & SF_NestedFrom)==0 ); assert( pE->pLeft!=0 ); assert( !ExprHasProperty(pE->pLeft, EP_IntValue) ); zTName = pE->pLeft->u.zToken; + assert( ExprUseWOfst(pE->pLeft) ); + iErrOfst = pE->pRight->w.iOfst; + }else{ + assert( ExprUseWOfst(pE) ); + iErrOfst = pE->w.iOfst; } for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; - Select *pSub = pFrom->pSelect; - char *zTabName = pFrom->zAlias; - const char *zSchemaName = 0; - int iDb; - if( zTabName==0 ){ + int nAdd; /* Number of cols including rowid */ + Table *pTab = pFrom->pSTab; /* Table for this data source */ + ExprList *pNestedFrom; /* Result-set of a nested FROM clause */ + char *zTabName; /* AS name for this data source */ + const char *zSchemaName = 0; /* Schema name for this data source */ + int iDb; /* Schema index for this data src */ + IdList *pUsing; /* USING clause for pFrom[1] */ + + if( (zTabName = pFrom->zAlias)==0 ){ zTabName = pTab->zName; } if( db->mallocFailed ) break; - if( pSub==0 || (pSub->selFlags & SF_NestedFrom)==0 ){ - pSub = 0; + assert( (int)pFrom->fg.isNestedFrom == IsNestedFrom(pFrom) ); + if( pFrom->fg.isNestedFrom ){ + assert( pFrom->fg.isSubquery && pFrom->u4.pSubq ); + assert( pFrom->u4.pSubq->pSelect!=0 ); + pNestedFrom = pFrom->u4.pSubq->pSelect->pEList; + assert( pNestedFrom!=0 ); + assert( pNestedFrom->nExpr==pTab->nCol ); + assert( VisibleRowid(pTab)==0 || ViewCanHaveRowid ); + }else{ if( zTName && sqlite3StrICmp(zTName, zTabName)!=0 ){ continue; } + pNestedFrom = 0; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - zSchemaName = iDb>=0 ? db->aDb[iDb].zName : "*"; + zSchemaName = iDb>=0 ? db->aDb[iDb].zDbSName : "*"; } - for(j=0; jnCol; j++){ - char *zName = pTab->aCol[j].zName; - char *zColname; /* The computed column name */ - char *zToFree; /* Malloced string that needs to be freed */ - Token sColname; /* Computed column name as a token */ - - assert( zName ); - if( zTName && pSub - && sqlite3MatchSpanName(pSub->pEList->a[j].zSpan, 0, zTName, 0)==0 - ){ - continue; + if( i+1nSrc + && pFrom[1].fg.isUsing + && (selFlags & SF_NestedFrom)!=0 + ){ + int ii; + pUsing = pFrom[1].u3.pUsing; + for(ii=0; iinId; ii++){ + const char *zUName = pUsing->a[ii].zName; + pRight = sqlite3Expr(db, TK_ID, zUName); + sqlite3ExprSetErrorOffset(pRight, iErrOfst); + pNew = sqlite3ExprListAppend(pParse, pNew, pRight); + if( pNew ){ + struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; + assert( pX->zEName==0 ); + pX->zEName = sqlite3MPrintf(db,"..%s", zUName); + pX->fg.eEName = ENAME_TAB; + pX->fg.bUsingTerm = 1; + } } + }else{ + pUsing = 0; + } - /* If a column is marked as 'hidden', omit it from the expanded - ** result-set list unless the SELECT has the SF_IncludeHidden - ** bit set. - */ - if( (p->selFlags & SF_IncludeHidden)==0 - && IsHiddenColumn(&pTab->aCol[j]) - ){ - continue; - } - tableSeen = 1; + nAdd = pTab->nCol; + if( VisibleRowid(pTab) && (selFlags & SF_NestedFrom)!=0 ) nAdd++; + for(j=0; j0 && zTName==0 ){ - if( (pFrom->fg.jointype & JT_NATURAL)!=0 - && tableAndColumnIndex(pTabList, i, zName, 0, 0) - ){ - /* In a NATURAL join, omit the join columns from the - ** table to the right of the join */ + if( j==pTab->nCol ){ + zName = sqlite3RowidAlias(pTab); + if( zName==0 ) continue; + }else{ + zName = pTab->aCol[j].zCnName; + + /* If pTab is actually an SF_NestedFrom sub-select, do not + ** expand any ENAME_ROWID columns. */ + if( pNestedFrom && pNestedFrom->a[j].fg.eEName==ENAME_ROWID ){ continue; } - if( sqlite3IdListIndex(pFrom->pUsing, zName)>=0 ){ + + if( zTName + && pNestedFrom + && sqlite3MatchEName(&pNestedFrom->a[j], 0, zTName, 0, 0)==0 + ){ + continue; + } + + /* If a column is marked as 'hidden', omit it from the expanded + ** result-set list unless the SELECT has the SF_IncludeHidden + ** bit set. + */ + if( (p->selFlags & SF_IncludeHidden)==0 + && IsHiddenColumn(&pTab->aCol[j]) + ){ + continue; + } + if( (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)!=0 + && zTName==0 + && (selFlags & (SF_NestedFrom))==0 + ){ + continue; + } + } + assert( zName ); + tableSeen = 1; + + if( i>0 && zTName==0 && (selFlags & SF_NestedFrom)==0 ){ + if( pFrom->fg.isUsing + && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0 + ){ /* In a join with a USING clause, omit columns in the ** using clause from the table on the right. */ continue; } } pRight = sqlite3Expr(db, TK_ID, zName); - zColname = zName; - zToFree = 0; - if( longNames || pTabList->nSrc>1 ){ + if( (pTabList->nSrc>1 + && ( (pFrom->fg.jointype & JT_LTORJ)==0 + || (selFlags & SF_NestedFrom)!=0 + || !inAnyUsingClause(zName,pFrom,pTabList->nSrc-i-1) + ) + ) + || IN_RENAME_OBJECT + ){ Expr *pLeft; pLeft = sqlite3Expr(db, TK_ID, zTabName); - pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + if( IN_RENAME_OBJECT && pE->pLeft ){ + sqlite3RenameTokenRemap(pParse, pLeft, pE->pLeft); + } if( zSchemaName ){ pLeft = sqlite3Expr(db, TK_ID, zSchemaName); - pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr, 0); - } - if( longNames ){ - zColname = sqlite3MPrintf(db, "%s.%s", zTabName, zName); - zToFree = zColname; + pExpr = sqlite3PExpr(pParse, TK_DOT, pLeft, pExpr); } }else{ pExpr = pRight; } + sqlite3ExprSetErrorOffset(pExpr, iErrOfst); pNew = sqlite3ExprListAppend(pParse, pNew, pExpr); - sqlite3TokenInit(&sColname, zColname); - sqlite3ExprListSetName(pParse, pNew, &sColname, 0); - if( pNew && (p->selFlags & SF_NestedFrom)!=0 ){ - struct ExprList_item *pX = &pNew->a[pNew->nExpr-1]; - if( pSub ){ - pX->zSpan = sqlite3DbStrDup(db, pSub->pEList->a[j].zSpan); - testcase( pX->zSpan==0 ); - }else{ - pX->zSpan = sqlite3MPrintf(db, "%s.%s.%s", - zSchemaName, zTabName, zColname); - testcase( pX->zSpan==0 ); - } - pX->bSpanIsTab = 1; + if( pNew==0 ){ + break; /* OOM */ + } + pX = &pNew->a[pNew->nExpr-1]; + assert( pX->zEName==0 ); + if( (selFlags & SF_NestedFrom)!=0 && !IN_RENAME_OBJECT ){ + if( pNestedFrom && (!ViewCanHaveRowid || jnExpr) ){ + assert( jnExpr ); + pX->zEName = sqlite3DbStrDup(db, pNestedFrom->a[j].zEName); + testcase( pX->zEName==0 ); + }else{ + pX->zEName = sqlite3MPrintf(db, "%s.%s.%s", + zSchemaName, zTabName, zName); + testcase( pX->zEName==0 ); + } + pX->fg.eEName = (j==pTab->nCol ? ENAME_ROWID : ENAME_TAB); + if( (pFrom->fg.isUsing + && sqlite3IdListIndex(pFrom->u3.pUsing, zName)>=0) + || (pUsing && sqlite3IdListIndex(pUsing, zName)>=0) + || (jnCol && (pTab->aCol[j].colFlags & COLFLAG_NOEXPAND)) + ){ + pX->fg.bNoExpand = 1; + } + }else if( longNames ){ + pX->zEName = sqlite3MPrintf(db, "%s.%s", zTabName, zName); + pX->fg.eEName = ENAME_NAME; + }else{ + pX->zEName = sqlite3DbStrDup(db, zName); + pX->fg.eEName = ENAME_NAME; } - sqlite3DbFree(db, zToFree); } } if( !tableSeen ){ @@ -117340,29 +150076,34 @@ static int selectExpander(Walker *pWalker, Select *p){ sqlite3ExprListDelete(db, pEList); p->pEList = pNew; } -#if SQLITE_MAX_COLUMN - if( p->pEList && p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ - sqlite3ErrorMsg(pParse, "too many columns in result set"); - return WRC_Abort; + if( p->pEList ){ + if( p->pEList->nExpr>db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns in result set"); + return WRC_Abort; + } + if( (elistFlags & (EP_HasFunc|EP_Subquery))!=0 ){ + p->selFlags |= SF_ComplexResult; + } + } +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x8 ){ + TREETRACE(0x8,pParse,p,("After result-set wildcard expansion:\n")); + sqlite3TreeViewSelect(0, p, 0); } #endif return WRC_Continue; } +#if SQLITE_DEBUG /* -** No-op routine for the parse-tree walker. -** -** When this routine is the Walker.xExprCallback then expression trees -** are walked without any actions being taken at each node. Presumably, -** when this routine is used for Walker.xExprCallback then -** Walker.xSelectCallback is set to do something useful for every -** subquery in the parser tree. +** Always assert. This xSelectCallback2 implementation proves that the +** xSelectCallback2 is never invoked. */ -SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ +SQLITE_PRIVATE void sqlite3SelectWalkAssert2(Walker *NotUsed, Select *NotUsed2){ UNUSED_PARAMETER2(NotUsed, NotUsed2); - return WRC_Continue; + assert( 0 ); } - +#endif /* ** This routine "expands" a SELECT statement and all of its subqueries. ** For additional information on what it means to "expand" a SELECT @@ -117378,17 +150119,16 @@ SQLITE_PRIVATE int sqlite3ExprWalkNoop(Walker *NotUsed, Expr *NotUsed2){ */ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ Walker w; - memset(&w, 0, sizeof(w)); w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; - if( pParse->hasCompound ){ + if( OK_IF_ALWAYS_TRUE(pParse->hasCompound) ){ w.xSelectCallback = convertCompoundSelectToSubquery; + w.xSelectCallback2 = 0; sqlite3WalkSelect(&w, pSelect); } w.xSelectCallback = selectExpander; - if( (pSelect->selFlags & SF_MultiValue)==0 ){ - w.xSelectCallback2 = selectPopWith; - } + w.xSelectCallback2 = sqlite3SelectPopWith; + w.eCode = 0; sqlite3WalkSelect(&w, pSelect); } @@ -117398,36 +150138,33 @@ static void sqlite3SelectExpand(Parse *pParse, Select *pSelect){ ** This is a Walker.xSelectCallback callback for the sqlite3SelectTypeInfo() ** interface. ** -** For each FROM-clause subquery, add Column.zType and Column.zColl -** information to the Table structure that represents the result set -** of that subquery. +** For each FROM-clause subquery, add Column.zType, Column.zColl, and +** Column.affinity information to the Table structure that represents +** the result set of that subquery. ** ** The Table structure that represents the result set was constructed -** by selectExpander() but the type and collation information was omitted -** at that point because identifiers had not yet been resolved. This -** routine is called after identifier resolution. +** by selectExpander() but the type and collation and affinity information +** was omitted at that point because identifiers had not yet been resolved. +** This routine is called after identifier resolution. */ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ Parse *pParse; int i; SrcList *pTabList; - struct SrcList_item *pFrom; + SrcItem *pFrom; - assert( p->selFlags & SF_Resolved ); - assert( (p->selFlags & SF_HasTypeInfo)==0 ); + if( p->selFlags & SF_HasTypeInfo ) return; p->selFlags |= SF_HasTypeInfo; pParse = pWalker->pParse; + assert( (p->selFlags & SF_Resolved) ); pTabList = p->pSrc; for(i=0, pFrom=pTabList->a; inSrc; i++, pFrom++){ - Table *pTab = pFrom->pTab; + Table *pTab = pFrom->pSTab; assert( pTab!=0 ); - if( (pTab->tabFlags & TF_Ephemeral)!=0 ){ + if( (pTab->tabFlags & TF_Ephemeral)!=0 && pFrom->fg.isSubquery ){ /* A sub-query in the FROM clause of a SELECT */ - Select *pSel = pFrom->pSelect; - if( pSel ){ - while( pSel->pPrior ) pSel = pSel->pPrior; - sqlite3SelectAddColumnTypeAndCollation(pParse, pTab, pSel); - } + Select *pSel = pFrom->u4.pSubq->pSelect; + sqlite3SubqueryColumnTypes(pParse, pTab, pSel, SQLITE_AFF_NONE); } } } @@ -117444,7 +150181,7 @@ static void selectAddSubqueryTypeInfo(Walker *pWalker, Select *p){ static void sqlite3SelectAddTypeInfo(Parse *pParse, Select *pSelect){ #ifndef SQLITE_OMIT_SUBQUERY Walker w; - memset(&w, 0, sizeof(w)); + w.xSelectCallback = sqlite3SelectWalkNoop; w.xSelectCallback2 = selectAddSubqueryTypeInfo; w.xExprCallback = sqlite3ExprWalkNoop; w.pParse = pParse; @@ -117470,18 +150207,197 @@ SQLITE_PRIVATE void sqlite3SelectPrep( Select *p, /* The SELECT statement being coded. */ NameContext *pOuterNC /* Name context for container */ ){ - sqlite3 *db; - if( NEVER(p==0) ) return; - db = pParse->db; - if( db->mallocFailed ) return; + assert( p!=0 || pParse->db->mallocFailed ); + assert( pParse->db->pParse==pParse ); + if( pParse->db->mallocFailed ) return; if( p->selFlags & SF_HasTypeInfo ) return; sqlite3SelectExpand(pParse, p); - if( pParse->nErr || db->mallocFailed ) return; + if( pParse->nErr ) return; sqlite3ResolveSelectNames(pParse, p, pOuterNC); - if( pParse->nErr || db->mallocFailed ) return; + if( pParse->nErr ) return; sqlite3SelectAddTypeInfo(pParse, p); } +#if TREETRACE_ENABLED +/* +** Display all information about an AggInfo object +*/ +static void printAggInfo(AggInfo *pAggInfo){ + int ii; + sqlite3DebugPrintf("AggInfo %d/%p:\n", + pAggInfo->selId, pAggInfo); + for(ii=0; iinColumn; ii++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[ii]; + sqlite3DebugPrintf( + "agg-column[%d] pTab=%s iTable=%d iColumn=%d iMem=%d" + " iSorterColumn=%d %s\n", + ii, pCol->pTab ? pCol->pTab->zName : "NULL", + pCol->iTable, pCol->iColumn, pAggInfo->iFirstReg+ii, + pCol->iSorterColumn, + ii>=pAggInfo->nAccumulator ? "" : " Accumulator"); + sqlite3TreeViewExpr(0, pAggInfo->aCol[ii].pCExpr, 0); + } + for(ii=0; iinFunc; ii++){ + sqlite3DebugPrintf("agg-func[%d]: iMem=%d\n", + ii, pAggInfo->iFirstReg+pAggInfo->nColumn+ii); + sqlite3TreeViewExpr(0, pAggInfo->aFunc[ii].pFExpr, 0); + } +} +#endif /* TREETRACE_ENABLED */ + +/* +** Analyze the arguments to aggregate functions. Create new pAggInfo->aCol[] +** entries for columns that are arguments to aggregate functions but which +** are not otherwise used. +** +** The aCol[] entries in AggInfo prior to nAccumulator are columns that +** are referenced outside of aggregate functions. These might be columns +** that are part of the GROUP by clause, for example. Other database engines +** would throw an error if there is a column reference that is not in the +** GROUP BY clause and that is not part of an aggregate function argument. +** But SQLite allows this. +** +** The aCol[] entries beginning with the aCol[nAccumulator] and following +** are column references that are used exclusively as arguments to +** aggregate functions. This routine is responsible for computing +** (or recomputing) those aCol[] entries. +*/ +static void analyzeAggFuncArgs( + AggInfo *pAggInfo, + NameContext *pNC +){ + int i; + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pNC->ncFlags |= NC_InAggFunc; + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr->op==TK_FUNCTION || pExpr->op==TK_AGG_FUNCTION ); + assert( ExprUseXList(pExpr) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->x.pList); + if( pExpr->pLeft ){ + assert( pExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pExpr->pLeft) ); + sqlite3ExprAnalyzeAggList(pNC, pExpr->pLeft->x.pList); + } +#ifndef SQLITE_OMIT_WINDOWFUNC + assert( !IsWindowFunc(pExpr) ); + if( ExprHasProperty(pExpr, EP_WinFunc) ){ + sqlite3ExprAnalyzeAggregates(pNC, pExpr->y.pWin->pFilter); + } +#endif + } + pNC->ncFlags &= ~NC_InAggFunc; +} + +/* +** An index on expressions is being used in the inner loop of an +** aggregate query with a GROUP BY clause. This routine attempts +** to adjust the AggInfo object to take advantage of index and to +** perhaps use the index as a covering index. +** +*/ +static void optimizeAggregateUseOfIndexedExpr( + Parse *pParse, /* Parsing context */ + Select *pSelect, /* The SELECT statement being processed */ + AggInfo *pAggInfo, /* The aggregate info */ + NameContext *pNC /* Name context used to resolve agg-func args */ +){ + assert( pAggInfo->iFirstReg==0 ); + assert( pSelect!=0 ); + assert( pSelect->pGroupBy!=0 ); + pAggInfo->nColumn = pAggInfo->nAccumulator; + if( ALWAYS(pAggInfo->nSortingColumn>0) ){ + int mx = pSelect->pGroupBy->nExpr - 1; + int j, k; + for(j=0; jnColumn; j++){ + k = pAggInfo->aCol[j].iSorterColumn; + if( k>mx ) mx = k; + } + pAggInfo->nSortingColumn = mx+1; + } + analyzeAggFuncArgs(pAggInfo, pNC); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + IndexedExpr *pIEpr; + TREETRACE(0x20, pParse, pSelect, + ("AggInfo (possibly) adjusted for Indexed Exprs\n")); + sqlite3TreeViewSelect(0, pSelect, 0); + for(pIEpr=pParse->pIdxEpr; pIEpr; pIEpr=pIEpr->pIENext){ + printf("data-cursor=%d index={%d,%d}\n", + pIEpr->iDataCur, pIEpr->iIdxCur, pIEpr->iIdxCol); + sqlite3TreeViewExpr(0, pIEpr->pExpr, 0); + } + printAggInfo(pAggInfo); + } +#else + UNUSED_PARAMETER(pSelect); + UNUSED_PARAMETER(pParse); +#endif +} + +/* +** Walker callback for aggregateConvertIndexedExprRefToColumn(). +*/ +static int aggregateIdxEprRefToColCallback(Walker *pWalker, Expr *pExpr){ + AggInfo *pAggInfo; + struct AggInfo_col *pCol; + UNUSED_PARAMETER(pWalker); + if( pExpr->pAggInfo==0 ) return WRC_Continue; + if( pExpr->op==TK_AGG_COLUMN ) return WRC_Continue; + if( pExpr->op==TK_AGG_FUNCTION ) return WRC_Continue; + if( pExpr->op==TK_IF_NULL_ROW ) return WRC_Continue; + pAggInfo = pExpr->pAggInfo; + if( NEVER(pExpr->iAgg>=pAggInfo->nColumn) ) return WRC_Continue; + assert( pExpr->iAgg>=0 ); + pCol = &pAggInfo->aCol[pExpr->iAgg]; + pExpr->op = TK_AGG_COLUMN; + pExpr->iTable = pCol->iTable; + pExpr->iColumn = pCol->iColumn; + ExprClearProperty(pExpr, EP_Skip|EP_Collate|EP_Unlikely); + return WRC_Prune; +} + +/* +** Convert every pAggInfo->aFunc[].pExpr such that any node within +** those expressions that has pAppInfo set is changed into a TK_AGG_COLUMN +** opcode. +*/ +static void aggregateConvertIndexedExprRefToColumn(AggInfo *pAggInfo){ + int i; + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = aggregateIdxEprRefToColCallback; + for(i=0; inFunc; i++){ + sqlite3WalkExpr(&w, pAggInfo->aFunc[i].pFExpr); + } +} + + +/* +** Allocate a block of registers so that there is one register for each +** pAggInfo->aCol[] and pAggInfo->aFunc[] entry in pAggInfo. The first +** register in this block is stored in pAggInfo->iFirstReg. +** +** This routine may only be called once for each AggInfo object. Prior +** to calling this routine: +** +** * The aCol[] and aFunc[] arrays may be modified +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may not be used +** +** After calling this routine: +** +** * The aCol[] and aFunc[] arrays are fixed +** * The AggInfoColumnReg() and AggInfoFuncReg() macros may be used +** +*/ +static void assignAggregateRegisters(Parse *pParse, AggInfo *pAggInfo){ + assert( pAggInfo!=0 ); + assert( pAggInfo->iFirstReg==0 ); + pAggInfo->iFirstReg = pParse->nMem + 1; + pParse->nMem += pAggInfo->nColumn + pAggInfo->nFunc; +} + /* ** Reset the aggregate accumulator. ** @@ -117495,35 +150411,59 @@ static void resetAccumulator(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pFunc; int nReg = pAggInfo->nFunc + pAggInfo->nColumn; + assert( pAggInfo->iFirstReg>0 ); + assert( pParse->db->pParse==pParse ); + assert( pParse->db->mallocFailed==0 || pParse->nErr!=0 ); if( nReg==0 ) return; -#ifdef SQLITE_DEBUG - /* Verify that all AggInfo registers are within the range specified by - ** AggInfo.mnReg..AggInfo.mxReg */ - assert( nReg==pAggInfo->mxReg-pAggInfo->mnReg+1 ); - for(i=0; inColumn; i++){ - assert( pAggInfo->aCol[i].iMem>=pAggInfo->mnReg - && pAggInfo->aCol[i].iMem<=pAggInfo->mxReg ); - } - for(i=0; inFunc; i++){ - assert( pAggInfo->aFunc[i].iMem>=pAggInfo->mnReg - && pAggInfo->aFunc[i].iMem<=pAggInfo->mxReg ); - } -#endif - sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->mnReg, pAggInfo->mxReg); + if( pParse->nErr ) return; + sqlite3VdbeAddOp3(v, OP_Null, 0, pAggInfo->iFirstReg, + pAggInfo->iFirstReg+nReg-1); for(pFunc=pAggInfo->aFunc, i=0; inFunc; i++, pFunc++){ if( pFunc->iDistinct>=0 ){ - Expr *pE = pFunc->pExpr; - assert( !ExprHasProperty(pE, EP_xIsSelect) ); + Expr *pE = pFunc->pFExpr; + assert( ExprUseXList(pE) ); if( pE->x.pList==0 || pE->x.pList->nExpr!=1 ){ sqlite3ErrorMsg(pParse, "DISTINCT aggregates must have exactly one " "argument"); pFunc->iDistinct = -1; }else{ - KeyInfo *pKeyInfo = keyInfoFromExprList(pParse, pE->x.pList, 0, 0); - sqlite3VdbeAddOp4(v, OP_OpenEphemeral, pFunc->iDistinct, 0, 0, - (char*)pKeyInfo, P4_KEYINFO); + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pE->x.pList,0,0); + pFunc->iDistAddr = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iDistinct, 0, 0, (char*)pKeyInfo, P4_KEYINFO); + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(DISTINCT)", + pFunc->pFunc->zName)); } } + if( pFunc->iOBTab>=0 ){ + ExprList *pOBList; + KeyInfo *pKeyInfo; + int nExtra = 0; + assert( pFunc->pFExpr->pLeft!=0 ); + assert( pFunc->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pFunc->pFExpr->pLeft) ); + assert( pFunc->pFunc!=0 ); + pOBList = pFunc->pFExpr->pLeft->x.pList; + if( !pFunc->bOBUnique ){ + nExtra++; /* One extra column for the OP_Sequence */ + } + if( pFunc->bOBPayload ){ + /* extra columns for the function arguments */ + assert( ExprUseXList(pFunc->pFExpr) ); + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + if( pFunc->bUseSubtype ){ + nExtra += pFunc->pFExpr->x.pList->nExpr; + } + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOBList, 0, nExtra); + if( !pFunc->bOBUnique && pParse->nErr==0 ){ + pKeyInfo->nKeyField++; + } + sqlite3VdbeAddOp4(v, OP_OpenEphemeral, + pFunc->iOBTab, pOBList->nExpr+nExtra, 0, + (char*)pKeyInfo, P4_KEYINFO); + ExplainQueryPlan((pParse, 0, "USE TEMP B-TREE FOR %s(ORDER BY)", + pFunc->pFunc->zName)); + } } } @@ -117536,18 +150476,82 @@ static void finalizeAggFunctions(Parse *pParse, AggInfo *pAggInfo){ int i; struct AggInfo_func *pF; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ - ExprList *pList = pF->pExpr->x.pList; - assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); - sqlite3VdbeAddOp4(v, OP_AggFinal, pF->iMem, pList ? pList->nExpr : 0, 0, - (void*)pF->pFunc, P4_FUNCDEF); + ExprList *pList; + assert( ExprUseXList(pF->pFExpr) ); + if( pParse->nErr ) return; + pList = pF->pFExpr->x.pList; + if( pF->iOBTab>=0 ){ + /* For an ORDER BY aggregate, calls to OP_AggStep were deferred. Inputs + ** were stored in emphermal table pF->iOBTab. Here, we extract those + ** inputs (in ORDER BY order) and make all calls to OP_AggStep + ** before doing the OP_AggFinal call. */ + int iTop; /* Start of loop for extracting columns */ + int nArg; /* Number of columns to extract */ + int nKey; /* Key columns to be skipped */ + int regAgg; /* Extract into this array */ + int j; /* Loop counter */ + + assert( pF->pFunc!=0 ); + nArg = pList->nExpr; + regAgg = sqlite3GetTempRange(pParse, nArg); + + if( pF->bOBPayload==0 ){ + nKey = 0; + }else{ + assert( pF->pFExpr->pLeft!=0 ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + assert( pF->pFExpr->pLeft->x.pList!=0 ); + nKey = pF->pFExpr->pLeft->x.pList->nExpr; + if( ALWAYS(!pF->bOBUnique) ) nKey++; + } + iTop = sqlite3VdbeAddOp1(v, OP_Rewind, pF->iOBTab); VdbeCoverage(v); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, nKey+j, regAgg+j); + } + if( pF->bUseSubtype ){ + int regSubtype = sqlite3GetTempReg(pParse); + int iBaseCol = nKey + nArg + (pF->bOBPayload==0 && pF->bOBUnique==0); + for(j=nArg-1; j>=0; j--){ + sqlite3VdbeAddOp3(v, OP_Column, pF->iOBTab, iBaseCol+j, regSubtype); + sqlite3VdbeAddOp2(v, OP_SetSubtype, regSubtype, regAgg+j); + } + sqlite3ReleaseTempReg(pParse, regSubtype); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3VdbeAddOp2(v, OP_Next, pF->iOBTab, iTop+1); VdbeCoverage(v); + sqlite3VdbeJumpHere(v, iTop); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); + } + sqlite3VdbeAddOp2(v, OP_AggFinal, AggInfoFuncReg(pAggInfo,i), + pList ? pList->nExpr : 0); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); } } /* -** Update the accumulator memory cells for an aggregate based on -** the current cursor position. +** Generate code that will update the accumulator memory cells for an +** aggregate based on the current cursor position. +** +** If regAcc is non-zero and there are no min() or max() aggregates +** in pAggInfo, then only populate the pAggInfo->nAccumulator accumulator +** registers if register regAcc contains 0. The caller will take care +** of setting and clearing regAcc. +** +** For an ORDER BY aggregate, the actual accumulator memory cell update +** is deferred until after all input rows have been received, so that they +** can be run in the requested order. In that case, instead of invoking +** OP_AggStep to update the accumulator, just add the arguments that would +** have been passed into OP_AggStep into the sorting ephemeral table +** (along with the appropriate sort key). */ -static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ +static void updateAccumulator( + Parse *pParse, + int regAcc, + AggInfo *pAggInfo, + int eDistinctType +){ Vdbe *v = pParse->pVdbe; int i; int regHit = 0; @@ -117555,73 +150559,151 @@ static void updateAccumulator(Parse *pParse, AggInfo *pAggInfo){ struct AggInfo_func *pF; struct AggInfo_col *pC; + assert( pAggInfo->iFirstReg>0 ); + if( pParse->nErr ) return; pAggInfo->directMode = 1; for(i=0, pF=pAggInfo->aFunc; inFunc; i++, pF++){ int nArg; int addrNext = 0; int regAgg; - ExprList *pList = pF->pExpr->x.pList; - assert( !ExprHasProperty(pF->pExpr, EP_xIsSelect) ); - if( pList ){ + int regAggSz = 0; + int regDistinct = 0; + ExprList *pList; + assert( ExprUseXList(pF->pFExpr) ); + assert( !IsWindowFunc(pF->pFExpr) ); + assert( pF->pFunc!=0 ); + pList = pF->pFExpr->x.pList; + if( ExprHasProperty(pF->pFExpr, EP_WinFunc) ){ + Expr *pFilter = pF->pFExpr->y.pWin->pFilter; + if( pAggInfo->nAccumulator + && (pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL) + && regAcc + ){ + /* If regAcc==0, there there exists some min() or max() function + ** without a FILTER clause that will ensure the magnet registers + ** are populated. */ + if( regHit==0 ) regHit = ++pParse->nMem; + /* If this is the first row of the group (regAcc contains 0), clear the + ** "magnet" register regHit so that the accumulator registers + ** are populated if the FILTER clause jumps over the the + ** invocation of min() or max() altogether. Or, if this is not + ** the first row (regAcc contains 1), set the magnet register so that + ** the accumulators are not populated unless the min()/max() is invoked + ** and indicates that they should be. */ + sqlite3VdbeAddOp2(v, OP_Copy, regAcc, regHit); + } + addrNext = sqlite3VdbeMakeLabel(pParse); + sqlite3ExprIfFalse(pParse, pFilter, addrNext, SQLITE_JUMPIFNULL); + } + if( pF->iOBTab>=0 ){ + /* Instead of invoking AggStep, we must push the arguments that would + ** have been passed to AggStep onto the sorting table. */ + int jj; /* Registered used so far in building the record */ + ExprList *pOBList; /* The ORDER BY clause */ + assert( pList!=0 ); + nArg = pList->nExpr; + assert( nArg>0 ); + assert( pF->pFExpr->pLeft!=0 ); + assert( pF->pFExpr->pLeft->op==TK_ORDER ); + assert( ExprUseXList(pF->pFExpr->pLeft) ); + pOBList = pF->pFExpr->pLeft->x.pList; + assert( pOBList!=0 ); + assert( pOBList->nExpr>0 ); + regAggSz = pOBList->nExpr; + if( !pF->bOBUnique ){ + regAggSz++; /* One register for OP_Sequence */ + } + if( pF->bOBPayload ){ + regAggSz += nArg; + } + if( pF->bUseSubtype ){ + regAggSz += nArg; + } + regAggSz++; /* One extra register to hold result of MakeRecord */ + regAgg = sqlite3GetTempRange(pParse, regAggSz); + regDistinct = regAgg; + sqlite3ExprCodeExprList(pParse, pOBList, regAgg, 0, SQLITE_ECEL_DUP); + jj = pOBList->nExpr; + if( !pF->bOBUnique ){ + sqlite3VdbeAddOp2(v, OP_Sequence, pF->iOBTab, regAgg+jj); + jj++; + } + if( pF->bOBPayload ){ + regDistinct = regAgg+jj; + sqlite3ExprCodeExprList(pParse, pList, regDistinct, 0, SQLITE_ECEL_DUP); + jj += nArg; + } + if( pF->bUseSubtype ){ + int kk; + int regBase = pF->bOBPayload ? regDistinct : regAgg; + for(kk=0; kknExpr; regAgg = sqlite3GetTempRange(pParse, nArg); + regDistinct = regAgg; sqlite3ExprCodeExprList(pParse, pList, regAgg, 0, SQLITE_ECEL_DUP); }else{ nArg = 0; regAgg = 0; } - if( pF->iDistinct>=0 ){ - addrNext = sqlite3VdbeMakeLabel(v); - testcase( nArg==0 ); /* Error condition */ - testcase( nArg>1 ); /* Also an error */ - codeDistinct(pParse, pF->iDistinct, addrNext, 1, regAgg); - } - if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ - CollSeq *pColl = 0; - struct ExprList_item *pItem; - int j; - assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ - for(j=0, pItem=pList->a; !pColl && jpExpr); + if( pF->iDistinct>=0 && pList ){ + if( addrNext==0 ){ + addrNext = sqlite3VdbeMakeLabel(pParse); } - if( !pColl ){ - pColl = pParse->db->pDfltColl; - } - if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; - sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, (char *)pColl, P4_COLLSEQ); + pF->iDistinct = codeDistinct(pParse, eDistinctType, + pF->iDistinct, addrNext, pList, regDistinct); + } + if( pF->iOBTab>=0 ){ + /* Insert a new record into the ORDER BY table */ + sqlite3VdbeAddOp3(v, OP_MakeRecord, regAgg, regAggSz-1, + regAgg+regAggSz-1); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pF->iOBTab, regAgg+regAggSz-1, + regAgg, regAggSz-1); + sqlite3ReleaseTempRange(pParse, regAgg, regAggSz); + }else{ + /* Invoke the AggStep function */ + if( pF->pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl = 0; + struct ExprList_item *pItem; + int j; + assert( pList!=0 ); /* pList!=0 if pF->pFunc has NEEDCOLL */ + for(j=0, pItem=pList->a; !pColl && jpExpr); + } + if( !pColl ){ + pColl = pParse->db->pDfltColl; + } + if( regHit==0 && pAggInfo->nAccumulator ) regHit = ++pParse->nMem; + sqlite3VdbeAddOp4(v, OP_CollSeq, regHit, 0, 0, + (char *)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, OP_AggStep, 0, regAgg, AggInfoFuncReg(pAggInfo,i)); + sqlite3VdbeAppendP4(v, pF->pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + sqlite3ReleaseTempRange(pParse, regAgg, nArg); } - sqlite3VdbeAddOp4(v, OP_AggStep0, 0, regAgg, pF->iMem, - (void*)pF->pFunc, P4_FUNCDEF); - sqlite3VdbeChangeP5(v, (u8)nArg); - sqlite3ExprCacheAffinityChange(pParse, regAgg, nArg); - sqlite3ReleaseTempRange(pParse, regAgg, nArg); if( addrNext ){ sqlite3VdbeResolveLabel(v, addrNext); - sqlite3ExprCacheClear(pParse); } + if( pParse->nErr ) return; + } + if( regHit==0 && pAggInfo->nAccumulator ){ + regHit = regAcc; } - - /* Before populating the accumulator registers, clear the column cache. - ** Otherwise, if any of the required column values are already present - ** in registers, sqlite3ExprCode() may use OP_SCopy to copy the value - ** to pC->iMem. But by the time the value is used, the original register - ** may have been used, invalidating the underlying buffer holding the - ** text or blob value. See ticket [883034dcb5]. - ** - ** Another solution would be to change the OP_SCopy used to copy cached - ** values to an OP_Copy. - */ if( regHit ){ addrHitTest = sqlite3VdbeAddOp1(v, OP_If, regHit); VdbeCoverage(v); } - sqlite3ExprCacheClear(pParse); for(i=0, pC=pAggInfo->aCol; inAccumulator; i++, pC++){ - sqlite3ExprCode(pParse, pC->pExpr, pC->iMem); + sqlite3ExprCode(pParse, pC->pCExpr, AggInfoColumnReg(pAggInfo,i)); + if( pParse->nErr ) return; } + pAggInfo->directMode = 0; - sqlite3ExprCacheClear(pParse); if( addrHitTest ){ - sqlite3VdbeJumpHere(v, addrHitTest); + sqlite3VdbeJumpHereOrPopInst(v, addrHitTest); } } @@ -117637,14 +150719,11 @@ static void explainSimpleCount( ){ if( pParse->explain==2 ){ int bCover = (pIdx!=0 && (HasRowid(pTab) || !IsPrimaryKeyIndex(pIdx))); - char *zEqp = sqlite3MPrintf(pParse->db, "SCAN TABLE %s%s%s", + sqlite3VdbeExplain(pParse, 0, "SCAN %s%s%s", pTab->zName, bCover ? " USING COVERING INDEX " : "", bCover ? pIdx->zName : "" ); - sqlite3VdbeAddOp4( - pParse->pVdbe, OP_Explain, pParse->iSelectId, 0, 0, zEqp, P4_DYNAMIC - ); } } #else @@ -117652,7 +150731,310 @@ static void explainSimpleCount( #endif /* -** Generate code for the SELECT statement given in the p argument. +** sqlite3WalkExpr() callback used by havingToWhere(). +** +** If the node passed to the callback is a TK_AND node, return +** WRC_Continue to tell sqlite3WalkExpr() to iterate through child nodes. +** +** Otherwise, return WRC_Prune. In this case, also check if the +** sub-expression matches the criteria for being moved to the WHERE +** clause. If so, add it to the WHERE clause and replace the sub-expression +** within the HAVING expression with a constant "1". +*/ +static int havingToWhereExprCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op!=TK_AND ){ + Select *pS = pWalker->u.pSelect; + /* This routine is called before the HAVING clause of the current + ** SELECT is analyzed for aggregates. So if pExpr->pAggInfo is set + ** here, it indicates that the expression is a correlated reference to a + ** column from an outer aggregate query, or an aggregate function that + ** belongs to an outer query. Do not move the expression to the WHERE + ** clause in this obscure case, as doing so may corrupt the outer Select + ** statements AggInfo structure. */ + if( sqlite3ExprIsConstantOrGroupBy(pWalker->pParse, pExpr, pS->pGroupBy) + && ExprAlwaysFalse(pExpr)==0 + && pExpr->pAggInfo==0 + ){ + sqlite3 *db = pWalker->pParse->db; + Expr *pNew = sqlite3Expr(db, TK_INTEGER, "1"); + if( pNew ){ + Expr *pWhere = pS->pWhere; + SWAP(Expr, *pNew, *pExpr); + pNew = sqlite3ExprAnd(pWalker->pParse, pWhere, pNew); + pS->pWhere = pNew; + pWalker->eCode = 1; + } + } + return WRC_Prune; + } + return WRC_Continue; +} + +/* +** Transfer eligible terms from the HAVING clause of a query, which is +** processed after grouping, to the WHERE clause, which is processed before +** grouping. For example, the query: +** +** SELECT * FROM WHERE a=? GROUP BY b HAVING b=? AND c=? +** +** can be rewritten as: +** +** SELECT * FROM WHERE a=? AND b=? GROUP BY b HAVING c=? +** +** A term of the HAVING expression is eligible for transfer if it consists +** entirely of constants and expressions that are also GROUP BY terms that +** use the "BINARY" collation sequence. +*/ +static void havingToWhere(Parse *pParse, Select *p){ + Walker sWalker; + memset(&sWalker, 0, sizeof(sWalker)); + sWalker.pParse = pParse; + sWalker.xExprCallback = havingToWhereExprCb; + sWalker.u.pSelect = p; + sqlite3WalkExpr(&sWalker, p->pHaving); +#if TREETRACE_ENABLED + if( sWalker.eCode && (sqlite3TreeTrace & 0x100)!=0 ){ + TREETRACE(0x100,pParse,p,("Move HAVING terms into WHERE:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif +} + +/* +** Check to see if the pThis entry of pTabList is a self-join of another view. +** Search FROM-clause entries in the range of iFirst..iEnd, including iFirst +** but stopping before iEnd. +** +** If pThis is a self-join, then return the SrcItem for the first other +** instance of that view found. If pThis is not a self-join then return 0. +*/ +static SrcItem *isSelfJoinView( + SrcList *pTabList, /* Search for self-joins in this FROM clause */ + SrcItem *pThis, /* Search for prior reference to this subquery */ + int iFirst, int iEnd /* Range of FROM-clause entries to search. */ +){ + SrcItem *pItem; + Select *pSel; + assert( pThis->fg.isSubquery ); + pSel = pThis->u4.pSubq->pSelect; + assert( pSel!=0 ); + if( pSel->selFlags & SF_PushDown ) return 0; + while( iFirsta[iFirst++]; + if( !pItem->fg.isSubquery ) continue; + if( pItem->fg.viaCoroutine ) continue; + if( pItem->zName==0 ) continue; + assert( pItem->pSTab!=0 ); + assert( pThis->pSTab!=0 ); + if( pItem->pSTab->pSchema!=pThis->pSTab->pSchema ) continue; + if( sqlite3_stricmp(pItem->zName, pThis->zName)!=0 ) continue; + pS1 = pItem->u4.pSubq->pSelect; + if( pItem->pSTab->pSchema==0 && pSel->selId!=pS1->selId ){ + /* The query flattener left two different CTE tables with identical + ** names in the same FROM clause. */ + continue; + } + if( pS1->selFlags & SF_PushDown ){ + /* The view was modified by some other optimization such as + ** pushDownWhereTerms() */ + continue; + } + return pItem; + } + return 0; +} + +/* +** Deallocate a single AggInfo object +*/ +static void agginfoFree(sqlite3 *db, void *pArg){ + AggInfo *p = (AggInfo*)pArg; + sqlite3DbFree(db, p->aCol); + sqlite3DbFree(db, p->aFunc); + sqlite3DbFreeNN(db, p); +} + +/* +** Attempt to transform a query of the form +** +** SELECT count(*) FROM (SELECT x FROM t1 UNION ALL SELECT y FROM t2) +** +** Into this: +** +** SELECT (SELECT count(*) FROM t1)+(SELECT count(*) FROM t2) +** +** The transformation only works if all of the following are true: +** +** * The subquery is a UNION ALL of two or more terms +** * The subquery does not have a LIMIT clause +** * There is no WHERE or GROUP BY or HAVING clauses on the subqueries +** * The outer query is a simple count(*) with no WHERE clause or other +** extraneous syntax. +** +** Return TRUE if the optimization is undertaken. +*/ +static int countOfViewOptimization(Parse *pParse, Select *p){ + Select *pSub, *pPrior; + Expr *pExpr; + Expr *pCount; + sqlite3 *db; + SrcItem *pFrom; + if( (p->selFlags & SF_Aggregate)==0 ) return 0; /* This is an aggregate */ + if( p->pEList->nExpr!=1 ) return 0; /* Single result column */ + if( p->pWhere ) return 0; + if( p->pHaving ) return 0; + if( p->pGroupBy ) return 0; + if( p->pOrderBy ) return 0; + pExpr = p->pEList->a[0].pExpr; + if( pExpr->op!=TK_AGG_FUNCTION ) return 0; /* Result is an aggregate */ + assert( ExprUseUToken(pExpr) ); + if( sqlite3_stricmp(pExpr->u.zToken,"count") ) return 0; /* Is count() */ + assert( ExprUseXList(pExpr) ); + if( pExpr->x.pList!=0 ) return 0; /* Must be count(*) */ + if( p->pSrc->nSrc!=1 ) return 0; /* One table in FROM */ + if( ExprHasProperty(pExpr, EP_WinFunc) ) return 0;/* Not a window function */ + pFrom = p->pSrc->a; + if( pFrom->fg.isSubquery==0 ) return 0; /* FROM is a subquery */ + pSub = pFrom->u4.pSubq->pSelect; + if( pSub->pPrior==0 ) return 0; /* Must be a compound */ + if( pSub->selFlags & SF_CopyCte ) return 0; /* Not a CTE */ + do{ + if( pSub->op!=TK_ALL && pSub->pPrior ) return 0; /* Must be UNION ALL */ + if( pSub->pWhere ) return 0; /* No WHERE clause */ + if( pSub->pLimit ) return 0; /* No LIMIT clause */ + if( pSub->selFlags & SF_Aggregate ) return 0; /* Not an aggregate */ + assert( pSub->pHaving==0 ); /* Due to the previous */ + pSub = pSub->pPrior; /* Repeat over compound */ + }while( pSub ); + + /* If we reach this point then it is OK to perform the transformation */ + + db = pParse->db; + pCount = pExpr; + pExpr = 0; + pSub = sqlite3SubqueryDetach(db, pFrom); + sqlite3SrcListDelete(db, p->pSrc); + p->pSrc = sqlite3DbMallocZero(pParse->db, sizeof(*p->pSrc)); + while( pSub ){ + Expr *pTerm; + pPrior = pSub->pPrior; + pSub->pPrior = 0; + pSub->pNext = 0; + pSub->selFlags |= SF_Aggregate; + pSub->selFlags &= ~SF_Compound; + pSub->nSelectRow = 0; + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, pSub->pEList); + pTerm = pPrior ? sqlite3ExprDup(db, pCount, 0) : pCount; + pSub->pEList = sqlite3ExprListAppend(pParse, 0, pTerm); + pTerm = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, pTerm, pSub); + if( pExpr==0 ){ + pExpr = pTerm; + }else{ + pExpr = sqlite3PExpr(pParse, TK_PLUS, pTerm, pExpr); + } + pSub = pPrior; + } + p->pEList->a[0].pExpr = pExpr; + p->selFlags &= ~SF_Aggregate; + +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x200 ){ + TREETRACE(0x200,pParse,p,("After count-of-view optimization:\n")); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + return 1; +} + +/* +** If any term of pSrc, or any SF_NestedFrom sub-query, is not the same +** as pSrcItem but has the same alias as p0, then return true. +** Otherwise return false. +*/ +static int sameSrcAlias(SrcItem *p0, SrcList *pSrc){ + int i; + for(i=0; inSrc; i++){ + SrcItem *p1 = &pSrc->a[i]; + if( p1==p0 ) continue; + if( p0->pSTab==p1->pSTab && 0==sqlite3_stricmp(p0->zAlias, p1->zAlias) ){ + return 1; + } + if( p1->fg.isSubquery + && (p1->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 + && sameSrcAlias(p0, p1->u4.pSubq->pSelect->pSrc) + ){ + return 1; + } + } + return 0; +} + +/* +** Return TRUE (non-zero) if the i-th entry in the pTabList SrcList can +** be implemented as a co-routine. The i-th entry is guaranteed to be +** a subquery. +** +** The subquery is implemented as a co-routine if all of the following are +** true: +** +** (1) The subquery will likely be implemented in the outer loop of +** the query. This will be the case if any one of the following +** conditions hold: +** (a) The subquery is the only term in the FROM clause +** (b) The subquery is the left-most term and a CROSS JOIN or similar +** requires it to be the outer loop +** (c) All of the following are true: +** (i) The subquery is the left-most subquery in the FROM clause +** (ii) There is nothing that would prevent the subquery from +** being used as the outer loop if the sqlite3WhereBegin() +** routine nominates it to that position. +** (iii) The query is not a UPDATE ... FROM +** (2) The subquery is not a CTE that should be materialized because +** (a) the AS MATERIALIZED keyword is used, or +** (b) the CTE is used multiple times and does not have the +** NOT MATERIALIZED keyword +** (3) The subquery is not part of a left operand for a RIGHT JOIN +** (4) The SQLITE_Coroutine optimization disable flag is not set +** (5) The subquery is not self-joined +*/ +static int fromClauseTermCanBeCoroutine( + Parse *pParse, /* Parsing context */ + SrcList *pTabList, /* FROM clause */ + int i, /* Which term of the FROM clause holds the subquery */ + int selFlags /* Flags on the SELECT statement */ +){ + SrcItem *pItem = &pTabList->a[i]; + if( pItem->fg.isCte ){ + const CteUse *pCteUse = pItem->u2.pCteUse; + if( pCteUse->eM10d==M10d_Yes ) return 0; /* (2a) */ + if( pCteUse->nUse>=2 && pCteUse->eM10d!=M10d_No ) return 0; /* (2b) */ + } + if( pTabList->a[0].fg.jointype & JT_LTORJ ) return 0; /* (3) */ + if( OptimizationDisabled(pParse->db, SQLITE_Coroutines) ) return 0; /* (4) */ + if( isSelfJoinView(pTabList, pItem, i+1, pTabList->nSrc)!=0 ){ + return 0; /* (5) */ + } + if( i==0 ){ + if( pTabList->nSrc==1 ) return 1; /* (1a) */ + if( pTabList->a[1].fg.jointype & JT_CROSS ) return 1; /* (1b) */ + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + return 1; + } + if( selFlags & SF_UpdateFrom ) return 0; /* (1c-iii) */ + while( 1 /*exit-by-break*/ ){ + if( pItem->fg.jointype & (JT_OUTER|JT_CROSS) ) return 0; /* (1c-ii) */ + if( i==0 ) break; + i--; + pItem--; + if( pItem->fg.isSubquery ) return 0; /* (1c-i) */ + } + return 1; +} + +/* +** Generate byte-code for the SELECT statement given in the p argument. ** ** The results are returned according to the SelectDest structure. ** See comments in sqliteInt.h for further information. @@ -117663,6 +151045,40 @@ static void explainSimpleCount( ** ** This routine does NOT free the Select structure passed in. The ** calling function needs to do that. +** +** This is a long function. The following is an outline of the processing +** steps, with tags referencing various milestones: +** +** * Resolve names and similar preparation tag-select-0100 +** * Scan of the FROM clause tag-select-0200 +** + OUTER JOIN strength reduction tag-select-0220 +** + Sub-query ORDER BY removal tag-select-0230 +** + Query flattening tag-select-0240 +** * Separate subroutine for compound-SELECT tag-select-0300 +** * WHERE-clause constant propagation tag-select-0330 +** * Count()-of-VIEW optimization tag-select-0350 +** * Scan of the FROM clause again tag-select-0400 +** + Authorize unreferenced tables tag-select-0410 +** + Predicate push-down optimization tag-select-0420 +** + Omit unused subquery columns optimization tag-select-0440 +** + Generate code to implement subqueries tag-select-0480 +** - Co-routines tag-select-0482 +** - Reuse previously computed CTE tag-select-0484 +** - REuse previously computed VIEW tag-select-0486 +** - Materialize a VIEW or CTE tag-select-0488 +** * DISTINCT ORDER BY -> GROUP BY optimization tag-select-0500 +** * Set up for ORDER BY tag-select-0600 +** * Create output table tag-select-0630 +** * Prepare registers for LIMIT tag-select-0650 +** * Setup for DISTINCT tag-select-0680 +** * Generate code for non-aggregate and non-GROUP BY tag-select-0700 +** * Generate code for aggregate and/or GROUP BY tag-select-0800 +** + GROUP BY queries tag-select-0810 +** + non-GROUP BY queries tag-select-0820 +** - Special case of count() w/o GROUP BY tag-select-0821 +** - General case of non-GROUP BY aggregates tag-select-0822 +** * Sort results, as needed tag-select-0900 +** * Internal self-checks tag-select-1000 */ SQLITE_PRIVATE int sqlite3Select( Parse *pParse, /* The parser context */ @@ -117678,81 +151094,187 @@ SQLITE_PRIVATE int sqlite3Select( Expr *pWhere; /* The WHERE clause. May be NULL */ ExprList *pGroupBy; /* The GROUP BY clause. May be NULL */ Expr *pHaving; /* The HAVING clause. May be NULL */ + AggInfo *pAggInfo = 0; /* Aggregate information */ int rc = 1; /* Value to return from this function */ DistinctCtx sDistinct; /* Info on how to code the DISTINCT keyword */ SortCtx sSort; /* Info on how to code the ORDER BY clause */ - AggInfo sAggInfo; /* Information used by aggregate queries */ int iEnd; /* Address of the end of the query */ sqlite3 *db; /* The database connection */ - -#ifndef SQLITE_OMIT_EXPLAIN - int iRestoreSelectId = pParse->iSelectId; - pParse->iSelectId = pParse->iNextSelectId++; -#endif + ExprList *pMinMaxOrderBy = 0; /* Added ORDER BY for min/max queries */ + u8 minMaxFlag; /* Flag for min/max queries */ db = pParse->db; - if( p==0 || db->mallocFailed || pParse->nErr ){ + assert( pParse==db->pParse ); + v = sqlite3GetVdbe(pParse); + if( p==0 || pParse->nErr ){ return 1; } + assert( db->mallocFailed==0 ); if( sqlite3AuthCheck(pParse, SQLITE_SELECT, 0, 0, 0) ) return 1; - memset(&sAggInfo, 0, sizeof(sAggInfo)); -#if SELECTTRACE_ENABLED - pParse->nSelectIndent++; - SELECTTRACE(1,pParse,p, ("begin processing:\n")); - if( sqlite3SelectTrace & 0x100 ){ - sqlite3TreeViewSelect(0, p, 0); +#if TREETRACE_ENABLED + TREETRACE(0x1,pParse,p, ("begin processing:\n", pParse->addrExplain)); + if( sqlite3TreeTrace & 0x10000 ){ + if( (sqlite3TreeTrace & 0x10001)==0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Select() at %s:%d", + __FILE__, __LINE__); + } + sqlite3ShowSelect(p); } #endif + /* tag-select-0100 */ assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistFifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Fifo ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_DistQueue ); assert( p->pOrderBy==0 || pDest->eDest!=SRT_Queue ); - if( IgnorableOrderby(pDest) ){ - assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || - pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || - pDest->eDest==SRT_Queue || pDest->eDest==SRT_DistFifo || - pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_Fifo); - /* If ORDER BY makes no difference in the output then neither does - ** DISTINCT so it can be removed too. */ - sqlite3ExprListDelete(db, p->pOrderBy); - p->pOrderBy = 0; + if( IgnorableDistinct(pDest) ){ + assert(pDest->eDest==SRT_Exists || pDest->eDest==SRT_Union || + pDest->eDest==SRT_Except || pDest->eDest==SRT_Discard || + pDest->eDest==SRT_DistQueue || pDest->eDest==SRT_DistFifo ); + /* All of these destinations are also able to ignore the ORDER BY clause */ + if( p->pOrderBy ){ +#if TREETRACE_ENABLED + TREETRACE(0x800,pParse,p, ("dropping superfluous ORDER BY:\n")); + if( sqlite3TreeTrace & 0x800 ){ + sqlite3TreeViewExprList(0, p->pOrderBy, 0, "ORDERBY"); + } +#endif + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + p->pOrderBy); + testcase( pParse->earlyCleanup ); + p->pOrderBy = 0; + } p->selFlags &= ~SF_Distinct; + p->selFlags |= SF_NoopOrderBy; } sqlite3SelectPrep(pParse, p, 0); - memset(&sSort, 0, sizeof(sSort)); - sSort.pOrderBy = p->pOrderBy; - pTabList = p->pSrc; - if( pParse->nErr || db->mallocFailed ){ + if( pParse->nErr ){ goto select_end; } + assert( db->mallocFailed==0 ); assert( p->pEList!=0 ); - isAgg = (p->selFlags & SF_Aggregate)!=0; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p, ("after name resolution:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10 ){ + TREETRACE(0x10,pParse,p, ("after name resolution:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif + /* If the SF_UFSrcCheck flag is set, then this function is being called + ** as part of populating the temp table for an UPDATE...FROM statement. + ** In this case, it is an error if the target object (pSrc->a[0]) name + ** or alias is duplicated within FROM clause (pSrc->a[1..n]). + ** + ** Postgres disallows this case too. The reason is that some other + ** systems handle this case differently, and not all the same way, + ** which is just confusing. To avoid this, we follow PG's lead and + ** disallow it altogether. */ + if( p->selFlags & SF_UFSrcCheck ){ + SrcItem *p0 = &p->pSrc->a[0]; + if( sameSrcAlias(p0, p->pSrc) ){ + sqlite3ErrorMsg(pParse, + "target object/alias may not appear in FROM clause: %s", + p0->zAlias ? p0->zAlias : p0->pSTab->zName + ); + goto select_end; + } - /* If writing to memory or generating a set - ** only a single column may be output. - */ -#ifndef SQLITE_OMIT_SUBQUERY - if( checkForMultiColumnSelectError(pParse, pDest, p->pEList->nExpr) ){ + /* Clear the SF_UFSrcCheck flag. The check has already been performed, + ** and leaving this flag set can cause errors if a compound sub-query + ** in p->pSrc is flattened into this query and this function called + ** again as part of compound SELECT processing. */ + p->selFlags &= ~SF_UFSrcCheck; + } + + if( pDest->eDest==SRT_Output ){ + sqlite3GenerateColumnNames(pParse, p); + } + +#ifndef SQLITE_OMIT_WINDOWFUNC + if( sqlite3WindowRewrite(pParse, p) ){ + assert( pParse->nErr ); goto select_end; } +#if TREETRACE_ENABLED + if( p->pWin && (sqlite3TreeTrace & 0x40)!=0 ){ + TREETRACE(0x40,pParse,p, ("after window rewrite:\n")); + sqlite3TreeViewSelect(0, p, 0); + } #endif +#endif /* SQLITE_OMIT_WINDOWFUNC */ + pTabList = p->pSrc; + isAgg = (p->selFlags & SF_Aggregate)!=0; + memset(&sSort, 0, sizeof(sSort)); + sSort.pOrderBy = p->pOrderBy; - /* Try to flatten subqueries in the FROM clause up into the main query + /* Try to do various optimizations (flattening subqueries, and strength + ** reduction of join operators) in the FROM clause up into the main query + ** tag-select-0200 */ #if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) for(i=0; !p->pPrior && inSrc; i++){ - struct SrcList_item *pItem = &pTabList->a[i]; - Select *pSub = pItem->pSelect; - int isAggSub; - Table *pTab = pItem->pTab; + SrcItem *pItem = &pTabList->a[i]; + Select *pSub = pItem->fg.isSubquery ? pItem->u4.pSubq->pSelect : 0; + Table *pTab = pItem->pSTab; + + /* The expander should have already created transient Table objects + ** even for FROM clause elements such as subqueries that do not correspond + ** to a real table */ + assert( pTab!=0 ); + + /* Try to simplify joins: + ** + ** LEFT JOIN -> JOIN + ** RIGHT JOIN -> JOIN + ** FULL JOIN -> RIGHT JOIN + ** + ** If terms of the i-th table are used in the WHERE clause in such a + ** way that the i-th table cannot be the NULL row of a join, then + ** perform the appropriate simplification. This is called + ** "OUTER JOIN strength reduction" in the SQLite documentation. + ** tag-select-0220 + */ + if( (pItem->fg.jointype & (JT_LEFT|JT_LTORJ))!=0 + && sqlite3ExprImpliesNonNullRow(p->pWhere, pItem->iCursor, + pItem->fg.jointype & JT_LTORJ) + && OptimizationEnabled(db, SQLITE_SimplifyJoin) + ){ + if( pItem->fg.jointype & JT_LEFT ){ + if( pItem->fg.jointype & JT_RIGHT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to RIGHT-JOIN on term %d\n",i)); + pItem->fg.jointype &= ~JT_LEFT; + }else{ + TREETRACE(0x1000,pParse,p, + ("LEFT-JOIN simplifies to JOIN on term %d\n",i)); + pItem->fg.jointype &= ~(JT_LEFT|JT_OUTER); + unsetJoinExpr(p->pWhere, pItem->iCursor, 0); + } + } + if( pItem->fg.jointype & JT_LTORJ ){ + for(j=i+1; jnSrc; j++){ + SrcItem *pI2 = &pTabList->a[j]; + if( pI2->fg.jointype & JT_RIGHT ){ + if( pI2->fg.jointype & JT_LEFT ){ + TREETRACE(0x1000,pParse,p, + ("FULL-JOIN simplifies to LEFT-JOIN on term %d\n",j)); + pI2->fg.jointype &= ~JT_RIGHT; + }else{ + TREETRACE(0x1000,pParse,p, + ("RIGHT-JOIN simplifies to JOIN on term %d\n",j)); + pI2->fg.jointype &= ~(JT_RIGHT|JT_OUTER); + unsetJoinExpr(p->pWhere, pI2->iCursor, 1); + } + } + } + for(j=pTabList->nSrc-1; j>=0; j--){ + pTabList->a[j].fg.jointype &= ~JT_LTORJ; + if( pTabList->a[j].fg.jointype & JT_RIGHT ) break; + } + } + } + + /* No further action if this term of the FROM clause is not a subquery */ if( pSub==0 ) continue; /* Catch mismatch in the declared columns of a view and the number of @@ -117763,13 +151285,95 @@ SQLITE_PRIVATE int sqlite3Select( goto select_end; } - isAggSub = (pSub->selFlags & SF_Aggregate)!=0; - if( flattenSubquery(pParse, p, i, isAgg, isAggSub) ){ + /* Do not attempt the usual optimizations (flattening and ORDER BY + ** elimination) on a MATERIALIZED common table expression because + ** a MATERIALIZED common table expression is an optimization fence. + */ + if( pItem->fg.isCte && pItem->u2.pCteUse->eM10d==M10d_Yes ){ + continue; + } + + /* Do not try to flatten an aggregate subquery. + ** + ** Flattening an aggregate subquery is only possible if the outer query + ** is not a join. But if the outer query is not a join, then the subquery + ** will be implemented as a co-routine and there is no advantage to + ** flattening in that case. + */ + if( (pSub->selFlags & SF_Aggregate)!=0 ) continue; + assert( pSub->pGroupBy==0 ); + + /* tag-select-0230: + ** If a FROM-clause subquery has an ORDER BY clause that is not + ** really doing anything, then delete it now so that it does not + ** interfere with query flattening. See the discussion at + ** https://sqlite.org/forum/forumpost/2d76f2bcf65d256a + ** + ** Beware of these cases where the ORDER BY clause may not be safely + ** omitted: + ** + ** (1) There is also a LIMIT clause + ** (2) The subquery was added to help with window-function + ** processing + ** (3) The subquery is in the FROM clause of an UPDATE + ** (4) The outer query uses an aggregate function other than + ** the built-in count(), min(), or max(). + ** (5) The ORDER BY isn't going to accomplish anything because + ** one of: + ** (a) The outer query has a different ORDER BY clause + ** (b) The subquery is part of a join + ** See forum post 062d576715d277c8 + ** (6) The subquery is not a recursive CTE. ORDER BY has a different + ** meaning for recursive CTEs and this optimization does not + ** apply. + ** + ** Also retain the ORDER BY if the OmitOrderBy optimization is disabled. + */ + if( pSub->pOrderBy!=0 + && (p->pOrderBy!=0 || pTabList->nSrc>1) /* Condition (5) */ + && pSub->pLimit==0 /* Condition (1) */ + && (pSub->selFlags & (SF_OrderByReqd|SF_Recursive))==0 /* (2) and (6) */ + && (p->selFlags & SF_OrderByReqd)==0 /* Condition (3) and (4) */ + && OptimizationEnabled(db, SQLITE_OmitOrderBy) + ){ + TREETRACE(0x800,pParse,p, + ("omit superfluous ORDER BY on %r FROM-clause subquery\n",i+1)); + sqlite3ParserAddCleanup(pParse, sqlite3ExprListDeleteGeneric, + pSub->pOrderBy); + pSub->pOrderBy = 0; + } + + /* If the outer query contains a "complex" result set (that is, + ** if the result set of the outer query uses functions or subqueries) + ** and if the subquery contains an ORDER BY clause and if + ** it will be implemented as a co-routine, then do not flatten. This + ** restriction allows SQL constructs like this: + ** + ** SELECT expensive_function(x) + ** FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + ** + ** The expensive_function() is only computed on the 10 rows that + ** are output, rather than every row of the table. + ** + ** The requirement that the outer query have a complex result set + ** means that flattening does occur on simpler SQL constraints without + ** the expensive_function() like: + ** + ** SELECT x FROM (SELECT x FROM tab ORDER BY y LIMIT 10); + */ + if( pSub->pOrderBy!=0 + && i==0 + && (p->selFlags & SF_ComplexResult)!=0 + && (pTabList->nSrc==1 + || (pTabList->a[1].fg.jointype&(JT_OUTER|JT_CROSS))!=0) + ){ + continue; + } + + /* tag-select-0240 */ + if( flattenSubquery(pParse, p, i, isAgg) ){ + if( pParse->nErr ) goto select_end; /* This subquery can be absorbed into its parent. */ - if( isAggSub ){ - isAgg = 1; - p->selFlags |= SF_Aggregate; - } i = -1; } pTabList = p->pSrc; @@ -117780,47 +151384,109 @@ SQLITE_PRIVATE int sqlite3Select( } #endif - /* Get a pointer the VDBE under construction, allocating a new VDBE if one - ** does not already exist */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto select_end; - #ifndef SQLITE_OMIT_COMPOUND_SELECT /* Handle compound SELECT statements using the separate multiSelect() - ** procedure. + ** procedure. tag-select-0300 */ if( p->pPrior ){ rc = multiSelect(pParse, p, pDest); - explainSetInteger(pParse->iSelectId, iRestoreSelectId); -#if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p,("end compound-select processing\n")); - pParse->nSelectIndent--; +#if TREETRACE_ENABLED + TREETRACE(0x400,pParse,p,("end compound-select processing\n")); + if( (sqlite3TreeTrace & 0x400)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + sqlite3TreeViewSelect(0, p, 0); + } #endif + if( p->pNext==0 ) ExplainQueryPlanPop(pParse); return rc; } #endif - /* Generate code for all sub-queries in the FROM clause + /* Do the WHERE-clause constant propagation optimization if this is + ** a join. No need to spend time on this operation for non-join queries + ** as the equivalent optimization will be handled by query planner in + ** sqlite3WhereBegin(). tag-select-0330 */ -#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) - for(i=0; inSrc; i++){ - struct SrcList_item *pItem = &pTabList->a[i]; - SelectDest dest; - Select *pSub = pItem->pSelect; - if( pSub==0 ) continue; - - /* Sometimes the code for a subquery will be generated more than - ** once, if the subquery is part of the WHERE clause in a LEFT JOIN, - ** for example. In that case, do not regenerate the code to manifest - ** a view or the co-routine to implement a view. The first instance - ** is sufficient, though the subroutine to manifest the view does need - ** to be invoked again. */ - if( pItem->addrFillSub ){ - if( pItem->fg.viaCoroutine==0 ){ - sqlite3VdbeAddOp2(v, OP_Gosub, pItem->regReturn, pItem->addrFillSub); - } - continue; + if( p->pWhere!=0 + && p->pWhere->op==TK_AND + && OptimizationEnabled(db, SQLITE_PropagateConst) + && propagateConstants(pParse, p) + ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x2000 ){ + TREETRACE(0x2000,pParse,p,("After constant propagation:\n")); + sqlite3TreeViewSelect(0, p, 0); } +#endif + }else{ + TREETRACE(0x2000,pParse,p,("Constant propagation not helpful\n")); + } + + /* tag-select-0350 */ + if( OptimizationEnabled(db, SQLITE_QueryFlattener|SQLITE_CountOfView) + && countOfViewOptimization(pParse, p) + ){ + if( db->mallocFailed ) goto select_end; + pTabList = p->pSrc; + } + + /* Loop over all terms in the FROM clause and do two things for each term: + ** + ** (1) Authorize unreferenced tables + ** (2) Generate code for all sub-queries + ** + ** tag-select-0400 + */ + for(i=0; inSrc; i++){ + SrcItem *pItem = &pTabList->a[i]; + SrcItem *pPrior; + SelectDest dest; + Subquery *pSubq; + Select *pSub; +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + const char *zSavedAuthContext; +#endif + + /* Authorized unreferenced tables. tag-select-0410 + ** + ** Issue SQLITE_READ authorizations with a fake column name for any + ** tables that are referenced but from which no values are extracted. + ** Examples of where these kinds of null SQLITE_READ authorizations + ** would occur: + ** + ** SELECT count(*) FROM t1; -- SQLITE_READ t1."" + ** SELECT t1.* FROM t1, t2; -- SQLITE_READ t2."" + ** + ** The fake column name is an empty string. It is possible for a table to + ** have a column named by the empty string, in which case there is no way to + ** distinguish between an unreferenced table and an actual reference to the + ** "" column. The original design was for the fake column name to be a NULL, + ** which would be unambiguous. But legacy authorization callbacks might + ** assume the column name is non-NULL and segfault. The use of an empty + ** string for the fake column name seems safer. + */ + if( pItem->colUsed==0 && pItem->zName!=0 ){ + const char *zDb; + if( pItem->fg.fixedSchema ){ + int iDb = sqlite3SchemaToIndex(pParse->db, pItem->u4.pSchema); + zDb = db->aDb[iDb].zDbSName; + }else if( pItem->fg.isSubquery ){ + zDb = 0; + }else{ + zDb = pItem->u4.zDatabase; + } + sqlite3AuthCheck(pParse, SQLITE_READ, pItem->zName, "", zDb); + } + +#if !defined(SQLITE_OMIT_SUBQUERY) || !defined(SQLITE_OMIT_VIEW) + /* Generate code for all sub-queries in the FROM clause + */ + if( pItem->fg.isSubquery==0 ) continue; + pSubq = pItem->u4.pSubq; + assert( pSubq!=0 ); + pSub = pSubq->pSelect; + + /* The code for a subquery should only be generated once. */ + if( pSubq->addrFillSub!=0 ) continue; /* Increment Parse.nHeight by the height of the largest expression ** tree referred to by this, the parent select. The child select @@ -117833,91 +151499,139 @@ SQLITE_PRIVATE int sqlite3Select( /* Make copies of constant WHERE-clause terms in the outer query down ** inside the subquery. This can help the subquery to run more efficiently. + ** This is the "predicate push-down optimization". tag-select-0420 */ - if( (pItem->fg.jointype & JT_OUTER)==0 - && pushDownWhereTerms(db, pSub, p->pWhere, pItem->iCursor) + if( OptimizationEnabled(db, SQLITE_PushDown) + && (pItem->fg.isCte==0 + || (pItem->u2.pCteUse->eM10d!=M10d_Yes && pItem->u2.pCteUse->nUse<2)) + && pushDownWhereTerms(pParse, pSub, p->pWhere, pTabList, i) ){ -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x100 ){ - SELECTTRACE(0x100,pParse,p,("After WHERE-clause push-down:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("After WHERE-clause push-down into subquery %d:\n", pSub->selId)); + sqlite3TreeViewSelect(0, p, 0); + } +#endif + assert( pSubq->pSelect && (pSub->selFlags & SF_PushDown)!=0 ); + }else{ + TREETRACE(0x4000,pParse,p,("WHERE-lcause push-down not possible\n")); + } + + /* Convert unused result columns of the subquery into simple NULL + ** expressions, to avoid unneeded searching and computation. + ** tag-select-0440 + */ + if( OptimizationEnabled(db, SQLITE_NullUnusedCols) + && disableUnusedSubqueryResultColumns(pItem) + ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x4000 ){ + TREETRACE(0x4000,pParse,p, + ("Change unused result columns to NULL for subquery %d:\n", + pSub->selId)); sqlite3TreeViewSelect(0, p, 0); } #endif } - /* Generate code to implement the subquery - ** - ** The subquery is implemented as a co-routine if all of these are true: - ** (1) The subquery is guaranteed to be the outer loop (so that it - ** does not need to be computed more than once) - ** (2) The ALL keyword after SELECT is omitted. (Applications are - ** allowed to say "SELECT ALL" instead of just "SELECT" to disable - ** the use of co-routines.) - ** (3) Co-routines are not disabled using sqlite3_test_control() - ** with SQLITE_TESTCTRL_OPTIMIZATIONS. - ** - ** TODO: Are there other reasons beside (1) to use a co-routine - ** implementation? + zSavedAuthContext = pParse->zAuthContext; + pParse->zAuthContext = pItem->zName; + + /* Generate byte-code to implement the subquery tag-select-0480 */ - if( i==0 - && (pTabList->nSrc==1 - || (pTabList->a[1].fg.jointype&(JT_LEFT|JT_CROSS))!=0) /* (1) */ - && (p->selFlags & SF_All)==0 /* (2) */ - && OptimizationEnabled(db, SQLITE_SubqCoroutine) /* (3) */ - ){ + if( fromClauseTermCanBeCoroutine(pParse, pTabList, i, p->selFlags) ){ /* Implement a co-routine that will return a single row of the result - ** set on each invocation. + ** set on each invocation. tag-select-0482 */ int addrTop = sqlite3VdbeCurrentAddr(v)+1; - pItem->regReturn = ++pParse->nMem; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, pItem->regReturn, 0, addrTop); - VdbeComment((v, "%s", pItem->pTab->zName)); - pItem->addrFillSub = addrTop; - sqlite3SelectDestInit(&dest, SRT_Coroutine, pItem->regReturn); - explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + + pSubq->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, pSubq->regReturn, 0, addrTop); + VdbeComment((v, "%!S", pItem)); + pSubq->addrFillSub = addrTop; + sqlite3SelectDestInit(&dest, SRT_Coroutine, pSubq->regReturn); + ExplainQueryPlan((pParse, 1, "CO-ROUTINE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; pItem->fg.viaCoroutine = 1; - pItem->regResult = dest.iSdst; - sqlite3VdbeEndCoroutine(v, pItem->regReturn); + pSubq->regResult = dest.iSdst; + sqlite3VdbeEndCoroutine(v, pSubq->regReturn); + VdbeComment((v, "end %!S", pItem)); sqlite3VdbeJumpHere(v, addrTop-1); sqlite3ClearTempRegCache(pParse); + }else if( pItem->fg.isCte && pItem->u2.pCteUse->addrM9e>0 ){ + /* This is a CTE for which materialization code has already been + ** generated. Invoke the subroutine to compute the materialization, + ** then make the pItem->iCursor be a copy of the ephemeral table that + ** holds the result of the materialization. tag-select-0484 */ + CteUse *pCteUse = pItem->u2.pCteUse; + sqlite3VdbeAddOp2(v, OP_Gosub, pCteUse->regRtn, pCteUse->addrM9e); + if( pItem->iCursor!=pCteUse->iCur ){ + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pCteUse->iCur); + VdbeComment((v, "%!S", pItem)); + } + pSub->nSelectRow = pCteUse->nRowEst; + }else if( (pPrior = isSelfJoinView(pTabList, pItem, 0, i))!=0 ){ + /* This view has already been materialized by a prior entry in + ** this same FROM clause. Reuse it. tag-select-0486 */ + Subquery *pPriorSubq; + assert( pPrior->fg.isSubquery ); + pPriorSubq = pPrior->u4.pSubq; + assert( pPriorSubq!=0 ); + if( pPriorSubq->addrFillSub ){ + sqlite3VdbeAddOp2(v, OP_Gosub, pPriorSubq->regReturn, + pPriorSubq->addrFillSub); + } + sqlite3VdbeAddOp2(v, OP_OpenDup, pItem->iCursor, pPrior->iCursor); + pSub->nSelectRow = pPriorSubq->pSelect->nSelectRow; }else{ - /* Generate a subroutine that will fill an ephemeral table with - ** the content of this subquery. pItem->addrFillSub will point - ** to the address of the generated subroutine. pItem->regReturn - ** is a register allocated to hold the subroutine return address - */ + /* Materialize the view. If the view is not correlated, generate a + ** subroutine to do the materialization so that subsequent uses of + ** the same view can reuse the materialization. tag-select-0488 */ int topAddr; int onceAddr = 0; - int retAddr; - assert( pItem->addrFillSub==0 ); - pItem->regReturn = ++pParse->nMem; - topAddr = sqlite3VdbeAddOp2(v, OP_Integer, 0, pItem->regReturn); - pItem->addrFillSub = topAddr+1; +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExplain; +#endif + + pSubq->regReturn = ++pParse->nMem; + topAddr = sqlite3VdbeAddOp0(v, OP_Goto); + pSubq->addrFillSub = topAddr+1; + pItem->fg.isMaterialized = 1; if( pItem->fg.isCorrelated==0 ){ /* If the subquery is not correlated and if we are not inside of ** a trigger, then we only need to compute the value of the subquery ** once. */ - onceAddr = sqlite3CodeOnce(pParse); VdbeCoverage(v); - VdbeComment((v, "materialize \"%s\"", pItem->pTab->zName)); + onceAddr = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + VdbeComment((v, "materialize %!S", pItem)); }else{ - VdbeNoopComment((v, "materialize \"%s\"", pItem->pTab->zName)); + VdbeNoopComment((v, "materialize %!S", pItem)); } sqlite3SelectDestInit(&dest, SRT_EphemTab, pItem->iCursor); - explainSetInteger(pItem->iSelectId, (u8)pParse->iNextSelectId); + + ExplainQueryPlan2(addrExplain, (pParse, 1, "MATERIALIZE %!S", pItem)); sqlite3Select(pParse, pSub, &dest); - pItem->pTab->nRowLogEst = pSub->nSelectRow; + pItem->pSTab->nRowLogEst = pSub->nSelectRow; if( onceAddr ) sqlite3VdbeJumpHere(v, onceAddr); - retAddr = sqlite3VdbeAddOp1(v, OP_Return, pItem->regReturn); - VdbeComment((v, "end %s", pItem->pTab->zName)); - sqlite3VdbeChangeP1(v, topAddr, retAddr); + sqlite3VdbeAddOp2(v, OP_Return, pSubq->regReturn, topAddr+1); + VdbeComment((v, "end %!S", pItem)); + sqlite3VdbeScanStatusRange(v, addrExplain, addrExplain, -1); + sqlite3VdbeJumpHere(v, topAddr); sqlite3ClearTempRegCache(pParse); + if( pItem->fg.isCte && pItem->fg.isCorrelated==0 ){ + CteUse *pCteUse = pItem->u2.pCteUse; + pCteUse->addrM9e = pSubq->addrFillSub; + pCteUse->regRtn = pSubq->regReturn; + pCteUse->iCur = pItem->iCursor; + pCteUse->nRowEst = pSub->nSelectRow; + } } if( db->mallocFailed ) goto select_end; pParse->nHeight -= sqlite3SelectExprHeight(p); - } + pParse->zAuthContext = zSavedAuthContext; #endif + } /* Various elements of the SELECT copied into local variables for ** convenience */ @@ -117927,14 +151641,16 @@ SQLITE_PRIVATE int sqlite3Select( pHaving = p->pHaving; sDistinct.isTnct = (p->selFlags & SF_Distinct)!=0; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("After all FROM-clause analysis:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x8000 ){ + TREETRACE(0x8000,pParse,p,("After all FROM-clause analysis:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif - /* If the query is DISTINCT with an ORDER BY but is not an aggregate, and + /* tag-select-0500 + ** + ** If the query is DISTINCT with an ORDER BY but is not an aggregate, and ** if the select-list is the same as the ORDER BY list, then this query ** can be rewritten as a GROUP BY. In other words, this: ** @@ -117944,24 +151660,35 @@ SQLITE_PRIVATE int sqlite3Select( ** ** SELECT xyz FROM ... GROUP BY xyz ORDER BY xyz ** - ** The second form is preferred as a single index (or temp-table) may be - ** used for both the ORDER BY and DISTINCT processing. As originally - ** written the query must use a temp-table for at least one of the ORDER + ** The second form is preferred as a single index (or temp-table) may be + ** used for both the ORDER BY and DISTINCT processing. As originally + ** written the query must use a temp-table for at least one of the ORDER ** BY and DISTINCT, and an index or separate temp-table for the other. */ - if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct + if( (p->selFlags & (SF_Distinct|SF_Aggregate))==SF_Distinct && sqlite3ExprListCompare(sSort.pOrderBy, pEList, -1)==0 + && OptimizationEnabled(db, SQLITE_GroupByOrder) +#ifndef SQLITE_OMIT_WINDOWFUNC + && p->pWin==0 +#endif ){ p->selFlags &= ~SF_Distinct; pGroupBy = p->pGroupBy = sqlite3ExprListDup(db, pEList, 0); + if( pGroupBy ){ + for(i=0; inExpr; i++){ + pGroupBy->a[i].u.x.iOrderByCol = i+1; + } + } + p->selFlags |= SF_Aggregate; /* Notice that even thought SF_Distinct has been cleared from p->selFlags, ** the sDistinct.isTnct is still set. Hence, isTnct represents the ** original setting of the SF_Distinct flag, not the current setting */ assert( sDistinct.isTnct ); + sDistinct.isTnct = 2; -#if SELECTTRACE_ENABLED - if( sqlite3SelectTrace & 0x400 ){ - SELECTTRACE(0x400,pParse,p,("Transform DISTINCT into GROUP BY:\n")); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20000 ){ + TREETRACE(0x20000,pParse,p,("Transform DISTINCT into GROUP BY:\n")); sqlite3TreeViewSelect(0, p, 0); } #endif @@ -117973,11 +151700,12 @@ SQLITE_PRIVATE int sqlite3Select( ** If that is the case, then the OP_OpenEphemeral instruction will be ** changed to an OP_Noop once we figure out that the sorting index is ** not needed. The sSort.addrSortIndex variable is used to facilitate - ** that change. + ** that change. tag-select-0600 */ if( sSort.pOrderBy ){ KeyInfo *pKeyInfo; - pKeyInfo = keyInfoFromExprList(pParse, sSort.pOrderBy, 0, pEList->nExpr); + pKeyInfo = sqlite3KeyInfoFromExprList( + pParse, sSort.pOrderBy, 0, pEList->nExpr); sSort.iECursor = pParse->nTab++; sSort.addrSortIndex = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, @@ -117989,29 +151717,44 @@ SQLITE_PRIVATE int sqlite3Select( } /* If the output is destined for a temporary table, open that table. + ** tag-select-0630 */ if( pDest->eDest==SRT_EphemTab ){ sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pDest->iSDParm, pEList->nExpr); + if( p->selFlags & SF_NestedFrom ){ + /* Delete or NULL-out result columns that will never be used */ + int ii; + for(ii=pEList->nExpr-1; ii>0 && pEList->a[ii].fg.bUsed==0; ii--){ + sqlite3ExprDelete(db, pEList->a[ii].pExpr); + sqlite3DbFree(db, pEList->a[ii].zEName); + pEList->nExpr--; + } + for(ii=0; iinExpr; ii++){ + if( pEList->a[ii].fg.bUsed==0 ) pEList->a[ii].pExpr->op = TK_NULL; + } + } } - /* Set the limiter. + /* Set the limiter. tag-select-0650 */ - iEnd = sqlite3VdbeMakeLabel(v); - p->nSelectRow = 320; /* 4 billion rows */ - computeLimitRegisters(pParse, p, iEnd); + iEnd = sqlite3VdbeMakeLabel(pParse); + if( (p->selFlags & SF_FixedLimit)==0 ){ + p->nSelectRow = 320; /* 4 billion rows */ + } + if( p->pLimit ) computeLimitRegisters(pParse, p, iEnd); if( p->iLimit==0 && sSort.addrSortIndex>=0 ){ sqlite3VdbeChangeOpcode(v, sSort.addrSortIndex, OP_SorterOpen); sSort.sortFlags |= SORTFLAG_UseSorter; } - /* Open an ephemeral index to use for the distinct set. + /* Open an ephemeral index to use for the distinct set. tag-select-0680 */ if( p->selFlags & SF_Distinct ){ sDistinct.tabTnct = pParse->nTab++; sDistinct.addrTnct = sqlite3VdbeAddOp4(v, OP_OpenEphemeral, - sDistinct.tabTnct, 0, 0, - (char*)keyInfoFromExprList(pParse, p->pEList,0,0), - P4_KEYINFO); + sDistinct.tabTnct, 0, 0, + (char*)sqlite3KeyInfoFromExprList(pParse, p->pEList,0,0), + P4_KEYINFO); sqlite3VdbeChangeP5(v, BTREE_UNORDERED); sDistinct.eTnctType = WHERE_DISTINCT_UNORDERED; }else{ @@ -118019,14 +151762,22 @@ SQLITE_PRIVATE int sqlite3Select( } if( !isAgg && pGroupBy==0 ){ - /* No aggregate functions and no GROUP BY clause */ - u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0); + /* No aggregate functions and no GROUP BY clause. tag-select-0700 */ + u16 wctrlFlags = (sDistinct.isTnct ? WHERE_WANT_DISTINCT : 0) + | (p->selFlags & SF_FixedLimit); +#ifndef SQLITE_OMIT_WINDOWFUNC + Window *pWin = p->pWin; /* Main window object (or NULL) */ + if( pWin ){ + sqlite3WindowCodeInit(pParse, p); + } +#endif assert( WHERE_USE_LIMIT==SF_FixedLimit ); - wctrlFlags |= p->selFlags & SF_FixedLimit; + /* Begin the database scan. */ + TREETRACE(0x2,pParse,p,("WhereBegin\n")); pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, sSort.pOrderBy, - p->pEList, wctrlFlags, p->nSelectRow); + p->pEList, p, wctrlFlags, p->nSelectRow); if( pWInfo==0 ) goto select_end; if( sqlite3WhereOutputRowCount(pWInfo) < p->nSelectRow ){ p->nSelectRow = sqlite3WhereOutputRowCount(pWInfo); @@ -118036,12 +151787,14 @@ SQLITE_PRIVATE int sqlite3Select( } if( sSort.pOrderBy ){ sSort.nOBSat = sqlite3WhereIsOrdered(pWInfo); + sSort.labelOBLopt = sqlite3WhereOrderByLimitOptLabel(pWInfo); if( sSort.nOBSat==sSort.pOrderBy->nExpr ){ sSort.pOrderBy = 0; } } + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); - /* If sorting index that was created by a prior OP_OpenEphemeral + /* If sorting index that was created by a prior OP_OpenEphemeral ** instruction ended up not being needed, then change the OP_OpenEphemeral ** into an OP_Noop. */ @@ -118049,17 +151802,41 @@ SQLITE_PRIVATE int sqlite3Select( sqlite3VdbeChangeToNoop(v, sSort.addrSortIndex); } - /* Use the standard inner loop. */ - selectInnerLoop(pParse, p, pEList, -1, &sSort, &sDistinct, pDest, - sqlite3WhereContinueLabel(pWInfo), - sqlite3WhereBreakLabel(pWInfo)); + assert( p->pEList==pEList ); +#ifndef SQLITE_OMIT_WINDOWFUNC + if( pWin ){ + int addrGosub = sqlite3VdbeMakeLabel(pParse); + int iCont = sqlite3VdbeMakeLabel(pParse); + int iBreak = sqlite3VdbeMakeLabel(pParse); + int regGosub = ++pParse->nMem; - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); + sqlite3WindowCodeStep(pParse, p, pWInfo, regGosub, addrGosub); + + sqlite3VdbeAddOp2(v, OP_Goto, 0, iBreak); + sqlite3VdbeResolveLabel(v, addrGosub); + VdbeNoopComment((v, "inner-loop subroutine")); + sSort.labelOBLopt = 0; + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, iCont, iBreak); + sqlite3VdbeResolveLabel(v, iCont); + sqlite3VdbeAddOp1(v, OP_Return, regGosub); + VdbeComment((v, "end inner-loop subroutine")); + sqlite3VdbeResolveLabel(v, iBreak); + }else +#endif /* SQLITE_OMIT_WINDOWFUNC */ + { + /* Use the standard inner loop. */ + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, + sqlite3WhereContinueLabel(pWInfo), + sqlite3WhereBreakLabel(pWInfo)); + + /* End the database scan loop. + */ + TREETRACE(0x2,pParse,p,("WhereEnd\n")); + sqlite3WhereEnd(pWInfo); + } }else{ - /* This case when there exist aggregate functions or a GROUP BY clause - ** or both */ + /* This case is for when there exist aggregate functions or a GROUP BY + ** clause or both. tag-select-0800 */ NameContext sNC; /* Name context for processing aggregate information */ int iAMem; /* First Mem address for storing current GROUP BY */ int iBMem; /* First Mem address for previous GROUP BY */ @@ -118088,58 +151865,101 @@ SQLITE_PRIVATE int sqlite3Select( } assert( 66==sqlite3LogEst(100) ); if( p->nSelectRow>66 ) p->nSelectRow = 66; + + /* If there is both a GROUP BY and an ORDER BY clause and they are + ** identical, then it may be possible to disable the ORDER BY clause + ** on the grounds that the GROUP BY will cause elements to come out + ** in the correct order. It also may not - the GROUP BY might use a + ** database index that causes rows to be grouped together as required + ** but not actually sorted. Either way, record the fact that the + ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp + ** variable. */ + if( sSort.pOrderBy && pGroupBy->nExpr==sSort.pOrderBy->nExpr ){ + int ii; + /* The GROUP BY processing doesn't care whether rows are delivered in + ** ASC or DESC order - only that each group is returned contiguously. + ** So set the ASC/DESC flags in the GROUP BY to match those in the + ** ORDER BY to maximize the chances of rows being delivered in an + ** order that makes the ORDER BY redundant. */ + for(ii=0; iinExpr; ii++){ + u8 sortFlags; + sortFlags = sSort.pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_DESC; + pGroupBy->a[ii].fg.sortFlags = sortFlags; + } + if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ + orderByGrp = 1; + } + } }else{ assert( 0==sqlite3LogEst(1) ); p->nSelectRow = 0; } - /* If there is both a GROUP BY and an ORDER BY clause and they are - ** identical, then it may be possible to disable the ORDER BY clause - ** on the grounds that the GROUP BY will cause elements to come out - ** in the correct order. It also may not - the GROUP BY might use a - ** database index that causes rows to be grouped together as required - ** but not actually sorted. Either way, record the fact that the - ** ORDER BY and GROUP BY clauses are the same by setting the orderByGrp - ** variable. */ - if( sqlite3ExprListCompare(pGroupBy, sSort.pOrderBy, -1)==0 ){ - orderByGrp = 1; - } - /* Create a label to jump to when we want to abort the query */ - addrEnd = sqlite3VdbeMakeLabel(v); + addrEnd = sqlite3VdbeMakeLabel(pParse); /* Convert TK_COLUMN nodes into TK_AGG_COLUMN and make entries in ** sAggInfo for all TK_AGG_FUNCTION nodes in expressions of the ** SELECT statement. */ + pAggInfo = sqlite3DbMallocZero(db, sizeof(*pAggInfo) ); + if( pAggInfo ){ + sqlite3ParserAddCleanup(pParse, agginfoFree, pAggInfo); + testcase( pParse->earlyCleanup ); + } + if( db->mallocFailed ){ + goto select_end; + } + pAggInfo->selId = p->selId; +#ifdef SQLITE_DEBUG + pAggInfo->pSelect = p; +#endif memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; - sNC.pAggInfo = &sAggInfo; - sAggInfo.mnReg = pParse->nMem+1; - sAggInfo.nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; - sAggInfo.pGroupBy = pGroupBy; + sNC.uNC.pAggInfo = pAggInfo; + VVA_ONLY( sNC.ncFlags = NC_UAggInfo; ) + pAggInfo->nSortingColumn = pGroupBy ? pGroupBy->nExpr : 0; + pAggInfo->pGroupBy = pGroupBy; sqlite3ExprAnalyzeAggList(&sNC, pEList); sqlite3ExprAnalyzeAggList(&sNC, sSort.pOrderBy); if( pHaving ){ + if( pGroupBy ){ + assert( pWhere==p->pWhere ); + assert( pHaving==p->pHaving ); + assert( pGroupBy==p->pGroupBy ); + havingToWhere(pParse, p); + pWhere = p->pWhere; + } sqlite3ExprAnalyzeAggregates(&sNC, pHaving); } - sAggInfo.nAccumulator = sAggInfo.nColumn; - for(i=0; ix.pList); - sNC.ncFlags &= ~NC_InAggFunc; + pAggInfo->nAccumulator = pAggInfo->nColumn; + if( p->pGroupBy==0 && p->pHaving==0 && pAggInfo->nFunc==1 ){ + minMaxFlag = minMaxQuery(db, pAggInfo->aFunc[0].pFExpr, &pMinMaxOrderBy); + }else{ + minMaxFlag = WHERE_ORDERBY_NORMAL; } - sAggInfo.mxReg = pParse->nMem; + analyzeAggFuncArgs(pAggInfo, &sNC); if( db->mallocFailed ) goto select_end; +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("After aggregate analysis %p:\n", pAggInfo)); + sqlite3TreeViewSelect(0, p, 0); + if( minMaxFlag ){ + sqlite3DebugPrintf("MIN/MAX Optimization (0x%02x) adds:\n", minMaxFlag); + sqlite3TreeViewExprList(0, pMinMaxOrderBy, 0, "ORDERBY"); + } + printAggInfo(pAggInfo); + } +#endif + /* Processing for aggregates with GROUP BY is very different and - ** much more complex than aggregates without a GROUP BY. + ** much more complex than aggregates without a GROUP BY. tag-select-0810 */ if( pGroupBy ){ KeyInfo *pKeyInfo; /* Keying information for the group by clause */ - int addr1; /* A-vs-B comparision jump */ + int addr1; /* A-vs-B comparison jump */ int addrOutputRow; /* Start of subroutine that outputs a result row */ int regOutputRow; /* Return address register for output subroutine */ int addrSetAbort; /* Set the abort flag and return */ @@ -118147,16 +151967,33 @@ SQLITE_PRIVATE int sqlite3Select( int addrSortingIdx; /* The OP_OpenEphemeral for the sorting index */ int addrReset; /* Subroutine for resetting the accumulator */ int regReset; /* Return address register for reset subroutine */ + ExprList *pDistinct = 0; + u16 distFlag = 0; + int eDist = WHERE_DISTINCT_NOOP; + + if( pAggInfo->nFunc==1 + && pAggInfo->aFunc[0].iDistinct>=0 + && ALWAYS(pAggInfo->aFunc[0].pFExpr!=0) + && ALWAYS(ExprUseXList(pAggInfo->aFunc[0].pFExpr)) + && pAggInfo->aFunc[0].pFExpr->x.pList!=0 + ){ + Expr *pExpr = pAggInfo->aFunc[0].pFExpr->x.pList->a[0].pExpr; + pExpr = sqlite3ExprDup(db, pExpr, 0); + pDistinct = sqlite3ExprListDup(db, pGroupBy, 0); + pDistinct = sqlite3ExprListAppend(pParse, pDistinct, pExpr); + distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; + } /* If there is a GROUP BY clause we might need a sorting index to ** implement it. Allocate that sorting index now. If it turns out ** that we do not need it after all, the OP_SorterOpen instruction - ** will be converted into a Noop. + ** will be converted into a Noop. */ - sAggInfo.sortingIdx = pParse->nTab++; - pKeyInfo = keyInfoFromExprList(pParse, pGroupBy, 0, sAggInfo.nColumn); - addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, - sAggInfo.sortingIdx, sAggInfo.nSortingColumn, + pAggInfo->sortingIdx = pParse->nTab++; + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pGroupBy, + 0, pAggInfo->nColumn); + addrSortingIdx = sqlite3VdbeAddOp4(v, OP_SorterOpen, + pAggInfo->sortingIdx, pAggInfo->nSortingColumn, 0, (char*)pKeyInfo, P4_KEYINFO); /* Initialize memory locations used by GROUP BY aggregate processing @@ -118164,17 +152001,15 @@ SQLITE_PRIVATE int sqlite3Select( iUseFlag = ++pParse->nMem; iAbortFlag = ++pParse->nMem; regOutputRow = ++pParse->nMem; - addrOutputRow = sqlite3VdbeMakeLabel(v); + addrOutputRow = sqlite3VdbeMakeLabel(pParse); regReset = ++pParse->nMem; - addrReset = sqlite3VdbeMakeLabel(v); + addrReset = sqlite3VdbeMakeLabel(pParse); iAMem = pParse->nMem + 1; pParse->nMem += pGroupBy->nExpr; iBMem = pParse->nMem + 1; pParse->nMem += pGroupBy->nExpr; sqlite3VdbeAddOp2(v, OP_Integer, 0, iAbortFlag); VdbeComment((v, "clear abort flag")); - sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); - VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp3(v, OP_Null, 0, iAMem, iAMem+pGroupBy->nExpr-1); /* Begin a loop that will extract all source rows in GROUP BY order. @@ -118183,10 +152018,21 @@ SQLITE_PRIVATE int sqlite3Select( ** in the right order to begin with. */ sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, 0, - WHERE_GROUPBY | (orderByGrp ? WHERE_SORTBYGROUP : 0), 0 + TREETRACE(0x2,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pGroupBy, pDistinct, + p, (sDistinct.isTnct==2 ? WHERE_DISTINCTBY : WHERE_GROUPBY) + | (orderByGrp ? WHERE_SORTBYGROUP : 0) | distFlag, 0 ); - if( pWInfo==0 ) goto select_end; + if( pWInfo==0 ){ + sqlite3ExprListDelete(db, pDistinct); + goto select_end; + } + if( pParse->pIdxEpr ){ + optimizeAggregateUseOfIndexedExpr(pParse, p, pAggInfo, &sNC); + } + assignAggregateRegisters(pParse, pAggInfo); + eDist = sqlite3WhereIsDistinct(pWInfo); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); if( sqlite3WhereIsOrdered(pWInfo)==pGroupBy->nExpr ){ /* The optimizer is able to deliver rows in group by order so ** we do not have to sort. The OP_OpenEphemeral table will be @@ -118204,47 +152050,71 @@ SQLITE_PRIVATE int sqlite3Select( int nCol; int nGroupBy; - explainTempTable(pParse, +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp; /* Address of OP_Explain instruction */ +#endif + ExplainQueryPlan2(addrExp, (pParse, 0, "USE TEMP B-TREE FOR %s", (sDistinct.isTnct && (p->selFlags&SF_Distinct)==0) ? - "DISTINCT" : "GROUP BY"); + "DISTINCT" : "GROUP BY" + )); groupBySort = 1; nGroupBy = pGroupBy->nExpr; nCol = nGroupBy; j = nGroupBy; - for(i=0; i=j ){ + for(i=0; inColumn; i++){ + if( pAggInfo->aCol[i].iSorterColumn>=j ){ nCol++; j++; } } regBase = sqlite3GetTempRange(pParse, nCol); - sqlite3ExprCacheClear(pParse); sqlite3ExprCodeExprList(pParse, pGroupBy, regBase, 0, 0); j = nGroupBy; - for(i=0; idirectMode = 1; + for(i=0; inColumn; i++){ + struct AggInfo_col *pCol = &pAggInfo->aCol[i]; if( pCol->iSorterColumn>=j ){ - int r1 = j + regBase; - sqlite3ExprCodeGetColumnToReg(pParse, - pCol->pTab, pCol->iColumn, pCol->iTable, r1); + sqlite3ExprCode(pParse, pCol->pCExpr, j + regBase); j++; } } + pAggInfo->directMode = 0; regRecord = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, 0, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp3(v, OP_MakeRecord, regBase, nCol, regRecord); - sqlite3VdbeAddOp2(v, OP_SorterInsert, sAggInfo.sortingIdx, regRecord); + sqlite3VdbeAddOp2(v, OP_SorterInsert, pAggInfo->sortingIdx, regRecord); + sqlite3VdbeScanStatusRange(v, addrExp, sqlite3VdbeCurrentAddr(v)-2, -1); sqlite3ReleaseTempReg(pParse, regRecord); sqlite3ReleaseTempRange(pParse, regBase, nCol); + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); - sAggInfo.sortingIdxPTab = sortPTab = pParse->nTab++; + pAggInfo->sortingIdxPTab = sortPTab = pParse->nTab++; sortOut = sqlite3GetTempReg(pParse); + sqlite3VdbeScanStatusCounters(v, addrExp, sqlite3VdbeCurrentAddr(v), 0); sqlite3VdbeAddOp3(v, OP_OpenPseudo, sortPTab, sortOut, nCol); - sqlite3VdbeAddOp2(v, OP_SorterSort, sAggInfo.sortingIdx, addrEnd); + sqlite3VdbeAddOp2(v, OP_SorterSort, pAggInfo->sortingIdx, addrEnd); VdbeComment((v, "GROUP BY sort")); VdbeCoverage(v); - sAggInfo.useSortingIdx = 1; - sqlite3ExprCacheClear(pParse); + pAggInfo->useSortingIdx = 1; + sqlite3VdbeScanStatusRange(v, addrExp, -1, sortPTab); + sqlite3VdbeScanStatusRange(v, addrExp, -1, pAggInfo->sortingIdx); + } + /* If there are entries in pAgggInfo->aFunc[] that contain subexpressions + ** that are indexed (and that were previously identified and tagged + ** in optimizeAggregateUseOfIndexedExpr()) then those subexpressions + ** must now be converted into a TK_AGG_COLUMN node so that the value + ** is correctly pulled from the index rather than being recomputed. */ + if( pParse->pIdxEpr ){ + aggregateConvertIndexedExprRefToColumn(pAggInfo); +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20, pParse, p, + ("AggInfo function expressions converted to reference index\n")); + sqlite3TreeViewSelect(0, p, 0); + printAggInfo(pAggInfo); + } +#endif } /* If the index or temporary table used by the GROUP BY sort @@ -118252,9 +152122,9 @@ SQLITE_PRIVATE int sqlite3Select( ** clause, cancel the ephemeral table open coded earlier. ** ** This is an optimization - the correct answer should result regardless. - ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to + ** Use the SQLITE_GroupByOrder flag with SQLITE_TESTCTRL_OPTIMIZER to ** disable this optimization for testing purposes. */ - if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) + if( orderByGrp && OptimizationEnabled(db, SQLITE_GroupByOrder) && (groupBySort || sqlite3WhereIsSorted(pWInfo)) ){ sSort.pOrderBy = 0; @@ -118267,18 +152137,30 @@ SQLITE_PRIVATE int sqlite3Select( ** from the previous row currently stored in a0, a1, a2... */ addrTopOfLoop = sqlite3VdbeCurrentAddr(v); - sqlite3ExprCacheClear(pParse); if( groupBySort ){ - sqlite3VdbeAddOp3(v, OP_SorterData, sAggInfo.sortingIdx, + sqlite3VdbeAddOp3(v, OP_SorterData, pAggInfo->sortingIdx, sortOut, sortPTab); } for(j=0; jnExpr; j++){ + int iOrderByCol = pGroupBy->a[j].u.x.iOrderByCol; + if( groupBySort ){ sqlite3VdbeAddOp3(v, OP_Column, sortPTab, j, iBMem+j); }else{ - sAggInfo.directMode = 1; + pAggInfo->directMode = 1; sqlite3ExprCode(pParse, pGroupBy->a[j].pExpr, iBMem+j); } + + if( iOrderByCol ){ + Expr *pX = p->pEList->a[iOrderByCol-1].pExpr; + Expr *pBase = sqlite3ExprSkipCollateAndLikely(pX); + if( ALWAYS(pBase!=0) + && pBase->op!=TK_AGG_COLUMN + && pBase->op!=TK_REGISTER + ){ + sqlite3ExprToRegister(pX, iAMem+j); + } + } } sqlite3VdbeAddOp4(v, OP_Compare, iAMem, iBMem, pGroupBy->nExpr, (char*)sqlite3KeyInfoRef(pKeyInfo), P4_KEYINFO); @@ -118294,9 +152176,9 @@ SQLITE_PRIVATE int sqlite3Select( ** and resets the aggregate accumulator registers in preparation ** for the next GROUP BY batch. */ - sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_Gosub, regOutputRow, addrOutputRow); VdbeComment((v, "output one row")); + sqlite3ExprCodeMove(pParse, iBMem, iAMem, pGroupBy->nExpr); sqlite3VdbeAddOp2(v, OP_IfPos, iAbortFlag, addrEnd); VdbeCoverage(v); VdbeComment((v, "check abort flag")); sqlite3VdbeAddOp2(v, OP_Gosub, regReset, addrReset); @@ -118306,19 +152188,21 @@ SQLITE_PRIVATE int sqlite3Select( ** the current row */ sqlite3VdbeJumpHere(v, addr1); - updateAccumulator(pParse, &sAggInfo); + updateAccumulator(pParse, iUseFlag, pAggInfo, eDist); sqlite3VdbeAddOp2(v, OP_Integer, 1, iUseFlag); VdbeComment((v, "indicate data in accumulator")); /* End of the loop */ if( groupBySort ){ - sqlite3VdbeAddOp2(v, OP_SorterNext, sAggInfo.sortingIdx, addrTopOfLoop); + sqlite3VdbeAddOp2(v, OP_SorterNext, pAggInfo->sortingIdx,addrTopOfLoop); VdbeCoverage(v); }else{ + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); sqlite3VdbeChangeToNoop(v, addrSortingIdx); } + sqlite3ExprListDelete(db, pDistinct); /* Output the final row of result */ @@ -118346,9 +152230,9 @@ SQLITE_PRIVATE int sqlite3Select( VdbeCoverage(v); VdbeComment((v, "Groupby result generator entry point")); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); - finalizeAggFunctions(pParse, &sAggInfo); + finalizeAggFunctions(pParse, pAggInfo); sqlite3ExprIfFalse(pParse, pHaving, addrOutputRow+1, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, &sSort, + selectInnerLoop(pParse, p, -1, &sSort, &sDistinct, pDest, addrOutputRow+1, addrSetAbort); sqlite3VdbeAddOp1(v, OP_Return, regOutputRow); @@ -118357,16 +152241,23 @@ SQLITE_PRIVATE int sqlite3Select( /* Generate a subroutine that will reset the group-by accumulator */ sqlite3VdbeResolveLabel(v, addrReset); - resetAccumulator(pParse, &sAggInfo); + resetAccumulator(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Integer, 0, iUseFlag); + VdbeComment((v, "indicate accumulator empty")); sqlite3VdbeAddOp1(v, OP_Return, regReset); - + + if( distFlag!=0 && eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = &pAggInfo->aFunc[0]; + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } } /* endif pGroupBy. Begin aggregate queries without GROUP BY: */ else { - ExprList *pDel = 0; -#ifndef SQLITE_OMIT_BTREECOUNT + /* Aggregate functions without GROUP BY. tag-select-0820 */ Table *pTab; - if( (pTab = isSimpleCount(p, &sAggInfo))!=0 ){ - /* If isSimpleCount() returns a pointer to a Table structure, then + if( (pTab = isSimpleCount(p, pAggInfo))!=0 ){ + /* tag-select-0821 + ** + ** If isSimpleCount() returns a pointer to a Table structure, then ** the SQL statement is of the form: ** ** SELECT count(*) FROM @@ -118384,7 +152275,7 @@ SQLITE_PRIVATE int sqlite3Select( Index *pIdx; /* Iterator variable */ KeyInfo *pKeyInfo = 0; /* Keyinfo for scanned index */ Index *pBest = 0; /* Best index found so far */ - int iRoot = pTab->tnum; /* Root page of scanned b-tree */ + Pgno iRoot = pTab->tnum; /* Root page of scanned b-tree */ sqlite3CodeVerifySchema(pParse, iDb); sqlite3TableLock(pParse, iDb, pTab->tnum, 0, pTab->zName); @@ -118395,17 +152286,19 @@ SQLITE_PRIVATE int sqlite3Select( ** ** (2013-10-03) Do not count the entries in a partial index. ** - ** In practice the KeyInfo structure will not be used. It is only + ** In practice the KeyInfo structure will not be used. It is only ** passed to keep OP_OpenRead happy. */ if( !HasRowid(pTab) ) pBest = sqlite3PrimaryKeyIndex(pTab); - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->bUnordered==0 - && pIdx->szIdxRowszTabRow - && pIdx->pPartIdxWhere==0 - && (!pBest || pIdx->szIdxRowszIdxRow) - ){ - pBest = pIdx; + if( !p->pSrc->a[0].fg.notIndexed ){ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->bUnordered==0 + && pIdx->szIdxRowszTabRow + && pIdx->pPartIdxWhere==0 + && (!pBest || pIdx->szIdxRowszIdxRow) + ){ + pBest = pIdx; + } } } if( pBest ){ @@ -118414,90 +152307,98 @@ SQLITE_PRIVATE int sqlite3Select( } /* Open a read-only cursor, execute the OP_Count, close the cursor. */ - sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, iRoot, iDb, 1); + sqlite3VdbeAddOp4Int(v, OP_OpenRead, iCsr, (int)iRoot, iDb, 1); if( pKeyInfo ){ sqlite3VdbeChangeP4(v, -1, (char *)pKeyInfo, P4_KEYINFO); } - sqlite3VdbeAddOp2(v, OP_Count, iCsr, sAggInfo.aFunc[0].iMem); + assignAggregateRegisters(pParse, pAggInfo); + sqlite3VdbeAddOp2(v, OP_Count, iCsr, AggInfoFuncReg(pAggInfo,0)); sqlite3VdbeAddOp1(v, OP_Close, iCsr); explainSimpleCount(pParse, pTab, pBest); - }else -#endif /* SQLITE_OMIT_BTREECOUNT */ - { - /* Check if the query is of one of the following forms: - ** - ** SELECT min(x) FROM ... - ** SELECT max(x) FROM ... - ** - ** If it is, then ask the code in where.c to attempt to sort results - ** as if there was an "ORDER ON x" or "ORDER ON x DESC" clause. - ** If where.c is able to produce results sorted in this order, then - ** add vdbe code to break out of the processing loop after the - ** first iteration (since the first iteration of the loop is - ** guaranteed to operate on the row with the minimum or maximum - ** value of x, the only row required). - ** - ** A special flag must be passed to sqlite3WhereBegin() to slightly - ** modify behavior as follows: - ** - ** + If the query is a "SELECT min(x)", then the loop coded by - ** where.c should not iterate over any values with a NULL value - ** for x. - ** - ** + The optimizer code in where.c (the thing that decides which - ** index or indices to use) should place a different priority on - ** satisfying the 'ORDER BY' clause than it does in other cases. - ** Refer to code and comments in where.c for details. - */ - ExprList *pMinMax = 0; - u8 flag = WHERE_ORDERBY_NORMAL; - - assert( p->pGroupBy==0 ); - assert( flag==0 ); - if( p->pHaving==0 ){ - flag = minMaxQuery(&sAggInfo, &pMinMax); - } - assert( flag==0 || (pMinMax!=0 && pMinMax->nExpr==1) ); + }else{ + /* The general case of an aggregate query without GROUP BY + ** tag-select-0822 */ + int regAcc = 0; /* "populate accumulators" flag */ + ExprList *pDistinct = 0; + u16 distFlag = 0; + int eDist; - if( flag ){ - pMinMax = sqlite3ExprListDup(db, pMinMax, 0); - pDel = pMinMax; - assert( db->mallocFailed || pMinMax!=0 ); - if( !db->mallocFailed ){ - pMinMax->a[0].sortOrder = flag!=WHERE_ORDERBY_MIN ?1:0; - pMinMax->a[0].pExpr->op = TK_COLUMN; + /* If there are accumulator registers but no min() or max() functions + ** without FILTER clauses, allocate register regAcc. Register regAcc + ** will contain 0 the first time the inner loop runs, and 1 thereafter. + ** The code generated by updateAccumulator() uses this to ensure + ** that the accumulator registers are (a) updated only once if + ** there are no min() or max functions or (b) always updated for the + ** first row visited by the aggregate, so that they are updated at + ** least once even if the FILTER clause means the min() or max() + ** function visits zero rows. */ + if( pAggInfo->nAccumulator ){ + for(i=0; inFunc; i++){ + if( ExprHasProperty(pAggInfo->aFunc[i].pFExpr, EP_WinFunc) ){ + continue; + } + if( pAggInfo->aFunc[i].pFunc->funcFlags&SQLITE_FUNC_NEEDCOLL ){ + break; + } } + if( i==pAggInfo->nFunc ){ + regAcc = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 0, regAcc); + } + }else if( pAggInfo->nFunc==1 && pAggInfo->aFunc[0].iDistinct>=0 ){ + assert( ExprUseXList(pAggInfo->aFunc[0].pFExpr) ); + pDistinct = pAggInfo->aFunc[0].pFExpr->x.pList; + distFlag = pDistinct ? (WHERE_WANT_DISTINCT|WHERE_AGG_DISTINCT) : 0; } - + assignAggregateRegisters(pParse, pAggInfo); + /* This case runs if the aggregate has no GROUP BY clause. The ** processing is much simpler since there is only a single row ** of output. */ - resetAccumulator(pParse, &sAggInfo); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMax,0,flag,0); + assert( p->pGroupBy==0 ); + resetAccumulator(pParse, pAggInfo); + + /* If this query is a candidate for the min/max optimization, then + ** minMaxFlag will have been previously set to either + ** WHERE_ORDERBY_MIN or WHERE_ORDERBY_MAX and pMinMaxOrderBy will + ** be an appropriate ORDER BY expression for the optimization. + */ + assert( minMaxFlag==WHERE_ORDERBY_NORMAL || pMinMaxOrderBy!=0 ); + assert( pMinMaxOrderBy==0 || pMinMaxOrderBy->nExpr==1 ); + + TREETRACE(0x2,pParse,p,("WhereBegin\n")); + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, pMinMaxOrderBy, + pDistinct, p, minMaxFlag|distFlag, 0); if( pWInfo==0 ){ - sqlite3ExprListDelete(db, pDel); goto select_end; } - updateAccumulator(pParse, &sAggInfo); - assert( pMinMax==0 || pMinMax->nExpr==1 ); - if( sqlite3WhereIsOrdered(pWInfo)>0 ){ - sqlite3VdbeGoto(v, sqlite3WhereBreakLabel(pWInfo)); - VdbeComment((v, "%s() by index", - (flag==WHERE_ORDERBY_MIN?"min":"max"))); + TREETRACE(0x2,pParse,p,("WhereBegin returns\n")); + eDist = sqlite3WhereIsDistinct(pWInfo); + updateAccumulator(pParse, regAcc, pAggInfo, eDist); + if( eDist!=WHERE_DISTINCT_NOOP ){ + struct AggInfo_func *pF = pAggInfo->aFunc; + if( pF ){ + fixDistinctOpenEph(pParse, eDist, pF->iDistinct, pF->iDistAddr); + } } + + if( regAcc ) sqlite3VdbeAddOp2(v, OP_Integer, 1, regAcc); + if( minMaxFlag ){ + sqlite3WhereMinMaxOptEarlyOut(v, pWInfo); + } + TREETRACE(0x2,pParse,p,("WhereEnd\n")); sqlite3WhereEnd(pWInfo); - finalizeAggFunctions(pParse, &sAggInfo); + finalizeAggFunctions(pParse, pAggInfo); } sSort.pOrderBy = 0; sqlite3ExprIfFalse(pParse, pHaving, addrEnd, SQLITE_JUMPIFNULL); - selectInnerLoop(pParse, p, p->pEList, -1, 0, 0, + selectInnerLoop(pParse, p, -1, 0, 0, pDest, addrEnd, addrEnd); - sqlite3ExprListDelete(db, pDel); } sqlite3VdbeResolveLabel(v, addrEnd); - + } /* endif aggregate query */ if( sDistinct.eTnctType==WHERE_DISTINCT_UNORDERED ){ @@ -118505,11 +152406,10 @@ SQLITE_PRIVATE int sqlite3Select( } /* If there is an ORDER BY clause, then we need to sort the results - ** and send them to the callback one by one. + ** and send them to the callback one by one. tag-select-0900 */ if( sSort.pOrderBy ){ - explainTempTable(pParse, - sSort.nOBSat>0 ? "RIGHT PART OF ORDER BY":"ORDER BY"); + assert( p->pEList==pEList ); generateSortTail(pParse, p, &sSort, pEList->nExpr, pDest); } @@ -118525,20 +152425,40 @@ SQLITE_PRIVATE int sqlite3Select( ** successful coding of the SELECT. */ select_end: - explainSetInteger(pParse->iSelectId, iRestoreSelectId); - - /* Identify column names if results of the SELECT are to be output. - */ - if( rc==SQLITE_OK && pDest->eDest==SRT_Output ){ - generateColumnNames(pParse, pTabList, pEList); - } - - sqlite3DbFree(db, sAggInfo.aCol); - sqlite3DbFree(db, sAggInfo.aFunc); -#if SELECTTRACE_ENABLED - SELECTTRACE(1,pParse,p,("end processing\n")); - pParse->nSelectIndent--; + assert( db->mallocFailed==0 || db->mallocFailed==1 ); + assert( db->mallocFailed==0 || pParse->nErr!=0 ); + sqlite3ExprListDelete(db, pMinMaxOrderBy); +#ifdef SQLITE_DEBUG + /* Internal self-checks. tag-select-1000 */ + if( pAggInfo && !db->mallocFailed ){ +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x20 ){ + TREETRACE(0x20,pParse,p,("Finished with AggInfo\n")); + printAggInfo(pAggInfo); + } #endif + for(i=0; inColumn; i++){ + Expr *pExpr = pAggInfo->aCol[i].pCExpr; + if( pExpr==0 ) continue; + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + for(i=0; inFunc; i++){ + Expr *pExpr = pAggInfo->aFunc[i].pFExpr; + assert( pExpr!=0 ); + assert( pExpr->pAggInfo==pAggInfo ); + assert( pExpr->iAgg==i ); + } + } +#endif + +#if TREETRACE_ENABLED + TREETRACE(0x1,pParse,p,("end processing\n")); + if( (sqlite3TreeTrace & 0x40000)!=0 && ExplainQueryPlanParent(pParse)==0 ){ + sqlite3TreeViewSelect(0, p, 0); + } +#endif + ExplainQueryPlanPop(pParse); return rc; } @@ -118563,8 +152483,6 @@ select_end: ** if they are not used. */ /* #include "sqliteInt.h" */ -/* #include */ -/* #include */ #ifndef SQLITE_OMIT_GET_TABLE @@ -118604,7 +152522,7 @@ static int sqlite3_get_table_cb(void *pArg, int nCol, char **argv, char **colv){ if( p->nData + need > p->nAlloc ){ char **azNew; p->nAlloc = p->nAlloc*2 + need; - azNew = sqlite3_realloc64( p->azResult, sizeof(char*)*p->nAlloc ); + azNew = sqlite3Realloc( p->azResult, sizeof(char*)*p->nAlloc ); if( azNew==0 ) goto malloc_failed; p->azResult = azNew; } @@ -118657,11 +152575,11 @@ malloc_failed: ** at the conclusion of the call. ** ** The result that is written to ***pazResult is held in memory obtained -** from malloc(). But the caller cannot free this memory directly. +** from malloc(). But the caller cannot free this memory directly. ** Instead, the entire table should be passed to sqlite3_free_table() when ** the calling procedure is finished using it. */ -SQLITE_API int SQLITE_STDCALL sqlite3_get_table( +SQLITE_API int sqlite3_get_table( sqlite3 *db, /* The database on which the SQL executes */ const char *zSql, /* The SQL to be executed */ char ***pazResult, /* Write the result table here */ @@ -118713,7 +152631,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_table( } if( res.nAlloc>res.nData ){ char **azNew; - azNew = sqlite3_realloc64( res.azResult, sizeof(char*)*res.nData ); + azNew = sqlite3Realloc( res.azResult, sizeof(char*)*res.nData ); if( azNew==0 ){ sqlite3_free_table(&res.azResult[1]); db->errCode = SQLITE_NOMEM; @@ -118730,7 +152648,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_table( /* ** This routine frees the space the sqlite3_get_table() malloced. */ -SQLITE_API void SQLITE_STDCALL sqlite3_free_table( +SQLITE_API void sqlite3_free_table( char **azResult /* Result returned from sqlite3_get_table() */ ){ if( azResult ){ @@ -118774,13 +152692,16 @@ SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerS sqlite3ExprListDelete(db, pTmp->pExprList); sqlite3SelectDelete(db, pTmp->pSelect); sqlite3IdListDelete(db, pTmp->pIdList); + sqlite3UpsertDelete(db, pTmp->pUpsert); + sqlite3SrcListDelete(db, pTmp->pFrom); + sqlite3DbFree(db, pTmp->zSpan); sqlite3DbFree(db, pTmp); } } /* -** Given table pTab, return a list of all the triggers attached to +** Given table pTab, return a list of all the triggers attached to ** the table. The list is connected by Trigger.pNext pointers. ** ** All of the triggers on pTab that are in the same database as pTab @@ -118794,28 +152715,48 @@ SQLITE_PRIVATE void sqlite3DeleteTriggerStep(sqlite3 *db, TriggerStep *pTriggerS ** pTab as well as the triggers lised in pTab->pTrigger. */ SQLITE_PRIVATE Trigger *sqlite3TriggerList(Parse *pParse, Table *pTab){ - Schema * const pTmpSchema = pParse->db->aDb[1].pSchema; - Trigger *pList = 0; /* List of triggers to return */ + Schema *pTmpSchema; /* Schema of the pTab table */ + Trigger *pList; /* List of triggers to return */ + HashElem *p; /* Loop variable for TEMP triggers */ - if( pParse->disableTriggers ){ - return 0; - } - - if( pTmpSchema!=pTab->pSchema ){ - HashElem *p; - assert( sqlite3SchemaMutexHeld(pParse->db, 0, pTmpSchema) ); - for(p=sqliteHashFirst(&pTmpSchema->trigHash); p; p=sqliteHashNext(p)){ - Trigger *pTrig = (Trigger *)sqliteHashData(p); - if( pTrig->pTabSchema==pTab->pSchema - && 0==sqlite3StrICmp(pTrig->table, pTab->zName) - ){ - pTrig->pNext = (pList ? pList : pTab->pTrigger); - pList = pTrig; - } + assert( pParse->disableTriggers==0 ); + pTmpSchema = pParse->db->aDb[1].pSchema; + p = sqliteHashFirst(&pTmpSchema->trigHash); + pList = pTab->pTrigger; + while( p ){ + Trigger *pTrig = (Trigger *)sqliteHashData(p); + if( pTrig->pTabSchema==pTab->pSchema + && pTrig->table + && 0==sqlite3StrICmp(pTrig->table, pTab->zName) + && (pTrig->pTabSchema!=pTmpSchema || pTrig->bReturning) + ){ + pTrig->pNext = pList; + pList = pTrig; + }else if( pTrig->op==TK_RETURNING ){ +#ifndef SQLITE_OMIT_VIRTUALTABLE + assert( pParse->db->pVtabCtx==0 ); +#endif + assert( pParse->bReturning ); + assert( &(pParse->u1.pReturning->retTrig) == pTrig ); + pTrig->table = pTab->zName; + pTrig->pTabSchema = pTab->pSchema; + pTrig->pNext = pList; + pList = pTrig; } + p = sqliteHashNext(p); } - - return (pList ? pList : pTab->pTrigger); +#if 0 + if( pList ){ + Trigger *pX; + printf("Triggers for %s:", pTab->zName); + for(pX=pList; pX; pX=pX->pNext){ + printf(" %s", pX->zName); + } + printf("\n"); + fflush(stdout); + } +#endif + return pList; } /* @@ -118845,7 +152786,6 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( int iDb; /* The database to store the trigger in */ Token *pName; /* The unqualified db name */ DbFixer sFix; /* State vector for the DB fixer */ - int iTabDb; /* Index of the database holding pTab */ assert( pName1!=0 ); /* pName1->z might be NULL, but not pName1 itself */ assert( pName2!=0 ); @@ -118876,11 +152816,13 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( ** ^^^^^^^^ ** ** To maintain backwards compatibility, ignore the database - ** name on pTableName if we are reparsing out of SQLITE_MASTER. + ** name on pTableName if we are reparsing out of the schema table */ if( db->init.busy && iDb!=1 ){ - sqlite3DbFree(db, pTableName->a[0].zDatabase); - pTableName->a[0].zDatabase = 0; + assert( pTableName->a[0].fg.fixedSchema==0 ); + assert( pTableName->a[0].fg.isSubquery==0 ); + sqlite3DbFree(db, pTableName->a[0].u4.zDatabase); + pTableName->a[0].u4.zDatabase = 0; } /* If the trigger name was unqualified, and the table is a temp table, @@ -118904,39 +152846,39 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( pTab = sqlite3SrcListLookup(pParse, pTableName); if( !pTab ){ /* The table does not exist. */ - if( db->init.iDb==1 ){ - /* Ticket #3810. - ** Normally, whenever a table is dropped, all associated triggers are - ** dropped too. But if a TEMP trigger is created on a non-TEMP table - ** and the table is dropped by a different database connection, the - ** trigger is not visible to the database connection that does the - ** drop so the trigger cannot be dropped. This results in an - ** "orphaned trigger" - a trigger whose associated table is missing. - */ - db->init.orphanTrigger = 1; - } - goto trigger_cleanup; + goto trigger_orphan_error; } if( IsVirtual(pTab) ){ sqlite3ErrorMsg(pParse, "cannot create triggers on virtual tables"); - goto trigger_cleanup; + goto trigger_orphan_error; + } + if( (pTab->tabFlags & TF_Shadow)!=0 && sqlite3ReadOnlyShadowTables(db) ){ + sqlite3ErrorMsg(pParse, "cannot create triggers on shadow tables"); + goto trigger_orphan_error; } /* Check that the trigger name is not reserved and that no trigger of the ** specified name exists */ zName = sqlite3NameFromToken(db, pName); - if( !zName || SQLITE_OK!=sqlite3CheckObjectName(pParse, zName) ){ + if( zName==0 ){ + assert( db->mallocFailed ); + goto trigger_cleanup; + } + if( sqlite3CheckObjectName(pParse, zName, "trigger", pTab->zName) ){ goto trigger_cleanup; } assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); - if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ - if( !noErr ){ - sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); - }else{ - assert( !db->init.busy ); - sqlite3CodeVerifySchema(pParse, iDb); + if( !IN_RENAME_OBJECT ){ + if( sqlite3HashFind(&(db->aDb[iDb].pSchema->trigHash),zName) ){ + if( !noErr ){ + sqlite3ErrorMsg(pParse, "trigger %T already exists", pName); + }else{ + assert( !db->init.busy ); + sqlite3CodeVerifySchema(pParse, iDb); + VVA_ONLY( pParse->ifNotExists = 1; ) + } + goto trigger_cleanup; } - goto trigger_cleanup; } /* Do not create a trigger on a system table */ @@ -118948,23 +152890,23 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( /* INSTEAD of triggers are only for views and views only support INSTEAD ** of triggers. */ - if( pTab->pSelect && tr_tm!=TK_INSTEAD ){ - sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", - (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName, 0); - goto trigger_cleanup; + if( IsView(pTab) && tr_tm!=TK_INSTEAD ){ + sqlite3ErrorMsg(pParse, "cannot create %s trigger on view: %S", + (tr_tm == TK_BEFORE)?"BEFORE":"AFTER", pTableName->a); + goto trigger_orphan_error; } - if( !pTab->pSelect && tr_tm==TK_INSTEAD ){ + if( !IsView(pTab) && tr_tm==TK_INSTEAD ){ sqlite3ErrorMsg(pParse, "cannot create INSTEAD OF" - " trigger on table: %S", pTableName, 0); - goto trigger_cleanup; + " trigger on table: %S", pTableName->a); + goto trigger_orphan_error; } - iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( !IN_RENAME_OBJECT ){ + int iTabDb = sqlite3SchemaToIndex(db, pTab->pSchema); int code = SQLITE_CREATE_TRIGGER; - const char *zDb = db->aDb[iTabDb].zName; - const char *zDbTrig = isTemp ? db->aDb[1].zName : zDb; + const char *zDb = db->aDb[iTabDb].zDbSName; + const char *zDbTrig = isTemp ? db->aDb[1].zDbSName : zDb; if( iTabDb==1 || isTemp ) code = SQLITE_CREATE_TEMP_TRIGGER; if( sqlite3AuthCheck(pParse, code, zName, pTab->zName, zDbTrig) ){ goto trigger_cleanup; @@ -118994,8 +152936,15 @@ SQLITE_PRIVATE void sqlite3BeginTrigger( pTrigger->pTabSchema = pTab->pSchema; pTrigger->op = (u8)op; pTrigger->tr_tm = tr_tm==TK_BEFORE ? TRIGGER_BEFORE : TRIGGER_AFTER; - pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); - pTrigger->pColumns = sqlite3IdListDup(db, pColumns); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, pTrigger->table, pTableName->a[0].zName); + pTrigger->pWhen = pWhen; + pWhen = 0; + }else{ + pTrigger->pWhen = sqlite3ExprDup(db, pWhen, EXPRDUP_REDUCE); + } + pTrigger->pColumns = pColumns; + pColumns = 0; assert( pParse->pNewTrigger==0 ); pParse->pNewTrigger = pTrigger; @@ -119009,6 +152958,23 @@ trigger_cleanup: }else{ assert( pParse->pNewTrigger==pTrigger ); } + return; + +trigger_orphan_error: + if( db->init.iDb==1 ){ + /* Ticket #3810. + ** Normally, whenever a table is dropped, all associated triggers are + ** dropped too. But if a TEMP trigger is created on a non-TEMP table + ** and the table is dropped by a different database connection, the + ** trigger is not visible to the database connection that does the + ** drop so the trigger cannot be dropped. This results in an + ** "orphaned trigger" - a trigger whose associated table is missing. + ** + ** 2020-11-05 see also https://sqlite.org/forum/forumpost/157dc791df + */ + db->init.orphanTrigger = 1; + } + goto trigger_cleanup; } /* @@ -119038,38 +153004,66 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( } sqlite3TokenInit(&nameToken, pTrig->zName); sqlite3FixInit(&sFix, pParse, iDb, "trigger", &nameToken); - if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) - || sqlite3FixExpr(&sFix, pTrig->pWhen) + if( sqlite3FixTriggerStep(&sFix, pTrig->step_list) + || sqlite3FixExpr(&sFix, pTrig->pWhen) ){ goto triggerfinish_cleanup; } +#ifndef SQLITE_OMIT_ALTERTABLE + if( IN_RENAME_OBJECT ){ + assert( !db->init.busy ); + pParse->pNewTrigger = pTrig; + pTrig = 0; + }else +#endif + /* if we are not initializing, - ** build the sqlite_master entry + ** build the sqlite_schema entry */ if( !db->init.busy ){ Vdbe *v; char *z; - /* Make an entry in the sqlite_master table */ + /* If this is a new CREATE TABLE statement, and if shadow tables + ** are read-only, and the trigger makes a change to a shadow table, + ** then raise an error - do not allow the trigger to be created. */ + if( sqlite3ReadOnlyShadowTables(db) ){ + TriggerStep *pStep; + for(pStep=pTrig->step_list; pStep; pStep=pStep->pNext){ + if( pStep->zTarget!=0 + && sqlite3ShadowTableName(db, pStep->zTarget) + ){ + sqlite3ErrorMsg(pParse, + "trigger \"%s\" may not write to shadow table \"%s\"", + pTrig->zName, pStep->zTarget); + goto triggerfinish_cleanup; + } + } + } + + /* Make an entry in the sqlite_schema table */ v = sqlite3GetVdbe(pParse); if( v==0 ) goto triggerfinish_cleanup; sqlite3BeginWriteOperation(pParse, 0, iDb); z = sqlite3DbStrNDup(db, (char*)pAll->z, pAll->n); + testcase( z==0 ); sqlite3NestedParse(pParse, - "INSERT INTO %Q.%s VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), zName, + "INSERT INTO %Q." LEGACY_SCHEMA_TABLE + " VALUES('trigger',%Q,%Q,0,'CREATE TRIGGER %q')", + db->aDb[iDb].zDbSName, zName, pTrig->table, z); sqlite3DbFree(db, z); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddParseSchemaOp(v, iDb, - sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName)); + sqlite3MPrintf(db, "type='trigger' AND name='%q'", zName), 0); } if( db->init.busy ){ Trigger *pLink = pTrig; Hash *pHash = &db->aDb[iDb].pSchema->trigHash; assert( sqlite3SchemaMutexHeld(db, iDb, 0) ); + assert( pLink!=0 ); pTrig = sqlite3HashInsert(pHash, zName, pTrig); if( pTrig ){ sqlite3OomFault(db); @@ -119084,18 +153078,34 @@ SQLITE_PRIVATE void sqlite3FinishTrigger( triggerfinish_cleanup: sqlite3DeleteTrigger(db, pTrig); - assert( !pParse->pNewTrigger ); + assert( IN_RENAME_OBJECT || !pParse->pNewTrigger ); sqlite3DeleteTriggerStep(db, pStepList); } +/* +** Duplicate a range of text from an SQL statement, then convert all +** whitespace characters into ordinary space characters. +*/ +static char *triggerSpanDup(sqlite3 *db, const char *zStart, const char *zEnd){ + char *z = sqlite3DbSpanDup(db, zStart, zEnd); + int i; + if( z ) for(i=0; z[i]; i++) if( sqlite3Isspace(z[i]) ) z[i] = ' '; + return z; +} + /* ** Turn a SELECT statement (that the pSelect parameter points to) into ** a trigger step. Return a pointer to a TriggerStep structure. ** ** The parser calls this routine when it finds a SELECT statement in -** body of a TRIGGER. +** body of a TRIGGER. */ -SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelect){ +SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep( + sqlite3 *db, /* Database connection */ + Select *pSelect, /* The SELECT statement */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ +){ TriggerStep *pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep)); if( pTriggerStep==0 ) { sqlite3SelectDelete(db, pSelect); @@ -119104,6 +153114,7 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelec pTriggerStep->op = TK_SELECT; pTriggerStep->pSelect = pSelect; pTriggerStep->orconf = OE_Default; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); return pTriggerStep; } @@ -119114,12 +153125,16 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerSelectStep(sqlite3 *db, Select *pSelec ** If an OOM error occurs, NULL is returned and db->mallocFailed is set. */ static TriggerStep *triggerStepAllocate( - sqlite3 *db, /* Database connection */ + Parse *pParse, /* Parser context */ u8 op, /* Trigger opcode */ - Token *pName /* The target name */ + Token *pName, /* The target name */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; + if( pParse->nErr ) return 0; pTriggerStep = sqlite3DbMallocZero(db, sizeof(TriggerStep) + pName->n + 1); if( pTriggerStep ){ char *z = (char*)&pTriggerStep[1]; @@ -119127,6 +153142,10 @@ static TriggerStep *triggerStepAllocate( sqlite3Dequote(z); pTriggerStep->zTarget = z; pTriggerStep->op = op; + pTriggerStep->zSpan = triggerSpanDup(db, zStart, zEnd); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenMap(pParse, pTriggerStep->zTarget, pName); + } } return pTriggerStep; } @@ -119139,23 +153158,39 @@ static TriggerStep *triggerStepAllocate( ** body of a trigger. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( - sqlite3 *db, /* The database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table into which we insert */ IdList *pColumn, /* List of columns in pTableName to insert into */ Select *pSelect, /* A SELECT statement that supplies values */ - u8 orconf /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ + u8 orconf, /* The conflict algorithm (OE_Abort, OE_Replace, etc.) */ + Upsert *pUpsert, /* ON CONFLICT clauses for upsert */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; assert(pSelect != 0 || db->mallocFailed); - pTriggerStep = triggerStepAllocate(db, TK_INSERT, pTableName); + pTriggerStep = triggerStepAllocate(pParse, TK_INSERT, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pSelect = pSelect; + pSelect = 0; + }else{ + pTriggerStep->pSelect = sqlite3SelectDup(db, pSelect, EXPRDUP_REDUCE); + } pTriggerStep->pIdList = pColumn; + pTriggerStep->pUpsert = pUpsert; pTriggerStep->orconf = orconf; + if( pUpsert ){ + sqlite3HasExplicitNulls(pParse, pUpsert->pUpsertTarget); + } }else{ + testcase( pColumn ); sqlite3IdListDelete(db, pColumn); + testcase( pUpsert ); + sqlite3UpsertDelete(db, pUpsert); } sqlite3SelectDelete(db, pSelect); @@ -119168,22 +153203,37 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerInsertStep( ** sees an UPDATE statement inside the body of a CREATE TRIGGER. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( - sqlite3 *db, /* The database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* Name of the table to be updated */ + SrcList *pFrom, /* FROM clause for an UPDATE-FROM, or NULL */ ExprList *pEList, /* The SET clause: list of column and new values */ Expr *pWhere, /* The WHERE clause */ - u8 orconf /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ + u8 orconf, /* The conflict algorithm. (OE_Abort, OE_Ignore, etc) */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(db, TK_UPDATE, pTableName); + pTriggerStep = triggerStepAllocate(pParse, TK_UPDATE, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pExprList = pEList; + pTriggerStep->pWhere = pWhere; + pTriggerStep->pFrom = pFrom; + pEList = 0; + pWhere = 0; + pFrom = 0; + }else{ + pTriggerStep->pExprList = sqlite3ExprListDup(db, pEList, EXPRDUP_REDUCE); + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + pTriggerStep->pFrom = sqlite3SrcListDup(db, pFrom, EXPRDUP_REDUCE); + } pTriggerStep->orconf = orconf; } sqlite3ExprListDelete(db, pEList); sqlite3ExprDelete(db, pWhere); + sqlite3SrcListDelete(db, pFrom); return pTriggerStep; } @@ -119193,26 +153243,34 @@ SQLITE_PRIVATE TriggerStep *sqlite3TriggerUpdateStep( ** sees a DELETE statement inside the body of a CREATE TRIGGER. */ SQLITE_PRIVATE TriggerStep *sqlite3TriggerDeleteStep( - sqlite3 *db, /* Database connection */ + Parse *pParse, /* Parser */ Token *pTableName, /* The table from which rows are deleted */ - Expr *pWhere /* The WHERE clause */ + Expr *pWhere, /* The WHERE clause */ + const char *zStart, /* Start of SQL text */ + const char *zEnd /* End of SQL text */ ){ + sqlite3 *db = pParse->db; TriggerStep *pTriggerStep; - pTriggerStep = triggerStepAllocate(db, TK_DELETE, pTableName); + pTriggerStep = triggerStepAllocate(pParse, TK_DELETE, pTableName,zStart,zEnd); if( pTriggerStep ){ - pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + if( IN_RENAME_OBJECT ){ + pTriggerStep->pWhere = pWhere; + pWhere = 0; + }else{ + pTriggerStep->pWhere = sqlite3ExprDup(db, pWhere, EXPRDUP_REDUCE); + } pTriggerStep->orconf = OE_Default; } sqlite3ExprDelete(db, pWhere); return pTriggerStep; } -/* +/* ** Recursively delete a Trigger structure */ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3 *db, Trigger *pTrigger){ - if( pTrigger==0 ) return; + if( pTrigger==0 || pTrigger->bReturning ) return; sqlite3DeleteTriggerStep(db, pTrigger->step_list); sqlite3DbFree(db, pTrigger->zName); sqlite3DbFree(db, pTrigger->table); @@ -119222,7 +153280,7 @@ SQLITE_PRIVATE void sqlite3DeleteTrigger(sqlite3 *db, Trigger *pTrigger){ } /* -** This function is called to drop a trigger from the database schema. +** This function is called to drop a trigger from the database schema. ** ** This may be called directly from the parser and therefore identifies ** the trigger by name. The sqlite3DropTriggerPtr() routine does the @@ -119242,19 +153300,20 @@ SQLITE_PRIVATE void sqlite3DropTrigger(Parse *pParse, SrcList *pName, int noErr) } assert( pName->nSrc==1 ); - zDb = pName->a[0].zDatabase; + assert( pName->a[0].fg.fixedSchema==0 && pName->a[0].fg.isSubquery==0 ); + zDb = pName->a[0].u4.zDatabase; zName = pName->a[0].zName; assert( zDb!=0 || sqlite3BtreeHoldsAllMutexes(db) ); for(i=OMIT_TEMPDB; inDb; i++){ int j = (i<2) ? i^1 : i; /* Search TEMP before MAIN */ - if( zDb && sqlite3StrICmp(db->aDb[j].zName, zDb) ) continue; + if( zDb && sqlite3DbIsNamed(db, j, zDb)==0 ) continue; assert( sqlite3SchemaMutexHeld(db, j, 0) ); pTrigger = sqlite3HashFind(&(db->aDb[j].pSchema->trigHash), zName); if( pTrigger ) break; } if( !pTrigger ){ if( !noErr ){ - sqlite3ErrorMsg(pParse, "no such trigger: %S", pName, 0); + sqlite3ErrorMsg(pParse, "no such trigger: %S", pName->a); }else{ sqlite3CodeVerifyNamedSchema(pParse, zDb); } @@ -119277,7 +153336,7 @@ static Table *tableOfTrigger(Trigger *pTrigger){ /* -** Drop a trigger given a pointer to that trigger. +** Drop a trigger given a pointer to that trigger. */ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ Table *pTable; @@ -119288,12 +153347,11 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ iDb = sqlite3SchemaToIndex(pParse->db, pTrigger->pSchema); assert( iDb>=0 && iDbnDb ); pTable = tableOfTrigger(pTrigger); - assert( pTable ); - assert( pTable->pSchema==pTrigger->pSchema || iDb==1 ); + assert( (pTable && pTable->pSchema==pTrigger->pSchema) || iDb==1 ); #ifndef SQLITE_OMIT_AUTHORIZATION - { + if( pTable ){ int code = SQLITE_DROP_TRIGGER; - const char *zDb = db->aDb[iDb].zName; + const char *zDb = db->aDb[iDb].zDbSName; const char *zTab = SCHEMA_TABLE(iDb); if( iDb==1 ) code = SQLITE_DROP_TEMP_TRIGGER; if( sqlite3AuthCheck(pParse, code, pTrigger->zName, pTable->zName, zDb) || @@ -119305,11 +153363,10 @@ SQLITE_PRIVATE void sqlite3DropTriggerPtr(Parse *pParse, Trigger *pTrigger){ /* Generate code to destroy the database record of the trigger. */ - assert( pTable!=0 ); if( (v = sqlite3GetVdbe(pParse))!=0 ){ sqlite3NestedParse(pParse, - "DELETE FROM %Q.%s WHERE name=%Q AND type='trigger'", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), pTrigger->zName + "DELETE FROM %Q." LEGACY_SCHEMA_TABLE " WHERE name=%Q AND type='trigger'", + db->aDb[iDb].zDbSName, pTrigger->zName ); sqlite3ChangeCookie(pParse, iDb); sqlite3VdbeAddOp4(v, OP_DropTrigger, iDb, 0, 0, pTrigger->zName, 0); @@ -119329,12 +153386,18 @@ SQLITE_PRIVATE void sqlite3UnlinkAndDeleteTrigger(sqlite3 *db, int iDb, const ch if( ALWAYS(pTrigger) ){ if( pTrigger->pSchema==pTrigger->pTabSchema ){ Table *pTab = tableOfTrigger(pTrigger); - Trigger **pp; - for(pp=&pTab->pTrigger; *pp!=pTrigger; pp=&((*pp)->pNext)); - *pp = (*pp)->pNext; + if( pTab ){ + Trigger **pp; + for(pp=&pTab->pTrigger; *pp; pp=&((*pp)->pNext)){ + if( *pp==pTrigger ){ + *pp = (*pp)->pNext; + break; + } + } + } } sqlite3DeleteTrigger(db, pTrigger); - db->flags |= SQLITE_InternChanges; + db->mDbFlags |= DBFLAG_SchemaChange; } } @@ -119351,18 +153414,27 @@ static int checkColumnOverlap(IdList *pIdList, ExprList *pEList){ int e; if( pIdList==0 || NEVER(pEList==0) ) return 1; for(e=0; enExpr; e++){ - if( sqlite3IdListIndex(pIdList, pEList->a[e].zName)>=0 ) return 1; + if( sqlite3IdListIndex(pIdList, pEList->a[e].zEName)>=0 ) return 1; } - return 0; + return 0; +} + +/* +** Return true if any TEMP triggers exist +*/ +static int tempTriggersExist(sqlite3 *db){ + if( NEVER(db->aDb[1].pSchema==0) ) return 0; + if( sqliteHashFirst(&db->aDb[1].pSchema->trigHash)==0 ) return 0; + return 1; } /* ** Return a list of all triggers on table pTab if there exists at least -** one trigger that must be fired when an operation of type 'op' is +** one trigger that must be fired when an operation of type 'op' is ** performed on the table, and, if that operation is an UPDATE, if at ** least one of the columns in pChanges is being modified. */ -SQLITE_PRIVATE Trigger *sqlite3TriggersExist( +static SQLITE_NOINLINE Trigger *triggersReallyExist( Parse *pParse, /* Parse context */ Table *pTab, /* The table the contains the triggers */ int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ @@ -119373,20 +153445,74 @@ SQLITE_PRIVATE Trigger *sqlite3TriggersExist( Trigger *pList = 0; Trigger *p; - if( (pParse->db->flags & SQLITE_EnableTrigger)!=0 ){ - pList = sqlite3TriggerList(pParse, pTab); - } - assert( pList==0 || IsVirtual(pTab)==0 ); - for(p=pList; p; p=p->pNext){ - if( p->op==op && checkColumnOverlap(p->pColumns, pChanges) ){ - mask |= p->tr_tm; + pList = sqlite3TriggerList(pParse, pTab); + assert( pList==0 || IsVirtual(pTab)==0 + || (pList->bReturning && pList->pNext==0) ); + if( pList!=0 ){ + p = pList; + if( (pParse->db->flags & SQLITE_EnableTrigger)==0 + && pTab->pTrigger!=0 + ){ + /* The SQLITE_DBCONFIG_ENABLE_TRIGGER setting is off. That means that + ** only TEMP triggers are allowed. Truncate the pList so that it + ** includes only TEMP triggers */ + if( pList==pTab->pTrigger ){ + pList = 0; + goto exit_triggers_exist; + } + while( ALWAYS(p->pNext) && p->pNext!=pTab->pTrigger ) p = p->pNext; + p->pNext = 0; + p = pList; } + do{ + if( p->op==op && checkColumnOverlap(p->pColumns, pChanges) ){ + mask |= p->tr_tm; + }else if( p->op==TK_RETURNING ){ + /* The first time a RETURNING trigger is seen, the "op" value tells + ** us what time of trigger it should be. */ + assert( sqlite3IsToplevel(pParse) ); + p->op = op; + if( IsVirtual(pTab) ){ + if( op!=TK_INSERT ){ + sqlite3ErrorMsg(pParse, + "%s RETURNING is not available on virtual tables", + op==TK_DELETE ? "DELETE" : "UPDATE"); + } + p->tr_tm = TRIGGER_BEFORE; + }else{ + p->tr_tm = TRIGGER_AFTER; + } + mask |= p->tr_tm; + }else if( p->bReturning && p->op==TK_INSERT && op==TK_UPDATE + && sqlite3IsToplevel(pParse) ){ + /* Also fire a RETURNING trigger for an UPSERT */ + mask |= p->tr_tm; + } + p = p->pNext; + }while( p ); } +exit_triggers_exist: if( pMask ){ *pMask = mask; } return (mask ? pList : 0); } +SQLITE_PRIVATE Trigger *sqlite3TriggersExist( + Parse *pParse, /* Parse context */ + Table *pTab, /* The table the contains the triggers */ + int op, /* one of TK_DELETE, TK_INSERT, TK_UPDATE */ + ExprList *pChanges, /* Columns that change in an UPDATE statement */ + int *pMask /* OUT: Mask of TRIGGER_BEFORE|TRIGGER_AFTER */ +){ + assert( pTab!=0 ); + if( (pTab->pTrigger==0 && !tempTriggersExist(pParse->db)) + || pParse->disableTriggers + ){ + if( pMask ) *pMask = 0; + return 0; + } + return triggersReallyExist(pParse,pTab,op,pChanges,pMask); +} /* ** Convert the pStep->zTarget string into a SrcList and return a pointer @@ -119398,35 +153524,265 @@ SQLITE_PRIVATE Trigger *sqlite3TriggersExist( ** trigger is in TEMP in which case it can refer to any other database it ** wants. */ -static SrcList *targetSrcList( +SQLITE_PRIVATE SrcList *sqlite3TriggerStepSrc( Parse *pParse, /* The parsing context */ TriggerStep *pStep /* The trigger containing the target token */ ){ sqlite3 *db = pParse->db; - int iDb; /* Index of the database to use */ - SrcList *pSrc; /* SrcList to be returned */ - - pSrc = sqlite3SrcListAppend(db, 0, 0, 0); + SrcList *pSrc; /* SrcList to be returned */ + char *zName = sqlite3DbStrDup(db, pStep->zTarget); + pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + assert( pSrc==0 || pSrc->nSrc==1 ); + assert( zName || pSrc==0 ); if( pSrc ){ - assert( pSrc->nSrc>0 ); - pSrc->a[pSrc->nSrc-1].zName = sqlite3DbStrDup(db, pStep->zTarget); - iDb = sqlite3SchemaToIndex(db, pStep->pTrig->pSchema); - if( iDb==0 || iDb>=2 ){ - assert( iDbnDb ); - pSrc->a[pSrc->nSrc-1].zDatabase = sqlite3DbStrDup(db, db->aDb[iDb].zName); + Schema *pSchema = pStep->pTrig->pSchema; + pSrc->a[0].zName = zName; + if( pSchema!=db->aDb[1].pSchema ){ + assert( pSrc->a[0].fg.fixedSchema || pSrc->a[0].u4.zDatabase==0 ); + pSrc->a[0].u4.pSchema = pSchema; + pSrc->a[0].fg.fixedSchema = 1; } + if( pStep->pFrom ){ + SrcList *pDup = sqlite3SrcListDup(db, pStep->pFrom, 0); + if( pDup && pDup->nSrc>1 && !IN_RENAME_OBJECT ){ + Select *pSubquery; + Token as; + pSubquery = sqlite3SelectNew(pParse,0,pDup,0,0,0,0,SF_NestedFrom,0); + as.n = 0; + as.z = 0; + pDup = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); + } + pSrc = sqlite3SrcListAppendList(pParse, pSrc, pDup); + } + }else{ + sqlite3DbFree(db, zName); } return pSrc; } /* -** Generate VDBE code for the statements inside the body of a single +** Return true if the pExpr term from the RETURNING clause argument +** list is of the form "*". Raise an error if the terms if of the +** form "table.*". +*/ +static int isAsteriskTerm( + Parse *pParse, /* Parsing context */ + Expr *pTerm /* A term in the RETURNING clause */ +){ + assert( pTerm!=0 ); + if( pTerm->op==TK_ASTERISK ) return 1; + if( pTerm->op!=TK_DOT ) return 0; + assert( pTerm->pRight!=0 ); + assert( pTerm->pLeft!=0 ); + if( pTerm->pRight->op!=TK_ASTERISK ) return 0; + sqlite3ErrorMsg(pParse, "RETURNING may not use \"TABLE.*\" wildcards"); + return 1; +} + +/* The input list pList is the list of result set terms from a RETURNING +** clause. The table that we are returning from is pTab. +** +** This routine makes a copy of the pList, and at the same time expands +** any "*" wildcards to be the complete set of columns from pTab. +*/ +static ExprList *sqlite3ExpandReturning( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* The arguments to RETURNING */ + Table *pTab /* The table being updated */ +){ + ExprList *pNew = 0; + sqlite3 *db = pParse->db; + int i; + + for(i=0; inExpr; i++){ + Expr *pOldExpr = pList->a[i].pExpr; + if( NEVER(pOldExpr==0) ) continue; + if( isAsteriskTerm(pParse, pOldExpr) ){ + int jj; + for(jj=0; jjnCol; jj++){ + Expr *pNewExpr; + if( IsHiddenColumn(pTab->aCol+jj) ) continue; + pNewExpr = sqlite3Expr(db, TK_ID, pTab->aCol[jj].zCnName); + pNew = sqlite3ExprListAppend(pParse, pNew, pNewExpr); + if( !db->mallocFailed ){ + struct ExprList_item *pItem = &pNew->a[pNew->nExpr-1]; + pItem->zEName = sqlite3DbStrDup(db, pTab->aCol[jj].zCnName); + pItem->fg.eEName = ENAME_NAME; + } + } + }else{ + Expr *pNewExpr = sqlite3ExprDup(db, pOldExpr, 0); + pNew = sqlite3ExprListAppend(pParse, pNew, pNewExpr); + if( !db->mallocFailed && ALWAYS(pList->a[i].zEName!=0) ){ + struct ExprList_item *pItem = &pNew->a[pNew->nExpr-1]; + pItem->zEName = sqlite3DbStrDup(db, pList->a[i].zEName); + pItem->fg.eEName = pList->a[i].fg.eEName; + } + } + } + return pNew; +} + +/* If the Expr node is a subquery or an EXISTS operator or an IN operator that +** uses a subquery, and if the subquery is SF_Correlated, then mark the +** expression as EP_VarSelect. +*/ +static int sqlite3ReturningSubqueryVarSelect(Walker *NotUsed, Expr *pExpr){ + UNUSED_PARAMETER(NotUsed); + if( ExprUseXSelect(pExpr) + && (pExpr->x.pSelect->selFlags & SF_Correlated)!=0 + ){ + testcase( ExprHasProperty(pExpr, EP_VarSelect) ); + ExprSetProperty(pExpr, EP_VarSelect); + } + return WRC_Continue; +} + + +/* +** If the SELECT references the table pWalker->u.pTab, then do two things: +** +** (1) Mark the SELECT as as SF_Correlated. +** (2) Set pWalker->eCode to non-zero so that the caller will know +** that (1) has happened. +*/ +static int sqlite3ReturningSubqueryCorrelated(Walker *pWalker, Select *pSelect){ + int i; + SrcList *pSrc; + assert( pSelect!=0 ); + pSrc = pSelect->pSrc; + assert( pSrc!=0 ); + for(i=0; inSrc; i++){ + if( pSrc->a[i].pSTab==pWalker->u.pTab ){ + testcase( pSelect->selFlags & SF_Correlated ); + pSelect->selFlags |= SF_Correlated; + pWalker->eCode = 1; + break; + } + } + return WRC_Continue; +} + +/* +** Scan the expression list that is the argument to RETURNING looking +** for subqueries that depend on the table which is being modified in the +** statement that is hosting the RETURNING clause (pTab). Mark all such +** subqueries as SF_Correlated. If the subqueries are part of an +** expression, mark the expression as EP_VarSelect. +** +** https://sqlite.org/forum/forumpost/2c83569ce8945d39 +*/ +static void sqlite3ProcessReturningSubqueries( + ExprList *pEList, + Table *pTab +){ + Walker w; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3ExprWalkNoop; + w.xSelectCallback = sqlite3ReturningSubqueryCorrelated; + w.u.pTab = pTab; + sqlite3WalkExprList(&w, pEList); + if( w.eCode ){ + w.xExprCallback = sqlite3ReturningSubqueryVarSelect; + w.xSelectCallback = sqlite3SelectWalkNoop; + sqlite3WalkExprList(&w, pEList); + } +} + +/* +** Generate code for the RETURNING trigger. Unlike other triggers +** that invoke a subprogram in the bytecode, the code for RETURNING +** is generated in-line. +*/ +static void codeReturningTrigger( + Parse *pParse, /* Parse context */ + Trigger *pTrigger, /* The trigger step that defines the RETURNING */ + Table *pTab, /* The table to code triggers from */ + int regIn /* The first in an array of registers */ +){ + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + ExprList *pNew; + Returning *pReturning; + Select sSelect; + SrcList sFrom; + + assert( v!=0 ); + if( !pParse->bReturning ){ + /* This RETURNING trigger must be for a different statement as + ** this statement lacks a RETURNING clause. */ + return; + } + assert( db->pParse==pParse ); + pReturning = pParse->u1.pReturning; + if( pTrigger != &(pReturning->retTrig) ){ + /* This RETURNING trigger is for a different statement */ + return; + } + memset(&sSelect, 0, sizeof(sSelect)); + memset(&sFrom, 0, sizeof(sFrom)); + sSelect.pEList = sqlite3ExprListDup(db, pReturning->pReturnEL, 0); + sSelect.pSrc = &sFrom; + sFrom.nSrc = 1; + sFrom.a[0].pSTab = pTab; + sFrom.a[0].zName = pTab->zName; /* tag-20240424-1 */ + sFrom.a[0].iCursor = -1; + sqlite3SelectPrep(pParse, &sSelect, 0); + if( pParse->nErr==0 ){ + assert( db->mallocFailed==0 ); + sqlite3GenerateColumnNames(pParse, &sSelect); + } + sqlite3ExprListDelete(db, sSelect.pEList); + pNew = sqlite3ExpandReturning(pParse, pReturning->pReturnEL, pTab); + if( pParse->nErr==0 ){ + NameContext sNC; + memset(&sNC, 0, sizeof(sNC)); + if( pReturning->nRetCol==0 ){ + pReturning->nRetCol = pNew->nExpr; + pReturning->iRetCur = pParse->nTab++; + } + sNC.pParse = pParse; + sNC.uNC.iBaseReg = regIn; + sNC.ncFlags = NC_UBaseReg; + pParse->eTriggerOp = pTrigger->op; + pParse->pTriggerTab = pTab; + if( sqlite3ResolveExprListNames(&sNC, pNew)==SQLITE_OK + && ALWAYS(!db->mallocFailed) + ){ + int i; + int nCol = pNew->nExpr; + int reg = pParse->nMem+1; + sqlite3ProcessReturningSubqueries(pNew, pTab); + pParse->nMem += nCol+2; + pReturning->iRetReg = reg; + for(i=0; ia[i].pExpr; + assert( pCol!=0 ); /* Due to !db->mallocFailed ~9 lines above */ + sqlite3ExprCodeFactorable(pParse, pCol, reg+i); + if( sqlite3ExprAffinity(pCol)==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, reg+i); + } + } + sqlite3VdbeAddOp3(v, OP_MakeRecord, reg, i, reg+i); + sqlite3VdbeAddOp2(v, OP_NewRowid, pReturning->iRetCur, reg+i+1); + sqlite3VdbeAddOp3(v, OP_Insert, pReturning->iRetCur, reg+i, reg+i+1); + } + } + sqlite3ExprListDelete(db, pNew); + pParse->eTriggerOp = 0; + pParse->pTriggerTab = 0; +} + + + +/* +** Generate VDBE code for the statements inside the body of a single ** trigger. */ static int codeTriggerProgram( Parse *pParse, /* The parser context */ TriggerStep *pStepList, /* List of statements inside the trigger body */ - int orconf /* Conflict algorithm. (OE_Abort, etc) */ + int orconf /* Conflict algorithm. (OE_Abort, etc) */ ){ TriggerStep *pStep; Vdbe *v = pParse->pVdbe; @@ -119452,30 +153808,42 @@ static int codeTriggerProgram( pParse->eOrconf = (orconf==OE_Default)?pStep->orconf:(u8)orconf; assert( pParse->okConstFactor==0 ); +#ifndef SQLITE_OMIT_TRACE + if( pStep->zSpan ){ + sqlite3VdbeAddOp4(v, OP_Trace, 0x7fffffff, 1, 0, + sqlite3MPrintf(db, "-- %s", pStep->zSpan), + P4_DYNAMIC); + } +#endif + switch( pStep->op ){ case TK_UPDATE: { - sqlite3Update(pParse, - targetSrcList(pParse, pStep), - sqlite3ExprListDup(db, pStep->pExprList, 0), - sqlite3ExprDup(db, pStep->pWhere, 0), - pParse->eOrconf + sqlite3Update(pParse, + sqlite3TriggerStepSrc(pParse, pStep), + sqlite3ExprListDup(db, pStep->pExprList, 0), + sqlite3ExprDup(db, pStep->pWhere, 0), + pParse->eOrconf, 0, 0, 0 ); + sqlite3VdbeAddOp0(v, OP_ResetCount); break; } case TK_INSERT: { - sqlite3Insert(pParse, - targetSrcList(pParse, pStep), - sqlite3SelectDup(db, pStep->pSelect, 0), - sqlite3IdListDup(db, pStep->pIdList), - pParse->eOrconf + sqlite3Insert(pParse, + sqlite3TriggerStepSrc(pParse, pStep), + sqlite3SelectDup(db, pStep->pSelect, 0), + sqlite3IdListDup(db, pStep->pIdList), + pParse->eOrconf, + sqlite3UpsertDup(db, pStep->pUpsert) ); + sqlite3VdbeAddOp0(v, OP_ResetCount); break; } case TK_DELETE: { - sqlite3DeleteFrom(pParse, - targetSrcList(pParse, pStep), - sqlite3ExprDup(db, pStep->pWhere, 0) + sqlite3DeleteFrom(pParse, + sqlite3TriggerStepSrc(pParse, pStep), + sqlite3ExprDup(db, pStep->pWhere, 0), 0, 0 ); + sqlite3VdbeAddOp0(v, OP_ResetCount); break; } default: assert( pStep->op==TK_SELECT ); { @@ -119486,9 +153854,6 @@ static int codeTriggerProgram( sqlite3SelectDelete(db, pSelect); break; } - } - if( pStep->op!=TK_SELECT ){ - sqlite3VdbeAddOp0(v, OP_ResetCount); } } @@ -119531,7 +153896,7 @@ static void transferParseError(Parse *pTo, Parse *pFrom){ } /* -** Create and populate a new TriggerPrg object with a sub-program +** Create and populate a new TriggerPrg object with a sub-program ** implementing trigger pTrigger with ON CONFLICT policy orconf. */ static TriggerPrg *codeRowTrigger( @@ -119547,14 +153912,14 @@ static TriggerPrg *codeRowTrigger( Vdbe *v; /* Temporary VM */ NameContext sNC; /* Name context for sub-vdbe */ SubProgram *pProgram = 0; /* Sub-vdbe for trigger program */ - Parse *pSubParse; /* Parse context for sub-vdbe */ int iEndTrigger = 0; /* Label to jump to if WHEN is false */ + Parse sSubParse; /* Parse context for sub-vdbe */ assert( pTrigger->zName==0 || pTab==tableOfTrigger(pTrigger) ); assert( pTop->pVdbe ); /* Allocate the TriggerPrg and SubProgram objects. To ensure that they - ** are freed if an error occurs, link them into the Parse.pTriggerPrg + ** are freed if an error occurs, link them into the Parse.pTriggerPrg ** list of the top-level Parse object sooner rather than later. */ pPrg = sqlite3DbMallocZero(db, sizeof(TriggerPrg)); if( !pPrg ) return 0; @@ -119568,22 +153933,21 @@ static TriggerPrg *codeRowTrigger( pPrg->aColmask[0] = 0xffffffff; pPrg->aColmask[1] = 0xffffffff; - /* Allocate and populate a new Parse context to use for coding the + /* Allocate and populate a new Parse context to use for coding the ** trigger sub-program. */ - pSubParse = sqlite3StackAllocZero(db, sizeof(Parse)); - if( !pSubParse ) return 0; + sqlite3ParseObjectInit(&sSubParse, db); memset(&sNC, 0, sizeof(sNC)); - sNC.pParse = pSubParse; - pSubParse->db = db; - pSubParse->pTriggerTab = pTab; - pSubParse->pToplevel = pTop; - pSubParse->zAuthContext = pTrigger->zName; - pSubParse->eTriggerOp = pTrigger->op; - pSubParse->nQueryLoop = pParse->nQueryLoop; + sNC.pParse = &sSubParse; + sSubParse.pTriggerTab = pTab; + sSubParse.pToplevel = pTop; + sSubParse.zAuthContext = pTrigger->zName; + sSubParse.eTriggerOp = pTrigger->op; + sSubParse.nQueryLoop = pParse->nQueryLoop; + sSubParse.prepFlags = pParse->prepFlags; - v = sqlite3GetVdbe(pSubParse); + v = sqlite3GetVdbe(&sSubParse); if( v ){ - VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", + VdbeComment((v, "Start: %s.%s (%s %s%s%s ON %s)", pTrigger->zName, onErrorText(orconf), (pTrigger->tr_tm==TRIGGER_BEFORE ? "BEFORE" : "AFTER"), (pTrigger->op==TK_UPDATE ? "UPDATE" : ""), @@ -119592,27 +153956,29 @@ static TriggerPrg *codeRowTrigger( pTab->zName )); #ifndef SQLITE_OMIT_TRACE - sqlite3VdbeChangeP4(v, -1, - sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC - ); + if( pTrigger->zName ){ + sqlite3VdbeChangeP4(v, -1, + sqlite3MPrintf(db, "-- TRIGGER %s", pTrigger->zName), P4_DYNAMIC + ); + } #endif /* If one was specified, code the WHEN clause. If it evaluates to false - ** (or NULL) the sub-vdbe is immediately halted by jumping to the + ** (or NULL) the sub-vdbe is immediately halted by jumping to the ** OP_Halt inserted at the end of the program. */ if( pTrigger->pWhen ){ pWhen = sqlite3ExprDup(db, pTrigger->pWhen, 0); - if( SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) - && db->mallocFailed==0 + if( db->mallocFailed==0 + && SQLITE_OK==sqlite3ResolveExprNames(&sNC, pWhen) ){ - iEndTrigger = sqlite3VdbeMakeLabel(v); - sqlite3ExprIfFalse(pSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); + iEndTrigger = sqlite3VdbeMakeLabel(&sSubParse); + sqlite3ExprIfFalse(&sSubParse, pWhen, iEndTrigger, SQLITE_JUMPIFNULL); } sqlite3ExprDelete(db, pWhen); } /* Code the trigger program into the sub-vdbe. */ - codeTriggerProgram(pSubParse, pTrigger->step_list, orconf); + codeTriggerProgram(&sSubParse, pTrigger->step_list, orconf); /* Insert an OP_Halt at the end of the sub-program. */ if( iEndTrigger ){ @@ -119620,28 +153986,27 @@ static TriggerPrg *codeRowTrigger( } sqlite3VdbeAddOp0(v, OP_Halt); VdbeComment((v, "End: %s.%s", pTrigger->zName, onErrorText(orconf))); + transferParseError(pParse, &sSubParse); - transferParseError(pParse, pSubParse); - if( db->mallocFailed==0 ){ + if( pParse->nErr==0 ){ + assert( db->mallocFailed==0 ); pProgram->aOp = sqlite3VdbeTakeOpArray(v, &pProgram->nOp, &pTop->nMaxArg); } - pProgram->nMem = pSubParse->nMem; - pProgram->nCsr = pSubParse->nTab; - pProgram->nOnce = pSubParse->nOnce; + pProgram->nMem = sSubParse.nMem; + pProgram->nCsr = sSubParse.nTab; pProgram->token = (void *)pTrigger; - pPrg->aColmask[0] = pSubParse->oldmask; - pPrg->aColmask[1] = pSubParse->newmask; + pPrg->aColmask[0] = sSubParse.oldmask; + pPrg->aColmask[1] = sSubParse.newmask; sqlite3VdbeDelete(v); + }else{ + transferParseError(pParse, &sSubParse); } - assert( !pSubParse->pAinc && !pSubParse->pZombieTab ); - assert( !pSubParse->pTriggerPrg && !pSubParse->nMaxArg ); - sqlite3ParserReset(pSubParse); - sqlite3StackFree(db, pSubParse); - + assert( !sSubParse.pTriggerPrg && !sSubParse.nMaxArg ); + sqlite3ParseObjectReset(&sSubParse); return pPrg; } - + /* ** Return a pointer to a TriggerPrg object containing the sub-program for ** trigger pTrigger with default ON CONFLICT algorithm orconf. If no such @@ -119663,21 +154028,22 @@ static TriggerPrg *getRowTrigger( ** process of being coded). If this is the case, then an entry with ** a matching TriggerPrg.pTrigger field will be present somewhere ** in the Parse.pTriggerPrg list. Search for such an entry. */ - for(pPrg=pRoot->pTriggerPrg; - pPrg && (pPrg->pTrigger!=pTrigger || pPrg->orconf!=orconf); + for(pPrg=pRoot->pTriggerPrg; + pPrg && (pPrg->pTrigger!=pTrigger || pPrg->orconf!=orconf); pPrg=pPrg->pNext ); /* If an existing TriggerPrg could not be located, create a new one. */ if( !pPrg ){ pPrg = codeRowTrigger(pParse, pTrigger, pTab, orconf); + pParse->db->errByteOffset = -1; } return pPrg; } /* -** Generate code for the trigger program associated with trigger p on +** Generate code for the trigger program associated with trigger p on ** table pTab. The reg, orconf and ignoreJump parameters passed to this ** function are the same as those described in the header function for ** sqlite3CodeRowTrigger() @@ -119693,9 +154059,9 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( Vdbe *v = sqlite3GetVdbe(pParse); /* Main VM */ TriggerPrg *pPrg; pPrg = getRowTrigger(pParse, p, pTab, orconf); - assert( pPrg || pParse->nErr || pParse->db->mallocFailed ); + assert( pPrg || pParse->nErr ); - /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program + /* Code the OP_Program opcode in the parent VDBE. P4 of the OP_Program ** is a pointer to the sub-vdbe containing the trigger program. */ if( pPrg ){ int bRecursive = (p->zName && 0==(pParse->db->flags&SQLITE_RecTriggers)); @@ -119724,7 +154090,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( ** If there are no triggers that fire at the specified time for the specified ** operation on pTab, this function is a no-op. ** -** The reg argument is the address of the first in an array of registers +** The reg argument is the address of the first in an array of registers ** that contain the values substituted for the new.* and old.* references ** in the trigger program. If N is the number of columns in table pTab ** (a copy of pTab->nCol), then registers are populated as follows: @@ -119736,17 +154102,17 @@ SQLITE_PRIVATE void sqlite3CodeRowTriggerDirect( ** ... ... ** reg+N OLD.* value of right-most column of pTab ** reg+N+1 NEW.rowid -** reg+N+2 OLD.* value of left-most column of pTab +** reg+N+2 NEW.* value of left-most column of pTab ** ... ... ** reg+N+N+1 NEW.* value of right-most column of pTab ** ** For ON DELETE triggers, the registers containing the NEW.* values will -** never be accessed by the trigger program, so they are not allocated or -** populated by the caller (there is no data to populate them with anyway). +** never be accessed by the trigger program, so they are not allocated or +** populated by the caller (there is no data to populate them with anyway). ** Similarly, for ON INSERT triggers the values stored in the OLD.* registers ** are never accessed, and so are not allocated by the caller. So, for an ** ON INSERT trigger, the value passed to this function as parameter reg -** is not a readable register, although registers (reg+N) through +** is not a readable register, although registers (reg+N) through ** (reg+N+N+1) are. ** ** Parameter orconf is the default conflict resolution algorithm for the @@ -119778,23 +154144,31 @@ SQLITE_PRIVATE void sqlite3CodeRowTrigger( ** or else it must be a TEMP trigger. */ assert( p->pSchema!=0 ); assert( p->pTabSchema!=0 ); - assert( p->pSchema==p->pTabSchema + assert( p->pSchema==p->pTabSchema || p->pSchema==pParse->db->aDb[1].pSchema ); - /* Determine whether we should code this trigger */ - if( p->op==op - && p->tr_tm==tr_tm + /* Determine whether we should code this trigger. One of two choices: + ** 1. The trigger is an exact match to the current DML statement + ** 2. This is a RETURNING trigger for INSERT but we are currently + ** doing the UPDATE part of an UPSERT. + */ + if( (p->op==op || (p->bReturning && p->op==TK_INSERT && op==TK_UPDATE)) + && p->tr_tm==tr_tm && checkColumnOverlap(p->pColumns, pChanges) ){ - sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); + if( !p->bReturning ){ + sqlite3CodeRowTriggerDirect(pParse, p, pTab, reg, orconf, ignoreJump); + }else if( sqlite3IsToplevel(pParse) ){ + codeReturningTrigger(pParse, p, pTab, reg); + } } } } /* -** Triggers may access values stored in the old.* or new.* pseudo-table. -** This function returns a 32-bit bitmask indicating which columns of the -** old.* or new.* tables actually are used by triggers. This information +** Triggers may access values stored in the old.* or new.* pseudo-table. +** This function returns a 32-bit bitmask indicating which columns of the +** old.* or new.* tables actually are used by triggers. This information ** may be used by the caller, for example, to avoid having to load the entire ** old.* record into memory when executing an UPDATE or DELETE command. ** @@ -119804,7 +154178,7 @@ SQLITE_PRIVATE void sqlite3CodeRowTrigger( ** are more than 32 columns in the table, and at least one of the columns ** with an index greater than 32 may be accessed, 0xffffffff is returned. ** -** It is not possible to determine if the old.rowid or new.rowid column is +** It is not possible to determine if the old.rowid or new.rowid column is ** accessed by triggers. The caller must always assume that it is. ** ** Parameter isNew must be either 1 or 0. If it is 0, then the mask returned @@ -119830,14 +154204,22 @@ SQLITE_PRIVATE u32 sqlite3TriggerColmask( Trigger *p; assert( isNew==1 || isNew==0 ); + if( IsView(pTab) ){ + return 0xffffffff; + } for(p=pTrigger; p; p=p->pNext){ - if( p->op==op && (tr_tm&p->tr_tm) + if( p->op==op + && (tr_tm&p->tr_tm) && checkColumnOverlap(p->pColumns,pChanges) ){ - TriggerPrg *pPrg; - pPrg = getRowTrigger(pParse, p, pTab, orconf); - if( pPrg ){ - mask |= pPrg->aColmask[isNew]; + if( p->bReturning ){ + mask = 0xffffffff; + }else{ + TriggerPrg *pPrg; + pPrg = getRowTrigger(pParse, p, pTab, orconf); + if( pPrg ){ + mask |= pPrg->aColmask[isNew]; + } } } } @@ -119881,10 +154263,10 @@ static void updateVirtualTable( /* ** The most recently coded instruction was an OP_Column to retrieve the -** i-th column of table pTab. This routine sets the P4 parameter of the +** i-th column of table pTab. This routine sets the P4 parameter of the ** OP_Column to the default value, if any. ** -** The default value of a column is specified by a DEFAULT clause in the +** The default value of a column is specified by a DEFAULT clause in the ** column definition. This was either supplied by the user when the table ** was created, or added later to the table definition by an ALTER TABLE ** command. If the latter, then the row-records in the table btree on disk @@ -119893,70 +154275,270 @@ static void updateVirtualTable( ** If the former, then all row-records are guaranteed to include a value ** for the column and the P4 value is not required. ** -** Column definitions created by an ALTER TABLE command may only have +** Column definitions created by an ALTER TABLE command may only have ** literal default values specified: a number, null or a string. (If a more -** complicated default expression value was provided, it is evaluated +** complicated default expression value was provided, it is evaluated ** when the ALTER TABLE is executed and one of the literal values written -** into the sqlite_master table.) +** into the sqlite_schema table.) ** ** Therefore, the P4 parameter is only required if the default value for ** the column is a literal number, string or null. The sqlite3ValueFromExpr() ** function is capable of transforming these types of expressions into ** sqlite3_value objects. ** -** If parameter iReg is not negative, code an OP_RealAffinity instruction -** on register iReg. This is used when an equivalent integer value is -** stored in place of an 8-byte floating point value in order to save -** space. +** If column as REAL affinity and the table is an ordinary b-tree table +** (not a virtual table) then the value might have been stored as an +** integer. In that case, add an OP_RealAffinity opcode to make sure +** it has been converted into REAL. */ SQLITE_PRIVATE void sqlite3ColumnDefault(Vdbe *v, Table *pTab, int i, int iReg){ + Column *pCol; assert( pTab!=0 ); - if( !pTab->pSelect ){ + assert( pTab->nCol>i ); + pCol = &pTab->aCol[i]; + if( pCol->iDflt ){ sqlite3_value *pValue = 0; u8 enc = ENC(sqlite3VdbeDb(v)); - Column *pCol = &pTab->aCol[i]; - VdbeComment((v, "%s.%s", pTab->zName, pCol->zName)); + assert( !IsView(pTab) ); + VdbeComment((v, "%s.%s", pTab->zName, pCol->zCnName)); assert( inCol ); - sqlite3ValueFromExpr(sqlite3VdbeDb(v), pCol->pDflt, enc, + sqlite3ValueFromExpr(sqlite3VdbeDb(v), + sqlite3ColumnExpr(pTab,pCol), enc, pCol->affinity, &pValue); if( pValue ){ - sqlite3VdbeChangeP4(v, -1, (const char *)pValue, P4_MEM); + sqlite3VdbeAppendP4(v, pValue, P4_MEM); } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ - sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + if( pCol->affinity==SQLITE_AFF_REAL && !IsVirtual(pTab) ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, iReg); + } +#endif +} + +/* +** Check to see if column iCol of index pIdx references any of the +** columns defined by aXRef and chngRowid. Return true if it does +** and false if not. This is an optimization. False-positives are a +** performance degradation, but false-negatives can result in a corrupt +** index and incorrect answers. +** +** aXRef[j] will be non-negative if column j of the original table is +** being updated. chngRowid will be true if the rowid of the table is +** being updated. +*/ +static int indexColumnIsBeingUpdated( + Index *pIdx, /* The index to check */ + int iCol, /* Which column of the index to check */ + int *aXRef, /* aXRef[j]>=0 if column j is being updated */ + int chngRowid /* true if the rowid is being updated */ +){ + i16 iIdxCol = pIdx->aiColumn[iCol]; + assert( iIdxCol!=XN_ROWID ); /* Cannot index rowid */ + if( iIdxCol>=0 ){ + return aXRef[iIdxCol]>=0; + } + assert( iIdxCol==XN_EXPR ); + assert( pIdx->aColExpr!=0 ); + assert( pIdx->aColExpr->a[iCol].pExpr!=0 ); + return sqlite3ExprReferencesUpdatedColumn(pIdx->aColExpr->a[iCol].pExpr, + aXRef,chngRowid); +} + +/* +** Check to see if index pIdx is a partial index whose conditional +** expression might change values due to an UPDATE. Return true if +** the index is subject to change and false if the index is guaranteed +** to be unchanged. This is an optimization. False-positives are a +** performance degradation, but false-negatives can result in a corrupt +** index and incorrect answers. +** +** aXRef[j] will be non-negative if column j of the original table is +** being updated. chngRowid will be true if the rowid of the table is +** being updated. +*/ +static int indexWhereClauseMightChange( + Index *pIdx, /* The index to check */ + int *aXRef, /* aXRef[j]>=0 if column j is being updated */ + int chngRowid /* true if the rowid is being updated */ +){ + if( pIdx->pPartIdxWhere==0 ) return 0; + return sqlite3ExprReferencesUpdatedColumn(pIdx->pPartIdxWhere, + aXRef, chngRowid); +} + +/* +** Allocate and return a pointer to an expression of type TK_ROW with +** Expr.iColumn set to value (iCol+1). The resolver will modify the +** expression to be a TK_COLUMN reading column iCol of the first +** table in the source-list (pSrc->a[0]). +*/ +static Expr *exprRowColumn(Parse *pParse, int iCol){ + Expr *pRet = sqlite3PExpr(pParse, TK_ROW, 0, 0); + if( pRet ) pRet->iColumn = iCol+1; + return pRet; +} + +/* +** Assuming both the pLimit and pOrderBy parameters are NULL, this function +** generates VM code to run the query: +** +** SELECT , pChanges FROM pTabList WHERE pWhere +** +** and write the results to the ephemeral table already opened as cursor +** iEph. None of pChanges, pTabList or pWhere are modified or consumed by +** this function, they must be deleted by the caller. +** +** Or, if pLimit and pOrderBy are not NULL, and pTab is not a view: +** +** SELECT , pChanges FROM pTabList +** WHERE pWhere +** GROUP BY +** ORDER BY pOrderBy LIMIT pLimit +** +** If pTab is a view, the GROUP BY clause is omitted. +** +** Exactly how results are written to table iEph, and exactly what +** the in the query above are is determined by the type +** of table pTabList->a[0].pTab. +** +** If the table is a WITHOUT ROWID table, then argument pPk must be its +** PRIMARY KEY. In this case are the primary key columns +** of the table, in order. The results of the query are written to ephemeral +** table iEph as index keys, using OP_IdxInsert. +** +** If the table is actually a view, then are all columns of +** the view. The results are written to the ephemeral table iEph as records +** with automatically assigned integer keys. +** +** If the table is a virtual or ordinary intkey table, then +** is its rowid. For a virtual table, the results are written to iEph as +** records with automatically assigned integer keys For intkey tables, the +** rowid value in is used as the integer key, and the +** remaining fields make up the table record. +*/ +static void updateFromSelect( + Parse *pParse, /* Parse context */ + int iEph, /* Cursor for open eph. table */ + Index *pPk, /* PK if table 0 is WITHOUT ROWID */ + ExprList *pChanges, /* List of expressions to return */ + SrcList *pTabList, /* List of tables to select from */ + Expr *pWhere, /* WHERE clause for query */ + ExprList *pOrderBy, /* ORDER BY clause */ + Expr *pLimit /* LIMIT clause */ +){ + int i; + SelectDest dest; + Select *pSelect = 0; + ExprList *pList = 0; + ExprList *pGrp = 0; + Expr *pLimit2 = 0; + ExprList *pOrderBy2 = 0; + sqlite3 *db = pParse->db; + Table *pTab = pTabList->a[0].pSTab; + SrcList *pSrc; + Expr *pWhere2; + int eDest; + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( pOrderBy && pLimit==0 ) { + sqlite3ErrorMsg(pParse, "ORDER BY without LIMIT on UPDATE"); + return; + } + pOrderBy2 = sqlite3ExprListDup(db, pOrderBy, 0); + pLimit2 = sqlite3ExprDup(db, pLimit, 0); +#else + UNUSED_PARAMETER(pOrderBy); + UNUSED_PARAMETER(pLimit); +#endif + + pSrc = sqlite3SrcListDup(db, pTabList, 0); + pWhere2 = sqlite3ExprDup(db, pWhere, 0); + + assert( pTabList->nSrc>1 ); + if( pSrc ){ + assert( pSrc->a[0].fg.notCte ); + pSrc->a[0].iCursor = -1; + pSrc->a[0].pSTab->nTabRef--; + pSrc->a[0].pSTab = 0; + } + if( pPk ){ + for(i=0; inKeyCol; i++){ + Expr *pNew = exprRowColumn(pParse, pPk->aiColumn[i]); +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( pLimit ){ + pGrp = sqlite3ExprListAppend(pParse, pGrp, sqlite3ExprDup(db, pNew, 0)); + } +#endif + pList = sqlite3ExprListAppend(pParse, pList, pNew); + } + eDest = IsVirtual(pTab) ? SRT_Table : SRT_Upfrom; + }else if( IsView(pTab) ){ + for(i=0; inCol; i++){ + pList = sqlite3ExprListAppend(pParse, pList, exprRowColumn(pParse, i)); + } + eDest = SRT_Table; + }else{ + eDest = IsVirtual(pTab) ? SRT_Table : SRT_Upfrom; + pList = sqlite3ExprListAppend(pParse, 0, sqlite3PExpr(pParse,TK_ROW,0,0)); +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( pLimit ){ + pGrp = sqlite3ExprListAppend(pParse, 0, sqlite3PExpr(pParse,TK_ROW,0,0)); } #endif } + assert( pChanges!=0 || pParse->db->mallocFailed ); + if( pChanges ){ + for(i=0; inExpr; i++){ + pList = sqlite3ExprListAppend(pParse, pList, + sqlite3ExprDup(db, pChanges->a[i].pExpr, 0) + ); + } + } + pSelect = sqlite3SelectNew(pParse, pList, + pSrc, pWhere2, pGrp, 0, pOrderBy2, + SF_UFSrcCheck|SF_IncludeHidden|SF_UpdateFrom, pLimit2 + ); + if( pSelect ) pSelect->selFlags |= SF_OrderByReqd; + sqlite3SelectDestInit(&dest, eDest, iEph); + dest.iSDParm2 = (pPk ? pPk->nKeyCol : -1); + sqlite3Select(pParse, pSelect, &dest); + sqlite3SelectDelete(db, pSelect); } /* ** Process an UPDATE statement. ** -** UPDATE OR IGNORE table_wxyz SET a=b, c=d WHERE e<5 AND f NOT NULL; -** \_______/ \________/ \______/ \________________/ -* onError pTabList pChanges pWhere +** UPDATE OR IGNORE tbl SET a=b, c=d FROM tbl2... WHERE e<5 AND f NOT NULL; +** \_______/ \_/ \______/ \_____/ \________________/ +** onError | pChanges | pWhere +** \_______________________/ +** pTabList */ SQLITE_PRIVATE void sqlite3Update( Parse *pParse, /* The parser context */ SrcList *pTabList, /* The table in which we should change things */ ExprList *pChanges, /* Things to be changed */ Expr *pWhere, /* The WHERE clause. May be null */ - int onError /* How to handle constraint errors */ + int onError, /* How to handle constraint errors */ + ExprList *pOrderBy, /* ORDER BY clause. May be null */ + Expr *pLimit, /* LIMIT clause. May be null */ + Upsert *pUpsert /* ON CONFLICT clause, or null */ ){ - int i, j; /* Loop counters */ + int i, j, k; /* Loop counters */ Table *pTab; /* The table to be updated */ int addrTop = 0; /* VDBE instruction address of the start of the loop */ - WhereInfo *pWInfo; /* Information about the WHERE clause */ + WhereInfo *pWInfo = 0; /* Information about the WHERE clause */ Vdbe *v; /* The virtual database engine */ Index *pIdx; /* For looping over indices */ Index *pPk; /* The PRIMARY KEY index for WITHOUT ROWID tables */ int nIdx; /* Number of indices that need updating */ + int nAllIdx; /* Total number of indexes */ int iBaseCur; /* Base cursor number */ int iDataCur; /* Cursor for the canonical data btree */ int iIdxCur; /* Cursor for the first index */ sqlite3 *db; /* The database structure */ - int *aRegIdx = 0; /* One register assigned to each index to be updated */ + int *aRegIdx = 0; /* Registers for to each index and the main table */ int *aXRef = 0; /* aXRef[i] is the index in pChanges->a[] of the ** an expression for the i-th column of the table. ** aXRef[i]==-1 if the i-th column is not changed. */ @@ -119965,13 +154547,15 @@ SQLITE_PRIVATE void sqlite3Update( u8 chngRowid; /* Rowid changed in a normal table */ u8 chngKey; /* Either chngPk or chngRowid */ Expr *pRowidExpr = 0; /* Expression defining the new record number */ + int iRowidExpr = -1; /* Index of "rowid=" (or IPK) assignment in pChanges */ AuthContext sContext; /* The authorization context */ NameContext sNC; /* The name-context to resolve expressions in */ int iDb; /* Database containing the table being updated */ - int okOnePass; /* True for one-pass algorithm without the FIFO */ + int eOnePass; /* ONEPASS_XXX value from where.c */ int hasFK; /* True if foreign key processing is required */ int labelBreak; /* Jump here to break out of UPDATE loop */ int labelContinue; /* Jump here to continue next step of UPDATE loop */ + int flags; /* Flags for sqlite3WhereBegin() */ #ifndef SQLITE_OMIT_TRIGGER int isView; /* True when updating a view (INSTEAD OF trigger) */ @@ -119982,6 +154566,12 @@ SQLITE_PRIVATE void sqlite3Update( int iEph = 0; /* Ephemeral table holding all primary key values */ int nKey = 0; /* Number of elements in regKey for WITHOUT ROWID */ int aiCurOnePass[2]; /* The write cursors opened by WHERE_ONEPASS */ + int addrOpen = 0; /* Address of OP_OpenEphemeral */ + int iPk = 0; /* First of nPk cells holding PRIMARY KEY value */ + i16 nPk = 0; /* Number of components of the PRIMARY KEY */ + int bReplace = 0; /* True if REPLACE conflict resolution might happen */ + int bFinishSeek = 1; /* The OP_FinishSeek opcode is needed */ + int nChangeFrom = 0; /* If there is a FROM, pChanges->nExpr, else 0 */ /* Register Allocations */ int regRowCount = 0; /* A count of rows changed */ @@ -119994,12 +154584,13 @@ SQLITE_PRIVATE void sqlite3Update( memset(&sContext, 0, sizeof(sContext)); db = pParse->db; - if( pParse->nErr || db->mallocFailed ){ + assert( db->pParse==pParse ); + if( pParse->nErr ){ goto update_cleanup; } - assert( pTabList->nSrc==1 ); + assert( db->mallocFailed==0 ); - /* Locate the table which we want to update. + /* Locate the table which we want to update. */ pTab = sqlite3SrcListLookup(pParse, pTabList); if( pTab==0 ) goto update_cleanup; @@ -120010,7 +154601,7 @@ SQLITE_PRIVATE void sqlite3Update( */ #ifndef SQLITE_OMIT_TRIGGER pTrigger = sqlite3TriggersExist(pParse, pTab, TK_UPDATE, pChanges, &tmask); - isView = pTab->pSelect!=0; + isView = IsView(pTab); assert( pTrigger || tmask==0 ); #else # define pTrigger 0 @@ -120022,10 +154613,35 @@ SQLITE_PRIVATE void sqlite3Update( # define isView 0 #endif +#if TREETRACE_ENABLED + if( sqlite3TreeTrace & 0x10000 ){ + sqlite3TreeViewLine(0, "In sqlite3Update() at %s:%d", __FILE__, __LINE__); + sqlite3TreeViewUpdate(pParse->pWith, pTabList, pChanges, pWhere, + onError, pOrderBy, pLimit, pUpsert, pTrigger); + } +#endif + + /* If there was a FROM clause, set nChangeFrom to the number of expressions + ** in the change-list. Otherwise, set it to 0. There cannot be a FROM + ** clause if this function is being called to generate code for part of + ** an UPSERT statement. */ + nChangeFrom = (pTabList->nSrc>1) ? pChanges->nExpr : 0; + assert( nChangeFrom==0 || pUpsert==0 ); + +#ifdef SQLITE_ENABLE_UPDATE_DELETE_LIMIT + if( !isView && nChangeFrom==0 ){ + pWhere = sqlite3LimitWhere( + pParse, pTabList, pWhere, pOrderBy, pLimit, "UPDATE" + ); + pOrderBy = 0; + pLimit = 0; + } +#endif + if( sqlite3ViewGetColumnNames(pParse, pTab) ){ goto update_cleanup; } - if( sqlite3IsReadOnly(pParse, pTab, tmask) ){ + if( sqlite3IsReadOnly(pParse, pTab, pTrigger) ){ goto update_cleanup; } @@ -120034,24 +154650,31 @@ SQLITE_PRIVATE void sqlite3Update( ** need to occur right after the database cursor. So go ahead and ** allocate enough space, just in case. */ - pTabList->a[0].iCursor = iBaseCur = iDataCur = pParse->nTab++; + iBaseCur = iDataCur = pParse->nTab++; iIdxCur = iDataCur+1; pPk = HasRowid(pTab) ? 0 : sqlite3PrimaryKeyIndex(pTab); + testcase( pPk!=0 && pPk!=pTab->pIndex ); for(nIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nIdx++){ - if( IsPrimaryKeyIndex(pIdx) && pPk!=0 ){ + if( pPk==pIdx ){ iDataCur = pParse->nTab; - pTabList->a[0].iCursor = iDataCur; } pParse->nTab++; } + if( pUpsert ){ + /* On an UPSERT, reuse the same cursors already opened by INSERT */ + iDataCur = pUpsert->iDataCur; + iIdxCur = pUpsert->iIdxCur; + pParse->nTab = iBaseCur; + } + pTabList->a[0].iCursor = iDataCur; - /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. + /* Allocate space for aXRef[], aRegIdx[], and aToOpen[]. ** Initialize aXRef[] and aToOpen[] to their default values. */ - aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx) + nIdx+2 ); + aXRef = sqlite3DbMallocRawNN(db, sizeof(int) * (pTab->nCol+nIdx+1) + nIdx+2 ); if( aXRef==0 ) goto update_cleanup; aRegIdx = aXRef+pTab->nCol; - aToOpen = (u8*)(aRegIdx+nIdx); + aToOpen = (u8*)(aRegIdx+nIdx+1); memset(aToOpen, 1, nIdx+1); aToOpen[nIdx+1] = 0; for(i=0; inCol; i++) aXRef[i] = -1; @@ -120060,6 +154683,12 @@ SQLITE_PRIVATE void sqlite3Update( memset(&sNC, 0, sizeof(sNC)); sNC.pParse = pParse; sNC.pSrcList = pTabList; + sNC.uNC.pUpsert = pUpsert; + sNC.ncFlags = NC_UUpsert; + + /* Begin generating code. */ + v = sqlite3GetVdbe(pParse); + if( v==0 ) goto update_cleanup; /* Resolve the column names in all the expressions of the ** of the UPDATE statement. Also find the column index @@ -120069,28 +154698,45 @@ SQLITE_PRIVATE void sqlite3Update( */ chngRowid = chngPk = 0; for(i=0; inExpr; i++){ - if( sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ + u8 hCol = sqlite3StrIHash(pChanges->a[i].zEName); + /* If this is an UPDATE with a FROM clause, do not resolve expressions + ** here. The call to sqlite3Select() below will do that. */ + if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pChanges->a[i].pExpr) ){ goto update_cleanup; } for(j=0; jnCol; j++){ - if( sqlite3StrICmp(pTab->aCol[j].zName, pChanges->a[i].zName)==0 ){ + if( pTab->aCol[j].hName==hCol + && sqlite3StrICmp(pTab->aCol[j].zCnName, pChanges->a[i].zEName)==0 + ){ if( j==pTab->iPKey ){ chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; + iRowidExpr = i; }else if( pPk && (pTab->aCol[j].colFlags & COLFLAG_PRIMKEY)!=0 ){ chngPk = 1; } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + else if( pTab->aCol[j].colFlags & COLFLAG_GENERATED ){ + testcase( pTab->aCol[j].colFlags & COLFLAG_VIRTUAL ); + testcase( pTab->aCol[j].colFlags & COLFLAG_STORED ); + sqlite3ErrorMsg(pParse, + "cannot UPDATE generated column \"%s\"", + pTab->aCol[j].zCnName); + goto update_cleanup; + } +#endif aXRef[j] = i; break; } } if( j>=pTab->nCol ){ - if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zName) ){ + if( pPk==0 && sqlite3IsRowid(pChanges->a[i].zEName) ){ j = -1; chngRowid = 1; pRowidExpr = pChanges->a[i].pExpr; + iRowidExpr = i; }else{ - sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zName); + sqlite3ErrorMsg(pParse, "no such column: %s", pChanges->a[i].zEName); pParse->checkSchema = 1; goto update_cleanup; } @@ -120099,8 +154745,8 @@ SQLITE_PRIVATE void sqlite3Update( { int rc; rc = sqlite3AuthCheck(pParse, SQLITE_UPDATE, pTab->zName, - j<0 ? "ROWID" : pTab->aCol[j].zName, - db->aDb[iDb].zName); + j<0 ? "ROWID" : pTab->aCol[j].zCnName, + db->aDb[iDb].zDbSName); if( rc==SQLITE_DENY ){ goto update_cleanup; }else if( rc==SQLITE_IGNORE ){ @@ -120114,7 +154760,36 @@ SQLITE_PRIVATE void sqlite3Update( assert( chngPk==0 || chngPk==1 ); chngKey = chngRowid + chngPk; - /* The SET expressions are not actually used inside the WHERE loop. +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + /* Mark generated columns as changing if their generator expressions + ** reference any changing column. The actual aXRef[] value for + ** generated expressions is not used, other than to check to see that it + ** is non-negative, so the value of aXRef[] for generated columns can be + ** set to any non-negative number. We use 99999 so that the value is + ** obvious when looking at aXRef[] in a symbolic debugger. + */ + if( pTab->tabFlags & TF_HasGenerated ){ + int bProgress; + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + do{ + bProgress = 0; + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ) continue; + if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ) continue; + if( sqlite3ExprReferencesUpdatedColumn( + sqlite3ColumnExpr(pTab, &pTab->aCol[i]), + aXRef, chngRowid) + ){ + aXRef[i] = 99999; + bProgress = 1; + } + } + }while( bProgress ); + } +#endif + + /* The SET expressions are not actually used inside the WHERE loop. ** So reset the colUsed mask. Unless this is a virtual table. In that ** case, set all bits of the colUsed mask (to ensure that the virtual ** table implementation makes all columns available). @@ -120126,36 +154801,50 @@ SQLITE_PRIVATE void sqlite3Update( /* There is one entry in the aRegIdx[] array for each index on the table ** being updated. Fill in aRegIdx[] with a register number that will hold ** the key for accessing each index. - ** - ** FIXME: Be smarter about omitting indexes that use expressions. */ - for(j=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, j++){ + if( onError==OE_Replace ) bReplace = 1; + for(nAllIdx=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, nAllIdx++){ int reg; - if( chngKey || hasFK || pIdx->pPartIdxWhere || pIdx==pPk ){ + if( chngKey || hasFK>1 || pIdx==pPk + || indexWhereClauseMightChange(pIdx,aXRef,chngRowid) + ){ reg = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; }else{ reg = 0; for(i=0; inKeyCol; i++){ - i16 iIdxCol = pIdx->aiColumn[i]; - if( iIdxCol<0 || aXRef[iIdxCol]>=0 ){ + if( indexColumnIsBeingUpdated(pIdx, i, aXRef, chngRowid) ){ reg = ++pParse->nMem; + pParse->nMem += pIdx->nColumn; + if( onError==OE_Default && pIdx->onError==OE_Replace ){ + bReplace = 1; + } break; } } } - if( reg==0 ) aToOpen[j+1] = 0; - aRegIdx[j] = reg; + if( reg==0 ) aToOpen[nAllIdx+1] = 0; + aRegIdx[nAllIdx] = reg; + } + aRegIdx[nAllIdx] = ++pParse->nMem; /* Register storing the table record */ + if( bReplace ){ + /* If REPLACE conflict resolution might be invoked, open cursors on all + ** indexes in case they are needed to delete records. */ + memset(aToOpen, 1, nIdx+1); } - /* Begin generating code. */ - v = sqlite3GetVdbe(pParse); - if( v==0 ) goto update_cleanup; if( pParse->nested==0 ) sqlite3VdbeCountChanges(v); - sqlite3BeginWriteOperation(pParse, 1, iDb); + sqlite3BeginWriteOperation(pParse, pTrigger || hasFK, iDb); /* Allocate required registers. */ if( !IsVirtual(pTab) ){ - regRowSet = ++pParse->nMem; + /* For now, regRowSet and aRegIdx[nAllIdx] share the same register. + ** If regRowSet turns out to be needed, then aRegIdx[nAllIdx] will be + ** reallocated. aRegIdx[nAllIdx] is the register in which the main + ** table record is written. regRowSet holds the RowSet for the + ** two-pass update algorithm. */ + assert( aRegIdx[nAllIdx]==pParse->nMem ); + regRowSet = aRegIdx[nAllIdx]; regOldRowid = regNewRowid = ++pParse->nMem; if( chngPk || pTrigger || hasFK ){ regOld = pParse->nMem + 1; @@ -120177,15 +154866,19 @@ SQLITE_PRIVATE void sqlite3Update( ** an ephemeral table. */ #if !defined(SQLITE_OMIT_VIEW) && !defined(SQLITE_OMIT_TRIGGER) - if( isView ){ - sqlite3MaterializeView(pParse, pTab, pWhere, iDataCur); + if( nChangeFrom==0 && isView ){ + sqlite3MaterializeView(pParse, pTab, + pWhere, pOrderBy, pLimit, iDataCur + ); + pOrderBy = 0; + pLimit = 0; } #endif /* Resolve the column names in all the expressions in the ** WHERE clause. */ - if( sqlite3ResolveExprNames(&sNC, pWhere) ){ + if( nChangeFrom==0 && sqlite3ResolveExprNames(&sNC, pWhere) ){ goto update_cleanup; } @@ -120198,127 +154891,245 @@ SQLITE_PRIVATE void sqlite3Update( } #endif - /* Begin the database scan - */ - if( HasRowid(pTab) ){ - sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); - pWInfo = sqlite3WhereBegin( - pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED | WHERE_SEEK_TABLE, iIdxCur - ); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); - - /* Remember the rowid of every item to be updated. - */ - sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); - if( !okOnePass ){ - sqlite3VdbeAddOp2(v, OP_RowSetAdd, regRowSet, regOldRowid); - } - - /* End the database scan loop. - */ - sqlite3WhereEnd(pWInfo); - }else{ - int iPk; /* First of nPk memory cells holding PRIMARY KEY value */ - i16 nPk; /* Number of components of the PRIMARY KEY */ - int addrOpen; /* Address of the OpenEphemeral instruction */ + /* Jump to labelBreak to abandon further processing of this UPDATE */ + labelContinue = labelBreak = sqlite3VdbeMakeLabel(pParse); - assert( pPk!=0 ); - nPk = pPk->nKeyCol; - iPk = pParse->nMem+1; - pParse->nMem += nPk; - regKey = ++pParse->nMem; - iEph = pParse->nTab++; - sqlite3VdbeAddOp2(v, OP_Null, 0, iPk); - addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nPk); - sqlite3VdbeSetP4KeyInfo(pParse, pPk); - pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere, 0, 0, - WHERE_ONEPASS_DESIRED, iIdxCur); - if( pWInfo==0 ) goto update_cleanup; - okOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); - for(i=0; iaiColumn[i]>=0 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, pPk->aiColumn[i], - iPk+i); - } - if( okOnePass ){ - sqlite3VdbeChangeToNoop(v, addrOpen); - nKey = nPk; - regKey = iPk; - }else{ - sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, - sqlite3IndexAffinityStr(db, pPk), nPk); - sqlite3VdbeAddOp2(v, OP_IdxInsert, iEph, regKey); - } - sqlite3WhereEnd(pWInfo); - } - - /* Initialize the count of updated rows - */ - if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab ){ + /* Not an UPSERT. Normal processing. Begin by + ** initialize the count of updated rows */ + if( (db->flags&SQLITE_CountRows)!=0 + && !pParse->pTriggerTab + && !pParse->nested + && !pParse->bReturning + && pUpsert==0 + ){ regRowCount = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, regRowCount); } - labelBreak = sqlite3VdbeMakeLabel(v); - if( !isView ){ - /* - ** Open every index that needs updating. Note that if any - ** index could potentially invoke a REPLACE conflict resolution - ** action, then we need to open all indices because we might need - ** to be deleting some records. - */ - if( onError==OE_Replace ){ - memset(aToOpen, 1, nIdx+1); + if( nChangeFrom==0 && HasRowid(pTab) ){ + sqlite3VdbeAddOp3(v, OP_Null, 0, regRowSet, regOldRowid); + iEph = pParse->nTab++; + addrOpen = sqlite3VdbeAddOp3(v, OP_OpenEphemeral, iEph, 0, regRowSet); + }else{ + assert( pPk!=0 || HasRowid(pTab) ); + nPk = pPk ? pPk->nKeyCol : 0; + iPk = pParse->nMem+1; + pParse->nMem += nPk; + pParse->nMem += nChangeFrom; + regKey = ++pParse->nMem; + if( pUpsert==0 ){ + int nEphCol = nPk + nChangeFrom + (isView ? pTab->nCol : 0); + iEph = pParse->nTab++; + if( pPk ) sqlite3VdbeAddOp3(v, OP_Null, 0, iPk, iPk+nPk-1); + addrOpen = sqlite3VdbeAddOp2(v, OP_OpenEphemeral, iEph, nEphCol); + if( pPk ){ + KeyInfo *pKeyInfo = sqlite3KeyInfoOfIndex(pParse, pPk); + if( pKeyInfo ){ + pKeyInfo->nAllField = nEphCol; + sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); + } + } + if( nChangeFrom ){ + updateFromSelect( + pParse, iEph, pPk, pChanges, pTabList, pWhere, pOrderBy, pLimit + ); +#ifndef SQLITE_OMIT_SUBQUERY + if( isView ) iDataCur = iEph; +#endif + } + } + } + + if( nChangeFrom ){ + sqlite3MultiWrite(pParse); + eOnePass = ONEPASS_OFF; + nKey = nPk; + regKey = iPk; + }else{ + if( pUpsert ){ + /* If this is an UPSERT, then all cursors have already been opened by + ** the outer INSERT and the data cursor should be pointing at the row + ** that is to be updated. So bypass the code that searches for the + ** row(s) to be updated. + */ + pWInfo = 0; + eOnePass = ONEPASS_SINGLE; + sqlite3ExprIfFalse(pParse, pWhere, labelBreak, SQLITE_JUMPIFNULL); + bFinishSeek = 0; }else{ - for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->onError==OE_Replace ){ - memset(aToOpen, 1, nIdx+1); - break; + /* Begin the database scan. + ** + ** Do not consider a single-pass strategy for a multi-row update if + ** there is anything that might disrupt the cursor being used to do + ** the UPDATE: + ** (1) This is a nested UPDATE + ** (2) There are triggers + ** (3) There are FOREIGN KEY constraints + ** (4) There are REPLACE conflict handlers + ** (5) There are subqueries in the WHERE clause + */ + flags = WHERE_ONEPASS_DESIRED; + if( !pParse->nested + && !pTrigger + && !hasFK + && !chngKey + && !bReplace + && (pWhere==0 || !ExprHasProperty(pWhere, EP_Subquery)) + ){ + flags |= WHERE_ONEPASS_MULTIROW; + } + pWInfo = sqlite3WhereBegin(pParse, pTabList, pWhere,0,0,0,flags,iIdxCur); + if( pWInfo==0 ) goto update_cleanup; + + /* A one-pass strategy that might update more than one row may not + ** be used if any column of the index used for the scan is being + ** updated. Otherwise, if there is an index on "b", statements like + ** the following could create an infinite loop: + ** + ** UPDATE t1 SET b=b+1 WHERE b>? + ** + ** Fall back to ONEPASS_OFF if where.c has selected a ONEPASS_MULTI + ** strategy that uses an index for which one or more columns are being + ** updated. */ + eOnePass = sqlite3WhereOkOnePass(pWInfo, aiCurOnePass); + bFinishSeek = sqlite3WhereUsesDeferredSeek(pWInfo); + if( eOnePass!=ONEPASS_SINGLE ){ + sqlite3MultiWrite(pParse); + if( eOnePass==ONEPASS_MULTI ){ + int iCur = aiCurOnePass[1]; + if( iCur>=0 && iCur!=iDataCur && aToOpen[iCur-iBaseCur] ){ + eOnePass = ONEPASS_OFF; + } + assert( iCur!=iDataCur || !HasRowid(pTab) ); } } } - if( okOnePass ){ - if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; - if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; + + if( HasRowid(pTab) ){ + /* Read the rowid of the current row of the WHERE scan. In ONEPASS_OFF + ** mode, write the rowid into the FIFO. In either of the one-pass modes, + ** leave it in register regOldRowid. */ + sqlite3VdbeAddOp2(v, OP_Rowid, iDataCur, regOldRowid); + if( eOnePass==ONEPASS_OFF ){ + aRegIdx[nAllIdx] = ++pParse->nMem; + sqlite3VdbeAddOp3(v, OP_Insert, iEph, regRowSet, regOldRowid); + }else{ + if( ALWAYS(addrOpen) ) sqlite3VdbeChangeToNoop(v, addrOpen); + } + }else{ + /* Read the PK of the current row into an array of registers. In + ** ONEPASS_OFF mode, serialize the array into a record and store it in + ** the ephemeral table. Or, in ONEPASS_SINGLE or MULTI mode, change + ** the OP_OpenEphemeral instruction to a Noop (the ephemeral table + ** is not required) and leave the PK fields in the array of registers. */ + for(i=0; iaiColumn[i]>=0 ); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, + pPk->aiColumn[i], iPk+i); + } + if( eOnePass ){ + if( addrOpen ) sqlite3VdbeChangeToNoop(v, addrOpen); + nKey = nPk; + regKey = iPk; + }else{ + sqlite3VdbeAddOp4(v, OP_MakeRecord, iPk, nPk, regKey, + sqlite3IndexAffinityStr(db, pPk), nPk); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, iEph, regKey, iPk, nPk); + } } - sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, aToOpen, - 0, 0); } - /* Top of the update loop */ - if( okOnePass ){ - if( aToOpen[iDataCur-iBaseCur] && !isView ){ - assert( pPk ); - sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey, nKey); - VdbeCoverageNeverTaken(v); + if( pUpsert==0 ){ + if( nChangeFrom==0 && eOnePass!=ONEPASS_MULTI ){ + sqlite3WhereEnd(pWInfo); + } + + if( !isView ){ + int addrOnce = 0; + int iNotUsed1 = 0; + int iNotUsed2 = 0; + + /* Open every index that needs updating. */ + if( eOnePass!=ONEPASS_OFF ){ + if( aiCurOnePass[0]>=0 ) aToOpen[aiCurOnePass[0]-iBaseCur] = 0; + if( aiCurOnePass[1]>=0 ) aToOpen[aiCurOnePass[1]-iBaseCur] = 0; + } + + if( eOnePass==ONEPASS_MULTI && (nIdx-(aiCurOnePass[1]>=0))>0 ){ + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + } + sqlite3OpenTableAndIndices(pParse, pTab, OP_OpenWrite, 0, iBaseCur, + aToOpen, &iNotUsed1, &iNotUsed2); + if( addrOnce ){ + sqlite3VdbeJumpHereOrPopInst(v, addrOnce); + } + } + + /* Top of the update loop */ + if( eOnePass!=ONEPASS_OFF ){ + if( aiCurOnePass[0]!=iDataCur + && aiCurOnePass[1]!=iDataCur +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + && !isView +#endif + ){ + assert( pPk ); + sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelBreak, regKey,nKey); + VdbeCoverage(v); + } + if( eOnePass!=ONEPASS_SINGLE ){ + labelContinue = sqlite3VdbeMakeLabel(pParse); + } + sqlite3VdbeAddOp2(v, OP_IsNull, pPk ? regKey : regOldRowid, labelBreak); + VdbeCoverageIf(v, pPk==0); + VdbeCoverageIf(v, pPk!=0); + }else if( pPk || nChangeFrom ){ + labelContinue = sqlite3VdbeMakeLabel(pParse); + sqlite3VdbeAddOp2(v, OP_Rewind, iEph, labelBreak); VdbeCoverage(v); + addrTop = sqlite3VdbeCurrentAddr(v); + if( nChangeFrom ){ + if( !isView ){ + if( pPk ){ + for(i=0; i=0 ); + if( nChangeFrom==0 ){ + sqlite3ExprCode(pParse, pRowidExpr, regNewRowid); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph, iRowidExpr, regNewRowid); + } sqlite3VdbeAddOp1(v, OP_MustBeInt, regNewRowid); VdbeCoverage(v); } @@ -120326,21 +155137,26 @@ SQLITE_PRIVATE void sqlite3Update( ** information is needed */ if( chngPk || hasFK || pTrigger ){ u32 oldmask = (hasFK ? sqlite3FkOldmask(pParse, pTab) : 0); - oldmask |= sqlite3TriggerColmask(pParse, + oldmask |= sqlite3TriggerColmask(pParse, pTrigger, pChanges, 0, TRIGGER_BEFORE|TRIGGER_AFTER, pTab, onError ); for(i=0; inCol; i++){ + u32 colFlags = pTab->aCol[i].colFlags; + k = sqlite3TableColumnToStorage(pTab, i) + regOld; if( oldmask==0xffffffff || (i<32 && (oldmask & MASKBIT32(i))!=0) - || (pTab->aCol[i].colFlags & COLFLAG_PRIMKEY)!=0 + || (colFlags & COLFLAG_PRIMKEY)!=0 ){ testcase( oldmask!=0xffffffff && i==31 ); - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regOld+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, regOld+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } if( chngRowid==0 && pPk==0 ){ +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( isView ) sqlite3VdbeAddOp2(v, OP_Null, 0, regOldRowid); +#endif sqlite3VdbeAddOp2(v, OP_Copy, regOldRowid, regNewRowid); } } @@ -120353,79 +155169,119 @@ SQLITE_PRIVATE void sqlite3Update( ** If there are one or more BEFORE triggers, then do not populate the ** registers associated with columns that are (a) not modified by ** this UPDATE statement and (b) not accessed by new.* references. The - ** values for registers not modified by the UPDATE must be reloaded from - ** the database after the BEFORE triggers are fired anyway (as the trigger + ** values for registers not modified by the UPDATE must be reloaded from + ** the database after the BEFORE triggers are fired anyway (as the trigger ** may have modified them). So not loading those that are not going to ** be used eliminates some redundant opcodes. */ newmask = sqlite3TriggerColmask( pParse, pTrigger, pChanges, 1, TRIGGER_BEFORE, pTab, onError ); - for(i=0; inCol; i++){ + for(i=0, k=regNew; inCol; i++, k++){ if( i==pTab->iPKey ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); + }else if( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)!=0 ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; }else{ j = aXRef[i]; if( j>=0 ){ - sqlite3ExprCode(pParse, pChanges->a[j].pExpr, regNew+i); + if( nChangeFrom ){ + int nOff = (isView ? pTab->nCol : nPk); + assert( eOnePass==ONEPASS_OFF ); + sqlite3VdbeAddOp3(v, OP_Column, iEph, nOff+j, k); + }else{ + sqlite3ExprCode(pParse, pChanges->a[j].pExpr, k); + } }else if( 0==(tmask&TRIGGER_BEFORE) || i>31 || (newmask & MASKBIT32(i)) ){ - /* This branch loads the value of a column that will not be changed + /* This branch loads the value of a column that will not be changed ** into a register. This is done if there are no BEFORE triggers, or ** if there are one or more BEFORE triggers that use this value via ** a new.* reference in a trigger program. */ testcase( i==31 ); testcase( i==32 ); - sqlite3ExprCodeGetColumnToReg(pParse, pTab, i, iDataCur, regNew+i); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); + bFinishSeek = 0; }else{ - sqlite3VdbeAddOp2(v, OP_Null, 0, regNew+i); + sqlite3VdbeAddOp2(v, OP_Null, 0, k); } } } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif /* Fire any BEFORE UPDATE triggers. This happens before constraints are ** verified. One could argue that this is wrong. */ if( tmask&TRIGGER_BEFORE ){ sqlite3TableAffinity(v, pTab, regNew); - sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, TRIGGER_BEFORE, pTab, regOldRowid, onError, labelContinue); - /* The row-trigger may have deleted the row being updated. In this - ** case, jump to the next row. No updates or AFTER triggers are - ** required. This behavior - what happens when the row being updated - ** is deleted or renamed by a BEFORE trigger - is left undefined in the - ** documentation. - */ - if( pPk ){ - sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, labelContinue,regKey,nKey); - VdbeCoverage(v); - }else{ - sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue, regOldRowid); - VdbeCoverage(v); - } - - /* If it did not delete it, the row-trigger may still have modified - ** some of the columns of the row being updated. Load the values for - ** all columns not modified by the update statement into their - ** registers in case this has happened. - */ - for(i=0; inCol; i++){ - if( aXRef[i]<0 && i!=pTab->iPKey ){ - sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, regNew+i); + if( !isView ){ + /* The row-trigger may have deleted the row being updated. In this + ** case, jump to the next row. No updates or AFTER triggers are + ** required. This behavior - what happens when the row being updated + ** is deleted or renamed by a BEFORE trigger - is left undefined in the + ** documentation. + */ + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_NotFound,iDataCur,labelContinue,regKey,nKey); + VdbeCoverage(v); + }else{ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); + VdbeCoverage(v); } + + /* After-BEFORE-trigger-reload-loop: + ** If it did not delete it, the BEFORE trigger may still have modified + ** some of the columns of the row being updated. Load the values for + ** all columns not modified by the update statement into their registers + ** in case this has happened. Only unmodified columns are reloaded. + ** The values computed for modified columns use the values before the + ** BEFORE trigger runs. See test case trigger1-18.0 (added 2018-04-26) + ** for an example. + */ + for(i=0, k=regNew; inCol; i++, k++){ + if( pTab->aCol[i].colFlags & COLFLAG_GENERATED ){ + if( pTab->aCol[i].colFlags & COLFLAG_VIRTUAL ) k--; + }else if( aXRef[i]<0 && i!=pTab->iPKey ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iDataCur, i, k); + } + } +#ifndef SQLITE_OMIT_GENERATED_COLUMNS + if( pTab->tabFlags & TF_HasGenerated ){ + testcase( pTab->tabFlags & TF_HasVirtual ); + testcase( pTab->tabFlags & TF_HasStored ); + sqlite3ComputeGeneratedColumns(pParse, regNew, pTab); + } +#endif } } if( !isView ){ - int addr1 = 0; /* Address of jump instruction */ - int bReplace = 0; /* True if REPLACE conflict resolution might happen */ - /* Do constraint checks. */ assert( regOldRowid>0 ); sqlite3GenerateConstraintChecks(pParse, pTab, aRegIdx, iDataCur, iIdxCur, regNewRowid, regOldRowid, chngKey, onError, labelContinue, &bReplace, - aXRef); + aXRef, 0); + + /* If REPLACE conflict handling may have been used, or if the PK of the + ** row is changing, then the GenerateConstraintChecks() above may have + ** moved cursor iDataCur. Reseek it. */ + if( bReplace || chngKey ){ + if( pPk ){ + sqlite3VdbeAddOp4Int(v, OP_NotFound,iDataCur,labelContinue,regKey,nKey); + }else{ + sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, labelContinue,regOldRowid); + } + VdbeCoverage(v); + } /* Do FK constraint checks. */ if( hasFK ){ @@ -120433,21 +155289,22 @@ SQLITE_PRIVATE void sqlite3Update( } /* Delete the index entries associated with the current record. */ - if( bReplace || chngKey ){ - if( pPk ){ - addr1 = sqlite3VdbeAddOp4Int(v, OP_NotFound, iDataCur, 0, regKey, nKey); - }else{ - addr1 = sqlite3VdbeAddOp3(v, OP_NotExists, iDataCur, 0, regOldRowid); - } - VdbeCoverageNeverTaken(v); - } sqlite3GenerateRowIndexDelete(pParse, pTab, iDataCur, iIdxCur, aRegIdx, -1); + /* We must run the OP_FinishSeek opcode to resolve a prior + ** OP_DeferredSeek if there is any possibility that there have been + ** no OP_Column opcodes since the OP_DeferredSeek was issued. But + ** we want to avoid the OP_FinishSeek if possible, as running it + ** costs CPU cycles. */ + if( bFinishSeek ){ + sqlite3VdbeAddOp1(v, OP_FinishSeek, iDataCur); + } + /* If changing the rowid value, or if there are foreign key constraints ** to process, delete the old record. Otherwise, add a noop OP_Delete ** to invoke the pre-update hook. ** - ** That (regNew==regnewRowid+1) is true is also important for the + ** That (regNew==regnewRowid+1) is true is also important for the ** pre-update hook. If the caller invokes preupdate_new(), the returned ** value is copied from memory cell (regNewRowid+1+iCol), where iCol ** is the column index supplied by the user. @@ -120455,85 +155312,80 @@ SQLITE_PRIVATE void sqlite3Update( assert( regNew==regNewRowid+1 ); #ifdef SQLITE_ENABLE_PREUPDATE_HOOK sqlite3VdbeAddOp3(v, OP_Delete, iDataCur, - OPFLAG_ISUPDATE | ((hasFK || chngKey || pPk!=0) ? 0 : OPFLAG_ISNOOP), + OPFLAG_ISUPDATE | ((hasFK>1 || chngKey) ? 0 : OPFLAG_ISNOOP), regNewRowid ); + if( eOnePass==ONEPASS_MULTI ){ + assert( hasFK==0 && chngKey==0 ); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } if( !pParse->nested ){ - sqlite3VdbeChangeP4(v, -1, (char*)pTab, P4_TABLE); + sqlite3VdbeAppendP4(v, pTab, P4_TABLE); } #else - if( hasFK || chngKey || pPk!=0 ){ + if( hasFK>1 || chngKey ){ sqlite3VdbeAddOp2(v, OP_Delete, iDataCur, 0); } #endif - if( bReplace || chngKey ){ - sqlite3VdbeJumpHere(v, addr1); - } if( hasFK ){ sqlite3FkCheck(pParse, pTab, 0, regNewRowid, aXRef, chngKey); } - + /* Insert the new index entries and the new record. */ - sqlite3CompleteInsertion(pParse, pTab, iDataCur, iIdxCur, - regNewRowid, aRegIdx, 1, 0, 0); + sqlite3CompleteInsertion( + pParse, pTab, iDataCur, iIdxCur, regNewRowid, aRegIdx, + OPFLAG_ISUPDATE | (eOnePass==ONEPASS_MULTI ? OPFLAG_SAVEPOSITION : 0), + 0, 0 + ); /* Do any ON CASCADE, SET NULL or SET DEFAULT operations required to ** handle rows (possibly in other tables) that refer via a foreign key - ** to the row just updated. */ + ** to the row just updated. */ if( hasFK ){ sqlite3FkActions(pParse, pTab, pChanges, regOldRowid, aXRef, chngKey); } } - /* Increment the row counter + /* Increment the row counter */ - if( (db->flags & SQLITE_CountRows) && !pParse->pTriggerTab){ + if( regRowCount ){ sqlite3VdbeAddOp2(v, OP_AddImm, regRowCount, 1); } - sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, - TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + if( pTrigger ){ + sqlite3CodeRowTrigger(pParse, pTrigger, TK_UPDATE, pChanges, + TRIGGER_AFTER, pTab, regOldRowid, onError, labelContinue); + } /* Repeat the above with the next record to be updated, until ** all record selected by the WHERE clause have been updated. */ - if( okOnePass ){ + if( eOnePass==ONEPASS_SINGLE ){ /* Nothing to do at end-of-loop for a single-pass */ - }else if( pPk ){ + }else if( eOnePass==ONEPASS_MULTI ){ + sqlite3VdbeResolveLabel(v, labelContinue); + sqlite3WhereEnd(pWInfo); + }else{ sqlite3VdbeResolveLabel(v, labelContinue); sqlite3VdbeAddOp2(v, OP_Next, iEph, addrTop); VdbeCoverage(v); - }else{ - sqlite3VdbeGoto(v, labelContinue); } sqlite3VdbeResolveLabel(v, labelBreak); - /* Close all tables */ - for(i=0, pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext, i++){ - assert( aRegIdx ); - if( aToOpen[i+1] ){ - sqlite3VdbeAddOp2(v, OP_Close, iIdxCur+i, 0); - } - } - if( iDataCurnested==0 && pParse->pTriggerTab==0 ){ + if( pParse->nested==0 && pParse->pTriggerTab==0 && pUpsert==0 ){ sqlite3AutoincrementEnd(pParse); } /* - ** Return the number of rows that were changed. If this routine is - ** generating code because of a call to sqlite3NestedParse(), do not - ** invoke the callback function. + ** Return the number of rows that were changed, if we are tracking + ** that information. */ - if( (db->flags&SQLITE_CountRows) && !pParse->pTriggerTab && !pParse->nested ){ - sqlite3VdbeAddOp2(v, OP_ResultRow, regRowCount, 1); - sqlite3VdbeSetNumCols(v, 1); - sqlite3VdbeSetColName(v, 0, COLNAME_NAME, "rows updated", SQLITE_STATIC); + if( regRowCount ){ + sqlite3CodeChangeCount(v, regRowCount, "rows updated"); } update_cleanup: @@ -120542,6 +155394,10 @@ update_cleanup: sqlite3SrcListDelete(db, pTabList); sqlite3ExprListDelete(db, pChanges); sqlite3ExprDelete(db, pWhere); +#if defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) + sqlite3ExprListDelete(db, pOrderBy); + sqlite3ExprDelete(db, pLimit); +#endif return; } /* Make sure "isView" and other macros defined above are undefined. Otherwise @@ -120558,8 +155414,8 @@ update_cleanup: /* ** Generate code for an UPDATE of a virtual table. ** -** There are two possible strategies - the default and the special -** "onepass" strategy. Onepass is only used if the virtual table +** There are two possible strategies - the default and the special +** "onepass" strategy. Onepass is only used if the virtual table ** implementation indicates that pWhere may match at most one row. ** ** The default strategy is to create an ephemeral table that contains @@ -120591,17 +155447,17 @@ static void updateVirtualTable( int i; /* Loop counter */ sqlite3 *db = pParse->db; /* Database connection */ const char *pVTab = (const char*)sqlite3GetVTable(db, pTab); - WhereInfo *pWInfo; + WhereInfo *pWInfo = 0; int nArg = 2 + pTab->nCol; /* Number of arguments to VUpdate */ int regArg; /* First register in VUpdate arg array */ int regRec; /* Register in which to assemble record */ - int regRowid; /* Register for ephem table rowid */ + int regRowid; /* Register for ephemeral table rowid */ int iCsr = pSrc->a[0].iCursor; /* Cursor used for virtual table scan */ int aDummy[2]; /* Unused arg for sqlite3WhereOkOnePass() */ - int bOnePass; /* True to use onepass strategy */ + int eOnePass; /* True to use onepass strategy */ int addr; /* Address of OP_OpenEphemeral */ - /* Allocate nArg registers to martial the arguments to VUpdate. Then + /* Allocate nArg registers in which to gather the arguments for VUpdate. Then ** create and open the ephemeral table in which the records created from ** these arguments will be temporarily stored. */ assert( v ); @@ -120609,55 +155465,119 @@ static void updateVirtualTable( addr= sqlite3VdbeAddOp2(v, OP_OpenEphemeral, ephemTab, nArg); regArg = pParse->nMem + 1; pParse->nMem += nArg; - regRec = ++pParse->nMem; - regRowid = ++pParse->nMem; - - /* Start scanning the virtual table */ - pWInfo = sqlite3WhereBegin(pParse, pSrc, pWhere, 0,0,WHERE_ONEPASS_DESIRED,0); - if( pWInfo==0 ) return; - - /* Populate the argument registers. */ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); - if( pRowid ){ - sqlite3ExprCode(pParse, pRowid, regArg+1); - }else{ - sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); - } - for(i=0; inCol; i++){ - if( aXRef[i]>=0 ){ - sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); + if( pSrc->nSrc>1 ){ + Index *pPk = 0; + Expr *pRow; + ExprList *pList; + if( HasRowid(pTab) ){ + if( pRowid ){ + pRow = sqlite3ExprDup(db, pRowid, 0); + }else{ + pRow = sqlite3PExpr(pParse, TK_ROW, 0, 0); + } }else{ - sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); + i16 iPk; /* PRIMARY KEY column */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol==1 ); + iPk = pPk->aiColumn[0]; + if( aXRef[iPk]>=0 ){ + pRow = sqlite3ExprDup(db, pChanges->a[aXRef[iPk]].pExpr, 0); + }else{ + pRow = exprRowColumn(pParse, iPk); + } } - } + pList = sqlite3ExprListAppend(pParse, 0, pRow); - bOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); - - if( bOnePass ){ - /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded - ** above. Also, if this is a top-level parse (not a trigger), clear the - ** multi-write flag so that the VM does not open a statement journal */ - sqlite3VdbeChangeToNoop(v, addr); - if( sqlite3IsToplevel(pParse) ){ - pParse->isMultiWrite = 0; + for(i=0; inCol; i++){ + if( aXRef[i]>=0 ){ + pList = sqlite3ExprListAppend(pParse, pList, + sqlite3ExprDup(db, pChanges->a[aXRef[i]].pExpr, 0) + ); + }else{ + Expr *pRowExpr = exprRowColumn(pParse, i); + if( pRowExpr ) pRowExpr->op2 = OPFLAG_NOCHNG; + pList = sqlite3ExprListAppend(pParse, pList, pRowExpr); + } } + + updateFromSelect(pParse, ephemTab, pPk, pList, pSrc, pWhere, 0, 0); + sqlite3ExprListDelete(db, pList); + eOnePass = ONEPASS_OFF; }else{ - /* Create a record from the argument register contents and insert it into - ** the ephemeral table. */ - sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); - sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); - sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + regRec = ++pParse->nMem; + regRowid = ++pParse->nMem; + + /* Start scanning the virtual table */ + pWInfo = sqlite3WhereBegin( + pParse, pSrc, pWhere, 0, 0, 0, WHERE_ONEPASS_DESIRED, 0 + ); + if( pWInfo==0 ) return; + + /* Populate the argument registers. */ + for(i=0; inCol; i++){ + assert( (pTab->aCol[i].colFlags & COLFLAG_GENERATED)==0 ); + if( aXRef[i]>=0 ){ + sqlite3ExprCode(pParse, pChanges->a[aXRef[i]].pExpr, regArg+2+i); + }else{ + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, i, regArg+2+i); + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG);/* For sqlite3_vtab_nochange() */ + } + } + if( HasRowid(pTab) ){ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg); + if( pRowid ){ + sqlite3ExprCode(pParse, pRowid, regArg+1); + }else{ + sqlite3VdbeAddOp2(v, OP_Rowid, iCsr, regArg+1); + } + }else{ + Index *pPk; /* PRIMARY KEY index */ + i16 iPk; /* PRIMARY KEY column */ + pPk = sqlite3PrimaryKeyIndex(pTab); + assert( pPk!=0 ); + assert( pPk->nKeyCol==1 ); + iPk = pPk->aiColumn[0]; + sqlite3VdbeAddOp3(v, OP_VColumn, iCsr, iPk, regArg); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg+2+iPk, regArg+1); + } + + eOnePass = sqlite3WhereOkOnePass(pWInfo, aDummy); + + /* There is no ONEPASS_MULTI on virtual tables */ + assert( eOnePass==ONEPASS_OFF || eOnePass==ONEPASS_SINGLE ); + + if( eOnePass ){ + /* If using the onepass strategy, no-op out the OP_OpenEphemeral coded + ** above. */ + sqlite3VdbeChangeToNoop(v, addr); + sqlite3VdbeAddOp1(v, OP_Close, iCsr); + }else{ + /* Create a record from the argument register contents and insert it into + ** the ephemeral table. */ + sqlite3MultiWrite(pParse); + sqlite3VdbeAddOp3(v, OP_MakeRecord, regArg, nArg, regRec); +#if defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_NULL_TRIM) + /* Signal an assert() within OP_MakeRecord that it is allowed to + ** accept no-change records with serial_type 10 */ + sqlite3VdbeChangeP5(v, OPFLAG_NOCHNG_MAGIC); +#endif + sqlite3VdbeAddOp2(v, OP_NewRowid, ephemTab, regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, ephemTab, regRec, regRowid); + } } - if( bOnePass==0 ){ + if( eOnePass==ONEPASS_OFF ){ /* End the virtual table scan */ - sqlite3WhereEnd(pWInfo); + if( pSrc->nSrc==1 ){ + sqlite3WhereEnd(pWInfo); + } - /* Begin scannning through the ephemeral table. */ + /* Begin scanning through the ephemeral table. */ addr = sqlite3VdbeAddOp1(v, OP_Rewind, ephemTab); VdbeCoverage(v); - /* Extract arguments from the current row of the ephemeral table and + /* Extract arguments from the current row of the ephemeral table and ** invoke the VUpdate method. */ for(i=0; ipNextUpsert; + sqlite3ExprListDelete(db, p->pUpsertTarget); + sqlite3ExprDelete(db, p->pUpsertTargetWhere); + sqlite3ExprListDelete(db, p->pUpsertSet); + sqlite3ExprDelete(db, p->pUpsertWhere); + sqlite3DbFree(db, p->pToFree); + sqlite3DbFree(db, p); + p = pNext; + }while( p ); +} +SQLITE_PRIVATE void sqlite3UpsertDelete(sqlite3 *db, Upsert *p){ + if( p ) upsertDelete(db, p); +} + + +/* +** Duplicate an Upsert object. +*/ +SQLITE_PRIVATE Upsert *sqlite3UpsertDup(sqlite3 *db, Upsert *p){ + if( p==0 ) return 0; + return sqlite3UpsertNew(db, + sqlite3ExprListDup(db, p->pUpsertTarget, 0), + sqlite3ExprDup(db, p->pUpsertTargetWhere, 0), + sqlite3ExprListDup(db, p->pUpsertSet, 0), + sqlite3ExprDup(db, p->pUpsertWhere, 0), + sqlite3UpsertDup(db, p->pNextUpsert) + ); +} + +/* +** Create a new Upsert object. +*/ +SQLITE_PRIVATE Upsert *sqlite3UpsertNew( + sqlite3 *db, /* Determines which memory allocator to use */ + ExprList *pTarget, /* Target argument to ON CONFLICT, or NULL */ + Expr *pTargetWhere, /* Optional WHERE clause on the target */ + ExprList *pSet, /* UPDATE columns, or NULL for a DO NOTHING */ + Expr *pWhere, /* WHERE clause for the ON CONFLICT UPDATE */ + Upsert *pNext /* Next ON CONFLICT clause in the list */ +){ + Upsert *pNew; + pNew = sqlite3DbMallocZero(db, sizeof(Upsert)); + if( pNew==0 ){ + sqlite3ExprListDelete(db, pTarget); + sqlite3ExprDelete(db, pTargetWhere); + sqlite3ExprListDelete(db, pSet); + sqlite3ExprDelete(db, pWhere); + sqlite3UpsertDelete(db, pNext); + return 0; + }else{ + pNew->pUpsertTarget = pTarget; + pNew->pUpsertTargetWhere = pTargetWhere; + pNew->pUpsertSet = pSet; + pNew->pUpsertWhere = pWhere; + pNew->isDoUpdate = pSet!=0; + pNew->pNextUpsert = pNext; + } + return pNew; +} + +/* +** Analyze the ON CONFLICT clause described by pUpsert. Resolve all +** symbols in the conflict-target. +** +** Return SQLITE_OK if everything works, or an error code is something +** is wrong. +*/ +SQLITE_PRIVATE int sqlite3UpsertAnalyzeTarget( + Parse *pParse, /* The parsing context */ + SrcList *pTabList, /* Table into which we are inserting */ + Upsert *pUpsert, /* The ON CONFLICT clauses */ + Upsert *pAll /* Complete list of all ON CONFLICT clauses */ +){ + Table *pTab; /* That table into which we are inserting */ + int rc; /* Result code */ + int iCursor; /* Cursor used by pTab */ + Index *pIdx; /* One of the indexes of pTab */ + ExprList *pTarget; /* The conflict-target clause */ + Expr *pTerm; /* One term of the conflict-target clause */ + NameContext sNC; /* Context for resolving symbolic names */ + Expr sCol[2]; /* Index column converted into an Expr */ + int nClause = 0; /* Counter of ON CONFLICT clauses */ + + assert( pTabList->nSrc==1 ); + assert( pTabList->a[0].pSTab!=0 ); + assert( pUpsert!=0 ); + assert( pUpsert->pUpsertTarget!=0 ); + + /* Resolve all symbolic names in the conflict-target clause, which + ** includes both the list of columns and the optional partial-index + ** WHERE clause. + */ + memset(&sNC, 0, sizeof(sNC)); + sNC.pParse = pParse; + sNC.pSrcList = pTabList; + for(; pUpsert && pUpsert->pUpsertTarget; + pUpsert=pUpsert->pNextUpsert, nClause++){ + rc = sqlite3ResolveExprListNames(&sNC, pUpsert->pUpsertTarget); + if( rc ) return rc; + rc = sqlite3ResolveExprNames(&sNC, pUpsert->pUpsertTargetWhere); + if( rc ) return rc; + + /* Check to see if the conflict target matches the rowid. */ + pTab = pTabList->a[0].pSTab; + pTarget = pUpsert->pUpsertTarget; + iCursor = pTabList->a[0].iCursor; + if( HasRowid(pTab) + && pTarget->nExpr==1 + && (pTerm = pTarget->a[0].pExpr)->op==TK_COLUMN + && pTerm->iColumn==XN_ROWID + ){ + /* The conflict-target is the rowid of the primary table */ + assert( pUpsert->pUpsertIdx==0 ); + continue; + } + + /* Initialize sCol[0..1] to be an expression parse tree for a + ** single column of an index. The sCol[0] node will be the TK_COLLATE + ** operator and sCol[1] will be the TK_COLUMN operator. Code below + ** will populate the specific collation and column number values + ** prior to comparing against the conflict-target expression. + */ + memset(sCol, 0, sizeof(sCol)); + sCol[0].op = TK_COLLATE; + sCol[0].pLeft = &sCol[1]; + sCol[1].op = TK_COLUMN; + sCol[1].iTable = pTabList->a[0].iCursor; + + /* Check for matches against other indexes */ + for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ + int ii, jj, nn; + if( !IsUniqueIndex(pIdx) ) continue; + if( pTarget->nExpr!=pIdx->nKeyCol ) continue; + if( pIdx->pPartIdxWhere ){ + if( pUpsert->pUpsertTargetWhere==0 ) continue; + if( sqlite3ExprCompare(pParse, pUpsert->pUpsertTargetWhere, + pIdx->pPartIdxWhere, iCursor)!=0 ){ + continue; + } + } + nn = pIdx->nKeyCol; + for(ii=0; iiazColl[ii]; + if( pIdx->aiColumn[ii]==XN_EXPR ){ + assert( pIdx->aColExpr!=0 ); + assert( pIdx->aColExpr->nExpr>ii ); + assert( pIdx->bHasExpr ); + pExpr = pIdx->aColExpr->a[ii].pExpr; + if( pExpr->op!=TK_COLLATE ){ + sCol[0].pLeft = pExpr; + pExpr = &sCol[0]; + } + }else{ + sCol[0].pLeft = &sCol[1]; + sCol[1].iColumn = pIdx->aiColumn[ii]; + pExpr = &sCol[0]; + } + for(jj=0; jja[jj].pExpr,pExpr,iCursor)<2 ){ + break; /* Column ii of the index matches column jj of target */ + } + } + if( jj>=nn ){ + /* The target contains no match for column jj of the index */ + break; + } + } + if( iipUpsertIdx = pIdx; + if( sqlite3UpsertOfIndex(pAll,pIdx)!=pUpsert ){ + /* Really this should be an error. The isDup ON CONFLICT clause will + ** never fire. But this problem was not discovered until three years + ** after multi-CONFLICT upsert was added, and so we silently ignore + ** the problem to prevent breaking applications that might actually + ** have redundant ON CONFLICT clauses. */ + pUpsert->isDup = 1; + } + break; + } + if( pUpsert->pUpsertIdx==0 ){ + char zWhich[16]; + if( nClause==0 && pUpsert->pNextUpsert==0 ){ + zWhich[0] = 0; + }else{ + sqlite3_snprintf(sizeof(zWhich),zWhich,"%r ", nClause+1); + } + sqlite3ErrorMsg(pParse, "%sON CONFLICT clause does not match any " + "PRIMARY KEY or UNIQUE constraint", zWhich); + return SQLITE_ERROR; + } + } + return SQLITE_OK; +} + +/* +** Return true if pUpsert is the last ON CONFLICT clause with a +** conflict target, or if pUpsert is followed by another ON CONFLICT +** clause that targets the INTEGER PRIMARY KEY. +*/ +SQLITE_PRIVATE int sqlite3UpsertNextIsIPK(Upsert *pUpsert){ + Upsert *pNext; + if( NEVER(pUpsert==0) ) return 0; + pNext = pUpsert->pNextUpsert; + while( 1 /*exit-by-return*/ ){ + if( pNext==0 ) return 1; + if( pNext->pUpsertTarget==0 ) return 1; + if( pNext->pUpsertIdx==0 ) return 1; + if( !pNext->isDup ) return 0; + pNext = pNext->pNextUpsert; + } + return 0; +} + +/* +** Given the list of ON CONFLICT clauses described by pUpsert, and +** a particular index pIdx, return a pointer to the particular ON CONFLICT +** clause that applies to the index. Or, if the index is not subject to +** any ON CONFLICT clause, return NULL. +*/ +SQLITE_PRIVATE Upsert *sqlite3UpsertOfIndex(Upsert *pUpsert, Index *pIdx){ + while( + pUpsert + && pUpsert->pUpsertTarget!=0 + && pUpsert->pUpsertIdx!=pIdx + ){ + pUpsert = pUpsert->pNextUpsert; + } + return pUpsert; +} + +/* +** Generate bytecode that does an UPDATE as part of an upsert. +** +** If pIdx is NULL, then the UNIQUE constraint that failed was the IPK. +** In this case parameter iCur is a cursor open on the table b-tree that +** currently points to the conflicting table row. Otherwise, if pIdx +** is not NULL, then pIdx is the constraint that failed and iCur is a +** cursor points to the conflicting row. +*/ +SQLITE_PRIVATE void sqlite3UpsertDoUpdate( + Parse *pParse, /* The parsing and code-generating context */ + Upsert *pUpsert, /* The ON CONFLICT clause for the upsert */ + Table *pTab, /* The table being updated */ + Index *pIdx, /* The UNIQUE constraint that failed */ + int iCur /* Cursor for pIdx (or pTab if pIdx==NULL) */ +){ + Vdbe *v = pParse->pVdbe; + sqlite3 *db = pParse->db; + SrcList *pSrc; /* FROM clause for the UPDATE */ + int iDataCur; + int i; + Upsert *pTop = pUpsert; + + assert( v!=0 ); + assert( pUpsert!=0 ); + iDataCur = pUpsert->iDataCur; + pUpsert = sqlite3UpsertOfIndex(pTop, pIdx); + VdbeNoopComment((v, "Begin DO UPDATE of UPSERT")); + if( pIdx && iCur!=iDataCur ){ + if( HasRowid(pTab) ){ + int regRowid = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_IdxRowid, iCur, regRowid); + sqlite3VdbeAddOp3(v, OP_SeekRowid, iDataCur, 0, regRowid); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regRowid); + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + int nPk = pPk->nKeyCol; + int iPk = pParse->nMem+1; + pParse->nMem += nPk; + for(i=0; iaiColumn[i]>=0 ); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[i]); + sqlite3VdbeAddOp3(v, OP_Column, iCur, k, iPk+i); + VdbeComment((v, "%s.%s", pIdx->zName, + pTab->aCol[pPk->aiColumn[i]].zCnName)); + } + sqlite3VdbeVerifyAbortable(v, OE_Abort); + i = sqlite3VdbeAddOp4Int(v, OP_Found, iDataCur, 0, iPk, nPk); + VdbeCoverage(v); + sqlite3VdbeAddOp4(v, OP_Halt, SQLITE_CORRUPT, OE_Abort, 0, + "corrupt database", P4_STATIC); + sqlite3MayAbort(pParse); + sqlite3VdbeJumpHere(v, i); + } + } + /* pUpsert does not own pTop->pUpsertSrc - the outer INSERT statement does. + ** So we have to make a copy before passing it down into sqlite3Update() */ + pSrc = sqlite3SrcListDup(db, pTop->pUpsertSrc, 0); + /* excluded.* columns of type REAL need to be converted to a hard real */ + for(i=0; inCol; i++){ + if( pTab->aCol[i].affinity==SQLITE_AFF_REAL ){ + sqlite3VdbeAddOp1(v, OP_RealAffinity, pTop->regData+i); + } + } + sqlite3Update(pParse, pSrc, sqlite3ExprListDup(db,pUpsert->pUpsertSet,0), + sqlite3ExprDup(db,pUpsert->pUpsertWhere,0), OE_Abort, 0, 0, pUpsert); + VdbeNoopComment((v, "End DO UPDATE of UPSERT")); +} + +#endif /* SQLITE_OMIT_UPSERT */ + +/************** End of upsert.c **********************************************/ /************** Begin file vacuum.c ******************************************/ /* ** 2003 April 6 @@ -120702,57 +155954,58 @@ static void updateVirtualTable( /* #include "vdbeInt.h" */ #if !defined(SQLITE_OMIT_VACUUM) && !defined(SQLITE_OMIT_ATTACH) -/* -** Finalize a prepared statement. If there was an error, store the -** text of the error message in *pzErrMsg. Return the result code. -*/ -static int vacuumFinalize(sqlite3 *db, sqlite3_stmt *pStmt, char **pzErrMsg){ - int rc; - rc = sqlite3VdbeFinalize((Vdbe*)pStmt); - if( rc ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - } - return rc; -} /* -** Execute zSql on database db. Return an error code. +** Execute zSql on database db. +** +** If zSql returns rows, then each row will have exactly one +** column. (This will only happen if zSql begins with "SELECT".) +** Take each row of result and call execSql() again recursively. +** +** The execSqlF() routine does the same thing, except it accepts +** a format string as its third argument */ static int execSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ - sqlite3_stmt *pStmt; - VVA_ONLY( int rc; ) - if( !zSql ){ - return SQLITE_NOMEM_BKPT; - } - if( SQLITE_OK!=sqlite3_prepare(db, zSql, -1, &pStmt, 0) ){ - sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); - return sqlite3_errcode(db); - } - VVA_ONLY( rc = ) sqlite3_step(pStmt); - assert( rc!=SQLITE_ROW || (db->flags&SQLITE_CountRows) ); - return vacuumFinalize(db, pStmt, pzErrMsg); -} - -/* -** Execute zSql on database db. The statement returns exactly -** one column. Execute this as SQL on the same database. -*/ -static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ sqlite3_stmt *pStmt; int rc; - rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0); + /* printf("SQL: [%s]\n", zSql); fflush(stdout); */ + rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; - - while( SQLITE_ROW==sqlite3_step(pStmt) ){ - rc = execSql(db, pzErrMsg, (char*)sqlite3_column_text(pStmt, 0)); - if( rc!=SQLITE_OK ){ - vacuumFinalize(db, pStmt, pzErrMsg); - return rc; + while( SQLITE_ROW==(rc = sqlite3_step(pStmt)) ){ + const char *zSubSql = (const char*)sqlite3_column_text(pStmt,0); + assert( sqlite3_strnicmp(zSql,"SELECT",6)==0 ); + /* The secondary SQL must be one of CREATE TABLE, CREATE INDEX, + ** or INSERT. Historically there have been attacks that first + ** corrupt the sqlite_schema.sql field with other kinds of statements + ** then run VACUUM to get those statements to execute at inappropriate + ** times. */ + if( zSubSql + && (strncmp(zSubSql,"CRE",3)==0 || strncmp(zSubSql,"INS",3)==0) + ){ + rc = execSql(db, pzErrMsg, zSubSql); + if( rc!=SQLITE_OK ) break; } } - - return vacuumFinalize(db, pStmt, pzErrMsg); + assert( rc!=SQLITE_ROW ); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + if( rc ){ + sqlite3SetString(pzErrMsg, db, sqlite3_errmsg(db)); + } + (void)sqlite3_finalize(pStmt); + return rc; +} +static int execSqlF(sqlite3 *db, char **pzErrMsg, const char *zSql, ...){ + char *z; + va_list ap; + int rc; + va_start(ap, zSql); + z = sqlite3VMPrintf(db, zSql, ap); + va_end(ap); + if( z==0 ) return SQLITE_NOMEM; + rc = execSql(db, pzErrMsg, z); + sqlite3DbFree(db, z); + return rc; } /* @@ -120785,117 +156038,168 @@ static int execExecSql(sqlite3 *db, char **pzErrMsg, const char *zSql){ ** transient would cause the database file to appear to be deleted ** following reboot. */ -SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse){ +SQLITE_PRIVATE void sqlite3Vacuum(Parse *pParse, Token *pNm, Expr *pInto){ Vdbe *v = sqlite3GetVdbe(pParse); - if( v ){ - sqlite3VdbeAddOp2(v, OP_Vacuum, 0, 0); - sqlite3VdbeUsesBtree(v, 0); + int iDb = 0; + if( v==0 ) goto build_vacuum_end; + if( pParse->nErr ) goto build_vacuum_end; + if( pNm ){ +#ifndef SQLITE_BUG_COMPATIBLE_20160819 + /* Default behavior: Report an error if the argument to VACUUM is + ** not recognized */ + iDb = sqlite3TwoPartName(pParse, pNm, pNm, &pNm); + if( iDb<0 ) goto build_vacuum_end; +#else + /* When SQLITE_BUG_COMPATIBLE_20160819 is defined, unrecognized arguments + ** to VACUUM are silently ignored. This is a back-out of a bug fix that + ** occurred on 2016-08-19 (https://www.sqlite.org/src/info/083f9e6270). + ** The buggy behavior is required for binary compatibility with some + ** legacy applications. */ + iDb = sqlite3FindDb(pParse->db, pNm); + if( iDb<0 ) iDb = 0; +#endif } + if( iDb!=1 ){ + int iIntoReg = 0; + if( pInto && sqlite3ResolveSelfReference(pParse,0,0,pInto,0)==0 ){ + iIntoReg = ++pParse->nMem; + sqlite3ExprCode(pParse, pInto, iIntoReg); + } + sqlite3VdbeAddOp2(v, OP_Vacuum, iDb, iIntoReg); + sqlite3VdbeUsesBtree(v, iDb); + } +build_vacuum_end: + sqlite3ExprDelete(pParse->db, pInto); return; } /* ** This routine implements the OP_Vacuum opcode of the VDBE. */ -SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ +SQLITE_PRIVATE SQLITE_NOINLINE int sqlite3RunVacuum( + char **pzErrMsg, /* Write error message here */ + sqlite3 *db, /* Database connection */ + int iDb, /* Which attached DB to vacuum */ + sqlite3_value *pOut /* Write results here, if not NULL. VACUUM INTO */ +){ int rc = SQLITE_OK; /* Return code from service routines */ Btree *pMain; /* The database being vacuumed */ Btree *pTemp; /* The temporary database we vacuum into */ - char *zSql = 0; /* SQL statements */ - int saved_flags; /* Saved value of the db->flags */ - int saved_nChange; /* Saved value of db->nChange */ - int saved_nTotalChange; /* Saved value of db->nTotalChange */ - void (*saved_xTrace)(void*,const char*); /* Saved db->xTrace */ + u32 saved_mDbFlags; /* Saved value of db->mDbFlags */ + u64 saved_flags; /* Saved value of db->flags */ + i64 saved_nChange; /* Saved value of db->nChange */ + i64 saved_nTotalChange; /* Saved value of db->nTotalChange */ + u32 saved_openFlags; /* Saved value of db->openFlags */ + u8 saved_mTrace; /* Saved trace settings */ Db *pDb = 0; /* Database to detach at end of vacuum */ int isMemDb; /* True if vacuuming a :memory: database */ int nRes; /* Bytes of reserved space at the end of each page */ int nDb; /* Number of attached databases */ + const char *zDbMain; /* Schema name of database to vacuum */ + const char *zOut; /* Name of output file */ + u32 pgflags = PAGER_SYNCHRONOUS_OFF; /* sync flags for output db */ + u64 iRandom; /* Random value used for zDbVacuum[] */ + char zDbVacuum[42]; /* Name of the ATTACH-ed database used for vacuum */ + if( !db->autoCommit ){ sqlite3SetString(pzErrMsg, db, "cannot VACUUM from within a transaction"); - return SQLITE_ERROR; + return SQLITE_ERROR; /* IMP: R-12218-18073 */ } if( db->nVdbeActive>1 ){ sqlite3SetString(pzErrMsg, db,"cannot VACUUM - SQL statements in progress"); - return SQLITE_ERROR; + return SQLITE_ERROR; /* IMP: R-15610-35227 */ + } + saved_openFlags = db->openFlags; + if( pOut ){ + if( sqlite3_value_type(pOut)!=SQLITE_TEXT ){ + sqlite3SetString(pzErrMsg, db, "non-text filename"); + return SQLITE_ERROR; + } + zOut = (const char*)sqlite3_value_text(pOut); + db->openFlags &= ~SQLITE_OPEN_READONLY; + db->openFlags |= SQLITE_OPEN_CREATE|SQLITE_OPEN_READWRITE; + }else{ + zOut = ""; } - /* Save the current value of the database flags so that it can be + /* Save the current value of the database flags so that it can be ** restored before returning. Then set the writable-schema flag, and ** disable CHECK and foreign key constraints. */ saved_flags = db->flags; + saved_mDbFlags = db->mDbFlags; saved_nChange = db->nChange; saved_nTotalChange = db->nTotalChange; - saved_xTrace = db->xTrace; - db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks | SQLITE_PreferBuiltin; - db->flags &= ~(SQLITE_ForeignKeys | SQLITE_ReverseOrder); - db->xTrace = 0; + saved_mTrace = db->mTrace; + db->flags |= SQLITE_WriteSchema | SQLITE_IgnoreChecks; + db->mDbFlags |= DBFLAG_PreferBuiltin | DBFLAG_Vacuum; + db->flags &= ~(u64)(SQLITE_ForeignKeys | SQLITE_ReverseOrder + | SQLITE_Defensive | SQLITE_CountRows); + db->mTrace = 0; - pMain = db->aDb[0].pBt; + zDbMain = db->aDb[iDb].zDbSName; + pMain = db->aDb[iDb].pBt; isMemDb = sqlite3PagerIsMemdb(sqlite3BtreePager(pMain)); - /* Attach the temporary database as 'vacuum_db'. The synchronous pragma + /* Attach the temporary database as 'vacuum_XXXXXX'. The synchronous pragma ** can be set to 'off' for this file, as it is not recovered if a crash ** occurs anyway. The integrity of the database is maintained by a ** (possibly synchronous) transaction opened on the main database before ** sqlite3BtreeCopyFile() is called. ** - ** An optimisation would be to use a non-journaled pager. - ** (Later:) I tried setting "PRAGMA vacuum_db.journal_mode=OFF" but + ** An optimization would be to use a non-journaled pager. + ** (Later:) I tried setting "PRAGMA vacuum_XXXXXX.journal_mode=OFF" but ** that actually made the VACUUM run slower. Very little journalling ** actually occurs when doing a vacuum since the vacuum_db is initially ** empty. Only the journal header is written. Apparently it takes more ** time to parse and run the PRAGMA to turn journalling off than it does ** to write the journal header file. */ + sqlite3_randomness(sizeof(iRandom),&iRandom); + sqlite3_snprintf(sizeof(zDbVacuum), zDbVacuum, "vacuum_%016llx", iRandom); nDb = db->nDb; - if( sqlite3TempInMemory(db) ){ - zSql = "ATTACH ':memory:' AS vacuum_db;"; - }else{ - zSql = "ATTACH '' AS vacuum_db;"; - } - rc = execSql(db, pzErrMsg, zSql); - if( db->nDb>nDb ){ - pDb = &db->aDb[db->nDb-1]; - assert( strcmp(pDb->zName,"vacuum_db")==0 ); - } + rc = execSqlF(db, pzErrMsg, "ATTACH %Q AS %s", zOut, zDbVacuum); + db->openFlags = saved_openFlags; if( rc!=SQLITE_OK ) goto end_of_vacuum; - pTemp = db->aDb[db->nDb-1].pBt; + assert( (db->nDb-1)==nDb ); + pDb = &db->aDb[nDb]; + assert( strcmp(pDb->zDbSName,zDbVacuum)==0 ); + pTemp = pDb->pBt; + if( pOut ){ + sqlite3_file *id = sqlite3PagerFile(sqlite3BtreePager(pTemp)); + i64 sz = 0; + if( id->pMethods!=0 && (sqlite3OsFileSize(id, &sz)!=SQLITE_OK || sz>0) ){ + rc = SQLITE_ERROR; + sqlite3SetString(pzErrMsg, db, "output file already exists"); + goto end_of_vacuum; + } + db->mDbFlags |= DBFLAG_VacuumInto; - /* The call to execSql() to attach the temp database has left the file - ** locked (as there was more than one active statement when the transaction - ** to read the schema was concluded. Unlock it here so that this doesn't - ** cause problems for the call to BtreeSetPageSize() below. */ - sqlite3BtreeCommit(pTemp); - - nRes = sqlite3BtreeGetOptimalReserve(pMain); - - /* A VACUUM cannot change the pagesize of an encrypted database. */ -#ifdef SQLITE_HAS_CODEC - if( db->nextPagesize ){ - extern void sqlite3CodecGetKey(sqlite3*, int, void**, int*); - int nKey; - char *zKey; - sqlite3CodecGetKey(db, 0, (void**)&zKey, &nKey); - if( nKey ) db->nextPagesize = 0; + /* For a VACUUM INTO, the pager-flags are set to the same values as + ** they are for the database being vacuumed, except that PAGER_CACHESPILL + ** is always set. */ + pgflags = db->aDb[iDb].safety_level | (db->flags & PAGER_FLAGS_MASK); } -#endif + nRes = sqlite3BtreeGetRequestedReserve(pMain); - rc = execSql(db, pzErrMsg, "PRAGMA vacuum_db.synchronous=OFF"); - if( rc!=SQLITE_OK ) goto end_of_vacuum; + sqlite3BtreeSetCacheSize(pTemp, db->aDb[iDb].pSchema->cache_size); + sqlite3BtreeSetSpillSize(pTemp, sqlite3BtreeSetSpillSize(pMain,0)); + sqlite3BtreeSetPagerFlags(pTemp, pgflags|PAGER_CACHESPILL); /* Begin a transaction and take an exclusive lock on the main database ** file. This is done before the sqlite3BtreeGetPageSize(pMain) call below, ** to ensure that we do not try to change the page-size on a WAL database. */ - rc = execSql(db, pzErrMsg, "BEGIN;"); + rc = execSql(db, pzErrMsg, "BEGIN"); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = sqlite3BtreeBeginTrans(pMain, 2); + rc = sqlite3BtreeBeginTrans(pMain, pOut==0 ? 2 : 0, 0); if( rc!=SQLITE_OK ) goto end_of_vacuum; /* Do not attempt to change the page size for a WAL database */ if( sqlite3PagerGetJournalMode(sqlite3BtreePager(pMain)) - ==PAGER_JOURNALMODE_WAL ){ + ==PAGER_JOURNALMODE_WAL + && pOut==0 + ){ db->nextPagesize = 0; } @@ -120915,68 +156219,52 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ /* Query the schema of the main database. Create a mirror schema ** in the temporary database. */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE TABLE vacuum_db.' || substr(sql,14) " - " FROM sqlite_master WHERE type='table' AND name!='sqlite_sequence'" - " AND coalesce(rootpage,1)>0" + db->init.iDb = nDb; /* force new CREATE statements into vacuum_db */ + rc = execSqlF(db, pzErrMsg, + "SELECT sql FROM \"%w\".sqlite_schema" + " WHERE type='table'AND name<>'sqlite_sequence'" + " AND coalesce(rootpage,1)>0", + zDbMain ); if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE INDEX vacuum_db.' || substr(sql,14)" - " FROM sqlite_master WHERE sql LIKE 'CREATE INDEX %' "); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'CREATE UNIQUE INDEX vacuum_db.' || substr(sql,21) " - " FROM sqlite_master WHERE sql LIKE 'CREATE UNIQUE INDEX %'"); + rc = execSqlF(db, pzErrMsg, + "SELECT sql FROM \"%w\".sqlite_schema" + " WHERE type='index'", + zDbMain + ); if( rc!=SQLITE_OK ) goto end_of_vacuum; + db->init.iDb = 0; /* Loop through the tables in the main database. For each, do ** an "INSERT INTO vacuum_db.xxx SELECT * FROM main.xxx;" to copy ** the contents to the temporary database. */ - assert( (db->flags & SQLITE_Vacuum)==0 ); - db->flags |= SQLITE_Vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'INSERT INTO vacuum_db.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';'" - "FROM main.sqlite_master " - "WHERE type = 'table' AND name!='sqlite_sequence' " - " AND coalesce(rootpage,1)>0" + rc = execSqlF(db, pzErrMsg, + "SELECT'INSERT INTO %s.'||quote(name)" + "||' SELECT*FROM\"%w\".'||quote(name)" + "FROM %s.sqlite_schema " + "WHERE type='table'AND coalesce(rootpage,1)>0", + zDbVacuum, zDbMain, zDbVacuum ); - assert( (db->flags & SQLITE_Vacuum)!=0 ); - db->flags &= ~SQLITE_Vacuum; + assert( (db->mDbFlags & DBFLAG_Vacuum)!=0 ); + db->mDbFlags &= ~DBFLAG_Vacuum; if( rc!=SQLITE_OK ) goto end_of_vacuum; - /* Copy over the sequence table - */ - rc = execExecSql(db, pzErrMsg, - "SELECT 'DELETE FROM vacuum_db.' || quote(name) || ';' " - "FROM vacuum_db.sqlite_master WHERE name='sqlite_sequence' " - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - rc = execExecSql(db, pzErrMsg, - "SELECT 'INSERT INTO vacuum_db.' || quote(name) " - "|| ' SELECT * FROM main.' || quote(name) || ';' " - "FROM vacuum_db.sqlite_master WHERE name=='sqlite_sequence';" - ); - if( rc!=SQLITE_OK ) goto end_of_vacuum; - - /* Copy the triggers, views, and virtual tables from the main database ** over to the temporary database. None of these objects has any ** associated storage, so all we have to do is copy their entries - ** from the SQLITE_MASTER table. + ** from the schema table. */ - rc = execSql(db, pzErrMsg, - "INSERT INTO vacuum_db.sqlite_master " - " SELECT type, name, tbl_name, rootpage, sql" - " FROM main.sqlite_master" - " WHERE type='view' OR type='trigger'" - " OR (type='table' AND rootpage=0)" + rc = execSqlF(db, pzErrMsg, + "INSERT INTO %s.sqlite_schema" + " SELECT*FROM \"%w\".sqlite_schema" + " WHERE type IN('view','trigger')" + " OR(type='table'AND rootpage=0)", + zDbVacuum, zDbMain ); if( rc ) goto end_of_vacuum; - /* At this point, there is a write transaction open on both the + /* At this point, there is a write transaction open on both the ** vacuum database and the main database. Assuming no error occurs, ** both transactions are closed by this block - the main database ** transaction by sqlite3BtreeCopyFile() and the other by an explicit @@ -121000,8 +156288,8 @@ SQLITE_PRIVATE int sqlite3RunVacuum(char **pzErrMsg, sqlite3 *db){ BTREE_APPLICATION_ID, 0, /* Preserve the application id */ }; - assert( 1==sqlite3BtreeIsInTrans(pTemp) ); - assert( 1==sqlite3BtreeIsInTrans(pMain) ); + assert( SQLITE_TXN_WRITE==sqlite3BtreeTxnState(pTemp) ); + assert( pOut!=0 || SQLITE_TXN_WRITE==sqlite3BtreeTxnState(pMain) ); /* Copy Btree meta values */ for(i=0; iflags */ + db->init.iDb = 0; + db->mDbFlags = saved_mDbFlags; db->flags = saved_flags; db->nChange = saved_nChange; db->nTotalChange = saved_nTotalChange; - db->xTrace = saved_xTrace; - sqlite3BtreeSetPageSize(pMain, -1, -1, 1); + db->mTrace = saved_mTrace; + sqlite3BtreeSetPageSize(pMain, -1, 0, 1); /* Currently there is an SQL level transaction open on the vacuum ** database. No locks are held on any other files (since the main file @@ -121048,7 +156345,7 @@ end_of_vacuum: } /* This both clears the schemas and reduces the size of the db->aDb[] - ** array. */ + ** array. */ sqlite3ResetAllSchemasOfConnection(db); return rc; @@ -121077,7 +156374,7 @@ end_of_vacuum: /* ** Before a virtual table xCreate() or xConnect() method is invoked, the ** sqlite3.pVtabCtx member variable is set to point to an instance of -** this struct allocated on the stack. It is used by the implementation of +** this struct allocated on the stack. It is used by the implementation of ** the sqlite3_declare_vtab() and sqlite3_vtab_config() APIs, both of which ** are invoked only from within xCreate and xConnect methods. */ @@ -121088,6 +156385,57 @@ struct VtabCtx { int bDeclared; /* True after sqlite3_declare_vtab() is called */ }; +/* +** Construct and install a Module object for a virtual table. When this +** routine is called, it is guaranteed that all appropriate locks are held +** and the module is not already part of the connection. +** +** If there already exists a module with zName, replace it with the new one. +** If pModule==0, then delete the module zName if it exists. +*/ +SQLITE_PRIVATE Module *sqlite3VtabCreateModule( + sqlite3 *db, /* Database in which module is registered */ + const char *zName, /* Name assigned to this module */ + const sqlite3_module *pModule, /* The definition of the module */ + void *pAux, /* Context pointer for xCreate/xConnect */ + void (*xDestroy)(void *) /* Module destructor function */ +){ + Module *pMod; + Module *pDel; + char *zCopy; + if( pModule==0 ){ + zCopy = (char*)zName; + pMod = 0; + }else{ + int nName = sqlite3Strlen30(zName); + pMod = (Module *)sqlite3Malloc(sizeof(Module) + nName + 1); + if( pMod==0 ){ + sqlite3OomFault(db); + return 0; + } + zCopy = (char *)(&pMod[1]); + memcpy(zCopy, zName, nName+1); + pMod->zName = zCopy; + pMod->pModule = pModule; + pMod->pAux = pAux; + pMod->xDestroy = xDestroy; + pMod->pEpoTab = 0; + pMod->nRefModule = 1; + } + pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); + if( pDel ){ + if( pDel==pMod ){ + sqlite3OomFault(db); + sqlite3DbFree(db, pDel); + pMod = 0; + }else{ + sqlite3VtabEponymousTableClear(db, pDel); + sqlite3VtabModuleUnref(db, pDel); + } + } + return pMod; +} + /* ** The actual function that does the work of creating a new module. ** This function implements the sqlite3_create_module() and @@ -121101,35 +156449,11 @@ static int createModule( void (*xDestroy)(void *) /* Module destructor function */ ){ int rc = SQLITE_OK; - int nName; sqlite3_mutex_enter(db->mutex); - nName = sqlite3Strlen30(zName); - if( sqlite3HashFind(&db->aModule, zName) ){ - rc = SQLITE_MISUSE_BKPT; - }else{ - Module *pMod; - pMod = (Module *)sqlite3DbMallocRawNN(db, sizeof(Module) + nName + 1); - if( pMod ){ - Module *pDel; - char *zCopy = (char *)(&pMod[1]); - memcpy(zCopy, zName, nName+1); - pMod->zName = zCopy; - pMod->pModule = pModule; - pMod->pAux = pAux; - pMod->xDestroy = xDestroy; - pMod->pEpoTab = 0; - pDel = (Module *)sqlite3HashInsert(&db->aModule,zCopy,(void*)pMod); - assert( pDel==0 || pDel==pMod ); - if( pDel ){ - sqlite3OomFault(db); - sqlite3DbFree(db, pDel); - } - } - } + (void)sqlite3VtabCreateModule(db, zName, pModule, pAux, xDestroy); rc = sqlite3ApiExit(db, rc); if( rc!=SQLITE_OK && xDestroy ) xDestroy(pAux); - sqlite3_mutex_leave(db->mutex); return rc; } @@ -121138,7 +156462,7 @@ static int createModule( /* ** External API function used to create a new virtual-table module. */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_module( +SQLITE_API int sqlite3_create_module( sqlite3 *db, /* Database in which module is registered */ const char *zName, /* Name assigned to this module */ const sqlite3_module *pModule, /* The definition of the module */ @@ -121153,7 +156477,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_module( /* ** External API function used to create a new virtual-table module. */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2( +SQLITE_API int sqlite3_create_module_v2( sqlite3 *db, /* Database in which module is registered */ const char *zName, /* Name assigned to this module */ const sqlite3_module *pModule, /* The definition of the module */ @@ -121166,10 +156490,48 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_module_v2( return createModule(db, zName, pModule, pAux, xDestroy); } +/* +** External API to drop all virtual-table modules, except those named +** on the azNames list. +*/ +SQLITE_API int sqlite3_drop_modules(sqlite3 *db, const char** azNames){ + HashElem *pThis, *pNext; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + for(pThis=sqliteHashFirst(&db->aModule); pThis; pThis=pNext){ + Module *pMod = (Module*)sqliteHashData(pThis); + pNext = sqliteHashNext(pThis); + if( azNames ){ + int ii; + for(ii=0; azNames[ii]!=0 && strcmp(azNames[ii],pMod->zName)!=0; ii++){} + if( azNames[ii]!=0 ) continue; + } + createModule(db, pMod->zName, 0, 0, 0); + } + return SQLITE_OK; +} + +/* +** Decrement the reference count on a Module object. Destroy the +** module when the reference count reaches zero. +*/ +SQLITE_PRIVATE void sqlite3VtabModuleUnref(sqlite3 *db, Module *pMod){ + assert( pMod->nRefModule>0 ); + pMod->nRefModule--; + if( pMod->nRefModule==0 ){ + if( pMod->xDestroy ){ + pMod->xDestroy(pMod->pAux); + } + assert( pMod->pEpoTab==0 ); + sqlite3DbFree(db, pMod); + } +} + /* ** Lock the virtual table so that it cannot be disconnected. ** Locks nest. Every lock should have a corresponding unlock. -** If an unlock is omitted, resources leaks will occur. +** If an unlock is omitted, resources leaks will occur. ** ** If a disconnect is attempted while a virtual table is locked, ** the disconnect is deferred until all locks have been removed. @@ -121181,13 +156543,13 @@ SQLITE_PRIVATE void sqlite3VtabLock(VTable *pVTab){ /* ** pTab is a pointer to a Table structure representing a virtual-table. -** Return a pointer to the VTable object used by connection db to access +** Return a pointer to the VTable object used by connection db to access ** this virtual-table, if one has been created, or NULL otherwise. */ SQLITE_PRIVATE VTable *sqlite3GetVTable(sqlite3 *db, Table *pTab){ VTable *pVtab; assert( IsVirtual(pTab) ); - for(pVtab=pTab->pVTable; pVtab && pVtab->db!=db; pVtab=pVtab->pNext); + for(pVtab=pTab->u.vtab.p; pVtab && pVtab->db!=db; pVtab=pVtab->pNext); return pVtab; } @@ -121200,7 +156562,8 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ assert( db ); assert( pVTab->nRef>0 ); - assert( db->magic==SQLITE_MAGIC_OPEN || db->magic==SQLITE_MAGIC_ZOMBIE ); + assert( db->eOpenState==SQLITE_STATE_OPEN + || db->eOpenState==SQLITE_STATE_ZOMBIE ); pVTab->nRef--; if( pVTab->nRef==0 ){ @@ -121208,27 +156571,31 @@ SQLITE_PRIVATE void sqlite3VtabUnlock(VTable *pVTab){ if( p ){ p->pModule->xDisconnect(p); } + sqlite3VtabModuleUnref(pVTab->db, pVTab->pMod); sqlite3DbFree(db, pVTab); } } /* ** Table p is a virtual table. This function moves all elements in the -** p->pVTable list to the sqlite3.pDisconnect lists of their associated -** database connections to be disconnected at the next opportunity. +** p->u.vtab.p list to the sqlite3.pDisconnect lists of their associated +** database connections to be disconnected at the next opportunity. ** Except, if argument db is not NULL, then the entry associated with -** connection db is left in the p->pVTable list. +** connection db is left in the p->u.vtab.p list. */ static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ VTable *pRet = 0; - VTable *pVTable = p->pVTable; - p->pVTable = 0; + VTable *pVTable; - /* Assert that the mutex (if any) associated with the BtShared database - ** that contains table p is held by the caller. See header comments + assert( IsVirtual(p) ); + pVTable = p->u.vtab.p; + p->u.vtab.p = 0; + + /* Assert that the mutex (if any) associated with the BtShared database + ** that contains table p is held by the caller. See header comments ** above function sqlite3VtabUnlockList() for an explanation of why ** this makes it safe to access the sqlite3.pDisconnect list of any - ** database connection that may have an entry in the p->pVTable list. + ** database connection that may have an entry in the p->u.vtab.p list. */ assert( db==0 || sqlite3SchemaMutexHeld(db, 0, p->pSchema) ); @@ -121238,7 +156605,7 @@ static VTable *vtabDisconnectAll(sqlite3 *db, Table *p){ assert( db2 ); if( db2==db ){ pRet = pVTable; - p->pVTable = pRet; + p->u.vtab.p = pRet; pRet->pNext = 0; }else{ pVTable->pNext = db2->pDisconnect; @@ -121266,7 +156633,7 @@ SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3_mutex_held(db->mutex) ); - for(ppVTab=&p->pVTable; *ppVTab; ppVTab=&(*ppVTab)->pNext){ + for(ppVTab=&p->u.vtab.p; *ppVTab; ppVTab=&(*ppVTab)->pNext){ if( (*ppVTab)->db==db ){ VTable *pVTab = *ppVTab; *ppVTab = pVTab->pNext; @@ -121281,7 +156648,7 @@ SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ ** Disconnect all the virtual table objects in the sqlite3.pDisconnect list. ** ** This function may only be called when the mutexes associated with all -** shared b-tree databases opened using connection db are held by the +** shared b-tree databases opened using connection db are held by the ** caller. This is done to protect the sqlite3.pDisconnect list. The ** sqlite3.pDisconnect list is accessed only as follows: ** @@ -121294,18 +156661,17 @@ SQLITE_PRIVATE void sqlite3VtabDisconnect(sqlite3 *db, Table *p){ ** or, if the virtual table is stored in a non-sharable database, then ** the database handle mutex is held. ** -** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously +** As a result, a sqlite3.pDisconnect cannot be accessed simultaneously ** by multiple threads. It is thread-safe. */ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ VTable *p = db->pDisconnect; - db->pDisconnect = 0; assert( sqlite3BtreeHoldsAllMutexes(db) ); assert( sqlite3_mutex_held(db->mutex) ); if( p ){ - sqlite3ExpirePreparedStatements(db); + db->pDisconnect = 0; do { VTable *pNext = p->pNext; sqlite3VtabUnlock(p); @@ -121320,42 +156686,51 @@ SQLITE_PRIVATE void sqlite3VtabUnlockList(sqlite3 *db){ ** record. ** ** Since it is a virtual-table, the Table structure contains a pointer -** to the head of a linked list of VTable structures. Each VTable +** to the head of a linked list of VTable structures. Each VTable ** structure is associated with a single sqlite3* user of the schema. -** The reference count of the VTable structure associated with database -** connection db is decremented immediately (which may lead to the +** The reference count of the VTable structure associated with database +** connection db is decremented immediately (which may lead to the ** structure being xDisconnected and free). Any other VTable structures -** in the list are moved to the sqlite3.pDisconnect list of the associated +** in the list are moved to the sqlite3.pDisconnect list of the associated ** database connection. */ SQLITE_PRIVATE void sqlite3VtabClear(sqlite3 *db, Table *p){ - if( !db || db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); - if( p->azModuleArg ){ + assert( IsVirtual(p) ); + assert( db!=0 ); + if( db->pnBytesFreed==0 ) vtabDisconnectAll(0, p); + if( p->u.vtab.azArg ){ int i; - for(i=0; inModuleArg; i++){ - if( i!=1 ) sqlite3DbFree(db, p->azModuleArg[i]); + for(i=0; iu.vtab.nArg; i++){ + if( i!=1 ) sqlite3DbFree(db, p->u.vtab.azArg[i]); } - sqlite3DbFree(db, p->azModuleArg); + sqlite3DbFree(db, p->u.vtab.azArg); } } /* -** Add a new module argument to pTable->azModuleArg[]. +** Add a new module argument to pTable->u.vtab.azArg[]. ** The string is not copied - the pointer is stored. The ** string will be freed automatically when the table is ** deleted. */ -static void addModuleArgument(sqlite3 *db, Table *pTable, char *zArg){ - int nBytes = sizeof(char *)*(2+pTable->nModuleArg); +static void addModuleArgument(Parse *pParse, Table *pTable, char *zArg){ + sqlite3_int64 nBytes; char **azModuleArg; - azModuleArg = sqlite3DbRealloc(db, pTable->azModuleArg, nBytes); + sqlite3 *db = pParse->db; + + assert( IsVirtual(pTable) ); + nBytes = sizeof(char *)*(2+pTable->u.vtab.nArg); + if( pTable->u.vtab.nArg+3>=db->aLimit[SQLITE_LIMIT_COLUMN] ){ + sqlite3ErrorMsg(pParse, "too many columns on %s", pTable->zName); + } + azModuleArg = sqlite3DbRealloc(db, pTable->u.vtab.azArg, nBytes); if( azModuleArg==0 ){ sqlite3DbFree(db, zArg); }else{ - int i = pTable->nModuleArg++; + int i = pTable->u.vtab.nArg++; azModuleArg[i] = zArg; azModuleArg[i+1] = 0; - pTable->azModuleArg = azModuleArg; + pTable->u.vtab.azArg = azModuleArg; } } @@ -121371,7 +156746,6 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( Token *pModuleName, /* Name of the module for the virtual table */ int ifNotExists /* No error if the table already exists */ ){ - int iDb; /* The database the table is being created in */ Table *pTable; /* The new virtual table */ sqlite3 *db; /* Database connection */ @@ -121379,16 +156753,14 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( pTable = pParse->pNewTable; if( pTable==0 ) return; assert( 0==pTable->pIndex ); + pTable->eTabType = TABTYP_VTAB; db = pParse->db; - iDb = sqlite3SchemaToIndex(db, pTable->pSchema); - assert( iDb>=0 ); - pTable->tabFlags |= TF_Virtual; - pTable->nModuleArg = 0; - addModuleArgument(db, pTable, sqlite3NameFromToken(db, pModuleName)); - addModuleArgument(db, pTable, 0); - addModuleArgument(db, pTable, sqlite3DbStrDup(db, pTable->zName)); + assert( pTable->u.vtab.nArg==0 ); + addModuleArgument(pParse, pTable, sqlite3NameFromToken(db, pModuleName)); + addModuleArgument(pParse, pTable, 0); + addModuleArgument(pParse, pTable, sqlite3DbStrDup(db, pTable->zName)); assert( (pParse->sNameToken.z==pName2->z && pName2->z!=0) || (pParse->sNameToken.z==pName1->z && pName2->z==0) ); @@ -121399,12 +156771,14 @@ SQLITE_PRIVATE void sqlite3VtabBeginParse( #ifndef SQLITE_OMIT_AUTHORIZATION /* Creating a virtual table invokes the authorization callback twice. ** The first invocation, to obtain permission to INSERT a row into the - ** sqlite_master table, has already been made by sqlite3StartTable(). + ** sqlite_schema table, has already been made by sqlite3StartTable(). ** The second call, to obtain permission to create the table, is made now. */ - if( pTable->azModuleArg ){ - sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, - pTable->azModuleArg[0], pParse->db->aDb[iDb].zName); + if( pTable->u.vtab.azArg ){ + int iDb = sqlite3SchemaToIndex(db, pTable->pSchema); + assert( iDb>=0 ); /* The database the table is being created in */ + sqlite3AuthCheck(pParse, SQLITE_CREATE_VTABLE, pTable->zName, + pTable->u.vtab.azArg[0], pParse->db->aDb[iDb].zDbSName); } #endif } @@ -121419,7 +156793,7 @@ static void addArgumentToVtab(Parse *pParse){ const char *z = (const char*)pParse->sArg.z; int n = pParse->sArg.n; sqlite3 *db = pParse->db; - addModuleArgument(db, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); + addModuleArgument(pParse, pParse->pNewTable, sqlite3DbStrNDup(db, z, n)); } } @@ -121432,15 +156806,16 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ sqlite3 *db = pParse->db; /* The database connection */ if( pTab==0 ) return; + assert( IsVirtual(pTab) ); addArgumentToVtab(pParse); pParse->sArg.z = 0; - if( pTab->nModuleArg<1 ) return; - + if( pTab->u.vtab.nArg<1 ) return; + /* If the CREATE VIRTUAL TABLE statement is being entered for the ** first time (in other words if the virtual table is actually being - ** created now instead of just being read out of sqlite_master) then + ** created now instead of just being read out of sqlite_schema) then ** do additional initialization work and store the statement text - ** in the sqlite_master table. + ** in the sqlite_schema table. */ if( !db->init.busy ){ char *zStmt; @@ -121449,54 +156824,52 @@ SQLITE_PRIVATE void sqlite3VtabFinishParse(Parse *pParse, Token *pEnd){ int iReg; Vdbe *v; + sqlite3MayAbort(pParse); + /* Compute the complete text of the CREATE VIRTUAL TABLE statement */ if( pEnd ){ pParse->sNameToken.n = (int)(pEnd->z - pParse->sNameToken.z) + pEnd->n; } zStmt = sqlite3MPrintf(db, "CREATE VIRTUAL TABLE %T", &pParse->sNameToken); - /* A slot for the record has already been allocated in the - ** SQLITE_MASTER table. We just need to update that slot with all - ** the information we've collected. + /* A slot for the record has already been allocated in the + ** schema table. We just need to update that slot with all + ** the information we've collected. ** ** The VM register number pParse->regRowid holds the rowid of an - ** entry in the sqlite_master table tht was created for this vtab + ** entry in the sqlite_schema table that was created for this vtab ** by sqlite3StartTable(). */ iDb = sqlite3SchemaToIndex(db, pTab->pSchema); sqlite3NestedParse(pParse, - "UPDATE %Q.%s " + "UPDATE %Q." LEGACY_SCHEMA_TABLE " " "SET type='table', name=%Q, tbl_name=%Q, rootpage=0, sql=%Q " "WHERE rowid=#%d", - db->aDb[iDb].zName, SCHEMA_TABLE(iDb), + db->aDb[iDb].zDbSName, pTab->zName, pTab->zName, zStmt, pParse->regRowid ); - sqlite3DbFree(db, zStmt); v = sqlite3GetVdbe(pParse); sqlite3ChangeCookie(pParse, iDb); - sqlite3VdbeAddOp2(v, OP_Expire, 0, 0); - zWhere = sqlite3MPrintf(db, "name='%q' AND type='table'", pTab->zName); - sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere); + sqlite3VdbeAddOp0(v, OP_Expire); + zWhere = sqlite3MPrintf(db, "name=%Q AND sql=%Q", pTab->zName, zStmt); + sqlite3VdbeAddParseSchemaOp(v, iDb, zWhere, 0); + sqlite3DbFree(db, zStmt); iReg = ++pParse->nMem; sqlite3VdbeLoadString(v, iReg, pTab->zName); sqlite3VdbeAddOp2(v, OP_VCreate, iDb, iReg); - } - - /* If we are rereading the sqlite_master table create the in-memory - ** record of the table. The xConnect() method is not called until - ** the first time the virtual table is used in an SQL statement. This - ** allows a schema that contains virtual tables to be loaded before - ** the required virtual table implementations are registered. */ - else { + }else{ + /* If we are rereading the sqlite_schema table create the in-memory + ** record of the table. */ Table *pOld; Schema *pSchema = pTab->pSchema; const char *zName = pTab->zName; - assert( sqlite3SchemaMutexHeld(db, 0, pSchema) ); + assert( zName!=0 ); + sqlite3MarkAllShadowTablesOf(db, pTab); pOld = sqlite3HashInsert(&pSchema->tblHash, zName, pTab); if( pOld ){ sqlite3OomFault(db); @@ -121538,7 +156911,7 @@ SQLITE_PRIVATE void sqlite3VtabArgExtend(Parse *pParse, Token *p){ ** to this procedure. */ static int vtabCallConstructor( - sqlite3 *db, + sqlite3 *db, Table *pTab, Module *pMod, int (*xConstruct)(sqlite3*,void*,int,const char*const*,sqlite3_vtab**,char**), @@ -121547,38 +156920,43 @@ static int vtabCallConstructor( VtabCtx sCtx; VTable *pVTable; int rc; - const char *const*azArg = (const char *const*)pTab->azModuleArg; - int nArg = pTab->nModuleArg; + const char *const*azArg; + int nArg = pTab->u.vtab.nArg; char *zErr = 0; char *zModuleName; int iDb; VtabCtx *pCtx; + assert( IsVirtual(pTab) ); + azArg = (const char *const*)pTab->u.vtab.azArg; + /* Check that the virtual-table is not already being initialized */ for(pCtx=db->pVtabCtx; pCtx; pCtx=pCtx->pPrior){ if( pCtx->pTab==pTab ){ - *pzErr = sqlite3MPrintf(db, + *pzErr = sqlite3MPrintf(db, "vtable constructor called recursively: %s", pTab->zName ); return SQLITE_LOCKED; } } - zModuleName = sqlite3MPrintf(db, "%s", pTab->zName); + zModuleName = sqlite3DbStrDup(db, pTab->zName); if( !zModuleName ){ return SQLITE_NOMEM_BKPT; } - pVTable = sqlite3DbMallocZero(db, sizeof(VTable)); + pVTable = sqlite3MallocZero(sizeof(VTable)); if( !pVTable ){ + sqlite3OomFault(db); sqlite3DbFree(db, zModuleName); return SQLITE_NOMEM_BKPT; } pVTable->db = db; pVTable->pMod = pMod; + pVTable->eVtabRisk = SQLITE_VTABRISK_Normal; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); - pTab->azModuleArg[1] = db->aDb[iDb].zName; + pTab->u.vtab.azArg[1] = db->aDb[iDb].zDbSName; /* Invoke the virtual table constructor */ assert( &db->pVtabCtx ); @@ -121588,7 +156966,11 @@ static int vtabCallConstructor( sCtx.pPrior = db->pVtabCtx; sCtx.bDeclared = 0; db->pVtabCtx = &sCtx; + pTab->nTabRef++; rc = xConstruct(db, pMod->pAux, nArg, azArg, &pVTable->pVtab, &zErr); + assert( pTab!=0 ); + assert( pTab->nTabRef>1 || rc!=SQLITE_OK ); + sqlite3DeleteTable(db, pTab); db->pVtabCtx = sCtx.pPrior; if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); assert( sCtx.pTab==pTab ); @@ -121606,22 +156988,23 @@ static int vtabCallConstructor( ** the sqlite3_vtab object if successful. */ memset(pVTable->pVtab, 0, sizeof(pVTable->pVtab[0])); pVTable->pVtab->pModule = pMod->pModule; + pMod->nRefModule++; pVTable->nRef = 1; if( sCtx.bDeclared==0 ){ const char *zFormat = "vtable constructor did not declare schema: %s"; - *pzErr = sqlite3MPrintf(db, zFormat, pTab->zName); + *pzErr = sqlite3MPrintf(db, zFormat, zModuleName); sqlite3VtabUnlock(pVTable); rc = SQLITE_ERROR; }else{ int iCol; - u8 oooHidden = 0; + u16 oooHidden = 0; /* If everything went according to plan, link the new VTable structure - ** into the linked list headed by pTab->pVTable. Then loop through the + ** into the linked list headed by pTab->u.vtab.p. Then loop through the ** columns of the table to see if any of them contain the token "hidden". ** If so, set the Column COLFLAG_HIDDEN flag and remove the token from ** the type string. */ - pVTable->pNext = pTab->pVTable; - pTab->pVTable = pVTable; + pVTable->pNext = pTab->u.vtab.p; + pTab->u.vtab.p = pVTable; for(iCol=0; iColnCol; iCol++){ char *zType = sqlite3ColumnType(&pTab->aCol[iCol], ""); @@ -121647,6 +157030,7 @@ static int vtabCallConstructor( zType[i-1] = '\0'; } pTab->aCol[iCol].colFlags |= COLFLAG_HIDDEN; + pTab->tabFlags |= TF_HasHidden; oooHidden = TF_OOOHidden; }else{ pTab->tabFlags |= oooHidden; @@ -121661,7 +157045,7 @@ static int vtabCallConstructor( /* ** This function is invoked by the parser to call the xConnect() method -** of the virtual table pTab. If an error occurs, an error code is returned +** of the virtual table pTab. If an error occurs, an error code is returned ** and an error left in pParse. ** ** This call is a no-op if table pTab is not a virtual table. @@ -121673,16 +157057,17 @@ SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ int rc; assert( pTab ); - if( (pTab->tabFlags & TF_Virtual)==0 || sqlite3GetVTable(db, pTab) ){ + assert( IsVirtual(pTab) ); + if( sqlite3GetVTable(db, pTab) ){ return SQLITE_OK; } /* Locate the required virtual table module */ - zMod = pTab->azModuleArg[0]; + zMod = pTab->u.vtab.azArg[0]; pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); if( !pMod ){ - const char *zModule = pTab->azModuleArg[0]; + const char *zModule = pTab->u.vtab.azArg[0]; sqlite3ErrorMsg(pParse, "no such module: %s", zModule); rc = SQLITE_ERROR; }else{ @@ -121690,6 +157075,7 @@ SQLITE_PRIVATE int sqlite3VtabCallConnect(Parse *pParse, Table *pTab){ rc = vtabCallConstructor(db, pTab, pMod, pMod->pModule->xConnect, &zErr); if( rc!=SQLITE_OK ){ sqlite3ErrorMsg(pParse, "%s", zErr); + pParse->rc = rc; } sqlite3DbFree(db, zErr); } @@ -121706,7 +157092,8 @@ static int growVTrans(sqlite3 *db){ /* Grow the sqlite3.aVTrans array if required */ if( (db->nVTrans%ARRAY_INCR)==0 ){ VTable **aVTrans; - int nBytes = sizeof(sqlite3_vtab *) * (db->nVTrans + ARRAY_INCR); + sqlite3_int64 nBytes = sizeof(sqlite3_vtab*)* + ((sqlite3_int64)db->nVTrans + ARRAY_INCR); aVTrans = sqlite3DbRealloc(db, (void *)db->aVTrans, nBytes); if( !aVTrans ){ return SQLITE_NOMEM_BKPT; @@ -121730,9 +157117,9 @@ static void addToVTrans(sqlite3 *db, VTable *pVTab){ /* ** This function is invoked by the vdbe to call the xCreate method -** of the virtual table named zTab in database iDb. +** of the virtual table named zTab in database iDb. ** -** If an error occurs, *pzErr is set to point an an English language +** If an error occurs, *pzErr is set to point to an English language ** description of the error and an SQLITE_XXX error code is returned. ** In this case the caller must call sqlite3DbFree(db, ) on *pzErr. */ @@ -121742,15 +157129,15 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, Module *pMod; const char *zMod; - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); - assert( pTab && (pTab->tabFlags & TF_Virtual)!=0 && !pTab->pVTable ); + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); + assert( pTab && IsVirtual(pTab) && !pTab->u.vtab.p ); /* Locate the required virtual table module */ - zMod = pTab->azModuleArg[0]; + zMod = pTab->u.vtab.azArg[0]; pMod = (Module*)sqlite3HashFind(&db->aModule, zMod); - /* If the module has been registered and includes a Create method, - ** invoke it now. If the module has not been registered, return an + /* If the module has been registered and includes a Create method, + ** invoke it now. If the module has not been registered, return an ** error. Otherwise, do nothing. */ if( pMod==0 || pMod->pModule->xCreate==0 || pMod->pModule->xDestroy==0 ){ @@ -121777,63 +157164,105 @@ SQLITE_PRIVATE int sqlite3VtabCallCreate(sqlite3 *db, int iDb, const char *zTab, ** valid to call this function from within the xCreate() or xConnect() of a ** virtual table module. */ -SQLITE_API int SQLITE_STDCALL sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ +SQLITE_API int sqlite3_declare_vtab(sqlite3 *db, const char *zCreateTable){ VtabCtx *pCtx; - Parse *pParse; int rc = SQLITE_OK; Table *pTab; - char *zErr = 0; + Parse sParse; + int initBusy; + int i; + const unsigned char *z; + static const u8 aKeyword[] = { TK_CREATE, TK_TABLE, 0 }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zCreateTable==0 ){ return SQLITE_MISUSE_BKPT; } #endif + + /* Verify that the first two keywords in the CREATE TABLE statement + ** really are "CREATE" and "TABLE". If this is not the case, then + ** sqlite3_declare_vtab() is being misused. + */ + z = (const unsigned char*)zCreateTable; + for(i=0; aKeyword[i]; i++){ + int tokenType = 0; + do{ z += sqlite3GetToken(z, &tokenType); }while( tokenType==TK_SPACE ); + if( tokenType!=aKeyword[i] ){ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, "syntax error"); + return SQLITE_ERROR; + } + } + sqlite3_mutex_enter(db->mutex); pCtx = db->pVtabCtx; if( !pCtx || pCtx->bDeclared ){ - sqlite3Error(db, SQLITE_MISUSE); + sqlite3Error(db, SQLITE_MISUSE_BKPT); sqlite3_mutex_leave(db->mutex); return SQLITE_MISUSE_BKPT; } - pTab = pCtx->pTab; - assert( (pTab->tabFlags & TF_Virtual)!=0 ); - pParse = sqlite3StackAllocZero(db, sizeof(*pParse)); - if( pParse==0 ){ - rc = SQLITE_NOMEM_BKPT; - }else{ - pParse->declareVtab = 1; - pParse->db = db; - pParse->nQueryLoop = 1; - - if( SQLITE_OK==sqlite3RunParser(pParse, zCreateTable, &zErr) - && pParse->pNewTable - && !db->mallocFailed - && !pParse->pNewTable->pSelect - && (pParse->pNewTable->tabFlags & TF_Virtual)==0 - ){ - if( !pTab->aCol ){ - pTab->aCol = pParse->pNewTable->aCol; - pTab->nCol = pParse->pNewTable->nCol; - pParse->pNewTable->nCol = 0; - pParse->pNewTable->aCol = 0; + pTab = pCtx->pTab; + assert( IsVirtual(pTab) ); + + sqlite3ParseObjectInit(&sParse, db); + sParse.eParseMode = PARSE_MODE_DECLARE_VTAB; + sParse.disableTriggers = 1; + /* We should never be able to reach this point while loading the + ** schema. Nevertheless, defend against that (turn off db->init.busy) + ** in case a bug arises. */ + assert( db->init.busy==0 ); + initBusy = db->init.busy; + db->init.busy = 0; + sParse.nQueryLoop = 1; + if( SQLITE_OK==sqlite3RunParser(&sParse, zCreateTable) ){ + assert( sParse.pNewTable!=0 ); + assert( !db->mallocFailed ); + assert( IsOrdinaryTable(sParse.pNewTable) ); + assert( sParse.zErrMsg==0 ); + if( !pTab->aCol ){ + Table *pNew = sParse.pNewTable; + Index *pIdx; + pTab->aCol = pNew->aCol; + assert( IsOrdinaryTable(pNew) ); + sqlite3ExprListDelete(db, pNew->u.tab.pDfltList); + pTab->nNVCol = pTab->nCol = pNew->nCol; + pTab->tabFlags |= pNew->tabFlags & (TF_WithoutRowid|TF_NoVisibleRowid); + pNew->nCol = 0; + pNew->aCol = 0; + assert( pTab->pIndex==0 ); + assert( HasRowid(pNew) || sqlite3PrimaryKeyIndex(pNew)!=0 ); + if( !HasRowid(pNew) + && pCtx->pVTable->pMod->pModule->xUpdate!=0 + && sqlite3PrimaryKeyIndex(pNew)->nKeyCol!=1 + ){ + /* WITHOUT ROWID virtual tables must either be read-only (xUpdate==0) + ** or else must have a single-column PRIMARY KEY */ + rc = SQLITE_ERROR; + } + pIdx = pNew->pIndex; + if( pIdx ){ + assert( pIdx->pNext==0 ); + pTab->pIndex = pIdx; + pNew->pIndex = 0; + pIdx->pTable = pTab; } - pCtx->bDeclared = 1; - }else{ - sqlite3ErrorWithMsg(db, SQLITE_ERROR, (zErr ? "%s" : 0), zErr); - sqlite3DbFree(db, zErr); - rc = SQLITE_ERROR; } - pParse->declareVtab = 0; - - if( pParse->pVdbe ){ - sqlite3VdbeFinalize(pParse->pVdbe); - } - sqlite3DeleteTable(db, pParse->pNewTable); - sqlite3ParserReset(pParse); - sqlite3StackFree(db, pParse); + pCtx->bDeclared = 1; + }else{ + sqlite3ErrorWithMsg(db, SQLITE_ERROR, + (sParse.zErrMsg ? "%s" : 0), sParse.zErrMsg); + sqlite3DbFree(db, sParse.zErrMsg); + rc = SQLITE_ERROR; } + sParse.eParseMode = PARSE_MODE_NORMAL; + + if( sParse.pVdbe ){ + sqlite3VdbeFinalize(sParse.pVdbe); + } + sqlite3DeleteTable(db, sParse.pNewTable); + sqlite3ParseObjectReset(&sParse); + db->init.busy = initBusy; assert( (rc&0xff)==rc ); rc = sqlite3ApiExit(db, rc); @@ -121852,11 +157281,14 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab int rc = SQLITE_OK; Table *pTab; - pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zName); - if( ALWAYS(pTab!=0 && pTab->pVTable!=0) ){ + pTab = sqlite3FindTable(db, zTab, db->aDb[iDb].zDbSName); + if( ALWAYS(pTab!=0) + && ALWAYS(IsVirtual(pTab)) + && ALWAYS(pTab->u.vtab.p!=0) + ){ VTable *p; int (*xDestroy)(sqlite3_vtab *); - for(p=pTab->pVTable; p; p=p->pNext){ + for(p=pTab->u.vtab.p; p; p=p->pNext){ assert( p->pVtab ); if( p->pVtab->nRef>0 ){ return SQLITE_LOCKED; @@ -121864,15 +157296,18 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab } p = vtabDisconnectAll(db, pTab); xDestroy = p->pMod->pModule->xDestroy; - assert( xDestroy!=0 ); /* Checked before the virtual table is created */ + if( xDestroy==0 ) xDestroy = p->pMod->pModule->xDisconnect; + assert( xDestroy!=0 ); + pTab->nTabRef++; rc = xDestroy(p->pVtab); /* Remove the sqlite3_vtab* from the aVTrans[] array, if applicable */ if( rc==SQLITE_OK ){ - assert( pTab->pVTable==p && p->pNext==0 ); + assert( pTab->u.vtab.p==p && p->pNext==0 ); p->pVtab = 0; - pTab->pVTable = 0; + pTab->u.vtab.p = 0; sqlite3VtabUnlock(p); } + sqlite3DeleteTable(db, pTab); } return rc; @@ -121884,7 +157319,7 @@ SQLITE_PRIVATE int sqlite3VtabCallDestroy(sqlite3 *db, int iDb, const char *zTab ** called is identified by the second argument, "offset", which is ** the offset of the method to call in the sqlite3_module structure. ** -** The array is cleared after invoking the callbacks. +** The array is cleared after invoking the callbacks. */ static void callFinaliser(sqlite3 *db, int offset){ int i; @@ -121933,7 +157368,7 @@ SQLITE_PRIVATE int sqlite3VtabSync(sqlite3 *db, Vdbe *p){ } /* -** Invoke the xRollback method of all virtual tables in the +** Invoke the xRollback method of all virtual tables in the ** sqlite3.aVTrans array. Then clear the array itself. */ SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db){ @@ -121942,7 +157377,7 @@ SQLITE_PRIVATE int sqlite3VtabRollback(sqlite3 *db){ } /* -** Invoke the xCommit method of all virtual tables in the +** Invoke the xCommit method of all virtual tables in the ** sqlite3.aVTrans array. Then clear the array itself. */ SQLITE_PRIVATE int sqlite3VtabCommit(sqlite3 *db){ @@ -121964,7 +157399,7 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ /* Special case: If db->aVTrans is NULL and db->nVTrans is greater ** than zero, then this function is being called from within a - ** virtual module xSync() callback. It is illegal to write to + ** virtual module xSync() callback. It is illegal to write to ** virtual module tables in this case, so return SQLITE_LOCKED. */ if( sqlite3VtabInSync(db) ){ @@ -121972,7 +157407,7 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ } if( !pVTab ){ return SQLITE_OK; - } + } pModule = pVTab->pVtab->pModule; if( pModule->xBegin ){ @@ -121985,7 +157420,7 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ } } - /* Invoke the xBegin method. If successful, add the vtab to the + /* Invoke the xBegin method. If successful, add the vtab to the ** sqlite3.aVTrans[] array. */ rc = growVTrans(db); if( rc==SQLITE_OK ){ @@ -121993,7 +157428,10 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ if( rc==SQLITE_OK ){ int iSvpt = db->nStatement + db->nSavepoint; addToVTrans(db, pVTab); - if( iSvpt ) rc = sqlite3VtabSavepoint(db, SAVEPOINT_BEGIN, iSvpt-1); + if( iSvpt && pModule->xSavepoint ){ + pVTab->iSavepoint = iSvpt; + rc = pModule->xSavepoint(pVTab->pVtab, iSvpt-1); + } } } } @@ -122006,11 +157444,11 @@ SQLITE_PRIVATE int sqlite3VtabBegin(sqlite3 *db, VTable *pVTab){ ** as the second argument to the virtual table method invoked. ** ** If op is SAVEPOINT_BEGIN, the xSavepoint method is invoked. If it is -** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is +** SAVEPOINT_ROLLBACK, the xRollbackTo method. Otherwise, if op is ** SAVEPOINT_RELEASE, then the xRelease method of each virtual table with ** an open transaction is invoked. ** -** If any virtual table method returns an error code other than SQLITE_OK, +** If any virtual table method returns an error code other than SQLITE_OK, ** processing is abandoned and the error returned to the caller of this ** function immediately. If all calls to virtual table methods are successful, ** SQLITE_OK is returned. @@ -122027,6 +157465,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ const sqlite3_module *pMod = pVTab->pMod->pModule; if( pVTab->pVtab && pMod->iVersion>=2 ){ int (*xMethod)(sqlite3_vtab *, int); + sqlite3VtabLock(pVTab); switch( op ){ case SAVEPOINT_BEGIN: xMethod = pMod->xSavepoint; @@ -122040,8 +157479,12 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ break; } if( xMethod && pVTab->iSavepoint>iSavepoint ){ + u64 savedFlags = (db->flags & SQLITE_Defensive); + db->flags &= ~(u64)SQLITE_Defensive; rc = xMethod(pVTab->pVtab, iSavepoint); + db->flags |= savedFlags; } + sqlite3VtabUnlock(pVTab); } } } @@ -122057,7 +157500,7 @@ SQLITE_PRIVATE int sqlite3VtabSavepoint(sqlite3 *db, int op, int iSavepoint){ ** This routine is used to allow virtual table implementations to ** overload MATCH, LIKE, GLOB, and REGEXP operators. ** -** Return either the pDef argument (indicating no change) or a +** Return either the pDef argument (indicating no change) or a ** new FuncDef structure that is marked as ephemeral using the ** SQLITE_FUNC_EPHEM flag. */ @@ -122074,33 +157517,37 @@ SQLITE_PRIVATE FuncDef *sqlite3VtabOverloadFunction( void *pArg = 0; FuncDef *pNew; int rc = 0; - char *zLowerName; - unsigned char *z; - /* Check to see the left operand is a column in a virtual table */ if( NEVER(pExpr==0) ) return pDef; if( pExpr->op!=TK_COLUMN ) return pDef; - pTab = pExpr->pTab; + assert( ExprUseYTab(pExpr) ); + pTab = pExpr->y.pTab; if( NEVER(pTab==0) ) return pDef; - if( (pTab->tabFlags & TF_Virtual)==0 ) return pDef; + if( !IsVirtual(pTab) ) return pDef; pVtab = sqlite3GetVTable(db, pTab)->pVtab; assert( pVtab!=0 ); assert( pVtab->pModule!=0 ); pMod = (sqlite3_module *)pVtab->pModule; if( pMod->xFindFunction==0 ) return pDef; - + /* Call the xFindFunction method on the virtual table implementation - ** to see if the implementation wants to overload this function + ** to see if the implementation wants to overload this function. + ** + ** Though undocumented, we have historically always invoked xFindFunction + ** with an all lower-case function name. Continue in this tradition to + ** avoid any chance of an incompatibility. */ - zLowerName = sqlite3DbStrDup(db, pDef->zName); - if( zLowerName ){ - for(z=(unsigned char*)zLowerName; *z; z++){ - *z = sqlite3UpperToLower[*z]; +#ifdef SQLITE_DEBUG + { + int i; + for(i=0; pDef->zName[i]; i++){ + unsigned char x = (unsigned char)pDef->zName[i]; + assert( x==sqlite3UpperToLower[x] ); } - rc = pMod->xFindFunction(pVtab, nArg, zLowerName, &xSFunc, &pArg); - sqlite3DbFree(db, zLowerName); } +#endif + rc = pMod->xFindFunction(pVtab, nArg, pDef->zName, &xSFunc, &pArg); if( rc==0 ){ return pDef; } @@ -122137,7 +157584,7 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ if( pTab==pToplevel->apVtabLock[i] ) return; } n = (pToplevel->nVtabLock+1)*sizeof(pToplevel->apVtabLock[0]); - apVtabLock = sqlite3_realloc64(pToplevel->apVtabLock, n); + apVtabLock = sqlite3Realloc(pToplevel->apVtabLock, n); if( apVtabLock ){ pToplevel->apVtabLock = apVtabLock; pToplevel->apVtabLock[pToplevel->nVtabLock++] = pTab; @@ -122147,14 +157594,15 @@ SQLITE_PRIVATE void sqlite3VtabMakeWritable(Parse *pParse, Table *pTab){ } /* -** Check to see if virtual tale module pMod can be have an eponymous +** Check to see if virtual table module pMod can be have an eponymous ** virtual table instance. If it can, create one if one does not already -** exist. Return non-zero if the eponymous virtual table instance exists -** when this routine returns, and return zero if it does not exist. +** exist. Return non-zero if either the eponymous virtual table instance +** exists when this routine returns or if an attempt to create it failed +** and an error message was left in pParse. ** ** An eponymous virtual table instance is one that is named after its ** module, and more importantly, does not require a CREATE VIRTUAL TABLE -** statement in order to come into existance. Eponymous virtual table +** statement in order to come into existence. Eponymous virtual table ** instances always exist. They cannot be DROP-ed. ** ** Any virtual table module for which xConnect and xCreate are the same @@ -122164,31 +157612,32 @@ SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ const sqlite3_module *pModule = pMod->pModule; Table *pTab; char *zErr = 0; - int nName; int rc; sqlite3 *db = pParse->db; if( pMod->pEpoTab ) return 1; if( pModule->xCreate!=0 && pModule->xCreate!=pModule->xConnect ) return 0; - nName = sqlite3Strlen30(pMod->zName) + 1; - pTab = sqlite3DbMallocZero(db, sizeof(Table) + nName); + pTab = sqlite3DbMallocZero(db, sizeof(Table)); if( pTab==0 ) return 0; + pTab->zName = sqlite3DbStrDup(db, pMod->zName); + if( pTab->zName==0 ){ + sqlite3DbFree(db, pTab); + return 0; + } pMod->pEpoTab = pTab; - pTab->zName = (char*)&pTab[1]; - memcpy(pTab->zName, pMod->zName, nName); - pTab->nRef = 1; + pTab->nTabRef = 1; + pTab->eTabType = TABTYP_VTAB; pTab->pSchema = db->aDb[0].pSchema; - pTab->tabFlags |= TF_Virtual; - pTab->nModuleArg = 0; + assert( pTab->u.vtab.nArg==0 ); pTab->iPKey = -1; - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); - addModuleArgument(db, pTab, 0); - addModuleArgument(db, pTab, sqlite3DbStrDup(db, pTab->zName)); + pTab->tabFlags |= TF_Eponymous; + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); + addModuleArgument(pParse, pTab, 0); + addModuleArgument(pParse, pTab, sqlite3DbStrDup(db, pTab->zName)); rc = vtabCallConstructor(db, pTab, pMod, pModule->xConnect, &zErr); if( rc ){ sqlite3ErrorMsg(pParse, "%s", zErr); sqlite3DbFree(db, zErr); sqlite3VtabEponymousTableClear(db, pMod); - return 0; } return 1; } @@ -122200,9 +157649,11 @@ SQLITE_PRIVATE int sqlite3VtabEponymousTableInit(Parse *pParse, Module *pMod){ SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ Table *pTab = pMod->pEpoTab; if( pTab!=0 ){ - sqlite3DeleteColumnNames(db, pTab); - sqlite3VtabClear(db, pTab); - sqlite3DbFree(db, pTab); + /* Mark the table as Ephemeral prior to deleting it, so that the + ** sqlite3DeleteTable() routine will know that it is not stored in + ** the schema. */ + pTab->tabFlags |= TF_Ephemeral; + sqlite3DeleteTable(db, pTab); pMod->pEpoTab = 0; } } @@ -122214,9 +157665,9 @@ SQLITE_PRIVATE void sqlite3VtabEponymousTableClear(sqlite3 *db, Module *pMod){ ** The results of this routine are undefined unless it is called from ** within an xUpdate method. */ -SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *db){ - static const unsigned char aMap[] = { - SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *db){ + static const unsigned char aMap[] = { + SQLITE_ROLLBACK, SQLITE_ABORT, SQLITE_FAIL, SQLITE_IGNORE, SQLITE_REPLACE }; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -122228,35 +157679,49 @@ SQLITE_API int SQLITE_STDCALL sqlite3_vtab_on_conflict(sqlite3 *db){ } /* -** Call from within the xCreate() or xConnect() methods to provide +** Call from within the xCreate() or xConnect() methods to provide ** the SQLite core with additional information about the behavior ** of the virtual table being implemented. */ -SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ +SQLITE_API int sqlite3_vtab_config(sqlite3 *db, int op, ...){ va_list ap; int rc = SQLITE_OK; + VtabCtx *p; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif sqlite3_mutex_enter(db->mutex); - va_start(ap, op); - switch( op ){ - case SQLITE_VTAB_CONSTRAINT_SUPPORT: { - VtabCtx *p = db->pVtabCtx; - if( !p ){ - rc = SQLITE_MISUSE_BKPT; - }else{ - assert( p->pTab==0 || (p->pTab->tabFlags & TF_Virtual)!=0 ); + p = db->pVtabCtx; + if( !p ){ + rc = SQLITE_MISUSE_BKPT; + }else{ + assert( p->pTab==0 || IsVirtual(p->pTab) ); + va_start(ap, op); + switch( op ){ + case SQLITE_VTAB_CONSTRAINT_SUPPORT: { p->pVTable->bConstraint = (u8)va_arg(ap, int); + break; + } + case SQLITE_VTAB_INNOCUOUS: { + p->pVTable->eVtabRisk = SQLITE_VTABRISK_Low; + break; + } + case SQLITE_VTAB_DIRECTONLY: { + p->pVTable->eVtabRisk = SQLITE_VTABRISK_High; + break; + } + case SQLITE_VTAB_USES_ALL_SCHEMAS: { + p->pVTable->bAllSchemas = 1; + break; + } + default: { + rc = SQLITE_MISUSE_BKPT; + break; } - break; } - default: - rc = SQLITE_MISUSE_BKPT; - break; + va_end(ap); } - va_end(ap); if( rc!=SQLITE_OK ) sqlite3Error(db, rc); sqlite3_mutex_leave(db->mutex); @@ -122305,20 +157770,9 @@ SQLITE_API int SQLITE_CDECL sqlite3_vtab_config(sqlite3 *db, int op, ...){ ** planner logic in "where.c". These definitions are broken out into ** a separate source file for easier editing. */ +#ifndef SQLITE_WHEREINT_H +#define SQLITE_WHEREINT_H -/* -** Trace output macros -*/ -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace; -#endif -#if defined(SQLITE_DEBUG) \ - && (defined(SQLITE_TEST) || defined(SQLITE_ENABLE_WHERETRACE)) -# define WHERETRACE(K,X) if(sqlite3WhereTrace&(K)) sqlite3DebugPrintf X -# define WHERETRACE_ENABLED 1 -#else -# define WHERETRACE(K,X) -#endif /* Forward references */ @@ -122334,6 +157788,28 @@ typedef struct WhereLoopBuilder WhereLoopBuilder; typedef struct WhereScan WhereScan; typedef struct WhereOrCost WhereOrCost; typedef struct WhereOrSet WhereOrSet; +typedef struct WhereMemBlock WhereMemBlock; +typedef struct WhereRightJoin WhereRightJoin; + +/* +** This object is a header on a block of allocated memory that will be +** automatically freed when its WInfo object is destructed. +*/ +struct WhereMemBlock { + WhereMemBlock *pNext; /* Next block in the chain */ + u64 sz; /* Bytes of space */ +}; + +/* +** Extra information attached to a WhereLevel that is a RIGHT JOIN. +*/ +struct WhereRightJoin { + int iMatch; /* Cursor used to determine prior matched rows */ + int regBloom; /* Bloom filter for iRJMatch */ + int regReturn; /* Return register for the interior subroutine */ + int addrSubrtn; /* Starting address for the interior subroutine */ + int endSubrtn; /* The last opcode in the interior subroutine */ +}; /* ** This object contains information needed to implement a single nested @@ -122360,23 +157836,29 @@ struct WhereLevel { int addrCont; /* Jump here to continue with the next loop cycle */ int addrFirst; /* First instruction of interior of the loop */ int addrBody; /* Beginning of the body of this loop */ + int regBignull; /* big-null flag reg. True if a NULL-scan is needed */ + int addrBignull; /* Jump here for next part of big-null scan */ #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS u32 iLikeRepCntr; /* LIKE range processing counter register (times 2) */ int addrLikeRep; /* LIKE range processing address */ #endif + int regFilter; /* Bloom filter */ + WhereRightJoin *pRJ; /* Extra information for RIGHT JOIN */ u8 iFrom; /* Which entry in the FROM clause */ u8 op, p3, p5; /* Opcode, P3 & P5 of the opcode that ends the loop */ - int p1, p2; /* Operands of the opcode used to ends the loop */ + int p1, p2; /* Operands of the opcode used to end the loop */ union { /* Information that depends on pWLoop->wsFlags */ struct { int nIn; /* Number of entries in aInLoop[] */ struct InLoop { int iCur; /* The VDBE cursor used by this IN operator */ int addrInTop; /* Top of the IN loop */ + int iBase; /* Base register of multi-key index record */ + int nPrefix; /* Number of prior entries in the key */ u8 eEndLoopOp; /* IN Loop terminator. OP_Next or OP_Prev */ } *aInLoop; /* Information about each nested IN operator */ } in; /* Used when pWLoop->wsFlags&WHERE_IN_ABLE */ - Index *pCovidx; /* Possible covering index for WHERE_MULTI_OR */ + Index *pCoveringIdx; /* Possible covering index for WHERE_MULTI_OR */ } u; struct WhereLoop *pWLoop; /* The selected WhereLoop object */ Bitmask notReady; /* FROM entries not usable at this level */ @@ -122413,14 +157895,21 @@ struct WhereLoop { union { struct { /* Information for internal btree tables */ u16 nEq; /* Number of equality constraints */ + u16 nBtm; /* Size of BTM vector */ + u16 nTop; /* Size of TOP vector */ + u16 nDistinctCol; /* Index columns used to sort for DISTINCT */ Index *pIndex; /* Index used, or NULL */ + ExprList *pOrderBy; /* ORDER BY clause if this is really a subquery */ } btree; struct { /* Information for virtual tables */ int idxNum; /* Index number */ - u8 needFree; /* True if sqlite3_free(idxStr) is needed */ + u32 needFree : 1; /* True if sqlite3_free(idxStr) is needed */ + u32 bOmitOffset : 1; /* True to let virtual table handle offset */ + u32 bIdxNumHex : 1; /* Show idxNum as hex in EXPLAIN QUERY PLAN */ i8 isOrdered; /* True if satisfies ORDER BY */ u16 omitMask; /* Terms that may be omitted */ char *idxStr; /* Index identifier string */ + u32 mHandleIn; /* Terms to handle as IN(...) instead of == */ } vtab; } u; u32 wsFlags; /* WHERE_* flags describing the plan */ @@ -122429,6 +157918,8 @@ struct WhereLoop { /**** whereLoopXfer() copies fields above ***********************/ # define WHERE_LOOP_XFER_SZ offsetof(WhereLoop,nLSlot) u16 nLSlot; /* Number of slots allocated for aLTerm[] */ + LogEst rStarDelta; /* Cost delta due to star-schema heuristic. Not + ** initialized unless pWInfo->nOutStarDelta>0 */ WhereTerm **aLTerm; /* WhereTerms used */ WhereLoop *pNextLoop; /* Next WhereLoop object in the WhereClause */ WhereTerm *aLTermSpace[3]; /* Initial aLTerm[] space */ @@ -122436,7 +157927,7 @@ struct WhereLoop { /* This object holds the prerequisites and the cost of running a ** subquery on one operand of an OR operator in the WHERE clause. -** See WhereOrSet for additional information +** See WhereOrSet for additional information */ struct WhereOrCost { Bitmask prereq; /* Prerequisites */ @@ -122488,7 +157979,7 @@ struct WherePath { ** clause subexpression is separated from the others by AND operators, ** usually, or sometimes subexpressions separated by OR. ** -** All WhereTerms are collected into a single WhereClause structure. +** All WhereTerms are collected into a single WhereClause structure. ** The following identity holds: ** ** WhereTerm.pWC->a[WhereTerm.idx] == WhereTerm @@ -122535,19 +158026,22 @@ struct WherePath { */ struct WhereTerm { Expr *pExpr; /* Pointer to the subexpression that is this term */ + WhereClause *pWC; /* The clause this term is part of */ + LogEst truthProb; /* Probability of truth for this expression */ + u16 wtFlags; /* TERM_xxx bit flags. See below */ + u16 eOperator; /* A WO_xx value describing */ + u8 nChild; /* Number of children that must disable us */ + u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ int iParent; /* Disable pWC->a[iParent] when this term disabled */ int leftCursor; /* Cursor number of X in "X " */ union { - int leftColumn; /* Column number of X in "X " */ + struct { + int leftColumn; /* Column number of X in "X " */ + int iField; /* Field in (?,?,?) IN (SELECT...) vector */ + } x; /* Opcode other than OP_OR or OP_AND */ WhereOrInfo *pOrInfo; /* Extra information if (eOperator & WO_OR)!=0 */ WhereAndInfo *pAndInfo; /* Extra information if (eOperator& WO_AND)!=0 */ } u; - LogEst truthProb; /* Probability of truth for this expression */ - u16 eOperator; /* A WO_xx value describing */ - u16 wtFlags; /* TERM_xxx bit flags. See below */ - u8 nChild; /* Number of children that must disable us */ - u8 eMatchOp; /* Op for vtab MATCH/LIKE/GLOB/REGEXP terms */ - WhereClause *pWC; /* The clause this term is part of */ Bitmask prereqRight; /* Bitmask of tables used by pExpr->pRight */ Bitmask prereqAll; /* Bitmask of tables referenced by pExpr */ }; @@ -122555,22 +158049,26 @@ struct WhereTerm { /* ** Allowed values of WhereTerm.wtFlags */ -#define TERM_DYNAMIC 0x01 /* Need to call sqlite3ExprDelete(db, pExpr) */ -#define TERM_VIRTUAL 0x02 /* Added by the optimizer. Do not code */ -#define TERM_CODED 0x04 /* This term is already coded */ -#define TERM_COPIED 0x08 /* Has a child */ -#define TERM_ORINFO 0x10 /* Need to free the WhereTerm.u.pOrInfo object */ -#define TERM_ANDINFO 0x20 /* Need to free the WhereTerm.u.pAndInfo obj */ -#define TERM_OR_OK 0x40 /* Used during OR-clause processing */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -# define TERM_VNULL 0x80 /* Manufactured x>NULL or x<=NULL term */ +#define TERM_DYNAMIC 0x0001 /* Need to call sqlite3ExprDelete(db, pExpr) */ +#define TERM_VIRTUAL 0x0002 /* Added by the optimizer. Do not code */ +#define TERM_CODED 0x0004 /* This term is already coded */ +#define TERM_COPIED 0x0008 /* Has a child */ +#define TERM_ORINFO 0x0010 /* Need to free the WhereTerm.u.pOrInfo object */ +#define TERM_ANDINFO 0x0020 /* Need to free the WhereTerm.u.pAndInfo obj */ +#define TERM_OK 0x0040 /* Used during OR-clause processing */ +#define TERM_VNULL 0x0080 /* Manufactured x>NULL or x<=NULL term */ +#define TERM_LIKEOPT 0x0100 /* Virtual terms from the LIKE optimization */ +#define TERM_LIKECOND 0x0200 /* Conditionally this LIKE operator term */ +#define TERM_LIKE 0x0400 /* The original LIKE operator */ +#define TERM_IS 0x0800 /* Term.pExpr is an IS operator */ +#define TERM_VARSELECT 0x1000 /* Term.pExpr contains a correlated sub-query */ +#define TERM_HEURTRUTH 0x2000 /* Heuristic truthProb used */ +#ifdef SQLITE_ENABLE_STAT4 +# define TERM_HIGHTRUTH 0x4000 /* Term excludes few rows */ #else -# define TERM_VNULL 0x00 /* Disabled if not using stat3 */ +# define TERM_HIGHTRUTH 0 /* Only used with STAT4 */ #endif -#define TERM_LIKEOPT 0x100 /* Virtual terms from the LIKE optimization */ -#define TERM_LIKECOND 0x200 /* Conditionally this LIKE operator term */ -#define TERM_LIKE 0x400 /* The original LIKE operator */ -#define TERM_IS 0x800 /* Term.pExpr is an IS operator */ +#define TERM_SLICE 0x8000 /* One slice of a row-value/vector comparison */ /* ** An instance of the WhereScan object is used as an iterator for locating @@ -122581,11 +158079,11 @@ struct WhereScan { WhereClause *pWC; /* WhereClause currently being scanned */ const char *zCollName; /* Required collating sequence, if not NULL */ Expr *pIdxExpr; /* Search for this index expression */ - char idxaff; /* Must match this affinity, if zCollName!=NULL */ - unsigned char nEquiv; /* Number of entries in aEquiv[] */ - unsigned char iEquiv; /* Next unused slot in aEquiv[] */ - u32 opMask; /* Acceptable operators */ int k; /* Resume scanning at this->pWC->a[this->k] */ + u32 opMask; /* Acceptable operators */ + char idxaff; /* Must match this affinity, if zCollName!=NULL */ + unsigned char iEquiv; /* Current slot in aiCur[] and aiColumn[] */ + unsigned char nEquiv; /* Number of entries in aiCur[] and aiColumn[] */ int aiCur[11]; /* Cursors in the equivalence class */ i16 aiColumn[11]; /* Corresponding column number in the eq-class */ }; @@ -122606,9 +158104,11 @@ struct WhereClause { WhereInfo *pWInfo; /* WHERE clause processing context */ WhereClause *pOuter; /* Outer conjunction */ u8 op; /* Split operator. TK_AND or TK_OR */ + u8 hasOr; /* True if any a[].eOperator is WO_OR */ int nTerm; /* Number of terms */ int nSlot; /* Number of entries in a[] */ - WhereTerm *a; /* Each a[] describes a term of the WHERE cluase */ + int nBase; /* Number of terms through the last non-Virtual */ + WhereTerm *a; /* Each a[] describes a term of the WHERE clause */ #if defined(SQLITE_SMALL_STACK) WhereTerm aStatic[1]; /* Initial static space for a[] */ #else @@ -122637,8 +158137,8 @@ struct WhereAndInfo { ** An instance of the following structure keeps track of a mapping ** between VDBE cursor numbers and bits of the bitmasks in WhereTerm. ** -** The VDBE cursor numbers are small integers contained in -** SrcList_item.iCursor and Expr.iTable fields. For any given WHERE +** The VDBE cursor numbers are small integers contained in +** SrcItem.iCursor and Expr.iTable fields. For any given WHERE ** clause, the cursor numbers might not begin with 0 and they might ** contain gaps in the numbering sequence. But we want to make maximum ** use of the bits in our bitmasks. This structure provides a mapping @@ -122660,15 +158160,11 @@ struct WhereAndInfo { ** no gaps. */ struct WhereMaskSet { + int bVarSelect; /* Used by sqlite3WhereExprUsage() */ int n; /* Number of assigned cursor values */ int ix[BMS]; /* Cursor assigned to each bit */ }; -/* -** Initialize a WhereMaskSet object -*/ -#define initMaskSet(P) (P)->n=0 - /* ** This object is a convenience wrapper holding all information needed ** to construct WhereLoop objects for a particular query. @@ -122676,15 +158172,43 @@ struct WhereMaskSet { struct WhereLoopBuilder { WhereInfo *pWInfo; /* Information about this WHERE */ WhereClause *pWC; /* WHERE clause terms */ - ExprList *pOrderBy; /* ORDER BY clause */ WhereLoop *pNew; /* Template WhereLoop */ WhereOrSet *pOrSet; /* Record best loops here, if not NULL */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 UnpackedRecord *pRec; /* Probe for stat4 (if required) */ int nRecValid; /* Number of valid fields currently in pRec */ #endif + unsigned char bldFlags1; /* First set of SQLITE_BLDF_* flags */ + unsigned char bldFlags2; /* Second set of SQLITE_BLDF_* flags */ + unsigned int iPlanLimit; /* Search limiter */ }; +/* Allowed values for WhereLoopBuider.bldFlags */ +#define SQLITE_BLDF1_INDEXED 0x0001 /* An index is used */ +#define SQLITE_BLDF1_UNIQUE 0x0002 /* All keys of a UNIQUE index used */ + +#define SQLITE_BLDF2_2NDPASS 0x0004 /* Second builder pass needed */ + +/* The WhereLoopBuilder.iPlanLimit is used to limit the number of +** index+constraint combinations the query planner will consider for a +** particular query. If this parameter is unlimited, then certain +** pathological queries can spend excess time in the sqlite3WhereBegin() +** routine. The limit is high enough that is should not impact real-world +** queries. +** +** SQLITE_QUERY_PLANNER_LIMIT is the baseline limit. The limit is +** increased by SQLITE_QUERY_PLANNER_LIMIT_INCR before each term of the FROM +** clause is processed, so that every table in a join is guaranteed to be +** able to propose a some index+constraint combinations even if the initial +** baseline limit was exhausted by prior tables of the join. +*/ +#ifndef SQLITE_QUERY_PLANNER_LIMIT +# define SQLITE_QUERY_PLANNER_LIMIT 20000 +#endif +#ifndef SQLITE_QUERY_PLANNER_LIMIT_INCR +# define SQLITE_QUERY_PLANNER_LIMIT_INCR 1000 +#endif + /* ** The WHERE clause processing routine has two halves. The ** first part does the start of the WHERE loop and the second @@ -122699,25 +158223,34 @@ struct WhereInfo { Parse *pParse; /* Parsing and code generating context */ SrcList *pTabList; /* List of tables in the join */ ExprList *pOrderBy; /* The ORDER BY clause or NULL */ - ExprList *pDistinctSet; /* DISTINCT over all these values */ - WhereLoop *pLoops; /* List of all WhereLoop objects */ - Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ - LogEst nRowOut; /* Estimated number of output rows */ - LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ - u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ - i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ - u8 sorted; /* True if really sorted (not just grouped) */ - u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ - u8 untestedTerms; /* Not all WHERE terms resolved by outer loop */ - u8 eDistinct; /* One of the WHERE_DISTINCT_* values below */ - u8 nLevel; /* Number of nested loop */ - int iTop; /* The very beginning of the WHERE loop */ + ExprList *pResultSet; /* Result set of the query */ +#if WHERETRACE_ENABLED + Expr *pWhere; /* The complete WHERE clause */ +#endif + Select *pSelect; /* The entire SELECT statement containing WHERE */ + int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ int iContinue; /* Jump here to continue with next record */ int iBreak; /* Jump here to break out of the loop */ int savedNQueryLoop; /* pParse->nQueryLoop outside the WHERE loop */ - int aiCurOnePass[2]; /* OP_OpenWrite cursors for the ONEPASS opt */ - WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ + u16 wctrlFlags; /* Flags originally passed to sqlite3WhereBegin() */ + LogEst iLimit; /* LIMIT if wctrlFlags has WHERE_USE_LIMIT */ + u8 nLevel; /* Number of nested loop */ + i8 nOBSat; /* Number of ORDER BY terms satisfied by indices */ + u8 eOnePass; /* ONEPASS_OFF, or _SINGLE, or _MULTI */ + u8 eDistinct; /* One of the WHERE_DISTINCT_* values */ + unsigned bDeferredSeek :1; /* Uses OP_DeferredSeek */ + unsigned untestedTerms :1; /* Not all WHERE terms resolved by outer loop */ + unsigned bOrderedInnerLoop:1;/* True if only the inner-most loop is ordered */ + unsigned sorted :1; /* True if really sorted (not just grouped) */ + LogEst nOutStarDelta; /* Artifical nOut reduction for star-query */ + LogEst nRowOut; /* Estimated number of output rows */ + int iTop; /* The very beginning of the WHERE loop */ + int iEndWhere; /* End of the WHERE clause itself */ + WhereLoop *pLoops; /* List of all WhereLoop objects */ + WhereMemBlock *pMemToFree;/* Memory to free when this object destroyed */ + Bitmask revMask; /* Mask of ORDER BY terms that need reversing */ WhereClause sWC; /* Decomposition of the WHERE clause */ + WhereMaskSet sMaskSet; /* Map cursor numbers to bitmasks */ WhereLevel a[1]; /* Information about each nest loop in WHERE */ }; @@ -122727,6 +158260,11 @@ struct WhereInfo { ** where.c: */ SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet*,int); +#ifdef WHERETRACE_ENABLED +SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC); +SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm); +SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC); +#endif SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( WhereClause *pWC, /* The WHERE clause to be searched */ int iCur, /* Cursor number of LHS */ @@ -122735,6 +158273,8 @@ SQLITE_PRIVATE WhereTerm *sqlite3WhereFindTerm( u32 op, /* Mask of WO_xx values describing operator */ Index *pIdx /* Must be compatible with this index, if not NULL */ ); +SQLITE_PRIVATE void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte); +SQLITE_PRIVATE void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte); /* wherecode.c: */ #ifndef SQLITE_OMIT_EXPLAIN @@ -122742,12 +158282,16 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ); +SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( + const Parse *pParse, /* Parse context */ + const WhereInfo *pWInfo, /* WHERE clause */ + const WhereLevel *pLevel /* Bloom filter on this level */ +); #else -# define sqlite3WhereExplainOneScan(u,v,w,x,y,z) 0 +# define sqlite3WhereExplainOneScan(u,v,w,x) 0 +# define sqlite3WhereExplainBloomFilter(u,v,w) 0 #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS SQLITE_PRIVATE void sqlite3WhereAddScanStatus( @@ -122760,19 +158304,29 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( # define sqlite3WhereAddScanStatus(a, b, c, d) ((void)d) #endif SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + Parse *pParse, /* Parsing context */ + Vdbe *v, /* Prepared statement under construction */ WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ + WhereLevel *pLevel, /* The current level pointer */ Bitmask notReady /* Which tables are currently available */ ); +SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( + WhereInfo *pWInfo, + int iLevel, + WhereLevel *pLevel +); /* whereexpr.c: */ SQLITE_PRIVATE void sqlite3WhereClauseInit(WhereClause*,WhereInfo*); SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause*); SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause*,Expr*,u8); +SQLITE_PRIVATE void sqlite3WhereAddLimit(WhereClause*, Select*); SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet*, Expr*); +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet*, Expr*); SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet*, ExprList*); SQLITE_PRIVATE void sqlite3WhereExprAnalyze(SrcList*, WhereClause*); -SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereClause*); +SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, SrcItem*, WhereClause*); @@ -122790,7 +158344,6 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC ** WO_LE == SQLITE_INDEX_CONSTRAINT_LE ** WO_GT == SQLITE_INDEX_CONSTRAINT_GT ** WO_GE == SQLITE_INDEX_CONSTRAINT_GE -** WO_MATCH == SQLITE_INDEX_CONSTRAINT_MATCH */ #define WO_IN 0x0001 #define WO_EQ 0x0002 @@ -122798,15 +158351,16 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC #define WO_LE (WO_EQ<<(TK_LE-TK_EQ)) #define WO_GT (WO_EQ<<(TK_GT-TK_EQ)) #define WO_GE (WO_EQ<<(TK_GE-TK_EQ)) -#define WO_MATCH 0x0040 +#define WO_AUX 0x0040 /* Op useful to virtual tables only */ #define WO_IS 0x0080 #define WO_ISNULL 0x0100 #define WO_OR 0x0200 /* Two or more OR-connected terms */ #define WO_AND 0x0400 /* Two or more AND-connected terms */ #define WO_EQUIV 0x0800 /* Of the form A==B, both columns */ #define WO_NOOP 0x1000 /* This term does not restrict search space */ +#define WO_ROWVAL 0x2000 /* A row-value term */ -#define WO_ALL 0x1fff /* Mask of all possible WO_* values */ +#define WO_ALL 0x3fff /* Mask of all possible WO_* values */ #define WO_SINGLE 0x01ff /* Mask of all non-compound WO_* values */ /* @@ -122833,11 +158387,34 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC #define WHERE_SKIPSCAN 0x00008000 /* Uses the skip-scan algorithm */ #define WHERE_UNQ_WANTED 0x00010000 /* WHERE_ONEROW would have been helpful*/ #define WHERE_PARTIALIDX 0x00020000 /* The automatic index is partial */ +#define WHERE_IN_EARLYOUT 0x00040000 /* Perhaps quit IN loops early */ +#define WHERE_BIGNULL_SORT 0x00080000 /* Column nEq of index is BIGNULL */ +#define WHERE_IN_SEEKSCAN 0x00100000 /* Seek-scan optimization for IN */ +#define WHERE_TRANSCONS 0x00200000 /* Uses a transitive constraint */ +#define WHERE_BLOOMFILTER 0x00400000 /* Consider using a Bloom-filter */ +#define WHERE_SELFCULL 0x00800000 /* nOut reduced by extra WHERE terms */ +#define WHERE_OMIT_OFFSET 0x01000000 /* Set offset counter to zero */ +#define WHERE_COROUTINE 0x02000000 /* Implemented by co-routine. + ** NB: False-negatives are possible */ +#define WHERE_EXPRIDX 0x04000000 /* Uses an index-on-expressions */ + +#endif /* !defined(SQLITE_WHEREINT_H) */ /************** End of whereInt.h ********************************************/ /************** Continuing where we left off in wherecode.c ******************/ #ifndef SQLITE_OMIT_EXPLAIN + +/* +** Return the name of the i-th column of the pIdx index. +*/ +static const char *explainIndexColumnName(Index *pIdx, int i){ + i = pIdx->aiColumn[i]; + if( i==XN_EXPR ) return ""; + if( i==XN_ROWID ) return "rowid"; + return pIdx->pTable->aCol[i].zCnName; +} + /* ** This routine is a helper for explainIndexRange() below ** @@ -122848,28 +158425,36 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs(Parse*, struct SrcList_item*, WhereC */ static void explainAppendTerm( StrAccum *pStr, /* The text expression being built */ - int iTerm, /* Index of this term. First is zero */ - const char *zColumn, /* Name of the column */ + Index *pIdx, /* Index to read column names from */ + int nTerm, /* Number of terms */ + int iTerm, /* Zero-based index of first term. */ + int bAnd, /* Non-zero to append " AND " */ const char *zOp /* Name of the operator */ ){ - if( iTerm ) sqlite3StrAccumAppend(pStr, " AND ", 5); - sqlite3StrAccumAppendAll(pStr, zColumn); - sqlite3StrAccumAppend(pStr, zOp, 1); - sqlite3StrAccumAppend(pStr, "?", 1); + int i; + + assert( nTerm>=1 ); + if( bAnd ) sqlite3_str_append(pStr, " AND ", 5); + + if( nTerm>1 ) sqlite3_str_append(pStr, "(", 1); + for(i=0; i1 ) sqlite3_str_append(pStr, ")", 1); + + sqlite3_str_append(pStr, zOp, 1); + + if( nTerm>1 ) sqlite3_str_append(pStr, "(", 1); + for(i=0; i1 ) sqlite3_str_append(pStr, ")", 1); } /* -** Return the name of the i-th column of the pIdx index. -*/ -static const char *explainIndexColumnName(Index *pIdx, int i){ - i = pIdx->aiColumn[i]; - if( i==XN_EXPR ) return ""; - if( i==XN_ROWID ) return "rowid"; - return pIdx->pTable->aCol[i].zName; -} - -/* -** Argument pLevel describes a strategy for scanning table pTab. This +** Argument pLevel describes a strategy for scanning table pTab. This ** function appends text to pStr that describes the subset of table ** rows scanned by the strategy in the form of an SQL expression. ** @@ -122889,30 +158474,29 @@ static void explainIndexRange(StrAccum *pStr, WhereLoop *pLoop){ int i, j; if( nEq==0 && (pLoop->wsFlags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))==0 ) return; - sqlite3StrAccumAppend(pStr, " (", 2); + sqlite3_str_append(pStr, " (", 2); for(i=0; i=nSkip ? "%s=?" : "ANY(%s)", z); + if( i ) sqlite3_str_append(pStr, " AND ", 5); + sqlite3_str_appendf(pStr, i>=nSkip ? "%s=?" : "ANY(%s)", z); } j = i; if( pLoop->wsFlags&WHERE_BTM_LIMIT ){ - const char *z = explainIndexColumnName(pIndex, i); - explainAppendTerm(pStr, i++, z, ">"); + explainAppendTerm(pStr, pIndex, pLoop->u.btree.nBtm, j, i, ">"); + i = 1; } if( pLoop->wsFlags&WHERE_TOP_LIMIT ){ - const char *z = explainIndexColumnName(pIndex, j); - explainAppendTerm(pStr, i, z, "<"); + explainAppendTerm(pStr, pIndex, pLoop->u.btree.nTop, j, i, "<"); } - sqlite3StrAccumAppend(pStr, ")", 1); + sqlite3_str_append(pStr, ")", 1); } /* ** This function is a no-op unless currently processing an EXPLAIN QUERY PLAN -** command, or if either SQLITE_DEBUG or SQLITE_ENABLE_STMT_SCANSTATUS was -** defined at compile-time. If it is not a no-op, a single OP_Explain opcode -** is added to the output to describe the table scan strategy in pLevel. +** command, or if stmt_scanstatus_v2() stats are enabled, or if SQLITE_DEBUG +** was defined at compile-time. If it is not a no-op, a single OP_Explain +** opcode is added to the output to describe the table scan strategy in pLevel. ** ** If an OP_Explain opcode is added to the VM, its address is returned. ** Otherwise, if no OP_Explain is coded, zero is returned. @@ -122921,19 +158505,16 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( Parse *pParse, /* Parse context */ SrcList *pTabList, /* Table list this loop refers to */ WhereLevel *pLevel, /* Scan to write OP_Explain opcode for */ - int iLevel, /* Value for "level" column of output */ - int iFrom, /* Value for "from" column of output */ u16 wctrlFlags /* Flags passed to sqlite3WhereBegin() */ ){ int ret = 0; -#if !defined(SQLITE_DEBUG) && !defined(SQLITE_ENABLE_STMT_SCANSTATUS) - if( pParse->explain==2 ) +#if !defined(SQLITE_DEBUG) + if( sqlite3ParseToplevel(pParse)->explain==2 || IS_STMT_SCANSTATUS(pParse->db) ) #endif { - struct SrcList_item *pItem = &pTabList->a[pLevel->iFrom]; + SrcItem *pItem = &pTabList->a[pLevel->iFrom]; Vdbe *v = pParse->pVdbe; /* VM being constructed */ sqlite3 *db = pParse->db; /* Database handle */ - int iId = pParse->iSelectId; /* Select id (left-most output column) */ int isSearch; /* True for a SEARCH. False for SCAN. */ WhereLoop *pLoop; /* The controlling WhereLoop object */ u32 flags; /* Flags that describe this loop */ @@ -122943,23 +158524,15 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( pLoop = pLevel->pWLoop; flags = pLoop->wsFlags; - if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_ONETABLE_ONLY) ) return 0; + if( (flags&WHERE_MULTI_OR) || (wctrlFlags&WHERE_OR_SUBCLAUSE) ) return 0; isSearch = (flags&(WHERE_BTM_LIMIT|WHERE_TOP_LIMIT))!=0 || ((flags&WHERE_VIRTUALTABLE)==0 && (pLoop->u.btree.nEq>0)) || (wctrlFlags&(WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX)); sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); - sqlite3StrAccumAppendAll(&str, isSearch ? "SEARCH" : "SCAN"); - if( pItem->pSelect ){ - sqlite3XPrintf(&str, " SUBQUERY %d", pItem->iSelectId); - }else{ - sqlite3XPrintf(&str, " TABLE %s", pItem->zName); - } - - if( pItem->zAlias ){ - sqlite3XPrintf(&str, " AS %s", pItem->zAlias); - } + str.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3_str_appendf(&str, "%s %S", isSearch ? "SEARCH" : "SCAN", pItem); if( (flags & (WHERE_IPK|WHERE_VIRTUALTABLE))==0 ){ const char *zFmt = 0; Index *pIdx; @@ -122967,7 +158540,7 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( assert( pLoop->u.btree.pIndex!=0 ); pIdx = pLoop->u.btree.pIndex; assert( !(flags&WHERE_AUTO_INDEX) || (flags&WHERE_IDX_ONLY) ); - if( !HasRowid(pItem->pTab) && IsPrimaryKeyIndex(pIdx) ){ + if( !HasRowid(pItem->pSTab) && IsPrimaryKeyIndex(pIdx) ){ if( isSearch ){ zFmt = "PRIMARY KEY"; } @@ -122981,52 +158554,122 @@ SQLITE_PRIVATE int sqlite3WhereExplainOneScan( zFmt = "INDEX %s"; } if( zFmt ){ - sqlite3StrAccumAppend(&str, " USING ", 7); - sqlite3XPrintf(&str, zFmt, pIdx->zName); + sqlite3_str_append(&str, " USING ", 7); + sqlite3_str_appendf(&str, zFmt, pIdx->zName); explainIndexRange(&str, pLoop); } }else if( (flags & WHERE_IPK)!=0 && (flags & WHERE_CONSTRAINT)!=0 ){ - const char *zRangeOp; + char cRangeOp; +#if 0 /* Better output, but breaks many tests */ + const Table *pTab = pItem->pTab; + const char *zRowid = pTab->iPKey>=0 ? pTab->aCol[pTab->iPKey].zCnName: + "rowid"; +#else + const char *zRowid = "rowid"; +#endif + sqlite3_str_appendf(&str, " USING INTEGER PRIMARY KEY (%s", zRowid); if( flags&(WHERE_COLUMN_EQ|WHERE_COLUMN_IN) ){ - zRangeOp = "="; + cRangeOp = '='; }else if( (flags&WHERE_BOTH_LIMIT)==WHERE_BOTH_LIMIT ){ - zRangeOp = ">? AND rowid<"; + sqlite3_str_appendf(&str, ">? AND %s", zRowid); + cRangeOp = '<'; }else if( flags&WHERE_BTM_LIMIT ){ - zRangeOp = ">"; + cRangeOp = '>'; }else{ assert( flags&WHERE_TOP_LIMIT); - zRangeOp = "<"; + cRangeOp = '<'; } - sqlite3XPrintf(&str, " USING INTEGER PRIMARY KEY (rowid%s?)",zRangeOp); + sqlite3_str_appendf(&str, "%c?)", cRangeOp); } #ifndef SQLITE_OMIT_VIRTUALTABLE else if( (flags & WHERE_VIRTUALTABLE)!=0 ){ - sqlite3XPrintf(&str, " VIRTUAL TABLE INDEX %d:%s", + sqlite3_str_appendall(&str, " VIRTUAL TABLE INDEX "); + sqlite3_str_appendf(&str, + pLoop->u.vtab.bIdxNumHex ? "0x%x:%s" : "%d:%s", pLoop->u.vtab.idxNum, pLoop->u.vtab.idxStr); } #endif + if( pItem->fg.jointype & JT_LEFT ){ + sqlite3_str_appendf(&str, " LEFT-JOIN"); + } #ifdef SQLITE_EXPLAIN_ESTIMATED_ROWS if( pLoop->nOut>=10 ){ - sqlite3XPrintf(&str, " (~%llu rows)", sqlite3LogEstToInt(pLoop->nOut)); + sqlite3_str_appendf(&str, " (~%llu rows)", + sqlite3LogEstToInt(pLoop->nOut)); }else{ - sqlite3StrAccumAppend(&str, " (~1 row)", 9); + sqlite3_str_append(&str, " (~1 row)", 9); } #endif zMsg = sqlite3StrAccumFinish(&str); - ret = sqlite3VdbeAddOp4(v, OP_Explain, iId, iLevel, iFrom, zMsg,P4_DYNAMIC); + sqlite3ExplainBreakpoint("",zMsg); + ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), + pParse->addrExplain, pLoop->rRun, + zMsg, P4_DYNAMIC); } return ret; } + +/* +** Add a single OP_Explain opcode that describes a Bloom filter. +** +** Or if not processing EXPLAIN QUERY PLAN and not in a SQLITE_DEBUG and/or +** SQLITE_ENABLE_STMT_SCANSTATUS build, then OP_Explain opcodes are not +** required and this routine is a no-op. +** +** If an OP_Explain opcode is added to the VM, its address is returned. +** Otherwise, if no OP_Explain is coded, zero is returned. +*/ +SQLITE_PRIVATE int sqlite3WhereExplainBloomFilter( + const Parse *pParse, /* Parse context */ + const WhereInfo *pWInfo, /* WHERE clause */ + const WhereLevel *pLevel /* Bloom filter on this level */ +){ + int ret = 0; + SrcItem *pItem = &pWInfo->pTabList->a[pLevel->iFrom]; + Vdbe *v = pParse->pVdbe; /* VM being constructed */ + sqlite3 *db = pParse->db; /* Database handle */ + char *zMsg; /* Text to add to EQP output */ + int i; /* Loop counter */ + WhereLoop *pLoop; /* The where loop */ + StrAccum str; /* EQP output string */ + char zBuf[100]; /* Initial space for EQP output string */ + + sqlite3StrAccumInit(&str, db, zBuf, sizeof(zBuf), SQLITE_MAX_LENGTH); + str.printfFlags = SQLITE_PRINTF_INTERNAL; + sqlite3_str_appendf(&str, "BLOOM FILTER ON %S (", pItem); + pLoop = pLevel->pWLoop; + if( pLoop->wsFlags & WHERE_IPK ){ + const Table *pTab = pItem->pSTab; + if( pTab->iPKey>=0 ){ + sqlite3_str_appendf(&str, "%s=?", pTab->aCol[pTab->iPKey].zCnName); + }else{ + sqlite3_str_appendf(&str, "rowid=?"); + } + }else{ + for(i=pLoop->nSkip; iu.btree.nEq; i++){ + const char *z = explainIndexColumnName(pLoop->u.btree.pIndex, i); + if( i>pLoop->nSkip ) sqlite3_str_append(&str, " AND ", 5); + sqlite3_str_appendf(&str, "%s=?", z); + } + } + sqlite3_str_append(&str, ")", 1); + zMsg = sqlite3StrAccumFinish(&str); + ret = sqlite3VdbeAddOp4(v, OP_Explain, sqlite3VdbeCurrentAddr(v), + pParse->addrExplain, 0, zMsg,P4_DYNAMIC); + + sqlite3VdbeScanStatus(v, sqlite3VdbeCurrentAddr(v)-1, 0, 0, 0, 0); + return ret; +} #endif /* SQLITE_OMIT_EXPLAIN */ #ifdef SQLITE_ENABLE_STMT_SCANSTATUS /* ** Configure the VM passed as the first argument with an -** sqlite3_stmt_scanstatus() entry corresponding to the scan used to -** implement level pLvl. Argument pSrclist is a pointer to the FROM +** sqlite3_stmt_scanstatus() entry corresponding to the scan used to +** implement level pLvl. Argument pSrclist is a pointer to the FROM ** clause that the scan reads data from. ** -** If argument addrExplain is not 0, it must be the address of an +** If argument addrExplain is not 0, it must be the address of an ** OP_Explain instruction that describes the same loop. */ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( @@ -123035,16 +158678,39 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( WhereLevel *pLvl, /* Level to add scanstatus() entry for */ int addrExplain /* Address of OP_Explain (or 0) */ ){ - const char *zObj = 0; - WhereLoop *pLoop = pLvl->pWLoop; - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ - zObj = pLoop->u.btree.pIndex->zName; - }else{ - zObj = pSrclist->a[pLvl->iFrom].zName; + if( IS_STMT_SCANSTATUS( sqlite3VdbeDb(v) ) ){ + const char *zObj = 0; + WhereLoop *pLoop = pLvl->pWLoop; + int wsFlags = pLoop->wsFlags; + int viaCoroutine = 0; + + if( (wsFlags & WHERE_VIRTUALTABLE)==0 && pLoop->u.btree.pIndex!=0 ){ + zObj = pLoop->u.btree.pIndex->zName; + }else{ + zObj = pSrclist->a[pLvl->iFrom].zName; + viaCoroutine = pSrclist->a[pLvl->iFrom].fg.viaCoroutine; + } + sqlite3VdbeScanStatus( + v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj + ); + + if( viaCoroutine==0 ){ + if( (wsFlags & (WHERE_MULTI_OR|WHERE_AUTO_INDEX))==0 ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iTabCur); + } + if( wsFlags & WHERE_INDEXED ){ + sqlite3VdbeScanStatusRange(v, addrExplain, -1, pLvl->iIdxCur); + } + }else{ + int addr; + assert( pSrclist->a[pLvl->iFrom].fg.isSubquery ); + addr = pSrclist->a[pLvl->iFrom].u4.pSubq->addrFillSub; + VdbeOp *pOp = sqlite3VdbeGetOp(v, addr-1); + assert( sqlite3VdbeDb(v)->mallocFailed || pOp->opcode==OP_InitCoroutine ); + assert( sqlite3VdbeDb(v)->mallocFailed || pOp->p2>addr ); + sqlite3VdbeScanStatusRange(v, addrExplain, addr, pOp->p2-1); + } } - sqlite3VdbeScanStatus( - v, addrExplain, pLvl->addrBody, pLvl->addrVisit, pLoop->nOut, zObj - ); } #endif @@ -123082,7 +158748,7 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( ** ** Only the parent term was in the original WHERE clause. The child1 ** and child2 terms were added by the LIKE optimization. If both of -** the virtual child terms are valid, then testing of the parent can be +** the virtual child terms are valid, then testing of the parent can be ** skipped. ** ** Usually the parent term is marked as TERM_CODED. But if the parent @@ -123093,9 +158759,9 @@ SQLITE_PRIVATE void sqlite3WhereAddScanStatus( */ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ int nLoop = 0; - while( pTerm - && (pTerm->wtFlags & TERM_CODED)==0 - && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + assert( pTerm!=0 ); + while( (pTerm->wtFlags & TERM_CODED)==0 + && (pLevel->iLeftJoin==0 || ExprHasProperty(pTerm->pExpr, EP_OuterON)) && (pLevel->notReady & pTerm->prereqAll)==0 ){ if( nLoop && (pTerm->wtFlags & TERM_LIKE)!=0 ){ @@ -123103,8 +158769,15 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ }else{ pTerm->wtFlags |= TERM_CODED; } +#ifdef WHERETRACE_ENABLED + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ + sqlite3DebugPrintf("DISABLE-"); + sqlite3WhereTermPrint(pTerm, (int)(pTerm - (pTerm->pWC->a))); + } +#endif if( pTerm->iParent<0 ) break; pTerm = &pTerm->pWC->a[pTerm->iParent]; + assert( pTerm!=0 ); pTerm->nChild--; if( pTerm->nChild!=0 ) break; nLoop++; @@ -123113,11 +158786,11 @@ static void disableTerm(WhereLevel *pLevel, WhereTerm *pTerm){ /* ** Code an OP_Affinity opcode to apply the column affinity string zAff -** to the n registers starting at base. +** to the n registers starting at base. ** -** As an optimization, SQLITE_AFF_BLOB entries (which are no-ops) at the -** beginning and end of zAff are ignored. If all entries in zAff are -** SQLITE_AFF_BLOB, then no code gets generated. +** As an optimization, SQLITE_AFF_BLOB and SQLITE_AFF_NONE entries (which +** are no-ops) at the beginning and end of zAff are ignored. If all entries +** in zAff are SQLITE_AFF_BLOB or SQLITE_AFF_NONE, then no code gets generated. ** ** This routine makes its own copy of zAff so that the caller is free ** to modify zAff after this routine returns. @@ -123130,35 +158803,349 @@ static void codeApplyAffinity(Parse *pParse, int base, int n, char *zAff){ } assert( v!=0 ); - /* Adjust base and n to skip over SQLITE_AFF_BLOB entries at the beginning - ** and end of the affinity string. + /* Adjust base and n to skip over SQLITE_AFF_BLOB and SQLITE_AFF_NONE + ** entries at the beginning and end of the affinity string. */ - while( n>0 && zAff[0]==SQLITE_AFF_BLOB ){ + assert( SQLITE_AFF_NONE0 && zAff[0]<=SQLITE_AFF_BLOB ){ n--; base++; zAff++; } - while( n>1 && zAff[n-1]==SQLITE_AFF_BLOB ){ + while( n>1 && zAff[n-1]<=SQLITE_AFF_BLOB ){ n--; } /* Code the OP_Affinity opcode if there is anything left to do. */ if( n>0 ){ sqlite3VdbeAddOp4(v, OP_Affinity, base, n, 0, zAff, n); - sqlite3ExprCacheAffinityChange(pParse, base, n); + } +} + +/* +** Expression pRight, which is the RHS of a comparison operation, is +** either a vector of n elements or, if n==1, a scalar expression. +** Before the comparison operation, affinity zAff is to be applied +** to the pRight values. This function modifies characters within the +** affinity string to SQLITE_AFF_BLOB if either: +** +** * the comparison will be performed with no affinity, or +** * the affinity change in zAff is guaranteed not to change the value. +*/ +static void updateRangeAffinityStr( + Expr *pRight, /* RHS of comparison */ + int n, /* Number of vector elements in comparison */ + char *zAff /* Affinity string to modify */ +){ + int i; + for(i=0; ia[].u.x.iOrderByCol values might be incorrect because +** columns might have been rearranged in the result set. This routine +** fixes them up. +** +** pEList is the new result set. The pEList->a[].u.x.iOrderByCol values +** contain the *old* locations of each expression. This is a temporary +** use of u.x.iOrderByCol, not its intended use. The caller must reset +** u.x.iOrderByCol back to zero for all entries in pEList before the +** caller returns. +** +** This routine changes pOrderBy->a[].u.x.iOrderByCol values from +** pEList->a[N].u.x.iOrderByCol into N+1. (The "+1" is because of the 1-based +** indexing used by iOrderByCol.) Or if no match, iOrderByCol is set to zero. +*/ +static void adjustOrderByCol(ExprList *pOrderBy, ExprList *pEList){ + int i, j; + if( pOrderBy==0 ) return; + for(i=0; inExpr; i++){ + int t = pOrderBy->a[i].u.x.iOrderByCol; + if( t==0 ) continue; + for(j=0; jnExpr; j++){ + if( pEList->a[j].u.x.iOrderByCol==t ){ + pOrderBy->a[i].u.x.iOrderByCol = j+1; + break; + } + } + if( j>=pEList->nExpr ){ + pOrderBy->a[i].u.x.iOrderByCol = 0; + } } } +/* +** pX is an expression of the form: (vector) IN (SELECT ...) +** In other words, it is a vector IN operator with a SELECT clause on the +** LHS. But not all terms in the vector are indexable and the terms might +** not be in the correct order for indexing. +** +** This routine makes a copy of the input pX expression and then adjusts +** the vector on the LHS with corresponding changes to the SELECT so that +** the vector contains only index terms and those terms are in the correct +** order. The modified IN expression is returned. The caller is responsible +** for deleting the returned expression. +** +** Example: +** +** CREATE TABLE t1(a,b,c,d,e,f); +** CREATE INDEX t1x1 ON t1(e,c); +** SELECT * FROM t1 WHERE (a,b,c,d,e) IN (SELECT v,w,x,y,z FROM t2) +** \_______________________________________/ +** The pX expression +** +** Since only columns e and c can be used with the index, in that order, +** the modified IN expression that is returned will be: +** +** (e,c) IN (SELECT z,x FROM t2) +** +** The reduced pX is different from the original (obviously) and thus is +** only used for indexing, to improve performance. The original unaltered +** IN expression must also be run on each output row for correctness. +*/ +static Expr *removeUnindexableInClauseTerms( + Parse *pParse, /* The parsing context */ + int iEq, /* Look at loop terms starting here */ + WhereLoop *pLoop, /* The current loop */ + Expr *pX /* The IN expression to be reduced */ +){ + sqlite3 *db = pParse->db; + Select *pSelect; /* Pointer to the SELECT on the RHS */ + Expr *pNew; + pNew = sqlite3ExprDup(db, pX, 0); + if( db->mallocFailed==0 ){ + for(pSelect=pNew->x.pSelect; pSelect; pSelect=pSelect->pPrior){ + ExprList *pOrigRhs; /* Original unmodified RHS */ + ExprList *pOrigLhs = 0; /* Original unmodified LHS */ + ExprList *pRhs = 0; /* New RHS after modifications */ + ExprList *pLhs = 0; /* New LHS after mods */ + int i; /* Loop counter */ + + assert( ExprUseXSelect(pNew) ); + pOrigRhs = pSelect->pEList; + assert( pNew->pLeft!=0 ); + assert( ExprUseXList(pNew->pLeft) ); + if( pSelect==pNew->x.pSelect ){ + pOrigLhs = pNew->pLeft->x.pList; + } + for(i=iEq; inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iField; + assert( (pLoop->aLTerm[i]->eOperator & (WO_OR|WO_AND))==0 ); + iField = pLoop->aLTerm[i]->u.x.iField - 1; + if( pOrigRhs->a[iField].pExpr==0 ) continue; /* Duplicate PK column */ + pRhs = sqlite3ExprListAppend(pParse, pRhs, pOrigRhs->a[iField].pExpr); + pOrigRhs->a[iField].pExpr = 0; + if( pRhs ) pRhs->a[pRhs->nExpr-1].u.x.iOrderByCol = iField+1; + if( pOrigLhs ){ + assert( pOrigLhs->a[iField].pExpr!=0 ); + pLhs = sqlite3ExprListAppend(pParse,pLhs,pOrigLhs->a[iField].pExpr); + pOrigLhs->a[iField].pExpr = 0; + } + } + } + sqlite3ExprListDelete(db, pOrigRhs); + if( pOrigLhs ){ + sqlite3ExprListDelete(db, pOrigLhs); + pNew->pLeft->x.pList = pLhs; + } + pSelect->pEList = pRhs; + pSelect->selId = ++pParse->nSelect; /* Req'd for SubrtnSig validity */ + if( pLhs && pLhs->nExpr==1 ){ + /* Take care here not to generate a TK_VECTOR containing only a + ** single value. Since the parser never creates such a vector, some + ** of the subroutines do not handle this case. */ + Expr *p = pLhs->a[0].pExpr; + pLhs->a[0].pExpr = 0; + sqlite3ExprDelete(db, pNew->pLeft); + pNew->pLeft = p; + } + + /* If either the ORDER BY clause or the GROUP BY clause contains + ** references to result-set columns, those references might now be + ** obsolete. So fix them up. + */ + assert( pRhs!=0 || db->mallocFailed ); + if( pRhs ){ + adjustOrderByCol(pSelect->pOrderBy, pRhs); + adjustOrderByCol(pSelect->pGroupBy, pRhs); + for(i=0; inExpr; i++) pRhs->a[i].u.x.iOrderByCol = 0; + } + +#if 0 + printf("For indexing, change the IN expr:\n"); + sqlite3TreeViewExpr(0, pX, 0); + printf("Into:\n"); + sqlite3TreeViewExpr(0, pNew, 0); +#endif + } + } + return pNew; +} + + +#ifndef SQLITE_OMIT_SUBQUERY +/* +** Generate code for a single X IN (....) term of the WHERE clause. +** +** This is a special-case of codeEqualityTerm() that works for IN operators +** only. It is broken out into a subroutine because this case is +** uncommon and by splitting it off into a subroutine, the common case +** runs faster. +** +** The current value for the constraint is left in register iTarget. +** This routine sets up a loop that will iterate over all values of X. +*/ +static SQLITE_NOINLINE void codeINTerm( + Parse *pParse, /* The parsing context */ + WhereTerm *pTerm, /* The term of the WHERE clause to be coded */ + WhereLevel *pLevel, /* The level of the FROM clause we are working on */ + int iEq, /* Index of the equality term within this level */ + int bRev, /* True for reverse-order IN operations */ + int iTarget /* Attempt to leave results in this register */ +){ + Expr *pX = pTerm->pExpr; + int eType = IN_INDEX_NOOP; + int iTab; + struct InLoop *pIn; + WhereLoop *pLoop = pLevel->pWLoop; + Vdbe *v = pParse->pVdbe; + int i; + int nEq = 0; + int *aiMap = 0; + + if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && pLoop->u.btree.pIndex!=0 + && pLoop->u.btree.pIndex->aSortOrder[iEq] + ){ + testcase( iEq==0 ); + testcase( bRev ); + bRev = !bRev; + } + assert( pX->op==TK_IN ); + + for(i=0; iaLTerm[i] && pLoop->aLTerm[i]->pExpr==pX ){ + disableTerm(pLevel, pTerm); + return; + } + } + for(i=iEq;inLTerm; i++){ + assert( pLoop->aLTerm[i]!=0 ); + if( pLoop->aLTerm[i]->pExpr==pX ) nEq++; + } + + iTab = 0; + if( !ExprUseXSelect(pX) || pX->x.pSelect->pEList->nExpr==1 ){ + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, 0, &iTab); + }else{ + Expr *pExpr = pTerm->pExpr; + if( pExpr->iTable==0 || !ExprHasProperty(pExpr, EP_Subrtn) ){ + sqlite3 *db = pParse->db; + pX = removeUnindexableInClauseTerms(pParse, iEq, pLoop, pX); + if( !db->mallocFailed ){ + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*nEq); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap,&iTab); + pExpr->iTable = iTab; + } + sqlite3ExprDelete(db, pX); + }else{ + int n = sqlite3ExprVectorSize(pX->pLeft); + aiMap = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*MAX(nEq,n)); + eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0, aiMap, &iTab); + } + pX = pExpr; + } + + if( eType==IN_INDEX_INDEX_DESC ){ + testcase( bRev ); + bRev = !bRev; + } + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); + VdbeCoverageIf(v, bRev); + VdbeCoverageIf(v, !bRev); + + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + pLoop->wsFlags |= WHERE_IN_ABLE; + if( pLevel->u.in.nIn==0 ){ + pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + } + if( iEq>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 ){ + pLoop->wsFlags |= WHERE_IN_EARLYOUT; + } + + i = pLevel->u.in.nIn; + pLevel->u.in.nIn += nEq; + pLevel->u.in.aInLoop = + sqlite3WhereRealloc(pTerm->pWC->pWInfo, + pLevel->u.in.aInLoop, + sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); + pIn = pLevel->u.in.aInLoop; + if( pIn ){ + int iMap = 0; /* Index in aiMap[] */ + pIn += i; + for(i=iEq;inLTerm; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + int iOut = iTarget + i - iEq; + if( eType==IN_INDEX_ROWID ){ + pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iOut); + }else{ + int iCol = aiMap ? aiMap[iMap++] : 0; + pIn->addrInTop = sqlite3VdbeAddOp3(v,OP_Column,iTab, iCol, iOut); + } + sqlite3VdbeAddOp1(v, OP_IsNull, iOut); VdbeCoverage(v); + if( i==iEq ){ + pIn->iCur = iTab; + pIn->eEndLoopOp = bRev ? OP_Prev : OP_Next; + if( iEq>0 ){ + pIn->iBase = iTarget - i; + pIn->nPrefix = i; + }else{ + pIn->nPrefix = 0; + } + }else{ + pIn->eEndLoopOp = OP_Noop; + } + pIn++; + } + } + testcase( iEq>0 + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 + && (pLoop->wsFlags & WHERE_VIRTUALTABLE)!=0 ); + if( iEq>0 + && (pLoop->wsFlags & (WHERE_IN_SEEKSCAN|WHERE_VIRTUALTABLE))==0 + ){ + sqlite3VdbeAddOp3(v, OP_SeekHit, pLevel->iIdxCur, 0, iEq); + } + }else{ + pLevel->u.in.nIn = 0; + } + sqlite3DbFree(pParse->db, aiMap); +} +#endif + + /* ** Generate code for a single equality term of the WHERE clause. An equality -** term can be either X=expr or X IN (...). pTerm is the term to be +** term can be either X=expr or X IN (...). pTerm is the term to be ** coded. ** -** The current value for the constraint is left in register iReg. +** The current value for the constraint is left in a register, the index +** of which is returned. An attempt is made store the result in iTarget but +** this is only guaranteed for TK_ISNULL and TK_IN constraints. If the +** constraint is a TK_EQ or TK_IS, then the current value might be left in +** some other register and it is the caller's responsibility to compensate. ** -** For a constraint of the form X=expr, the expression is evaluated and its -** result is left on the stack. For constraints of the form X IN (...) +** For a constraint of the form X=expr, the expression is evaluated in +** straight-line code. For constraints of the form X IN (...) ** this routine sets up a loop that will iterate over all values of X. */ static int codeEqualityTerm( @@ -123170,67 +159157,38 @@ static int codeEqualityTerm( int iTarget /* Attempt to leave results in this register */ ){ Expr *pX = pTerm->pExpr; - Vdbe *v = pParse->pVdbe; int iReg; /* Register holding results */ + assert( pLevel->pWLoop->aLTerm[iEq]==pTerm ); assert( iTarget>0 ); if( pX->op==TK_EQ || pX->op==TK_IS ){ iReg = sqlite3ExprCodeTarget(pParse, pX->pRight, iTarget); }else if( pX->op==TK_ISNULL ){ iReg = iTarget; - sqlite3VdbeAddOp2(v, OP_Null, 0, iReg); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_Null, 0, iReg); #ifndef SQLITE_OMIT_SUBQUERY }else{ - int eType; - int iTab; - struct InLoop *pIn; - WhereLoop *pLoop = pLevel->pWLoop; - - if( (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 - && pLoop->u.btree.pIndex!=0 - && pLoop->u.btree.pIndex->aSortOrder[iEq] - ){ - testcase( iEq==0 ); - testcase( bRev ); - bRev = !bRev; - } assert( pX->op==TK_IN ); iReg = iTarget; - eType = sqlite3FindInIndex(pParse, pX, IN_INDEX_LOOP, 0); - if( eType==IN_INDEX_INDEX_DESC ){ - testcase( bRev ); - bRev = !bRev; - } - iTab = pX->iTable; - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iTab, 0); - VdbeCoverageIf(v, bRev); - VdbeCoverageIf(v, !bRev); - assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); - pLoop->wsFlags |= WHERE_IN_ABLE; - if( pLevel->u.in.nIn==0 ){ - pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - } - pLevel->u.in.nIn++; - pLevel->u.in.aInLoop = - sqlite3DbReallocOrFree(pParse->db, pLevel->u.in.aInLoop, - sizeof(pLevel->u.in.aInLoop[0])*pLevel->u.in.nIn); - pIn = pLevel->u.in.aInLoop; - if( pIn ){ - pIn += pLevel->u.in.nIn - 1; - pIn->iCur = iTab; - if( eType==IN_INDEX_ROWID ){ - pIn->addrInTop = sqlite3VdbeAddOp2(v, OP_Rowid, iTab, iReg); - }else{ - pIn->addrInTop = sqlite3VdbeAddOp3(v, OP_Column, iTab, 0, iReg); - } - pIn->eEndLoopOp = bRev ? OP_PrevIfOpen : OP_NextIfOpen; - sqlite3VdbeAddOp1(v, OP_IsNull, iReg); VdbeCoverage(v); - }else{ - pLevel->u.in.nIn = 0; - } + codeINTerm(pParse, pTerm, pLevel, iEq, bRev, iTarget); #endif } - disableTerm(pLevel, pTerm); + + /* As an optimization, try to disable the WHERE clause term that is + ** driving the index as it will always be true. The correct answer is + ** obtained regardless, but we might get the answer with fewer CPU cycles + ** by omitting the term. + ** + ** But do not disable the term unless we are certain that the term is + ** not a transitive constraint. For an example of where that does not + ** work, see https://sqlite.org/forum/forumpost/eb8613976a (2021-05-04) + */ + if( (pLevel->pWLoop->wsFlags & WHERE_TRANSCONS)==0 + || (pTerm->eOperator & WO_EQUIV)==0 + ){ + disableTerm(pLevel, pTerm); + } + return iReg; } @@ -123241,7 +159199,7 @@ static int codeEqualityTerm( ** For example, consider table t1(a,b,c,d,e,f) with index i1(a,b,c). ** Suppose the WHERE clause is this: a==5 AND b IN (1,2,3) AND c>5 AND c<10 ** The index has as many as three equality constraints, but in this -** example, the third "c" value is an inequality. So only two +** example, the third "c" value is an inequality. So only two ** constraints are coded. This routine will generate code to evaluate ** a==5 and b IN (1,2,3). The current values for a and b will be stored ** in consecutive registers and the index of the first register is returned. @@ -123308,7 +159266,7 @@ static int codeAllEqualityTerms( /* Figure out how many memory cells we will need then allocate them. */ regBase = pParse->nMem + 1; - nReg = pLoop->u.btree.nEq + nExtraReg; + nReg = nEq + nExtraReg; pParse->nMem += nReg; zAff = sqlite3DbStrDup(pParse->db,sqlite3IndexAffinityStr(pParse->db,pIdx)); @@ -123316,11 +159274,13 @@ static int codeAllEqualityTerms( if( nSkip ){ int iIdxCur = pLevel->iIdxCur; + sqlite3VdbeAddOp3(v, OP_Null, 0, regBase, regBase+nSkip-1); sqlite3VdbeAddOp1(v, (bRev?OP_Last:OP_Rewind), iIdxCur); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); VdbeComment((v, "begin skip-scan on %s", pIdx->zName)); j = sqlite3VdbeAddOp0(v, OP_Goto); + assert( pLevel->addrSkip==0 ); pLevel->addrSkip = sqlite3VdbeAddOp4Int(v, (bRev?OP_SeekLT:OP_SeekGT), iIdxCur, 0, regBase, nSkip); VdbeCoverageIf(v, bRev==0); @@ -123331,7 +159291,7 @@ static int codeAllEqualityTerms( testcase( pIdx->aiColumn[j]==XN_EXPR ); VdbeComment((v, "%s", explainIndexColumnName(pIdx, j))); } - } + } /* Evaluate the equality constraints */ @@ -123340,7 +159300,7 @@ static int codeAllEqualityTerms( int r1; pTerm = pLoop->aLTerm[j]; assert( pTerm!=0 ); - /* The following testcase is true for indices with redundant columns. + /* The following testcase is true for indices with redundant columns. ** Ex: CREATE INDEX i1 ON t1(a,b,a); SELECT * FROM t1 WHERE a=0 AND b=0; */ testcase( (pTerm->wtFlags & TERM_CODED)!=0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); @@ -123350,18 +159310,25 @@ static int codeAllEqualityTerms( sqlite3ReleaseTempReg(pParse, regBase); regBase = r1; }else{ - sqlite3VdbeAddOp2(v, OP_SCopy, r1, regBase+j); + sqlite3VdbeAddOp2(v, OP_Copy, r1, regBase+j); } } - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IN ); - if( (pTerm->eOperator & (WO_ISNULL|WO_IN))==0 ){ + if( pTerm->eOperator & WO_IN ){ + if( pTerm->pExpr->flags & EP_xIsSelect ){ + /* No affinity ever needs to be (or should be) applied to a value + ** from the RHS of an "? IN (SELECT ...)" expression. The + ** sqlite3FindInIndex() routine has already ensured that the + ** affinity of the comparison has been applied to the value. */ + if( zAff ) zAff[j] = SQLITE_AFF_BLOB; + } + }else if( (pTerm->eOperator & WO_ISNULL)==0 ){ Expr *pRight = pTerm->pExpr->pRight; if( (pTerm->wtFlags & TERM_IS)==0 && sqlite3ExprCanBeNull(pRight) ){ sqlite3VdbeAddOp2(v, OP_IsNull, regBase+j, pLevel->addrBrk); VdbeCoverage(v); } - if( zAff ){ + if( pParse->nErr==0 ){ + assert( pParse->db->mallocFailed==0 ); if( sqlite3CompareAffinity(pRight, zAff[j])==SQLITE_AFF_BLOB ){ zAff[j] = SQLITE_AFF_BLOB; } @@ -123378,7 +159345,7 @@ static int codeAllEqualityTerms( #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS /* ** If the most recently coded instruction is a constant range constraint -** (a string literal) that originated from the LIKE optimization, then +** (a string literal) that originated from the LIKE optimization, then ** set P3 and P5 on the OP_String opcode so that the string will be cast ** to a BLOB at appropriate times. ** @@ -123386,7 +159353,7 @@ static int codeAllEqualityTerms( ** expression: "x>='ABC' AND x<'abd'". But this requires that the range ** scan loop run twice, once for strings and a second time for BLOBs. ** The OP_String opcodes on the second pass convert the upper and lower -** bound string contants to blobs. This routine makes the necessary changes +** bound string constants to blobs. This routine makes the necessary changes ** to the OP_String opcodes for that to happen. ** ** Except, of course, if SQLITE_LIKE_DOESNT_MATCH_BLOBS is defined, then @@ -123401,9 +159368,9 @@ static void whereLikeOptimizationStringFixup( if( pTerm->wtFlags & TERM_LIKEOPT ){ VdbeOp *pOp; assert( pLevel->iLikeRepCntr>0 ); - pOp = sqlite3VdbeGetOp(v, -1); + pOp = sqlite3VdbeGetLastOp(v); assert( pOp!=0 ); - assert( pOp->opcode==OP_String8 + assert( pOp->opcode==OP_String8 || pTerm->pWC->pWInfo->pParse->db->mallocFailed ); pOp->p3 = (int)(pLevel->iLikeRepCntr>>1); /* Register holding counter */ pOp->p5 = (u8)(pLevel->iLikeRepCntr&1); /* ASC or DESC */ @@ -123436,13 +159403,45 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ assert( pHint->pIdx!=0 ); if( pExpr->op==TK_COLUMN && pExpr->iTable==pHint->iTabCur - && sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn)<0 + && sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn)<0 ){ pWalker->eCode = 1; } return WRC_Continue; } +/* +** Test whether or not expression pExpr, which was part of a WHERE clause, +** should be included in the cursor-hint for a table that is on the rhs +** of a LEFT JOIN. Set Walker.eCode to non-zero before returning if the +** expression is not suitable. +** +** An expression is unsuitable if it might evaluate to non NULL even if +** a TK_COLUMN node that does affect the value of the expression is set +** to NULL. For example: +** +** col IS NULL +** col IS NOT NULL +** coalesce(col, 1) +** CASE WHEN col THEN 0 ELSE 1 END +*/ +static int codeCursorHintIsOrFunction(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_IS + || pExpr->op==TK_ISNULL || pExpr->op==TK_ISNOT + || pExpr->op==TK_NOTNULL || pExpr->op==TK_CASE + ){ + pWalker->eCode = 1; + }else if( pExpr->op==TK_FUNCTION ){ + int d1; + char d2[4]; + if( 0==sqlite3IsLikeFunction(pWalker->pParse->db, pExpr, &d1, d2) ){ + pWalker->eCode = 1; + } + } + + return WRC_Continue; +} + /* ** This function is called on every node of an expression tree used as an @@ -123450,43 +159449,41 @@ static int codeCursorHintCheckExpr(Walker *pWalker, Expr *pExpr){ ** that accesses any table other than the one identified by ** CCurHint.iTabCur, then do the following: ** -** 1) allocate a register and code an OP_Column instruction to read +** 1) allocate a register and code an OP_Column instruction to read ** the specified column into the new register, and ** -** 2) transform the expression node to a TK_REGISTER node that reads +** 2) transform the expression node to a TK_REGISTER node that reads ** from the newly populated register. ** -** Also, if the node is a TK_COLUMN that does access the table idenified +** Also, if the node is a TK_COLUMN that does access the table identified ** by pCCurHint.iTabCur, and an index is being used (which we will ** know because CCurHint.pIdx!=0) then transform the TK_COLUMN into ** an access of the index rather than the original table. */ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ int rc = WRC_Continue; + int reg; struct CCurHint *pHint = pWalker->u.pCCurHint; if( pExpr->op==TK_COLUMN ){ if( pExpr->iTable!=pHint->iTabCur ){ - Vdbe *v = pWalker->pParse->pVdbe; - int reg = ++pWalker->pParse->nMem; /* Register for column value */ - sqlite3ExprCodeGetColumnOfTable( - v, pExpr->pTab, pExpr->iTable, pExpr->iColumn, reg - ); + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); pExpr->op = TK_REGISTER; pExpr->iTable = reg; }else if( pHint->pIdx!=0 ){ pExpr->iTable = pHint->iIdxCur; - pExpr->iColumn = sqlite3ColumnOfIndex(pHint->pIdx, pExpr->iColumn); + pExpr->iColumn = sqlite3TableColumnToIndex(pHint->pIdx, pExpr->iColumn); assert( pExpr->iColumn>=0 ); } - }else if( pExpr->op==TK_AGG_FUNCTION ){ - /* An aggregate function in the WHERE clause of a query means this must - ** be a correlated sub-query, and expression pExpr is an aggregate from - ** the parent context. Do not walk the function arguments in this case. - ** - ** todo: It should be possible to replace this node with a TK_REGISTER - ** expression, as the result of the expression must be stored in a - ** register at this point. The same holds for TK_AGG_COLUMN nodes. */ + }else if( pExpr->pAggInfo ){ rc = WRC_Prune; + reg = ++pWalker->pParse->nMem; /* Register for column value */ + reg = sqlite3ExprCodeTarget(pWalker->pParse, pExpr, reg); + pExpr->op = TK_REGISTER; + pExpr->iTable = reg; + }else if( pExpr->op==TK_TRUEFALSE ){ + /* Do not walk disabled expressions. tag-20230504-1 */ + return WRC_Prune; } return rc; } @@ -123495,6 +159492,7 @@ static int codeCursorHintFixExpr(Walker *pWalker, Expr *pExpr){ ** Insert an OP_CursorHint instruction if it is appropriate to do so. */ static void codeCursorHint( + SrcItem *pTabItem, /* FROM clause item */ WhereInfo *pWInfo, /* The where clause */ WhereLevel *pLevel, /* Which loop to provide hints for */ WhereTerm *pEndRange /* Hint this end-of-scan boundary term if not NULL */ @@ -123521,11 +159519,46 @@ static void codeCursorHint( sWalker.pParse = pParse; sWalker.u.pCCurHint = &sHint; pWC = &pWInfo->sWC; - for(i=0; inTerm; i++){ + for(i=0; inBase; i++){ pTerm = &pWC->a[i]; if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( pTerm->prereqAll & pLevel->notReady ) continue; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) continue; + + /* Any terms specified as part of the ON(...) clause for any LEFT + ** JOIN for which the current table is not the rhs are omitted + ** from the cursor-hint. + ** + ** If this table is the rhs of a LEFT JOIN, "IS" or "IS NULL" terms + ** that were specified as part of the WHERE clause must be excluded. + ** This is to address the following: + ** + ** SELECT ... t1 LEFT JOIN t2 ON (t1.a=t2.b) WHERE t2.c IS NULL; + ** + ** Say there is a single row in t2 that matches (t1.a=t2.b), but its + ** t2.c values is not NULL. If the (t2.c IS NULL) constraint is + ** pushed down to the cursor, this row is filtered out, causing + ** SQLite to synthesize a row of NULL values. Which does match the + ** WHERE clause, and so the query returns a row. Which is incorrect. + ** + ** For the same reason, WHERE terms such as: + ** + ** WHERE 1 = (t2.c IS NULL) + ** + ** are also excluded. See codeCursorHintIsOrFunction() for details. + */ + if( pTabItem->fg.jointype & JT_LEFT ){ + Expr *pExpr = pTerm->pExpr; + if( !ExprHasProperty(pExpr, EP_OuterON) + || pExpr->w.iJoin!=pTabItem->iCursor + ){ + sWalker.eCode = 0; + sWalker.xExprCallback = codeCursorHintIsOrFunction; + sqlite3WalkExpr(&sWalker, pTerm->pExpr); + if( sWalker.eCode ) continue; + } + }else{ + if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) continue; + } /* All terms in pWLoop->aLTerm[] except pEndRange are used to initialize ** the cursor. These terms are not needed as hints for a pure range @@ -123548,37 +159581,45 @@ static void codeCursorHint( } /* If we survive all prior tests, that means this term is worth hinting */ - pExpr = sqlite3ExprAnd(db, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); + pExpr = sqlite3ExprAnd(pParse, pExpr, sqlite3ExprDup(db, pTerm->pExpr, 0)); } if( pExpr!=0 ){ sWalker.xExprCallback = codeCursorHintFixExpr; - sqlite3WalkExpr(&sWalker, pExpr); - sqlite3VdbeAddOp4(v, OP_CursorHint, + if( pParse->nErr==0 ) sqlite3WalkExpr(&sWalker, pExpr); + sqlite3VdbeAddOp4(v, OP_CursorHint, (sHint.pIdx ? sHint.iIdxCur : sHint.iTabCur), 0, 0, (const char*)pExpr, P4_EXPR); } } #else -# define codeCursorHint(A,B,C) /* No-op */ +# define codeCursorHint(A,B,C,D) /* No-op */ #endif /* SQLITE_ENABLE_CURSOR_HINTS */ /* ** Cursor iCur is open on an intkey b-tree (a table). Register iRowid contains ** a rowid value just read from cursor iIdxCur, open on index pIdx. This -** function generates code to do a deferred seek of cursor iCur to the +** function generates code to do a deferred seek of cursor iCur to the ** rowid stored in register iRowid. ** ** Normally, this is just: ** -** OP_Seek $iCur $iRowid +** OP_DeferredSeek $iCur $iRowid +** +** Which causes a seek on $iCur to the row with rowid $iRowid. ** ** However, if the scan currently being coded is a branch of an OR-loop and -** the statement currently being coded is a SELECT, then P3 of the OP_Seek -** is set to iIdxCur and P4 is set to point to an array of integers -** containing one entry for each column of the table cursor iCur is open -** on. For each table column, if the column is the i'th column of the -** index, then the corresponding array entry is set to (i+1). If the column -** does not appear in the index at all, the array entry is set to 0. +** the statement currently being coded is a SELECT, then additional information +** is added that might allow OP_Column to omit the seek and instead do its +** lookup on the index, thus avoiding an expensive seek operation. To +** enable this optimization, the P3 of OP_DeferredSeek is set to iIdxCur +** and P4 is set to an array of integers containing one entry for each column +** in the table. For each table column, if the column is the i'th +** column of the index, then the corresponding array entry is set to (i+1). +** If the column does not appear in the index at all, the array entry is set +** to 0. The OP_Column opcode can check this array to see if the column it +** wants is in the index and if it is, it will substitute the index cursor +** and column number and continue with those new values, rather than seeking +** the table cursor. */ static void codeDeferredSeek( WhereInfo *pWInfo, /* Where clause context */ @@ -123591,65 +159632,235 @@ static void codeDeferredSeek( assert( iIdxCur>0 ); assert( pIdx->aiColumn[pIdx->nColumn-1]==-1 ); - - sqlite3VdbeAddOp3(v, OP_Seek, iIdxCur, 0, iCur); - if( (pWInfo->wctrlFlags & WHERE_FORCE_TABLE) + + pWInfo->bDeferredSeek = 1; + sqlite3VdbeAddOp3(v, OP_DeferredSeek, iIdxCur, 0, iCur); + if( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN)) && DbMaskAllZero(sqlite3ParseToplevel(pParse)->writeMask) ){ int i; Table *pTab = pIdx->pTable; - int *ai = (int*)sqlite3DbMallocZero(pParse->db, sizeof(int)*(pTab->nCol+1)); + u32 *ai = (u32*)sqlite3DbMallocZero(pParse->db, sizeof(u32)*(pTab->nCol+1)); if( ai ){ ai[0] = pTab->nCol; for(i=0; inColumn-1; i++){ + int x1, x2; assert( pIdx->aiColumn[i]nCol ); - if( pIdx->aiColumn[i]>=0 ) ai[pIdx->aiColumn[i]+1] = i+1; + x1 = pIdx->aiColumn[i]; + x2 = sqlite3TableColumnToStorage(pTab, x1); + testcase( x1!=x2 ); + if( x1>=0 ) ai[x2+1] = i+1; } sqlite3VdbeChangeP4(v, -1, (char*)ai, P4_INTARRAY); } } } +/* +** If the expression passed as the second argument is a vector, generate +** code to write the first nReg elements of the vector into an array +** of registers starting with iReg. +** +** If the expression is not a vector, then nReg must be passed 1. In +** this case, generate code to evaluate the expression and leave the +** result in register iReg. +*/ +static void codeExprOrVector(Parse *pParse, Expr *p, int iReg, int nReg){ + assert( nReg>0 ); + if( p && sqlite3ExprIsVector(p) ){ +#ifndef SQLITE_OMIT_SUBQUERY + if( ExprUseXSelect(p) ){ + Vdbe *v = pParse->pVdbe; + int iSelect; + assert( p->op==TK_SELECT ); + iSelect = sqlite3CodeSubselect(pParse, p); + sqlite3VdbeAddOp3(v, OP_Copy, iSelect, iReg, nReg-1); + }else +#endif + { + int i; + const ExprList *pList; + assert( ExprUseXList(p) ); + pList = p->x.pList; + assert( nReg<=pList->nExpr ); + for(i=0; ia[i].pExpr, iReg+i); + } + } + }else{ + assert( nReg==1 || pParse->nErr ); + sqlite3ExprCode(pParse, p, iReg); + } +} + +/* +** The pTruth expression is always true because it is the WHERE clause +** a partial index that is driving a query loop. Look through all of the +** WHERE clause terms on the query, and if any of those terms must be +** true because pTruth is true, then mark those WHERE clause terms as +** coded. +*/ +static void whereApplyPartialIndexConstraints( + Expr *pTruth, + int iTabCur, + WhereClause *pWC +){ + int i; + WhereTerm *pTerm; + while( pTruth->op==TK_AND ){ + whereApplyPartialIndexConstraints(pTruth->pLeft, iTabCur, pWC); + pTruth = pTruth->pRight; + } + for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ + Expr *pExpr; + if( pTerm->wtFlags & TERM_CODED ) continue; + pExpr = pTerm->pExpr; + if( sqlite3ExprCompare(0, pExpr, pTruth, iTabCur)==0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } +} + +/* +** This routine is called right after An OP_Filter has been generated and +** before the corresponding index search has been performed. This routine +** checks to see if there are additional Bloom filters in inner loops that +** can be checked prior to doing the index lookup. If there are available +** inner-loop Bloom filters, then evaluate those filters now, before the +** index lookup. The idea is that a Bloom filter check is way faster than +** an index lookup, and the Bloom filter might return false, meaning that +** the index lookup can be skipped. +** +** We know that an inner loop uses a Bloom filter because it has the +** WhereLevel.regFilter set. If an inner-loop Bloom filter is checked, +** then clear the WhereLevel.regFilter value to prevent the Bloom filter +** from being checked a second time when the inner loop is evaluated. +*/ +static SQLITE_NOINLINE void filterPullDown( + Parse *pParse, /* Parsing context */ + WhereInfo *pWInfo, /* Complete information about the WHERE clause */ + int iLevel, /* Which level of pWInfo->a[] should be coded */ + int addrNxt, /* Jump here to bypass inner loops */ + Bitmask notReady /* Loops that are not ready */ +){ + while( ++iLevel < pWInfo->nLevel ){ + WhereLevel *pLevel = &pWInfo->a[iLevel]; + WhereLoop *pLoop = pLevel->pWLoop; + if( pLevel->regFilter==0 ) continue; + if( pLevel->pWLoop->nSkip ) continue; + /* ,--- Because sqlite3ConstructBloomFilter() has will not have set + ** vvvvv--' pLevel->regFilter if this were true. */ + if( NEVER(pLoop->prereq & notReady) ) continue; + assert( pLevel->addrBrk==0 ); + pLevel->addrBrk = addrNxt; + if( pLoop->wsFlags & WHERE_IPK ){ + WhereTerm *pTerm = pLoop->aLTerm[0]; + int regRowid; + assert( pTerm!=0 ); + assert( pTerm->pExpr!=0 ); + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + regRowid = sqlite3GetTempReg(pParse); + regRowid = codeEqualityTerm(pParse, pTerm, pLevel, 0, 0, regRowid); + sqlite3VdbeAddOp2(pParse->pVdbe, OP_MustBeInt, regRowid, addrNxt); + VdbeCoverage(pParse->pVdbe); + sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, + addrNxt, regRowid, 1); + VdbeCoverage(pParse->pVdbe); + }else{ + u16 nEq = pLoop->u.btree.nEq; + int r1; + char *zStartAff; + + assert( pLoop->wsFlags & WHERE_INDEXED ); + assert( (pLoop->wsFlags & WHERE_COLUMN_IN)==0 ); + r1 = codeAllEqualityTerms(pParse,pLevel,0,0,&zStartAff); + codeApplyAffinity(pParse, r1, nEq, zStartAff); + sqlite3DbFree(pParse->db, zStartAff); + sqlite3VdbeAddOp4Int(pParse->pVdbe, OP_Filter, pLevel->regFilter, + addrNxt, r1, nEq); + VdbeCoverage(pParse->pVdbe); + } + pLevel->regFilter = 0; + pLevel->addrBrk = 0; + } +} + +/* +** Loop pLoop is a WHERE_INDEXED level that uses at least one IN(...) +** operator. Return true if level pLoop is guaranteed to visit only one +** row for each key generated for the index. +*/ +static int whereLoopIsOneRow(WhereLoop *pLoop){ + if( pLoop->u.btree.pIndex->onError + && pLoop->nSkip==0 + && pLoop->u.btree.nEq==pLoop->u.btree.pIndex->nKeyCol + ){ + int ii; + for(ii=0; iiu.btree.nEq; ii++){ + if( pLoop->aLTerm[ii]->eOperator & (WO_IS|WO_ISNULL) ){ + return 0; + } + } + return 1; + } + return 0; +} + /* ** Generate code for the start of the iLevel-th loop in the WHERE clause ** implementation described by pWInfo. */ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( + Parse *pParse, /* Parsing context */ + Vdbe *v, /* Prepared statement under construction */ WhereInfo *pWInfo, /* Complete information about the WHERE clause */ int iLevel, /* Which level of pWInfo->a[] should be coded */ + WhereLevel *pLevel, /* The current level pointer */ Bitmask notReady /* Which tables are currently available */ ){ int j, k; /* Loop counters */ int iCur; /* The VDBE cursor for the table */ int addrNxt; /* Where to jump to continue with the next IN case */ - int omitTable; /* True if we use the index only */ int bRev; /* True if we need to scan in reverse order */ - WhereLevel *pLevel; /* The where level to be coded */ WhereLoop *pLoop; /* The WhereLoop object being coded */ WhereClause *pWC; /* Decomposition of the entire WHERE clause */ WhereTerm *pTerm; /* A WHERE clause term */ - Parse *pParse; /* Parsing context */ sqlite3 *db; /* Database connection */ - Vdbe *v; /* The prepared stmt under constructions */ - struct SrcList_item *pTabItem; /* FROM clause term being coded */ + SrcItem *pTabItem; /* FROM clause term being coded */ int addrBrk; /* Jump here to break out of the loop */ + int addrHalt; /* addrBrk for the outermost loop */ int addrCont; /* Jump here to continue with next cycle */ int iRowidReg = 0; /* Rowid is stored in this register, if not zero */ int iReleaseReg = 0; /* Temp register to free before returning */ + Index *pIdx = 0; /* Index used by loop (if any) */ + int iLoop; /* Iteration of constraint generator loop */ - pParse = pWInfo->pParse; - v = pParse->pVdbe; pWC = &pWInfo->sWC; db = pParse->db; - pLevel = &pWInfo->a[iLevel]; pLoop = pLevel->pWLoop; pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; iCur = pTabItem->iCursor; pLevel->notReady = notReady & ~sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); bRev = (pWInfo->revMask>>iLevel)&1; - omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 - && (pWInfo->wctrlFlags & WHERE_FORCE_TABLE)==0; - VdbeModuleComment((v, "Begin WHERE-loop%d: %s",iLevel,pTabItem->pTab->zName)); + VdbeModuleComment((v, "Begin WHERE-loop%d: %s", + iLevel, pTabItem->pSTab->zName)); +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x1 ){ + sqlite3DebugPrintf("Coding level %d of %d: notReady=%llx iFrom=%d\n", + iLevel, pWInfo->nLevel, (u64)notReady, pLevel->iFrom); + if( sqlite3WhereTrace & 0x1000 ){ + sqlite3WhereLoopPrint(pLoop, pWC); + } + } + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ + if( iLevel==0 ){ + sqlite3DebugPrintf("WHERE clause being coded:\n"); + sqlite3TreeViewExpr(0, pWInfo->pWhere, 0); + } + sqlite3DebugPrintf("All WHERE-clause terms before coding:\n"); + sqlite3WhereClausePrint(pWC); + } +#endif /* Create labels for the "break" and "continue" instructions ** for the current loop. Jump to addrBrk to break out of a loop. @@ -123661,26 +159872,41 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** there are no IN operators in the constraints, the "addrNxt" label ** is the same as "addrBrk". */ - addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(v); - addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(v); + addrBrk = pLevel->addrBrk = pLevel->addrNxt = sqlite3VdbeMakeLabel(pParse); + addrCont = pLevel->addrCont = sqlite3VdbeMakeLabel(pParse); /* If this is the right table of a LEFT OUTER JOIN, allocate and ** initialize a memory cell that records if this table matches any ** row of the left table of the join. */ + assert( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN)) + || pLevel->iFrom>0 || (pTabItem[0].fg.jointype & JT_LEFT)==0 + ); if( pLevel->iFrom>0 && (pTabItem[0].fg.jointype & JT_LEFT)!=0 ){ pLevel->iLeftJoin = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Integer, 0, pLevel->iLeftJoin); - VdbeComment((v, "init LEFT JOIN no-match flag")); + VdbeComment((v, "init LEFT JOIN match flag")); } + /* Compute a safe address to jump to if we discover that the table for + ** this loop is empty and can never contribute content. */ + for(j=iLevel; j>0; j--){ + if( pWInfo->a[j].iLeftJoin ) break; + if( pWInfo->a[j].pRJ ) break; + } + addrHalt = pWInfo->a[j].addrBrk; + /* Special case of a FROM clause subquery implemented as a co-routine */ if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + int regYield; + Subquery *pSubq; + assert( pTabItem->fg.isSubquery && pTabItem->u4.pSubq!=0 ); + pSubq = pTabItem->u4.pSubq; + regYield = pSubq->regReturn; + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); pLevel->p2 = sqlite3VdbeAddOp2(v, OP_Yield, regYield, addrBrk); VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pTabItem->pSTab->zName)); pLevel->op = OP_Goto; }else @@ -123692,9 +159918,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int iReg; /* P3 Value for OP_VFilter */ int addrNotFound; int nConstraint = pLoop->nLTerm; - int iIn; /* Counter for IN constraints */ - sqlite3ExprCachePush(pParse); iReg = sqlite3GetTempRange(pParse, nConstraint+2); addrNotFound = pLevel->addrBrk; for(j=0; jaLTerm[j]; if( NEVER(pTerm==0) ) continue; if( pTerm->eOperator & WO_IN ){ - codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); - addrNotFound = pLevel->addrNxt; + if( SMASKBIT32(j) & pLoop->u.vtab.mHandleIn ){ + int iTab = pParse->nTab++; + int iCache = ++pParse->nMem; + sqlite3CodeRhsOfIN(pParse, pTerm->pExpr, iTab); + sqlite3VdbeAddOp3(v, OP_VInitIn, iTab, iTarget, iCache); + }else{ + codeEqualityTerm(pParse, pTerm, pLevel, j, bRev, iTarget); + addrNotFound = pLevel->addrNxt; + } }else{ - sqlite3ExprCode(pParse, pTerm->pExpr->pRight, iTarget); + Expr *pRight = pTerm->pExpr->pRight; + codeExprOrVector(pParse, pRight, iTarget, 1); + if( pTerm->eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET + && pLoop->u.vtab.bOmitOffset + ){ + assert( pTerm->eOperator==WO_AUX ); + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->iOffset>0 ); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWInfo->pSelect->iOffset); + VdbeComment((v,"Zero OFFSET counter")); + } } } sqlite3VdbeAddOp2(v, OP_Integer, pLoop->u.vtab.idxNum, iReg); sqlite3VdbeAddOp2(v, OP_Integer, nConstraint, iReg+1); sqlite3VdbeAddOp4(v, OP_VFilter, iCur, addrNotFound, iReg, pLoop->u.vtab.idxStr, - pLoop->u.vtab.needFree ? P4_MPRINTF : P4_STATIC); + pLoop->u.vtab.needFree ? P4_DYNAMIC : P4_STATIC); VdbeCoverage(v); pLoop->u.vtab.needFree = 0; + /* An OOM inside of AddOp4(OP_VFilter) instruction above might have freed + ** the u.vtab.idxStr. NULL it out to prevent a use-after-free */ + if( db->mallocFailed ) pLoop->u.vtab.idxStr = 0; pLevel->p1 = iCur; pLevel->op = pWInfo->eOnePass ? OP_Noop : OP_VNext; pLevel->p2 = sqlite3VdbeCurrentAddr(v); - iIn = pLevel->u.in.nIn; - for(j=nConstraint-1; j>=0; j--){ + assert( (pLoop->wsFlags & WHERE_MULTI_OR)==0 ); + + for(j=0; jaLTerm[j]; if( j<16 && (pLoop->u.vtab.omitMask>>j)&1 ){ disableTerm(pLevel, pTerm); - }else if( (pTerm->eOperator & WO_IN)!=0 ){ + continue; + } + if( (pTerm->eOperator & WO_IN)!=0 + && (SMASKBIT32(j) & pLoop->u.vtab.mHandleIn)==0 + && !db->mallocFailed + ){ Expr *pCompare; /* The comparison operator */ Expr *pRight; /* RHS of the comparison */ VdbeOp *pOp; /* Opcode to access the value of the IN constraint */ + int iIn; /* IN loop corresponding to the j-th constraint */ /* Reload the constraint value into reg[iReg+j+2]. The same value ** was loaded into the same register prior to the OP_VFilter, but ** the xFilter implementation might have changed the datatype or - ** encoding of the value in the register, so it *must* be reloaded. */ - assert( pLevel->u.in.aInLoop!=0 || db->mallocFailed ); - if( !db->mallocFailed ){ - assert( iIn>0 ); - pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[--iIn].addrInTop); - assert( pOp->opcode==OP_Column || pOp->opcode==OP_Rowid ); - assert( pOp->opcode!=OP_Column || pOp->p3==iReg+j+2 ); - assert( pOp->opcode!=OP_Rowid || pOp->p2==iReg+j+2 ); - testcase( pOp->opcode==OP_Rowid ); - sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + ** encoding of the value in the register, so it *must* be reloaded. + */ + for(iIn=0; ALWAYS(iInu.in.nIn); iIn++){ + pOp = sqlite3VdbeGetOp(v, pLevel->u.in.aInLoop[iIn].addrInTop); + if( (pOp->opcode==OP_Column && pOp->p3==iReg+j+2) + || (pOp->opcode==OP_Rowid && pOp->p2==iReg+j+2) + ){ + testcase( pOp->opcode==OP_Rowid ); + sqlite3VdbeAddOp3(v, pOp->opcode, pOp->p1, pOp->p2, pOp->p3); + break; + } } - /* Generate code that will continue to the next row if - ** the IN constraint is not satisfied */ - pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0, 0); - assert( pCompare!=0 || db->mallocFailed ); - if( pCompare ){ - pCompare->pLeft = pTerm->pExpr->pLeft; + /* Generate code that will continue to the next row if + ** the IN constraint is not satisfied + */ + pCompare = sqlite3PExpr(pParse, TK_EQ, 0, 0); + if( !db->mallocFailed ){ + int iFld = pTerm->u.x.iField; + Expr *pLeft = pTerm->pExpr->pLeft; + assert( pLeft!=0 ); + if( iFld>0 ){ + assert( pLeft->op==TK_VECTOR ); + assert( ExprUseXList(pLeft) ); + assert( iFld<=pLeft->x.pList->nExpr ); + pCompare->pLeft = pLeft->x.pList->a[iFld-1].pExpr; + }else{ + pCompare->pLeft = pLeft; + } pCompare->pRight = pRight = sqlite3Expr(db, TK_REGISTER, 0); if( pRight ){ pRight->iTable = iReg+j+2; - sqlite3ExprIfFalse(pParse, pCompare, pLevel->addrCont, 0); + sqlite3ExprIfFalse( + pParse, pCompare, pLevel->addrCont, SQLITE_JUMPIFNULL + ); } pCompare->pLeft = 0; - sqlite3ExprDelete(db, pCompare); } + sqlite3ExprDelete(db, pCompare); } } + /* These registers need to be preserved in case there is an IN operator ** loop. So we could deallocate the registers here (and potentially ** reuse them later) if (pLoop->wsFlags & WHERE_IN_ABLE)==0. But it seems @@ -123766,7 +160031,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** ** sqlite3ReleaseTempRange(pParse, iReg, nConstraint+2); */ - sqlite3ExprCachePop(pParse); }else #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -123782,18 +160046,21 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->pExpr!=0 ); - assert( omitTable==0 ); testcase( pTerm->wtFlags & TERM_VIRTUAL ); iReleaseReg = ++pParse->nMem; iRowidReg = codeEqualityTerm(pParse, pTerm, pLevel, 0, bRev, iReleaseReg); if( iRowidReg!=iReleaseReg ) sqlite3ReleaseTempReg(pParse, iReleaseReg); addrNxt = pLevel->addrNxt; - sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); VdbeCoverage(v); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, addrNxt, iRowidReg); + if( pLevel->regFilter ){ + sqlite3VdbeAddOp2(v, OP_MustBeInt, iRowidReg, addrNxt); + VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, + iRowidReg, 1); + VdbeCoverage(v); + filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady); + } + sqlite3VdbeAddOp3(v, OP_SeekRowid, iCur, addrNxt, iRowidReg); VdbeCoverage(v); - sqlite3ExprCacheAffinityChange(pParse, iRowidReg, 1); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - VdbeComment((v, "pk")); pLevel->op = OP_Noop; }else if( (pLoop->wsFlags & WHERE_IPK)!=0 && (pLoop->wsFlags & WHERE_COLUMN_RANGE)!=0 @@ -123805,7 +160072,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int memEndValue = 0; WhereTerm *pStart, *pEnd; - assert( omitTable==0 ); j = 0; pStart = pEnd = 0; if( pLoop->wsFlags & WHERE_BTM_LIMIT ) pStart = pLoop->aLTerm[j++]; @@ -123816,12 +160082,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pStart = pEnd; pEnd = pTerm; } - codeCursorHint(pWInfo, pLevel, pEnd); + codeCursorHint(pTabItem, pWInfo, pLevel, pEnd); if( pStart ){ Expr *pX; /* The expression that defines the start bound */ int r1, rTemp; /* Registers for holding the start boundary */ + int op; /* Cursor seek operation */ - /* The following constant maps TK_xx codes into corresponding + /* The following constant maps TK_xx codes into corresponding ** seek opcodes. It depends on a particular ordering of TK_xx */ const u8 aMoveOp[] = { @@ -123832,25 +160099,39 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( }; assert( TK_LE==TK_GT+1 ); /* Make sure the ordering.. */ assert( TK_LT==TK_GT+2 ); /* ... of the TK_xx values... */ - assert( TK_GE==TK_GT+3 ); /* ... is correcct. */ + assert( TK_GE==TK_GT+3 ); /* ... is correct. */ assert( (pStart->wtFlags & TERM_VNULL)==0 ); testcase( pStart->wtFlags & TERM_VIRTUAL ); pX = pStart->pExpr; assert( pX!=0 ); testcase( pStart->leftCursor!=iCur ); /* transitive constraints */ - r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); - sqlite3VdbeAddOp3(v, aMoveOp[pX->op-TK_GT], iCur, addrBrk, r1); + if( sqlite3ExprIsVector(pX->pRight) ){ + r1 = rTemp = sqlite3GetTempReg(pParse); + codeExprOrVector(pParse, pX->pRight, r1, 1); + testcase( pX->op==TK_GT ); + testcase( pX->op==TK_GE ); + testcase( pX->op==TK_LT ); + testcase( pX->op==TK_LE ); + op = aMoveOp[((pX->op - TK_GT - 1) & 0x3) | 0x1]; + assert( pX->op!=TK_GT || op==OP_SeekGE ); + assert( pX->op!=TK_GE || op==OP_SeekGE ); + assert( pX->op!=TK_LT || op==OP_SeekLE ); + assert( pX->op!=TK_LE || op==OP_SeekLE ); + }else{ + r1 = sqlite3ExprCodeTemp(pParse, pX->pRight, &rTemp); + disableTerm(pLevel, pStart); + op = aMoveOp[(pX->op - TK_GT)]; + } + sqlite3VdbeAddOp3(v, op, iCur, addrBrk, r1); VdbeComment((v, "pk")); VdbeCoverageIf(v, pX->op==TK_GT); VdbeCoverageIf(v, pX->op==TK_LE); VdbeCoverageIf(v, pX->op==TK_LT); VdbeCoverageIf(v, pX->op==TK_GE); - sqlite3ExprCacheAffinityChange(pParse, r1, 1); sqlite3ReleaseTempReg(pParse, rTemp); - disableTerm(pLevel, pStart); }else{ - sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrBrk); + sqlite3VdbeAddOp2(v, bRev ? OP_Last : OP_Rewind, iCur, addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); } @@ -123862,13 +160143,17 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( testcase( pEnd->leftCursor!=iCur ); /* Transitive constraints */ testcase( pEnd->wtFlags & TERM_VIRTUAL ); memEndValue = ++pParse->nMem; - sqlite3ExprCode(pParse, pX->pRight, memEndValue); - if( pX->op==TK_LT || pX->op==TK_GT ){ + codeExprOrVector(pParse, pX->pRight, memEndValue, 1); + if( 0==sqlite3ExprIsVector(pX->pRight) + && (pX->op==TK_LT || pX->op==TK_GT) + ){ testOp = bRev ? OP_Le : OP_Ge; }else{ testOp = bRev ? OP_Lt : OP_Gt; } - disableTerm(pLevel, pEnd); + if( 0==sqlite3ExprIsVector(pX->pRight) ){ + disableTerm(pLevel, pEnd); + } } start = sqlite3VdbeCurrentAddr(v); pLevel->op = bRev ? OP_Prev : OP_Next; @@ -123878,7 +160163,6 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( if( testOp!=OP_Noop ){ iRowidReg = ++pParse->nMem; sqlite3VdbeAddOp2(v, OP_Rowid, iCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); sqlite3VdbeAddOp3(v, testOp, memEndValue, addrBrk, iRowidReg); VdbeCoverageIf(v, testOp==OP_Le); VdbeCoverageIf(v, testOp==OP_Lt); @@ -123889,14 +160173,14 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( }else if( pLoop->wsFlags & WHERE_INDEXED ){ /* Case 4: A scan using an index. ** - ** The WHERE clause may contain zero or more equality + ** The WHERE clause may contain zero or more equality ** terms ("==" or "IN" operators) that refer to the N ** left-most columns of the index. It may also contain ** inequality constraints (>, <, >= or <=) on the indexed - ** column that immediately follows the N equalities. Only + ** column that immediately follows the N equalities. Only ** the right-most column can be an inequality - the rest must - ** use the "==" and "IN" operators. For example, if the - ** index is on (x,y,z), then the following clauses are all + ** use the "==" and "IN" operators. For example, if the + ** index is on (x,y,z), then the following clauses are all ** optimized: ** ** x=5 @@ -123917,7 +160201,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** This case is also used when there are no WHERE clause ** constraints but an index is selected anyway, in order ** to force the output order to conform to an ORDER BY. - */ + */ static const u8 aStartOp[] = { 0, 0, @@ -123935,6 +160219,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( OP_IdxLT, /* 3: (end_constraints && bRev && endEq) */ }; u16 nEq = pLoop->u.btree.nEq; /* Number of == or IN terms */ + u16 nBtm = pLoop->u.btree.nBtm; /* Length of BTM vector */ + u16 nTop = pLoop->u.btree.nTop; /* Length of TOP vector */ int regBase; /* Base register holding constraint values */ WhereTerm *pRangeStart = 0; /* Inequality constraint at range start */ WhereTerm *pRangeEnd = 0; /* Inequality constraint at range end */ @@ -123942,53 +160228,35 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int endEq; /* True if range end uses ==, >= or <= */ int start_constraints; /* Start of range is constrained */ int nConstraint; /* Number of constraint terms */ - Index *pIdx; /* The index we will be using */ int iIdxCur; /* The VDBE cursor for the index */ int nExtraReg = 0; /* Number of extra registers needed */ int op; /* Instruction opcode */ char *zStartAff; /* Affinity for start of range constraint */ - char cEndAff = 0; /* Affinity for end of range constraint */ + char *zEndAff = 0; /* Affinity for end of range constraint */ u8 bSeekPastNull = 0; /* True to seek past initial nulls */ u8 bStopAtNull = 0; /* Add condition to terminate at NULLs */ + int omitTable; /* True if we use the index only */ + int regBignull = 0; /* big-null flag register */ + int addrSeekScan = 0; /* Opcode of the OP_SeekScan, if any */ pIdx = pLoop->u.btree.pIndex; iIdxCur = pLevel->iIdxCur; assert( nEq>=pLoop->nSkip ); - /* If this loop satisfies a sort order (pOrderBy) request that - ** was passed to this function to implement a "SELECT min(x) ..." - ** query, then the caller will only allow the loop to run for - ** a single iteration. This means that the first row returned - ** should not have a NULL value stored in 'x'. If column 'x' is - ** the first one after the nEq equality constraints in the index, - ** this requires some special handling. - */ - assert( pWInfo->pOrderBy==0 - || pWInfo->pOrderBy->nExpr==1 - || (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 ); - if( (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)!=0 - && pWInfo->nOBSat>0 - && (pIdx->nKeyCol>nEq) - ){ - assert( pLoop->nSkip==0 ); - bSeekPastNull = 1; - nExtraReg = 1; - } - - /* Find any inequality constraint terms for the start and end - ** of the range. + /* Find any inequality constraint terms for the start and end + ** of the range. */ j = nEq; if( pLoop->wsFlags & WHERE_BTM_LIMIT ){ pRangeStart = pLoop->aLTerm[j++]; - nExtraReg = 1; + nExtraReg = MAX(nExtraReg, pLoop->u.btree.nBtm); /* Like optimization range constraints always occur in pairs */ - assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || + assert( (pRangeStart->wtFlags & TERM_LIKEOPT)==0 || (pLoop->wsFlags & WHERE_TOP_LIMIT)!=0 ); } if( pLoop->wsFlags & WHERE_TOP_LIMIT ){ pRangeEnd = pLoop->aLTerm[j++]; - nExtraReg = 1; + nExtraReg = MAX(nExtraReg, pLoop->u.btree.nTop); #ifndef SQLITE_LIKE_DOESNT_MATCH_BLOBS if( (pRangeEnd->wtFlags & TERM_LIKEOPT)!=0 ){ assert( pRangeStart!=0 ); /* LIKE opt constraints */ @@ -124006,35 +160274,64 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pLevel->iLikeRepCntr |= bRev ^ (pIdx->aSortOrder[nEq]==SQLITE_SO_DESC); } #endif - if( pRangeStart==0 - && (j = pIdx->aiColumn[nEq])>=0 - && pIdx->pTable->aCol[j].notNull==0 - ){ - bSeekPastNull = 1; + if( pRangeStart==0 ){ + j = pIdx->aiColumn[nEq]; + if( (j>=0 && pIdx->pTable->aCol[j].notNull==0) || j==XN_EXPR ){ + bSeekPastNull = 1; + } } } assert( pRangeEnd==0 || (pRangeEnd->wtFlags & TERM_VNULL)==0 ); + /* If the WHERE_BIGNULL_SORT flag is set, then index column nEq uses + ** a non-default "big-null" sort (either ASC NULLS LAST or DESC NULLS + ** FIRST). In both cases separate ordered scans are made of those + ** index entries for which the column is null and for those for which + ** it is not. For an ASC sort, the non-NULL entries are scanned first. + ** For DESC, NULL entries are scanned first. + */ + if( (pLoop->wsFlags & (WHERE_TOP_LIMIT|WHERE_BTM_LIMIT))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)!=0 + ){ + assert( bSeekPastNull==0 && nExtraReg==0 && nBtm==0 && nTop==0 ); + assert( pRangeEnd==0 && pRangeStart==0 ); + testcase( pLoop->nSkip>0 ); + nExtraReg = 1; + bSeekPastNull = 1; + pLevel->regBignull = regBignull = ++pParse->nMem; + if( pLevel->iLeftJoin ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, regBignull); + } + pLevel->addrBignull = sqlite3VdbeMakeLabel(pParse); + } + /* If we are doing a reverse order scan on an ascending index, or - ** a forward order scan on a descending index, interchange the + ** a forward order scan on a descending index, interchange the ** start and end terms (pRangeStart and pRangeEnd). */ - if( (nEqnKeyCol && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) - || (bRev && pIdx->nKeyCol==nEq) - ){ + if( (nEqnColumn && bRev==(pIdx->aSortOrder[nEq]==SQLITE_SO_ASC)) ){ SWAP(WhereTerm *, pRangeEnd, pRangeStart); SWAP(u8, bSeekPastNull, bStopAtNull); + SWAP(u8, nBtm, nTop); + } + + if( iLevel>0 && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 ){ + /* In case OP_SeekScan is used, ensure that the index cursor does not + ** point to a valid row for the first iteration of this loop. */ + sqlite3VdbeAddOp1(v, OP_NullRow, iIdxCur); } /* Generate code to evaluate all constraint terms using == or IN ** and store the values of those terms in an array of registers ** starting at regBase. */ - codeCursorHint(pWInfo, pLevel, pRangeEnd); + codeCursorHint(pTabItem, pWInfo, pLevel, pRangeEnd); regBase = codeAllEqualityTerms(pParse,pLevel,bRev,nExtraReg,&zStartAff); assert( zStartAff==0 || sqlite3Strlen30(zStartAff)>=nEq ); - if( zStartAff ) cEndAff = zStartAff[nEq]; - addrNxt = pLevel->addrNxt; + if( zStartAff && nTop ){ + zEndAff = sqlite3DbStrDup(db, &zStartAff[nEq]); + } + addrNxt = (regBignull ? pLevel->addrBignull : pLevel->addrNxt); testcase( pRangeStart && (pRangeStart->eOperator & WO_LE)!=0 ); testcase( pRangeStart && (pRangeStart->eOperator & WO_GE)!=0 ); @@ -124048,7 +160345,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( nConstraint = nEq; if( pRangeStart ){ Expr *pRight = pRangeStart->pExpr->pRight; - sqlite3ExprCode(pParse, pRight, regBase+nEq); + codeExprOrVector(pParse, pRight, regBase+nEq, nBtm); whereLikeOptimizationStringFixup(v, pLevel, pRangeStart); if( (pRangeStart->wtFlags & TERM_VNULL)==0 && sqlite3ExprCanBeNull(pRight) @@ -124057,23 +160354,25 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverage(v); } if( zStartAff ){ - if( sqlite3CompareAffinity(pRight, zStartAff[nEq])==SQLITE_AFF_BLOB){ - /* Since the comparison is to be performed with no conversions - ** applied to the operands, set the affinity to apply to pRight to - ** SQLITE_AFF_BLOB. */ - zStartAff[nEq] = SQLITE_AFF_BLOB; - } - if( sqlite3ExprNeedsNoAffinityChange(pRight, zStartAff[nEq]) ){ - zStartAff[nEq] = SQLITE_AFF_BLOB; - } - } - nConstraint++; + updateRangeAffinityStr(pRight, nBtm, &zStartAff[nEq]); + } + nConstraint += nBtm; testcase( pRangeStart->wtFlags & TERM_VIRTUAL ); + if( sqlite3ExprIsVector(pRight)==0 ){ + disableTerm(pLevel, pRangeStart); + }else{ + startEq = 1; + } + bSeekPastNull = 0; }else if( bSeekPastNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - nConstraint++; startEq = 0; + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); start_constraints = 1; + nConstraint++; + }else if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + start_constraints = 1; + nConstraint++; } codeApplyAffinity(pParse, regBase, nConstraint - bSeekPastNull, zStartAff); if( pLoop->nSkip>0 && nConstraint==pLoop->nSkip ){ @@ -124081,8 +160380,38 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** above has already left the cursor sitting on the correct row, ** so no further seeking is needed */ }else{ + if( regBignull ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, regBignull); + VdbeComment((v, "NULL-scan pass ctr")); + } + if( pLevel->regFilter ){ + sqlite3VdbeAddOp4Int(v, OP_Filter, pLevel->regFilter, addrNxt, + regBase, nEq); + VdbeCoverage(v); + filterPullDown(pParse, pWInfo, iLevel, addrNxt, notReady); + } + op = aStartOp[(start_constraints<<2) + (startEq<<1) + bRev]; assert( op!=0 ); + if( (pLoop->wsFlags & WHERE_IN_SEEKSCAN)!=0 && op==OP_SeekGE ){ + assert( regBignull==0 ); + /* TUNING: The OP_SeekScan opcode seeks to reduce the number + ** of expensive seek operations by replacing a single seek with + ** 1 or more step operations. The question is, how many steps + ** should we try before giving up and going with a seek. The cost + ** of a seek is proportional to the logarithm of the of the number + ** of entries in the tree, so basing the number of steps to try + ** on the estimated number of rows in the btree seems like a good + ** guess. */ + addrSeekScan = sqlite3VdbeAddOp1(v, OP_SeekScan, + (pIdx->aiRowLogEst[0]+9)/10); + if( pRangeStart || pRangeEnd ){ + sqlite3VdbeChangeP5(v, 1); + sqlite3VdbeChangeP2(v, addrSeekScan, sqlite3VdbeCurrentAddr(v)+1); + addrSeekScan = 0; + } + VdbeCoverage(v); + } sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); VdbeCoverage(v); VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); @@ -124091,16 +160420,34 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); VdbeCoverageIf(v, op==OP_SeekLT); testcase( op==OP_SeekLT ); + + assert( bSeekPastNull==0 || bStopAtNull==0 ); + if( regBignull ){ + assert( bSeekPastNull==1 || bStopAtNull==1 ); + assert( bSeekPastNull==!bStopAtNull ); + assert( bStopAtNull==startEq ); + sqlite3VdbeAddOp2(v, OP_Goto, 0, sqlite3VdbeCurrentAddr(v)+2); + op = aStartOp[(nConstraint>1)*4 + 2 + bRev]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint-startEq); + VdbeCoverage(v); + VdbeCoverageIf(v, op==OP_Rewind); testcase( op==OP_Rewind ); + VdbeCoverageIf(v, op==OP_Last); testcase( op==OP_Last ); + VdbeCoverageIf(v, op==OP_SeekGE); testcase( op==OP_SeekGE ); + VdbeCoverageIf(v, op==OP_SeekLE); testcase( op==OP_SeekLE ); + assert( op==OP_Rewind || op==OP_Last || op==OP_SeekGE || op==OP_SeekLE); + } } /* Load the value for the inequality constraint at the end of the ** range (if any). */ nConstraint = nEq; + assert( pLevel->p2==0 ); if( pRangeEnd ){ Expr *pRight = pRangeEnd->pExpr->pRight; - sqlite3ExprCacheRemove(pParse, regBase+nEq, 1); - sqlite3ExprCode(pParse, pRight, regBase+nEq); + assert( addrSeekScan==0 ); + codeExprOrVector(pParse, pRight, regBase+nEq, nTop); whereLikeOptimizationStringFixup(v, pLevel, pRangeEnd); if( (pRangeEnd->wtFlags & TERM_VNULL)==0 && sqlite3ExprCanBeNull(pRight) @@ -124108,63 +160455,112 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeAddOp2(v, OP_IsNull, regBase+nEq, addrNxt); VdbeCoverage(v); } - if( sqlite3CompareAffinity(pRight, cEndAff)!=SQLITE_AFF_BLOB - && !sqlite3ExprNeedsNoAffinityChange(pRight, cEndAff) - ){ - codeApplyAffinity(pParse, regBase+nEq, 1, &cEndAff); + if( zEndAff ){ + updateRangeAffinityStr(pRight, nTop, zEndAff); + codeApplyAffinity(pParse, regBase+nEq, nTop, zEndAff); + }else{ + assert( pParse->db->mallocFailed ); + } + nConstraint += nTop; + testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); + + if( sqlite3ExprIsVector(pRight)==0 ){ + disableTerm(pLevel, pRangeEnd); + }else{ + endEq = 1; + } + }else if( bStopAtNull ){ + if( regBignull==0 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); + endEq = 0; } nConstraint++; - testcase( pRangeEnd->wtFlags & TERM_VIRTUAL ); - }else if( bStopAtNull ){ - sqlite3VdbeAddOp2(v, OP_Null, 0, regBase+nEq); - endEq = 0; - nConstraint++; } - sqlite3DbFree(db, zStartAff); + if( zStartAff ) sqlite3DbNNFreeNN(db, zStartAff); + if( zEndAff ) sqlite3DbNNFreeNN(db, zEndAff); /* Top of the loop body */ pLevel->p2 = sqlite3VdbeCurrentAddr(v); /* Check if the index cursor is past the end of the range. */ if( nConstraint ){ + if( regBignull ){ + /* Except, skip the end-of-range check while doing the NULL-scan */ + sqlite3VdbeAddOp2(v, OP_IfNot, regBignull, sqlite3VdbeCurrentAddr(v)+3); + VdbeComment((v, "If NULL-scan 2nd pass")); + VdbeCoverage(v); + } op = aEndOp[bRev*2 + endEq]; sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, nConstraint); testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + if( addrSeekScan ) sqlite3VdbeJumpHere(v, addrSeekScan); + } + if( regBignull ){ + /* During a NULL-scan, check to see if we have reached the end of + ** the NULLs */ + assert( bSeekPastNull==!bStopAtNull ); + assert( bSeekPastNull+bStopAtNull==1 ); + assert( nConstraint+bSeekPastNull>0 ); + sqlite3VdbeAddOp2(v, OP_If, regBignull, sqlite3VdbeCurrentAddr(v)+2); + VdbeComment((v, "If NULL-scan 1st pass")); + VdbeCoverage(v); + op = aEndOp[bRev*2 + bSeekPastNull]; + sqlite3VdbeAddOp4Int(v, op, iIdxCur, addrNxt, regBase, + nConstraint+bSeekPastNull); + testcase( op==OP_IdxGT ); VdbeCoverageIf(v, op==OP_IdxGT ); + testcase( op==OP_IdxGE ); VdbeCoverageIf(v, op==OP_IdxGE ); + testcase( op==OP_IdxLT ); VdbeCoverageIf(v, op==OP_IdxLT ); + testcase( op==OP_IdxLE ); VdbeCoverageIf(v, op==OP_IdxLE ); + } + + if( (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0 ){ + sqlite3VdbeAddOp3(v, OP_SeekHit, iIdxCur, nEq, nEq); } /* Seek the table cursor, if required */ - disableTerm(pLevel, pRangeStart); - disableTerm(pLevel, pRangeEnd); + omitTable = (pLoop->wsFlags & WHERE_IDX_ONLY)!=0 + && (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0; if( omitTable ){ /* pIdx is a covering index. No need to access the main table. */ }else if( HasRowid(pIdx->pTable) ){ - if( (pWInfo->wctrlFlags & WHERE_SEEK_TABLE)!=0 ){ - iRowidReg = ++pParse->nMem; - sqlite3VdbeAddOp2(v, OP_IdxRowid, iIdxCur, iRowidReg); - sqlite3ExprCacheStore(pParse, iCur, -1, iRowidReg); - sqlite3VdbeAddOp3(v, OP_NotExists, iCur, 0, iRowidReg); - VdbeCoverage(v); - }else{ - codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur); - } + codeDeferredSeek(pWInfo, pIdx, iCur, iIdxCur); }else if( iCur!=iIdxCur ){ Index *pPk = sqlite3PrimaryKeyIndex(pIdx->pTable); iRowidReg = sqlite3GetTempRange(pParse, pPk->nKeyCol); for(j=0; jnKeyCol; j++){ - k = sqlite3ColumnOfIndex(pIdx, pPk->aiColumn[j]); + k = sqlite3TableColumnToIndex(pIdx, pPk->aiColumn[j]); sqlite3VdbeAddOp3(v, OP_Column, iIdxCur, k, iRowidReg+j); } sqlite3VdbeAddOp4Int(v, OP_NotFound, iCur, addrCont, iRowidReg, pPk->nKeyCol); VdbeCoverage(v); } - /* Record the instruction used to terminate the loop. Disable - ** WHERE clause terms made redundant by the index range scan. - */ - if( pLoop->wsFlags & WHERE_ONEROW ){ + if( pLevel->iLeftJoin==0 ){ + /* If a partial index is driving the loop, try to eliminate WHERE clause + ** terms from the query that must be true due to the WHERE clause of + ** the partial index. + ** + ** 2019-11-02 ticket 623eff57e76d45f6: This optimization does not work + ** for a LEFT JOIN. + */ + if( pIdx->pPartIdxWhere ){ + whereApplyPartialIndexConstraints(pIdx->pPartIdxWhere, iCur, pWC); + } + }else{ + testcase( pIdx->pPartIdxWhere ); + /* The following assert() is not a requirement, merely an observation: + ** The OR-optimization doesn't work for the right hand table of + ** a LEFT JOIN: */ + assert( (pWInfo->wctrlFlags & (WHERE_OR_SUBCLAUSE|WHERE_RIGHT_JOIN))==0 ); + } + + /* Record the instruction used to terminate the loop. */ + if( (pLoop->wsFlags & WHERE_ONEROW) + || (pLevel->u.in.nIn && regBignull==0 && whereLoopIsOneRow(pLoop)) + ){ pLevel->op = OP_Noop; }else if( bRev ){ pLevel->op = OP_Prev; @@ -124178,6 +160574,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( }else{ assert( pLevel->p5==0 ); } + if( omitTable ) pIdx = 0; }else #ifndef SQLITE_OMIT_OR_OPTIMIZATION @@ -124233,14 +160630,13 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( int regReturn = ++pParse->nMem; /* Register used with OP_Gosub */ int regRowset = 0; /* Register for RowSet object */ int regRowid = 0; /* Register holding rowid */ - int iLoopBody = sqlite3VdbeMakeLabel(v); /* Start of loop body */ + int iLoopBody = sqlite3VdbeMakeLabel(pParse);/* Start of loop body */ int iRetInit; /* Address of regReturn init */ int untestedTerms = 0; /* Some terms not completely tested */ int ii; /* Loop counter */ - u16 wctrlFlags; /* Flags for sub-WHERE clause */ Expr *pAndExpr = 0; /* An ".. AND (...)" expression */ - Table *pTab = pTabItem->pTab; - + Table *pTab = pTabItem->pSTab; + pTerm = pLoop->aLTerm[0]; assert( pTerm!=0 ); assert( pTerm->eOperator & WO_OR ); @@ -124255,9 +160651,9 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( */ if( pWInfo->nLevel>1 ){ int nNotReady; /* The number of notReady tables */ - struct SrcList_item *origSrc; /* Original list of tables */ + SrcItem *origSrc; /* Original list of tables */ nNotReady = pWInfo->nLevel - iLevel - 1; - pOrTab = sqlite3StackAllocRaw(db, + pOrTab = sqlite3DbMallocRawNN(db, sizeof(*pOrTab)+ nNotReady*sizeof(pOrTab->a[0])); if( pOrTab==0 ) return notReady; pOrTab->nAlloc = (u8)(nNotReady + 1); @@ -124271,15 +160667,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( pOrTab = pWInfo->pTabList; } - /* Initialize the rowset register to contain NULL. An SQL NULL is + /* Initialize the rowset register to contain NULL. An SQL NULL is ** equivalent to an empty rowset. Or, create an ephemeral index ** capable of holding primary keys in the case of a WITHOUT ROWID. ** - ** Also initialize regReturn to contain the address of the instruction + ** Also initialize regReturn to contain the address of the instruction ** immediately following the OP_Return at the bottom of the loop. This ** is required in a few obscure LEFT JOIN cases where control jumps - ** over the top of the loop into the body of it. In this case the - ** correct response for the end-of-loop code (the OP_Return) is to + ** over the top of the loop into the body of it. In this case the + ** correct response for the end-of-loop code (the OP_Return) is to ** fall through to the next instruction, just as an OP_Next does if ** called on an uninitialized cursor. */ @@ -124298,35 +160694,56 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( iRetInit = sqlite3VdbeAddOp2(v, OP_Integer, 0, regReturn); /* If the original WHERE clause is z of the form: (x1 OR x2 OR ...) AND y - ** Then for every term xN, evaluate as the subexpression: xN AND z + ** Then for every term xN, evaluate as the subexpression: xN AND y ** That way, terms in y that are factored into the disjunction will ** be picked up by the recursive calls to sqlite3WhereBegin() below. ** ** Actually, each subexpression is converted to "xN AND w" where w is ** the "interesting" terms of z - terms that did not originate in the - ** ON or USING clause of a LEFT JOIN, and terms that are usable as + ** ON or USING clause of a LEFT JOIN, and terms that are usable as ** indices. ** ** This optimization also only applies if the (x1 OR x2 OR ...) term ** is not contained in the ON clause of a LEFT JOIN. ** See ticket http://www.sqlite.org/src/info/f2369304e4 + ** + ** 2022-02-04: Do not push down slices of a row-value comparison. + ** In other words, "w" or "y" may not be a slice of a vector. Otherwise, + ** the initialization of the right-hand operand of the vector comparison + ** might not occur, or might occur only in an OR branch that is not + ** taken. dbsqlfuzz 80a9fade844b4fb43564efc972bcb2c68270f5d1. + ** + ** 2022-03-03: Do not push down expressions that involve subqueries. + ** The subquery might get coded as a subroutine. Any table-references + ** in the subquery might be resolved to index-references for the index on + ** the OR branch in which the subroutine is coded. But if the subroutine + ** is invoked from a different OR branch that uses a different index, such + ** index-references will not work. tag-20220303a + ** https://sqlite.org/forum/forumpost/36937b197273d403 */ if( pWC->nTerm>1 ){ int iTerm; for(iTerm=0; iTermnTerm; iTerm++){ Expr *pExpr = pWC->a[iTerm].pExpr; if( &pWC->a[iTerm] == pTerm ) continue; - if( ExprHasProperty(pExpr, EP_FromJoin) ) continue; testcase( pWC->a[iTerm].wtFlags & TERM_VIRTUAL ); testcase( pWC->a[iTerm].wtFlags & TERM_CODED ); - if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED))!=0 ) continue; + testcase( pWC->a[iTerm].wtFlags & TERM_SLICE ); + if( (pWC->a[iTerm].wtFlags & (TERM_VIRTUAL|TERM_CODED|TERM_SLICE))!=0 ){ + continue; + } if( (pWC->a[iTerm].eOperator & WO_ALL)==0 ) continue; - testcase( pWC->a[iTerm].wtFlags & TERM_ORINFO ); + if( ExprHasProperty(pExpr, EP_Subquery) ) continue; /* tag-20220303a */ pExpr = sqlite3ExprDup(db, pExpr, 0); - pAndExpr = sqlite3ExprAnd(db, pAndExpr, pExpr); + pAndExpr = sqlite3ExprAnd(pParse, pAndExpr, pExpr); } if( pAndExpr ){ - pAndExpr = sqlite3PExpr(pParse, TK_AND|TKFLG_DONTFOLD, 0, pAndExpr, 0); + /* The extra 0x10000 bit on the opcode is masked off and does not + ** become part of the new Expr.op. However, it does make the + ** op==TK_AND comparison inside of sqlite3PExpr() false, and this + ** prevents sqlite3PExpr() from applying the AND short-circuit + ** optimization, which we do not want here. */ + pAndExpr = sqlite3PExpr(pParse, TK_AND|0x10000, 0, pAndExpr); } } @@ -124334,30 +160751,36 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** eliminating duplicates from other WHERE clauses, the action for each ** sub-WHERE clause is to to invoke the main loop body as a subroutine. */ - wctrlFlags = WHERE_OMIT_OPEN_CLOSE - | WHERE_FORCE_TABLE - | WHERE_ONETABLE_ONLY - | WHERE_NO_AUTOINDEX - | (pWInfo->wctrlFlags & WHERE_SEEK_TABLE); + ExplainQueryPlan((pParse, 1, "MULTI-INDEX OR")); for(ii=0; iinTerm; ii++){ WhereTerm *pOrTerm = &pOrWc->a[ii]; if( pOrTerm->leftCursor==iCur || (pOrTerm->eOperator & WO_AND)!=0 ){ WhereInfo *pSubWInfo; /* Info for single OR-term scan */ Expr *pOrExpr = pOrTerm->pExpr; /* Current OR clause term */ + Expr *pDelete; /* Local copy of OR clause term */ int jmp1 = 0; /* Address of jump operation */ - if( pAndExpr && !ExprHasProperty(pOrExpr, EP_FromJoin) ){ + testcase( (pTabItem[0].fg.jointype & JT_LEFT)!=0 + && !ExprHasProperty(pOrExpr, EP_OuterON) + ); /* See TH3 vtab25.400 and ticket 614b25314c766238 */ + pDelete = pOrExpr = sqlite3ExprDup(db, pOrExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDelete); + continue; + } + if( pAndExpr ){ pAndExpr->pLeft = pOrExpr; pOrExpr = pAndExpr; } /* Loop through table entries that match term pOrTerm. */ - WHERETRACE(0xffff, ("Subplan for OR-clause:\n")); - pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, - wctrlFlags, iCovCur); - assert( pSubWInfo || pParse->nErr || db->mallocFailed ); + ExplainQueryPlan((pParse, 1, "INDEX %d", ii+1)); + WHERETRACE(0xffffffff, ("Subplan for OR-clause:\n")); + pSubWInfo = sqlite3WhereBegin(pParse, pOrTab, pOrExpr, 0, 0, 0, + WHERE_OR_SUBCLAUSE, iCovCur); + assert( pSubWInfo || pParse->nErr ); if( pSubWInfo ){ WhereLoop *pSubLoop; int addrExplain = sqlite3WhereExplainOneScan( - pParse, pOrTab, &pSubWInfo->a[0], iLevel, pLevel->iFrom, 0 + pParse, pOrTab, &pSubWInfo->a[0], 0 ); sqlite3WhereAddScanStatus(v, pOrTab, &pSubWInfo->a[0], addrExplain); @@ -124367,23 +160790,23 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** row will be skipped in subsequent sub-WHERE clauses. */ if( (pWInfo->wctrlFlags & WHERE_DUPLICATES_OK)==0 ){ - int r; int iSet = ((ii==pOrWc->nTerm-1)?-1:ii); if( HasRowid(pTab) ){ - r = sqlite3ExprCodeGetColumn(pParse, pTab, -1, iCur, regRowid, 0); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, regRowid); jmp1 = sqlite3VdbeAddOp4Int(v, OP_RowSetTest, regRowset, 0, - r,iSet); + regRowid, iSet); VdbeCoverage(v); }else{ Index *pPk = sqlite3PrimaryKeyIndex(pTab); int nPk = pPk->nKeyCol; int iPk; + int r; /* Read the PK into an array of temp registers. */ r = sqlite3GetTempRange(pParse, nPk); for(iPk=0; iPkaiColumn[iPk]; - sqlite3ExprCodeGetColumnToReg(pParse, pTab, iCol, iCur, r+iPk); + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); } /* Check if the temp table already contains this key. If so, @@ -124394,16 +160817,17 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** ** Use some of the same optimizations as OP_RowSetTest: If iSet ** is zero, assume that the key cannot already be present in - ** the temp table. And if iSet is -1, assume that there is no - ** need to insert the key into the temp table, as it will never - ** be tested for. */ + ** the temp table. And if iSet is -1, assume that there is no + ** need to insert the key into the temp table, as it will never + ** be tested for. */ if( iSet ){ jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, regRowset, 0, r, nPk); VdbeCoverage(v); } if( iSet>=0 ){ sqlite3VdbeAddOp3(v, OP_MakeRecord, r, nPk, regRowid); - sqlite3VdbeAddOp3(v, OP_IdxInsert, regRowset, regRowid, 0); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, regRowset, regRowid, + r, nPk); if( iSet ) sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); } @@ -124434,8 +160858,8 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** If the call to sqlite3WhereBegin() above resulted in a scan that ** uses an index, and this is either the first OR-connected term ** processed or the index is the same as that used by all previous - ** terms, set pCov to the candidate covering index. Otherwise, set - ** pCov to NULL to indicate that no candidate covering index will + ** terms, set pCov to the candidate covering index. Otherwise, set + ** pCov to NULL to indicate that no candidate covering index will ** be available. */ pSubLoop = pSubWInfo->a[0].pWLoop; @@ -124446,17 +160870,25 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ){ assert( pSubWInfo->a[0].iIdxCur==iCovCur ); pCov = pSubLoop->u.btree.pIndex; - wctrlFlags |= WHERE_REOPEN_IDX; }else{ pCov = 0; } + if( sqlite3WhereUsesDeferredSeek(pSubWInfo) ){ + pWInfo->bDeferredSeek = 1; + } /* Finish the loop through table entries that match term pOrTerm. */ sqlite3WhereEnd(pSubWInfo); + ExplainQueryPlanPop(pParse); } + sqlite3ExprDelete(db, pDelete); } } - pLevel->u.pCovidx = pCov; + ExplainQueryPlanPop(pParse); + assert( pLevel->pWLoop==pLoop ); + assert( (pLoop->wsFlags & WHERE_MULTI_OR)!=0 ); + assert( (pLoop->wsFlags & WHERE_IN_ABLE)==0 ); + pLevel->u.pCoveringIdx = pCov; if( pCov ) pLevel->iIdxCur = iCovCur; if( pAndExpr ){ pAndExpr->pLeft = 0; @@ -124466,7 +160898,15 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( sqlite3VdbeGoto(v, pLevel->addrBrk); sqlite3VdbeResolveLabel(v, iLoopBody); - if( pWInfo->nLevel>1 ) sqlite3StackFree(db, pOrTab); + /* Set the P2 operand of the OP_Return opcode that will end the current + ** loop to point to this spot, which is the top of the next containing + ** loop. The byte-code formatter will use that P2 value as a hint to + ** indent everything in between the this point and the final OP_Return. + ** See tag-20220407a in vdbe.c and shell.c */ + assert( pLevel->op==OP_Return ); + pLevel->p2 = sqlite3VdbeCurrentAddr(v); + + if( pWInfo->nLevel>1 ){ sqlite3DbFreeNN(db, pOrTab); } if( !untestedTerms ) disableTerm(pLevel, pTerm); }else #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ @@ -124483,10 +160923,10 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** a pseudo-cursor. No need to Rewind or Next such cursors. */ pLevel->op = OP_Noop; }else{ - codeCursorHint(pWInfo, pLevel, 0); + codeCursorHint(pTabItem, pWInfo, pLevel, 0); pLevel->op = aStep[bRev]; pLevel->p1 = iCur; - pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrBrk); + pLevel->p2 = 1 + sqlite3VdbeAddOp2(v, aStart[bRev], iCur, addrHalt); VdbeCoverageIf(v, bRev==0); VdbeCoverageIf(v, bRev!=0); pLevel->p5 = SQLITE_STMTSTATUS_FULLSCAN_STEP; @@ -124499,43 +160939,99 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( /* Insert code to test every subexpression that can be completely ** computed using the current set of tables. + ** + ** This loop may run between one and three times, depending on the + ** constraints to be generated. The value of stack variable iLoop + ** determines the constraints coded by each iteration, as follows: + ** + ** iLoop==1: Code only expressions that are entirely covered by pIdx. + ** iLoop==2: Code remaining expressions that do not contain correlated + ** sub-queries. + ** iLoop==3: Code all remaining expressions. + ** + ** An effort is made to skip unnecessary iterations of the loop. + ** + ** This optimization of causing simple query restrictions to occur before + ** more complex one is call the "push-down" optimization in MySQL. Here + ** in SQLite, the name is "MySQL push-down", since there is also another + ** totally unrelated optimization called "WHERE-clause push-down". + ** Sometimes the qualifier is omitted, resulting in an ambiguity, so beware. */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE; - int skipLikeAddr = 0; - testcase( pTerm->wtFlags & TERM_VIRTUAL ); - testcase( pTerm->wtFlags & TERM_CODED ); - if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; - if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ - testcase( pWInfo->untestedTerms==0 - && (pWInfo->wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ); - pWInfo->untestedTerms = 1; - continue; - } - pE = pTerm->pExpr; - assert( pE!=0 ); - if( pLevel->iLeftJoin && !ExprHasProperty(pE, EP_FromJoin) ){ - continue; - } - if( pTerm->wtFlags & TERM_LIKECOND ){ - /* If the TERM_LIKECOND flag is set, that means that the range search - ** is sufficient to guarantee that the LIKE operator is true, so we - ** can skip the call to the like(A,B) function. But this only works - ** for strings. So do not skip the call to the function on the pass - ** that compares BLOBs. */ + iLoop = (pIdx ? 1 : 2); + do{ + int iNext = 0; /* Next value for iLoop */ + for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ + Expr *pE; + int skipLikeAddr = 0; + testcase( pTerm->wtFlags & TERM_VIRTUAL ); + testcase( pTerm->wtFlags & TERM_CODED ); + if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; + if( (pTerm->prereqAll & pLevel->notReady)!=0 ){ + testcase( pWInfo->untestedTerms==0 + && (pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ); + pWInfo->untestedTerms = 1; + continue; + } + pE = pTerm->pExpr; + assert( pE!=0 ); + if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ){ + if( !ExprHasProperty(pE,EP_OuterON|EP_InnerON) ){ + /* Defer processing WHERE clause constraints until after outer + ** join processing. tag-20220513a */ + continue; + }else if( (pTabItem->fg.jointype & JT_LEFT)==JT_LEFT + && !ExprHasProperty(pE,EP_OuterON) ){ + continue; + }else{ + Bitmask m = sqlite3WhereGetMask(&pWInfo->sMaskSet, pE->w.iJoin); + if( m & pLevel->notReady ){ + /* An ON clause that is not ripe */ + continue; + } + } + } + if( iLoop==1 && !sqlite3ExprCoveredByIndex(pE, pLevel->iTabCur, pIdx) ){ + iNext = 2; + continue; + } + if( iLoop<3 && (pTerm->wtFlags & TERM_VARSELECT) ){ + if( iNext==0 ) iNext = 3; + continue; + } + + if( (pTerm->wtFlags & TERM_LIKECOND)!=0 ){ + /* If the TERM_LIKECOND flag is set, that means that the range search + ** is sufficient to guarantee that the LIKE operator is true, so we + ** can skip the call to the like(A,B) function. But this only works + ** for strings. So do not skip the call to the function on the pass + ** that compares BLOBs. */ #ifdef SQLITE_LIKE_DOESNT_MATCH_BLOBS - continue; + continue; #else - u32 x = pLevel->iLikeRepCntr; - assert( x>0 ); - skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)? OP_IfNot : OP_If, (int)(x>>1)); - VdbeCoverage(v); + u32 x = pLevel->iLikeRepCntr; + if( x>0 ){ + skipLikeAddr = sqlite3VdbeAddOp1(v, (x&1)?OP_IfNot:OP_If,(int)(x>>1)); + VdbeCoverageIf(v, (x&1)==1); + VdbeCoverageIf(v, (x&1)==0); + } #endif + } +#ifdef WHERETRACE_ENABLED /* 0xffffffff */ + if( sqlite3WhereTrace ){ + VdbeNoopComment((v, "WhereTerm[%d] (%p) priority=%d", + pWC->nTerm-j, pTerm, iLoop)); + } + if( sqlite3WhereTrace & 0x4000 ){ + sqlite3DebugPrintf("Coding auxiliary constraint:\n"); + sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); + } +#endif + sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); + if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); + pTerm->wtFlags |= TERM_CODED; } - sqlite3ExprIfFalse(pParse, pE, addrCont, SQLITE_JUMPIFNULL); - if( skipLikeAddr ) sqlite3VdbeJumpHere(v, skipLikeAddr); - pTerm->wtFlags |= TERM_CODED; - } + iLoop = iNext; + }while( iLoop>0 ); /* Insert code to test for implied constraints based on transitivity ** of the "==" operator. @@ -124545,43 +161041,117 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** then we cannot use the "t1.a=t2.b" constraint, but we can code ** the implied "t1.a=123" constraint. */ - for(pTerm=pWC->a, j=pWC->nTerm; j>0; j--, pTerm++){ - Expr *pE, *pEAlt; + for(pTerm=pWC->a, j=pWC->nBase; j>0; j--, pTerm++){ + Expr *pE, sEAlt; WhereTerm *pAlt; if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) continue; if( (pTerm->eOperator & WO_EQUIV)==0 ) continue; if( pTerm->leftCursor!=iCur ) continue; - if( pLevel->iLeftJoin ) continue; + if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT) ) continue; pE = pTerm->pExpr; - assert( !ExprHasProperty(pE, EP_FromJoin) ); +#ifdef WHERETRACE_ENABLED /* 0x4001 */ + if( (sqlite3WhereTrace & 0x4001)==0x4001 ){ + sqlite3DebugPrintf("Coding transitive constraint:\n"); + sqlite3WhereTermPrint(pTerm, pWC->nTerm-j); + } +#endif + assert( !ExprHasProperty(pE, EP_OuterON) ); assert( (pTerm->prereqRight & pLevel->notReady)!=0 ); - pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.leftColumn, notReady, + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + pAlt = sqlite3WhereFindTerm(pWC, iCur, pTerm->u.x.leftColumn, notReady, WO_EQ|WO_IN|WO_IS, 0); if( pAlt==0 ) continue; if( pAlt->wtFlags & (TERM_CODED) ) continue; + if( (pAlt->eOperator & WO_IN) + && ExprUseXSelect(pAlt->pExpr) + && (pAlt->pExpr->x.pSelect->pEList->nExpr>1) + ){ + continue; + } testcase( pAlt->eOperator & WO_EQ ); testcase( pAlt->eOperator & WO_IS ); testcase( pAlt->eOperator & WO_IN ); VdbeModuleComment((v, "begin transitive constraint")); - pEAlt = sqlite3StackAllocRaw(db, sizeof(*pEAlt)); - if( pEAlt ){ - *pEAlt = *pAlt->pExpr; - pEAlt->pLeft = pE->pLeft; - sqlite3ExprIfFalse(pParse, pEAlt, addrCont, SQLITE_JUMPIFNULL); - sqlite3StackFree(db, pEAlt); + sEAlt = *pAlt->pExpr; + sEAlt.pLeft = pE->pLeft; + sqlite3ExprIfFalse(pParse, &sEAlt, addrCont, SQLITE_JUMPIFNULL); + pAlt->wtFlags |= TERM_CODED; + } + + /* For a RIGHT OUTER JOIN, record the fact that the current row has + ** been matched at least once. + */ + if( pLevel->pRJ ){ + Table *pTab; + int nPk; + int r; + int jmp1 = 0; + WhereRightJoin *pRJ = pLevel->pRJ; + + /* pTab is the right-hand table of the RIGHT JOIN. Generate code that + ** will record that the current row of that table has been matched at + ** least once. This is accomplished by storing the PK for the row in + ** both the iMatch index and the regBloom Bloom filter. + */ + pTab = pWInfo->pTabList->a[pLevel->iFrom].pSTab; + if( HasRowid(pTab) ){ + r = sqlite3GetTempRange(pParse, 2); + sqlite3ExprCodeGetColumnOfTable(v, pTab, pLevel->iTabCur, -1, r+1); + nPk = 1; + }else{ + int iPk; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + nPk = pPk->nKeyCol; + r = sqlite3GetTempRange(pParse, nPk+1); + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+1+iPk); + } } + jmp1 = sqlite3VdbeAddOp4Int(v, OP_Found, pRJ->iMatch, 0, r+1, nPk); + VdbeCoverage(v); + VdbeComment((v, "match against %s", pTab->zName)); + sqlite3VdbeAddOp3(v, OP_MakeRecord, r+1, nPk, r); + sqlite3VdbeAddOp4Int(v, OP_IdxInsert, pRJ->iMatch, r, r+1, nPk); + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pRJ->regBloom, 0, r+1, nPk); + sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); + sqlite3VdbeJumpHere(v, jmp1); + sqlite3ReleaseTempRange(pParse, r, nPk+1); } /* For a LEFT OUTER JOIN, generate code that will record the fact that - ** at least one row of the right table has matched the left table. + ** at least one row of the right table has matched the left table. */ if( pLevel->iLeftJoin ){ pLevel->addrFirst = sqlite3VdbeCurrentAddr(v); sqlite3VdbeAddOp2(v, OP_Integer, 1, pLevel->iLeftJoin); VdbeComment((v, "record LEFT JOIN hit")); - sqlite3ExprCacheClear(pParse); - for(pTerm=pWC->a, j=0; jnTerm; j++, pTerm++){ + if( pLevel->pRJ==0 ){ + goto code_outer_join_constraints; /* WHERE clause constraints */ + } + } + + if( pLevel->pRJ ){ + /* Create a subroutine used to process all interior loops and code + ** of the RIGHT JOIN. During normal operation, the subroutine will + ** be in-line with the rest of the code. But at the end, a separate + ** loop will run that invokes this subroutine for unmatched rows + ** of pTab, with all tables to left begin set to NULL. + */ + WhereRightJoin *pRJ = pLevel->pRJ; + sqlite3VdbeAddOp2(v, OP_BeginSubrtn, 0, pRJ->regReturn); + pRJ->addrSubrtn = sqlite3VdbeCurrentAddr(v); + assert( pParse->withinRJSubrtn < 255 ); + pParse->withinRJSubrtn++; + + /* WHERE clause constraints must be deferred until after outer join + ** row elimination has completed, since WHERE clause constraints apply + ** to the results of the OUTER JOIN. The following loop generates the + ** appropriate WHERE clause constraint checks. tag-20220513a. + */ + code_outer_join_constraints: + for(pTerm=pWC->a, j=0; jnBase; j++, pTerm++){ testcase( pTerm->wtFlags & TERM_VIRTUAL ); testcase( pTerm->wtFlags & TERM_CODED ); if( pTerm->wtFlags & (TERM_VIRTUAL|TERM_CODED) ) continue; @@ -124589,15 +161159,130 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( assert( pWInfo->untestedTerms ); continue; } + if( pTabItem->fg.jointype & JT_LTORJ ) continue; assert( pTerm->pExpr ); sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); pTerm->wtFlags |= TERM_CODED; } } +#if WHERETRACE_ENABLED /* 0x4001 */ + if( sqlite3WhereTrace & 0x4000 ){ + sqlite3DebugPrintf("All WHERE-clause terms after coding level %d:\n", + iLevel); + sqlite3WhereClausePrint(pWC); + } + if( sqlite3WhereTrace & 0x1 ){ + sqlite3DebugPrintf("End Coding level %d: notReady=%llx\n", + iLevel, (u64)pLevel->notReady); + } +#endif return pLevel->notReady; } +/* +** Generate the code for the loop that finds all non-matched terms +** for a RIGHT JOIN. +*/ +SQLITE_PRIVATE SQLITE_NOINLINE void sqlite3WhereRightJoinLoop( + WhereInfo *pWInfo, + int iLevel, + WhereLevel *pLevel +){ + Parse *pParse = pWInfo->pParse; + Vdbe *v = pParse->pVdbe; + WhereRightJoin *pRJ = pLevel->pRJ; + Expr *pSubWhere = 0; + WhereClause *pWC = &pWInfo->sWC; + WhereInfo *pSubWInfo; + WhereLoop *pLoop = pLevel->pWLoop; + SrcItem *pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + SrcList sFrom; + Bitmask mAll = 0; + int k; + + ExplainQueryPlan((pParse, 1, "RIGHT-JOIN %s", pTabItem->pSTab->zName)); + sqlite3VdbeNoJumpsOutsideSubrtn(v, pRJ->addrSubrtn, pRJ->endSubrtn, + pRJ->regReturn); + for(k=0; ka[k].pWLoop->iTab == pWInfo->a[k].iFrom ); + pRight = &pWInfo->pTabList->a[pWInfo->a[k].iFrom]; + mAll |= pWInfo->a[k].pWLoop->maskSelf; + if( pRight->fg.viaCoroutine ){ + Subquery *pSubq; + assert( pRight->fg.isSubquery && pRight->u4.pSubq!=0 ); + pSubq = pRight->u4.pSubq; + assert( pSubq->pSelect!=0 && pSubq->pSelect->pEList!=0 ); + sqlite3VdbeAddOp3( + v, OP_Null, 0, pSubq->regResult, + pSubq->regResult + pSubq->pSelect->pEList->nExpr-1 + ); + } + sqlite3VdbeAddOp1(v, OP_NullRow, pWInfo->a[k].iTabCur); + iIdxCur = pWInfo->a[k].iIdxCur; + if( iIdxCur ){ + sqlite3VdbeAddOp1(v, OP_NullRow, iIdxCur); + } + } + if( (pTabItem->fg.jointype & JT_LTORJ)==0 ){ + mAll |= pLoop->maskSelf; + for(k=0; knTerm; k++){ + WhereTerm *pTerm = &pWC->a[k]; + if( (pTerm->wtFlags & (TERM_VIRTUAL|TERM_SLICE))!=0 + && pTerm->eOperator!=WO_ROWVAL + ){ + break; + } + if( pTerm->prereqAll & ~mAll ) continue; + if( ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) ) continue; + pSubWhere = sqlite3ExprAnd(pParse, pSubWhere, + sqlite3ExprDup(pParse->db, pTerm->pExpr, 0)); + } + } + sFrom.nSrc = 1; + sFrom.nAlloc = 1; + memcpy(&sFrom.a[0], pTabItem, sizeof(SrcItem)); + sFrom.a[0].fg.jointype = 0; + assert( pParse->withinRJSubrtn < 100 ); + pParse->withinRJSubrtn++; + pSubWInfo = sqlite3WhereBegin(pParse, &sFrom, pSubWhere, 0, 0, 0, + WHERE_RIGHT_JOIN, 0); + if( pSubWInfo ){ + int iCur = pLevel->iTabCur; + int r = ++pParse->nMem; + int nPk; + int jmp; + int addrCont = sqlite3WhereContinueLabel(pSubWInfo); + Table *pTab = pTabItem->pSTab; + if( HasRowid(pTab) ){ + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, -1, r); + nPk = 1; + }else{ + int iPk; + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + nPk = pPk->nKeyCol; + pParse->nMem += nPk - 1; + for(iPk=0; iPkaiColumn[iPk]; + sqlite3ExprCodeGetColumnOfTable(v, pTab, iCur, iCol,r+iPk); + } + } + jmp = sqlite3VdbeAddOp4Int(v, OP_Filter, pRJ->regBloom, 0, r, nPk); + VdbeCoverage(v); + sqlite3VdbeAddOp4Int(v, OP_Found, pRJ->iMatch, addrCont, r, nPk); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, jmp); + sqlite3VdbeAddOp2(v, OP_Gosub, pRJ->regReturn, pRJ->addrSubrtn); + sqlite3WhereEnd(pSubWInfo); + } + sqlite3ExprDelete(pParse->db, pSubWhere); + ExplainQueryPlanPop(pParse); + assert( pParse->withinRJSubrtn>0 ); + pParse->withinRJSubrtn--; +} + /************** End of wherecode.c *******************************************/ /************** Begin file whereexpr.c ***************************************/ /* @@ -124615,7 +161300,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereCodeOneLoopStart( ** the WHERE clause of SQL statements. ** ** This file was originally part of where.c but was split out to improve -** readability and editabiliity. This file contains utility routines for +** readability and editability. This file contains utility routines for ** analyzing Expr objects in the WHERE clause. */ /* #include "sqliteInt.h" */ @@ -124666,7 +161351,7 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ if( pWC->nTerm>=pWC->nSlot ){ WhereTerm *pOld = pWC->a; sqlite3 *db = pWC->pWInfo->pParse->db; - pWC->a = sqlite3DbMallocRawNN(db, sizeof(pWC->a[0])*pWC->nSlot*2 ); + pWC->a = sqlite3WhereMalloc(pWC->pWInfo, sizeof(pWC->a[0])*pWC->nSlot*2 ); if( pWC->a==0 ){ if( wtFlags & TERM_DYNAMIC ){ sqlite3ExprDelete(db, p); @@ -124675,66 +161360,53 @@ static int whereClauseInsert(WhereClause *pWC, Expr *p, u16 wtFlags){ return 0; } memcpy(pWC->a, pOld, sizeof(pWC->a[0])*pWC->nTerm); - if( pOld!=pWC->aStatic ){ - sqlite3DbFree(db, pOld); - } - pWC->nSlot = sqlite3DbMallocSize(db, pWC->a)/sizeof(pWC->a[0]); - memset(&pWC->a[pWC->nTerm], 0, sizeof(pWC->a[0])*(pWC->nSlot-pWC->nTerm)); + pWC->nSlot = pWC->nSlot*2; } pTerm = &pWC->a[idx = pWC->nTerm++]; + if( (wtFlags & TERM_VIRTUAL)==0 ) pWC->nBase = pWC->nTerm; if( p && ExprHasProperty(p, EP_Unlikely) ){ pTerm->truthProb = sqlite3LogEst(p->iTable) - 270; }else{ pTerm->truthProb = 1; } - pTerm->pExpr = sqlite3ExprSkipCollate(p); + pTerm->pExpr = sqlite3ExprSkipCollateAndLikely(p); pTerm->wtFlags = wtFlags; pTerm->pWC = pWC; pTerm->iParent = -1; + memset(&pTerm->eOperator, 0, + sizeof(WhereTerm) - offsetof(WhereTerm,eOperator)); return idx; } /* ** Return TRUE if the given operator is one of the operators that is ** allowed for an indexable WHERE clause term. The allowed operators are -** "=", "<", ">", "<=", ">=", "IN", and "IS NULL" +** "=", "<", ">", "<=", ">=", "IN", "IS", and "IS NULL" */ static int allowedOp(int op){ assert( TK_GT>TK_EQ && TK_GTTK_EQ && TK_LTTK_EQ && TK_LE=TK_EQ && op<=TK_GE) || op==TK_ISNULL || op==TK_IS; + assert( TK_INTK_GE ) return 0; + if( op>=TK_EQ ) return 1; + return op==TK_IN || op==TK_ISNULL || op==TK_IS; } /* ** Commute a comparison operator. Expressions of the form "X op Y" ** are converted into "Y op X". -** -** If left/right precedence rules come into play when determining the -** collating sequence, then COLLATE operators are adjusted to ensure -** that the collating sequence does not change. For example: -** "Y collate NOCASE op X" becomes "X op Y" because any collation sequence on -** the left hand side of a comparison overrides any collation sequence -** attached to the right. For the same reason the EP_Collate flag -** is not commuted. */ -static void exprCommute(Parse *pParse, Expr *pExpr){ - u16 expRight = (pExpr->pRight->flags & EP_Collate); - u16 expLeft = (pExpr->pLeft->flags & EP_Collate); - assert( allowedOp(pExpr->op) && pExpr->op!=TK_IN ); - if( expRight==expLeft ){ - /* Either X and Y both have COLLATE operator or neither do */ - if( expRight ){ - /* Both X and Y have COLLATE operators. Make sure X is always - ** used by clearing the EP_Collate flag from Y. */ - pExpr->pRight->flags &= ~EP_Collate; - }else if( sqlite3ExprCollSeq(pParse, pExpr->pLeft)!=0 ){ - /* Neither X nor Y have COLLATE operators, but X has a non-default - ** collating sequence. So add the EP_Collate marker on X to cause - ** it to be searched first. */ - pExpr->pLeft->flags |= EP_Collate; - } +static u16 exprCommute(Parse *pParse, Expr *pExpr){ + if( pExpr->pLeft->op==TK_VECTOR + || pExpr->pRight->op==TK_VECTOR + || sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight) != + sqlite3BinaryCompareCollSeq(pParse, pExpr->pRight, pExpr->pLeft) + ){ + pExpr->flags ^= EP_Commuted; } SWAP(Expr*,pExpr->pRight,pExpr->pLeft); if( pExpr->op>=TK_GT ){ @@ -124745,6 +161417,7 @@ static void exprCommute(Parse *pParse, Expr *pExpr){ assert( pExpr->op>=TK_GT && pExpr->op<=TK_GE ); pExpr->op = ((pExpr->op-TK_GT)^2)+TK_GT; } + return 0; } /* @@ -124753,15 +161426,16 @@ static void exprCommute(Parse *pParse, Expr *pExpr){ static u16 operatorMask(int op){ u16 c; assert( allowedOp(op) ); - if( op==TK_IN ){ + if( op>=TK_EQ ){ + assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); + c = (u16)(WO_EQ<<(op-TK_EQ)); + }else if( op==TK_IN ){ c = WO_IN; }else if( op==TK_ISNULL ){ c = WO_ISNULL; - }else if( op==TK_IS ){ - c = WO_IS; }else{ - assert( (WO_EQ<<(op-TK_EQ)) < 0x7fff ); - c = (u16)(WO_EQ<<(op-TK_EQ)); + assert( op==TK_IS ); + c = WO_IS; } assert( op!=TK_ISNULL || c==WO_ISNULL ); assert( op!=TK_IN || c==WO_IN ); @@ -124795,70 +161469,147 @@ static int isLikeOrGlob( int *pisComplete, /* True if the only wildcard is % in the last character */ int *pnoCase /* True if uppercase is equivalent to lowercase */ ){ - const char *z = 0; /* String on RHS of LIKE operator */ + const u8 *z = 0; /* String on RHS of LIKE operator */ Expr *pRight, *pLeft; /* Right and left size of LIKE operator */ ExprList *pList; /* List of operands to the LIKE operator */ - int c; /* One character in z[] */ + u8 c; /* One character in z[] */ int cnt; /* Number of non-wildcard prefix characters */ - char wc[3]; /* Wildcard characters */ + u8 wc[4]; /* Wildcard characters */ sqlite3 *db = pParse->db; /* Database connection */ sqlite3_value *pVal = 0; int op; /* Opcode of pRight */ int rc; /* Result code to return */ - if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, wc) ){ + if( !sqlite3IsLikeFunction(db, pExpr, pnoCase, (char*)wc) ){ return 0; } #ifdef SQLITE_EBCDIC if( *pnoCase ) return 0; #endif + assert( ExprUseXList(pExpr) ); pList = pExpr->x.pList; pLeft = pList->a[1].pExpr; - if( pLeft->op!=TK_COLUMN - || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT - || IsVirtual(pLeft->pTab) /* Value might be numeric */ - ){ - /* IMP: R-02065-49465 The left-hand side of the LIKE or GLOB operator must - ** be the name of an indexed column with TEXT affinity. */ - return 0; - } - assert( pLeft->iColumn!=(-1) ); /* Because IPK never has AFF_TEXT */ pRight = sqlite3ExprSkipCollate(pList->a[0].pExpr); op = pRight->op; - if( op==TK_VARIABLE ){ + if( op==TK_VARIABLE && (db->flags & SQLITE_EnableQPSG)==0 ){ Vdbe *pReprepare = pParse->pReprepare; int iCol = pRight->iColumn; pVal = sqlite3VdbeGetBoundValue(pReprepare, iCol, SQLITE_AFF_BLOB); if( pVal && sqlite3_value_type(pVal)==SQLITE_TEXT ){ - z = (char *)sqlite3_value_text(pVal); + z = sqlite3_value_text(pVal); } sqlite3VdbeSetVarmask(pParse->pVdbe, iCol); assert( pRight->op==TK_VARIABLE || pRight->op==TK_REGISTER ); }else if( op==TK_STRING ){ - z = pRight->u.zToken; + assert( !ExprHasProperty(pRight, EP_IntValue) ); + z = (u8*)pRight->u.zToken; } if( z ){ + /* Count the number of prefix bytes prior to the first wildcard. + ** or U+fffd character. If the underlying database has a UTF16LE + ** encoding, then only consider ASCII characters. Note that the + ** encoding of z[] is UTF8 - we are dealing with only UTF8 here in + ** this code, but the database engine itself might be processing + ** content using a different encoding. */ cnt = 0; while( (c=z[cnt])!=0 && c!=wc[0] && c!=wc[1] && c!=wc[2] ){ cnt++; + if( c==wc[3] && z[cnt]>0 && z[cnt]<0x80 ){ + cnt++; + }else if( c>=0x80 ){ + const u8 *z2 = z+cnt-1; + if( sqlite3Utf8Read(&z2)==0xfffd || ENC(db)==SQLITE_UTF16LE ){ + cnt--; + break; + }else{ + cnt = (int)(z2-z); + } + } } - if( cnt!=0 && 255!=(u8)z[cnt-1] ){ + + /* The optimization is possible only if (1) the pattern does not begin + ** with a wildcard and if (2) the non-wildcard prefix does not end with + ** an (illegal 0xff) character, or (3) the pattern does not consist of + ** a single escape character. The second condition is necessary so + ** that we can increment the prefix key to find an upper bound for the + ** range search. The third is because the caller assumes that the pattern + ** consists of at least one character after all escapes have been + ** removed. */ + if( (cnt>1 || (cnt>0 && z[0]!=wc[3])) && ALWAYS(255!=(u8)z[cnt-1]) ){ Expr *pPrefix; - *pisComplete = c==wc[0] && z[cnt+1]==0; - pPrefix = sqlite3Expr(db, TK_STRING, z); - if( pPrefix ) pPrefix->u.zToken[cnt] = 0; + + /* A "complete" match if the pattern ends with "*" or "%" */ + *pisComplete = c==wc[0] && z[cnt+1]==0 && ENC(db)!=SQLITE_UTF16LE; + + /* Get the pattern prefix. Remove all escapes from the prefix. */ + pPrefix = sqlite3Expr(db, TK_STRING, (char*)z); + if( pPrefix ){ + int iFrom, iTo; + char *zNew; + assert( !ExprHasProperty(pPrefix, EP_IntValue) ); + zNew = pPrefix->u.zToken; + zNew[cnt] = 0; + for(iFrom=iTo=0; iFrom0 ); + + /* If the LHS is not an ordinary column with TEXT affinity, then the + ** pattern prefix boundaries (both the start and end boundaries) must + ** not look like a number. Otherwise the pattern might be treated as + ** a number, which will invalidate the LIKE optimization. + ** + ** Getting this right has been a persistent source of bugs in the + ** LIKE optimization. See, for example: + ** 2018-09-10 https://sqlite.org/src/info/c94369cae9b561b1 + ** 2019-05-02 https://sqlite.org/src/info/b043a54c3de54b28 + ** 2019-06-10 https://sqlite.org/src/info/fd76310a5e843e07 + ** 2019-06-14 https://sqlite.org/src/info/ce8717f0885af975 + ** 2019-09-03 https://sqlite.org/src/info/0f0428096f17252a + */ + if( pLeft->op!=TK_COLUMN + || sqlite3ExprAffinity(pLeft)!=SQLITE_AFF_TEXT + || (ALWAYS( ExprUseYTab(pLeft) ) + && ALWAYS(pLeft->y.pTab) + && IsVirtual(pLeft->y.pTab)) /* Might be numeric */ + ){ + int isNum; + double rDummy; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + if( isNum<=0 ){ + if( iTo==1 && zNew[0]=='-' ){ + isNum = +1; + }else{ + zNew[iTo-1]++; + isNum = sqlite3AtoF(zNew, &rDummy, iTo, SQLITE_UTF8); + zNew[iTo-1]--; + } + } + if( isNum>0 ){ + sqlite3ExprDelete(db, pPrefix); + sqlite3ValueFree(pVal); + return 0; + } + } + } *ppPrefix = pPrefix; + + /* If the RHS pattern is a bound parameter, make arrangements to + ** reprepare the statement when that parameter is rebound */ if( op==TK_VARIABLE ){ Vdbe *v = pParse->pVdbe; sqlite3VdbeSetVarmask(v, pRight->iColumn); + assert( !ExprHasProperty(pRight, EP_IntValue) ); if( *pisComplete && pRight->u.zToken[1] ){ /* If the rhs of the LIKE expression is a variable, and the current ** value of the variable means there is no need to invoke the LIKE ** function, then no OP_Variable will be added to the program. ** This causes problems for the sqlite3_bind_parameter_name() ** API. To work around them, add a dummy OP_Variable here. - */ + */ int r1 = sqlite3GetTempReg(pParse); sqlite3ExprCodeTarget(pParse, pRight, r1); sqlite3VdbeChangeP3(v, sqlite3VdbeCurrentAddr(v)-1, 0); @@ -124879,48 +161630,139 @@ static int isLikeOrGlob( #ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Check to see if the given expression is of the form +** Check to see if the pExpr expression is a form that needs to be passed +** to the xBestIndex method of virtual tables. Forms of interest include: ** -** column OP expr +** Expression Virtual Table Operator +** ----------------------- --------------------------------- +** 1. column MATCH expr SQLITE_INDEX_CONSTRAINT_MATCH +** 2. column GLOB expr SQLITE_INDEX_CONSTRAINT_GLOB +** 3. column LIKE expr SQLITE_INDEX_CONSTRAINT_LIKE +** 4. column REGEXP expr SQLITE_INDEX_CONSTRAINT_REGEXP +** 5. column != expr SQLITE_INDEX_CONSTRAINT_NE +** 6. expr != column SQLITE_INDEX_CONSTRAINT_NE +** 7. column IS NOT expr SQLITE_INDEX_CONSTRAINT_ISNOT +** 8. expr IS NOT column SQLITE_INDEX_CONSTRAINT_ISNOT +** 9. column IS NOT NULL SQLITE_INDEX_CONSTRAINT_ISNOTNULL ** -** where OP is one of MATCH, GLOB, LIKE or REGEXP and "column" is a -** column of a virtual table. +** In every case, "column" must be a column of a virtual table. If there +** is a match, set *ppLeft to the "column" expression, set *ppRight to the +** "expr" expression (even though in forms (6) and (8) the column is on the +** right and the expression is on the left). Also set *peOp2 to the +** appropriate virtual table operator. The return value is 1 or 2 if there +** is a match. The usual return is 1, but if the RHS is also a column +** of virtual table in forms (5) or (7) then return 2. ** -** If it is then return TRUE. If not, return FALSE. +** If the expression matches none of the patterns above, return 0. */ -static int isMatchOfColumn( +static int isAuxiliaryVtabOperator( + sqlite3 *db, /* Parsing context */ Expr *pExpr, /* Test this expression */ - unsigned char *peOp2 /* OUT: 0 for MATCH, or else an op2 value */ + unsigned char *peOp2, /* OUT: 0 for MATCH, or else an op2 value */ + Expr **ppLeft, /* Column expression to left of MATCH/op2 */ + Expr **ppRight /* Expression to left of MATCH/op2 */ ){ - struct Op2 { - const char *zOp; - unsigned char eOp2; - } aOp[] = { - { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, - { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, - { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, - { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } - }; - ExprList *pList; - Expr *pCol; /* Column reference */ - int i; + if( pExpr->op==TK_FUNCTION ){ + static const struct Op2 { + const char *zOp; + unsigned char eOp2; + } aOp[] = { + { "match", SQLITE_INDEX_CONSTRAINT_MATCH }, + { "glob", SQLITE_INDEX_CONSTRAINT_GLOB }, + { "like", SQLITE_INDEX_CONSTRAINT_LIKE }, + { "regexp", SQLITE_INDEX_CONSTRAINT_REGEXP } + }; + ExprList *pList; + Expr *pCol; /* Column reference */ + int i; - if( pExpr->op!=TK_FUNCTION ){ - return 0; - } - pList = pExpr->x.pList; - if( pList==0 || pList->nExpr!=2 ){ - return 0; - } - pCol = pList->a[1].pExpr; - if( pCol->op!=TK_COLUMN || !IsVirtual(pCol->pTab) ){ - return 0; - } - for(i=0; iu.zToken, aOp[i].zOp)==0 ){ - *peOp2 = aOp[i].eOp2; - return 1; + assert( ExprUseXList(pExpr) ); + pList = pExpr->x.pList; + if( pList==0 || pList->nExpr!=2 ){ + return 0; } + + /* Built-in operators MATCH, GLOB, LIKE, and REGEXP attach to a + ** virtual table on their second argument, which is the same as + ** the left-hand side operand in their in-fix form. + ** + ** vtab_column MATCH expression + ** MATCH(expression,vtab_column) + */ + pCol = pList->a[1].pExpr; + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); + if( ExprIsVtab(pCol) ){ + for(i=0; iu.zToken, aOp[i].zOp)==0 ){ + *peOp2 = aOp[i].eOp2; + *ppRight = pList->a[0].pExpr; + *ppLeft = pCol; + return 1; + } + } + } + + /* We can also match against the first column of overloaded + ** functions where xFindFunction returns a value of at least + ** SQLITE_INDEX_CONSTRAINT_FUNCTION. + ** + ** OVERLOADED(vtab_column,expression) + ** + ** Historically, xFindFunction expected to see lower-case function + ** names. But for this use case, xFindFunction is expected to deal + ** with function names in an arbitrary case. + */ + pCol = pList->a[0].pExpr; + assert( pCol->op!=TK_COLUMN || ExprUseYTab(pCol) ); + assert( pCol->op!=TK_COLUMN || (ExprUseYTab(pCol) && pCol->y.pTab!=0) ); + if( ExprIsVtab(pCol) ){ + sqlite3_vtab *pVtab; + sqlite3_module *pMod; + void (*xNotUsed)(sqlite3_context*,int,sqlite3_value**); + void *pNotUsed; + pVtab = sqlite3GetVTable(db, pCol->y.pTab)->pVtab; + assert( pVtab!=0 ); + assert( pVtab->pModule!=0 ); + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pMod = (sqlite3_module *)pVtab->pModule; + if( pMod->xFindFunction!=0 ){ + i = pMod->xFindFunction(pVtab,2, pExpr->u.zToken, &xNotUsed, &pNotUsed); + if( i>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + *peOp2 = i; + *ppRight = pList->a[1].pExpr; + *ppLeft = pCol; + return 1; + } + } + } + }else if( pExpr->op>=TK_EQ ){ + /* Comparison operators are a common case. Save a few comparisons for + ** that common case by terminating early. */ + assert( TK_NE < TK_EQ ); + assert( TK_ISNOT < TK_EQ ); + assert( TK_NOTNULL < TK_EQ ); + return 0; + }else if( pExpr->op==TK_NE || pExpr->op==TK_ISNOT || pExpr->op==TK_NOTNULL ){ + int res = 0; + Expr *pLeft = pExpr->pLeft; + Expr *pRight = pExpr->pRight; + assert( pLeft->op!=TK_COLUMN || (ExprUseYTab(pLeft) && pLeft->y.pTab!=0) ); + if( ExprIsVtab(pLeft) ){ + res++; + } + assert( pRight==0 || pRight->op!=TK_COLUMN + || (ExprUseYTab(pRight) && pRight->y.pTab!=0) ); + if( pRight && ExprIsVtab(pRight) ){ + res++; + SWAP(Expr*, pLeft, pRight); + } + *ppLeft = pLeft; + *ppRight = pRight; + if( pExpr->op==TK_NE ) *peOp2 = SQLITE_INDEX_CONSTRAINT_NE; + if( pExpr->op==TK_ISNOT ) *peOp2 = SQLITE_INDEX_CONSTRAINT_ISNOT; + if( pExpr->op==TK_NOTNULL ) *peOp2 = SQLITE_INDEX_CONSTRAINT_ISNOTNULL; + return res; } return 0; } @@ -124931,9 +161773,9 @@ static int isMatchOfColumn( ** a join, then transfer the appropriate markings over to derived. */ static void transferJoinMarkings(Expr *pDerived, Expr *pBase){ - if( pDerived ){ - pDerived->flags |= pBase->flags & EP_FromJoin; - pDerived->iRightJoinTable = pBase->iRightJoinTable; + if( pDerived && ExprHasProperty(pBase, EP_OuterON|EP_InnerON) ){ + pDerived->flags |= pBase->flags & (EP_OuterON|EP_InnerON); + pDerived->w.iJoin = pBase->w.iJoin; } } @@ -124979,7 +161821,7 @@ static WhereTerm *whereNthSubterm(WhereTerm *pTerm, int N){ ** ** The following is NOT generated: ** -** xy --> x!=y +** xy --> x!=y */ static void whereCombineDisjuncts( SrcList *pSrc, /* the FROM clause */ @@ -124993,14 +161835,15 @@ static void whereCombineDisjuncts( int op; /* Operator for the combined expression */ int idxNew; /* Index in pWC of the next virtual term */ + if( (pOne->wtFlags | pTwo->wtFlags) & TERM_VNULL ) return; if( (pOne->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (pTwo->eOperator & (WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE))==0 ) return; if( (eOp & (WO_EQ|WO_LT|WO_LE))!=eOp && (eOp & (WO_EQ|WO_GT|WO_GE))!=eOp ) return; assert( pOne->pExpr->pLeft!=0 && pOne->pExpr->pRight!=0 ); assert( pTwo->pExpr->pLeft!=0 && pTwo->pExpr->pRight!=0 ); - if( sqlite3ExprCompare(pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; - if( sqlite3ExprCompare(pOne->pExpr->pRight, pTwo->pExpr->pRight, -1) )return; + if( sqlite3ExprCompare(0,pOne->pExpr->pLeft, pTwo->pExpr->pLeft, -1) ) return; + if( sqlite3ExprCompare(0,pOne->pExpr->pRight, pTwo->pExpr->pRight,-1) )return; /* If we reach this point, it means the two subterms can be combined */ if( (eOp & (eOp-1))!=0 ){ if( eOp & (WO_LT|WO_LE) ){ @@ -125076,10 +161919,10 @@ static void whereCombineDisjuncts( ** WhereTerm.u.pOrInfo->indexable |= the cursor number for table T ** ** A subterm is "indexable" if it is of the form -** "T.C " where C is any column of table T and +** "T.C " where C is any column of table T and ** is one of "=", "<", "<=", ">", ">=", "IS NULL", or "IN". ** A subterm is also indexable if it is an AND of two or more -** subsubterms at least one of which is indexable. Indexable AND +** subsubterms at least one of which is indexable. Indexable AND ** subterms have their eOperator set to WO_AND and they have ** u.pAndInfo set to a dynamically allocated WhereAndTerm object. ** @@ -125161,6 +162004,7 @@ static void exprAnalyzeOrTerm( pOrTerm->u.pAndInfo = pAndInfo; pOrTerm->wtFlags |= TERM_ANDINFO; pOrTerm->eOperator = WO_AND; + pOrTerm->leftCursor = -1; pAndWC = &pAndInfo->wc; memset(pAndWC->aStatic, 0, sizeof(pAndWC->aStatic)); sqlite3WhereClauseInit(pAndWC, pWC->pWInfo); @@ -125170,7 +162014,9 @@ static void exprAnalyzeOrTerm( if( !db->mallocFailed ){ for(j=0, pAndTerm=pAndWC->a; jnTerm; j++, pAndTerm++){ assert( pAndTerm->pExpr ); - if( allowedOp(pAndTerm->pExpr->op) ){ + if( allowedOp(pAndTerm->pExpr->op) + || pAndTerm->eOperator==WO_AUX + ){ b |= sqlite3WhereGetMask(&pWInfo->sMaskSet, pAndTerm->leftCursor); } } @@ -125201,7 +162047,11 @@ static void exprAnalyzeOrTerm( ** empty. */ pOrInfo->indexable = indexable; - pTerm->eOperator = indexable==0 ? 0 : WO_OR; + pTerm->eOperator = WO_OR; + pTerm->leftCursor = -1; + if( indexable ){ + pWC->hasOr = 1; + } /* For a two-way OR, attempt to implementation case 2. */ @@ -125251,10 +162101,11 @@ static void exprAnalyzeOrTerm( ** and column is found but leave okToChngToIN false if not found. */ for(j=0; j<2 && !okToChngToIN; j++){ + Expr *pLeft = 0; pOrTerm = pOrWc->a; for(i=pOrWc->nTerm-1; i>=0; i--, pOrTerm++){ assert( pOrTerm->eOperator & WO_EQ ); - pOrTerm->wtFlags &= ~TERM_OR_OK; + pOrTerm->wtFlags &= ~TERM_OK; if( pOrTerm->leftCursor==iCursor ){ /* This is the 2-bit case and we are on the second iteration and ** current term is from the first iteration. So skip this term. */ @@ -125265,15 +162116,17 @@ static void exprAnalyzeOrTerm( pOrTerm->leftCursor))==0 ){ /* This term must be of the form t1.a==t2.b where t2 is in the ** chngToIN set but t1 is not. This term will be either preceded - ** or follwed by an inverted copy (t2.b==t1.a). Skip this term + ** or followed by an inverted copy (t2.b==t1.a). Skip this term ** and use its inversion. */ testcase( pOrTerm->wtFlags & TERM_COPIED ); testcase( pOrTerm->wtFlags & TERM_VIRTUAL ); assert( pOrTerm->wtFlags & (TERM_COPIED|TERM_VIRTUAL) ); continue; } - iColumn = pOrTerm->u.leftColumn; + assert( (pOrTerm->eOperator & (WO_OR|WO_AND))==0 ); + iColumn = pOrTerm->u.x.leftColumn; iCursor = pOrTerm->leftCursor; + pLeft = pOrTerm->pExpr->pLeft; break; } if( i<0 ){ @@ -125291,9 +162144,12 @@ static void exprAnalyzeOrTerm( okToChngToIN = 1; for(; i>=0 && okToChngToIN; i--, pOrTerm++){ assert( pOrTerm->eOperator & WO_EQ ); + assert( (pOrTerm->eOperator & (WO_OR|WO_AND))==0 ); if( pOrTerm->leftCursor!=iCursor ){ - pOrTerm->wtFlags &= ~TERM_OR_OK; - }else if( pOrTerm->u.leftColumn!=iColumn ){ + pOrTerm->wtFlags &= ~TERM_OK; + }else if( pOrTerm->u.x.leftColumn!=iColumn || (iColumn==XN_EXPR + && sqlite3ExprCompare(pParse, pOrTerm->pExpr->pLeft, pLeft, -1) + )){ okToChngToIN = 0; }else{ int affLeft, affRight; @@ -125306,14 +162162,14 @@ static void exprAnalyzeOrTerm( if( affRight!=0 && affRight!=affLeft ){ okToChngToIN = 0; }else{ - pOrTerm->wtFlags |= TERM_OR_OK; + pOrTerm->wtFlags |= TERM_OK; } } } } /* At this point, okToChngToIN is true if original pTerm satisfies - ** case 1. In that case, construct a new virtual term that is + ** case 1. In that case, construct a new virtual term that is ** pTerm converted into an IN operator. */ if( okToChngToIN ){ @@ -125323,31 +162179,31 @@ static void exprAnalyzeOrTerm( Expr *pNew; /* The complete IN operator */ for(i=pOrWc->nTerm-1, pOrTerm=pOrWc->a; i>=0; i--, pOrTerm++){ - if( (pOrTerm->wtFlags & TERM_OR_OK)==0 ) continue; + if( (pOrTerm->wtFlags & TERM_OK)==0 ) continue; assert( pOrTerm->eOperator & WO_EQ ); + assert( (pOrTerm->eOperator & (WO_OR|WO_AND))==0 ); assert( pOrTerm->leftCursor==iCursor ); - assert( pOrTerm->u.leftColumn==iColumn ); + assert( pOrTerm->u.x.leftColumn==iColumn ); pDup = sqlite3ExprDup(db, pOrTerm->pExpr->pRight, 0); pList = sqlite3ExprListAppend(pWInfo->pParse, pList, pDup); pLeft = pOrTerm->pExpr->pLeft; } assert( pLeft!=0 ); pDup = sqlite3ExprDup(db, pLeft, 0); - pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0, 0); + pNew = sqlite3PExpr(pParse, TK_IN, pDup, 0); if( pNew ){ int idxNew; transferJoinMarkings(pNew, pExpr); - assert( !ExprHasProperty(pNew, EP_xIsSelect) ); + assert( ExprUseXList(pNew) ); pNew->x.pList = pList; idxNew = whereClauseInsert(pWC, pNew, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); exprAnalyze(pSrc, pWC, idxNew); - pTerm = &pWC->a[idxTerm]; + /* pTerm = &pWC->a[idxTerm]; // would be needed if pTerm where reused */ markTermAsChild(pWC, idxNew, idxTerm); }else{ sqlite3ExprListDelete(db, pList); } - pTerm->eOperator = WO_NOOP; /* case 1 trumps case 3 */ } } } @@ -125371,10 +162227,9 @@ static void exprAnalyzeOrTerm( static int termIsEquivalence(Parse *pParse, Expr *pExpr){ char aff1, aff2; CollSeq *pColl; - const char *zColl1, *zColl2; if( !OptimizationEnabled(pParse->db, SQLITE_Transitive) ) return 0; if( pExpr->op!=TK_EQ && pExpr->op!=TK_IS ) return 0; - if( ExprHasProperty(pExpr, EP_FromJoin) ) return 0; + if( ExprHasProperty(pExpr, EP_OuterON) ) return 0; aff1 = sqlite3ExprAffinity(pExpr->pLeft); aff2 = sqlite3ExprAffinity(pExpr->pRight); if( aff1!=aff2 @@ -125382,15 +162237,9 @@ static int termIsEquivalence(Parse *pParse, Expr *pExpr){ ){ return 0; } - pColl = sqlite3BinaryCompareCollSeq(pParse, pExpr->pLeft, pExpr->pRight); - if( pColl==0 || sqlite3StrICmp(pColl->zName, "BINARY")==0 ) return 1; - pColl = sqlite3ExprCollSeq(pParse, pExpr->pLeft); - /* Since pLeft and pRight are both a column references, their collating - ** sequence should always be defined. */ - zColl1 = ALWAYS(pColl) ? pColl->zName : 0; - pColl = sqlite3ExprCollSeq(pParse, pExpr->pRight); - zColl2 = ALWAYS(pColl) ? pColl->zName : 0; - return sqlite3StrICmp(zColl1, zColl2)==0; + pColl = sqlite3ExprCompareCollSeq(pParse, pExpr); + if( sqlite3IsBinary(pColl) ) return 1; + return sqlite3ExprCollSeqMatch(pParse, pExpr->pLeft, pExpr->pRight); } /* @@ -125410,8 +162259,15 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ if( ALWAYS(pSrc!=0) ){ int i; for(i=0; inSrc; i++){ - mask |= exprSelectUsage(pMaskSet, pSrc->a[i].pSelect); - mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].pOn); + if( pSrc->a[i].fg.isSubquery ){ + mask |= exprSelectUsage(pMaskSet, pSrc->a[i].u4.pSubq->pSelect); + } + if( pSrc->a[i].fg.isUsing==0 ){ + mask |= sqlite3WhereExprUsage(pMaskSet, pSrc->a[i].u3.pOn); + } + if( pSrc->a[i].fg.isTabFunc ){ + mask |= sqlite3WhereExprListUsage(pMaskSet, pSrc->a[i].u1.pFuncArg); + } } } pS = pS->pPrior; @@ -125423,47 +162279,80 @@ static Bitmask exprSelectUsage(WhereMaskSet *pMaskSet, Select *pS){ ** Expression pExpr is one operand of a comparison operator that might ** be useful for indexing. This routine checks to see if pExpr appears ** in any index. Return TRUE (1) if pExpr is an indexed term and return -** FALSE (0) if not. If TRUE is returned, also set *piCur to the cursor -** number of the table that is indexed and *piColumn to the column number -** of the column that is indexed, or -2 if an expression is being indexed. +** FALSE (0) if not. If TRUE is returned, also set aiCurCol[0] to the cursor +** number of the table that is indexed and aiCurCol[1] to the column number +** of the column that is indexed, or XN_EXPR (-2) if an expression is being +** indexed. ** ** If pExpr is a TK_COLUMN column reference, then this routine always returns ** true even if that particular column is not indexed, because the column ** might be added to an automatic index later. */ -static int exprMightBeIndexed( +static SQLITE_NOINLINE int exprMightBeIndexed2( SrcList *pFrom, /* The FROM clause */ - Bitmask mPrereq, /* Bitmask of FROM clause terms referenced by pExpr */ + int *aiCurCol, /* Write the referenced table cursor and column here */ Expr *pExpr, /* An operand of a comparison operator */ - int *piCur, /* Write the referenced table cursor number here */ - int *piColumn /* Write the referenced table column number here */ + int j /* Start looking with the j-th pFrom entry */ ){ Index *pIdx; int i; int iCur; + do{ + iCur = pFrom->a[j].iCursor; + for(pIdx=pFrom->a[j].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr==0 ) continue; + for(i=0; inKeyCol; i++){ + if( pIdx->aiColumn[i]!=XN_EXPR ) continue; + assert( pIdx->bHasExpr ); + if( sqlite3ExprCompareSkip(pExpr,pIdx->aColExpr->a[i].pExpr,iCur)==0 + && !sqlite3ExprIsConstant(0,pIdx->aColExpr->a[i].pExpr) + ){ + aiCurCol[0] = iCur; + aiCurCol[1] = XN_EXPR; + return 1; + } + } + } + }while( ++j < pFrom->nSrc ); + return 0; +} +static int exprMightBeIndexed( + SrcList *pFrom, /* The FROM clause */ + int *aiCurCol, /* Write the referenced table cursor & column here */ + Expr *pExpr, /* An operand of a comparison operator */ + int op /* The specific comparison operator */ +){ + int i; + + /* If this expression is a vector to the left or right of a + ** inequality constraint (>, <, >= or <=), perform the processing + ** on the first element of the vector. */ + assert( TK_GT+1==TK_LE && TK_GT+2==TK_LT && TK_GT+3==TK_GE ); + assert( TK_ISop==TK_VECTOR && (op>=TK_GT && ALWAYS(op<=TK_GE)) ){ + assert( ExprUseXList(pExpr) ); + pExpr = pExpr->x.pList->a[0].pExpr; + } + if( pExpr->op==TK_COLUMN ){ - *piCur = pExpr->iTable; - *piColumn = pExpr->iColumn; + aiCurCol[0] = pExpr->iTable; + aiCurCol[1] = pExpr->iColumn; return 1; } - if( mPrereq==0 ) return 0; /* No table references */ - if( (mPrereq&(mPrereq-1))!=0 ) return 0; /* Refs more than one table */ - for(i=0; mPrereq>1; i++, mPrereq>>=1){} - iCur = pFrom->a[i].iCursor; - for(pIdx=pFrom->a[i].pTab->pIndex; pIdx; pIdx=pIdx->pNext){ - if( pIdx->aColExpr==0 ) continue; - for(i=0; inKeyCol; i++){ - if( pIdx->aiColumn[i]!=(-2) ) continue; - if( sqlite3ExprCompare(pExpr, pIdx->aColExpr->a[i].pExpr, iCur)==0 ){ - *piCur = iCur; - *piColumn = -2; - return 1; + + for(i=0; inSrc; i++){ + Index *pIdx; + for(pIdx=pFrom->a[i].pSTab->pIndex; pIdx; pIdx=pIdx->pNext){ + if( pIdx->aColExpr ){ + return exprMightBeIndexed2(pFrom,aiCurCol,pExpr,i); } } } return 0; } + /* ** The input to this routine is an WhereTerm structure with only the ** "pExpr" field filled in. The job of this routine is to analyze the @@ -125491,8 +162380,8 @@ static void exprAnalyze( WhereTerm *pTerm; /* The term to be analyzed */ WhereMaskSet *pMaskSet; /* Set of table index masks */ Expr *pExpr; /* The expression to be analyzed */ - Bitmask prereqLeft; /* Prerequesites of the pExpr->pLeft */ - Bitmask prereqAll; /* Prerequesites of pExpr */ + Bitmask prereqLeft; /* Prerequisites of the pExpr->pLeft */ + Bitmask prereqAll; /* Prerequisites of pExpr */ Bitmask extraRight = 0; /* Extra dependencies on LEFT JOIN */ Expr *pStr1 = 0; /* RHS of LIKE/GLOB operator */ int isComplete = 0; /* RHS of LIKE/GLOB ends with wildcard */ @@ -125500,57 +162389,107 @@ static void exprAnalyze( int op; /* Top-level operator. pExpr->op */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection */ - unsigned char eOp2; /* op2 value for LIKE/REGEXP/GLOB */ + unsigned char eOp2 = 0; /* op2 value for LIKE/REGEXP/GLOB */ + int nLeft; /* Number of elements on left side vector */ if( db->mallocFailed ){ return; } + assert( pWC->nTerm > idxTerm ); pTerm = &pWC->a[idxTerm]; pMaskSet = &pWInfo->sMaskSet; pExpr = pTerm->pExpr; + assert( pExpr!=0 ); /* Because malloc() has not failed */ assert( pExpr->op!=TK_AS && pExpr->op!=TK_COLLATE ); + pMaskSet->bVarSelect = 0; prereqLeft = sqlite3WhereExprUsage(pMaskSet, pExpr->pLeft); op = pExpr->op; if( op==TK_IN ){ assert( pExpr->pRight==0 ); - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( sqlite3ExprCheckIN(pParse, pExpr) ) return; + if( ExprUseXSelect(pExpr) ){ pTerm->prereqRight = exprSelectUsage(pMaskSet, pExpr->x.pSelect); }else{ pTerm->prereqRight = sqlite3WhereExprListUsage(pMaskSet, pExpr->x.pList); } - }else if( op==TK_ISNULL ){ - pTerm->prereqRight = 0; + prereqAll = prereqLeft | pTerm->prereqRight; }else{ pTerm->prereqRight = sqlite3WhereExprUsage(pMaskSet, pExpr->pRight); + if( pExpr->pLeft==0 + || ExprHasProperty(pExpr, EP_xIsSelect|EP_IfNullRow) + || pExpr->x.pList!=0 + ){ + prereqAll = sqlite3WhereExprUsageNN(pMaskSet, pExpr); + }else{ + prereqAll = prereqLeft | pTerm->prereqRight; + } } - prereqAll = sqlite3WhereExprUsage(pMaskSet, pExpr); - if( ExprHasProperty(pExpr, EP_FromJoin) ){ - Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->iRightJoinTable); - prereqAll |= x; - extraRight = x-1; /* ON clause terms may not be used with an index - ** on left table of a LEFT JOIN. Ticket #3015 */ + if( pMaskSet->bVarSelect ) pTerm->wtFlags |= TERM_VARSELECT; + +#ifdef SQLITE_DEBUG + if( prereqAll!=sqlite3WhereExprUsageNN(pMaskSet, pExpr) ){ + printf("\n*** Incorrect prereqAll computed for:\n"); + sqlite3TreeViewExpr(0,pExpr,0); + assert( 0 ); + } +#endif + + if( ExprHasProperty(pExpr, EP_OuterON|EP_InnerON) ){ + Bitmask x = sqlite3WhereGetMask(pMaskSet, pExpr->w.iJoin); + if( ExprHasProperty(pExpr, EP_OuterON) ){ + prereqAll |= x; + extraRight = x-1; /* ON clause terms may not be used with an index + ** on left table of a LEFT JOIN. Ticket #3015 */ + if( (prereqAll>>1)>=x ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } + }else if( (prereqAll>>1)>=x ){ + /* The ON clause of an INNER JOIN references a table to its right. + ** Most other SQL database engines raise an error. But SQLite versions + ** 3.0 through 3.38 just put the ON clause constraint into the WHERE + ** clause and carried on. Beginning with 3.39, raise an error only + ** if there is a RIGHT or FULL JOIN in the query. This makes SQLite + ** more like other systems, and also preserves legacy. */ + if( ALWAYS(pSrc->nSrc>0) && (pSrc->a[0].fg.jointype & JT_LTORJ)!=0 ){ + sqlite3ErrorMsg(pParse, "ON clause references tables to its right"); + return; + } + ExprClearProperty(pExpr, EP_InnerON); + } } pTerm->prereqAll = prereqAll; pTerm->leftCursor = -1; pTerm->iParent = -1; pTerm->eOperator = 0; if( allowedOp(op) ){ - int iCur, iColumn; + int aiCurCol[2]; Expr *pLeft = sqlite3ExprSkipCollate(pExpr->pLeft); Expr *pRight = sqlite3ExprSkipCollate(pExpr->pRight); u16 opMask = (pTerm->prereqRight & prereqLeft)==0 ? WO_ALL : WO_EQUIV; - if( exprMightBeIndexed(pSrc, prereqLeft, pLeft, &iCur, &iColumn) ){ - pTerm->leftCursor = iCur; - pTerm->u.leftColumn = iColumn; + + if( pTerm->u.x.iField>0 ){ + assert( op==TK_IN ); + assert( pLeft->op==TK_VECTOR ); + assert( ExprUseXList(pLeft) ); + pLeft = pLeft->x.pList->a[pTerm->u.x.iField-1].pExpr; + } + + if( exprMightBeIndexed(pSrc, aiCurCol, pLeft, op) ){ + pTerm->leftCursor = aiCurCol[0]; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + pTerm->u.x.leftColumn = aiCurCol[1]; pTerm->eOperator = operatorMask(op) & opMask; } if( op==TK_IS ) pTerm->wtFlags |= TERM_IS; - if( pRight - && exprMightBeIndexed(pSrc, pTerm->prereqRight, pRight, &iCur, &iColumn) + if( pRight + && exprMightBeIndexed(pSrc, aiCurCol, pRight, op) + && !ExprHasProperty(pRight, EP_FixedCol) ){ WhereTerm *pNew; Expr *pDup; u16 eExtraOp = 0; /* Extra bits for pNew->eOperator */ + assert( pTerm->u.x.iField==0 ); if( pTerm->leftCursor>=0 ){ int idxNew; pDup = sqlite3ExprDup(db, pExpr, 0); @@ -125574,13 +162513,25 @@ static void exprAnalyze( pDup = pExpr; pNew = pTerm; } - exprCommute(pParse, pDup); - pNew->leftCursor = iCur; - pNew->u.leftColumn = iColumn; + pNew->wtFlags |= exprCommute(pParse, pDup); + pNew->leftCursor = aiCurCol[0]; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + pNew->u.x.leftColumn = aiCurCol[1]; testcase( (prereqLeft | extraRight) != prereqLeft ); pNew->prereqRight = prereqLeft | extraRight; pNew->prereqAll = prereqAll; pNew->eOperator = (operatorMask(pDup->op) + eExtraOp) & opMask; + }else + if( op==TK_ISNULL + && !ExprHasProperty(pExpr,EP_OuterON) + && 0==sqlite3ExprCanBeNull(pLeft) + ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + pExpr->op = TK_TRUEFALSE; /* See tag-20230504-1 */ + pExpr->u.zToken = "false"; + ExprSetProperty(pExpr, EP_IsFalse); + pTerm->prereqAll = 0; + pTerm->eOperator = 0; } } @@ -125601,17 +162552,19 @@ static void exprAnalyze( ** BETWEEN term is skipped. */ else if( pExpr->op==TK_BETWEEN && pWC->op==TK_AND ){ - ExprList *pList = pExpr->x.pList; + ExprList *pList; int i; static const u8 ops[] = {TK_GE, TK_LE}; + assert( ExprUseXList(pExpr) ); + pList = pExpr->x.pList; assert( pList!=0 ); assert( pList->nExpr==2 ); for(i=0; i<2; i++){ Expr *pNewExpr; int idxNew; - pNewExpr = sqlite3PExpr(pParse, ops[i], + pNewExpr = sqlite3PExpr(pParse, ops[i], sqlite3ExprDup(db, pExpr->pLeft, 0), - sqlite3ExprDup(db, pList->a[i].pExpr, 0), 0); + sqlite3ExprDup(db, pList->a[i].pExpr, 0)); transferJoinMarkings(pNewExpr, pExpr); idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); testcase( idxNew==0 ); @@ -125632,6 +162585,42 @@ static void exprAnalyze( pTerm = &pWC->a[idxTerm]; } #endif /* SQLITE_OMIT_OR_OPTIMIZATION */ + /* The form "x IS NOT NULL" can sometimes be evaluated more efficiently + ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a + ** virtual term of that form. + ** + ** The virtual term must be tagged with TERM_VNULL. + */ + else if( pExpr->op==TK_NOTNULL ){ + if( pExpr->pLeft->op==TK_COLUMN + && pExpr->pLeft->iColumn>=0 + && !ExprHasProperty(pExpr, EP_OuterON) + ){ + Expr *pNewExpr; + Expr *pLeft = pExpr->pLeft; + int idxNew; + WhereTerm *pNewTerm; + + pNewExpr = sqlite3PExpr(pParse, TK_GT, + sqlite3ExprDup(db, pLeft, 0), + sqlite3ExprAlloc(db, TK_NULL, 0, 0)); + + idxNew = whereClauseInsert(pWC, pNewExpr, + TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); + if( idxNew ){ + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = 0; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.x.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_GT; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + } + } + #ifndef SQLITE_OMIT_LIKE_OPTIMIZATION /* Add constraints to reduce the search space on a LIKE or GLOB @@ -125647,7 +162636,8 @@ static void exprAnalyze( ** bound is made all lowercase so that the bounds also work when comparing ** BLOBs. */ - if( pWC->op==TK_AND + else if( pExpr->op==TK_FUNCTION + && pWC->op==TK_AND && isLikeOrGlob(pParse, pExpr, &pStr1, &isComplete, &noCase) ){ Expr *pLeft; /* LHS of LIKE/GLOB operator */ @@ -125659,8 +162649,12 @@ static void exprAnalyze( const char *zCollSeqName; /* Name of collating sequence */ const u16 wtFlags = TERM_LIKEOPT | TERM_VIRTUAL | TERM_DYNAMIC; + assert( ExprUseXList(pExpr) ); pLeft = pExpr->x.pList->a[1].pExpr; pStr2 = sqlite3ExprDup(db, pStr1, 0); + assert( pStr1==0 || !ExprHasProperty(pStr1, EP_IntValue) ); + assert( pStr2==0 || !ExprHasProperty(pStr2, EP_IntValue) ); + /* Convert the lower bound to upper-case and the upper bound to ** lower-case (upper-case is less than lower-case in ASCII) so that @@ -125683,7 +162677,7 @@ static void exprAnalyze( if( noCase ){ /* The point is to increment the last character before the first ** wildcard. But if we increment '@', that will push it into the - ** alphabetic range where case conversions will mess up the + ** alphabetic range where case conversions will mess up the ** inequality. To avoid this, make sure to also run the full ** LIKE on all candidate expressions by clearing the isComplete flag */ @@ -125692,22 +162686,22 @@ static void exprAnalyze( } *pC = c + 1; } - zCollSeqName = noCase ? "NOCASE" : "BINARY"; + zCollSeqName = noCase ? "NOCASE" : sqlite3StrBINARY; pNewExpr1 = sqlite3ExprDup(db, pLeft, 0); pNewExpr1 = sqlite3PExpr(pParse, TK_GE, sqlite3ExprAddCollateString(pParse,pNewExpr1,zCollSeqName), - pStr1, 0); + pStr1); transferJoinMarkings(pNewExpr1, pExpr); idxNew1 = whereClauseInsert(pWC, pNewExpr1, wtFlags); testcase( idxNew1==0 ); - exprAnalyze(pSrc, pWC, idxNew1); pNewExpr2 = sqlite3ExprDup(db, pLeft, 0); pNewExpr2 = sqlite3PExpr(pParse, TK_LT, sqlite3ExprAddCollateString(pParse,pNewExpr2,zCollSeqName), - pStr2, 0); + pStr2); transferJoinMarkings(pNewExpr2, pExpr); idxNew2 = whereClauseInsert(pWC, pNewExpr2, wtFlags); testcase( idxNew2==0 ); + exprAnalyze(pSrc, pWC, idxNew1); exprAnalyze(pSrc, pWC, idxNew2); pTerm = &pWC->a[idxTerm]; if( isComplete ){ @@ -125717,84 +162711,120 @@ static void exprAnalyze( } #endif /* SQLITE_OMIT_LIKE_OPTIMIZATION */ + /* If there is a vector == or IS term - e.g. "(a, b) == (?, ?)" - create + ** new terms for each component comparison - "a = ?" and "b = ?". The + ** new terms completely replace the original vector comparison, which is + ** no longer used. + ** + ** This is only required if at least one side of the comparison operation + ** is not a sub-select. + ** + ** tag-20220128a + */ + if( (pExpr->op==TK_EQ || pExpr->op==TK_IS) + && (nLeft = sqlite3ExprVectorSize(pExpr->pLeft))>1 + && sqlite3ExprVectorSize(pExpr->pRight)==nLeft + && ( (pExpr->pLeft->flags & EP_xIsSelect)==0 + || (pExpr->pRight->flags & EP_xIsSelect)==0) + && pWC->op==TK_AND + ){ + int i; + for(i=0; ipLeft, i, nLeft); + Expr *pRight = sqlite3ExprForVectorField(pParse, pExpr->pRight, i, nLeft); + + pNew = sqlite3PExpr(pParse, pExpr->op, pLeft, pRight); + transferJoinMarkings(pNew, pExpr); + idxNew = whereClauseInsert(pWC, pNew, TERM_DYNAMIC|TERM_SLICE); + exprAnalyze(pSrc, pWC, idxNew); + } + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_CODED|TERM_VIRTUAL; /* Disable the original */ + pTerm->eOperator = WO_ROWVAL; + } + + /* If there is a vector IN term - e.g. "(a, b) IN (SELECT ...)" - create + ** a virtual term for each vector component. The expression object + ** used by each such virtual term is pExpr (the full vector IN(...) + ** expression). The WhereTerm.u.x.iField variable identifies the index within + ** the vector on the LHS that the virtual term represents. + ** + ** This only works if the RHS is a simple SELECT (not a compound) that does + ** not use window functions. + */ + else if( pExpr->op==TK_IN + && pTerm->u.x.iField==0 + && pExpr->pLeft->op==TK_VECTOR + && ALWAYS( ExprUseXSelect(pExpr) ) + && (pExpr->x.pSelect->pPrior==0 || (pExpr->x.pSelect->selFlags & SF_Values)) +#ifndef SQLITE_OMIT_WINDOWFUNC + && pExpr->x.pSelect->pWin==0 +#endif + && pWC->op==TK_AND + ){ + int i; + for(i=0; ipLeft); i++){ + int idxNew; + idxNew = whereClauseInsert(pWC, pExpr, TERM_VIRTUAL|TERM_SLICE); + pWC->a[idxNew].u.x.iField = i+1; + exprAnalyze(pSrc, pWC, idxNew); + markTermAsChild(pWC, idxNew, idxTerm); + } + } + #ifndef SQLITE_OMIT_VIRTUALTABLE - /* Add a WO_MATCH auxiliary term to the constraint set if the - ** current expression is of the form: column MATCH expr. + /* Add a WO_AUX auxiliary term to the constraint set if the + ** current expression is of the form "column OP expr" where OP + ** is an operator that gets passed into virtual tables but which is + ** not normally optimized for ordinary tables. In other words, OP + ** is one of MATCH, LIKE, GLOB, REGEXP, !=, IS, IS NOT, or NOT NULL. ** This information is used by the xBestIndex methods of ** virtual tables. The native query optimizer does not attempt ** to do anything with MATCH functions. */ - if( isMatchOfColumn(pExpr, &eOp2) ){ - int idxNew; - Expr *pRight, *pLeft; - WhereTerm *pNewTerm; - Bitmask prereqColumn, prereqExpr; + else if( pWC->op==TK_AND ){ + Expr *pRight = 0, *pLeft = 0; + int res = isAuxiliaryVtabOperator(db, pExpr, &eOp2, &pLeft, &pRight); + while( res-- > 0 ){ + int idxNew; + WhereTerm *pNewTerm; + Bitmask prereqColumn, prereqExpr; - pRight = pExpr->x.pList->a[0].pExpr; - pLeft = pExpr->x.pList->a[1].pExpr; - prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); - prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); - if( (prereqExpr & prereqColumn)==0 ){ - Expr *pNewExpr; - pNewExpr = sqlite3PExpr(pParse, TK_MATCH, - 0, sqlite3ExprDup(db, pRight, 0), 0); - idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); - testcase( idxNew==0 ); - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = prereqExpr; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_MATCH; - pNewTerm->eMatchOp = eOp2; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; + prereqExpr = sqlite3WhereExprUsage(pMaskSet, pRight); + prereqColumn = sqlite3WhereExprUsage(pMaskSet, pLeft); + if( (prereqExpr & prereqColumn)==0 ){ + Expr *pNewExpr; + pNewExpr = sqlite3PExpr(pParse, TK_MATCH, + 0, sqlite3ExprDup(db, pRight, 0)); + if( ExprHasProperty(pExpr, EP_OuterON) && pNewExpr ){ + ExprSetProperty(pNewExpr, EP_OuterON); + pNewExpr->w.iJoin = pExpr->w.iJoin; + } + idxNew = whereClauseInsert(pWC, pNewExpr, TERM_VIRTUAL|TERM_DYNAMIC); + testcase( idxNew==0 ); + pNewTerm = &pWC->a[idxNew]; + pNewTerm->prereqRight = prereqExpr; + pNewTerm->leftCursor = pLeft->iTable; + pNewTerm->u.x.leftColumn = pLeft->iColumn; + pNewTerm->eOperator = WO_AUX; + pNewTerm->eMatchOp = eOp2; + markTermAsChild(pWC, idxNew, idxTerm); + pTerm = &pWC->a[idxTerm]; + pTerm->wtFlags |= TERM_COPIED; + pNewTerm->prereqAll = pTerm->prereqAll; + } + SWAP(Expr*, pLeft, pRight); } } #endif /* SQLITE_OMIT_VIRTUALTABLE */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 - /* When sqlite_stat3 histogram data is available an operator of the - ** form "x IS NOT NULL" can sometimes be evaluated more efficiently - ** as "x>NULL" if x is not an INTEGER PRIMARY KEY. So construct a - ** virtual term of that form. - ** - ** Note that the virtual term must be tagged with TERM_VNULL. - */ - if( pExpr->op==TK_NOTNULL - && pExpr->pLeft->op==TK_COLUMN - && pExpr->pLeft->iColumn>=0 - && OptimizationEnabled(db, SQLITE_Stat34) - ){ - Expr *pNewExpr; - Expr *pLeft = pExpr->pLeft; - int idxNew; - WhereTerm *pNewTerm; - - pNewExpr = sqlite3PExpr(pParse, TK_GT, - sqlite3ExprDup(db, pLeft, 0), - sqlite3PExpr(pParse, TK_NULL, 0, 0, 0), 0); - - idxNew = whereClauseInsert(pWC, pNewExpr, - TERM_VIRTUAL|TERM_DYNAMIC|TERM_VNULL); - if( idxNew ){ - pNewTerm = &pWC->a[idxNew]; - pNewTerm->prereqRight = 0; - pNewTerm->leftCursor = pLeft->iTable; - pNewTerm->u.leftColumn = pLeft->iColumn; - pNewTerm->eOperator = WO_GT; - markTermAsChild(pWC, idxNew, idxTerm); - pTerm = &pWC->a[idxTerm]; - pTerm->wtFlags |= TERM_COPIED; - pNewTerm->prereqAll = pTerm->prereqAll; - } - } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ - /* Prevent ON clause terms of a LEFT JOIN from being used to drive ** an index for tables to the left of the join. */ + testcase( pTerm!=&pWC->a[idxTerm] ); + pTerm = &pWC->a[idxTerm]; pTerm->prereqRight |= extraRight; } @@ -125821,8 +162851,9 @@ static void exprAnalyze( ** all terms of the WHERE clause. */ SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ - Expr *pE2 = sqlite3ExprSkipCollate(pExpr); + Expr *pE2 = sqlite3ExprSkipCollateAndLikely(pExpr); pWC->op = op; + assert( pE2!=0 || pExpr==0 ); if( pE2==0 ) return; if( pE2->op!=op ){ whereClauseInsert(pWC, pExpr, 0); @@ -125832,6 +162863,123 @@ SQLITE_PRIVATE void sqlite3WhereSplit(WhereClause *pWC, Expr *pExpr, u8 op){ } } +/* +** Add either a LIMIT (if eMatchOp==SQLITE_INDEX_CONSTRAINT_LIMIT) or +** OFFSET (if eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET) term to the +** where-clause passed as the first argument. The value for the term +** is found in register iReg. +** +** In the common case where the value is a simple integer +** (example: "LIMIT 5 OFFSET 10") then the expression codes as a +** TK_INTEGER so that it will be available to sqlite3_vtab_rhs_value(). +** If not, then it codes as a TK_REGISTER expression. +*/ +static void whereAddLimitExpr( + WhereClause *pWC, /* Add the constraint to this WHERE clause */ + int iReg, /* Register that will hold value of the limit/offset */ + Expr *pExpr, /* Expression that defines the limit/offset */ + int iCsr, /* Cursor to which the constraint applies */ + int eMatchOp /* SQLITE_INDEX_CONSTRAINT_LIMIT or _OFFSET */ +){ + Parse *pParse = pWC->pWInfo->pParse; + sqlite3 *db = pParse->db; + Expr *pNew; + int iVal = 0; + + if( sqlite3ExprIsInteger(pExpr, &iVal, pParse) && iVal>=0 ){ + Expr *pVal = sqlite3Expr(db, TK_INTEGER, 0); + if( pVal==0 ) return; + ExprSetProperty(pVal, EP_IntValue); + pVal->u.iValue = iVal; + pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); + }else{ + Expr *pVal = sqlite3Expr(db, TK_REGISTER, 0); + if( pVal==0 ) return; + pVal->iTable = iReg; + pNew = sqlite3PExpr(pParse, TK_MATCH, 0, pVal); + } + if( pNew ){ + WhereTerm *pTerm; + int idx; + idx = whereClauseInsert(pWC, pNew, TERM_DYNAMIC|TERM_VIRTUAL); + pTerm = &pWC->a[idx]; + pTerm->leftCursor = iCsr; + pTerm->eOperator = WO_AUX; + pTerm->eMatchOp = eMatchOp; + } +} + +/* +** Possibly add terms corresponding to the LIMIT and OFFSET clauses of the +** SELECT statement passed as the second argument. These terms are only +** added if: +** +** 1. The SELECT statement has a LIMIT clause, and +** 2. The SELECT statement is not an aggregate or DISTINCT query, and +** 3. The SELECT statement has exactly one object in its from clause, and +** that object is a virtual table, and +** 4. There are no terms in the WHERE clause that will not be passed +** to the virtual table xBestIndex method. +** 5. The ORDER BY clause, if any, will be made available to the xBestIndex +** method. +** +** LIMIT and OFFSET terms are ignored by most of the planner code. They +** exist only so that they may be passed to the xBestIndex method of the +** single virtual table in the FROM clause of the SELECT. +*/ +SQLITE_PRIVATE void SQLITE_NOINLINE sqlite3WhereAddLimit(WhereClause *pWC, Select *p){ + assert( p!=0 && p->pLimit!=0 ); /* 1 -- checked by caller */ + if( p->pGroupBy==0 + && (p->selFlags & (SF_Distinct|SF_Aggregate))==0 /* 2 */ + && (p->pSrc->nSrc==1 && IsVirtual(p->pSrc->a[0].pSTab)) /* 3 */ + ){ + ExprList *pOrderBy = p->pOrderBy; + int iCsr = p->pSrc->a[0].iCursor; + int ii; + + /* Check condition (4). Return early if it is not met. */ + for(ii=0; iinTerm; ii++){ + if( pWC->a[ii].wtFlags & TERM_CODED ){ + /* This term is a vector operation that has been decomposed into + ** other, subsequent terms. It can be ignored. See tag-20220128a */ + assert( pWC->a[ii].wtFlags & TERM_VIRTUAL ); + assert( pWC->a[ii].eOperator==WO_ROWVAL ); + continue; + } + if( pWC->a[ii].nChild ){ + /* If this term has child terms, then they are also part of the + ** pWC->a[] array. So this term can be ignored, as a LIMIT clause + ** will only be added if each of the child terms passes the + ** (leftCursor==iCsr) test below. */ + continue; + } + if( pWC->a[ii].leftCursor!=iCsr ) return; + if( pWC->a[ii].prereqRight!=0 ) return; + } + + /* Check condition (5). Return early if it is not met. */ + if( pOrderBy ){ + for(ii=0; iinExpr; ii++){ + Expr *pExpr = pOrderBy->a[ii].pExpr; + if( pExpr->op!=TK_COLUMN ) return; + if( pExpr->iTable!=iCsr ) return; + if( pOrderBy->a[ii].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) return; + } + } + + /* All conditions are met. Add the terms to the where-clause object. */ + assert( p->pLimit->op==TK_LIMIT ); + if( p->iOffset!=0 && (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iOffset, p->pLimit->pRight, + iCsr, SQLITE_INDEX_CONSTRAINT_OFFSET); + } + if( p->iOffset==0 || (p->selFlags & SF_Compound)==0 ){ + whereAddLimitExpr(pWC, p->iLimit, p->pLimit->pLeft, + iCsr, SQLITE_INDEX_CONSTRAINT_LIMIT); + } + } +} + /* ** Initialize a preallocated WhereClause structure. */ @@ -125840,8 +162988,10 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( WhereInfo *pWInfo /* The WHERE processing context */ ){ pWC->pWInfo = pWInfo; + pWC->hasOr = 0; pWC->pOuter = 0; pWC->nTerm = 0; + pWC->nBase = 0; pWC->nSlot = ArraySize(pWC->aStatic); pWC->a = pWC->aStatic; } @@ -125852,22 +163002,36 @@ SQLITE_PRIVATE void sqlite3WhereClauseInit( ** sqlite3WhereClauseInit(). */ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ - int i; - WhereTerm *a; sqlite3 *db = pWC->pWInfo->pParse->db; - for(i=pWC->nTerm-1, a=pWC->a; i>=0; i--, a++){ - if( a->wtFlags & TERM_DYNAMIC ){ - sqlite3ExprDelete(db, a->pExpr); + assert( pWC->nTerm>=pWC->nBase ); + if( pWC->nTerm>0 ){ + WhereTerm *a = pWC->a; + WhereTerm *aLast = &pWC->a[pWC->nTerm-1]; +#ifdef SQLITE_DEBUG + int i; + /* Verify that every term past pWC->nBase is virtual */ + for(i=pWC->nBase; inTerm; i++){ + assert( (pWC->a[i].wtFlags & TERM_VIRTUAL)!=0 ); } - if( a->wtFlags & TERM_ORINFO ){ - whereOrInfoDelete(db, a->u.pOrInfo); - }else if( a->wtFlags & TERM_ANDINFO ){ - whereAndInfoDelete(db, a->u.pAndInfo); +#endif + while(1){ + assert( a->eMatchOp==0 || a->eOperator==WO_AUX ); + if( a->wtFlags & TERM_DYNAMIC ){ + sqlite3ExprDelete(db, a->pExpr); + } + if( a->wtFlags & (TERM_ORINFO|TERM_ANDINFO) ){ + if( a->wtFlags & TERM_ORINFO ){ + assert( (a->wtFlags & TERM_ANDINFO)==0 ); + whereOrInfoDelete(db, a->u.pOrInfo); + }else{ + assert( (a->wtFlags & TERM_ANDINFO)!=0 ); + whereAndInfoDelete(db, a->u.pAndInfo); + } + } + if( a==aLast ) break; + a++; } } - if( pWC->a!=pWC->aStatic ){ - sqlite3DbFree(db, pWC->a); - } } @@ -125875,23 +163039,71 @@ SQLITE_PRIVATE void sqlite3WhereClauseClear(WhereClause *pWC){ ** These routines walk (recursively) an expression tree and generate ** a bitmask indicating which tables are used in that expression ** tree. +** +** sqlite3WhereExprUsage(MaskSet, Expr) -> +** +** Return a Bitmask of all tables referenced by Expr. Expr can be +** be NULL, in which case 0 is returned. +** +** sqlite3WhereExprUsageNN(MaskSet, Expr) -> +** +** Same as sqlite3WhereExprUsage() except that Expr must not be +** NULL. The "NN" suffix on the name stands for "Not Null". +** +** sqlite3WhereExprListUsage(MaskSet, ExprList) -> +** +** Return a Bitmask of all tables referenced by every expression +** in the expression list ExprList. ExprList can be NULL, in which +** case 0 is returned. +** +** sqlite3WhereExprUsageFull(MaskSet, ExprList) -> +** +** Internal use only. Called only by sqlite3WhereExprUsageNN() for +** complex expressions that require pushing register values onto +** the stack. Many calls to sqlite3WhereExprUsageNN() do not need +** the more complex analysis done by this routine. Hence, the +** computations done by this routine are broken out into a separate +** "no-inline" function to avoid the stack push overhead in the +** common case where it is not needed. */ -SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ - Bitmask mask = 0; - if( p==0 ) return 0; - if( p->op==TK_COLUMN ){ - mask = sqlite3WhereGetMask(pMaskSet, p->iTable); - return mask; - } - mask = sqlite3WhereExprUsage(pMaskSet, p->pRight); - if( p->pLeft ) mask |= sqlite3WhereExprUsage(pMaskSet, p->pLeft); - if( ExprHasProperty(p, EP_xIsSelect) ){ +static SQLITE_NOINLINE Bitmask sqlite3WhereExprUsageFull( + WhereMaskSet *pMaskSet, + Expr *p +){ + Bitmask mask; + mask = (p->op==TK_IF_NULL_ROW) ? sqlite3WhereGetMask(pMaskSet, p->iTable) : 0; + if( p->pLeft ) mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pLeft); + if( p->pRight ){ + mask |= sqlite3WhereExprUsageNN(pMaskSet, p->pRight); + assert( p->x.pList==0 ); + }else if( ExprUseXSelect(p) ){ + if( ExprHasProperty(p, EP_VarSelect) ) pMaskSet->bVarSelect = 1; mask |= exprSelectUsage(pMaskSet, p->x.pSelect); }else if( p->x.pList ){ mask |= sqlite3WhereExprListUsage(pMaskSet, p->x.pList); } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( (p->op==TK_FUNCTION || p->op==TK_AGG_FUNCTION) && ExprUseYWin(p) ){ + assert( p->y.pWin!=0 ); + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pPartition); + mask |= sqlite3WhereExprListUsage(pMaskSet, p->y.pWin->pOrderBy); + mask |= sqlite3WhereExprUsage(pMaskSet, p->y.pWin->pFilter); + } +#endif return mask; } +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsageNN(WhereMaskSet *pMaskSet, Expr *p){ + if( p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ + return sqlite3WhereGetMask(pMaskSet, p->iTable); + }else if( ExprHasProperty(p, EP_TokenOnly|EP_Leaf) ){ + assert( p->op!=TK_IF_NULL_ROW ); + return 0; + } + return sqlite3WhereExprUsageFull(pMaskSet, p); +} +SQLITE_PRIVATE Bitmask sqlite3WhereExprUsage(WhereMaskSet *pMaskSet, Expr *p){ + return p ? sqlite3WhereExprUsageNN(pMaskSet,p) : 0; +} SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprList *pList){ int i; Bitmask mask = 0; @@ -125905,7 +163117,7 @@ SQLITE_PRIVATE Bitmask sqlite3WhereExprListUsage(WhereMaskSet *pMaskSet, ExprLis /* -** Call exprAnalyze on all terms in a WHERE clause. +** Call exprAnalyze on all terms in a WHERE clause. ** ** Note that exprAnalyze() might add new virtual terms onto the ** end of the WHERE clause. We do not want to analyze these new @@ -125924,14 +163136,14 @@ SQLITE_PRIVATE void sqlite3WhereExprAnalyze( /* ** For table-valued-functions, transform the function arguments into -** new WHERE clause terms. +** new WHERE clause terms. ** ** Each function argument translates into an equality constraint against ** a HIDDEN column in the table. */ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( Parse *pParse, /* Parsing context */ - struct SrcList_item *pItem, /* The FROM clause term to process */ + SrcItem *pItem, /* The FROM clause term to process */ WhereClause *pWC /* Xfer function arguments to here */ ){ Table *pTab; @@ -125940,24 +163152,38 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( Expr *pColRef; Expr *pTerm; if( pItem->fg.isTabFunc==0 ) return; - pTab = pItem->pTab; + pTab = pItem->pSTab; assert( pTab!=0 ); pArgs = pItem->u1.pFuncArg; if( pArgs==0 ) return; for(j=k=0; jnExpr; j++){ + Expr *pRhs; + u32 joinType; while( knCol && (pTab->aCol[k].colFlags & COLFLAG_HIDDEN)==0 ){k++;} if( k>=pTab->nCol ){ sqlite3ErrorMsg(pParse, "too many arguments on %s() - max %d", pTab->zName, j); return; } - pColRef = sqlite3PExpr(pParse, TK_COLUMN, 0, 0, 0); + pColRef = sqlite3ExprAlloc(pParse->db, TK_COLUMN, 0, 0); if( pColRef==0 ) return; pColRef->iTable = pItem->iCursor; pColRef->iColumn = k++; - pColRef->pTab = pTab; - pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, - sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + assert( ExprUseYTab(pColRef) ); + pColRef->y.pTab = pTab; + pItem->colUsed |= sqlite3ExprColUsed(pColRef); + pRhs = sqlite3PExpr(pParse, TK_UPLUS, + sqlite3ExprDup(pParse->db, pArgs->a[j].pExpr, 0), 0); + pTerm = sqlite3PExpr(pParse, TK_EQ, pColRef, pRhs); + if( pItem->fg.jointype & (JT_LEFT|JT_RIGHT) ){ + testcase( pItem->fg.jointype & JT_LEFT ); /* testtag-20230227a */ + testcase( pItem->fg.jointype & JT_RIGHT ); /* testtag-20230227b */ + joinType = EP_OuterON; + }else{ + testcase( pItem->fg.jointype & JT_LTORJ ); /* testtag-20230227c */ + joinType = EP_InnerON; + } + sqlite3SetJoinExpr(pTerm, pItem->iCursor, joinType); whereClauseInsert(pWC, pTerm, TERM_DYNAMIC); } } @@ -125985,15 +163211,30 @@ SQLITE_PRIVATE void sqlite3WhereTabFuncArgs( /* #include "sqliteInt.h" */ /* #include "whereInt.h" */ +/* +** Extra information appended to the end of sqlite3_index_info but not +** visible to the xBestIndex function, at least not directly. The +** sqlite3_vtab_collation() interface knows how to reach it, however. +** +** This object is not an API and can be changed from one release to the +** next. As long as allocateIndexInfo() and sqlite3_vtab_collation() +** agree on the structure, all will be well. +*/ +typedef struct HiddenIndexInfo HiddenIndexInfo; +struct HiddenIndexInfo { + WhereClause *pWC; /* The Where clause being analyzed */ + Parse *pParse; /* The parsing context */ + int eDistinct; /* Value to return from sqlite3_vtab_distinct() */ + u32 mIn; /* Mask of terms that are IN (...) */ + u32 mHandleIn; /* Terms that vtab will handle as IN (...) */ + sqlite3_value *aRhs[1]; /* RHS values for constraints. MUST BE LAST + ** because extra space is allocated to hold up + ** to nTerm such values */ +}; + /* Forward declaration of methods */ static int whereLoopResize(sqlite3*, WhereLoop*, int); -/* Test variable that can be set to enable WHERE tracing */ -#if defined(SQLITE_TEST) || defined(SQLITE_DEBUG) -/***/ int sqlite3WhereTrace = 0; -#endif - - /* ** Return the estimated number of output rows from a WHERE clause */ @@ -126010,11 +163251,76 @@ SQLITE_PRIVATE int sqlite3WhereIsDistinct(WhereInfo *pWInfo){ } /* -** Return TRUE if the WHERE clause returns rows in ORDER BY order. -** Return FALSE if the output needs to be sorted. +** Return the number of ORDER BY terms that are satisfied by the +** WHERE clause. A return of 0 means that the output must be +** completely sorted. A return equal to the number of ORDER BY +** terms means that no sorting is needed at all. A return that +** is positive but less than the number of ORDER BY terms means that +** block sorting is required. */ SQLITE_PRIVATE int sqlite3WhereIsOrdered(WhereInfo *pWInfo){ - return pWInfo->nOBSat; + return pWInfo->nOBSat<0 ? 0 : pWInfo->nOBSat; +} + +/* +** In the ORDER BY LIMIT optimization, if the inner-most loop is known +** to emit rows in increasing order, and if the last row emitted by the +** inner-most loop did not fit within the sorter, then we can skip all +** subsequent rows for the current iteration of the inner loop (because they +** will not fit in the sorter either) and continue with the second inner +** loop - the loop immediately outside the inner-most. +** +** When a row does not fit in the sorter (because the sorter already +** holds LIMIT+OFFSET rows that are smaller), then a jump is made to the +** label returned by this function. +** +** If the ORDER BY LIMIT optimization applies, the jump destination should +** be the continuation for the second-inner-most loop. If the ORDER BY +** LIMIT optimization does not apply, then the jump destination should +** be the continuation for the inner-most loop. +** +** It is always safe for this routine to return the continuation of the +** inner-most loop, in the sense that a correct answer will result. +** Returning the continuation the second inner loop is an optimization +** that might make the code run a little faster, but should not change +** the final answer. +*/ +SQLITE_PRIVATE int sqlite3WhereOrderByLimitOptLabel(WhereInfo *pWInfo){ + WhereLevel *pInner; + if( !pWInfo->bOrderedInnerLoop ){ + /* The ORDER BY LIMIT optimization does not apply. Jump to the + ** continuation of the inner-most loop. */ + return pWInfo->iContinue; + } + pInner = &pWInfo->a[pWInfo->nLevel-1]; + assert( pInner->addrNxt!=0 ); + return pInner->pRJ ? pWInfo->iContinue : pInner->addrNxt; +} + +/* +** While generating code for the min/max optimization, after handling +** the aggregate-step call to min() or max(), check to see if any +** additional looping is required. If the output order is such that +** we are certain that the correct answer has already been found, then +** code an OP_Goto to by pass subsequent processing. +** +** Any extra OP_Goto that is coded here is an optimization. The +** correct answer should be obtained regardless. This OP_Goto just +** makes the answer appear faster. +*/ +SQLITE_PRIVATE void sqlite3WhereMinMaxOptEarlyOut(Vdbe *v, WhereInfo *pWInfo){ + WhereLevel *pInner; + int i; + if( !pWInfo->bOrderedInnerLoop ) return; + if( pWInfo->nOBSat==0 ) return; + for(i=pWInfo->nLevel-1; i>=0; i--){ + pInner = &pWInfo->a[i]; + if( (pInner->pWLoop->wsFlags & WHERE_COLUMN_IN)!=0 ){ + sqlite3VdbeGoto(v, pInner->addrNxt); + return; + } + } + sqlite3VdbeGoto(v, pWInfo->iBreak); } /* @@ -126036,10 +163342,10 @@ SQLITE_PRIVATE int sqlite3WhereBreakLabel(WhereInfo *pWInfo){ /* ** Return ONEPASS_OFF (0) if an UPDATE or DELETE statement is unable to -** operate directly on the rowis returned by a WHERE clause. Return +** operate directly on the rowids returned by a WHERE clause. Return ** ONEPASS_SINGLE (1) if the statement can operation directly because only ** a single row is to be changed. Return ONEPASS_MULTI (2) if the one-pass -** optimization can be used on multiple +** optimization can be used on multiple ** ** If the ONEPASS optimization is used (if this routine returns true) ** then also write the indices of open cursors used by ONEPASS @@ -126063,6 +163369,14 @@ SQLITE_PRIVATE int sqlite3WhereOkOnePass(WhereInfo *pWInfo, int *aiCur){ return pWInfo->eOnePass; } +/* +** Return TRUE if the WHERE loop uses the OP_DeferredSeek opcode to move +** the data cursor to the row selected by the index cursor. +*/ +SQLITE_PRIVATE int sqlite3WhereUsesDeferredSeek(WhereInfo *pWInfo){ + return pWInfo->bDeferredSeek; +} + /* ** Move the content of pSrc into pDest */ @@ -126118,7 +163432,12 @@ whereOrInsert_done: SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ int i; assert( pMaskSet->n<=(int)sizeof(Bitmask)*8 ); - for(i=0; in; i++){ + assert( pMaskSet->n>0 || pMaskSet->ix[0]<0 ); + assert( iCursor>=-1 ); + if( pMaskSet->ix[0]==iCursor ){ + return 1; + } + for(i=1; in; i++){ if( pMaskSet->ix[i]==iCursor ){ return MASKBIT(i); } @@ -126126,6 +163445,30 @@ SQLITE_PRIVATE Bitmask sqlite3WhereGetMask(WhereMaskSet *pMaskSet, int iCursor){ return 0; } +/* Allocate memory that is automatically freed when pWInfo is freed. +*/ +SQLITE_PRIVATE void *sqlite3WhereMalloc(WhereInfo *pWInfo, u64 nByte){ + WhereMemBlock *pBlock; + pBlock = sqlite3DbMallocRawNN(pWInfo->pParse->db, nByte+sizeof(*pBlock)); + if( pBlock ){ + pBlock->pNext = pWInfo->pMemToFree; + pBlock->sz = nByte; + pWInfo->pMemToFree = pBlock; + pBlock++; + } + return (void*)pBlock; +} +SQLITE_PRIVATE void *sqlite3WhereRealloc(WhereInfo *pWInfo, void *pOld, u64 nByte){ + void *pNew = sqlite3WhereMalloc(pWInfo, nByte); + if( pNew && pOld ){ + WhereMemBlock *pOldBlk = (WhereMemBlock*)pOld; + pOldBlk--; + assert( pOldBlk->szsz); + } + return pNew; +} + /* ** Create a new mask for cursor iCursor. ** @@ -126139,6 +163482,54 @@ static void createMask(WhereMaskSet *pMaskSet, int iCursor){ pMaskSet->ix[pMaskSet->n++] = iCursor; } +/* +** If the right-hand branch of the expression is a TK_COLUMN, then return +** a pointer to the right-hand branch. Otherwise, return NULL. +*/ +static Expr *whereRightSubexprIsColumn(Expr *p){ + p = sqlite3ExprSkipCollateAndLikely(p->pRight); + if( ALWAYS(p!=0) && p->op==TK_COLUMN && !ExprHasProperty(p, EP_FixedCol) ){ + return p; + } + return 0; +} + +/* +** Term pTerm is guaranteed to be a WO_IN term. It may be a component term +** of a vector IN expression of the form "(x, y, ...) IN (SELECT ...)". +** This function checks to see if the term is compatible with an index +** column with affinity idxaff (one of the SQLITE_AFF_XYZ values). If so, +** it returns a pointer to the name of the collation sequence (e.g. "BINARY" +** or "NOCASE") used by the comparison in pTerm. If it is not compatible +** with affinity idxaff, NULL is returned. +*/ +static SQLITE_NOINLINE const char *indexInAffinityOk( + Parse *pParse, + WhereTerm *pTerm, + u8 idxaff +){ + Expr *pX = pTerm->pExpr; + Expr inexpr; + + assert( pTerm->eOperator & WO_IN ); + + if( sqlite3ExprIsVector(pX->pLeft) ){ + int iField = pTerm->u.x.iField - 1; + inexpr.flags = 0; + inexpr.op = TK_EQ; + inexpr.pLeft = pX->pLeft->x.pList->a[iField].pExpr; + assert( ExprUseXSelect(pX) ); + inexpr.pRight = pX->x.pSelect->pEList->a[iField].pExpr; + pX = &inexpr; + } + + if( sqlite3IndexAffinityOk(pX, idxaff) ){ + CollSeq *pRet = sqlite3ExprCompareCollSeq(pParse, pX); + return pRet ? pRet->zName : sqlite3StrBINARY; + } + return 0; +} + /* ** Advance to the next WhereTerm that matches according to the criteria ** established when the pScan object was initialized by whereScanInit(). @@ -126152,21 +163543,26 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ WhereTerm *pTerm; /* The term being tested */ int k = pScan->k; /* Where to start scanning */ - while( pScan->iEquiv<=pScan->nEquiv ){ - iCur = pScan->aiCur[pScan->iEquiv-1]; + assert( pScan->iEquiv<=pScan->nEquiv ); + pWC = pScan->pWC; + while(1){ iColumn = pScan->aiColumn[pScan->iEquiv-1]; - if( iColumn==XN_EXPR && pScan->pIdxExpr==0 ) return 0; - while( (pWC = pScan->pWC)!=0 ){ + iCur = pScan->aiCur[pScan->iEquiv-1]; + assert( pWC!=0 ); + assert( iCur>=0 ); + do{ for(pTerm=pWC->a+k; knTerm; k++, pTerm++){ + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 || pTerm->leftCursor<0 ); if( pTerm->leftCursor==iCur - && pTerm->u.leftColumn==iColumn + && pTerm->u.x.leftColumn==iColumn && (iColumn!=XN_EXPR - || sqlite3ExprCompare(pTerm->pExpr->pLeft,pScan->pIdxExpr,iCur)==0) - && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_FromJoin)) + || sqlite3ExprCompareSkip(pTerm->pExpr->pLeft, + pScan->pIdxExpr,iCur)==0) + && (pScan->iEquiv<=1 || !ExprHasProperty(pTerm->pExpr, EP_OuterON)) ){ if( (pTerm->eOperator & WO_EQUIV)!=0 && pScan->nEquivaiCur) - && (pX = sqlite3ExprSkipCollate(pTerm->pExpr->pRight))->op==TK_COLUMN + && (pX = whereRightSubexprIsColumn(pTerm->pExpr))!=0 ){ int j; for(j=0; jnEquiv; j++){ @@ -126184,43 +163580,76 @@ static WhereTerm *whereScanNext(WhereScan *pScan){ if( (pTerm->eOperator & pScan->opMask)!=0 ){ /* Verify the affinity and collating sequence match */ if( pScan->zCollName && (pTerm->eOperator & WO_ISNULL)==0 ){ - CollSeq *pColl; + const char *zCollName; Parse *pParse = pWC->pWInfo->pParse; pX = pTerm->pExpr; - if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ - continue; + + if( (pTerm->eOperator & WO_IN) ){ + zCollName = indexInAffinityOk(pParse, pTerm, pScan->idxaff); + if( !zCollName ) continue; + }else{ + CollSeq *pColl; + if( !sqlite3IndexAffinityOk(pX, pScan->idxaff) ){ + continue; + } + assert(pX->pLeft); + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + zCollName = pColl ? pColl->zName : sqlite3StrBINARY; } - assert(pX->pLeft); - pColl = sqlite3BinaryCompareCollSeq(pParse, - pX->pLeft, pX->pRight); - if( pColl==0 ) pColl = pParse->db->pDfltColl; - if( sqlite3StrICmp(pColl->zName, pScan->zCollName) ){ + + if( sqlite3StrICmp(zCollName, pScan->zCollName) ){ continue; } } if( (pTerm->eOperator & (WO_EQ|WO_IS))!=0 - && (pX = pTerm->pExpr->pRight)->op==TK_COLUMN + && (pX = pTerm->pExpr->pRight, ALWAYS(pX!=0)) + && pX->op==TK_COLUMN && pX->iTable==pScan->aiCur[0] && pX->iColumn==pScan->aiColumn[0] ){ testcase( pTerm->eOperator & WO_IS ); continue; } + pScan->pWC = pWC; pScan->k = k+1; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x20000 ){ + int ii; + sqlite3DebugPrintf("SCAN-TERM %p: nEquiv=%d", + pTerm, pScan->nEquiv); + for(ii=0; iinEquiv; ii++){ + sqlite3DebugPrintf(" {%d:%d}", + pScan->aiCur[ii], pScan->aiColumn[ii]); + } + sqlite3DebugPrintf("\n"); + } +#endif return pTerm; } } } - pScan->pWC = pScan->pWC->pOuter; + pWC = pWC->pOuter; k = 0; - } - pScan->pWC = pScan->pOrigWC; + }while( pWC!=0 ); + if( pScan->iEquiv>=pScan->nEquiv ) break; + pWC = pScan->pOrigWC; k = 0; pScan->iEquiv++; } return 0; } +/* +** This is whereScanInit() for the case of an index on an expression. +** It is factored out into a separate tail-recursion subroutine so that +** the normal whereScanInit() routine, which is a high-runner, does not +** need to push registers onto the stack as part of its prologue. +*/ +static SQLITE_NOINLINE WhereTerm *whereScanInitIndexExpr(WhereScan *pScan){ + pScan->idxaff = sqlite3ExprAffinity(pScan->pIdxExpr); + return whereScanNext(pScan); +} + /* ** Initialize a WHERE clause scanner object. Return a pointer to the ** first match. Return NULL if there are no matches. @@ -126248,31 +163677,34 @@ static WhereTerm *whereScanInit( u32 opMask, /* Operator(s) to scan for */ Index *pIdx /* Must be compatible with this index */ ){ - int j = 0; - - /* memset(pScan, 0, sizeof(*pScan)); */ pScan->pOrigWC = pWC; pScan->pWC = pWC; pScan->pIdxExpr = 0; - if( pIdx ){ - j = iColumn; - iColumn = pIdx->aiColumn[j]; - if( iColumn==XN_EXPR ) pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; - if( iColumn==pIdx->pTable->iPKey ) iColumn = XN_ROWID; - } - if( pIdx && iColumn>=0 ){ - pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; - pScan->zCollName = pIdx->azColl[j]; - }else{ - pScan->idxaff = 0; - pScan->zCollName = 0; - } + pScan->idxaff = 0; + pScan->zCollName = 0; pScan->opMask = opMask; pScan->k = 0; pScan->aiCur[0] = iCur; - pScan->aiColumn[0] = iColumn; pScan->nEquiv = 1; pScan->iEquiv = 1; + if( pIdx ){ + int j = iColumn; + iColumn = pIdx->aiColumn[j]; + if( iColumn==pIdx->pTable->iPKey ){ + iColumn = XN_ROWID; + }else if( iColumn>=0 ){ + pScan->idxaff = pIdx->pTable->aCol[iColumn].affinity; + pScan->zCollName = pIdx->azColl[j]; + }else if( iColumn==XN_EXPR ){ + pScan->pIdxExpr = pIdx->aColExpr->a[j].pExpr; + pScan->zCollName = pIdx->azColl[j]; + pScan->aiColumn[0] = XN_EXPR; + return whereScanInitIndexExpr(pScan); + } + }else if( iColumn==XN_EXPR ){ + return 0; + } + pScan->aiColumn[0] = iColumn; return whereScanNext(pScan); } @@ -126282,7 +163714,7 @@ static WhereTerm *whereScanInit( ** if pIdx!=0 and is one of the WO_xx operator codes specified by ** the op parameter. Return a pointer to the term. Return 0 if not found. ** -** If pIdx!=0 then it must be one of the indexes of table iCur. +** If pIdx!=0 then it must be one of the indexes of table iCur. ** Search for terms matching the iColumn-th column of pIdx ** rather than the iColumn-th column of table iCur. ** @@ -126346,13 +163778,14 @@ static int findIndexCol( const char *zColl = pIdx->azColl[iCol]; for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pList->a[i].pExpr); - if( p->op==TK_COLUMN + Expr *p = sqlite3ExprSkipCollateAndLikely(pList->a[i].pExpr); + if( ALWAYS(p!=0) + && (p->op==TK_COLUMN || p->op==TK_AGG_COLUMN) && p->iColumn==pIdx->aiColumn[iCol] && p->iTable==iBase ){ - CollSeq *pColl = sqlite3ExprCollSeq(pParse, pList->a[i].pExpr); - if( pColl && 0==sqlite3StrICmp(pColl->zName, zColl) ){ + CollSeq *pColl = sqlite3ExprNNCollSeq(pParse, pList->a[i].pExpr); + if( 0==sqlite3StrICmp(pColl->zName, zColl) ){ return i; } } @@ -126395,23 +163828,25 @@ static int isDistinctRedundant( ){ Table *pTab; Index *pIdx; - int i; + int i; int iBase; /* If there is more than one table or sub-select in the FROM clause of - ** this query, then it will not be possible to show that the DISTINCT + ** this query, then it will not be possible to show that the DISTINCT ** clause is redundant. */ if( pTabList->nSrc!=1 ) return 0; iBase = pTabList->a[0].iCursor; - pTab = pTabList->a[0].pTab; + pTab = pTabList->a[0].pSTab; - /* If any of the expressions is an IPK column on table iBase, then return + /* If any of the expressions is an IPK column on table iBase, then return ** true. Note: The (p->iTable==iBase) part of this test may be false if the ** current SELECT is a correlated sub-query. */ for(i=0; inExpr; i++){ - Expr *p = sqlite3ExprSkipCollate(pDistinct->a[i].pExpr); - if( p->op==TK_COLUMN && p->iTable==iBase && p->iColumn<0 ) return 1; + Expr *p = sqlite3ExprSkipCollateAndLikely(pDistinct->a[i].pExpr); + if( NEVER(p==0) ) continue; + if( p->op!=TK_COLUMN && p->op!=TK_AGG_COLUMN ) continue; + if( p->iTable==iBase && p->iColumn<0 ) return 1; } /* Loop through all indices on the table, checking each to see if it makes @@ -126429,6 +163864,7 @@ static int isDistinctRedundant( */ for(pIdx=pTab->pIndex; pIdx; pIdx=pIdx->pNext){ if( !IsUniqueIndex(pIdx) ) continue; + if( pIdx->pPartIdxWhere ) continue; for(i=0; inKeyCol; i++){ if( 0==sqlite3WhereFindTerm(pWC, iBase, i, ~(Bitmask)0, WO_EQ, pIdx) ){ if( findIndexCol(pParse, pDistinct, iBase, pIdx, i)<0 ) break; @@ -126456,41 +163892,58 @@ static LogEst estLog(LogEst N){ ** Convert OP_Column opcodes to OP_Copy in previously generated code. ** ** This routine runs over generated VDBE code and translates OP_Column -** opcodes into OP_Copy when the table is being accessed via co-routine +** opcodes into OP_Copy when the table is being accessed via co-routine ** instead of via table lookup. ** -** If the bIncrRowid parameter is 0, then any OP_Rowid instructions on -** cursor iTabCur are transformed into OP_Null. Or, if bIncrRowid is non-zero, -** then each OP_Rowid is transformed into an instruction to increment the -** value stored in its output register. +** If the iAutoidxCur is not zero, then any OP_Rowid instructions on +** cursor iTabCur are transformed into OP_Sequence opcode for the +** iAutoidxCur cursor, in order to generate unique rowids for the +** automatic index being generated. */ static void translateColumnToCopy( - Vdbe *v, /* The VDBE containing code to translate */ + Parse *pParse, /* Parsing context */ int iStart, /* Translate from this opcode to the end */ int iTabCur, /* OP_Column/OP_Rowid references to this table */ int iRegister, /* The first column is in this register */ - int bIncrRowid /* If non-zero, transform OP_rowid to OP_AddImm(1) */ + int iAutoidxCur /* If non-zero, cursor of autoindex being generated */ ){ + Vdbe *v = pParse->pVdbe; VdbeOp *pOp = sqlite3VdbeGetOp(v, iStart); int iEnd = sqlite3VdbeCurrentAddr(v); + if( pParse->db->mallocFailed ) return; +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("CHECKING for column-to-copy on cursor %d for %d..%d\n", + iTabCur, iStart, iEnd); + } +#endif for(; iStartp1!=iTabCur ) continue; if( pOp->opcode==OP_Column ){ +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE OP_Column to OP_Copy at %d\n", iStart); + } +#endif pOp->opcode = OP_Copy; pOp->p1 = pOp->p2 + iRegister; pOp->p2 = pOp->p3; pOp->p3 = 0; + pOp->p5 = 2; /* Cause the MEM_Subtype flag to be cleared */ }else if( pOp->opcode==OP_Rowid ){ - if( bIncrRowid ){ - /* Increment the value stored in the P2 operand of the OP_Rowid. */ - pOp->opcode = OP_AddImm; - pOp->p1 = pOp->p2; - pOp->p2 = 1; - }else{ +#ifdef SQLITE_DEBUG + if( pParse->db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE OP_Rowid to OP_Sequence at %d\n", iStart); + } +#endif + pOp->opcode = OP_Sequence; + pOp->p1 = iAutoidxCur; +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( iAutoidxCur==0 ){ pOp->opcode = OP_Null; - pOp->p1 = 0; pOp->p3 = 0; } +#endif } } } @@ -126502,16 +163955,22 @@ static void translateColumnToCopy( ** are no-ops. */ #if !defined(SQLITE_OMIT_VIRTUALTABLE) && defined(WHERETRACE_ENABLED) -static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ +static void whereTraceIndexInfoInputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info inputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ - sqlite3DebugPrintf(" constraint[%d]: col=%d termid=%d op=%d usabled=%d\n", + sqlite3DebugPrintf( + " constraint[%d]: col=%d termid=%d op=%d usabled=%d collseq=%s\n", i, p->aConstraint[i].iColumn, p->aConstraint[i].iTermOffset, p->aConstraint[i].op, - p->aConstraint[i].usable); + p->aConstraint[i].usable, + sqlite3_vtab_collation(p,i)); } for(i=0; inOrderBy; i++){ sqlite3DebugPrintf(" orderby[%d]: col=%d desc=%d\n", @@ -126520,9 +163979,13 @@ static void TRACE_IDX_INPUTS(sqlite3_index_info *p){ p->aOrderBy[i].desc); } } -static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ +static void whereTraceIndexInfoOutputs( + sqlite3_index_info *p, /* The IndexInfo object */ + Table *pTab /* The TABLE that is the virtual table */ +){ int i; - if( !sqlite3WhereTrace ) return; + if( (sqlite3WhereTrace & 0x10)==0 ) return; + sqlite3DebugPrintf("sqlite3_index_info outputs for %s:\n", pTab->zName); for(i=0; inConstraint; i++){ sqlite3DebugPrintf(" usage[%d]: argvIdx=%d omit=%d\n", i, @@ -126536,10 +163999,81 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ sqlite3DebugPrintf(" estimatedRows=%lld\n", p->estimatedRows); } #else -#define TRACE_IDX_INPUTS(A) -#define TRACE_IDX_OUTPUTS(A) +#define whereTraceIndexInfoInputs(A,B) +#define whereTraceIndexInfoOutputs(A,B) #endif +/* +** We know that pSrc is an operand of an outer join. Return true if +** pTerm is a constraint that is compatible with that join. +** +** pTerm must be EP_OuterON if pSrc is the right operand of an +** outer join. pTerm can be either EP_OuterON or EP_InnerON if pSrc +** is the left operand of a RIGHT join. +** +** See https://sqlite.org/forum/forumpost/206d99a16dd9212f +** for an example of a WHERE clause constraints that may not be used on +** the right table of a RIGHT JOIN because the constraint implies a +** not-NULL condition on the left table of the RIGHT JOIN. +*/ +static int constraintCompatibleWithOuterJoin( + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc /* Table we are trying to access */ +){ + assert( (pSrc->fg.jointype&(JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 ); /* By caller */ + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LEFT ); + testcase( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))==JT_LTORJ ); + testcase( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) + testcase( ExprHasProperty(pTerm->pExpr, EP_InnerON) ); + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON|EP_InnerON) + || pTerm->pExpr->w.iJoin != pSrc->iCursor + ){ + return 0; + } + if( (pSrc->fg.jointype & (JT_LEFT|JT_RIGHT))!=0 + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + ){ + return 0; + } + return 1; +} + +#ifndef SQLITE_OMIT_AUTOMATIC_INDEX +/* +** Return true if column iCol of table pTab seem like it might be a +** good column to use as part of a query-time index. +** +** Current algorithm (subject to improvement!): +** +** 1. If iCol is already the left-most column of some other index, +** then return false. +** +** 2. If iCol is part of an existing index that has an aiRowLogEst of +** more than 20, then return false. +** +** 3. If no disqualifying conditions above are found, return true. +*/ +static SQLITE_NOINLINE int columnIsGoodIndexCandidate( + const Table *pTab, + int iCol +){ + const Index *pIdx; + for(pIdx = pTab->pIndex; pIdx!=0; pIdx=pIdx->pNext){ + int j; + for(j=0; jnKeyCol; j++){ + if( pIdx->aiColumn[j]==iCol ){ + if( j==0 ) return 0; + if( pIdx->hasStat1 && pIdx->aiRowLogEst[j+1]>20 ) return 0; + break; + } + } + } + return 1; +} +#endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ + + + #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* ** Return TRUE if the WHERE clause term pTerm is of a form where it @@ -126547,34 +164081,94 @@ static void TRACE_IDX_OUTPUTS(sqlite3_index_info *p){ ** index existed. */ static int termCanDriveIndex( - WhereTerm *pTerm, /* WHERE clause term to check */ - struct SrcList_item *pSrc, /* Table we are trying to access */ - Bitmask notReady /* Tables in outer loops of the join */ + const WhereTerm *pTerm, /* WHERE clause term to check */ + const SrcItem *pSrc, /* Table we are trying to access */ + const Bitmask notReady /* Tables in outer loops of the join */ ){ char aff; + int leftCol; + if( pTerm->leftCursor!=pSrc->iCursor ) return 0; if( (pTerm->eOperator & (WO_EQ|WO_IS))==0 ) return 0; + assert( (pSrc->fg.jointype & JT_RIGHT)==0 ); + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + return 0; /* See https://sqlite.org/forum/forumpost/51e6959f61 */ + } if( (pTerm->prereqRight & notReady)!=0 ) return 0; - if( pTerm->u.leftColumn<0 ) return 0; - aff = pSrc->pTab->aCol[pTerm->u.leftColumn].affinity; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + leftCol = pTerm->u.x.leftColumn; + if( leftCol<0 ) return 0; + aff = pSrc->pSTab->aCol[leftCol].affinity; if( !sqlite3IndexAffinityOk(pTerm->pExpr, aff) ) return 0; testcase( pTerm->pExpr->op==TK_IS ); - return 1; + return columnIsGoodIndexCandidate(pSrc->pSTab, leftCol); } #endif #ifndef SQLITE_OMIT_AUTOMATIC_INDEX + +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS +/* +** Argument pIdx represents an automatic index that the current statement +** will create and populate. Add an OP_Explain with text of the form: +** +** CREATE AUTOMATIC INDEX ON
            () [WHERE ] +** +** This is only required if sqlite3_stmt_scanstatus() is enabled, to +** associate an SQLITE_SCANSTAT_NCYCLE and SQLITE_SCANSTAT_NLOOP +** values with. In order to avoid breaking legacy code and test cases, +** the OP_Explain is not added if this is an EXPLAIN QUERY PLAN command. +*/ +static void explainAutomaticIndex( + Parse *pParse, + Index *pIdx, /* Automatic index to explain */ + int bPartial, /* True if pIdx is a partial index */ + int *pAddrExplain /* OUT: Address of OP_Explain */ +){ + if( IS_STMT_SCANSTATUS(pParse->db) && pParse->explain!=2 ){ + Table *pTab = pIdx->pTable; + const char *zSep = ""; + char *zText = 0; + int ii = 0; + sqlite3_str *pStr = sqlite3_str_new(pParse->db); + sqlite3_str_appendf(pStr,"CREATE AUTOMATIC INDEX ON %s(", pTab->zName); + assert( pIdx->nColumn>1 ); + assert( pIdx->aiColumn[pIdx->nColumn-1]==XN_ROWID ); + for(ii=0; ii<(pIdx->nColumn-1); ii++){ + const char *zName = 0; + int iCol = pIdx->aiColumn[ii]; + + zName = pTab->aCol[iCol].zCnName; + sqlite3_str_appendf(pStr, "%s%s", zSep, zName); + zSep = ", "; + } + zText = sqlite3_str_finish(pStr); + if( zText==0 ){ + sqlite3OomFault(pParse->db); + }else{ + *pAddrExplain = sqlite3VdbeExplain( + pParse, 0, "%s)%s", zText, (bPartial ? " WHERE " : "") + ); + sqlite3_free(zText); + } + } +} +#else +# define explainAutomaticIndex(a,b,c,d) +#endif + /* ** Generate code to construct the Index object for an automatic index ** and to set up the WhereLevel object pLevel so that the code generator ** makes use of the automatic index. */ -static void constructAutomaticIndex( +static SQLITE_NOINLINE void constructAutomaticIndex( Parse *pParse, /* The parsing context */ WhereClause *pWC, /* The WHERE clause */ - struct SrcList_item *pSrc, /* The FROM clause term to get the next index */ - Bitmask notReady, /* Mask of cursors that are not available */ + const Bitmask notReady, /* Mask of cursors that are not available */ WhereLevel *pLevel /* Write new index here */ ){ int nKeyCol; /* Number of columns in the constructed index */ @@ -126594,47 +164188,56 @@ static void constructAutomaticIndex( char *zNotUsed; /* Extra space on the end of pIdx */ Bitmask idxCols; /* Bitmap of columns used for indexing */ Bitmask extraCols; /* Bitmap of additional columns */ - u8 sentWarning = 0; /* True if a warnning has been issued */ + u8 sentWarning = 0; /* True if a warning has been issued */ + u8 useBloomFilter = 0; /* True to also add a Bloom filter */ Expr *pPartial = 0; /* Partial Index Expression */ int iContinue = 0; /* Jump here to skip excluded rows */ - struct SrcList_item *pTabItem; /* FROM clause term being indexed */ + SrcList *pTabList; /* The complete FROM clause */ + SrcItem *pSrc; /* The FROM clause term to get the next index */ int addrCounter = 0; /* Address where integer counter is initialized */ int regBase; /* Array of registers where record is assembled */ +#ifdef SQLITE_ENABLE_STMT_SCANSTATUS + int addrExp = 0; /* Address of OP_Explain */ +#endif /* Generate code to skip over the creation and initialization of the ** transient index on 2nd and subsequent iterations of the loop. */ v = pParse->pVdbe; assert( v!=0 ); - addrInit = sqlite3CodeOnce(pParse); VdbeCoverage(v); + addrInit = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); /* Count the number of columns that will be added to the index ** and used to match WHERE clause constraints */ nKeyCol = 0; - pTable = pSrc->pTab; + pTabList = pWC->pWInfo->pTabList; + pSrc = &pTabList->a[pLevel->iFrom]; + pTable = pSrc->pSTab; pWCEnd = &pWC->a[pWC->nTerm]; pLoop = pLevel->pWLoop; idxCols = 0; for(pTerm=pWC->a; pTermpExpr; - assert( !ExprHasProperty(pExpr, EP_FromJoin) /* prereq always non-zero */ - || pExpr->iRightJoinTable!=pSrc->iCursor /* for the right-hand */ - || pLoop->prereq!=0 ); /* table of a LEFT JOIN */ - if( pLoop->prereq==0 - && (pTerm->wtFlags & TERM_VIRTUAL)==0 - && !ExprHasProperty(pExpr, EP_FromJoin) - && sqlite3ExprIsTableConstant(pExpr, pSrc->iCursor) ){ - pPartial = sqlite3ExprAnd(pParse->db, pPartial, + /* Make the automatic index a partial index if there are terms in the + ** WHERE clause (or the ON clause of a LEFT join) that constrain which + ** rows of the target table (pSrc) that can be used. */ + if( (pTerm->wtFlags & TERM_VIRTUAL)==0 + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, pLevel->iFrom, 0) + ){ + pPartial = sqlite3ExprAnd(pParse, pPartial, sqlite3ExprDup(pParse->db, pExpr, 0)); } if( termCanDriveIndex(pTerm, pSrc, notReady) ){ - int iCol = pTerm->u.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + int iCol; + Bitmask cMask; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + iCol = pTerm->u.x.leftColumn; + cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); testcase( iCol==BMS ); testcase( iCol==BMS-1 ); if( !sentWarning ){ sqlite3_log(SQLITE_WARNING_AUTOINDEX, "automatic index on %s(%s)", pTable->zName, - pTable->aCol[iCol].zName); + pTable->aCol[iCol].zCnName); sentWarning = 1; } if( (idxCols & cMask)==0 ){ @@ -126646,7 +164249,7 @@ static void constructAutomaticIndex( } } } - assert( nKeyCol>0 ); + assert( nKeyCol>0 || pParse->db->mallocFailed ); pLoop->u.btree.nEq = pLoop->nLTerm = nKeyCol; pLoop->wsFlags = WHERE_COLUMN_EQ | WHERE_IDX_ONLY | WHERE_INDEXED | WHERE_AUTO_INDEX; @@ -126659,7 +164262,11 @@ static void constructAutomaticIndex( ** original table changes and the index and table cannot both be used ** if they go out of sync. */ - extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + if( IsView(pTable) ){ + extraCols = ALLBITS & ~idxCols; + }else{ + extraCols = pSrc->colUsed & (~idxCols | MASKBIT(BMS-1)); + } mxBitCol = MIN(BMS-1,pTable->nCol); testcase( pTable->nCol==BMS-1 ); testcase( pTable->nCol==BMS-2 ); @@ -126680,17 +164287,31 @@ static void constructAutomaticIndex( idxCols = 0; for(pTerm=pWC->a; pTermu.leftColumn; - Bitmask cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); + int iCol; + Bitmask cMask; + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + iCol = pTerm->u.x.leftColumn; + cMask = iCol>=BMS ? MASKBIT(BMS-1) : MASKBIT(iCol); testcase( iCol==BMS-1 ); testcase( iCol==BMS ); if( (idxCols & cMask)==0 ){ Expr *pX = pTerm->pExpr; idxCols |= cMask; - pIdx->aiColumn[n] = pTerm->u.leftColumn; - pColl = sqlite3BinaryCompareCollSeq(pParse, pX->pLeft, pX->pRight); + pIdx->aiColumn[n] = pTerm->u.x.leftColumn; + pColl = sqlite3ExprCompareCollSeq(pParse, pX); + assert( pColl!=0 || pParse->nErr>0 ); /* TH3 collate01.800 */ pIdx->azColl[n] = pColl ? pColl->zName : sqlite3StrBINARY; n++; + if( ALWAYS(pX->pLeft!=0) + && sqlite3ExprAffinity(pX->pLeft)!=SQLITE_AFF_TEXT + ){ + /* TUNING: only use a Bloom filter on an automatic index + ** if one or more key columns has the ability to hold numeric + ** values, since strings all have the same hash in the Bloom + ** filter implementation and hence a Bloom filter on a text column + ** is not usually helpful. */ + useBloomFilter = 1; + } } } } @@ -126717,27 +164338,37 @@ static void constructAutomaticIndex( pIdx->azColl[n] = sqlite3StrBINARY; /* Create the automatic index */ + explainAutomaticIndex(pParse, pIdx, pPartial!=0, &addrExp); assert( pLevel->iIdxCur>=0 ); pLevel->iIdxCur = pParse->nTab++; sqlite3VdbeAddOp2(v, OP_OpenAutoindex, pLevel->iIdxCur, nKeyCol+1); sqlite3VdbeSetP4KeyInfo(pParse, pIdx); VdbeComment((v, "for %s", pTable->zName)); + if( OptimizationEnabled(pParse->db, SQLITE_BloomFilter) && useBloomFilter ){ + sqlite3WhereExplainBloomFilter(pParse, pWC->pWInfo, pLevel); + pLevel->regFilter = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Blob, 10000, pLevel->regFilter); + } /* Fill the automatic index with content */ - sqlite3ExprCachePush(pParse); - pTabItem = &pWC->pWInfo->pTabList->a[pLevel->iFrom]; - if( pTabItem->fg.viaCoroutine ){ - int regYield = pTabItem->regReturn; + assert( pSrc == &pWC->pWInfo->pTabList->a[pLevel->iFrom] ); + if( pSrc->fg.viaCoroutine ){ + int regYield; + Subquery *pSubq; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + assert( pSubq!=0 ); + regYield = pSubq->regReturn; addrCounter = sqlite3VdbeAddOp2(v, OP_Integer, 0, 0); - sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pTabItem->addrFillSub); + sqlite3VdbeAddOp3(v, OP_InitCoroutine, regYield, 0, pSubq->addrFillSub); addrTop = sqlite3VdbeAddOp1(v, OP_Yield, regYield); VdbeCoverage(v); - VdbeComment((v, "next row of \"%s\"", pTabItem->pTab->zName)); + VdbeComment((v, "next row of %s", pSrc->pSTab->zName)); }else{ addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, pLevel->iTabCur); VdbeCoverage(v); } if( pPartial ){ - iContinue = sqlite3VdbeMakeLabel(v); + iContinue = sqlite3VdbeMakeLabel(pParse); sqlite3ExprIfFalse(pParse, pPartial, iContinue, SQLITE_JUMPIFNULL); pLoop->wsFlags |= WHERE_PARTIALIDX; } @@ -126745,69 +164376,258 @@ static void constructAutomaticIndex( regBase = sqlite3GenerateIndexKey( pParse, pIdx, pLevel->iTabCur, regRecord, 0, 0, 0, 0 ); + if( pLevel->regFilter ){ + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, + regBase, pLoop->u.btree.nEq); + } + sqlite3VdbeScanStatusCounters(v, addrExp, addrExp, sqlite3VdbeCurrentAddr(v)); sqlite3VdbeAddOp2(v, OP_IdxInsert, pLevel->iIdxCur, regRecord); sqlite3VdbeChangeP5(v, OPFLAG_USESEEKRESULT); if( pPartial ) sqlite3VdbeResolveLabel(v, iContinue); - if( pTabItem->fg.viaCoroutine ){ + if( pSrc->fg.viaCoroutine ){ + assert( pSrc->fg.isSubquery && pSrc->u4.pSubq!=0 ); sqlite3VdbeChangeP2(v, addrCounter, regBase+n); - translateColumnToCopy(v, addrTop, pLevel->iTabCur, pTabItem->regResult, 1); + testcase( pParse->db->mallocFailed ); + assert( pLevel->iIdxCur>0 ); + translateColumnToCopy(pParse, addrTop, pLevel->iTabCur, + pSrc->u4.pSubq->regResult, pLevel->iIdxCur); sqlite3VdbeGoto(v, addrTop); - pTabItem->fg.viaCoroutine = 0; + pSrc->fg.viaCoroutine = 0; }else{ sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); VdbeCoverage(v); + sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); } - sqlite3VdbeChangeP5(v, SQLITE_STMTSTATUS_AUTOINDEX); sqlite3VdbeJumpHere(v, addrTop); sqlite3ReleaseTempReg(pParse, regRecord); - sqlite3ExprCachePop(pParse); - + /* Jump here when skipping the initialization */ sqlite3VdbeJumpHere(v, addrInit); + sqlite3VdbeScanStatusRange(v, addrExp, addrExp, -1); end_auto_index_create: sqlite3ExprDelete(pParse->db, pPartial); } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ +/* +** Generate bytecode that will initialize a Bloom filter that is appropriate +** for pLevel. +** +** If there are inner loops within pLevel that have the WHERE_BLOOMFILTER +** flag set, initialize a Bloomfilter for them as well. Except don't do +** this recursive initialization if the SQLITE_BloomPulldown optimization has +** been turned off. +** +** When the Bloom filter is initialized, the WHERE_BLOOMFILTER flag is cleared +** from the loop, but the regFilter value is set to a register that implements +** the Bloom filter. When regFilter is positive, the +** sqlite3WhereCodeOneLoopStart() will generate code to test the Bloom filter +** and skip the subsequence B-Tree seek if the Bloom filter indicates that +** no matching rows exist. +** +** This routine may only be called if it has previously been determined that +** the loop would benefit from a Bloom filter, and the WHERE_BLOOMFILTER bit +** is set. +*/ +static SQLITE_NOINLINE void sqlite3ConstructBloomFilter( + WhereInfo *pWInfo, /* The WHERE clause */ + int iLevel, /* Index in pWInfo->a[] that is pLevel */ + WhereLevel *pLevel, /* Make a Bloom filter for this FROM term */ + Bitmask notReady /* Loops that are not ready */ +){ + int addrOnce; /* Address of opening OP_Once */ + int addrTop; /* Address of OP_Rewind */ + int addrCont; /* Jump here to skip a row */ + const WhereTerm *pTerm; /* For looping over WHERE clause terms */ + const WhereTerm *pWCEnd; /* Last WHERE clause term */ + Parse *pParse = pWInfo->pParse; /* Parsing context */ + Vdbe *v = pParse->pVdbe; /* VDBE under construction */ + WhereLoop *pLoop = pLevel->pWLoop; /* The loop being coded */ + int iCur; /* Cursor for table getting the filter */ + IndexedExpr *saved_pIdxEpr; /* saved copy of Parse.pIdxEpr */ + IndexedExpr *saved_pIdxPartExpr; /* saved copy of Parse.pIdxPartExpr */ + + saved_pIdxEpr = pParse->pIdxEpr; + saved_pIdxPartExpr = pParse->pIdxPartExpr; + pParse->pIdxEpr = 0; + pParse->pIdxPartExpr = 0; + + assert( pLoop!=0 ); + assert( v!=0 ); + assert( pLoop->wsFlags & WHERE_BLOOMFILTER ); + assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ); + + addrOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + do{ + const SrcList *pTabList; + const SrcItem *pItem; + const Table *pTab; + u64 sz; + int iSrc; + sqlite3WhereExplainBloomFilter(pParse, pWInfo, pLevel); + addrCont = sqlite3VdbeMakeLabel(pParse); + iCur = pLevel->iTabCur; + pLevel->regFilter = ++pParse->nMem; + + /* The Bloom filter is a Blob held in a register. Initialize it + ** to zero-filled blob of at least 80K bits, but maybe more if the + ** estimated size of the table is larger. We could actually + ** measure the size of the table at run-time using OP_Count with + ** P3==1 and use that value to initialize the blob. But that makes + ** testing complicated. By basing the blob size on the value in the + ** sqlite_stat1 table, testing is much easier. + */ + pTabList = pWInfo->pTabList; + iSrc = pLevel->iFrom; + pItem = &pTabList->a[iSrc]; + assert( pItem!=0 ); + pTab = pItem->pSTab; + assert( pTab!=0 ); + sz = sqlite3LogEstToInt(pTab->nRowLogEst); + if( sz<10000 ){ + sz = 10000; + }else if( sz>10000000 ){ + sz = 10000000; + } + sqlite3VdbeAddOp2(v, OP_Blob, (int)sz, pLevel->regFilter); + + addrTop = sqlite3VdbeAddOp1(v, OP_Rewind, iCur); VdbeCoverage(v); + pWCEnd = &pWInfo->sWC.a[pWInfo->sWC.nTerm]; + for(pTerm=pWInfo->sWC.a; pTermpExpr; + if( (pTerm->wtFlags & TERM_VIRTUAL)==0 + && sqlite3ExprIsSingleTableConstraint(pExpr, pTabList, iSrc, 0) + ){ + sqlite3ExprIfFalse(pParse, pTerm->pExpr, addrCont, SQLITE_JUMPIFNULL); + } + } + if( pLoop->wsFlags & WHERE_IPK ){ + int r1 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Rowid, iCur, r1); + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, 1); + sqlite3ReleaseTempReg(pParse, r1); + }else{ + Index *pIdx = pLoop->u.btree.pIndex; + int n = pLoop->u.btree.nEq; + int r1 = sqlite3GetTempRange(pParse, n); + int jj; + for(jj=0; jjpTable==pItem->pSTab ); + sqlite3ExprCodeLoadIndexColumn(pParse, pIdx, iCur, jj, r1+jj); + } + sqlite3VdbeAddOp4Int(v, OP_FilterAdd, pLevel->regFilter, 0, r1, n); + sqlite3ReleaseTempRange(pParse, r1, n); + } + sqlite3VdbeResolveLabel(v, addrCont); + sqlite3VdbeAddOp2(v, OP_Next, pLevel->iTabCur, addrTop+1); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrTop); + pLoop->wsFlags &= ~WHERE_BLOOMFILTER; + if( OptimizationDisabled(pParse->db, SQLITE_BloomPulldown) ) break; + while( ++iLevel < pWInfo->nLevel ){ + const SrcItem *pTabItem; + pLevel = &pWInfo->a[iLevel]; + pTabItem = &pWInfo->pTabList->a[pLevel->iFrom]; + if( pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ) ) continue; + pLoop = pLevel->pWLoop; + if( NEVER(pLoop==0) ) continue; + if( pLoop->prereq & notReady ) continue; + if( (pLoop->wsFlags & (WHERE_BLOOMFILTER|WHERE_COLUMN_IN)) + ==WHERE_BLOOMFILTER + ){ + /* This is a candidate for bloom-filter pull-down (early evaluation). + ** The test that WHERE_COLUMN_IN is omitted is important, as we are + ** not able to do early evaluation of bloom filters that make use of + ** the IN operator */ + break; + } + } + }while( iLevel < pWInfo->nLevel ); + sqlite3VdbeJumpHere(v, addrOnce); + pParse->pIdxEpr = saved_pIdxEpr; + pParse->pIdxPartExpr = saved_pIdxPartExpr; +} + + #ifndef SQLITE_OMIT_VIRTUALTABLE /* -** Allocate and populate an sqlite3_index_info structure. It is the +** Return term iTerm of the WhereClause passed as the first argument. Terms +** are numbered from 0 upwards, starting with the terms in pWC->a[], then +** those in pWC->pOuter->a[] (if any), and so on. +*/ +static WhereTerm *termFromWhereClause(WhereClause *pWC, int iTerm){ + WhereClause *p; + for(p=pWC; p; p=p->pOuter){ + if( iTermnTerm ) return &p->a[iTerm]; + iTerm -= p->nTerm; + } + return 0; +} + +/* +** Allocate and populate an sqlite3_index_info structure. It is the ** responsibility of the caller to eventually release the structure -** by passing the pointer returned by this function to sqlite3_free(). +** by passing the pointer returned by this function to freeIndexInfo(). */ static sqlite3_index_info *allocateIndexInfo( - Parse *pParse, - WhereClause *pWC, + WhereInfo *pWInfo, /* The WHERE clause */ + WhereClause *pWC, /* The WHERE clause being analyzed */ Bitmask mUnusable, /* Ignore terms with these prereqs */ - struct SrcList_item *pSrc, - ExprList *pOrderBy + SrcItem *pSrc, /* The FROM clause term that is the vtab */ + u16 *pmNoOmit /* Mask of terms not to omit */ ){ int i, j; int nTerm; + Parse *pParse = pWInfo->pParse; struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_orderby *pIdxOrderBy; struct sqlite3_index_constraint_usage *pUsage; + struct HiddenIndexInfo *pHidden; WhereTerm *pTerm; int nOrderBy; sqlite3_index_info *pIdxInfo; + u16 mNoOmit = 0; + const Table *pTab; + int eDistinct = 0; + ExprList *pOrderBy = pWInfo->pOrderBy; + WhereClause *p; - /* Count the number of possible WHERE clause constraints referring - ** to this virtual table */ - for(i=nTerm=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - assert( pTerm->u.leftColumn>=(-1) ); - nTerm++; + assert( pSrc!=0 ); + pTab = pSrc->pSTab; + assert( pTab!=0 ); + assert( IsVirtual(pTab) ); + + /* Find all WHERE clause constraints referring to this virtual table. + ** Mark each term with the TERM_OK flag. Set nTerm to the number of + ** terms found. + */ + for(p=pWC, nTerm=0; p; p=p->pOuter){ + for(i=0, pTerm=p->a; inTerm; i++, pTerm++){ + pTerm->wtFlags &= ~TERM_OK; + if( pTerm->leftCursor != pSrc->iCursor ) continue; + if( pTerm->prereqRight & mUnusable ) continue; + assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); + testcase( pTerm->eOperator & WO_IN ); + testcase( pTerm->eOperator & WO_ISNULL ); + testcase( pTerm->eOperator & WO_IS ); + testcase( pTerm->eOperator & WO_ALL ); + if( (pTerm->eOperator & ~(WO_EQUIV))==0 ) continue; + if( pTerm->wtFlags & TERM_VNULL ) continue; + + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + assert( pTerm->u.x.leftColumn>=XN_ROWID ); + assert( pTerm->u.x.leftColumnnCol ); + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) + ){ + continue; + } + nTerm++; + pTerm->wtFlags |= TERM_OK; + } } - /* If the ORDER BY clause contains only columns in the current + /* If the ORDER BY clause contains only columns in the current ** virtual table then allocate space for the aOrderBy part of ** the sqlite3_index_info structure. */ @@ -126816,10 +164636,49 @@ static sqlite3_index_info *allocateIndexInfo( int n = pOrderBy->nExpr; for(i=0; ia[i].pExpr; - if( pExpr->op!=TK_COLUMN || pExpr->iTable!=pSrc->iCursor ) break; + Expr *pE2; + + /* Skip over constant terms in the ORDER BY clause */ + if( sqlite3ExprIsConstant(0, pExpr) ){ + continue; + } + + /* Virtual tables are unable to deal with NULLS FIRST */ + if( pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL ) break; + + /* First case - a direct column references without a COLLATE operator */ + if( pExpr->op==TK_COLUMN && pExpr->iTable==pSrc->iCursor ){ + assert( pExpr->iColumn>=XN_ROWID && pExpr->iColumnnCol ); + continue; + } + + /* 2nd case - a column reference with a COLLATE operator. Only match + ** of the COLLATE operator matches the collation of the column. */ + if( pExpr->op==TK_COLLATE + && (pE2 = pExpr->pLeft)->op==TK_COLUMN + && pE2->iTable==pSrc->iCursor + ){ + const char *zColl; /* The collating sequence name */ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + assert( pExpr->u.zToken!=0 ); + assert( pE2->iColumn>=XN_ROWID && pE2->iColumnnCol ); + pExpr->iColumn = pE2->iColumn; + if( pE2->iColumn<0 ) continue; /* Collseq does not matter for rowid */ + zColl = sqlite3ColumnColl(&pTab->aCol[pE2->iColumn]); + if( zColl==0 ) zColl = sqlite3StrBINARY; + if( sqlite3_stricmp(pExpr->u.zToken, zColl)==0 ) continue; + } + + /* No matches cause a break out of the loop */ + break; } - if( i==n){ + if( i==n ){ nOrderBy = n; + if( (pWInfo->wctrlFlags & WHERE_DISTINCTBY) && !pSrc->fg.rowidUsed ){ + eDistinct = 2 + ((pWInfo->wctrlFlags & WHERE_SORTBYGROUP)!=0); + }else if( pWInfo->wctrlFlags & WHERE_GROUPBY ){ + eDistinct = 1; + } } } @@ -126827,91 +164686,160 @@ static sqlite3_index_info *allocateIndexInfo( */ pIdxInfo = sqlite3DbMallocZero(pParse->db, sizeof(*pIdxInfo) + (sizeof(*pIdxCons) + sizeof(*pUsage))*nTerm - + sizeof(*pIdxOrderBy)*nOrderBy ); + + sizeof(*pIdxOrderBy)*nOrderBy + sizeof(*pHidden) + + sizeof(sqlite3_value*)*nTerm ); if( pIdxInfo==0 ){ sqlite3ErrorMsg(pParse, "out of memory"); return 0; } - - /* Initialize the structure. The sqlite3_index_info structure contains - ** many fields that are declared "const" to prevent xBestIndex from - ** changing them. We have to do some funky casting in order to - ** initialize those fields. - */ - pIdxCons = (struct sqlite3_index_constraint*)&pIdxInfo[1]; + pHidden = (struct HiddenIndexInfo*)&pIdxInfo[1]; + pIdxCons = (struct sqlite3_index_constraint*)&pHidden->aRhs[nTerm]; pIdxOrderBy = (struct sqlite3_index_orderby*)&pIdxCons[nTerm]; pUsage = (struct sqlite3_index_constraint_usage*)&pIdxOrderBy[nOrderBy]; - *(int*)&pIdxInfo->nConstraint = nTerm; - *(int*)&pIdxInfo->nOrderBy = nOrderBy; - *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint = pIdxCons; - *(struct sqlite3_index_orderby**)&pIdxInfo->aOrderBy = pIdxOrderBy; - *(struct sqlite3_index_constraint_usage**)&pIdxInfo->aConstraintUsage = - pUsage; - - for(i=j=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - u8 op; - if( pTerm->leftCursor != pSrc->iCursor ) continue; - if( pTerm->prereqRight & mUnusable ) continue; - assert( IsPowerOfTwo(pTerm->eOperator & ~WO_EQUIV) ); - testcase( pTerm->eOperator & WO_IN ); - testcase( pTerm->eOperator & WO_IS ); - testcase( pTerm->eOperator & WO_ISNULL ); - testcase( pTerm->eOperator & WO_ALL ); - if( (pTerm->eOperator & ~(WO_ISNULL|WO_EQUIV|WO_IS))==0 ) continue; - if( pTerm->wtFlags & TERM_VNULL ) continue; - assert( pTerm->u.leftColumn>=(-1) ); - pIdxCons[j].iColumn = pTerm->u.leftColumn; - pIdxCons[j].iTermOffset = i; - op = (u8)pTerm->eOperator & WO_ALL; - if( op==WO_IN ) op = WO_EQ; - if( op==WO_MATCH ){ - op = pTerm->eMatchOp; + pIdxInfo->aConstraint = pIdxCons; + pIdxInfo->aOrderBy = pIdxOrderBy; + pIdxInfo->aConstraintUsage = pUsage; + pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + if( HasRowid(pTab)==0 ){ + /* Ensure that all bits associated with PK columns are set. This is to + ** ensure they are available for cases like RIGHT joins or OR loops. */ + Index *pPk = sqlite3PrimaryKeyIndex((Table*)pTab); + assert( pPk!=0 ); + for(i=0; inKeyCol; i++){ + int iCol = pPk->aiColumn[i]; + assert( iCol>=0 ); + if( iCol>=BMS-1 ) iCol = BMS-1; + pIdxInfo->colUsed |= MASKBIT(iCol); } - pIdxCons[j].op = op; - /* The direct assignment in the previous line is possible only because - ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The - ** following asserts verify this fact. */ - assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); - assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); - assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); - assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); - assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); - assert( WO_MATCH==SQLITE_INDEX_CONSTRAINT_MATCH ); - assert( pTerm->eOperator & (WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_MATCH) ); + } + pHidden->pWC = pWC; + pHidden->pParse = pParse; + pHidden->eDistinct = eDistinct; + pHidden->mIn = 0; + for(p=pWC, i=j=0; p; p=p->pOuter){ + int nLast = i+p->nTerm;; + for(pTerm=p->a; iwtFlags & TERM_OK)==0 ) continue; + pIdxCons[j].iColumn = pTerm->u.x.leftColumn; + pIdxCons[j].iTermOffset = i; + op = pTerm->eOperator & WO_ALL; + if( op==WO_IN ){ + if( (pTerm->wtFlags & TERM_SLICE)==0 ){ + pHidden->mIn |= SMASKBIT32(j); + } + op = WO_EQ; + } + if( op==WO_AUX ){ + pIdxCons[j].op = pTerm->eMatchOp; + }else if( op & (WO_ISNULL|WO_IS) ){ + if( op==WO_ISNULL ){ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_ISNULL; + }else{ + pIdxCons[j].op = SQLITE_INDEX_CONSTRAINT_IS; + } + }else{ + pIdxCons[j].op = (u8)op; + /* The direct assignment in the previous line is possible only because + ** the WO_ and SQLITE_INDEX_CONSTRAINT_ codes are identical. The + ** following asserts verify this fact. */ + assert( WO_EQ==SQLITE_INDEX_CONSTRAINT_EQ ); + assert( WO_LT==SQLITE_INDEX_CONSTRAINT_LT ); + assert( WO_LE==SQLITE_INDEX_CONSTRAINT_LE ); + assert( WO_GT==SQLITE_INDEX_CONSTRAINT_GT ); + assert( WO_GE==SQLITE_INDEX_CONSTRAINT_GE ); + assert( pTerm->eOperator&(WO_IN|WO_EQ|WO_LT|WO_LE|WO_GT|WO_GE|WO_AUX) ); + + if( op & (WO_LT|WO_LE|WO_GT|WO_GE) + && sqlite3ExprIsVector(pTerm->pExpr->pRight) + ){ + testcase( j!=i ); + if( j<16 ) mNoOmit |= (1 << j); + if( op==WO_LT ) pIdxCons[j].op = WO_LE; + if( op==WO_GT ) pIdxCons[j].op = WO_GE; + } + } + + j++; + } + } + assert( j==nTerm ); + pIdxInfo->nConstraint = j; + for(i=j=0; ia[i].pExpr; + if( sqlite3ExprIsConstant(0, pExpr) ) continue; + assert( pExpr->op==TK_COLUMN + || (pExpr->op==TK_COLLATE && pExpr->pLeft->op==TK_COLUMN + && pExpr->iColumn==pExpr->pLeft->iColumn) ); + pIdxOrderBy[j].iColumn = pExpr->iColumn; + pIdxOrderBy[j].desc = pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC; j++; } - for(i=0; ia[i].pExpr; - pIdxOrderBy[i].iColumn = pExpr->iColumn; - pIdxOrderBy[i].desc = pOrderBy->a[i].sortOrder; - } + pIdxInfo->nOrderBy = j; + *pmNoOmit = mNoOmit; return pIdxInfo; } +/* +** Free and zero the sqlite3_index_info.idxStr value if needed. +*/ +static void freeIdxStr(sqlite3_index_info *pIdxInfo){ + if( pIdxInfo->needToFreeIdxStr ){ + sqlite3_free(pIdxInfo->idxStr); + pIdxInfo->idxStr = 0; + pIdxInfo->needToFreeIdxStr = 0; + } +} + +/* +** Free an sqlite3_index_info structure allocated by allocateIndexInfo() +** and possibly modified by xBestIndex methods. +*/ +static void freeIndexInfo(sqlite3 *db, sqlite3_index_info *pIdxInfo){ + HiddenIndexInfo *pHidden; + int i; + assert( pIdxInfo!=0 ); + pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + assert( pHidden->pParse!=0 ); + assert( pHidden->pParse->db==db ); + for(i=0; inConstraint; i++){ + sqlite3ValueFree(pHidden->aRhs[i]); /* IMP: R-14553-25174 */ + pHidden->aRhs[i] = 0; + } + freeIdxStr(pIdxInfo); + sqlite3DbFree(db, pIdxInfo); +} + /* ** The table object reference passed as the second argument to this function ** must represent a virtual table. This function invokes the xBestIndex() ** method of the virtual table with the sqlite3_index_info object that ** comes in as the 3rd argument to this function. ** -** If an error occurs, pParse is populated with an error message and a -** non-zero value is returned. Otherwise, 0 is returned and the output -** part of the sqlite3_index_info structure is left populated. +** If an error occurs, pParse is populated with an error message and an +** appropriate error code is returned. A return of SQLITE_CONSTRAINT from +** xBestIndex is not considered an error. SQLITE_CONSTRAINT indicates that +** the current configuration of "unusable" flags in sqlite3_index_info can +** not result in a valid plan. ** ** Whether or not an error is returned, it is the responsibility of the ** caller to eventually free p->idxStr if p->needToFreeIdxStr indicates ** that this is required. */ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ - sqlite3_vtab *pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; int rc; + sqlite3_vtab *pVtab; - TRACE_IDX_INPUTS(p); + assert( IsVirtual(pTab) ); + pVtab = sqlite3GetVTable(pParse->db, pTab)->pVtab; + whereTraceIndexInfoInputs(p, pTab); + pParse->db->nSchemaLock++; rc = pVtab->pModule->xBestIndex(pVtab, p); - TRACE_IDX_OUTPUTS(p); + pParse->db->nSchemaLock--; + whereTraceIndexInfoOutputs(p, pTab); - if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_OK && rc!=SQLITE_CONSTRAINT ){ if( rc==SQLITE_NOMEM ){ sqlite3OomFault(pParse->db); }else if( !pVtab->zErrMsg ){ @@ -126920,25 +164848,16 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ sqlite3ErrorMsg(pParse, "%s", pVtab->zErrMsg); } } + if( pTab->u.vtab.p->bAllSchemas ){ + sqlite3VtabUsesAllSchemas(pParse); + } sqlite3_free(pVtab->zErrMsg); pVtab->zErrMsg = 0; - -#if 0 - /* This error is now caught by the caller. - ** Search for "xBestIndex malfunction" below */ - for(i=0; inConstraint; i++){ - if( !p->aConstraint[i].usable && p->aConstraintUsage[i].argvIndex>0 ){ - sqlite3ErrorMsg(pParse, - "table %s: xBestIndex returned an invalid plan", pTab->zName); - } - } -#endif - - return pParse->nErr; + return rc; } #endif /* !defined(SQLITE_OMIT_VIRTUALTABLE) */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the location of a particular key among all keys in an ** index. Store the results in aStat as follows: @@ -126949,8 +164868,8 @@ static int vtabBestIndex(Parse *pParse, Table *pTab, sqlite3_index_info *p){ ** Return the index of the sample that is the smallest sample that ** is greater than or equal to pRec. Note that this index is not an index ** into the aSample[] array - it is an index into a virtual set of samples -** based on the contents of aSample[] and the number of fields in record -** pRec. +** based on the contents of aSample[] and the number of fields in record +** pRec. */ static int whereKeyStats( Parse *pParse, /* Database connection */ @@ -126974,7 +164893,8 @@ static int whereKeyStats( #endif assert( pRec!=0 ); assert( pIdx->nSample>0 ); - assert( pRec->nField>0 && pRec->nField<=pIdx->nSampleCol ); + assert( pRec->nField>0 ); + /* Do a binary search to find the first sample greater than or equal ** to pRec. If pRec contains a single field, the set of samples to search @@ -126986,41 +164906,46 @@ static int whereKeyStats( ** consider prefixes of those samples. For example, if the set of samples ** in aSample is: ** - ** aSample[0] = (a, 5) - ** aSample[1] = (a, 10) - ** aSample[2] = (b, 5) - ** aSample[3] = (c, 100) + ** aSample[0] = (a, 5) + ** aSample[1] = (a, 10) + ** aSample[2] = (b, 5) + ** aSample[3] = (c, 100) ** aSample[4] = (c, 105) ** - ** Then the search space should ideally be the samples above and the - ** unique prefixes [a], [b] and [c]. But since that is hard to organize, + ** Then the search space should ideally be the samples above and the + ** unique prefixes [a], [b] and [c]. But since that is hard to organize, ** the code actually searches this set: ** - ** 0: (a) - ** 1: (a, 5) - ** 2: (a, 10) - ** 3: (a, 10) - ** 4: (b) - ** 5: (b, 5) - ** 6: (c) - ** 7: (c, 100) + ** 0: (a) + ** 1: (a, 5) + ** 2: (a, 10) + ** 3: (a, 10) + ** 4: (b) + ** 5: (b, 5) + ** 6: (c) + ** 7: (c, 100) ** 8: (c, 105) ** 9: (c, 105) ** ** For each sample in the aSample[] array, N samples are present in the - ** effective sample array. In the above, samples 0 and 1 are based on + ** effective sample array. In the above, samples 0 and 1 are based on ** sample aSample[0]. Samples 2 and 3 on aSample[1] etc. ** ** Often, sample i of each block of N effective samples has (i+1) fields. ** Except, each sample may be extended to ensure that it is greater than or - ** equal to the previous sample in the array. For example, in the above, - ** sample 2 is the first sample of a block of N samples, so at first it - ** appears that it should be 1 field in size. However, that would make it - ** smaller than sample 1, so the binary search would not work. As a result, - ** it is extended to two fields. The duplicates that this creates do not + ** equal to the previous sample in the array. For example, in the above, + ** sample 2 is the first sample of a block of N samples, so at first it + ** appears that it should be 1 field in size. However, that would make it + ** smaller than sample 1, so the binary search would not work. As a result, + ** it is extended to two fields. The duplicates that this creates do not ** cause any problems. */ - nField = pRec->nField; + if( !HasRowid(pIdx->pTable) && IsPrimaryKeyIndex(pIdx) ){ + nField = pIdx->nKeyCol; + }else{ + nField = pIdx->nColumn; + } + nField = MIN(pRec->nField, nField); iCol = 0; iSample = pIdx->nSample * nField; do{ @@ -127031,7 +164956,7 @@ static int whereKeyStats( iSamp = iTest / nField; if( iSamp>0 ){ /* The proposed effective sample is a prefix of sample aSample[iSamp]. - ** Specifically, the shortest prefix of at least (1 + iTest%nField) + ** Specifically, the shortest prefix of at least (1 + iTest%nField) ** fields that is greater than the previous effective sample. */ for(n=(iTest % nField) + 1; nnSample ); assert( iCol==nField-1 ); pRec->nField = nField; - assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) - || pParse->db->mallocFailed + assert( 0==sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec) + || pParse->db->mallocFailed ); }else{ /* Unless i==pIdx->nSample, indicating that pRec is larger than @@ -127075,7 +165000,7 @@ static int whereKeyStats( ** (iCol+1) field prefix of sample i. */ assert( i<=pIdx->nSample && i>=0 ); pRec->nField = iCol+1; - assert( i==pIdx->nSample + assert( i==pIdx->nSample || sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)>0 || pParse->db->mallocFailed ); @@ -127086,12 +165011,12 @@ static int whereKeyStats( if( iCol>0 ){ pRec->nField = iCol; assert( sqlite3VdbeRecordCompare(aSample[i].n, aSample[i].p, pRec)<=0 - || pParse->db->mallocFailed ); + || pParse->db->mallocFailed || CORRUPT_DB ); } if( i>0 ){ pRec->nField = nField; assert( sqlite3VdbeRecordCompare(aSample[i-1].n, aSample[i-1].p, pRec)<0 - || pParse->db->mallocFailed ); + || pParse->db->mallocFailed || CORRUPT_DB ); } } } @@ -127103,12 +165028,12 @@ static int whereKeyStats( aStat[0] = aSample[i].anLt[iCol]; aStat[1] = aSample[i].anEq[iCol]; }else{ - /* At this point, the (iCol+1) field prefix of aSample[i] is the first + /* At this point, the (iCol+1) field prefix of aSample[i] is the first ** sample that is greater than pRec. Or, if i==pIdx->nSample then pRec ** is larger than all samples in the array. */ tRowcnt iUpper, iGap; if( i>=pIdx->nSample ){ - iUpper = sqlite3LogEstToInt(pIdx->aiRowLogEst[0]); + iUpper = pIdx->nRowEst0; }else{ iUpper = aSample[i].anLt[iCol]; } @@ -127124,18 +165049,18 @@ static int whereKeyStats( iGap = iGap/3; } aStat[0] = iLower + iGap; - aStat[1] = pIdx->aAvgEq[iCol]; + aStat[1] = pIdx->aAvgEq[nField-1]; } /* Restore the pRec->nField value before returning. */ pRec->nField = nField; return i; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** If it is not NULL, pTerm is a term that provides an upper or lower -** bound on a range scan. Without considering pTerm, it is estimated +** bound on a range scan. Without considering pTerm, it is estimated ** that the scan will visit nNew rows. This function returns the number ** estimated to be visited after taking pTerm into account. ** @@ -127157,33 +165082,34 @@ static LogEst whereRangeAdjust(WhereTerm *pTerm, LogEst nNew){ } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Return the affinity for a single column of an index. */ -static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ +SQLITE_PRIVATE char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ assert( iCol>=0 && iColnColumn ); if( !pIdx->zColAff ){ if( sqlite3IndexAffinityStr(db, pIdx)==0 ) return SQLITE_AFF_BLOB; } + assert( pIdx->zColAff[iCol]!=0 ); return pIdx->zColAff[iCol]; } #endif -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 -/* +#ifdef SQLITE_ENABLE_STAT4 +/* ** This function is called to estimate the number of rows visited by a ** range-scan on a skip-scan index. For example: ** ** CREATE INDEX i1 ON t1(a, b, c); ** SELECT * FROM t1 WHERE a=? AND c BETWEEN ? AND ?; ** -** Value pLoop->nOut is currently set to the estimated number of rows -** visited for scanning (a=? AND b=?). This function reduces that estimate +** Value pLoop->nOut is currently set to the estimated number of rows +** visited for scanning (a=? AND b=?). This function reduces that estimate ** by some factor to account for the (c BETWEEN ? AND ?) expression based -** on the stat4 data for the index. this scan will be peformed multiple -** times (once for each (a,b) combination that matches a=?) is dealt with +** on the stat4 data for the index. this scan will be performed multiple +** times (once for each (a,b) combination that matches a=?) is dealt with ** by the caller. ** ** It does this by scanning through all stat4 samples, comparing values @@ -127204,7 +165130,7 @@ static char sqlite3IndexColumnAffinity(sqlite3 *db, Index *pIdx, int iCol){ ** estimate of the number of rows delivered remains unchanged), *pbDone ** is left as is. ** -** If an error occurs, an SQLite error code is returned. Otherwise, +** If an error occurs, an SQLite error code is returned. Otherwise, ** SQLITE_OK. */ static int whereRangeSkipScanEst( @@ -127222,7 +165148,7 @@ static int whereRangeSkipScanEst( int rc = SQLITE_OK; u8 aff = sqlite3IndexColumnAffinity(db, p, nEq); CollSeq *pColl; - + sqlite3_value *p1 = 0; /* Value extracted from pLower */ sqlite3_value *p2 = 0; /* Value extracted from pUpper */ sqlite3_value *pVal = 0; /* Value extracted from record */ @@ -127254,7 +165180,7 @@ static int whereRangeSkipScanEst( nDiff = (nUpper - nLower); if( nDiff<=0 ) nDiff = 1; - /* If there is both an upper and lower bound specified, and the + /* If there is both an upper and lower bound specified, and the ** comparisons indicate that they are close together, use the fallback ** method (assume that the scan visits 1/64 of the rows) for estimating ** the number of rows visited. Otherwise, estimate the number of rows @@ -127263,7 +165189,7 @@ static int whereRangeSkipScanEst( int nAdjust = (sqlite3LogEst(p->nSample) - sqlite3LogEst(nDiff)); pLoop->nOut -= nAdjust; *pbDone = 1; - WHERETRACE(0x10, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", + WHERETRACE(0x20, ("range skip-scan regions: %u..%u adjust=%d est=%d\n", nLower, nUpper, nAdjust*-1, pLoop->nOut)); } @@ -127277,7 +165203,7 @@ static int whereRangeSkipScanEst( return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ /* ** This function is used to estimate the number of rows that will be visited @@ -127301,7 +165227,7 @@ static int whereRangeSkipScanEst( ** ** ... FROM t1 WHERE a = ? AND b > ? AND b < ? ... ** -** then nEq is set to 1 (as the range restricted column, b, is the second +** then nEq is set to 1 (as the range restricted column, b, is the second ** left-most column of the index). Or, if the query is: ** ** ... FROM t1 WHERE a > ? AND a < ? ... @@ -127309,13 +165235,13 @@ static int whereRangeSkipScanEst( ** then nEq is set to 0. ** ** When this function is called, *pnOut is set to the sqlite3LogEst() of the -** number of rows that the index scan is expected to visit without -** considering the range constraints. If nEq is 0, then *pnOut is the number of +** number of rows that the index scan is expected to visit without +** considering the range constraints. If nEq is 0, then *pnOut is the number of ** rows in the index. Assuming no error occurs, *pnOut is adjusted (reduced) ** to account for the range constraints pLower and pUpper. -** +** ** In the absence of sqlite_stat4 ANALYZE data, or if such data cannot be -** used, a single range inequality reduces the search space by a factor of 4. +** used, a single range inequality reduces the search space by a factor of 4. ** and a pair of constraints (x>? AND xnOut; LogEst nNew; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 Index *p = pLoop->u.btree.pIndex; int nEq = pLoop->u.btree.nEq; - if( p->nSample>0 && nEqnSampleCol ){ + if( p->nSample>0 && ALWAYS(nEqnSampleCol) + && OptimizationEnabled(pParse->db, SQLITE_Stat4) + ){ if( nEq==pBuilder->nRecValid ){ UnpackedRecord *pRec = pBuilder->pRec; tRowcnt a[2]; - u8 aff; + int nBtm = pLoop->u.btree.nBtm; + int nTop = pLoop->u.btree.nTop; - /* Variable iLower will be set to the estimate of the number of rows in + /* Variable iLower will be set to the estimate of the number of rows in ** the index that are less than the lower bound of the range query. The ** lower bound being the concatenation of $P and $L, where $P is the ** key-prefix formed by the nEq values matched against the nEq left-most @@ -127349,7 +165278,7 @@ static int whereRangeScanEst( ** Or, if pLower is NULL or $L cannot be extracted from it (because it ** is not a simple variable or literal value), the lower bound of the ** range is $P. Due to a quirk in the way whereKeyStats() works, even - ** if $L is available, whereKeyStats() is called for both ($P) and + ** if $L is available, whereKeyStats() is called for both ($P) and ** ($P:$L) and the larger of the two returned values is used. ** ** Similarly, iUpper is to be set to the estimate of the number of rows @@ -127368,14 +165297,12 @@ static int whereRangeScanEst( testcase( pRec->nField!=pBuilder->nRecValid ); pRec->nField = pBuilder->nRecValid; } - aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq); - assert( nEq!=p->nKeyCol || aff==SQLITE_AFF_INTEGER ); /* Determine iLower and iUpper using ($P) only. */ if( nEq==0 ){ iLower = 0; iUpper = p->nRowEst0; }else{ - /* Note: this call could be optimized away - since the same values must + /* Note: this call could be optimized away - since the same values must ** have been requested when testing key $P in whereEqualScanEst(). */ whereKeyStats(pParse, p, pRec, 0, a); iLower = a[0]; @@ -127388,17 +165315,20 @@ static int whereRangeScanEst( if( p->aSortOrder[nEq] ){ /* The roles of pLower and pUpper are swapped for a DESC index */ SWAP(WhereTerm*, pLower, pUpper); + SWAP(int, nBtm, nTop); } /* If possible, improve on the iLower estimate using ($P:$L). */ if( pLower ){ - int bOk; /* True if value is extracted from pExpr */ + int n; /* Values extracted from pExpr */ Expr *pExpr = pLower->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nBtm, nEq, &n); + if( rc==SQLITE_OK && n ){ tRowcnt iNew; + u16 mask = WO_GT|WO_LE; + if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); iLwrIdx = whereKeyStats(pParse, p, pRec, 0, a); - iNew = a[0] + ((pLower->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + iNew = a[0] + ((pLower->eOperator & mask) ? a[1] : 0); if( iNew>iLower ) iLower = iNew; nOut--; pLower = 0; @@ -127407,13 +165337,15 @@ static int whereRangeScanEst( /* If possible, improve on the iUpper estimate using ($P:$U). */ if( pUpper ){ - int bOk; /* True if value is extracted from pExpr */ + int n; /* Values extracted from pExpr */ Expr *pExpr = pUpper->pExpr->pRight; - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq, &bOk); - if( rc==SQLITE_OK && bOk ){ + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, nTop, nEq, &n); + if( rc==SQLITE_OK && n ){ tRowcnt iNew; + u16 mask = WO_GT|WO_LE; + if( sqlite3ExprVectorSize(pExpr)>n ) mask = (WO_LE|WO_LT); iUprIdx = whereKeyStats(pParse, p, pRec, 1, a); - iNew = a[0] + ((pUpper->eOperator & (WO_GT|WO_LE)) ? a[1] : 0); + iNew = a[0] + ((pUpper->eOperator & mask) ? a[1] : 0); if( iNewwtFlags & TERM_VNULL)==0 ); + assert( pUpper==0 || (pUpper->wtFlags & TERM_VNULL)==0 || pParse->nErr>0 ); nNew = whereRangeAdjust(pLower, nOut); nNew = whereRangeAdjust(pUpper, nNew); @@ -127458,7 +165391,7 @@ static int whereRangeScanEst( ** reduced by an additional 75%. This means that, by default, an open-ended ** range query (e.g. col > ?) is assumed to match 1/4 of the rows in the ** index. While a closed range (e.g. col BETWEEN ? AND ?) is estimated to - ** match 1/64 of the index. */ + ** match 1/64 of the index. */ if( pLower && pLower->truthProb>0 && pUpper && pUpper->truthProb>0 ){ nNew -= 20; } @@ -127468,7 +165401,7 @@ static int whereRangeScanEst( if( nNewnOut>nOut ){ - WHERETRACE(0x10,("Range scan lowers nOut from %d to %d\n", + WHERETRACE(0x20,("Range scan lowers nOut from %d to %d\n", pLoop->nOut, nOut)); } #endif @@ -127476,16 +165409,16 @@ static int whereRangeScanEst( return rc; } -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an equality constraint x=VALUE and where that VALUE occurs in ** the histogram data. This only works when x is the left-most -** column of an index and sqlite_stat3 histogram data is available +** column of an index and sqlite_stat4 histogram data is available ** for that index. When pExpr==NULL that means the constraint is ** "x IS NULL" instead of "x=VALUE". ** -** Write the estimated row count into *pnRow and return SQLITE_OK. +** Write the estimated row count into *pnRow and return SQLITE_OK. ** If unable to make an estimate, leave *pnRow unchanged and return ** non-zero. ** @@ -127503,7 +165436,6 @@ static int whereEqualScanEst( Index *p = pBuilder->pNew->u.btree.pIndex; int nEq = pBuilder->pNew->u.btree.nEq; UnpackedRecord *pRec = pBuilder->pRec; - u8 aff; /* Column affinity */ int rc; /* Subfunction return code */ tRowcnt a[2]; /* Statistics */ int bOk; @@ -127527,23 +165459,22 @@ static int whereEqualScanEst( return SQLITE_OK; } - aff = sqlite3IndexColumnAffinity(pParse->db, p, nEq-1); - rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, aff, nEq-1, &bOk); + rc = sqlite3Stat4ProbeSetValue(pParse, p, &pRec, pExpr, 1, nEq-1, &bOk); pBuilder->pRec = pRec; if( rc!=SQLITE_OK ) return rc; if( bOk==0 ) return SQLITE_NOTFOUND; pBuilder->nRecValid = nEq; whereKeyStats(pParse, p, pRec, 0, a); - WHERETRACE(0x10,("equality scan regions %s(%d): %d\n", + WHERETRACE(0x20,("equality scan regions %s(%d): %d\n", p->zName, nEq-1, (int)a[1])); *pnRow = a[1]; - + return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 /* ** Estimate the number of rows that will be returned based on ** an IN constraint where the right-hand side of the IN operator @@ -127551,7 +165482,7 @@ static int whereEqualScanEst( ** ** WHERE x IN (1,2,3,4) ** -** Write the estimated row count into *pnRow and return SQLITE_OK. +** Write the estimated row count into *pnRow and return SQLITE_OK. ** If unable to make an estimate, leave *pnRow unchanged and return ** non-zero. ** @@ -127583,33 +165514,57 @@ static int whereInScanEst( } if( rc==SQLITE_OK ){ - if( nRowEst > nRow0 ) nRowEst = nRow0; + if( nRowEst > (tRowcnt)nRow0 ) nRowEst = nRow0; *pnRow = nRowEst; - WHERETRACE(0x10,("IN row estimate: est=%d\n", nRowEst)); + WHERETRACE(0x20,("IN row estimate: est=%d\n", nRowEst)); } assert( pBuilder->nRecValid==nRecValid ); return rc; } -#endif /* SQLITE_ENABLE_STAT3_OR_STAT4 */ +#endif /* SQLITE_ENABLE_STAT4 */ #ifdef WHERETRACE_ENABLED /* ** Print the content of a WhereTerm object */ -static void whereTermPrint(WhereTerm *pTerm, int iTerm){ +SQLITE_PRIVATE void sqlite3WhereTermPrint(WhereTerm *pTerm, int iTerm){ if( pTerm==0 ){ sqlite3DebugPrintf("TERM-%-3d NULL\n", iTerm); }else{ - char zType[4]; - memcpy(zType, "...", 4); + char zType[8]; + char zLeft[50]; + memcpy(zType, "....", 5); if( pTerm->wtFlags & TERM_VIRTUAL ) zType[0] = 'V'; if( pTerm->eOperator & WO_EQUIV ) zType[1] = 'E'; - if( ExprHasProperty(pTerm->pExpr, EP_FromJoin) ) zType[2] = 'L'; + if( ExprHasProperty(pTerm->pExpr, EP_OuterON) ) zType[2] = 'L'; + if( pTerm->wtFlags & TERM_CODED ) zType[3] = 'C'; + if( pTerm->eOperator & WO_SINGLE ){ + assert( (pTerm->eOperator & (WO_OR|WO_AND))==0 ); + sqlite3_snprintf(sizeof(zLeft),zLeft,"left={%d:%d}", + pTerm->leftCursor, pTerm->u.x.leftColumn); + }else if( (pTerm->eOperator & WO_OR)!=0 && pTerm->u.pOrInfo!=0 ){ + sqlite3_snprintf(sizeof(zLeft),zLeft,"indexable=0x%llx", + pTerm->u.pOrInfo->indexable); + }else{ + sqlite3_snprintf(sizeof(zLeft),zLeft,"left=%d", pTerm->leftCursor); + } sqlite3DebugPrintf( - "TERM-%-3d %p %s cursor=%-3d prob=%-3d op=0x%03x wtFlags=0x%04x\n", - iTerm, pTerm, zType, pTerm->leftCursor, pTerm->truthProb, - pTerm->eOperator, pTerm->wtFlags); + "TERM-%-3d %p %s %-12s op=%03x wtFlags=%04x", + iTerm, pTerm, zType, zLeft, pTerm->eOperator, pTerm->wtFlags); + /* The 0x10000 .wheretrace flag causes extra information to be + ** shown about each Term */ + if( sqlite3WhereTrace & 0x10000 ){ + sqlite3DebugPrintf(" prob=%-3d prereq=%llx,%llx", + pTerm->truthProb, (u64)pTerm->prereqAll, (u64)pTerm->prereqRight); + } + if( (pTerm->eOperator & (WO_OR|WO_AND))==0 && pTerm->u.x.iField ){ + sqlite3DebugPrintf(" iField=%d", pTerm->u.x.iField); + } + if( pTerm->iParent>=0 ){ + sqlite3DebugPrintf(" iParent=%d", pTerm->iParent); + } + sqlite3DebugPrintf("\n"); sqlite3TreeViewExpr(0, pTerm->pExpr, 0); } } @@ -127617,18 +165572,47 @@ static void whereTermPrint(WhereTerm *pTerm, int iTerm){ #ifdef WHERETRACE_ENABLED /* -** Print a WhereLoop object for debugging purposes +** Show the complete content of a WhereClause */ -static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ - WhereInfo *pWInfo = pWC->pWInfo; - int nb = 1+(pWInfo->pTabList->nSrc+3)/4; - struct SrcList_item *pItem = pWInfo->pTabList->a + p->iTab; - Table *pTab = pItem->pTab; - Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; - sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, - p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); - sqlite3DebugPrintf(" %12s", - pItem->zAlias ? pItem->zAlias : pTab->zName); +SQLITE_PRIVATE void sqlite3WhereClausePrint(WhereClause *pWC){ + int i; + for(i=0; inTerm; i++){ + sqlite3WhereTermPrint(&pWC->a[i], i); + } +} +#endif + +#ifdef WHERETRACE_ENABLED +/* +** Print a WhereLoop object for debugging purposes +** +** Format example: +** +** .--- Position in WHERE clause rSetup, rRun, nOut ---. +** | | +** | .--- selfMask nTerm ------. | +** | | | | +** | | .-- prereq Idx wsFlags----. | | +** | | | Name | | | +** | | | __|__ nEq ---. ___|__ | __|__ +** | / \ / \ / \ | / \ / \ / \ +** 1.002.001 t2.t2xy 2 f 010241 N 2 cost 0,56,31 +*/ +SQLITE_PRIVATE void sqlite3WhereLoopPrint(const WhereLoop *p, const WhereClause *pWC){ + if( pWC ){ + WhereInfo *pWInfo = pWC->pWInfo; + int nb = 1+(pWInfo->pTabList->nSrc+3)/4; + SrcItem *pItem = pWInfo->pTabList->a + p->iTab; + Table *pTab = pItem->pSTab; + Bitmask mAll = (((Bitmask)1)<<(nb*4)) - 1; + sqlite3DebugPrintf("%c%2d.%0*llx.%0*llx", p->cId, + p->iTab, nb, p->maskSelf, nb, p->prereq & mAll); + sqlite3DebugPrintf(" %12s", + pItem->zAlias ? pItem->zAlias : pTab->zName); + }else{ + sqlite3DebugPrintf("%c%2d.%03llx.%03llx %c%d", + p->cId, p->iTab, p->maskSelf, p->prereq & 0xfff, p->cId, p->iTab); + } if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ const char *zName; if( p->u.btree.pIndex && (zName = p->u.btree.pIndex->zName)!=0 ){ @@ -127644,7 +165628,7 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ }else{ char *z; if( p->u.vtab.idxStr ){ - z = sqlite3_mprintf("(%d,\"%s\",%x)", + z = sqlite3_mprintf("(%d,\"%s\",%#x)", p->u.vtab.idxNum, p->u.vtab.idxStr, p->u.vtab.omitMask); }else{ z = sqlite3_mprintf("(%d,%x)", p->u.vtab.idxNum, p->u.vtab.omitMask); @@ -127653,18 +165637,27 @@ static void whereLoopPrint(WhereLoop *p, WhereClause *pWC){ sqlite3_free(z); } if( p->wsFlags & WHERE_SKIPSCAN ){ - sqlite3DebugPrintf(" f %05x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); + sqlite3DebugPrintf(" f %06x %d-%d", p->wsFlags, p->nLTerm,p->nSkip); }else{ - sqlite3DebugPrintf(" f %05x N %d", p->wsFlags, p->nLTerm); + sqlite3DebugPrintf(" f %06x N %d", p->wsFlags, p->nLTerm); } sqlite3DebugPrintf(" cost %d,%d,%d\n", p->rSetup, p->rRun, p->nOut); - if( p->nLTerm && (sqlite3WhereTrace & 0x100)!=0 ){ + if( p->nLTerm && (sqlite3WhereTrace & 0x4000)!=0 ){ int i; for(i=0; inLTerm; i++){ - whereTermPrint(p->aLTerm[i], i); + sqlite3WhereTermPrint(p->aLTerm[i], i); } } } +SQLITE_PRIVATE void sqlite3ShowWhereLoop(const WhereLoop *p){ + if( p ) sqlite3WhereLoopPrint(p, 0); +} +SQLITE_PRIVATE void sqlite3ShowWhereLoopList(const WhereLoop *p){ + while( p ){ + sqlite3ShowWhereLoop(p); + p = p->pNextLoop; + } +} #endif /* @@ -127689,19 +165682,25 @@ static void whereLoopClearUnion(sqlite3 *db, WhereLoop *p){ p->u.vtab.idxStr = 0; }else if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 && p->u.btree.pIndex!=0 ){ sqlite3DbFree(db, p->u.btree.pIndex->zColAff); - sqlite3DbFree(db, p->u.btree.pIndex); + sqlite3DbFreeNN(db, p->u.btree.pIndex); p->u.btree.pIndex = 0; } } } /* -** Deallocate internal memory used by a WhereLoop object +** Deallocate internal memory used by a WhereLoop object. Leave the +** object in an initialized state, as if it had been newly allocated. */ static void whereLoopClear(sqlite3 *db, WhereLoop *p){ - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + if( p->aLTerm!=p->aLTermSpace ){ + sqlite3DbFreeNN(db, p->aLTerm); + p->aLTerm = p->aLTermSpace; + p->nLSlot = ArraySize(p->aLTermSpace); + } whereLoopClearUnion(db, p); - whereLoopInit(p); + p->nLTerm = 0; + p->wsFlags = 0; } /* @@ -127714,7 +165713,7 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ paNew = sqlite3DbMallocRawNN(db, sizeof(p->aLTerm[0])*n); if( paNew==0 ) return SQLITE_NOMEM_BKPT; memcpy(paNew, p->aLTerm, sizeof(p->aLTerm[0])*p->nLSlot); - if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFree(db, p->aLTerm); + if( p->aLTerm!=p->aLTermSpace ) sqlite3DbFreeNN(db, p->aLTerm); p->aLTerm = paNew; p->nLSlot = n; return SQLITE_OK; @@ -127725,8 +165724,10 @@ static int whereLoopResize(sqlite3 *db, WhereLoop *p, int n){ */ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ whereLoopClearUnion(db, pTo); - if( whereLoopResize(db, pTo, pFrom->nLTerm) ){ - memset(&pTo->u, 0, sizeof(pTo->u)); + if( pFrom->nLTerm > pTo->nLSlot + && whereLoopResize(db, pTo, pFrom->nLTerm) + ){ + memset(pTo, 0, WHERE_LOOP_XFER_SZ); return SQLITE_NOMEM_BKPT; } memcpy(pTo, pFrom, WHERE_LOOP_XFER_SZ); @@ -127743,75 +165744,91 @@ static int whereLoopXfer(sqlite3 *db, WhereLoop *pTo, WhereLoop *pFrom){ ** Delete a WhereLoop object */ static void whereLoopDelete(sqlite3 *db, WhereLoop *p){ + assert( db!=0 ); whereLoopClear(db, p); - sqlite3DbFree(db, p); + sqlite3DbNNFreeNN(db, p); } /* ** Free a WhereInfo structure */ static void whereInfoFree(sqlite3 *db, WhereInfo *pWInfo){ - if( ALWAYS(pWInfo) ){ - int i; - for(i=0; inLevel; i++){ - WhereLevel *pLevel = &pWInfo->a[i]; - if( pLevel->pWLoop && (pLevel->pWLoop->wsFlags & WHERE_IN_ABLE) ){ - sqlite3DbFree(db, pLevel->u.in.aInLoop); - } - } - sqlite3WhereClauseClear(&pWInfo->sWC); - while( pWInfo->pLoops ){ - WhereLoop *p = pWInfo->pLoops; - pWInfo->pLoops = p->pNextLoop; - whereLoopDelete(db, p); - } - sqlite3DbFree(db, pWInfo); + assert( pWInfo!=0 ); + assert( db!=0 ); + sqlite3WhereClauseClear(&pWInfo->sWC); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); } + while( pWInfo->pMemToFree ){ + WhereMemBlock *pNext = pWInfo->pMemToFree->pNext; + sqlite3DbNNFreeNN(db, pWInfo->pMemToFree); + pWInfo->pMemToFree = pNext; + } + sqlite3DbNNFreeNN(db, pWInfo); } /* -** Return TRUE if all of the following are true: +** Return TRUE if X is a proper subset of Y but is of equal or less cost. +** In other words, return true if all constraints of X are also part of Y +** and Y has additional constraints that might speed the search that X lacks +** but the cost of running X is not more than the cost of running Y. ** -** (1) X has the same or lower cost that Y -** (2) X is a proper subset of Y -** (3) X skips at least as many columns as Y +** In other words, return true if the cost relationship between X and Y +** is inverted and needs to be adjusted. ** -** By "proper subset" we mean that X uses fewer WHERE clause terms -** than Y and that every WHERE clause term used by X is also used -** by Y. +** Case 1: ** -** If X is a proper subset of Y then Y is a better choice and ought -** to have a lower cost. This routine returns TRUE when that cost -** relationship is inverted and needs to be adjusted. The third rule -** was added because if X uses skip-scan less than Y it still might -** deserve a lower cost even if it is a proper subset of Y. +** (1a) X and Y use the same index. +** (1b) X has fewer == terms than Y +** (1c) Neither X nor Y use skip-scan +** (1d) X does not have a a greater cost than Y +** +** Case 2: +** +** (2a) X has the same or lower cost, or returns the same or fewer rows, +** than Y. +** (2b) X uses fewer WHERE clause terms than Y +** (2c) Every WHERE clause term used by X is also used by Y +** (2d) X skips at least as many columns as Y +** (2e) If X is a covering index, than Y is too */ static int whereLoopCheaperProperSubset( const WhereLoop *pX, /* First WhereLoop to compare */ const WhereLoop *pY /* Compare against this WhereLoop */ ){ int i, j; + if( pX->rRun>pY->rRun && pX->nOut>pY->nOut ) return 0; /* (1d) and (2a) */ + assert( (pX->wsFlags & WHERE_VIRTUALTABLE)==0 ); + assert( (pY->wsFlags & WHERE_VIRTUALTABLE)==0 ); + if( pX->u.btree.nEq < pY->u.btree.nEq /* (1b) */ + && pX->u.btree.pIndex==pY->u.btree.pIndex /* (1a) */ + && pX->nSkip==0 && pY->nSkip==0 /* (1c) */ + ){ + return 1; /* Case 1 is true */ + } if( pX->nLTerm-pX->nSkip >= pY->nLTerm-pY->nSkip ){ - return 0; /* X is not a subset of Y */ - } - if( pY->nSkip > pX->nSkip ) return 0; - if( pX->rRun >= pY->rRun ){ - if( pX->rRun > pY->rRun ) return 0; /* X costs more than Y */ - if( pX->nOut > pY->nOut ) return 0; /* X costs more than Y */ + return 0; /* (2b) */ } + if( pY->nSkip > pX->nSkip ) return 0; /* (2d) */ for(i=pX->nLTerm-1; i>=0; i--){ if( pX->aLTerm[i]==0 ) continue; for(j=pY->nLTerm-1; j>=0; j--){ if( pY->aLTerm[j]==pX->aLTerm[i] ) break; } - if( j<0 ) return 0; /* X not a subset of Y since term X[i] not used by Y */ + if( j<0 ) return 0; /* (2c) */ } - return 1; /* All conditions meet */ + if( (pX->wsFlags&WHERE_IDX_ONLY)!=0 + && (pY->wsFlags&WHERE_IDX_ONLY)==0 ){ + return 0; /* (2e) */ + } + return 1; /* Case 2 is true */ } /* -** Try to adjust the cost of WhereLoop pTemplate upwards or downwards so -** that: +** Try to adjust the cost and number of output rows of WhereLoop pTemplate +** upwards or downwards so that: ** ** (1) pTemplate costs less than any other WhereLoops that are a proper ** subset of pTemplate @@ -127829,35 +165846,40 @@ static void whereLoopAdjustCost(const WhereLoop *p, WhereLoop *pTemplate){ if( p->iTab!=pTemplate->iTab ) continue; if( (p->wsFlags & WHERE_INDEXED)==0 ) continue; if( whereLoopCheaperProperSubset(p, pTemplate) ){ - /* Adjust pTemplate cost downward so that it is cheaper than its + /* Adjust pTemplate cost downward so that it is cheaper than its ** subset p. */ WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", - pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut-1)); - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut - 1; + pTemplate->rRun, pTemplate->nOut, + MIN(p->rRun, pTemplate->rRun), + MIN(p->nOut - 1, pTemplate->nOut))); + pTemplate->rRun = MIN(p->rRun, pTemplate->rRun); + pTemplate->nOut = MIN(p->nOut - 1, pTemplate->nOut); }else if( whereLoopCheaperProperSubset(pTemplate, p) ){ /* Adjust pTemplate cost upward so that it is costlier than p since ** pTemplate is a proper subset of p */ WHERETRACE(0x80,("subset cost adjustment %d,%d to %d,%d\n", - pTemplate->rRun, pTemplate->nOut, p->rRun, p->nOut+1)); - pTemplate->rRun = p->rRun; - pTemplate->nOut = p->nOut + 1; + pTemplate->rRun, pTemplate->nOut, + MAX(p->rRun, pTemplate->rRun), + MAX(p->nOut + 1, pTemplate->nOut))); + pTemplate->rRun = MAX(p->rRun, pTemplate->rRun); + pTemplate->nOut = MAX(p->nOut + 1, pTemplate->nOut); } } } /* ** Search the list of WhereLoops in *ppPrev looking for one that can be -** supplanted by pTemplate. +** replaced by pTemplate. ** -** Return NULL if the WhereLoop list contains an entry that can supplant -** pTemplate, in other words if pTemplate does not belong on the list. +** Return NULL if pTemplate does not belong on the WhereLoop list. +** In other words if pTemplate ought to be dropped from further consideration. ** -** If pX is a WhereLoop that pTemplate can supplant, then return the +** If pX is a WhereLoop that pTemplate can replace, then return the ** link that points to pX. ** -** If pTemplate cannot supplant any existing element of the list but needs -** to be added to the list, then return a pointer to the tail of the list. +** If pTemplate cannot replace any existing element of the list but needs +** to be added to the list as a new entry, then return a pointer to the +** tail of the list. */ static WhereLoop **whereLoopFindLesser( WhereLoop **ppPrev, @@ -127874,7 +165896,7 @@ static WhereLoop **whereLoopFindLesser( /* In the current implementation, the rSetup value is either zero ** or the cost of building an automatic index (NlogN) and the NlogN ** is the same for compatible WhereLoops. */ - assert( p->rSetup==0 || pTemplate->rSetup==0 + assert( p->rSetup==0 || pTemplate->rSetup==0 || p->rSetup==pTemplate->rSetup ); /* whereLoopAddBtree() always generates and inserts the automatic index @@ -127882,7 +165904,7 @@ static WhereLoop **whereLoopFindLesser( ** rSetup. Call this SETUP-INVARIANT */ assert( p->rSetup>=pTemplate->rSetup ); - /* Any loop using an appliation-defined index (or PRIMARY KEY or + /* Any loop using an application-defined index (or PRIMARY KEY or ** UNIQUE constraint) with one or more == constraints is better ** than an automatic index. Unless it is a skip-scan. */ if( (p->wsFlags & WHERE_AUTO_INDEX)!=0 @@ -127909,7 +165931,7 @@ static WhereLoop **whereLoopFindLesser( /* If pTemplate is always better than p, then cause p to be overwritten ** with pTemplate. pTemplate is better than p if: - ** (1) pTemplate has no more dependences than p, and + ** (1) pTemplate has no more dependencies than p, and ** (2) pTemplate has an equal or lower cost than p. */ if( (p->prereq & pTemplate->prereq)==pTemplate->prereq /* (1) */ @@ -127939,7 +165961,7 @@ static WhereLoop **whereLoopFindLesser( ** ** When accumulating multiple loops (when pBuilder->pOrSet is NULL) we ** still might overwrite similar loops with the new template if the -** new template is better. Loops may be overwritten if the following +** new template is better. Loops may be overwritten if the following ** conditions are met: ** ** (1) They have the same iTab. @@ -127953,6 +165975,16 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ sqlite3 *db = pWInfo->pParse->db; int rc; + /* Stop the search once we hit the query planner search limit */ + if( pBuilder->iPlanLimit==0 ){ + WHERETRACE(0xffffffff,("=== query planner search limit reached ===\n")); + if( pBuilder->pOrSet ) pBuilder->pOrSet->n = 0; + return SQLITE_DONE; + } + pBuilder->iPlanLimit--; + + whereLoopAdjustCost(pWInfo->pLoops, pTemplate); + /* If pBuilder->pOrSet is defined, then only keep track of the costs ** and prereqs. */ @@ -127967,7 +165999,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf(x?" or-%d: ":" or-X: ", n); - whereLoopPrint(pTemplate, pBuilder->pWC); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); } #endif } @@ -127976,7 +166008,6 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ /* Look for an existing WhereLoop to replace with pTemplate */ - whereLoopAdjustCost(pWInfo->pLoops, pTemplate); ppPrev = whereLoopFindLesser(&pWInfo->pLoops, pTemplate); if( ppPrev==0 ){ @@ -127985,10 +166016,10 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf(" skip: "); - whereLoopPrint(pTemplate, pBuilder->pWC); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); } #endif - return SQLITE_OK; + return SQLITE_OK; }else{ p = *ppPrev; } @@ -128001,10 +166032,12 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ if( sqlite3WhereTrace & 0x8 ){ if( p!=0 ){ sqlite3DebugPrintf("replace: "); - whereLoopPrint(p, pBuilder->pWC); + sqlite3WhereLoopPrint(p, pBuilder->pWC); + sqlite3DebugPrintf(" with: "); + }else{ + sqlite3DebugPrintf(" add: "); } - sqlite3DebugPrintf(" add: "); - whereLoopPrint(pTemplate, pBuilder->pWC); + sqlite3WhereLoopPrint(pTemplate, pBuilder->pWC); } #endif if( p==0 ){ @@ -128016,7 +166049,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ }else{ /* We will be overwriting WhereLoop p[]. But before we do, first ** go through the rest of the list and delete any other entries besides - ** p[] that are also supplated by pTemplate */ + ** p[] that are also supplanted by pTemplate */ WhereLoop **ppTail = &p->pNextLoop; WhereLoop *pToDel; while( *ppTail ){ @@ -128028,7 +166061,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ #if WHERETRACE_ENABLED /* 0x8 */ if( sqlite3WhereTrace & 0x8 ){ sqlite3DebugPrintf(" delete: "); - whereLoopPrint(pToDel, pBuilder->pWC); + sqlite3WhereLoopPrint(pToDel, pBuilder->pWC); } #endif whereLoopDelete(db, pToDel); @@ -128037,7 +166070,7 @@ static int whereLoopInsert(WhereLoopBuilder *pBuilder, WhereLoop *pTemplate){ rc = whereLoopXfer(db, p, pTemplate); if( (p->wsFlags & WHERE_VIRTUALTABLE)==0 ){ Index *pIndex = p->u.btree.pIndex; - if( pIndex && pIndex->tnum==0 ){ + if( pIndex && pIndex->idxType==SQLITE_IDXTYPE_IPK ){ p->u.btree.pIndex = 0; } } @@ -128080,14 +166113,15 @@ static void whereLoopOutputAdjust( ){ WhereTerm *pTerm, *pX; Bitmask notAllowed = ~(pLoop->prereq|pLoop->maskSelf); - int i, j, k; + int i, j; LogEst iReduce = 0; /* pLoop->nOut should not exceed nRow-iReduce */ assert( (pLoop->wsFlags & WHERE_AUTO_INDEX)==0 ); - for(i=pWC->nTerm, pTerm=pWC->a; i>0; i--, pTerm++){ - if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) break; - if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + for(i=pWC->nBase, pTerm=pWC->a; i>0; i--, pTerm++){ + assert( pTerm!=0 ); if( (pTerm->prereqAll & notAllowed)!=0 ) continue; + if( (pTerm->prereqAll & pLoop->maskSelf)==0 ) continue; + if( (pTerm->wtFlags & TERM_VIRTUAL)!=0 ) continue; for(j=pLoop->nLTerm-1; j>=0; j--){ pX = pLoop->aLTerm[j]; if( pX==0 ) continue; @@ -128095,6 +166129,24 @@ static void whereLoopOutputAdjust( if( pX->iParent>=0 && (&pWC->a[pX->iParent])==pTerm ) break; } if( j<0 ){ + sqlite3ProgressCheck(pWC->pWInfo->pParse); + if( pLoop->maskSelf==pTerm->prereqAll ){ + /* If there are extra terms in the WHERE clause not used by an index + ** that depend only on the table being scanned, and that will tend to + ** cause many rows to be omitted, then mark that table as + ** "self-culling". + ** + ** 2022-03-24: Self-culling only applies if either the extra terms + ** are straight comparison operators that are non-true with NULL + ** operand, or if the loop is not an OUTER JOIN. + */ + if( (pTerm->eOperator & 0x3f)!=0 + || (pWC->pWInfo->pTabList->a[pLoop->iTab].fg.jointype + & (JT_LEFT|JT_LTORJ))==0 + ){ + pLoop->wsFlags |= WHERE_SELFCULL; + } + } if( pTerm->truthProb<=0 ){ /* If a truth probability is specified using the likelihood() hints, ** then use the probability provided by the application. */ @@ -128103,24 +166155,101 @@ static void whereLoopOutputAdjust( /* In the absence of explicit truth probabilities, use heuristics to ** guess a reasonable truth probability. */ pLoop->nOut--; - if( pTerm->eOperator&(WO_EQ|WO_IS) ){ + if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 + && (pTerm->wtFlags & TERM_HIGHTRUTH)==0 /* tag-20200224-1 */ + ){ Expr *pRight = pTerm->pExpr->pRight; + int k = 0; testcase( pTerm->pExpr->op==TK_IS ); - if( sqlite3ExprIsInteger(pRight, &k) && k>=(-1) && k<=1 ){ + if( sqlite3ExprIsInteger(pRight, &k, 0) && k>=(-1) && k<=1 ){ k = 10; }else{ k = 20; } - if( iReducewtFlags |= TERM_HEURTRUTH; + iReduce = k; + } } } } } - if( pLoop->nOut > nRow-iReduce ) pLoop->nOut = nRow - iReduce; + if( pLoop->nOut > nRow-iReduce ){ + pLoop->nOut = nRow - iReduce; + } } /* -** Adjust the cost C by the costMult facter T. This only occurs if +** Term pTerm is a vector range comparison operation. The first comparison +** in the vector can be optimized using column nEq of the index. This +** function returns the total number of vector elements that can be used +** as part of the range comparison. +** +** For example, if the query is: +** +** WHERE a = ? AND (b, c, d) > (?, ?, ?) +** +** and the index: +** +** CREATE INDEX ... ON (a, b, c, d, e) +** +** then this function would be invoked with nEq=1. The value returned in +** this case is 3. +*/ +static int whereRangeVectorLen( + Parse *pParse, /* Parsing context */ + int iCur, /* Cursor open on pIdx */ + Index *pIdx, /* The index to be used for a inequality constraint */ + int nEq, /* Number of prior equality constraints on same index */ + WhereTerm *pTerm /* The vector inequality constraint */ +){ + int nCmp = sqlite3ExprVectorSize(pTerm->pExpr->pLeft); + int i; + + nCmp = MIN(nCmp, (pIdx->nColumn - nEq)); + for(i=1; ipExpr->pLeft) ); + pLhs = pTerm->pExpr->pLeft->x.pList->a[i].pExpr; + pRhs = pTerm->pExpr->pRight; + if( ExprUseXSelect(pRhs) ){ + pRhs = pRhs->x.pSelect->pEList->a[i].pExpr; + }else{ + pRhs = pRhs->x.pList->a[i].pExpr; + } + + /* Check that the LHS of the comparison is a column reference to + ** the right column of the right source table. And that the sort + ** order of the index column is the same as the sort order of the + ** leftmost index column. */ + if( pLhs->op!=TK_COLUMN + || pLhs->iTable!=iCur + || pLhs->iColumn!=pIdx->aiColumn[i+nEq] + || pIdx->aSortOrder[i+nEq]!=pIdx->aSortOrder[nEq] + ){ + break; + } + + testcase( pLhs->iColumn==XN_ROWID ); + aff = sqlite3CompareAffinity(pRhs, sqlite3ExprAffinity(pLhs)); + idxaff = sqlite3TableColumnAffinity(pIdx->pTable, pLhs->iColumn); + if( aff!=idxaff ) break; + + pColl = sqlite3BinaryCompareCollSeq(pParse, pLhs, pRhs); + if( pColl==0 ) break; + if( sqlite3StrICmp(pColl->zName, pIdx->azColl[i+nEq]) ) break; + } + return i; +} + +/* +** Adjust the cost C by the costMult factor T. This only occurs if ** compiled with -DSQLITE_ENABLE_COSTMULT */ #ifdef SQLITE_ENABLE_COSTMULT @@ -128130,24 +166259,24 @@ static void whereLoopOutputAdjust( #endif /* -** We have so far matched pBuilder->pNew->u.btree.nEq terms of the +** We have so far matched pBuilder->pNew->u.btree.nEq terms of the ** index pIndex. Try to match one more. ** -** When this function is called, pBuilder->pNew->nOut contains the -** number of rows expected to be visited by filtering using the nEq -** terms only. If it is modified, this value is restored before this +** When this function is called, pBuilder->pNew->nOut contains the +** number of rows expected to be visited by filtering using the nEq +** terms only. If it is modified, this value is restored before this ** function returns. ** -** If pProbe->tnum==0, that means pIndex is a fake index used for the -** INTEGER PRIMARY KEY. +** If pProbe->idxType==SQLITE_IDXTYPE_IPK, that means pIndex is +** a fake index used for the INTEGER PRIMARY KEY. */ static int whereLoopAddBtreeIndex( WhereLoopBuilder *pBuilder, /* The WhereLoop factory */ - struct SrcList_item *pSrc, /* FROM clause term being analyzed */ + SrcItem *pSrc, /* FROM clause term being analyzed */ Index *pProbe, /* An index on pSrc */ LogEst nInMul /* log(Number of iterations due to IN) */ ){ - WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyse context */ + WhereInfo *pWInfo = pBuilder->pWInfo; /* WHERE analyze context */ Parse *pParse = pWInfo->pParse; /* Parsing context */ sqlite3 *db = pParse->db; /* Database connection malloc context */ WhereLoop *pNew; /* Template WhereLoop under construction */ @@ -128157,6 +166286,8 @@ static int whereLoopAddBtreeIndex( Bitmask saved_prereq; /* Original value of pNew->prereq */ u16 saved_nLTerm; /* Original value of pNew->nLTerm */ u16 saved_nEq; /* Original value of pNew->u.btree.nEq */ + u16 saved_nBtm; /* Original value of pNew->u.btree.nBtm */ + u16 saved_nTop; /* Original value of pNew->u.btree.nTop */ u16 saved_nSkip; /* Original value of pNew->nSkip */ u32 saved_wsFlags; /* Original value of pNew->wsFlags */ LogEst saved_nOut; /* Original value of pNew->nOut */ @@ -128166,20 +166297,36 @@ static int whereLoopAddBtreeIndex( WhereTerm *pTop = 0, *pBtm = 0; /* Top and bottom range constraints */ pNew = pBuilder->pNew; - if( db->mallocFailed ) return SQLITE_NOMEM_BKPT; + assert( db->mallocFailed==0 || pParse->nErr>0 ); + if( pParse->nErr ){ + return pParse->rc; + } + WHERETRACE(0x800, ("BEGIN %s.addBtreeIdx(%s), nEq=%d, nSkip=%d, rRun=%d\n", + pProbe->pTable->zName,pProbe->zName, + pNew->u.btree.nEq, pNew->nSkip, pNew->rRun)); assert( (pNew->wsFlags & WHERE_VIRTUALTABLE)==0 ); assert( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 ); if( pNew->wsFlags & WHERE_BTM_LIMIT ){ opMask = WO_LT|WO_LE; }else{ + assert( pNew->u.btree.nBtm==0 ); opMask = WO_EQ|WO_IN|WO_GT|WO_GE|WO_LT|WO_LE|WO_ISNULL|WO_IS; } - if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + if( pProbe->bUnordered || pProbe->bLowQual ){ + if( pProbe->bUnordered ) opMask &= ~(WO_GT|WO_GE|WO_LT|WO_LE); + if( pProbe->bLowQual && pSrc->fg.isIndexedBy==0 ){ + opMask &= ~(WO_EQ|WO_IN|WO_IS); + } + } assert( pNew->u.btree.nEqnColumn ); + assert( pNew->u.btree.nEqnKeyCol + || pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY ); saved_nEq = pNew->u.btree.nEq; + saved_nBtm = pNew->u.btree.nBtm; + saved_nTop = pNew->u.btree.nTop; saved_nSkip = pNew->nSkip; saved_nLTerm = pNew->nLTerm; saved_wsFlags = pNew->wsFlags; @@ -128195,7 +166342,7 @@ static int whereLoopAddBtreeIndex( LogEst rCostIdx; LogEst nOutUnadjusted; /* nOut before IN() and WHERE adjustments */ int nIn = 0; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 int nRecValid = pBuilder->nRecValid; #endif if( (eOp==WO_ISNULL || (pTerm->wtFlags&TERM_VNULL)!=0) @@ -128209,94 +166356,161 @@ static int whereLoopAddBtreeIndex( ** to mix with a lower range bound from some other source */ if( pTerm->wtFlags & TERM_LIKEOPT && pTerm->eOperator==WO_LT ) continue; - /* Do not allow IS constraints from the WHERE clause to be used by the - ** right table of a LEFT JOIN. Only constraints in the ON clause are - ** allowed */ - if( (pSrc->fg.jointype & JT_LEFT)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - && (eOp & (WO_IS|WO_ISNULL))!=0 + if( (pSrc->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0 + && !constraintCompatibleWithOuterJoin(pTerm,pSrc) ){ - testcase( eOp & WO_IS ); - testcase( eOp & WO_ISNULL ); continue; } - + if( IsUniqueIndex(pProbe) && saved_nEq==pProbe->nKeyCol-1 ){ + pBuilder->bldFlags1 |= SQLITE_BLDF1_UNIQUE; + }else{ + pBuilder->bldFlags1 |= SQLITE_BLDF1_INDEXED; + } pNew->wsFlags = saved_wsFlags; pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nBtm = saved_nBtm; + pNew->u.btree.nTop = saved_nTop; pNew->nLTerm = saved_nLTerm; - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + if( pNew->nLTerm>=pNew->nLSlot + && whereLoopResize(db, pNew, pNew->nLTerm+1) + ){ + break; /* OOM while trying to enlarge the pNew->aLTerm array */ + } pNew->aLTerm[pNew->nLTerm++] = pTerm; pNew->prereq = (saved_prereq | pTerm->prereqRight) & ~pNew->maskSelf; assert( nInMul==0 - || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 - || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 - || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 + || (pNew->wsFlags & WHERE_COLUMN_NULL)!=0 + || (pNew->wsFlags & WHERE_COLUMN_IN)!=0 + || (pNew->wsFlags & WHERE_SKIPSCAN)!=0 ); if( eOp & WO_IN ){ Expr *pExpr = pTerm->pExpr; - pNew->wsFlags |= WHERE_COLUMN_IN; - if( ExprHasProperty(pExpr, EP_xIsSelect) ){ + if( ExprUseXSelect(pExpr) ){ /* "x IN (SELECT ...)": TUNING: the SELECT returns 25 rows */ + int i; nIn = 46; assert( 46==sqlite3LogEst(25) ); + + /* The expression may actually be of the form (x, y) IN (SELECT...). + ** In this case there is a separate term for each of (x) and (y). + ** However, the nIn multiplier should only be applied once, not once + ** for each such term. The following loop checks that pTerm is the + ** first such term in use, and sets nIn back to 0 if it is not. */ + for(i=0; inLTerm-1; i++){ + if( pNew->aLTerm[i] && pNew->aLTerm[i]->pExpr==pExpr ) nIn = 0; + } }else if( ALWAYS(pExpr->x.pList && pExpr->x.pList->nExpr) ){ /* "x IN (value, value, ...)" */ nIn = sqlite3LogEst(pExpr->x.pList->nExpr); } - assert( nIn>0 ); /* RHS always has 2 or more terms... The parser - ** changes "x IN (?)" into "x=?". */ - + if( pProbe->hasStat1 && rLogSize>=10 ){ + LogEst M, logK, x; + /* Let: + ** N = the total number of rows in the table + ** K = the number of entries on the RHS of the IN operator + ** M = the number of rows in the table that match terms to the + ** to the left in the same index. If the IN operator is on + ** the left-most index column, M==N. + ** + ** Given the definitions above, it is better to omit the IN operator + ** from the index lookup and instead do a scan of the M elements, + ** testing each scanned row against the IN operator separately, if: + ** + ** M*log(K) < K*log(N) + ** + ** Our estimates for M, K, and N might be inaccurate, so we build in + ** a safety margin of 2 (LogEst: 10) that favors using the IN operator + ** with the index, as using an index has better worst-case behavior. + ** If we do not have real sqlite_stat1 data, always prefer to use + ** the index. Do not bother with this optimization on very small + ** tables (less than 2 rows) as it is pointless in that case. + */ + M = pProbe->aiRowLogEst[saved_nEq]; + logK = estLog(nIn); + /* TUNING v----- 10 to bias toward indexed IN */ + x = M + logK + 10 - (nIn + rLogSize); + if( x>=0 ){ + WHERETRACE(0x40, + ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d) " + "prefers indexed lookup\n", + saved_nEq, M, logK, nIn, rLogSize, x)); + }else if( nInMul<2 && OptimizationEnabled(db, SQLITE_SeekScan) ){ + WHERETRACE(0x40, + ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" + " nInMul=%d) prefers skip-scan\n", + saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); + pNew->wsFlags |= WHERE_IN_SEEKSCAN; + }else{ + WHERETRACE(0x40, + ("IN operator (N=%d M=%d logK=%d nIn=%d rLogSize=%d x=%d" + " nInMul=%d) prefers normal scan\n", + saved_nEq, M, logK, nIn, rLogSize, x, nInMul)); + continue; + } + } + pNew->wsFlags |= WHERE_COLUMN_IN; }else if( eOp & (WO_EQ|WO_IS) ){ int iCol = pProbe->aiColumn[saved_nEq]; pNew->wsFlags |= WHERE_COLUMN_EQ; assert( saved_nEq==pNew->u.btree.nEq ); - if( iCol==XN_ROWID - || (iCol>0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) + if( iCol==XN_ROWID + || (iCol>=0 && nInMul==0 && saved_nEq==pProbe->nKeyCol-1) ){ - if( iCol>=0 && pProbe->uniqNotNull==0 ){ - pNew->wsFlags |= WHERE_UNQ_WANTED; - }else{ + if( iCol==XN_ROWID || pProbe->uniqNotNull + || (pProbe->nKeyCol==1 && pProbe->onError && (eOp & WO_EQ)) + ){ pNew->wsFlags |= WHERE_ONEROW; + }else{ + pNew->wsFlags |= WHERE_UNQ_WANTED; } } + if( scan.iEquiv>1 ) pNew->wsFlags |= WHERE_TRANSCONS; }else if( eOp & WO_ISNULL ){ pNew->wsFlags |= WHERE_COLUMN_NULL; - }else if( eOp & (WO_GT|WO_GE) ){ - testcase( eOp & WO_GT ); - testcase( eOp & WO_GE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; - pBtm = pTerm; - pTop = 0; - if( pTerm->wtFlags & TERM_LIKEOPT ){ - /* Range contraints that come from the LIKE optimization are - ** always used in pairs. */ - pTop = &pTerm[1]; - assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); - assert( pTop->wtFlags & TERM_LIKEOPT ); - assert( pTop->eOperator==WO_LT ); - if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ - pNew->aLTerm[pNew->nLTerm++] = pTop; - pNew->wsFlags |= WHERE_TOP_LIMIT; - } }else{ - assert( eOp & (WO_LT|WO_LE) ); - testcase( eOp & WO_LT ); - testcase( eOp & WO_LE ); - pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; - pTop = pTerm; - pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? - pNew->aLTerm[pNew->nLTerm-2] : 0; + int nVecLen = whereRangeVectorLen( + pParse, pSrc->iCursor, pProbe, saved_nEq, pTerm + ); + if( eOp & (WO_GT|WO_GE) ){ + testcase( eOp & WO_GT ); + testcase( eOp & WO_GE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_BTM_LIMIT; + pNew->u.btree.nBtm = nVecLen; + pBtm = pTerm; + pTop = 0; + if( pTerm->wtFlags & TERM_LIKEOPT ){ + /* Range constraints that come from the LIKE optimization are + ** always used in pairs. */ + pTop = &pTerm[1]; + assert( (pTop-(pTerm->pWC->a))pWC->nTerm ); + assert( pTop->wtFlags & TERM_LIKEOPT ); + assert( pTop->eOperator==WO_LT ); + if( whereLoopResize(db, pNew, pNew->nLTerm+1) ) break; /* OOM */ + pNew->aLTerm[pNew->nLTerm++] = pTop; + pNew->wsFlags |= WHERE_TOP_LIMIT; + pNew->u.btree.nTop = 1; + } + }else{ + assert( eOp & (WO_LT|WO_LE) ); + testcase( eOp & WO_LT ); + testcase( eOp & WO_LE ); + pNew->wsFlags |= WHERE_COLUMN_RANGE|WHERE_TOP_LIMIT; + pNew->u.btree.nTop = nVecLen; + pTop = pTerm; + pBtm = (pNew->wsFlags & WHERE_BTM_LIMIT)!=0 ? + pNew->aLTerm[pNew->nLTerm-2] : 0; + } } /* At this point pNew->nOut is set to the number of rows expected to ** be visited by the index scan before considering term pTerm, or the - ** values of nIn and nInMul. In other words, assuming that all + ** values of nIn and nInMul. In other words, assuming that all ** "x IN(...)" terms are replaced with "x = ?". This block updates ** the value of pNew->nOut to account for pTerm (but not nIn/nInMul). */ assert( pNew->nOut==saved_nOut ); if( pNew->wsFlags & WHERE_COLUMN_RANGE ){ - /* Adjust nOut using stat3/stat4 data. Or, if there is no stat3/stat4 + /* Adjust nOut using stat4 data. Or, if there is no stat4 ** data, using some other estimate. */ whereRangeScanEst(pParse, pBuilder, pBtm, pTop, pNew); }else{ @@ -128310,12 +166524,13 @@ static int whereLoopAddBtreeIndex( pNew->nOut += pTerm->truthProb; pNew->nOut -= nIn; }else{ -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 tRowcnt nOut = 0; - if( nInMul==0 - && pProbe->nSample - && pNew->u.btree.nEq<=pProbe->nSampleCol - && ((eOp & WO_IN)==0 || !ExprHasProperty(pTerm->pExpr, EP_xIsSelect)) + if( nInMul==0 + && pProbe->nSample + && ALWAYS(pNew->u.btree.nEq<=pProbe->nSampleCol) + && ((eOp & WO_IN)==0 || ExprUseXList(pTerm->pExpr)) + && OptimizationEnabled(db, SQLITE_Stat4) ){ Expr *pExpr = pTerm->pExpr; if( (eOp & (WO_EQ|WO_ISNULL|WO_IS))!=0 ){ @@ -128330,6 +166545,27 @@ static int whereLoopAddBtreeIndex( if( rc!=SQLITE_OK ) break; /* Jump out of the pTerm loop */ if( nOut ){ pNew->nOut = sqlite3LogEst(nOut); + if( nEq==1 + /* TUNING: Mark terms as "low selectivity" if they seem likely + ** to be true for half or more of the rows in the table. + ** See tag-202002240-1 */ + && pNew->nOut+10 > pProbe->aiRowLogEst[0] + ){ +#if WHERETRACE_ENABLED /* 0x01 */ + if( sqlite3WhereTrace & 0x20 ){ + sqlite3DebugPrintf( + "STAT4 determines term has low selectivity:\n"); + sqlite3WhereTermPrint(pTerm, 999); + } +#endif + pTerm->wtFlags |= TERM_HIGHTRUTH; + if( pTerm->wtFlags & TERM_HEURTRUTH ){ + /* If the term has previously been used with an assumption of + ** higher selectivity, then set the flag to rerun the + ** loop computations. */ + pBuilder->bldFlags2 |= SQLITE_BLDF2_2NDPASS; + } + } if( pNew->nOut>saved_nOut ) pNew->nOut = saved_nOut; pNew->nOut -= nIn; } @@ -128339,8 +166575,8 @@ static int whereLoopAddBtreeIndex( { pNew->nOut += (pProbe->aiRowLogEst[nEq] - pProbe->aiRowLogEst[nEq-1]); if( eOp & WO_ISNULL ){ - /* TUNING: If there is no likelihood() value, assume that a - ** "col IS NULL" expression matches twice as many rows + /* TUNING: If there is no likelihood() value, assume that a + ** "col IS NULL" expression matches twice as many rows ** as (col=?). */ pNew->nOut += 10; } @@ -128348,13 +166584,33 @@ static int whereLoopAddBtreeIndex( } } - /* Set rCostIdx to the cost of visiting selected rows in index. Add - ** it to pNew->rRun, which is currently set to the cost of the index - ** seek only. Then, if this is a non-covering index, add the cost of - ** visiting the rows in the main table. */ - rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pTab->szTabRow; - pNew->rRun = sqlite3LogEstAdd(rLogSize, rCostIdx); - if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK))==0 ){ + /* Set rCostIdx to the estimated cost of visiting selected rows in the + ** index. The estimate is the sum of two values: + ** 1. The cost of doing one search-by-key to find the first matching + ** entry + ** 2. Stepping forward in the index pNew->nOut times to find all + ** additional matching entries. + */ + assert( pSrc->pSTab->szTabRow>0 ); + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ + /* The pProbe->szIdxRow is low for an IPK table since the interior + ** pages are small. Thus szIdxRow gives a good estimate of seek cost. + ** But the leaf pages are full-size, so pProbe->szIdxRow would badly + ** under-estimate the scanning cost. */ + rCostIdx = pNew->nOut + 16; + }else{ + rCostIdx = pNew->nOut + 1 + (15*pProbe->szIdxRow)/pSrc->pSTab->szTabRow; + } + rCostIdx = sqlite3LogEstAdd(rLogSize, rCostIdx); + + /* Estimate the cost of running the loop. If all data is coming + ** from the index, then this is just the cost of doing the index + ** lookup and scan. But if some data is coming out of the main table, + ** we also have to add in the cost of doing pNew->nOut searches to + ** locate the row in the main table that corresponds to the index entry. + */ + pNew->rRun = rCostIdx; + if( (pNew->wsFlags & (WHERE_IDX_ONLY|WHERE_IPK|WHERE_EXPRIDX))==0 ){ pNew->rRun = sqlite3LogEstAdd(pNew->rRun, pNew->nOut + 16); } ApplyCostMultiplier(pNew->rRun, pProbe->pTable->costMult); @@ -128373,16 +166629,23 @@ static int whereLoopAddBtreeIndex( if( (pNew->wsFlags & WHERE_TOP_LIMIT)==0 && pNew->u.btree.nEqnColumn + && (pNew->u.btree.nEqnKeyCol || + pProbe->idxType!=SQLITE_IDXTYPE_PRIMARYKEY) ){ + if( pNew->u.btree.nEq>3 ){ + sqlite3ProgressCheck(pParse); + } whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, nInMul+nIn); } pNew->nOut = saved_nOut; -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 +#ifdef SQLITE_ENABLE_STAT4 pBuilder->nRecValid = nRecValid; #endif } pNew->prereq = saved_prereq; pNew->u.btree.nEq = saved_nEq; + pNew->u.btree.nBtm = saved_nBtm; + pNew->u.btree.nTop = saved_nTop; pNew->nSkip = saved_nSkip; pNew->wsFlags = saved_wsFlags; pNew->nOut = saved_nOut; @@ -128390,18 +166653,21 @@ static int whereLoopAddBtreeIndex( /* Consider using a skip-scan if there are no WHERE clause constraints ** available for the left-most terms of the index, and if the average - ** number of repeats in the left-most terms is at least 18. + ** number of repeats in the left-most terms is at least 18. ** ** The magic number 18 is selected on the basis that scanning 17 rows ** is almost always quicker than an index seek (even though if the index ** contains fewer than 2^17 rows we assume otherwise in other parts of - ** the code). And, even if it is not, it should not be too much slower. + ** the code). And, even if it is not, it should not be too much slower. ** On the other hand, the extra seeks could end up being significantly ** more expensive. */ assert( 42==sqlite3LogEst(18) ); if( saved_nEq==saved_nSkip && saved_nEq+1nKeyCol + && saved_nEq==pNew->nLTerm && pProbe->noSkipScan==0 + && pProbe->hasStat1!=0 + && OptimizationEnabled(db, SQLITE_SkipScan) && pProbe->aiRowLogEst[saved_nEq+1]>=42 /* TUNING: Minimum for skip-scan */ && (rc = whereLoopResize(db, pNew, pNew->nLTerm+1))==SQLITE_OK ){ @@ -128422,6 +166688,8 @@ static int whereLoopAddBtreeIndex( pNew->wsFlags = saved_wsFlags; } + WHERETRACE(0x800, ("END %s.addBtreeIdx(%s), nEq=%d, rc=%d\n", + pProbe->pTable->zName, pProbe->zName, saved_nEq, rc)); return rc; } @@ -128445,8 +166713,11 @@ static int indexMightHelpWithOrderBy( if( pIndex->bUnordered ) return 0; if( (pOB = pBuilder->pWInfo->pOrderBy)==0 ) return 0; for(ii=0; iinExpr; ii++){ - Expr *pExpr = sqlite3ExprSkipCollate(pOB->a[ii].pExpr); - if( pExpr->op==TK_COLUMN && pExpr->iTable==iCursor ){ + Expr *pExpr = sqlite3ExprSkipCollateAndLikely(pOB->a[ii].pExpr); + if( NEVER(pExpr==0) ) continue; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) + && pExpr->iTable==iCursor + ){ if( pExpr->iColumn<0 ) return 1; for(jj=0; jjnKeyCol; jj++){ if( pExpr->iColumn==pIndex->aiColumn[jj] ) return 1; @@ -128454,7 +166725,7 @@ static int indexMightHelpWithOrderBy( }else if( (aColExpr = pIndex->aColExpr)!=0 ){ for(jj=0; jjnKeyCol; jj++){ if( pIndex->aiColumn[jj]!=XN_EXPR ) continue; - if( sqlite3ExprCompare(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ + if( sqlite3ExprCompareSkip(pExpr,aColExpr->a[jj].pExpr,iCursor)==0 ){ return 1; } } @@ -128463,38 +166734,33 @@ static int indexMightHelpWithOrderBy( return 0; } -/* -** Return a bitmask where 1s indicate that the corresponding column of -** the table is used by an index. Only the first 63 columns are considered. -*/ -static Bitmask columnsInIndex(Index *pIdx){ - Bitmask m = 0; - int j; - for(j=pIdx->nColumn-1; j>=0; j--){ - int x = pIdx->aiColumn[j]; - if( x>=0 ){ - testcase( x==BMS-1 ); - testcase( x==BMS-2 ); - if( xpWInfo->pParse; while( pWhere->op==TK_AND ){ - if( !whereUsablePartialIndex(iTab,pWC,pWhere->pLeft) ) return 0; + if( !whereUsablePartialIndex(iTab,jointype,pWC,pWhere->pLeft) ) return 0; pWhere = pWhere->pRight; } + if( pParse->db->flags & SQLITE_EnableQPSG ) pParse = 0; for(i=0, pTerm=pWC->a; inTerm; i++, pTerm++){ - Expr *pExpr = pTerm->pExpr; - if( sqlite3ExprImpliesExpr(pExpr, pWhere, iTab) - && (!ExprHasProperty(pExpr, EP_FromJoin) || pExpr->iRightJoinTable==iTab) + Expr *pExpr; + pExpr = pTerm->pExpr; + if( (!ExprHasProperty(pExpr, EP_OuterON) || pExpr->w.iJoin==iTab) + && ((jointype & JT_OUTER)==0 || ExprHasProperty(pExpr, EP_OuterON)) + && sqlite3ExprImpliesExpr(pParse, pExpr, pWhere, iTab) + && (pTerm->wtFlags & TERM_VNULL)==0 ){ return 1; } @@ -128502,9 +166768,246 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ return 0; } +/* +** pIdx is an index containing expressions. Check it see if any of the +** expressions in the index match the pExpr expression. +*/ +static int exprIsCoveredByIndex( + const Expr *pExpr, + const Index *pIdx, + int iTabCur +){ + int i; + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]==XN_EXPR + && sqlite3ExprCompare(0, pExpr, pIdx->aColExpr->a[i].pExpr, iTabCur)==0 + ){ + return 1; + } + } + return 0; +} + +/* +** Structure passed to the whereIsCoveringIndex Walker callback. +*/ +typedef struct CoveringIndexCheck CoveringIndexCheck; +struct CoveringIndexCheck { + Index *pIdx; /* The index */ + int iTabCur; /* Cursor number for the corresponding table */ + u8 bExpr; /* Uses an indexed expression */ + u8 bUnidx; /* Uses an unindexed column not within an indexed expr */ +}; + +/* +** Information passed in is pWalk->u.pCovIdxCk. Call it pCk. +** +** If the Expr node references the table with cursor pCk->iTabCur, then +** make sure that column is covered by the index pCk->pIdx. We know that +** all columns less than 63 (really BMS-1) are covered, so we don't need +** to check them. But we do need to check any column at 63 or greater. +** +** If the index does not cover the column, then set pWalk->eCode to +** non-zero and return WRC_Abort to stop the search. +** +** If this node does not disprove that the index can be a covering index, +** then just return WRC_Continue, to continue the search. +** +** If pCk->pIdx contains indexed expressions and one of those expressions +** matches pExpr, then prune the search. +*/ +static int whereIsCoveringIndexWalkCallback(Walker *pWalk, Expr *pExpr){ + int i; /* Loop counter */ + const Index *pIdx; /* The index of interest */ + const i16 *aiColumn; /* Columns contained in the index */ + u16 nColumn; /* Number of columns in the index */ + CoveringIndexCheck *pCk; /* Info about this search */ + + pCk = pWalk->u.pCovIdxCk; + pIdx = pCk->pIdx; + if( (pExpr->op==TK_COLUMN || pExpr->op==TK_AGG_COLUMN) ){ + /* if( pExpr->iColumn<(BMS-1) && pIdx->bHasExpr==0 ) return WRC_Continue;*/ + if( pExpr->iTable!=pCk->iTabCur ) return WRC_Continue; + pIdx = pWalk->u.pCovIdxCk->pIdx; + aiColumn = pIdx->aiColumn; + nColumn = pIdx->nColumn; + for(i=0; iiColumn ) return WRC_Continue; + } + pCk->bUnidx = 1; + return WRC_Abort; + }else if( pIdx->bHasExpr + && exprIsCoveredByIndex(pExpr, pIdx, pWalk->u.pCovIdxCk->iTabCur) ){ + pCk->bExpr = 1; + return WRC_Prune; + } + return WRC_Continue; +} + + +/* +** pIdx is an index that covers all of the low-number columns used by +** pWInfo->pSelect (columns from 0 through 62) or an index that has +** expressions terms. Hence, we cannot determine whether or not it is +** a covering index by using the colUsed bitmasks. We have to do a search +** to see if the index is covering. This routine does that search. +** +** The return value is one of these: +** +** 0 The index is definitely not a covering index +** +** WHERE_IDX_ONLY The index is definitely a covering index +** +** WHERE_EXPRIDX The index is likely a covering index, but it is +** difficult to determine precisely because of the +** expressions that are indexed. Score it as a +** covering index, but still keep the main table open +** just in case we need it. +** +** This routine is an optimization. It is always safe to return zero. +** But returning one of the other two values when zero should have been +** returned can lead to incorrect bytecode and assertion faults. +*/ +static SQLITE_NOINLINE u32 whereIsCoveringIndex( + WhereInfo *pWInfo, /* The WHERE clause context */ + Index *pIdx, /* Index that is being tested */ + int iTabCur /* Cursor for the table being indexed */ +){ + int i, rc; + struct CoveringIndexCheck ck; + Walker w; + if( pWInfo->pSelect==0 ){ + /* We don't have access to the full query, so we cannot check to see + ** if pIdx is covering. Assume it is not. */ + return 0; + } + if( pIdx->bHasExpr==0 ){ + for(i=0; inColumn; i++){ + if( pIdx->aiColumn[i]>=BMS-1 ) break; + } + if( i>=pIdx->nColumn ){ + /* pIdx does not index any columns greater than 62, but we know from + ** colMask that columns greater than 62 are used, so this is not a + ** covering index */ + return 0; + } + } + ck.pIdx = pIdx; + ck.iTabCur = iTabCur; + ck.bExpr = 0; + ck.bUnidx = 0; + memset(&w, 0, sizeof(w)); + w.xExprCallback = whereIsCoveringIndexWalkCallback; + w.xSelectCallback = sqlite3SelectWalkNoop; + w.u.pCovIdxCk = &ck; + sqlite3WalkSelect(&w, pWInfo->pSelect); + if( ck.bUnidx ){ + rc = 0; + }else if( ck.bExpr ){ + rc = WHERE_EXPRIDX; + }else{ + rc = WHERE_IDX_ONLY; + } + return rc; +} + +/* +** This is an sqlite3ParserAddCleanup() callback that is invoked to +** free the Parse->pIdxEpr list when the Parse object is destroyed. +*/ +static void whereIndexedExprCleanup(sqlite3 *db, void *pObject){ + IndexedExpr **pp = (IndexedExpr**)pObject; + while( *pp!=0 ){ + IndexedExpr *p = *pp; + *pp = p->pIENext; + sqlite3ExprDelete(db, p->pExpr); + sqlite3DbFreeNN(db, p); + } +} + +/* +** This function is called for a partial index - one with a WHERE clause - in +** two scenarios. In both cases, it determines whether or not the WHERE +** clause on the index implies that a column of the table may be safely +** replaced by a constant expression. For example, in the following +** SELECT: +** +** CREATE INDEX i1 ON t1(b, c) WHERE a=; +** SELECT a, b, c FROM t1 WHERE a= AND b=?; +** +** The "a" in the select-list may be replaced by , iff: +** +** (a) is a constant expression, and +** (b) The (a=) comparison uses the BINARY collation sequence, and +** (c) Column "a" has an affinity other than NONE or BLOB. +** +** If argument pItem is NULL, then pMask must not be NULL. In this case this +** function is being called as part of determining whether or not pIdx +** is a covering index. This function clears any bits in (*pMask) +** corresponding to columns that may be replaced by constants as described +** above. +** +** Otherwise, if pItem is not NULL, then this function is being called +** as part of coding a loop that uses index pIdx. In this case, add entries +** to the Parse.pIdxPartExpr list for each column that can be replaced +** by a constant. +*/ +static void wherePartIdxExpr( + Parse *pParse, /* Parse context */ + Index *pIdx, /* Partial index being processed */ + Expr *pPart, /* WHERE clause being processed */ + Bitmask *pMask, /* Mask to clear bits in */ + int iIdxCur, /* Cursor number for index */ + SrcItem *pItem /* The FROM clause entry for the table */ +){ + assert( pItem==0 || (pItem->fg.jointype & JT_RIGHT)==0 ); + assert( (pItem==0 || pMask==0) && (pMask!=0 || pItem!=0) ); + + if( pPart->op==TK_AND ){ + wherePartIdxExpr(pParse, pIdx, pPart->pRight, pMask, iIdxCur, pItem); + pPart = pPart->pLeft; + } + + if( (pPart->op==TK_EQ || pPart->op==TK_IS) ){ + Expr *pLeft = pPart->pLeft; + Expr *pRight = pPart->pRight; + u8 aff; + + if( pLeft->op!=TK_COLUMN ) return; + if( !sqlite3ExprIsConstant(0, pRight) ) return; + if( !sqlite3IsBinary(sqlite3ExprCompareCollSeq(pParse, pPart)) ) return; + if( pLeft->iColumn<0 ) return; + aff = pIdx->pTable->aCol[pLeft->iColumn].affinity; + if( aff>=SQLITE_AFF_TEXT ){ + if( pItem ){ + sqlite3 *db = pParse->db; + IndexedExpr *p = (IndexedExpr*)sqlite3DbMallocRaw(db, sizeof(*p)); + if( p ){ + int bNullRow = (pItem->fg.jointype&(JT_LEFT|JT_LTORJ))!=0; + p->pExpr = sqlite3ExprDup(db, pRight, 0); + p->iDataCur = pItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = pLeft->iColumn; + p->bMaybeNullRow = bNullRow; + p->pIENext = pParse->pIdxPartExpr; + p->aff = aff; + pParse->pIdxPartExpr = p; + if( p->pIENext==0 ){ + void *pArg = (void*)&pParse->pIdxPartExpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); + } + } + }else if( pLeft->iColumn<(BMS-1) ){ + *pMask &= ~((Bitmask)1 << pLeft->iColumn); + } + } + } +} + + /* ** Add all WhereLoop objects for a single table of the join where the table -** is idenfied by pBuilder->pNew->iTab. That table is guaranteed to be +** is identified by pBuilder->pNew->iTab. That table is guaranteed to be ** a b-tree table, not a virtual table. ** ** The costs (WhereLoop.rRun) of the b-tree loops added by this function @@ -128516,18 +167019,18 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ ** cost = nRow * K // scan of covering index ** cost = nRow * (K+3.0) // scan of non-covering index ** -** where K is a value between 1.1 and 3.0 set based on the relative +** where K is a value between 1.1 and 3.0 set based on the relative ** estimated average size of the index and table records. ** ** For an index scan, where nVisit is the number of index rows visited -** by the scan, and nSeek is the number of seek operations required on +** by the scan, and nSeek is the number of seek operations required on ** the index b-tree: ** ** cost = nSeek * (log(nRow) + K * nVisit) // covering index ** cost = nSeek * (log(nRow) + (K+3.0) * nVisit) // non-covering index ** -** Normally, nSeek is 1. nSeek values greater than 1 come about if the -** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when +** Normally, nSeek is 1. nSeek values greater than 1 come about if the +** WHERE clause includes "x IN (....)" terms used in place of "x=?". Or when ** implicit "x IN (SELECT x FROM tbl)" terms are added for skip-scans. ** ** The estimated values (nRow, nVisit, nSeek) often contain a large amount @@ -128540,7 +167043,7 @@ static int whereUsablePartialIndex(int iTab, WhereClause *pWC, Expr *pWhere){ */ static int whereLoopAddBtree( WhereLoopBuilder *pBuilder, /* WHERE clause information */ - Bitmask mPrereq /* Extra prerequesites for using this table */ + Bitmask mPrereq /* Extra prerequisites for using this table */ ){ WhereInfo *pWInfo; /* WHERE analysis context */ Index *pProbe; /* An index we are evaluating */ @@ -128548,27 +167051,27 @@ static int whereLoopAddBtree( LogEst aiRowEstPk[2]; /* The aiRowLogEst[] value for the sPk index */ i16 aiColumnPk = -1; /* The aColumn[] value for the sPk index */ SrcList *pTabList; /* The FROM clause */ - struct SrcList_item *pSrc; /* The FROM clause btree term to add */ + SrcItem *pSrc; /* The FROM clause btree term to add */ WhereLoop *pNew; /* Template WhereLoop object */ int rc = SQLITE_OK; /* Return code */ int iSortIdx = 1; /* Index number */ int b; /* A boolean value */ LogEst rSize; /* number of rows in the table */ - LogEst rLogSize; /* Logarithm of the number of rows in the table */ WhereClause *pWC; /* The parsed WHERE clause */ Table *pTab; /* Table being queried */ - + pNew = pBuilder->pNew; pWInfo = pBuilder->pWInfo; pTabList = pWInfo->pTabList; pSrc = pTabList->a + pNew->iTab; - pTab = pSrc->pTab; + pTab = pSrc->pSTab; pWC = pBuilder->pWC; - assert( !IsVirtual(pSrc->pTab) ); + assert( !IsVirtual(pSrc->pSTab) ); - if( pSrc->pIBIndex ){ + if( pSrc->fg.isIndexedBy ){ + assert( pSrc->fg.isCte==0 ); /* An INDEXED BY clause specifies a particular index to use */ - pProbe = pSrc->pIBIndex; + pProbe = pSrc->u2.pIBIndex; }else if( !HasRowid(pTab) ){ pProbe = pTab->pIndex; }else{ @@ -128584,10 +167087,11 @@ static int whereLoopAddBtree( sPk.aiRowLogEst = aiRowEstPk; sPk.onError = OE_Replace; sPk.pTable = pTab; - sPk.szIdxRow = pTab->szTabRow; + sPk.szIdxRow = 3; /* TUNING: Interior rows of IPK table are very small */ + sPk.idxType = SQLITE_IDXTYPE_IPK; aiRowEstPk[0] = pTab->nRowLogEst; aiRowEstPk[1] = 0; - pFirst = pSrc->pTab->pIndex; + pFirst = pSrc->pSTab->pIndex; if( pSrc->fg.notIndexed==0 ){ /* The real indices of the table are only considered if the ** NOT INDEXED qualifier is omitted from the FROM clause */ @@ -128596,22 +167100,24 @@ static int whereLoopAddBtree( pProbe = &sPk; } rSize = pTab->nRowLogEst; - rLogSize = estLog(rSize); #ifndef SQLITE_OMIT_AUTOMATIC_INDEX /* Automatic indexes */ if( !pBuilder->pOrSet /* Not part of an OR optimization */ - && (pWInfo->wctrlFlags & WHERE_NO_AUTOINDEX)==0 + && (pWInfo->wctrlFlags & (WHERE_RIGHT_JOIN|WHERE_OR_SUBCLAUSE))==0 && (pWInfo->pParse->db->flags & SQLITE_AutoIndex)!=0 - && pSrc->pIBIndex==0 /* Has no INDEXED BY clause */ + && !pSrc->fg.isIndexedBy /* Has no INDEXED BY clause */ && !pSrc->fg.notIndexed /* Has no NOT INDEXED clause */ && HasRowid(pTab) /* Not WITHOUT ROWID table. (FIXME: Why not?) */ && !pSrc->fg.isCorrelated /* Not a correlated subquery */ && !pSrc->fg.isRecursive /* Not a recursive common table expression. */ + && (pSrc->fg.jointype & JT_RIGHT)==0 /* Not the right tab of a RIGHT JOIN */ ){ /* Generate auto-index WhereLoops */ + LogEst rLogSize; /* Logarithm of the number of rows in the table */ WhereTerm *pTerm; WhereTerm *pWCEnd = pWC->a + pWC->nTerm; + rLogSize = estLog(rSize); for(pTerm=pWC->a; rc==SQLITE_OK && pTermprereqRight & pNew->maskSelf ) continue; if( termCanDriveIndex(pTerm, pSrc, 0) ){ @@ -128623,16 +167129,20 @@ static int whereLoopAddBtree( /* TUNING: One-time cost for computing the automatic index is ** estimated to be X*N*log2(N) where N is the number of rows in ** the table being indexed and where X is 7 (LogEst=28) for normal - ** tables or 1.375 (LogEst=4) for views and subqueries. The value + ** tables or 0.5 (LogEst=-10) for views and subqueries. The value ** of X is smaller for views and subqueries so that the query planner ** will be more aggressive about generating automatic indexes for ** those objects, since there is no opportunity to add schema ** indexes on subqueries and views. */ - pNew->rSetup = rLogSize + rSize + 4; - if( pTab->pSelect==0 && (pTab->tabFlags & TF_Ephemeral)==0 ){ - pNew->rSetup += 24; + pNew->rSetup = rLogSize + rSize; + if( !IsView(pTab) && (pTab->tabFlags & TF_Ephemeral)==0 ){ + pNew->rSetup += 28; + }else{ + pNew->rSetup -= 25; /* Greatly reduced setup cost for auto indexes + ** on ephemeral materializations of views */ } ApplyCostMultiplier(pNew->rSetup, pTab->costMult); + if( pNew->rSetup<0 ) pNew->rSetup = 0; /* TUNING: Each index lookup yields 20 rows in the table. This ** is more than the usual guess of 10 rows, since we have no way ** of knowing how selective the index will ultimately be. It would @@ -128647,16 +167157,23 @@ static int whereLoopAddBtree( } #endif /* SQLITE_OMIT_AUTOMATIC_INDEX */ - /* Loop over all indices - */ - for(; rc==SQLITE_OK && pProbe; pProbe=pProbe->pNext, iSortIdx++){ + /* Loop over all indices. If there was an INDEXED BY clause, then only + ** consider index pProbe. */ + for(; rc==SQLITE_OK && pProbe; + pProbe=(pSrc->fg.isIndexedBy ? 0 : pProbe->pNext), iSortIdx++ + ){ if( pProbe->pPartIdxWhere!=0 - && !whereUsablePartialIndex(pSrc->iCursor, pWC, pProbe->pPartIdxWhere) ){ + && !whereUsablePartialIndex(pSrc->iCursor, pSrc->fg.jointype, pWC, + pProbe->pPartIdxWhere) + ){ testcase( pNew->iTab!=pSrc->iCursor ); /* See ticket [98d973b8f5] */ continue; /* Partial index inappropriate for this query */ } + if( pProbe->bNoQuery ) continue; rSize = pProbe->aiRowLogEst[0]; pNew->u.btree.nEq = 0; + pNew->u.btree.nBtm = 0; + pNew->u.btree.nTop = 0; pNew->nSkip = 0; pNew->nLTerm = 0; pNew->iSortIdx = 0; @@ -128664,35 +167181,92 @@ static int whereLoopAddBtree( pNew->prereq = mPrereq; pNew->nOut = rSize; pNew->u.btree.pIndex = pProbe; + pNew->u.btree.pOrderBy = 0; b = indexMightHelpWithOrderBy(pBuilder, pProbe, pSrc->iCursor); + /* The ONEPASS_DESIRED flags never occurs together with ORDER BY */ assert( (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || b==0 ); - if( pProbe->tnum<=0 ){ + if( pProbe->idxType==SQLITE_IDXTYPE_IPK ){ /* Integer primary key index */ pNew->wsFlags = WHERE_IPK; /* Full table scan */ pNew->iSortIdx = b ? iSortIdx : 0; - /* TUNING: Cost of full table scan is (N*3.0). */ + /* TUNING: Cost of full table scan is 3.0*N. The 3.0 factor is an + ** extra cost designed to discourage the use of full table scans, + ** since index lookups have better worst-case performance if our + ** stat guesses are wrong. Reduce the 3.0 penalty slightly + ** (to 2.75) if we have valid STAT4 information for the table. + ** At 2.75, a full table scan is preferred over using an index on + ** a column with just two distinct values where each value has about + ** an equal number of appearances. Without STAT4 data, we still want + ** to use an index in that case, since the constraint might be for + ** the scarcer of the two values, and in that case an index lookup is + ** better. + */ +#ifdef SQLITE_ENABLE_STAT4 + pNew->rRun = rSize + 16 - 2*((pTab->tabFlags & TF_HasStat4)!=0); +#else pNew->rRun = rSize + 16; +#endif ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); + if( pSrc->fg.isSubquery ){ + if( pSrc->fg.viaCoroutine ) pNew->wsFlags |= WHERE_COROUTINE; + pNew->u.btree.pOrderBy = pSrc->u4.pSubq->pSelect->pOrderBy; + } rc = whereLoopInsert(pBuilder, pNew); pNew->nOut = rSize; if( rc ) break; }else{ Bitmask m; if( pProbe->isCovering ){ - pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; m = 0; + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; }else{ - m = pSrc->colUsed & ~columnsInIndex(pProbe); - pNew->wsFlags = (m==0) ? (WHERE_IDX_ONLY|WHERE_INDEXED) : WHERE_INDEXED; + m = pSrc->colUsed & pProbe->colNotIdxed; + if( pProbe->pPartIdxWhere ){ + wherePartIdxExpr( + pWInfo->pParse, pProbe, pProbe->pPartIdxWhere, &m, 0, 0 + ); + } + pNew->wsFlags = WHERE_INDEXED; + if( m==TOPBIT || (pProbe->bHasExpr && !pProbe->bHasVCol && m!=0) ){ + u32 isCov = whereIsCoveringIndex(pWInfo, pProbe, pSrc->iCursor); + if( isCov==0 ){ + WHERETRACE(0x200, + ("-> %s is not a covering index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + assert( m!=0 ); + }else{ + m = 0; + pNew->wsFlags |= isCov; + if( isCov & WHERE_IDX_ONLY ){ + WHERETRACE(0x200, + ("-> %s is a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + }else{ + assert( isCov==WHERE_EXPRIDX ); + WHERETRACE(0x200, + ("-> %s might be a covering expression index" + " according to whereIsCoveringIndex()\n", pProbe->zName)); + } + } + }else if( m==0 + && (HasRowid(pTab) || pWInfo->pSelect!=0 || sqlite3FaultSim(700)) + ){ + WHERETRACE(0x200, + ("-> %s a covering index according to bitmasks\n", + pProbe->zName, m==0 ? "is" : "is not")); + pNew->wsFlags = WHERE_IDX_ONLY | WHERE_INDEXED; + } } /* Full scan via index */ if( b || !HasRowid(pTab) + || pProbe->pPartIdxWhere!=0 + || pSrc->fg.isIndexedBy || ( m==0 && pProbe->bUnordered==0 && (pProbe->szIdxRowszTabRow) @@ -128705,36 +167279,94 @@ static int whereLoopAddBtree( /* The cost of visiting the index rows is N*K, where K is ** between 1.1 and 3.0, depending on the relative sizes of the - ** index and table rows. If this is a non-covering index scan, - ** also add the cost of visiting table rows (N*3.0). */ + ** index and table rows. */ pNew->rRun = rSize + 1 + (15*pProbe->szIdxRow)/pTab->szTabRow; if( m!=0 ){ - pNew->rRun = sqlite3LogEstAdd(pNew->rRun, rSize+16); + /* If this is a non-covering index scan, add in the cost of + ** doing table lookups. The cost will be 3x the number of + ** lookups. Take into account WHERE clause terms that can be + ** satisfied using just the index, and that do not require a + ** table lookup. */ + LogEst nLookup = rSize + 16; /* Base cost: N*3 */ + int ii; + int iCur = pSrc->iCursor; + WhereClause *pWC2 = &pWInfo->sWC; + for(ii=0; iinTerm; ii++){ + WhereTerm *pTerm = &pWC2->a[ii]; + if( !sqlite3ExprCoveredByIndex(pTerm->pExpr, iCur, pProbe) ){ + break; + } + /* pTerm can be evaluated using just the index. So reduce + ** the expected number of table lookups accordingly */ + if( pTerm->truthProb<=0 ){ + nLookup += pTerm->truthProb; + }else{ + nLookup--; + if( pTerm->eOperator & (WO_EQ|WO_IS) ) nLookup -= 19; + } + } + + pNew->rRun = sqlite3LogEstAdd(pNew->rRun, nLookup); } ApplyCostMultiplier(pNew->rRun, pTab->costMult); whereLoopOutputAdjust(pWC, pNew, rSize); - rc = whereLoopInsert(pBuilder, pNew); + if( (pSrc->fg.jointype & JT_RIGHT)!=0 && pProbe->aColExpr ){ + /* Do not do an SCAN of a index-on-expression in a RIGHT JOIN + ** because the cursor used to access the index might not be + ** positioned to the correct row during the right-join no-match + ** loop. */ + }else{ + rc = whereLoopInsert(pBuilder, pNew); + } pNew->nOut = rSize; if( rc ) break; } } + pBuilder->bldFlags1 = 0; rc = whereLoopAddBtreeIndex(pBuilder, pSrc, pProbe, 0); -#ifdef SQLITE_ENABLE_STAT3_OR_STAT4 + if( pBuilder->bldFlags1==SQLITE_BLDF1_INDEXED ){ + /* If a non-unique index is used, or if a prefix of the key for + ** unique index is used (making the index functionally non-unique) + ** then the sqlite_stat1 data becomes important for scoring the + ** plan */ + pTab->tabFlags |= TF_MaybeReanalyze; + } +#ifdef SQLITE_ENABLE_STAT4 sqlite3Stat4ProbeFree(pBuilder->pRec); pBuilder->nRecValid = 0; pBuilder->pRec = 0; #endif - - /* If there was an INDEXED BY clause, then only that one index is - ** considered. */ - if( pSrc->pIBIndex ) break; } return rc; } #ifndef SQLITE_OMIT_VIRTUALTABLE +/* +** Return true if pTerm is a virtual table LIMIT or OFFSET term. +*/ +static int isLimitTerm(WhereTerm *pTerm){ + assert( pTerm->eOperator==WO_AUX || pTerm->eMatchOp==0 ); + return pTerm->eMatchOp>=SQLITE_INDEX_CONSTRAINT_LIMIT + && pTerm->eMatchOp<=SQLITE_INDEX_CONSTRAINT_OFFSET; +} + +/* +** Return true if the first nCons constraints in the pUsage array are +** marked as in-use (have argvIndex>0). False otherwise. +*/ +static int allConstraintsUsed( + struct sqlite3_index_constraint_usage *aUsage, + int nCons +){ + int ii; + for(ii=0; iipNew->iTab. This @@ -128761,9 +167393,12 @@ static int whereLoopAddVirtualOne( Bitmask mUsable, /* Mask of usable tables */ u16 mExclude, /* Exclude terms using these operators */ sqlite3_index_info *pIdxInfo, /* Populated object for xBestIndex */ - int *pbIn /* OUT: True if plan uses an IN(...) op */ + u16 mNoOmit, /* Do not omit these constraints */ + int *pbIn, /* OUT: True if plan uses an IN(...) op */ + int *pbRetryLimit /* OUT: Retry without LIMIT/OFFSET */ ){ WhereClause *pWC = pBuilder->pWC; + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; struct sqlite3_index_constraint *pIdxCons; struct sqlite3_index_constraint_usage *pUsage = pIdxInfo->aConstraintUsage; int i; @@ -128771,21 +167406,22 @@ static int whereLoopAddVirtualOne( int rc = SQLITE_OK; WhereLoop *pNew = pBuilder->pNew; Parse *pParse = pBuilder->pWInfo->pParse; - struct SrcList_item *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; + SrcItem *pSrc = &pBuilder->pWInfo->pTabList->a[pNew->iTab]; int nConstraint = pIdxInfo->nConstraint; assert( (mUsable & mPrereq)==mPrereq ); *pbIn = 0; pNew->prereq = mPrereq; - /* Set the usable flag on the subset of constraints identified by + /* Set the usable flag on the subset of constraints identified by ** arguments mUsable and mExclude. */ pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; ia[pIdxCons->iTermOffset]; + WhereTerm *pTerm = termFromWhereClause(pWC, pIdxCons->iTermOffset); pIdxCons->usable = 0; - if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight + if( (pTerm->prereqRight & mUsable)==pTerm->prereqRight && (pTerm->eOperator & mExclude)==0 + && (pbRetryLimit || !isLimitTerm(pTerm)) ){ pIdxCons->usable = 1; } @@ -128800,16 +167436,27 @@ static int whereLoopAddVirtualOne( pIdxInfo->estimatedCost = SQLITE_BIG_DBL / (double)2; pIdxInfo->estimatedRows = 25; pIdxInfo->idxFlags = 0; - pIdxInfo->colUsed = (sqlite3_int64)pSrc->colUsed; + pHidden->mHandleIn = 0; /* Invoke the virtual table xBestIndex() method */ - rc = vtabBestIndex(pParse, pSrc->pTab, pIdxInfo); - if( rc ) return rc; + rc = vtabBestIndex(pParse, pSrc->pSTab, pIdxInfo); + if( rc ){ + if( rc==SQLITE_CONSTRAINT ){ + /* If the xBestIndex method returns SQLITE_CONSTRAINT, that means + ** that the particular combination of parameters provided is unusable. + ** Make no entries in the loop table. + */ + WHERETRACE(0xffffffff, (" ^^^^--- non-viable plan rejected!\n")); + freeIdxStr(pIdxInfo); + return SQLITE_OK; + } + return rc; + } mxTerm = -1; assert( pNew->nLSlot>=nConstraint ); - for(i=0; iaLTerm[i] = 0; - pNew->u.vtab.omitMask = 0; + memset(pNew->aLTerm, 0, sizeof(pNew->aLTerm[0])*nConstraint ); + memset(&pNew->u.vtab, 0, sizeof(pNew->u.vtab)); pIdxCons = *(struct sqlite3_index_constraint**)&pIdxInfo->aConstraint; for(i=0; iiTermOffset; if( iTerm>=nConstraint || j<0 - || j>=pWC->nTerm + || (pTerm = termFromWhereClause(pWC, j))==0 || pNew->aLTerm[iTerm]!=0 || pIdxCons->usable==0 ){ - rc = SQLITE_ERROR; - sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pTab->zName); - return rc; + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); + freeIdxStr(pIdxInfo); + return SQLITE_ERROR; } testcase( iTerm==nConstraint-1 ); testcase( j==0 ); testcase( j==pWC->nTerm-1 ); - pTerm = &pWC->a[j]; pNew->prereq |= pTerm->prereqRight; assert( iTermnLSlot ); pNew->aLTerm[iTerm] = pTerm; if( iTerm>mxTerm ) mxTerm = iTerm; testcase( iTerm==15 ); testcase( iTerm==16 ); - if( iTerm<16 && pUsage[i].omit ) pNew->u.vtab.omitMask |= 1<eOperator & WO_IN)!=0 ){ + if( pUsage[i].omit ){ + if( i<16 && ((1<u.vtab.omitMask |= 1<eMatchOp==SQLITE_INDEX_CONSTRAINT_OFFSET ){ + pNew->u.vtab.bOmitOffset = 1; + } + } + if( SMASKBIT32(i) & pHidden->mHandleIn ){ + pNew->u.vtab.mHandleIn |= MASKBIT32(iTerm); + }else if( (pTerm->eOperator & WO_IN)!=0 ){ /* A virtual table that is constrained by an IN clause may not ** consume the ORDER BY clause because (1) the order of IN terms ** is not necessarily related to the order of output terms and @@ -128847,10 +167505,38 @@ static int whereLoopAddVirtualOne( pIdxInfo->idxFlags &= ~SQLITE_INDEX_SCAN_UNIQUE; *pbIn = 1; assert( (mExclude & WO_IN)==0 ); } + + /* Unless pbRetryLimit is non-NULL, there should be no LIMIT/OFFSET + ** terms. And if there are any, they should follow all other terms. */ + assert( pbRetryLimit || !isLimitTerm(pTerm) ); + assert( !isLimitTerm(pTerm) || i>=nConstraint-2 ); + assert( !isLimitTerm(pTerm) || i==nConstraint-1 || isLimitTerm(pTerm+1) ); + + if( isLimitTerm(pTerm) && (*pbIn || !allConstraintsUsed(pUsage, i)) ){ + /* If there is an IN(...) term handled as an == (separate call to + ** xFilter for each value on the RHS of the IN) and a LIMIT or + ** OFFSET term handled as well, the plan is unusable. Similarly, + ** if there is a LIMIT/OFFSET and there are other unused terms, + ** the plan cannot be used. In these cases set variable *pbRetryLimit + ** to true to tell the caller to retry with LIMIT and OFFSET + ** disabled. */ + freeIdxStr(pIdxInfo); + *pbRetryLimit = 1; + return SQLITE_OK; + } } } pNew->nLTerm = mxTerm+1; + for(i=0; i<=mxTerm; i++){ + if( pNew->aLTerm[i]==0 ){ + /* The non-zero argvIdx values must be contiguous. Raise an + ** error if they are not */ + sqlite3ErrorMsg(pParse,"%s.xBestIndex malfunction",pSrc->pSTab->zName); + freeIdxStr(pIdxInfo); + return SQLITE_ERROR; + } + } assert( pNew->nLTerm<=pNew->nLSlot ); pNew->u.vtab.idxNum = pIdxInfo->idxNum; pNew->u.vtab.needFree = pIdxInfo->needToFreeIdxStr; @@ -128858,6 +167544,7 @@ static int whereLoopAddVirtualOne( pNew->u.vtab.idxStr = pIdxInfo->idxStr; pNew->u.vtab.isOrdered = (i8)(pIdxInfo->orderByConsumed ? pIdxInfo->nOrderBy : 0); + pNew->u.vtab.bIdxNumHex = (pIdxInfo->idxFlags&SQLITE_INDEX_SCAN_HEX)!=0; pNew->rSetup = 0; pNew->rRun = sqlite3LogEstFromDouble(pIdxInfo->estimatedCost); pNew->nOut = sqlite3LogEst(pIdxInfo->estimatedRows); @@ -128874,13 +167561,130 @@ static int whereLoopAddVirtualOne( sqlite3_free(pNew->u.vtab.idxStr); pNew->u.vtab.needFree = 0; } - WHERETRACE(0xffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", + WHERETRACE(0xffffffff, (" bIn=%d prereqIn=%04llx prereqOut=%04llx\n", *pbIn, (sqlite3_uint64)mPrereq, (sqlite3_uint64)(pNew->prereq & ~mPrereq))); return rc; } +/* +** Return the collating sequence for a constraint passed into xBestIndex. +** +** pIdxInfo must be an sqlite3_index_info structure passed into xBestIndex. +** This routine depends on there being a HiddenIndexInfo structure immediately +** following the sqlite3_index_info structure. +** +** Return a pointer to the collation name: +** +** 1. If there is an explicit COLLATE operator on the constraint, return it. +** +** 2. Else, if the column has an alternative collation, return that. +** +** 3. Otherwise, return "BINARY". +*/ +SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info *pIdxInfo, int iCons){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + const char *zRet = 0; + if( iCons>=0 && iConsnConstraint ){ + CollSeq *pC = 0; + int iTerm = pIdxInfo->aConstraint[iCons].iTermOffset; + Expr *pX = termFromWhereClause(pHidden->pWC, iTerm)->pExpr; + if( pX->pLeft ){ + pC = sqlite3ExprCompareCollSeq(pHidden->pParse, pX); + } + zRet = (pC ? pC->zName : sqlite3StrBINARY); + } + return zRet; +} + +/* +** Return true if constraint iCons is really an IN(...) constraint, or +** false otherwise. If iCons is an IN(...) constraint, set (if bHandle!=0) +** or clear (if bHandle==0) the flag to handle it using an iterator. +*/ +SQLITE_API int sqlite3_vtab_in(sqlite3_index_info *pIdxInfo, int iCons, int bHandle){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + u32 m = SMASKBIT32(iCons); + if( m & pHidden->mIn ){ + if( bHandle==0 ){ + pHidden->mHandleIn &= ~m; + }else if( bHandle>0 ){ + pHidden->mHandleIn |= m; + } + return 1; + } + return 0; +} + +/* +** This interface is callable from within the xBestIndex callback only. +** +** If possible, set (*ppVal) to point to an object containing the value +** on the right-hand-side of constraint iCons. +*/ +SQLITE_API int sqlite3_vtab_rhs_value( + sqlite3_index_info *pIdxInfo, /* Copy of first argument to xBestIndex */ + int iCons, /* Constraint for which RHS is wanted */ + sqlite3_value **ppVal /* Write value extracted here */ +){ + HiddenIndexInfo *pH = (HiddenIndexInfo*)&pIdxInfo[1]; + sqlite3_value *pVal = 0; + int rc = SQLITE_OK; + if( iCons<0 || iCons>=pIdxInfo->nConstraint ){ + rc = SQLITE_MISUSE_BKPT; /* EV: R-30545-25046 */ + }else{ + if( pH->aRhs[iCons]==0 ){ + WhereTerm *pTerm = termFromWhereClause( + pH->pWC, pIdxInfo->aConstraint[iCons].iTermOffset + ); + rc = sqlite3ValueFromExpr( + pH->pParse->db, pTerm->pExpr->pRight, ENC(pH->pParse->db), + SQLITE_AFF_BLOB, &pH->aRhs[iCons] + ); + testcase( rc!=SQLITE_OK ); + } + pVal = pH->aRhs[iCons]; + } + *ppVal = pVal; + + if( rc==SQLITE_OK && pVal==0 ){ /* IMP: R-19933-32160 */ + rc = SQLITE_NOTFOUND; /* IMP: R-36424-56542 */ + } + + return rc; +} + +/* +** Return true if ORDER BY clause may be handled as DISTINCT. +*/ +SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info *pIdxInfo){ + HiddenIndexInfo *pHidden = (HiddenIndexInfo*)&pIdxInfo[1]; + assert( pHidden->eDistinct>=0 && pHidden->eDistinct<=3 ); + return pHidden->eDistinct; +} + +/* +** Cause the prepared statement that is associated with a call to +** xBestIndex to potentially use all schemas. If the statement being +** prepared is read-only, then just start read transactions on all +** schemas. But if this is a write operation, start writes on all +** schemas. +** +** This is used by the (built-in) sqlite_dbpage virtual table. +*/ +SQLITE_PRIVATE void sqlite3VtabUsesAllSchemas(Parse *pParse){ + int nDb = pParse->db->nDb; + int i; + for(i=0; iwriteMask) ){ + for(i=0; ipWInfo; @@ -128929,8 +167735,8 @@ static int whereLoopAddVirtual( pWC = pBuilder->pWC; pNew = pBuilder->pNew; pSrc = &pWInfo->pTabList->a[pNew->iTab]; - assert( IsVirtual(pSrc->pTab) ); - p = allocateIndexInfo(pParse, pWC, mUnusable, pSrc, pBuilder->pOrderBy); + assert( IsVirtual(pSrc->pSTab) ); + p = allocateIndexInfo(pWInfo, pWC, mUnusable, pSrc, &mNoOmit); if( p==0 ) return SQLITE_NOMEM_BKPT; pNew->rSetup = 0; pNew->wsFlags = WHERE_VIRTUALTABLE; @@ -128938,20 +167744,29 @@ static int whereLoopAddVirtual( pNew->u.vtab.needFree = 0; nConstraint = p->nConstraint; if( whereLoopResize(pParse->db, pNew, nConstraint) ){ - sqlite3DbFree(pParse->db, p); + freeIndexInfo(pParse->db, p); return SQLITE_NOMEM_BKPT; } /* First call xBestIndex() with all constraints usable. */ - WHERETRACE(0x40, (" VirtualOne: all usable\n")); - rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, 0, p, &bIn); + WHERETRACE(0x800, ("BEGIN %s.addVirtual()\n", pSrc->pSTab->zName)); + WHERETRACE(0x800, (" VirtualOne: all usable\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, &bRetry + ); + if( bRetry ){ + assert( rc==SQLITE_OK ); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, 0, p, mNoOmit, &bIn, 0 + ); + } /* If the call to xBestIndex() with all terms enabled produced a plan - ** that does not require any source tables (IOW: a plan with mBest==0), - ** then there is no point in making any further calls to xBestIndex() - ** since they will all return the same result (if the xBestIndex() - ** implementation is sane). */ - if( rc==SQLITE_OK && (mBest = (pNew->prereq & ~mPrereq))!=0 ){ + ** that does not require any source tables (IOW: a plan with mBest==0) + ** and does not use an IN(...) operator, then there is no point in making + ** any further calls to xBestIndex() since they will all return the same + ** result (if the xBestIndex() implementation is sane). */ + if( rc==SQLITE_OK && ((mBest = (pNew->prereq & ~mPrereq))!=0 || bIn) ){ int seenZero = 0; /* True if a plan with no prereqs seen */ int seenZeroNoIN = 0; /* Plan with no prereqs and no IN(...) seen */ Bitmask mPrev = 0; @@ -128960,8 +167775,9 @@ static int whereLoopAddVirtual( /* If the plan produced by the earlier call uses an IN(...) term, call ** xBestIndex again, this time with IN(...) terms disabled. */ if( bIn ){ - WHERETRACE(0x40, (" VirtualOne: all usable w/o IN\n")); - rc = whereLoopAddVirtualOne(pBuilder, mPrereq, ALLBITS, WO_IN, p, &bIn); + WHERETRACE(0x800, (" VirtualOne: all usable w/o IN\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, ALLBITS, WO_IN, p, mNoOmit, &bIn, 0); assert( bIn==0 ); mBestNoIn = pNew->prereq & ~mPrereq; if( mBestNoIn==0 ){ @@ -128970,24 +167786,24 @@ static int whereLoopAddVirtual( } } - /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) + /* Call xBestIndex once for each distinct value of (prereqRight & ~mPrereq) ** in the set of terms that apply to the current virtual table. */ while( rc==SQLITE_OK ){ int i; Bitmask mNext = ALLBITS; assert( mNext>0 ); for(i=0; ia[p->aConstraint[i].iTermOffset].prereqRight & ~mPrereq - ); + int iTerm = p->aConstraint[i].iTermOffset; + Bitmask mThis = termFromWhereClause(pWC, iTerm)->prereqRight & ~mPrereq; if( mThis>mPrev && mThisprereq==mPrereq ){ seenZero = 1; if( bIn==0 ) seenZeroNoIN = 1; @@ -128998,8 +167814,9 @@ static int whereLoopAddVirtual( ** that requires no source tables at all (i.e. one guaranteed to be ** usable), make a call here with all source tables disabled */ if( rc==SQLITE_OK && seenZero==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled\n")); - rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mPrereq, 0, p, &bIn); + WHERETRACE(0x800, (" VirtualOne: all disabled\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, mPrereq, 0, p, mNoOmit, &bIn, 0); if( bIn==0 ) seenZeroNoIN = 1; } @@ -129007,13 +167824,14 @@ static int whereLoopAddVirtual( ** that requires no source tables at all and does not use an IN(...) ** operator, make a final call to obtain one here. */ if( rc==SQLITE_OK && seenZeroNoIN==0 ){ - WHERETRACE(0x40, (" VirtualOne: all disabled and w/o IN\n")); - rc = whereLoopAddVirtualOne(pBuilder, mPrereq, mPrereq, WO_IN, p, &bIn); + WHERETRACE(0x800, (" VirtualOne: all disabled and w/o IN\n")); + rc = whereLoopAddVirtualOne( + pBuilder, mPrereq, mPrereq, WO_IN, p, mNoOmit, &bIn, 0); } } - if( p->needToFreeIdxStr ) sqlite3_free(p->idxStr); - sqlite3DbFree(pParse->db, p); + freeIndexInfo(pParse->db, p); + WHERETRACE(0x800, ("END %s.addVirtual(), rc=%d\n", pSrc->pSTab->zName, rc)); return rc; } #endif /* SQLITE_OMIT_VIRTUALTABLE */ @@ -129023,8 +167841,8 @@ static int whereLoopAddVirtual( ** btrees or virtual tables. */ static int whereLoopAddOr( - WhereLoopBuilder *pBuilder, - Bitmask mPrereq, + WhereLoopBuilder *pBuilder, + Bitmask mPrereq, Bitmask mUnusable ){ WhereInfo *pWInfo = pBuilder->pWInfo; @@ -129036,8 +167854,8 @@ static int whereLoopAddOr( WhereClause tempWC; WhereLoopBuilder sSubBuild; WhereOrSet sSum, sCur; - struct SrcList_item *pItem; - + SrcItem *pItem; + pWC = pBuilder->pWC; pWCEnd = pWC->a + pWC->nTerm; pNew = pBuilder->pNew; @@ -129045,21 +167863,23 @@ static int whereLoopAddOr( pItem = pWInfo->pTabList->a + pNew->iTab; iCur = pItem->iCursor; + /* The multi-index OR optimization does not work for RIGHT and FULL JOIN */ + if( pItem->fg.jointype & JT_RIGHT ) return SQLITE_OK; + for(pTerm=pWC->a; pTermeOperator & WO_OR)!=0 - && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 + && (pTerm->u.pOrInfo->indexable & pNew->maskSelf)!=0 ){ WhereClause * const pOrWC = &pTerm->u.pOrInfo->wc; WhereTerm * const pOrWCEnd = &pOrWC->a[pOrWC->nTerm]; WhereTerm *pOrTerm; int once = 1; int i, j; - + sSubBuild = *pBuilder; - sSubBuild.pOrderBy = 0; sSubBuild.pOrSet = &sCur; - WHERETRACE(0x200, ("Begin processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("Begin processing OR-clause %p\n", pTerm)); for(pOrTerm=pOrWC->a; pOrTermeOperator & WO_AND)!=0 ){ sSubBuild.pWC = &pOrTerm->u.pAndInfo->wc; @@ -129068,6 +167888,7 @@ static int whereLoopAddOr( tempWC.pOuter = pWC; tempWC.op = TK_AND; tempWC.nTerm = 1; + tempWC.nBase = 1; tempWC.a = pOrTerm; sSubBuild.pWC = &tempWC; }else{ @@ -129075,16 +167896,14 @@ static int whereLoopAddOr( } sCur.n = 0; #ifdef WHERETRACE_ENABLED - WHERETRACE(0x200, ("OR-term %d of %p has %d subterms:\n", + WHERETRACE(0x400, ("OR-term %d of %p has %d subterms:\n", (int)(pOrTerm-pOrWC->a), pTerm, sSubBuild.pWC->nTerm)); - if( sqlite3WhereTrace & 0x400 ){ - for(i=0; inTerm; i++){ - whereTermPrint(&sSubBuild.pWC->a[i], i); - } + if( sqlite3WhereTrace & 0x20000 ){ + sqlite3WhereClausePrint(sSubBuild.pWC); } #endif #ifndef SQLITE_OMIT_VIRTUALTABLE - if( IsVirtual(pItem->pTab) ){ + if( IsVirtual(pItem->pSTab) ){ rc = whereLoopAddVirtual(&sSubBuild, mPrereq, mUnusable); }else #endif @@ -129094,7 +167913,8 @@ static int whereLoopAddOr( if( rc==SQLITE_OK ){ rc = whereLoopAddOr(&sSubBuild, mPrereq, mUnusable); } - assert( rc==SQLITE_OK || sCur.n==0 ); + testcase( rc==SQLITE_NOMEM && sCur.n>0 ); + testcase( rc==SQLITE_DONE ); if( sCur.n==0 ){ sSum.n = 0; break; @@ -129124,8 +167944,8 @@ static int whereLoopAddOr( /* TUNING: Currently sSum.a[i].rRun is set to the sum of the costs ** of all sub-scans required by the OR-scan. However, due to rounding ** errors, it may be that the cost of the OR-scan is equal to its - ** most expensive sub-scan. Add the smallest possible penalty - ** (equivalent to multiplying the cost by 1.07) to ensure that + ** most expensive sub-scan. Add the smallest possible penalty + ** (equivalent to multiplying the cost by 1.07) to ensure that ** this does not happen. Otherwise, for WHERE clauses such as the ** following where there is an index on "y": ** @@ -129138,14 +167958,14 @@ static int whereLoopAddOr( pNew->prereq = sSum.a[i].prereq; rc = whereLoopInsert(pBuilder, pNew); } - WHERETRACE(0x200, ("End processing OR-clause %p\n", pTerm)); + WHERETRACE(0x400, ("End processing OR-clause %p\n", pTerm)); } } return rc; } /* -** Add all WhereLoop objects for all tables +** Add all WhereLoop objects for all tables */ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ WhereInfo *pWInfo = pBuilder->pWInfo; @@ -129153,70 +167973,195 @@ static int whereLoopAddAll(WhereLoopBuilder *pBuilder){ Bitmask mPrior = 0; int iTab; SrcList *pTabList = pWInfo->pTabList; - struct SrcList_item *pItem; - struct SrcList_item *pEnd = &pTabList->a[pWInfo->nLevel]; + SrcItem *pItem; + SrcItem *pEnd = &pTabList->a[pWInfo->nLevel]; sqlite3 *db = pWInfo->pParse->db; int rc = SQLITE_OK; + int bFirstPastRJ = 0; + int hasRightJoin = 0; WhereLoop *pNew; - u8 priorJointype = 0; + /* Loop over the tables in the join, from left to right */ pNew = pBuilder->pNew; - whereLoopInit(pNew); + + /* Verify that pNew has already been initialized */ + assert( pNew->nLTerm==0 ); + assert( pNew->wsFlags==0 ); + assert( pNew->nLSlot>=ArraySize(pNew->aLTermSpace) ); + assert( pNew->aLTerm!=0 ); + + pBuilder->iPlanLimit = SQLITE_QUERY_PLANNER_LIMIT; for(iTab=0, pItem=pTabList->a; pItemiTab = iTab; + pBuilder->iPlanLimit += SQLITE_QUERY_PLANNER_LIMIT_INCR; pNew->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, pItem->iCursor); - if( ((pItem->fg.jointype|priorJointype) & (JT_LEFT|JT_CROSS))!=0 ){ - /* This condition is true when pItem is the FROM clause term on the - ** right-hand-side of a LEFT or CROSS JOIN. */ - mPrereq = mPrior; + if( bFirstPastRJ + || (pItem->fg.jointype & (JT_OUTER|JT_CROSS|JT_LTORJ))!=0 + ){ + /* Add prerequisites to prevent reordering of FROM clause terms + ** across CROSS joins and outer joins. The bFirstPastRJ boolean + ** prevents the right operand of a RIGHT JOIN from being swapped with + ** other elements even further to the right. + ** + ** The JT_LTORJ case and the hasRightJoin flag work together to + ** prevent FROM-clause terms from moving from the right side of + ** a LEFT JOIN over to the left side of that join if the LEFT JOIN + ** is itself on the left side of a RIGHT JOIN. + */ + if( pItem->fg.jointype & JT_LTORJ ) hasRightJoin = 1; + mPrereq |= mPrior; + bFirstPastRJ = (pItem->fg.jointype & JT_RIGHT)!=0; + }else if( !hasRightJoin ){ + mPrereq = 0; } - priorJointype = pItem->fg.jointype; - if( IsVirtual(pItem->pTab) ){ - struct SrcList_item *p; +#ifndef SQLITE_OMIT_VIRTUALTABLE + if( IsVirtual(pItem->pSTab) ){ + SrcItem *p; for(p=&pItem[1]; pfg.jointype & (JT_LEFT|JT_CROSS)) ){ + if( mUnusable || (p->fg.jointype & (JT_OUTER|JT_CROSS)) ){ mUnusable |= sqlite3WhereGetMask(&pWInfo->sMaskSet, p->iCursor); } } rc = whereLoopAddVirtual(pBuilder, mPrereq, mUnusable); - }else{ + }else +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + { rc = whereLoopAddBtree(pBuilder, mPrereq); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pBuilder->pWC->hasOr ){ rc = whereLoopAddOr(pBuilder, mPrereq, mUnusable); } mPrior |= pNew->maskSelf; - if( rc || db->mallocFailed ) break; + if( rc || db->mallocFailed ){ + if( rc==SQLITE_DONE ){ + /* We hit the query planner search limit set by iPlanLimit */ + sqlite3_log(SQLITE_WARNING, "abbreviated query algorithm search"); + rc = SQLITE_OK; + }else{ + break; + } + } } whereLoopClear(db, pNew); return rc; } +/* Implementation of the order-by-subquery optimization: +** +** WhereLoop pLoop, which the iLoop-th term of the nested loop, is really +** a subquery or CTE that has an ORDER BY clause. See if any of the terms +** in the subquery ORDER BY clause will satisfy pOrderBy from the outer +** query. Mark off all satisfied terms (by setting bits in *pOBSat) and +** return TRUE if they do. If not, return false. +** +** Example: +** +** CREATE TABLE t1(a,b,c, PRIMARY KEY(a,b)); +** CREATE TABLE t2(x,y); +** WITH t3(p,q) AS MATERIALIZED (SELECT x+y, x-y FROM t2 ORDER BY x+y) +** SELECT * FROM t3 JOIN t1 ON a=q ORDER BY p, b; +** +** The CTE named "t3" comes out in the natural order of "p", so the first +** first them of "ORDER BY p,b" is satisfied by a sequential scan of "t3" +** and sorting only needs to occur on the second term "b". +** +** Limitations: +** +** (1) The optimization is not applied if the outer ORDER BY contains +** a COLLATE clause. The optimization might be applied if the +** outer ORDER BY uses NULLS FIRST, NULLS LAST, ASC, and/or DESC as +** long as the subquery ORDER BY does the same. But if the +** outer ORDER BY uses COLLATE, even a redundant COLLATE, the +** optimization is bypassed. +** +** (2) The subquery ORDER BY terms must exactly match subquery result +** columns, including any COLLATE annotations. This routine relies +** on iOrderByCol to do matching between order by terms and result +** columns, and iOrderByCol will not be set if the result column +** and ORDER BY collations differ. +** +** (3) The subquery and outer ORDER BY can be in opposite directions as +** long as the subquery is materialized. If the subquery is +** implemented as a co-routine, the sort orders must be in the same +** direction because there is no way to run a co-routine backwards. +*/ +static SQLITE_NOINLINE int wherePathMatchSubqueryOB( + WhereInfo *pWInfo, /* The WHERE clause */ + WhereLoop *pLoop, /* The nested loop term that is a subquery */ + int iLoop, /* Which level of the nested loop. 0==outermost */ + int iCur, /* Cursor used by the this loop */ + ExprList *pOrderBy, /* The ORDER BY clause on the whole query */ + Bitmask *pRevMask, /* When loops need to go in reverse order */ + Bitmask *pOBSat /* Which terms of pOrderBy are satisfied so far */ +){ + int iOB; /* Index into pOrderBy->a[] */ + int jSub; /* Index into pSubOB->a[] */ + u8 rev = 0; /* True if iOB and jSub sort in opposite directions */ + u8 revIdx = 0; /* Sort direction for jSub */ + Expr *pOBExpr; /* Current term of outer ORDER BY */ + ExprList *pSubOB; /* Complete ORDER BY on the subquery */ + + pSubOB = pLoop->u.btree.pOrderBy; + assert( pSubOB!=0 ); + for(iOB=0; (MASKBIT(iOB) & *pOBSat)!=0; iOB++){} + for(jSub=0; jSubnExpr && iOBnExpr; jSub++, iOB++){ + if( pSubOB->a[jSub].u.x.iOrderByCol==0 ) break; + pOBExpr = pOrderBy->a[iOB].pExpr; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) break; + if( pOBExpr->iTable!=iCur ) break; + if( pOBExpr->iColumn!=pSubOB->a[jSub].u.x.iOrderByCol-1 ) break; + if( (pWInfo->wctrlFlags & WHERE_GROUPBY)==0 ){ + u8 sfOB = pOrderBy->a[iOB].fg.sortFlags; /* sortFlags for iOB */ + u8 sfSub = pSubOB->a[jSub].fg.sortFlags; /* sortFlags for jSub */ + if( (sfSub & KEYINFO_ORDER_BIGNULL) != (sfOB & KEYINFO_ORDER_BIGNULL) ){ + break; + } + revIdx = sfSub & KEYINFO_ORDER_DESC; + if( jSub>0 ){ + if( (rev^revIdx)!=(sfOB & KEYINFO_ORDER_DESC) ){ + break; + } + }else{ + rev = revIdx ^ (sfOB & KEYINFO_ORDER_DESC); + if( rev ){ + if( (pLoop->wsFlags & WHERE_COROUTINE)!=0 ){ + /* Cannot run a co-routine in reverse order */ + break; + } + *pRevMask |= MASKBIT(iLoop); + } + } + } + *pOBSat |= MASKBIT(iOB); + } + return jSub>0; +} + /* -** Examine a WherePath (with the addition of the extra WhereLoop of the 5th +** Examine a WherePath (with the addition of the extra WhereLoop of the 6th ** parameters) to see if it outputs rows in the requested ORDER BY ** (or GROUP BY) without requiring a separate sort operation. Return N: -** +** ** N>0: N terms of the ORDER BY clause are satisfied ** N==0: No terms of the ORDER BY clause are satisfied -** N<0: Unknown yet how many terms of ORDER BY might be satisfied. +** N<0: Unknown yet how many terms of ORDER BY might be satisfied. ** ** Note that processing for WHERE_GROUPBY and WHERE_DISTINCTBY is not as ** strict. With GROUP BY and DISTINCT the only requirement is that ** equivalent rows appear immediately adjacent to one another. GROUP BY ** and DISTINCT do not require rows to appear in any particular order as long ** as equivalent rows are grouped together. Thus for GROUP BY and DISTINCT -** the pOrderBy terms can be matched in any order. With ORDER BY, the +** the pOrderBy terms can be matched in any order. With ORDER BY, the ** pOrderBy terms must be matched in strict left-to-right order. */ static i8 wherePathSatisfiesOrderBy( WhereInfo *pWInfo, /* The WHERE clause */ ExprList *pOrderBy, /* ORDER BY or GROUP BY or DISTINCT clause to check */ WherePath *pPath, /* The WherePath to check */ - u16 wctrlFlags, /* Might contain WHERE_GROUPBY or WHERE_DISTINCTBY */ + u16 wctrlFlags, /* WHERE_GROUPBY or _DISTINCTBY or _ORDERBY_LIMIT */ u16 nLoop, /* Number of entries in pPath->aLoop[] */ WhereLoop *pLast, /* Add this WhereLoop to the end of pPath->aLoop[] */ Bitmask *pRevMask /* OUT: Mask of WhereLoops to run in reverse order */ @@ -129227,6 +168172,7 @@ static i8 wherePathSatisfiesOrderBy( u8 isOrderDistinct; /* All prior WhereLoops are order-distinct */ u8 distinctColumns; /* True if the loop has UNIQUE NOT NULL columns */ u8 isMatch; /* iColumn matches a term of the ORDER BY clause */ + u16 eqOpMask; /* Allowed equality operators */ u16 nKeyCol; /* Number of key columns in pIndex */ u16 nColumn; /* Total number of ordered columns in the index */ u16 nOrderBy; /* Number terms in the ORDER BY clause */ @@ -129258,7 +168204,7 @@ static i8 wherePathSatisfiesOrderBy( ** row of the WhereLoop. Every one-row WhereLoop is automatically ** order-distinct. A WhereLoop that has no columns in the ORDER BY clause ** is not order-distinct. To be order-distinct is not quite the same as being - ** UNIQUE since a UNIQUE column or index can have multiple rows that + ** UNIQUE since a UNIQUE column or index can have multiple rows that ** are NULL and NULL values are equivalent for the purpose of order-distinct. ** To be order-distinct, the columns must be UNIQUE and NOT NULL. ** @@ -129277,12 +168223,27 @@ static i8 wherePathSatisfiesOrderBy( obDone = MASKBIT(nOrderBy)-1; orderDistinctMask = 0; ready = 0; + eqOpMask = WO_EQ | WO_IS | WO_ISNULL; + if( wctrlFlags & (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MAX|WHERE_ORDERBY_MIN) ){ + eqOpMask |= WO_IN; + } for(iLoop=0; isOrderDistinct && obSat0 ) ready |= pLoop->maskSelf; - pLoop = iLoopaLoop[iLoop] : pLast; + if( iLoopaLoop[iLoop]; + if( wctrlFlags & WHERE_ORDERBY_LIMIT ) continue; + }else{ + pLoop = pLast; + } if( pLoop->wsFlags & WHERE_VIRTUALTABLE ){ - if( pLoop->u.vtab.isOrdered ) obSat = obDone; + if( pLoop->u.vtab.isOrdered + && ((wctrlFlags&(WHERE_DISTINCTBY|WHERE_SORTBYGROUP))!=WHERE_DISTINCTBY) + ){ + obSat = obDone; + } break; + }else if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = 0; } iCur = pWInfo->pTabList->a[pLoop->iTab].iCursor; @@ -129293,21 +168254,30 @@ static i8 wherePathSatisfiesOrderBy( */ for(i=0; ia[i].pExpr); - if( pOBExpr->op!=TK_COLUMN ) continue; + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); + if( NEVER(pOBExpr==0) ) continue; + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; pTerm = sqlite3WhereFindTerm(&pWInfo->sWC, iCur, pOBExpr->iColumn, - ~ready, WO_EQ|WO_ISNULL|WO_IS, 0); + ~ready, eqOpMask, 0); if( pTerm==0 ) continue; + if( pTerm->eOperator==WO_IN ){ + /* IN terms are only valid for sorting in the ORDER BY LIMIT + ** optimization, and then only if they are actually used + ** by the query plan */ + assert( wctrlFlags & + (WHERE_ORDERBY_LIMIT|WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX) ); + for(j=0; jnLTerm && pTerm!=pLoop->aLTerm[j]; j++){} + if( j>=pLoop->nLTerm ) continue; + } if( (pTerm->eOperator&(WO_EQ|WO_IS))!=0 && pOBExpr->iColumn>=0 ){ - const char *z1, *z2; - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); - if( !pColl ) pColl = db->pDfltColl; - z1 = pColl->zName; - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pTerm->pExpr); - if( !pColl ) pColl = db->pDfltColl; - z2 = pColl->zName; - if( sqlite3StrICmp(z1, z2)!=0 ) continue; + Parse *pParse = pWInfo->pParse; + CollSeq *pColl1 = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[i].pExpr); + CollSeq *pColl2 = sqlite3ExprCompareCollSeq(pParse, pTerm->pExpr); + assert( pColl1 ); + if( pColl2==0 || sqlite3StrICmp(pColl1->zName, pColl2->zName) ){ + continue; + } testcase( pTerm->pExpr->op==TK_IS ); } obSat |= MASKBIT(i); @@ -129315,9 +168285,18 @@ static i8 wherePathSatisfiesOrderBy( if( (pLoop->wsFlags & WHERE_ONEROW)==0 ){ if( pLoop->wsFlags & WHERE_IPK ){ + if( pLoop->u.btree.pOrderBy + && OptimizationEnabled(db, SQLITE_OrderBySubq) + && wherePathMatchSubqueryOB(pWInfo,pLoop,iLoop,iCur, + pOrderBy,pRevMask, &obSat) + ){ + nColumn = 0; + isOrderDistinct = 0; + }else{ + nColumn = 1; + } pIndex = 0; nKeyCol = 0; - nColumn = 1; }else if( (pIndex = pLoop->u.btree.pIndex)==0 || pIndex->bUnordered ){ return 0; }else{ @@ -129326,7 +168305,12 @@ static i8 wherePathSatisfiesOrderBy( assert( nColumn==nKeyCol+1 || !HasRowid(pIndex->pTable) ); assert( pIndex->aiColumn[nColumn-1]==XN_ROWID || !HasRowid(pIndex->pTable)); - isOrderDistinct = IsUniqueIndex(pIndex); + /* All relevant terms of the index must also be non-NULL in order + ** for isOrderDistinct to be true. So the isOrderDistinct value + ** computed here might be a false positive. Corrections will be + ** made at tag-20210426-1 below */ + isOrderDistinct = IsUniqueIndex(pIndex) + && (pLoop->wsFlags & WHERE_SKIPSCAN)==0; } /* Loop through all columns of the index and deal with the ones @@ -129335,18 +168319,48 @@ static i8 wherePathSatisfiesOrderBy( rev = revSet = 0; distinctColumns = 0; for(j=0; ju.btree.nEq - && pLoop->nSkip==0 - && ((i = pLoop->aLTerm[j]->eOperator) & (WO_EQ|WO_ISNULL|WO_IS))!=0 - ){ - if( i & WO_ISNULL ){ - testcase( isOrderDistinct ); - isOrderDistinct = 0; + assert( j>=pLoop->u.btree.nEq + || (pLoop->aLTerm[j]==0)==(jnSkip) + ); + if( ju.btree.nEq && j>=pLoop->nSkip ){ + u16 eOp = pLoop->aLTerm[j]->eOperator; + + /* Skip over == and IS and ISNULL terms. (Also skip IN terms when + ** doing WHERE_ORDERBY_LIMIT processing). Except, IS and ISNULL + ** terms imply that the index is not UNIQUE NOT NULL in which case + ** the loop need to be marked as not order-distinct because it can + ** have repeated NULL rows. + ** + ** If the current term is a column of an ((?,?) IN (SELECT...)) + ** expression for which the SELECT returns more than one column, + ** check that it is the only column used by this loop. Otherwise, + ** if it is one of two or more, none of the columns can be + ** considered to match an ORDER BY term. + */ + if( (eOp & eqOpMask)!=0 ){ + if( eOp & (WO_ISNULL|WO_IS) ){ + testcase( eOp & WO_ISNULL ); + testcase( eOp & WO_IS ); + testcase( isOrderDistinct ); + isOrderDistinct = 0; + } + continue; + }else if( ALWAYS(eOp & WO_IN) ){ + /* ALWAYS() justification: eOp is an equality operator due to the + ** ju.btree.nEq constraint above. Any equality other + ** than WO_IN is captured by the previous "if". So this one + ** always has to be WO_IN. */ + Expr *pX = pLoop->aLTerm[j]->pExpr; + for(i=j+1; iu.btree.nEq; i++){ + if( pLoop->aLTerm[i]->pExpr==pX ){ + assert( (pLoop->aLTerm[i]->eOperator & WO_IN) ); + bOnce = 0; + break; + } + } } - continue; } /* Get the column number in the table (iColumn) and sort order @@ -129354,49 +168368,56 @@ static i8 wherePathSatisfiesOrderBy( */ if( pIndex ){ iColumn = pIndex->aiColumn[j]; - revIdx = pIndex->aSortOrder[j]; - if( iColumn==pIndex->pTable->iPKey ) iColumn = -1; + revIdx = pIndex->aSortOrder[j] & KEYINFO_ORDER_DESC; + if( iColumn==pIndex->pTable->iPKey ) iColumn = XN_ROWID; }else{ iColumn = XN_ROWID; revIdx = 0; } /* An unconstrained column that might be NULL means that this - ** WhereLoop is not well-ordered + ** WhereLoop is not well-ordered. tag-20210426-1 */ - if( isOrderDistinct - && iColumn>=0 - && j>=pLoop->u.btree.nEq - && pIndex->pTable->aCol[iColumn].notNull==0 - ){ - isOrderDistinct = 0; + if( isOrderDistinct ){ + if( iColumn>=0 + && j>=pLoop->u.btree.nEq + && pIndex->pTable->aCol[iColumn].notNull==0 + ){ + isOrderDistinct = 0; + } + if( iColumn==XN_EXPR ){ + isOrderDistinct = 0; + } } /* Find the ORDER BY term that corresponds to the j-th column - ** of the index and mark that ORDER BY term off + ** of the index and mark that ORDER BY term having been satisfied. */ - bOnce = 1; isMatch = 0; for(i=0; bOnce && ia[i].pExpr); + pOBExpr = sqlite3ExprSkipCollateAndLikely(pOrderBy->a[i].pExpr); testcase( wctrlFlags & WHERE_GROUPBY ); testcase( wctrlFlags & WHERE_DISTINCTBY ); + if( NEVER(pOBExpr==0) ) continue; if( (wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY))==0 ) bOnce = 0; - if( iColumn>=(-1) ){ - if( pOBExpr->op!=TK_COLUMN ) continue; + if( iColumn>=XN_ROWID ){ + if( pOBExpr->op!=TK_COLUMN && pOBExpr->op!=TK_AGG_COLUMN ) continue; if( pOBExpr->iTable!=iCur ) continue; if( pOBExpr->iColumn!=iColumn ) continue; }else{ - if( sqlite3ExprCompare(pOBExpr,pIndex->aColExpr->a[j].pExpr,iCur) ){ + Expr *pIxExpr = pIndex->aColExpr->a[j].pExpr; + if( sqlite3ExprCompareSkip(pOBExpr, pIxExpr, iCur) ){ continue; } } - if( iColumn>=0 ){ - pColl = sqlite3ExprCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); - if( !pColl ) pColl = db->pDfltColl; + if( iColumn!=XN_ROWID ){ + pColl = sqlite3ExprNNCollSeq(pWInfo->pParse, pOrderBy->a[i].pExpr); if( sqlite3StrICmp(pColl->zName, pIndex->azColl[j])!=0 ) continue; } + if( wctrlFlags & WHERE_DISTINCTBY ){ + pLoop->u.btree.nDistinctCol = j+1; + } isMatch = 1; break; } @@ -129404,15 +168425,26 @@ static i8 wherePathSatisfiesOrderBy( /* Make sure the sort order is compatible in an ORDER BY clause. ** Sort order is irrelevant for a GROUP BY clause. */ if( revSet ){ - if( (rev ^ revIdx)!=pOrderBy->a[i].sortOrder ) isMatch = 0; + if( (rev ^ revIdx) + != (pOrderBy->a[i].fg.sortFlags&KEYINFO_ORDER_DESC) + ){ + isMatch = 0; + } }else{ - rev = revIdx ^ pOrderBy->a[i].sortOrder; + rev = revIdx ^ (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_DESC); if( rev ) *pRevMask |= MASKBIT(iLoop); revSet = 1; } } + if( isMatch && (pOrderBy->a[i].fg.sortFlags & KEYINFO_ORDER_BIGNULL) ){ + if( j==pLoop->u.btree.nEq ){ + pLoop->wsFlags |= WHERE_BIGNULL_SORT; + }else{ + isMatch = 0; + } + } if( isMatch ){ - if( iColumn<0 ){ + if( iColumn==XN_ROWID ){ testcase( distinctColumns==0 ); distinctColumns = 1; } @@ -129441,7 +168473,7 @@ static i8 wherePathSatisfiesOrderBy( if( MASKBIT(i) & obSat ) continue; p = pOrderBy->a[i].pExpr; mTerm = sqlite3WhereExprUsage(&pWInfo->sMaskSet,p); - if( mTerm==0 && !sqlite3ExprIsConstant(p) ) continue; + if( mTerm==0 && !sqlite3ExprIsConstant(0,p) ) continue; if( (mTerm&~orderDistinctMask)==0 ){ obSat |= MASKBIT(i); } @@ -129451,7 +168483,7 @@ static i8 wherePathSatisfiesOrderBy( if( obSat==obDone ) return (i8)nOrderBy; if( !isOrderDistinct ){ for(i=nOrderBy-1; i>0; i--){ - Bitmask m = MASKBIT(i) - 1; + Bitmask m = ALWAYS(iwctrlFlags & WHERE_GROUPBY ); + assert( pWInfo->wctrlFlags & (WHERE_GROUPBY|WHERE_DISTINCTBY) ); assert( pWInfo->wctrlFlags & WHERE_SORTBYGROUP ); return pWInfo->sorted; } @@ -129502,43 +168534,147 @@ static const char *wherePathName(WherePath *pPath, int nLoop, WhereLoop *pLast){ #endif /* -** Return the cost of sorting nRow rows, assuming that the keys have +** Return the cost of sorting nRow rows, assuming that the keys have ** nOrderby columns and that the first nSorted columns are already in ** order. */ static LogEst whereSortingCost( - WhereInfo *pWInfo, - LogEst nRow, - int nOrderBy, - int nSorted + WhereInfo *pWInfo, /* Query planning context */ + LogEst nRow, /* Estimated number of rows to sort */ + int nOrderBy, /* Number of ORDER BY clause terms */ + int nSorted /* Number of initial ORDER BY terms naturally in order */ ){ - /* TUNING: Estimated cost of a full external sort, where N is + /* Estimated cost of a full external sort, where N is ** the number of rows to sort is: ** - ** cost = (3.0 * N * log(N)). - ** - ** Or, if the order-by clause has X terms but only the last Y - ** terms are out of order, then block-sorting will reduce the + ** cost = (K * N * log(N)). + ** + ** Or, if the order-by clause has X terms but only the last Y + ** terms are out of order, then block-sorting will reduce the ** sorting cost to: ** - ** cost = (3.0 * N * log(N)) * (Y/X) + ** cost = (K * N * log(N)) * (Y/X) ** - ** The (Y/X) term is implemented using stack variable rScale - ** below. */ - LogEst rScale, rSortCost; - assert( nOrderBy>0 && 66==sqlite3LogEst(100) ); - rScale = sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; - rSortCost = nRow + rScale + 16; + ** The constant K is at least 2.0 but will be larger if there are a + ** large number of columns to be sorted, as the sorting time is + ** proportional to the amount of content to be sorted. The algorithm + ** does not currently distinguish between fat columns (BLOBs and TEXTs) + ** and skinny columns (INTs). It just uses the number of columns as + ** an approximation for the row width. + ** + ** And extra factor of 2.0 or 3.0 is added to the sorting cost if the sort + ** is built using OP_IdxInsert and OP_Sort rather than with OP_SorterInsert. + */ + LogEst rSortCost, nCol; + assert( pWInfo->pSelect!=0 ); + assert( pWInfo->pSelect->pEList!=0 ); + /* TUNING: sorting cost proportional to the number of output columns: */ + nCol = sqlite3LogEst((pWInfo->pSelect->pEList->nExpr+59)/30); + rSortCost = nRow + nCol; + if( nSorted>0 ){ + /* Scale the result by (Y/X) */ + rSortCost += sqlite3LogEst((nOrderBy-nSorted)*100/nOrderBy) - 66; + } /* Multiple by log(M) where M is the number of output rows. - ** Use the LIMIT for M if it is smaller */ - if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 && pWInfo->iLimitiLimit; + ** Use the LIMIT for M if it is smaller. Or if this sort is for + ** a DISTINCT operator, M will be the number of distinct output + ** rows, so fudge it downwards a bit. + */ + if( (pWInfo->wctrlFlags & WHERE_USE_LIMIT)!=0 ){ + rSortCost += 10; /* TUNING: Extra 2.0x if using LIMIT */ + if( nSorted!=0 ){ + rSortCost += 6; /* TUNING: Extra 1.5x if also using partial sort */ + } + if( pWInfo->iLimitiLimit; + } + }else if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT) ){ + /* TUNING: In the sort for a DISTINCT operator, assume that the DISTINCT + ** reduces the number of output rows by a factor of 2 */ + if( nRow>10 ){ nRow -= 10; assert( 10==sqlite3LogEst(2) ); } } rSortCost += estLog(nRow); return rSortCost; } +/* +** Compute the maximum number of paths in the solver algorithm, for +** queries that have three or more terms in the FROM clause. Queries with +** two or fewer FROM clause terms are handled by the caller. +** +** Query planning is NP-hard. We must limit the number of paths at +** each step of the solver search algorithm to avoid exponential behavior. +** +** The value returned is a tuning parameter. Currently the value is: +** +** 18 for star queries +** 12 otherwise +** +** For the purposes of SQLite, a star-query is defined as a query +** with a large central table that is joined against four or more +** smaller tables. The central table is called the "fact" table. +** The smaller tables that get joined are "dimension tables". +** +** SIDE EFFECT: (and really the whole point of this subroutine) +** +** If pWInfo describes a star-query, then the cost on WhereLoops for the +** fact table is reduced. This heuristic helps keep fact tables in +** outer loops. Without this heuristic, paths with fact tables in outer +** loops tend to get pruned by the mxChoice limit on the number of paths, +** resulting in poor query plans. The total amount of heuristic cost +** adjustment is stored in pWInfo->nOutStarDelta and the cost adjustment +** for each WhereLoop is stored in its rStarDelta field. +*/ +static int computeMxChoice(WhereInfo *pWInfo, LogEst nRowEst){ + int nLoop = pWInfo->nLevel; /* Number of terms in the join */ + if( nRowEst==0 && nLoop>=5 ){ + /* Check to see if we are dealing with a star schema and if so, reduce + ** the cost of fact tables relative to dimension tables, as a heuristic + ** to help keep the fact tables in outer loops. + */ + int iLoop; /* Counter over join terms */ + Bitmask m; /* Bitmask for current loop */ + assert( pWInfo->nOutStarDelta==0 ); + for(iLoop=0, m=1; iLooppLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( (pWLoop->prereq & m)!=0 && (pWLoop->maskSelf & mSeen)==0 ){ + nDep++; + mSeen |= pWLoop->maskSelf; + } + } + if( nDep<=3 ) continue; + rDelta = 15*(nDep-3); +#ifdef WHERETRACE_ENABLED /* 0x4 */ + if( sqlite3WhereTrace&0x4 ){ + SrcItem *pItem = pWInfo->pTabList->a + iLoop; + sqlite3DebugPrintf("Fact-table %s: %d dimensions, cost reduced %d\n", + pItem->zAlias ? pItem->zAlias : pItem->pSTab->zName, + nDep, rDelta); + } +#endif + if( pWInfo->nOutStarDelta==0 ){ + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + pWLoop->rStarDelta = 0; + } + } + pWInfo->nOutStarDelta += rDelta; + for(pWLoop=pWInfo->pLoops; pWLoop; pWLoop=pWLoop->pNextLoop){ + if( pWLoop->maskSelf==m ){ + pWLoop->rRun -= rDelta; + pWLoop->nOut -= rDelta; + pWLoop->rStarDelta = rDelta; + } + } + } + } + return pWInfo->nOutStarDelta>0 ? 18 : 12; +} + /* ** Given the list of WhereLoop objects at pWInfo->pLoops, this routine ** attempts to find the lowest cost path that visits each WhereLoop @@ -129555,7 +168691,6 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int mxChoice; /* Maximum number of simultaneous paths tracked */ int nLoop; /* Number of terms in the join */ Parse *pParse; /* Parsing context */ - sqlite3 *db; /* The database connection */ int iLoop; /* Loop counter over the terms of the join */ int ii, jj; /* Loop counters */ int mxI = 0; /* Index of next entry to replace */ @@ -129574,14 +168709,26 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ int nSpace; /* Bytes of space allocated at pSpace */ pParse = pWInfo->pParse; - db = pParse->db; nLoop = pWInfo->nLevel; - /* TUNING: For simple queries, only the best path is tracked. - ** For 2-way joins, the 5 best paths are followed. - ** For joins of 3 or more tables, track the 10 best paths */ - mxChoice = (nLoop<=1) ? 1 : (nLoop==2 ? 5 : 10); + WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d, nQueryLoop=%d)\n", + nRowEst, pParse->nQueryLoop)); + /* TUNING: mxChoice is the maximum number of possible paths to preserve + ** at each step. Based on the number of loops in the FROM clause: + ** + ** nLoop mxChoice + ** ----- -------- + ** 1 1 // the most common case + ** 2 5 + ** 3+ 12 or 18 // see computeMxChoice() + */ + if( nLoop<=1 ){ + mxChoice = 1; + }else if( nLoop==2 ){ + mxChoice = 5; + }else{ + mxChoice = computeMxChoice(pWInfo, nRowEst); + } assert( nLoop<=pWInfo->pTabList->nSrc ); - WHERETRACE(0x002, ("---- begin solver. (nRowEst=%d)\n", nRowEst)); /* If nRowEst is zero and there is an ORDER BY clause, ignore it. In this ** case the purpose of this call is to estimate the number of rows returned @@ -129597,7 +168744,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ /* Allocate and initialize space for aTo, aFrom and aSortCost[] */ nSpace = (sizeof(WherePath)+sizeof(WhereLoop*)*nLoop)*mxChoice*2; nSpace += sizeof(LogEst) * nOrderBy; - pSpace = sqlite3DbMallocRawNN(db, nSpace); + pSpace = sqlite3StackAllocRawNN(pParse->db, nSpace); if( pSpace==0 ) return SQLITE_NOMEM_BKPT; aTo = (WherePath*)pSpace; aFrom = aTo+mxChoice; @@ -129611,7 +168758,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** space for the aSortCost[] array. Each element of the aSortCost array ** is either zero - meaning it has not yet been initialized - or the ** cost of sorting nRowEst rows of data where the first X terms of - ** the ORDER BY clause are already in order, where X is the array + ** the ORDER BY clause are already in order, where X is the array ** index. */ aSortCost = (LogEst*)pX; memset(aSortCost, 0, sizeof(LogEst) * nOrderBy); @@ -129632,7 +168779,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** in this case the query may return a maximum of one row, the results ** are already in the requested order. Set isOrdered to nOrderBy to ** indicate this. Or, if nLoop is greater than zero, set isOrdered to - ** -1, indicating that the result set may or may not be ordered, + ** -1, indicating that the result set may or may not be ordered, ** depending on the loops added to the current plan. */ aFrom[0].isOrdered = nLoop>0 ? -1 : nOrderBy; } @@ -129647,25 +168794,33 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ LogEst nOut; /* Rows visited by (pFrom+pWLoop) */ LogEst rCost; /* Cost of path (pFrom+pWLoop) */ LogEst rUnsorted; /* Unsorted cost of (pFrom+pWLoop) */ - i8 isOrdered = pFrom->isOrdered; /* isOrdered for (pFrom+pWLoop) */ + i8 isOrdered; /* isOrdered for (pFrom+pWLoop) */ Bitmask maskNew; /* Mask of src visited by (..) */ - Bitmask revMask = 0; /* Mask of rev-order loops for (..) */ + Bitmask revMask; /* Mask of rev-order loops for (..) */ if( (pWLoop->prereq & ~pFrom->maskLoop)!=0 ) continue; if( (pWLoop->maskSelf & pFrom->maskLoop)!=0 ) continue; - if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<10 ){ + if( (pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 && pFrom->nRow<3 ){ /* Do not use an automatic index if the this loop is expected - ** to run less than 2 times. */ + ** to run less than 1.25 times. It is tempting to also exclude + ** automatic index usage on an outer loop, but sometimes an automatic + ** index is useful in the outer loop of a correlated subquery. */ assert( 10==sqlite3LogEst(2) ); continue; } - /* At this point, pWLoop is a candidate to be the next loop. + + /* At this point, pWLoop is a candidate to be the next loop. ** Compute its cost */ - rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup,pWLoop->rRun + pFrom->nRow); + rUnsorted = pWLoop->rRun + pFrom->nRow; + if( pWLoop->rSetup ){ + rUnsorted = sqlite3LogEstAdd(pWLoop->rSetup, rUnsorted); + } rUnsorted = sqlite3LogEstAdd(rUnsorted, pFrom->rUnsorted); nOut = pFrom->nRow + pWLoop->nOut; maskNew = pFrom->maskLoop | pWLoop->maskSelf; + isOrdered = pFrom->isOrdered; if( isOrdered<0 ){ + revMask = 0; isOrdered = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, pWInfo->wctrlFlags, iLoop, pWLoop, &revMask); @@ -129678,14 +168833,19 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pWInfo, nRowEst, nOrderBy, isOrdered ); } - rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]); + /* TUNING: Add a small extra penalty (3) to sorting as an + ** extra encouragement to the query planner to select a plan + ** where the rows emerge in the correct order without any sorting + ** required. */ + rCost = sqlite3LogEstAdd(rUnsorted, aSortCost[isOrdered]) + 3; WHERETRACE(0x002, ("---- sort cost=%-3d (%d/%d) increases cost %3d to %-3d\n", - aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, + aSortCost[isOrdered], (nOrderBy-isOrdered), nOrderBy, rUnsorted, rCost)); }else{ rCost = rUnsorted; + rUnsorted -= 2; /* TUNING: Slight bias in favor of no-sort plans */ } /* Check to see if pWLoop should be added to the set of @@ -129699,6 +168859,7 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** to (pTo->isOrdered==(-1))==(isOrdered==(-1))" for the range ** of legal values for isOrdered, -1..64. */ + testcase( nTo==0 ); for(jj=0, pTo=aTo; jjmaskLoop==maskNew && ((pTo->isOrdered^isOrdered)&0x80)==0 @@ -129717,8 +168878,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** this candidate as not viable. */ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf("Skip %s cost=%-3d,%3d order=%c\n", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + sqlite3DebugPrintf("Skip %s cost=%-3d,%3d,%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, isOrdered>=0 ? isOrdered+'0' : '?'); } #endif @@ -129736,26 +168897,36 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ pTo = &aTo[jj]; #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ - sqlite3DebugPrintf("New %s cost=%-3d,%3d order=%c\n", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + sqlite3DebugPrintf("New %s cost=%-3d,%3d,%3d order=%c\n", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, isOrdered>=0 ? isOrdered+'0' : '?'); } #endif }else{ /* Control reaches here if best-so-far path pTo=aTo[jj] covers the - ** same set of loops and has the sam isOrdered setting as the + ** same set of loops and has the same isOrdered setting as the ** candidate path. Check to see if the candidate should replace - ** pTo or if the candidate should be skipped */ - if( pTo->rCostrCost==rCost && pTo->nRow<=nOut) ){ + ** pTo or if the candidate should be skipped. + ** + ** The conditional is an expanded vector comparison equivalent to: + ** (pTo->rCost,pTo->nRow,pTo->rUnsorted) <= (rCost,nOut,rUnsorted) + */ + if( pTo->rCostrCost==rCost + && (pTo->nRownRow==nOut && pTo->rUnsorted<=rUnsorted) + ) + ) + ){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( - "Skip %s cost=%-3d,%3d order=%c", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + "Skip %s cost=%-3d,%3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, isOrdered>=0 ? isOrdered+'0' : '?'); - sqlite3DebugPrintf(" vs %s cost=%-3d,%d order=%c\n", + sqlite3DebugPrintf(" vs %s cost=%-3d,%3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif /* Discard the candidate path from further consideration */ @@ -129768,12 +168939,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #ifdef WHERETRACE_ENABLED /* 0x4 */ if( sqlite3WhereTrace&0x4 ){ sqlite3DebugPrintf( - "Update %s cost=%-3d,%3d order=%c", - wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, + "Update %s cost=%-3d,%3d,%3d order=%c", + wherePathName(pFrom, iLoop, pWLoop), rCost, nOut, rUnsorted, isOrdered>=0 ? isOrdered+'0' : '?'); - sqlite3DebugPrintf(" was %s cost=%-3d,%3d order=%c\n", + sqlite3DebugPrintf(" was %s cost=%-3d,%3d,%3d order=%c\n", wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, - pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); + pTo->rUnsorted, pTo->isOrdered>=0 ? pTo->isOrdered+'0' : '?'); } #endif } @@ -129791,8 +168962,8 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ mxCost = aTo[0].rCost; mxUnsorted = aTo[0].nRow; for(jj=1, pTo=&aTo[1]; jjrCost>mxCost - || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) + if( pTo->rCost>mxCost + || (pTo->rCost==mxCost && pTo->rUnsorted>mxUnsorted) ){ mxCost = pTo->rCost; mxUnsorted = pTo->rUnsorted; @@ -129805,16 +168976,28 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ #ifdef WHERETRACE_ENABLED /* >=2 */ if( sqlite3WhereTrace & 0x02 ){ + LogEst rMin, rFloor = 0; + int nDone = 0; sqlite3DebugPrintf("---- after round %d ----\n", iLoop); - for(ii=0, pTo=aTo; iirCost, pTo->nRow, - pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); - if( pTo->isOrdered>0 ){ - sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); - }else{ - sqlite3DebugPrintf("\n"); + while( nDonerCost>rFloor && pTo->rCostrCost; } + for(ii=0, pTo=aTo; iirCost==rMin ){ + sqlite3DebugPrintf(" %s cost=%-3d nrow=%-3d order=%c", + wherePathName(pTo, iLoop+1, 0), pTo->rCost, pTo->nRow, + pTo->isOrdered>=0 ? (pTo->isOrdered+'0') : '?'); + if( pTo->isOrdered>0 ){ + sqlite3DebugPrintf(" rev=0x%llx\n", pTo->revLoop); + }else{ + sqlite3DebugPrintf("\n"); + } + nDone++; + } + } + rFloor = rMin; } } #endif @@ -129828,10 +169011,10 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ if( nFrom==0 ){ sqlite3ErrorMsg(pParse, "no query solution"); - sqlite3DbFree(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_ERROR; } - + /* Find the lowest cost path. pFrom will be left pointing to that path */ pFrom = aFrom; for(ii=1; iipDistinctSet, pFrom, + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pResultSet, pFrom, WHERE_DISTINCTBY, nLoop-1, pFrom->aLoop[nLoop-1], ¬Used); - if( rc==pWInfo->pDistinctSet->nExpr ){ + if( rc==pWInfo->pResultSet->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } } + pWInfo->bOrderedInnerLoop = 0; if( pWInfo->pOrderBy ){ + pWInfo->nOBSat = pFrom->isOrdered; if( pWInfo->wctrlFlags & WHERE_DISTINCTBY ){ if( pFrom->isOrdered==pWInfo->pOrderBy->nExpr ){ pWInfo->eDistinct = WHERE_DISTINCT_ORDERED; } + /* vvv--- See check-in [12ad822d9b827777] on 2023-03-16 ---vvv */ + assert( pWInfo->pSelect->pOrderBy==0 + || pWInfo->nOBSat <= pWInfo->pSelect->pOrderBy->nExpr ); }else{ - pWInfo->nOBSat = pFrom->isOrdered; - if( pWInfo->nOBSat<0 ) pWInfo->nOBSat = 0; pWInfo->revMask = pFrom->revLoop; + if( pWInfo->nOBSat<=0 ){ + pWInfo->nOBSat = 0; + if( nLoop>0 ){ + u32 wsFlags = pFrom->aLoop[nLoop-1]->wsFlags; + if( (wsFlags & WHERE_ONEROW)==0 + && (wsFlags&(WHERE_IPK|WHERE_COLUMN_IN))!=(WHERE_IPK|WHERE_COLUMN_IN) + ){ + Bitmask m = 0; + int rc = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, + WHERE_ORDERBY_LIMIT, nLoop-1, pFrom->aLoop[nLoop-1], &m); + testcase( wsFlags & WHERE_IPK ); + testcase( wsFlags & WHERE_COLUMN_IN ); + if( rc==pWInfo->pOrderBy->nExpr ){ + pWInfo->bOrderedInnerLoop = 1; + pWInfo->revMask = m; + } + } + } + }else if( nLoop + && pWInfo->nOBSat==1 + && (pWInfo->wctrlFlags & (WHERE_ORDERBY_MIN|WHERE_ORDERBY_MAX))!=0 + ){ + pWInfo->bOrderedInnerLoop = 1; + } } if( (pWInfo->wctrlFlags & WHERE_SORTBYGROUP) && pWInfo->nOBSat==pWInfo->pOrderBy->nExpr && nLoop>0 ){ Bitmask revMask = 0; - int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, + int nOrder = wherePathSatisfiesOrderBy(pWInfo, pWInfo->pOrderBy, pFrom, 0, nLoop-1, pFrom->aLoop[nLoop-1], &revMask ); assert( pWInfo->sorted==0 ); @@ -129882,14 +169092,90 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ } } - - pWInfo->nRowOut = pFrom->nRow; + pWInfo->nRowOut = pFrom->nRow + pWInfo->nOutStarDelta; /* Free temporary memory and return success */ - sqlite3DbFree(db, pSpace); + sqlite3StackFreeNN(pParse->db, pSpace); return SQLITE_OK; } +/* +** This routine implements a heuristic designed to improve query planning. +** This routine is called in between the first and second call to +** wherePathSolver(). Hence the name "Interstage" "Heuristic". +** +** The first call to wherePathSolver() (hereafter just "solver()") computes +** the best path without regard to the order of the outputs. The second call +** to the solver() builds upon the first call to try to find an alternative +** path that satisfies the ORDER BY clause. +** +** This routine looks at the results of the first solver() run, and for +** every FROM clause term in the resulting query plan that uses an equality +** constraint against an index, disable other WhereLoops for that same +** FROM clause term that would try to do a full-table scan. This prevents +** an index search from being converted into a full-table scan in order to +** satisfy an ORDER BY clause, since even though we might get slightly better +** performance using the full-scan without sorting if the output size +** estimates are very precise, we might also get severe performance +** degradation using the full-scan if the output size estimate is too large. +** It is better to err on the side of caution. +** +** Except, if the first solver() call generated a full-table scan in an outer +** loop then stop this analysis at the first full-scan, since the second +** solver() run might try to swap that full-scan for another in order to +** get the output into the correct order. In other words, we allow a +** rewrite like this: +** +** First Solver() Second Solver() +** |-- SCAN t1 |-- SCAN t2 +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** The purpose of this routine is to disallow rewrites such as: +** +** First Solver() Second Solver() +** |-- SEARCH t1 |-- SCAN t2 <--- bad! +** |-- SEARCH t2 `-- SEARCH t1 +** `-- SORT USING B-TREE +** +** See test cases in test/whereN.test for the real-world query that +** originally provoked this heuristic. +*/ +static SQLITE_NOINLINE void whereInterstageHeuristic(WhereInfo *pWInfo){ + int i; +#ifdef WHERETRACE_ENABLED + int once = 0; +#endif + for(i=0; inLevel; i++){ + WhereLoop *p = pWInfo->a[i].pWLoop; + if( p==0 ) break; + if( (p->wsFlags & WHERE_VIRTUALTABLE)!=0 ) continue; + if( (p->wsFlags & (WHERE_COLUMN_EQ|WHERE_COLUMN_NULL|WHERE_COLUMN_IN))!=0 ){ + u8 iTab = p->iTab; + WhereLoop *pLoop; + for(pLoop=pWInfo->pLoops; pLoop; pLoop=pLoop->pNextLoop){ + if( pLoop->iTab!=iTab ) continue; + if( (pLoop->wsFlags & (WHERE_CONSTRAINT|WHERE_AUTO_INDEX))!=0 ){ + /* Auto-index and index-constrained loops allowed to remain */ + continue; + } +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x80 ){ + if( once==0 ){ + sqlite3DebugPrintf("Loops disabled by interstage heuristic:\n"); + once = 1; + } + sqlite3WhereLoopPrint(pLoop, &pWInfo->sWC); + } +#endif /* WHERETRACE_ENABLED */ + pLoop->prereq = ALLBITS; /* Prevent 2nd solver() from using this one */ + } + }else{ + break; + } + } +} + /* ** Most queries use only a single table (they are not joins) and have ** simple == constraints against indexed fields. This routine attempts @@ -129898,12 +169184,12 @@ static int wherePathSolver(WhereInfo *pWInfo, LogEst nRowEst){ ** times for the common case. ** ** Return non-zero on success, if this query can be handled by this -** no-frills query planner. Return zero if this query needs the +** no-frills query planner. Return zero if this query needs the ** general-purpose query planner. */ static int whereShortCut(WhereLoopBuilder *pBuilder){ WhereInfo *pWInfo; - struct SrcList_item *pItem; + SrcItem *pItem; WhereClause *pWC; WhereTerm *pTerm; WhereLoop *pLoop; @@ -129911,20 +169197,26 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ int j; Table *pTab; Index *pIdx; + WhereScan scan; pWInfo = pBuilder->pWInfo; - if( pWInfo->wctrlFlags & WHERE_FORCE_TABLE ) return 0; + if( pWInfo->wctrlFlags & WHERE_OR_SUBCLAUSE ) return 0; assert( pWInfo->pTabList->nSrc>=1 ); pItem = pWInfo->pTabList->a; - pTab = pItem->pTab; + pTab = pItem->pSTab; if( IsVirtual(pTab) ) return 0; - if( pItem->fg.isIndexedBy ) return 0; + if( pItem->fg.isIndexedBy || pItem->fg.notIndexed ){ + testcase( pItem->fg.isIndexedBy ); + testcase( pItem->fg.notIndexed ); + return 0; + } iCur = pItem->iCursor; pWC = &pWInfo->sWC; pLoop = pBuilder->pNew; pLoop->wsFlags = 0; pLoop->nSkip = 0; - pTerm = sqlite3WhereFindTerm(pWC, iCur, -1, 0, WO_EQ|WO_IS, 0); + pTerm = whereScanInit(&scan, pWC, iCur, -1, WO_EQ|WO_IS, 0); + while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); if( pTerm ){ testcase( pTerm->eOperator & WO_IS ); pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_IPK|WHERE_ONEROW; @@ -129938,19 +169230,20 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ int opMask; assert( pLoop->aLTermSpace==pLoop->aLTerm ); if( !IsUniqueIndex(pIdx) - || pIdx->pPartIdxWhere!=0 - || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) + || pIdx->pPartIdxWhere!=0 + || pIdx->nKeyCol>ArraySize(pLoop->aLTermSpace) ) continue; opMask = pIdx->uniqNotNull ? (WO_EQ|WO_IS) : WO_EQ; for(j=0; jnKeyCol; j++){ - pTerm = sqlite3WhereFindTerm(pWC, iCur, j, 0, opMask, pIdx); + pTerm = whereScanInit(&scan, pWC, iCur, j, opMask, pIdx); + while( pTerm && pTerm->prereqRight ) pTerm = whereScanNext(&scan); if( pTerm==0 ) break; testcase( pTerm->eOperator & WO_IS ); pLoop->aLTerm[j] = pTerm; } if( j!=pIdx->nKeyCol ) continue; pLoop->wsFlags = WHERE_COLUMN_EQ|WHERE_ONEROW|WHERE_INDEXED; - if( pIdx->isCovering || (pItem->colUsed & ~columnsInIndex(pIdx))==0 ){ + if( pIdx->isCovering || (pItem->colUsed & pIdx->colNotIdxed)==0 ){ pLoop->wsFlags |= WHERE_IDX_ONLY; } pLoop->nLTerm = j; @@ -129964,21 +169257,326 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ if( pLoop->wsFlags ){ pLoop->nOut = (LogEst)1; pWInfo->a[0].pWLoop = pLoop; - pLoop->maskSelf = sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); + assert( pWInfo->sMaskSet.n==1 && iCur==pWInfo->sMaskSet.ix[0] ); + pLoop->maskSelf = 1; /* sqlite3WhereGetMask(&pWInfo->sMaskSet, iCur); */ pWInfo->a[0].iTabCur = iCur; pWInfo->nRowOut = 1; if( pWInfo->pOrderBy ) pWInfo->nOBSat = pWInfo->pOrderBy->nExpr; if( pWInfo->wctrlFlags & WHERE_WANT_DISTINCT ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } + if( scan.iEquiv>1 ) pLoop->wsFlags |= WHERE_TRANSCONS; #ifdef SQLITE_DEBUG pLoop->cId = '0'; +#endif +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x02 ){ + sqlite3DebugPrintf("whereShortCut() used to compute solution\n"); + } #endif return 1; } return 0; } +/* +** Helper function for exprIsDeterministic(). +*/ +static int exprNodeIsDeterministic(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_FUNCTION && ExprHasProperty(pExpr, EP_ConstFunc)==0 ){ + pWalker->eCode = 0; + return WRC_Abort; + } + return WRC_Continue; +} + +/* +** Return true if the expression contains no non-deterministic SQL +** functions. Do not consider non-deterministic SQL functions that are +** part of sub-select statements. +*/ +static int exprIsDeterministic(Expr *p){ + Walker w; + memset(&w, 0, sizeof(w)); + w.eCode = 1; + w.xExprCallback = exprNodeIsDeterministic; + w.xSelectCallback = sqlite3SelectWalkFail; + sqlite3WalkExpr(&w, p); + return w.eCode; +} + + +#ifdef WHERETRACE_ENABLED +/* +** Display all WhereLoops in pWInfo +*/ +static void showAllWhereLoops(WhereInfo *pWInfo, WhereClause *pWC){ + if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ + WhereLoop *p; + int i; + static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" + "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; + for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ + p->cId = zLabel[i%(sizeof(zLabel)-1)]; + sqlite3WhereLoopPrint(p, pWC); + } + } +} +# define WHERETRACE_ALL_LOOPS(W,C) showAllWhereLoops(W,C) +#else +# define WHERETRACE_ALL_LOOPS(W,C) +#endif + +/* Attempt to omit tables from a join that do not affect the result. +** For a table to not affect the result, the following must be true: +** +** 1) The query must not be an aggregate. +** 2) The table must be the RHS of a LEFT JOIN. +** 3) Either the query must be DISTINCT, or else the ON or USING clause +** must contain a constraint that limits the scan of the table to +** at most a single row. +** 4) The table must not be referenced by any part of the query apart +** from its own USING or ON clause. +** 5) The table must not have an inner-join ON or USING clause if there is +** a RIGHT JOIN anywhere in the query. Otherwise the ON/USING clause +** might move from the right side to the left side of the RIGHT JOIN. +** Note: Due to (2), this condition can only arise if the table is +** the right-most table of a subquery that was flattened into the +** main query and that subquery was the right-hand operand of an +** inner join that held an ON or USING clause. +** 6) The ORDER BY clause has 63 or fewer terms +** 7) The omit-noop-join optimization is enabled. +** +** Items (1), (6), and (7) are checked by the caller. +** +** For example, given: +** +** CREATE TABLE t1(ipk INTEGER PRIMARY KEY, v1); +** CREATE TABLE t2(ipk INTEGER PRIMARY KEY, v2); +** CREATE TABLE t3(ipk INTEGER PRIMARY KEY, v3); +** +** then table t2 can be omitted from the following: +** +** SELECT v1, v3 FROM t1 +** LEFT JOIN t2 ON (t1.ipk=t2.ipk) +** LEFT JOIN t3 ON (t1.ipk=t3.ipk) +** +** or from: +** +** SELECT DISTINCT v1, v3 FROM t1 +** LEFT JOIN t2 +** LEFT JOIN t3 ON (t1.ipk=t3.ipk) +*/ +static SQLITE_NOINLINE Bitmask whereOmitNoopJoin( + WhereInfo *pWInfo, + Bitmask notReady +){ + int i; + Bitmask tabUsed; + int hasRightJoin; + + /* Preconditions checked by the caller */ + assert( pWInfo->nLevel>=2 ); + assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_OmitNoopJoin) ); + + /* These two preconditions checked by the caller combine to guarantee + ** condition (1) of the header comment */ + assert( pWInfo->pResultSet!=0 ); + assert( 0==(pWInfo->wctrlFlags & WHERE_AGG_DISTINCT) ); + + tabUsed = sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pResultSet); + if( pWInfo->pOrderBy ){ + tabUsed |= sqlite3WhereExprListUsage(&pWInfo->sMaskSet, pWInfo->pOrderBy); + } + hasRightJoin = (pWInfo->pTabList->a[0].fg.jointype & JT_LTORJ)!=0; + for(i=pWInfo->nLevel-1; i>=1; i--){ + WhereTerm *pTerm, *pEnd; + SrcItem *pItem; + WhereLoop *pLoop; + Bitmask m1; + pLoop = pWInfo->a[i].pWLoop; + pItem = &pWInfo->pTabList->a[pLoop->iTab]; + if( (pItem->fg.jointype & (JT_LEFT|JT_RIGHT))!=JT_LEFT ) continue; + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)==0 + && (pLoop->wsFlags & WHERE_ONEROW)==0 + ){ + continue; + } + if( (tabUsed & pLoop->maskSelf)!=0 ) continue; + pEnd = pWInfo->sWC.a + pWInfo->sWC.nTerm; + for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + if( !ExprHasProperty(pTerm->pExpr, EP_OuterON) + || pTerm->pExpr->w.iJoin!=pItem->iCursor + ){ + break; + } + } + if( hasRightJoin + && ExprHasProperty(pTerm->pExpr, EP_InnerON) + && pTerm->pExpr->w.iJoin==pItem->iCursor + ){ + break; /* restriction (5) */ + } + } + if( pTerm omit unused FROM-clause term %c\n",pLoop->cId)); + m1 = MASKBIT(i)-1; + testcase( ((pWInfo->revMask>>1) & ~m1)!=0 ); + pWInfo->revMask = (m1 & pWInfo->revMask) | ((pWInfo->revMask>>1) & ~m1); + notReady &= ~pLoop->maskSelf; + for(pTerm=pWInfo->sWC.a; pTermprereqAll & pLoop->maskSelf)!=0 ){ + pTerm->wtFlags |= TERM_CODED; + } + } + if( i!=pWInfo->nLevel-1 ){ + int nByte = (pWInfo->nLevel-1-i) * sizeof(WhereLevel); + memmove(&pWInfo->a[i], &pWInfo->a[i+1], nByte); + } + pWInfo->nLevel--; + assert( pWInfo->nLevel>0 ); + } + return notReady; +} + +/* +** Check to see if there are any SEARCH loops that might benefit from +** using a Bloom filter. Consider a Bloom filter if: +** +** (1) The SEARCH happens more than N times where N is the number +** of rows in the table that is being considered for the Bloom +** filter. +** (2) Some searches are expected to find zero rows. (This is determined +** by the WHERE_SELFCULL flag on the term.) +** (3) Bloom-filter processing is not disabled. (Checked by the +** caller.) +** (4) The size of the table being searched is known by ANALYZE. +** +** This block of code merely checks to see if a Bloom filter would be +** appropriate, and if so sets the WHERE_BLOOMFILTER flag on the +** WhereLoop. The implementation of the Bloom filter comes further +** down where the code for each WhereLoop is generated. +*/ +static SQLITE_NOINLINE void whereCheckIfBloomFilterIsUseful( + const WhereInfo *pWInfo +){ + int i; + LogEst nSearch = 0; + + assert( pWInfo->nLevel>=2 ); + assert( OptimizationEnabled(pWInfo->pParse->db, SQLITE_BloomFilter) ); + for(i=0; inLevel; i++){ + WhereLoop *pLoop = pWInfo->a[i].pWLoop; + const unsigned int reqFlags = (WHERE_SELFCULL|WHERE_COLUMN_EQ); + SrcItem *pItem = &pWInfo->pTabList->a[pLoop->iTab]; + Table *pTab = pItem->pSTab; + if( (pTab->tabFlags & TF_HasStat1)==0 ) break; + pTab->tabFlags |= TF_MaybeReanalyze; + if( i>=1 + && (pLoop->wsFlags & reqFlags)==reqFlags + /* vvvvvv--- Always the case if WHERE_COLUMN_EQ is defined */ + && ALWAYS((pLoop->wsFlags & (WHERE_IPK|WHERE_INDEXED))!=0) + ){ + if( nSearch > pTab->nRowLogEst ){ + testcase( pItem->fg.jointype & JT_LEFT ); + pLoop->wsFlags |= WHERE_BLOOMFILTER; + pLoop->wsFlags &= ~WHERE_IDX_ONLY; + WHERETRACE(0xffffffff, ( + "-> use Bloom-filter on loop %c because there are ~%.1e " + "lookups into %s which has only ~%.1e rows\n", + pLoop->cId, (double)sqlite3LogEstToInt(nSearch), pTab->zName, + (double)sqlite3LogEstToInt(pTab->nRowLogEst))); + } + } + nSearch += pLoop->nOut; + if( pWInfo->nOutStarDelta ) nSearch += pLoop->rStarDelta; + } +} + +/* +** The index pIdx is used by a query and contains one or more expressions. +** In other words pIdx is an index on an expression. iIdxCur is the cursor +** number for the index and iDataCur is the cursor number for the corresponding +** table. +** +** This routine adds IndexedExpr entries to the Parse->pIdxEpr field for +** each of the expressions in the index so that the expression code generator +** will know to replace occurrences of the indexed expression with +** references to the corresponding column of the index. +*/ +static SQLITE_NOINLINE void whereAddIndexedExpr( + Parse *pParse, /* Add IndexedExpr entries to pParse->pIdxEpr */ + Index *pIdx, /* The index-on-expression that contains the expressions */ + int iIdxCur, /* Cursor number for pIdx */ + SrcItem *pTabItem /* The FROM clause entry for the table */ +){ + int i; + IndexedExpr *p; + Table *pTab; + assert( pIdx->bHasExpr ); + pTab = pIdx->pTable; + for(i=0; inColumn; i++){ + Expr *pExpr; + int j = pIdx->aiColumn[i]; + if( j==XN_EXPR ){ + pExpr = pIdx->aColExpr->a[i].pExpr; + }else if( j>=0 && (pTab->aCol[j].colFlags & COLFLAG_VIRTUAL)!=0 ){ + pExpr = sqlite3ColumnExpr(pTab, &pTab->aCol[j]); + }else{ + continue; + } + if( sqlite3ExprIsConstant(0,pExpr) ) continue; + p = sqlite3DbMallocRaw(pParse->db, sizeof(IndexedExpr)); + if( p==0 ) break; + p->pIENext = pParse->pIdxEpr; +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("New pParse->pIdxEpr term {%d,%d}\n", iIdxCur, i); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(pExpr); + } +#endif + p->pExpr = sqlite3ExprDup(pParse->db, pExpr, 0); + p->iDataCur = pTabItem->iCursor; + p->iIdxCur = iIdxCur; + p->iIdxCol = i; + p->bMaybeNullRow = (pTabItem->fg.jointype & (JT_LEFT|JT_LTORJ|JT_RIGHT))!=0; + if( sqlite3IndexAffinityStr(pParse->db, pIdx) ){ + p->aff = pIdx->zColAff[i]; + } +#ifdef SQLITE_ENABLE_EXPLAIN_COMMENTS + p->zIdxName = pIdx->zName; +#endif + pParse->pIdxEpr = p; + if( p->pIENext==0 ){ + void *pArg = (void*)&pParse->pIdxEpr; + sqlite3ParserAddCleanup(pParse, whereIndexedExprCleanup, pArg); + } + } +} + +/* +** Set the reverse-scan order mask to one for all tables in the query +** with the exception of MATERIALIZED common table expressions that have +** their own internal ORDER BY clauses. +** +** This implements the PRAGMA reverse_unordered_selects=ON setting. +** (Also SQLITE_DBCONFIG_REVERSE_SCANORDER). +*/ +static SQLITE_NOINLINE void whereReverseScanOrder(WhereInfo *pWInfo){ + int ii; + for(ii=0; iipTabList->nSrc; ii++){ + SrcItem *pItem = &pWInfo->pTabList->a[ii]; + if( !pItem->fg.isCte + || pItem->u2.pCteUse->eM10d!=M10d_Yes + || NEVER(pItem->fg.isSubquery==0) + || pItem->u4.pSubq->pSelect->pOrderBy==0 + ){ + pWInfo->revMask |= MASKBIT(ii); + } + } +} + /* ** Generate the beginning of the loop used for WHERE clause processing. ** The return value is a pointer to an opaque structure that contains @@ -130037,7 +169635,7 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ** ** OUTER JOINS ** -** An outer join of tables t1 and t2 is conceptally coded as follows: +** An outer join of tables t1 and t2 is conceptually coded as follows: ** ** foreach row1 in t1 do ** flag = 0 @@ -130059,8 +169657,8 @@ static int whereShortCut(WhereLoopBuilder *pBuilder){ ** if there is one. If there is no ORDER BY clause or if this routine ** is called from an UPDATE or DELETE statement, then pOrderBy is NULL. ** -** The iIdxCur parameter is the cursor number of an index. If -** WHERE_ONETABLE_ONLY is set, iIdxCur is the cursor number of an index +** The iIdxCur parameter is the cursor number of an index. If +** WHERE_OR_SUBCLAUSE is set, iIdxCur is the cursor number of an index ** to use for OR clause processing. The WHERE clause should use this ** specific cursor. If WHERE_ONEPASS_DESIRED is set, then iIdxCur is ** the first cursor in an array of cursors for all indices. iIdxCur should @@ -130072,9 +169670,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( SrcList *pTabList, /* FROM clause: A list of all tables to be scanned */ Expr *pWhere, /* The WHERE clause */ ExprList *pOrderBy, /* An ORDER BY (or GROUP BY) clause, or NULL */ - ExprList *pDistinctSet, /* Try not to output two rows that duplicate these */ + ExprList *pResultSet, /* Query result set. Req'd for DISTINCT */ + Select *pSelect, /* The entire SELECT statement */ u16 wctrlFlags, /* The WHERE_* flags defined in sqliteInt.h */ - int iAuxArg /* If WHERE_ONETABLE_ONLY is set, index cursor number + int iAuxArg /* If WHERE_OR_SUBCLAUSE is set, index cursor number ** If WHERE_USE_LIMIT, then the limit amount */ ){ int nByteWInfo; /* Num. bytes allocated for WhereInfo struct */ @@ -130092,12 +169691,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( u8 bFordelete = 0; /* OPFLAG_FORDELETE or zero, as appropriate */ assert( (wctrlFlags & WHERE_ONEPASS_MULTIROW)==0 || ( - (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 - && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 + (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 )); - /* Only one of WHERE_ONETABLE_ONLY or WHERE_USE_LIMIT */ - assert( (wctrlFlags & WHERE_ONETABLE_ONLY)==0 + /* Only one of WHERE_OR_SUBCLAUSE or WHERE_USE_LIMIT */ + assert( (wctrlFlags & WHERE_OR_SUBCLAUSE)==0 || (wctrlFlags & WHERE_USE_LIMIT)==0 ); /* Variable initialization */ @@ -130106,17 +169705,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* An ORDER/GROUP BY clause of more than 63 terms cannot be optimized */ testcase( pOrderBy && pOrderBy->nExpr==BMS-1 ); - if( pOrderBy && pOrderBy->nExpr>=BMS ) pOrderBy = 0; - sWLB.pOrderBy = pOrderBy; - - /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via - ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ - if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + if( pOrderBy && pOrderBy->nExpr>=BMS ){ + pOrderBy = 0; wctrlFlags &= ~WHERE_WANT_DISTINCT; + wctrlFlags |= WHERE_KEEP_ALL_JOINS; /* Disable omit-noop-join opt */ } /* The number of tables in the FROM clause is limited by the number of - ** bits in a Bitmask + ** bits in a Bitmask */ testcase( pTabList->nSrc==BMS ); if( pTabList->nSrc>BMS ){ @@ -130124,12 +169720,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( return 0; } - /* This function normally generates a nested loop for all tables in - ** pTabList. But if the WHERE_ONETABLE_ONLY flag is set, then we should + /* This function normally generates a nested loop for all tables in + ** pTabList. But if the WHERE_OR_SUBCLAUSE flag is set, then we should ** only generate code for the first table in pTabList and assume that ** any cursors associated with subsequent tables are uninitialized. */ - nTabList = (wctrlFlags & WHERE_ONETABLE_ONLY) ? 1 : pTabList->nSrc; + nTabList = (wctrlFlags & WHERE_OR_SUBCLAUSE) ? 1 : pTabList->nSrc; /* Allocate and initialize the WhereInfo structure that will become the ** return value. A single allocation is used to store the WhereInfo @@ -130138,25 +169734,39 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** field (type Bitmask) it must be aligned on an 8-byte boundary on ** some architectures. Hence the ROUND8() below. */ - nByteWInfo = ROUND8(sizeof(WhereInfo)+(nTabList-1)*sizeof(WhereLevel)); - pWInfo = sqlite3DbMallocZero(db, nByteWInfo + sizeof(WhereLoop)); + nByteWInfo = ROUND8P(sizeof(WhereInfo)); + if( nTabList>1 ){ + nByteWInfo = ROUND8P(nByteWInfo + (nTabList-1)*sizeof(WhereLevel)); + } + pWInfo = sqlite3DbMallocRawNN(db, nByteWInfo + sizeof(WhereLoop)); if( db->mallocFailed ){ sqlite3DbFree(db, pWInfo); pWInfo = 0; goto whereBeginError; } - pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; - pWInfo->nLevel = nTabList; pWInfo->pParse = pParse; pWInfo->pTabList = pTabList; pWInfo->pOrderBy = pOrderBy; - pWInfo->pDistinctSet = pDistinctSet; - pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(v); +#if WHERETRACE_ENABLED + pWInfo->pWhere = pWhere; +#endif + pWInfo->pResultSet = pResultSet; + pWInfo->aiCurOnePass[0] = pWInfo->aiCurOnePass[1] = -1; + pWInfo->nLevel = nTabList; + pWInfo->iBreak = pWInfo->iContinue = sqlite3VdbeMakeLabel(pParse); pWInfo->wctrlFlags = wctrlFlags; pWInfo->iLimit = iAuxArg; pWInfo->savedNQueryLoop = pParse->nQueryLoop; + pWInfo->pSelect = pSelect; + memset(&pWInfo->nOBSat, 0, + offsetof(WhereInfo,sWC) - offsetof(WhereInfo,nOBSat)); + memset(&pWInfo->a[0], 0, sizeof(WhereLoop)+nTabList*sizeof(WhereLevel)); assert( pWInfo->eOnePass==ONEPASS_OFF ); /* ONEPASS defaults to OFF */ pMaskSet = &pWInfo->sMaskSet; + pMaskSet->n = 0; + pMaskSet->ix[0] = -99; /* Initialize ix[0] to a value that can never be + ** a valid cursor number, to avoid an initial + ** test for pMaskSet->n==0 in sqlite3WhereGetMask() */ sWLB.pWInfo = pWInfo; sWLB.pWC = &pWInfo->sWC; sWLB.pNew = (WhereLoop*)(((char*)pWInfo)+nByteWInfo); @@ -130169,83 +169779,142 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* Split the WHERE clause into separate subexpressions where each ** subexpression is separated by an AND operator. */ - initMaskSet(pMaskSet); sqlite3WhereClauseInit(&pWInfo->sWC, pWInfo); sqlite3WhereSplit(&pWInfo->sWC, pWhere, TK_AND); - - /* Special case: a WHERE clause that is constant. Evaluate the - ** expression and either jump over all of the code or fall thru. - */ - for(ii=0; iinTerm; ii++){ - if( nTabList==0 || sqlite3ExprIsConstantNotJoin(sWLB.pWC->a[ii].pExpr) ){ - sqlite3ExprIfFalse(pParse, sWLB.pWC->a[ii].pExpr, pWInfo->iBreak, - SQLITE_JUMPIFNULL); - sWLB.pWC->a[ii].wtFlags |= TERM_CODED; - } - } /* Special case: No FROM clause */ if( nTabList==0 ){ if( pOrderBy ) pWInfo->nOBSat = pOrderBy->nExpr; - if( wctrlFlags & WHERE_WANT_DISTINCT ){ + if( (wctrlFlags & WHERE_WANT_DISTINCT)!=0 + && OptimizationEnabled(db, SQLITE_DistinctOpt) + ){ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; } + if( ALWAYS(pWInfo->pSelect) + && (pWInfo->pSelect->selFlags & SF_MultiValue)==0 + ){ + ExplainQueryPlan((pParse, 0, "SCAN CONSTANT ROW")); + } + }else{ + /* Assign a bit from the bitmask to every term in the FROM clause. + ** + ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in + ** pTabList, not just the first nTabList tables. nTabList is normally + ** equal to pTabList->nSrc but might be shortened to 1 if the + ** WHERE_OR_SUBCLAUSE flag is set. + */ + ii = 0; + do{ + createMask(pMaskSet, pTabList->a[ii].iCursor); + sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); + }while( (++ii)nSrc ); + #ifdef SQLITE_DEBUG + { + Bitmask mx = 0; + for(ii=0; iinSrc; ii++){ + Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); + assert( m>=mx ); + mx = m; + } + } + #endif } - /* Assign a bit from the bitmask to every term in the FROM clause. - ** - ** The N-th term of the FROM clause is assigned a bitmask of 1<nSrc tables in - ** pTabList, not just the first nTabList tables. nTabList is normally - ** equal to pTabList->nSrc but might be shortened to 1 if the - ** WHERE_ONETABLE_ONLY flag is set. - */ - for(ii=0; iinSrc; ii++){ - createMask(pMaskSet, pTabList->a[ii].iCursor); - sqlite3WhereTabFuncArgs(pParse, &pTabList->a[ii], &pWInfo->sWC); - } -#ifdef SQLITE_DEBUG - for(ii=0; iinSrc; ii++){ - Bitmask m = sqlite3WhereGetMask(pMaskSet, pTabList->a[ii].iCursor); - assert( m==MASKBIT(ii) ); - } -#endif - /* Analyze all of the subexpressions. */ sqlite3WhereExprAnalyze(pTabList, &pWInfo->sWC); - if( db->mallocFailed ) goto whereBeginError; + if( pSelect && pSelect->pLimit ){ + sqlite3WhereAddLimit(&pWInfo->sWC, pSelect); + } + if( pParse->nErr ) goto whereBeginError; + + /* The False-WHERE-Term-Bypass optimization: + ** + ** If there are WHERE terms that are false, then no rows will be output, + ** so skip over all of the code generated here. + ** + ** Conditions: + ** + ** (1) The WHERE term must not refer to any tables in the join. + ** (2) The term must not come from an ON clause on the + ** right-hand side of a LEFT or FULL JOIN. + ** (3) The term must not come from an ON clause, or there must be + ** no RIGHT or FULL OUTER joins in pTabList. + ** (4) If the expression contains non-deterministic functions + ** that are not within a sub-select. This is not required + ** for correctness but rather to preserves SQLite's legacy + ** behaviour in the following two cases: + ** + ** WHERE random()>0; -- eval random() once per row + ** WHERE (SELECT random())>0; -- eval random() just once overall + ** + ** Note that the Where term need not be a constant in order for this + ** optimization to apply, though it does need to be constant relative to + ** the current subquery (condition 1). The term might include variables + ** from outer queries so that the value of the term changes from one + ** invocation of the current subquery to the next. + */ + for(ii=0; iinBase; ii++){ + WhereTerm *pT = &sWLB.pWC->a[ii]; /* A term of the WHERE clause */ + Expr *pX; /* The expression of pT */ + if( pT->wtFlags & TERM_VIRTUAL ) continue; + pX = pT->pExpr; + assert( pX!=0 ); + assert( pT->prereqAll!=0 || !ExprHasProperty(pX, EP_OuterON) ); + if( pT->prereqAll==0 /* Conditions (1) and (2) */ + && (nTabList==0 || exprIsDeterministic(pX)) /* Condition (4) */ + && !(ExprHasProperty(pX, EP_InnerON) /* Condition (3) */ + && (pTabList->a[0].fg.jointype & JT_LTORJ)!=0 ) + ){ + sqlite3ExprIfFalse(pParse, pX, pWInfo->iBreak, SQLITE_JUMPIFNULL); + pT->wtFlags |= TERM_CODED; + } + } if( wctrlFlags & WHERE_WANT_DISTINCT ){ - if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pDistinctSet) ){ + if( OptimizationDisabled(db, SQLITE_DistinctOpt) ){ + /* Disable the DISTINCT optimization if SQLITE_DistinctOpt is set via + ** sqlite3_test_ctrl(SQLITE_TESTCTRL_OPTIMIZATIONS,...) */ + wctrlFlags &= ~WHERE_WANT_DISTINCT; + pWInfo->wctrlFlags &= ~WHERE_WANT_DISTINCT; + }else if( isDistinctRedundant(pParse, pTabList, &pWInfo->sWC, pResultSet) ){ /* The DISTINCT marking is pointless. Ignore it. */ pWInfo->eDistinct = WHERE_DISTINCT_UNIQUE; }else if( pOrderBy==0 ){ /* Try to ORDER BY the result set to make distinct processing easier */ pWInfo->wctrlFlags |= WHERE_DISTINCTBY; - pWInfo->pOrderBy = pDistinctSet; + pWInfo->pOrderBy = pResultSet; } } /* Construct the WhereLoop objects */ #if defined(WHERETRACE_ENABLED) - if( sqlite3WhereTrace & 0xffff ){ + if( sqlite3WhereTrace & 0xffffffff ){ sqlite3DebugPrintf("*** Optimizer Start *** (wctrlFlags: 0x%x",wctrlFlags); if( wctrlFlags & WHERE_USE_LIMIT ){ sqlite3DebugPrintf(", limit: %d", iAuxArg); } sqlite3DebugPrintf(")\n"); - } - if( sqlite3WhereTrace & 0x100 ){ /* Display all terms of the WHERE clause */ - int i; - for(i=0; inTerm; i++){ - whereTermPrint(&sWLB.pWC->a[i], i); + if( sqlite3WhereTrace & 0x8000 ){ + Select sSelect; + memset(&sSelect, 0, sizeof(sSelect)); + sSelect.selFlags = SF_WhereBegin; + sSelect.pSrc = pTabList; + sSelect.pWhere = pWhere; + sSelect.pOrderBy = pOrderBy; + sSelect.pEList = pResultSet; + sqlite3TreeViewSelect(0, &sSelect, 0); + } + if( sqlite3WhereTrace & 0x4000 ){ /* Display all WHERE clause terms */ + sqlite3DebugPrintf("---- WHERE clause at start of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); } } #endif @@ -130253,33 +169922,55 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( if( nTabList!=1 || whereShortCut(&sWLB)==0 ){ rc = whereLoopAddAll(&sWLB); if( rc ) goto whereBeginError; - -#ifdef WHERETRACE_ENABLED - if( sqlite3WhereTrace ){ /* Display all of the WhereLoop objects */ - WhereLoop *p; - int i; - static const char zLabel[] = "0123456789abcdefghijklmnopqrstuvwyxz" - "ABCDEFGHIJKLMNOPQRSTUVWYXZ"; - for(p=pWInfo->pLoops, i=0; p; p=p->pNextLoop, i++){ - p->cId = zLabel[i%sizeof(zLabel)]; - whereLoopPrint(p, sWLB.pWC); + +#ifdef SQLITE_ENABLE_STAT4 + /* If one or more WhereTerm.truthProb values were used in estimating + ** loop parameters, but then those truthProb values were subsequently + ** changed based on STAT4 information while computing subsequent loops, + ** then we need to rerun the whole loop building process so that all + ** loops will be built using the revised truthProb values. */ + if( sWLB.bldFlags2 & SQLITE_BLDF2_2NDPASS ){ + WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); + WHERETRACE(0xffffffff, + ("**** Redo all loop computations due to" + " TERM_HIGHTRUTH changes ****\n")); + while( pWInfo->pLoops ){ + WhereLoop *p = pWInfo->pLoops; + pWInfo->pLoops = p->pNextLoop; + whereLoopDelete(db, p); } + rc = whereLoopAddAll(&sWLB); + if( rc ) goto whereBeginError; } #endif - + WHERETRACE_ALL_LOOPS(pWInfo, sWLB.pWC); + wherePathSolver(pWInfo, 0); if( db->mallocFailed ) goto whereBeginError; if( pWInfo->pOrderBy ){ - wherePathSolver(pWInfo, pWInfo->nRowOut+1); + whereInterstageHeuristic(pWInfo); + wherePathSolver(pWInfo, pWInfo->nRowOut<0 ? 1 : pWInfo->nRowOut+1); if( db->mallocFailed ) goto whereBeginError; } + + /* TUNING: Assume that a DISTINCT clause on a subquery reduces + ** the output size by a factor of 8 (LogEst -30). + */ + if( (pWInfo->wctrlFlags & WHERE_WANT_DISTINCT)!=0 ){ + WHERETRACE(0x0080,("nRowOut reduced from %d to %d due to DISTINCT\n", + pWInfo->nRowOut, pWInfo->nRowOut-30)); + pWInfo->nRowOut -= 30; + } + } + assert( pWInfo->pTabList!=0 ); if( pWInfo->pOrderBy==0 && (db->flags & SQLITE_ReverseOrder)!=0 ){ - pWInfo->revMask = ALLBITS; + whereReverseScanOrder(pWInfo); } - if( pParse->nErr || NEVER(db->mallocFailed) ){ + if( pParse->nErr ){ goto whereBeginError; } + assert( db->mallocFailed==0 ); #ifdef WHERETRACE_ENABLED if( sqlite3WhereTrace ){ sqlite3DebugPrintf("---- Solution nRow=%d", pWInfo->nRowOut); @@ -130302,59 +169993,82 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } sqlite3DebugPrintf("\n"); for(ii=0; iinLevel; ii++){ - whereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); + sqlite3WhereLoopPrint(pWInfo->a[ii].pWLoop, sWLB.pWC); } } #endif - /* Attempt to omit tables from the join that do not effect the result */ - if( pWInfo->nLevel>=2 - && pDistinctSet!=0 - && OptimizationEnabled(db, SQLITE_OmitNoopJoin) + + /* Attempt to omit tables from a join that do not affect the result. + ** See the comment on whereOmitNoopJoin() for further information. + ** + ** This query optimization is factored out into a separate "no-inline" + ** procedure to keep the sqlite3WhereBegin() procedure from becoming + ** too large. If sqlite3WhereBegin() becomes too large, that prevents + ** some C-compiler optimizers from in-lining the + ** sqlite3WhereCodeOneLoopStart() procedure, and it is important to + ** in-line sqlite3WhereCodeOneLoopStart() for performance reasons. + */ + notReady = ~(Bitmask)0; + if( pWInfo->nLevel>=2 /* Must be a join, or this opt8n is pointless */ + && pResultSet!=0 /* Condition (1) */ + && 0==(wctrlFlags & (WHERE_AGG_DISTINCT|WHERE_KEEP_ALL_JOINS)) /* (1),(6) */ + && OptimizationEnabled(db, SQLITE_OmitNoopJoin) /* (7) */ ){ - Bitmask tabUsed = sqlite3WhereExprListUsage(pMaskSet, pDistinctSet); - if( sWLB.pOrderBy ){ - tabUsed |= sqlite3WhereExprListUsage(pMaskSet, sWLB.pOrderBy); - } - while( pWInfo->nLevel>=2 ){ - WhereTerm *pTerm, *pEnd; - pLoop = pWInfo->a[pWInfo->nLevel-1].pWLoop; - if( (pWInfo->pTabList->a[pLoop->iTab].fg.jointype & JT_LEFT)==0 ) break; - if( (wctrlFlags & WHERE_WANT_DISTINCT)==0 - && (pLoop->wsFlags & WHERE_ONEROW)==0 - ){ - break; - } - if( (tabUsed & pLoop->maskSelf)!=0 ) break; - pEnd = sWLB.pWC->a + sWLB.pWC->nTerm; - for(pTerm=sWLB.pWC->a; pTermprereqAll & pLoop->maskSelf)!=0 - && !ExprHasProperty(pTerm->pExpr, EP_FromJoin) - ){ - break; - } - } - if( pTerm drop loop %c not used\n", pLoop->cId)); - pWInfo->nLevel--; - nTabList--; - } + notReady = whereOmitNoopJoin(pWInfo, notReady); + nTabList = pWInfo->nLevel; + assert( nTabList>0 ); } - WHERETRACE(0xffff,("*** Optimizer Finished ***\n")); + + /* Check to see if there are any SEARCH loops that might benefit from + ** using a Bloom filter. + */ + if( pWInfo->nLevel>=2 + && OptimizationEnabled(db, SQLITE_BloomFilter) + ){ + whereCheckIfBloomFilterIsUseful(pWInfo); + } + +#if defined(WHERETRACE_ENABLED) + if( sqlite3WhereTrace & 0x4000 ){ /* Display all terms of the WHERE clause */ + sqlite3DebugPrintf("---- WHERE clause at end of analysis:\n"); + sqlite3WhereClausePrint(sWLB.pWC); + } + WHERETRACE(0xffffffff,("*** Optimizer Finished ***\n")); +#endif pWInfo->pParse->nQueryLoop += pWInfo->nRowOut; /* If the caller is an UPDATE or DELETE statement that is requesting ** to use a one-pass algorithm, determine if this is appropriate. + ** + ** A one-pass approach can be used if the caller has requested one + ** and either (a) the scan visits at most one row or (b) each + ** of the following are true: + ** + ** * the caller has indicated that a one-pass approach can be used + ** with multiple rows (by setting WHERE_ONEPASS_MULTIROW), and + ** * the table is not a virtual table, and + ** * either the scan does not use the OR optimization or the caller + ** is a DELETE operation (WHERE_DUPLICATES_OK is only specified + ** for DELETE). + ** + ** The last qualification is because an UPDATE statement uses + ** WhereInfo.aiCurOnePass[1] to determine whether or not it really can + ** use a one-pass approach, and this is not set accurately for scans + ** that use the OR optimization. */ assert( (wctrlFlags & WHERE_ONEPASS_DESIRED)==0 || pWInfo->nLevel==1 ); if( (wctrlFlags & WHERE_ONEPASS_DESIRED)!=0 ){ int wsFlags = pWInfo->a[0].pWLoop->wsFlags; int bOnerow = (wsFlags & WHERE_ONEROW)!=0; - if( bOnerow - || ((wctrlFlags & WHERE_ONEPASS_MULTIROW)!=0 - && 0==(wsFlags & WHERE_VIRTUALTABLE)) - ){ + assert( !(wsFlags&WHERE_VIRTUALTABLE) || IsVirtual(pTabList->a[0].pSTab) ); + if( bOnerow || ( + 0!=(wctrlFlags & WHERE_ONEPASS_MULTIROW) + && !IsVirtual(pTabList->a[0].pSTab) + && (0==(wsFlags & WHERE_MULTI_OR) || (wctrlFlags & WHERE_DUPLICATES_OK)) + && OptimizationEnabled(db, SQLITE_OnePass) + )){ pWInfo->eOnePass = bOnerow ? ONEPASS_SINGLE : ONEPASS_MULTI; - if( HasRowid(pTabList->a[0].pTab) && (wsFlags & WHERE_IDX_ONLY) ){ + if( HasRowid(pTabList->a[0].pSTab) && (wsFlags & WHERE_IDX_ONLY) ){ if( wctrlFlags & WHERE_ONEPASS_MULTIROW ){ bFordelete = OPFLAG_FORDELETE; } @@ -130369,13 +170083,13 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( for(ii=0, pLevel=pWInfo->a; iia[pLevel->iFrom]; - pTab = pTabItem->pTab; + pTab = pTabItem->pSTab; iDb = sqlite3SchemaToIndex(db, pTab->pSchema); pLoop = pLevel->pWLoop; - if( (pTab->tabFlags & TF_Ephemeral)!=0 || pTab->pSelect ){ + if( (pTab->tabFlags & TF_Ephemeral)!=0 || IsView(pTab) ){ /* Do nothing */ }else #ifndef SQLITE_OMIT_VIRTUALTABLE @@ -130387,8 +170101,10 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( /* noop */ }else #endif - if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - && (wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 ){ + if( ((pLoop->wsFlags & WHERE_IDX_ONLY)==0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)==0) + || (pTabItem->fg.jointype & (JT_LTORJ|JT_RIGHT))!=0 + ){ int op = OP_OpenRead; if( pWInfo->eOnePass!=ONEPASS_OFF ){ op = OP_OpenWrite; @@ -130398,7 +170114,14 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( pTabItem->iCursor==pLevel->iTabCur ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS-1 ); testcase( pWInfo->eOnePass==ONEPASS_OFF && pTab->nCol==BMS ); - if( pWInfo->eOnePass==ONEPASS_OFF && pTab->nColeOnePass==ONEPASS_OFF + && pTab->nColtabFlags & (TF_HasGenerated|TF_WithoutRowid))==0 + && (pLoop->wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))==0 + ){ + /* If we know that only a prefix of the record will be used, + ** it is advantageous to reduce the "column count" field in + ** the P4 operand of the OP_OpenRead/Write opcode. */ Bitmask b = pTabItem->colUsed; int n = 0; for(; b; b=b>>1, n++){} @@ -130406,7 +170129,7 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( assert( n<=pTab->nCol ); } #ifdef SQLITE_ENABLE_CURSOR_HINTS - if( pLoop->u.btree.pIndex!=0 ){ + if( pLoop->u.btree.pIndex!=0 && (pTab->tabFlags & TF_WithoutRowid)==0 ){ sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ|bFordelete); }else #endif @@ -130424,17 +170147,17 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( Index *pIx = pLoop->u.btree.pIndex; int iIndexCur; int op = OP_OpenRead; - /* iAuxArg is always set if to a positive value if ONEPASS is possible */ + /* iAuxArg is always set to a positive value if ONEPASS is possible */ assert( iAuxArg!=0 || (pWInfo->wctrlFlags & WHERE_ONEPASS_DESIRED)==0 ); if( !HasRowid(pTab) && IsPrimaryKeyIndex(pIx) - && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 + && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ /* This is one term of an OR-optimization using the PRIMARY KEY of a ** WITHOUT ROWID table. No need for a separate index */ iIndexCur = pLevel->iTabCur; op = 0; }else if( pWInfo->eOnePass!=ONEPASS_OFF ){ - Index *pJ = pTabItem->pTab->pIndex; + Index *pJ = pTabItem->pSTab->pIndex; iIndexCur = iAuxArg; assert( wctrlFlags & WHERE_ONEPASS_DESIRED ); while( ALWAYS(pJ) && pJ!=pIx ){ @@ -130443,13 +170166,22 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } op = OP_OpenWrite; pWInfo->aiCurOnePass[1] = iIndexCur; - }else if( iAuxArg && (wctrlFlags & WHERE_ONETABLE_ONLY)!=0 ){ + }else if( iAuxArg && (wctrlFlags & WHERE_OR_SUBCLAUSE)!=0 ){ iIndexCur = iAuxArg; - if( wctrlFlags & WHERE_REOPEN_IDX ) op = OP_ReopenIdx; + op = OP_ReopenIdx; }else{ iIndexCur = pParse->nTab++; + if( pIx->bHasExpr && OptimizationEnabled(db, SQLITE_IndexedExpr) ){ + whereAddIndexedExpr(pParse, pIx, iIndexCur, pTabItem); + } + if( pIx->pPartIdxWhere && (pTabItem->fg.jointype & JT_RIGHT)==0 ){ + wherePartIdxExpr( + pParse, pIx, pIx->pPartIdxWhere, 0, iIndexCur, pTabItem + ); + } } pLevel->iIdxCur = iIndexCur; + assert( pIx!=0 ); assert( pIx->pSchema==pTab->pSchema ); assert( iIndexCur>=0 ); if( op ){ @@ -130457,9 +170189,12 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( sqlite3VdbeSetP4KeyInfo(pParse, pIx); if( (pLoop->wsFlags & WHERE_CONSTRAINT)!=0 && (pLoop->wsFlags & (WHERE_COLUMN_RANGE|WHERE_SKIPSCAN))==0 + && (pLoop->wsFlags & WHERE_BIGNULL_SORT)==0 + && (pLoop->wsFlags & WHERE_IN_SEEKSCAN)==0 && (pWInfo->wctrlFlags&WHERE_ORDERBY_MIN)==0 + && pWInfo->eDistinct!=WHERE_DISTINCT_ORDERED ){ - sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); /* Hint to COMDB2 */ + sqlite3VdbeChangeP5(v, OPFLAG_SEEKEQ); } VdbeComment((v, "%s", pIx->zName)); #ifdef SQLITE_ENABLE_COLUMN_USED_MASK @@ -130480,6 +170215,37 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( } } if( iDb>=0 ) sqlite3CodeVerifySchema(pParse, iDb); + if( (pTabItem->fg.jointype & JT_RIGHT)!=0 + && (pLevel->pRJ = sqlite3WhereMalloc(pWInfo, sizeof(WhereRightJoin)))!=0 + ){ + WhereRightJoin *pRJ = pLevel->pRJ; + pRJ->iMatch = pParse->nTab++; + pRJ->regBloom = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Blob, 65536, pRJ->regBloom); + pRJ->regReturn = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, pRJ->regReturn); + assert( pTab==pTabItem->pSTab ); + if( HasRowid(pTab) ){ + KeyInfo *pInfo; + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, 1); + pInfo = sqlite3KeyInfoAlloc(pParse->db, 1, 0); + if( pInfo ){ + pInfo->aColl[0] = 0; + pInfo->aSortFlags[0] = 0; + sqlite3VdbeAppendP4(v, pInfo, P4_KEYINFO); + } + }else{ + Index *pPk = sqlite3PrimaryKeyIndex(pTab); + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pRJ->iMatch, pPk->nKeyCol); + sqlite3VdbeSetP4KeyInfo(pParse, pPk); + } + pLoop->wsFlags &= ~WHERE_IDX_ONLY; + /* The nature of RIGHT JOIN processing is such that it messes up + ** the output order. So omit any ORDER BY/GROUP BY elimination + ** optimizations. We need to do an actual sort for RIGHT JOIN. */ + pWInfo->nOBSat = 0; + pWInfo->eDistinct = WHERE_DISTINCT_UNORDERED; + } } pWInfo->iTop = sqlite3VdbeCurrentAddr(v); if( db->mallocFailed ) goto whereBeginError; @@ -130488,32 +170254,53 @@ SQLITE_PRIVATE WhereInfo *sqlite3WhereBegin( ** loop below generates code for a single nested loop of the VM ** program. */ - notReady = ~(Bitmask)0; for(ii=0; iinErr ) goto whereBeginError; pLevel = &pWInfo->a[ii]; wsFlags = pLevel->pWLoop->wsFlags; + pSrc = &pTabList->a[pLevel->iFrom]; + if( pSrc->fg.isMaterialized ){ + Subquery *pSubq; + int iOnce = 0; + assert( pSrc->fg.isSubquery ); + pSubq = pSrc->u4.pSubq; + if( pSrc->fg.isCorrelated==0 ){ + iOnce = sqlite3VdbeAddOp0(v, OP_Once); VdbeCoverage(v); + }else{ + iOnce = 0; + } + sqlite3VdbeAddOp2(v, OP_Gosub, pSubq->regReturn, pSubq->addrFillSub); + VdbeComment((v, "materialize %!S", pSrc)); + if( iOnce ) sqlite3VdbeJumpHere(v, iOnce); + } + assert( pTabList == pWInfo->pTabList ); + if( (wsFlags & (WHERE_AUTO_INDEX|WHERE_BLOOMFILTER))!=0 ){ + if( (wsFlags & WHERE_AUTO_INDEX)!=0 ){ #ifndef SQLITE_OMIT_AUTOMATIC_INDEX - if( (pLevel->pWLoop->wsFlags & WHERE_AUTO_INDEX)!=0 ){ - constructAutomaticIndex(pParse, &pWInfo->sWC, - &pTabList->a[pLevel->iFrom], notReady, pLevel); + constructAutomaticIndex(pParse, &pWInfo->sWC, notReady, pLevel); +#endif + }else{ + sqlite3ConstructBloomFilter(pWInfo, ii, pLevel, notReady); + } if( db->mallocFailed ) goto whereBeginError; } -#endif addrExplain = sqlite3WhereExplainOneScan( - pParse, pTabList, pLevel, ii, pLevel->iFrom, wctrlFlags + pParse, pTabList, pLevel, wctrlFlags ); pLevel->addrBody = sqlite3VdbeCurrentAddr(v); - notReady = sqlite3WhereCodeOneLoopStart(pWInfo, ii, notReady); + notReady = sqlite3WhereCodeOneLoopStart(pParse,v,pWInfo,ii,pLevel,notReady); pWInfo->iContinue = pLevel->addrCont; - if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_ONETABLE_ONLY)==0 ){ + if( (wsFlags&WHERE_MULTI_OR)==0 && (wctrlFlags&WHERE_OR_SUBCLAUSE)==0 ){ sqlite3WhereAddScanStatus(v, pTabList, pLevel, addrExplain); } } /* Done. */ VdbeModuleComment((v, "Begin WHERE-core")); + pWInfo->iEndWhere = sqlite3VdbeCurrentAddr(v); return pWInfo; /* Jump here if malloc fails */ @@ -130522,11 +170309,36 @@ whereBeginError: pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); } +#ifdef WHERETRACE_ENABLED + /* Prevent harmless compiler warnings about debugging routines + ** being declared but never used */ + sqlite3ShowWhereLoopList(0); +#endif /* WHERETRACE_ENABLED */ return 0; } /* -** Generate the end of the WHERE loop. See comments on +** Part of sqlite3WhereEnd() will rewrite opcodes to reference the +** index rather than the main table. In SQLITE_DEBUG mode, we want +** to trace those changes if PRAGMA vdbe_addoptrace=on. This routine +** does that. +*/ +#ifndef SQLITE_DEBUG +# define OpcodeRewriteTrace(D,K,P) /* no-op */ +#else +# define OpcodeRewriteTrace(D,K,P) sqlite3WhereOpcodeRewriteTrace(D,K,P) + static void sqlite3WhereOpcodeRewriteTrace( + sqlite3 *db, + int pc, + VdbeOp *pOp + ){ + if( (db->flags & SQLITE_VdbeAddopTrace)==0 ) return; + sqlite3VdbePrintOp(0, pc, pOp); + } +#endif + +/* +** Generate the end of the WHERE loop. See comments on ** sqlite3WhereBegin() for additional information. */ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ @@ -130537,38 +170349,122 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ WhereLoop *pLoop; SrcList *pTabList = pWInfo->pTabList; sqlite3 *db = pParse->db; + int iEnd = sqlite3VdbeCurrentAddr(v); + int nRJ = 0; /* Generate loop termination code. */ VdbeModuleComment((v, "End WHERE-core")); - sqlite3ExprCacheClear(pParse); for(i=pWInfo->nLevel-1; i>=0; i--){ int addr; pLevel = &pWInfo->a[i]; + if( pLevel->pRJ ){ + /* Terminate the subroutine that forms the interior of the loop of + ** the RIGHT JOIN table */ + WhereRightJoin *pRJ = pLevel->pRJ; + sqlite3VdbeResolveLabel(v, pLevel->addrCont); + pLevel->addrCont = 0; + pRJ->endSubrtn = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp3(v, OP_Return, pRJ->regReturn, pRJ->addrSubrtn, 1); + VdbeCoverage(v); + nRJ++; + } pLoop = pLevel->pWLoop; - sqlite3VdbeResolveLabel(v, pLevel->addrCont); if( pLevel->op!=OP_Noop ){ +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + int addrSeek = 0; + Index *pIdx; + int n; + if( pWInfo->eDistinct==WHERE_DISTINCT_ORDERED + && i==pWInfo->nLevel-1 /* Ticket [ef9318757b152e3] 2017-10-21 */ + && (pLoop->wsFlags & WHERE_INDEXED)!=0 + && (pIdx = pLoop->u.btree.pIndex)->hasStat1 + && (n = pLoop->u.btree.nDistinctCol)>0 + && pIdx->aiRowLogEst[n]>=36 + ){ + int r1 = pParse->nMem+1; + int j, op; + for(j=0; jiIdxCur, j, r1+j); + } + pParse->nMem += n+1; + op = pLevel->op==OP_Prev ? OP_SeekLT : OP_SeekGT; + addrSeek = sqlite3VdbeAddOp4Int(v, op, pLevel->iIdxCur, 0, r1, n); + VdbeCoverageIf(v, op==OP_SeekLT); + VdbeCoverageIf(v, op==OP_SeekGT); + sqlite3VdbeAddOp2(v, OP_Goto, 1, pLevel->p2); + } +#endif /* SQLITE_DISABLE_SKIPAHEAD_DISTINCT */ + /* The common case: Advance to the next row */ + if( pLevel->addrCont ) sqlite3VdbeResolveLabel(v, pLevel->addrCont); sqlite3VdbeAddOp3(v, pLevel->op, pLevel->p1, pLevel->p2, pLevel->p3); sqlite3VdbeChangeP5(v, pLevel->p5); VdbeCoverage(v); VdbeCoverageIf(v, pLevel->op==OP_Next); VdbeCoverageIf(v, pLevel->op==OP_Prev); VdbeCoverageIf(v, pLevel->op==OP_VNext); + if( pLevel->regBignull ){ + sqlite3VdbeResolveLabel(v, pLevel->addrBignull); + sqlite3VdbeAddOp2(v, OP_DecrJumpZero, pLevel->regBignull, pLevel->p2-1); + VdbeCoverage(v); + } +#ifndef SQLITE_DISABLE_SKIPAHEAD_DISTINCT + if( addrSeek ) sqlite3VdbeJumpHere(v, addrSeek); +#endif + }else if( pLevel->addrCont ){ + sqlite3VdbeResolveLabel(v, pLevel->addrCont); } - if( pLoop->wsFlags & WHERE_IN_ABLE && pLevel->u.in.nIn>0 ){ + if( (pLoop->wsFlags & WHERE_IN_ABLE)!=0 && pLevel->u.in.nIn>0 ){ struct InLoop *pIn; int j; sqlite3VdbeResolveLabel(v, pLevel->addrNxt); for(j=pLevel->u.in.nIn, pIn=&pLevel->u.in.aInLoop[j-1]; j>0; j--, pIn--){ + assert( sqlite3VdbeGetOp(v, pIn->addrInTop+1)->opcode==OP_IsNull + || pParse->db->mallocFailed ); sqlite3VdbeJumpHere(v, pIn->addrInTop+1); - sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); - VdbeCoverage(v); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_PrevIfOpen); - VdbeCoverageIf(v, pIn->eEndLoopOp==OP_NextIfOpen); + if( pIn->eEndLoopOp!=OP_Noop ){ + if( pIn->nPrefix ){ + int bEarlyOut = + (pLoop->wsFlags & WHERE_VIRTUALTABLE)==0 + && (pLoop->wsFlags & WHERE_IN_EARLYOUT)!=0; + if( pLevel->iLeftJoin ){ + /* For LEFT JOIN queries, cursor pIn->iCur may not have been + ** opened yet. This occurs for WHERE clauses such as + ** "a = ? AND b IN (...)", where the index is on (a, b). If + ** the RHS of the (a=?) is NULL, then the "b IN (...)" may + ** never have been coded, but the body of the loop run to + ** return the null-row. So, if the cursor is not open yet, + ** jump over the OP_Next or OP_Prev instruction about to + ** be coded. */ + sqlite3VdbeAddOp2(v, OP_IfNotOpen, pIn->iCur, + sqlite3VdbeCurrentAddr(v) + 2 + bEarlyOut); + VdbeCoverage(v); + } + if( bEarlyOut ){ + sqlite3VdbeAddOp4Int(v, OP_IfNoHope, pLevel->iIdxCur, + sqlite3VdbeCurrentAddr(v)+2, + pIn->iBase, pIn->nPrefix); + VdbeCoverage(v); + /* Retarget the OP_IsNull against the left operand of IN so + ** it jumps past the OP_IfNoHope. This is because the + ** OP_IsNull also bypasses the OP_Affinity opcode that is + ** required by OP_IfNoHope. */ + sqlite3VdbeJumpHere(v, pIn->addrInTop+1); + } + } + sqlite3VdbeAddOp2(v, pIn->eEndLoopOp, pIn->iCur, pIn->addrInTop); + VdbeCoverage(v); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Prev); + VdbeCoverageIf(v, pIn->eEndLoopOp==OP_Next); + } sqlite3VdbeJumpHere(v, pIn->addrInTop-1); } } sqlite3VdbeResolveLabel(v, pLevel->addrBrk); + if( pLevel->pRJ ){ + sqlite3VdbeAddOp3(v, OP_Return, pLevel->pRJ->regReturn, 0, 1); + VdbeCoverage(v); + } if( pLevel->addrSkip ){ sqlite3VdbeGoto(v, pLevel->addrSkip); VdbeComment((v, "next skip-scan on %s", pLoop->u.btree.pIndex->zName)); @@ -130583,13 +170479,31 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ } #endif if( pLevel->iLeftJoin ){ + int ws = pLoop->wsFlags; addr = sqlite3VdbeAddOp1(v, OP_IfPos, pLevel->iLeftJoin); VdbeCoverage(v); - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 - || (pLoop->wsFlags & WHERE_INDEXED)!=0 ); - if( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_NullRow, pTabList->a[i].iCursor); + assert( (ws & WHERE_IDX_ONLY)==0 || (ws & WHERE_INDEXED)!=0 ); + if( (ws & WHERE_IDX_ONLY)==0 ){ + SrcItem *pSrc = &pTabList->a[pLevel->iFrom]; + assert( pLevel->iTabCur==pSrc->iCursor ); + if( pSrc->fg.viaCoroutine ){ + int m, n; + assert( pSrc->fg.isSubquery ); + n = pSrc->u4.pSubq->regResult; + assert( pSrc->pSTab!=0 ); + m = pSrc->pSTab->nCol; + sqlite3VdbeAddOp3(v, OP_Null, 0, n, n+m-1); + } + sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iTabCur); } - if( pLoop->wsFlags & WHERE_INDEXED ){ + if( (ws & WHERE_INDEXED) + || ((ws & WHERE_MULTI_OR) && pLevel->u.pCoveringIdx) + ){ + if( ws & WHERE_MULTI_OR ){ + Index *pIx = pLevel->u.pCoveringIdx; + int iDb = sqlite3SchemaToIndex(db, pIx->pSchema); + sqlite3VdbeAddOp3(v, OP_ReopenIdx, pLevel->iIdxCur, pIx->tnum, iDb); + sqlite3VdbeSetP4KeyInfo(pParse, pIx); + } sqlite3VdbeAddOp1(v, OP_NullRow, pLevel->iIdxCur); } if( pLevel->op==OP_Return ){ @@ -130600,60 +170514,46 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ sqlite3VdbeJumpHere(v, addr); } VdbeModuleComment((v, "End WHERE-loop%d: %s", i, - pWInfo->pTabList->a[pLevel->iFrom].pTab->zName)); + pWInfo->pTabList->a[pLevel->iFrom].pSTab->zName)); } - /* The "break" point is here, just past the end of the outer loop. - ** Set it. - */ - sqlite3VdbeResolveLabel(v, pWInfo->iBreak); - assert( pWInfo->nLevel<=pTabList->nSrc ); for(i=0, pLevel=pWInfo->a; inLevel; i++, pLevel++){ int k, last; - VdbeOp *pOp; + VdbeOp *pOp, *pLastOp; Index *pIdx = 0; - struct SrcList_item *pTabItem = &pTabList->a[pLevel->iFrom]; - Table *pTab = pTabItem->pTab; + SrcItem *pTabItem = &pTabList->a[pLevel->iFrom]; + Table *pTab = pTabItem->pSTab; assert( pTab!=0 ); pLoop = pLevel->pWLoop; + /* Do RIGHT JOIN processing. Generate code that will output the + ** unmatched rows of the right operand of the RIGHT JOIN with + ** all of the columns of the left operand set to NULL. + */ + if( pLevel->pRJ ){ + sqlite3WhereRightJoinLoop(pWInfo, i, pLevel); + continue; + } + /* For a co-routine, change all OP_Column references to the table of ** the co-routine into OP_Copy of result contained in a register. ** OP_Rowid becomes OP_Null. */ - if( pTabItem->fg.viaCoroutine && !db->mallocFailed ){ - translateColumnToCopy(v, pLevel->addrBody, pLevel->iTabCur, - pTabItem->regResult, 0); + if( pTabItem->fg.viaCoroutine ){ + testcase( pParse->db->mallocFailed ); + assert( pTabItem->fg.isSubquery ); + assert( pTabItem->u4.pSubq->regResult>=0 ); + translateColumnToCopy(pParse, pLevel->addrBody, pLevel->iTabCur, + pTabItem->u4.pSubq->regResult, 0); continue; } - /* Close all of the cursors that were opened by sqlite3WhereBegin. - ** Except, do not close cursors that will be reused by the OR optimization - ** (WHERE_OMIT_OPEN_CLOSE). And do not close the OP_OpenWrite cursors - ** created for the ONEPASS optimization. - */ - if( (pTab->tabFlags & TF_Ephemeral)==0 - && pTab->pSelect==0 - && (pWInfo->wctrlFlags & WHERE_OMIT_OPEN_CLOSE)==0 - ){ - int ws = pLoop->wsFlags; - if( pWInfo->eOnePass==ONEPASS_OFF && (ws & WHERE_IDX_ONLY)==0 ){ - sqlite3VdbeAddOp1(v, OP_Close, pTabItem->iCursor); - } - if( (ws & WHERE_INDEXED)!=0 - && (ws & (WHERE_IPK|WHERE_AUTO_INDEX))==0 - && pLevel->iIdxCur!=pWInfo->aiCurOnePass[1] - ){ - sqlite3VdbeAddOp1(v, OP_Close, pLevel->iIdxCur); - } - } - /* If this scan uses an index, make VDBE code substitutions to read data ** from the index instead of from the table where possible. In some cases ** this optimization prevents the table from ever being read, which can ** yield a significant performance boost. - ** + ** ** Calls to the code generator in between sqlite3WhereBegin and ** sqlite3WhereEnd will have created code that references the table ** directly. This loop scans all that code looking for opcodes @@ -130663,50 +170563,121 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ if( pLoop->wsFlags & (WHERE_INDEXED|WHERE_IDX_ONLY) ){ pIdx = pLoop->u.btree.pIndex; }else if( pLoop->wsFlags & WHERE_MULTI_OR ){ - pIdx = pLevel->u.pCovidx; + pIdx = pLevel->u.pCoveringIdx; } if( pIdx - && (pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable)) && !db->mallocFailed ){ - last = sqlite3VdbeCurrentAddr(v); - k = pLevel->addrBody; + if( pWInfo->eOnePass==ONEPASS_OFF || !HasRowid(pIdx->pTable) ){ + last = iEnd; + }else{ + last = pWInfo->iEndWhere; + } + if( pIdx->bHasExpr ){ + IndexedExpr *p = pParse->pIdxEpr; + while( p ){ + if( p->iIdxCur==pLevel->iIdxCur ){ +#ifdef WHERETRACE_ENABLED + if( sqlite3WhereTrace & 0x200 ){ + sqlite3DebugPrintf("Disable pParse->pIdxEpr term {%d,%d}\n", + p->iIdxCur, p->iIdxCol); + if( sqlite3WhereTrace & 0x5000 ) sqlite3ShowExpr(p->pExpr); + } +#endif + p->iDataCur = -1; + p->iIdxCur = -1; + } + p = p->pIENext; + } + } + k = pLevel->addrBody + 1; +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_VdbeAddopTrace ){ + printf("TRANSLATE cursor %d->%d in opcode range %d..%d\n", + pLevel->iTabCur, pLevel->iIdxCur, k, last-1); + } + /* Proof that the "+1" on the k value above is safe */ + pOp = sqlite3VdbeGetOp(v, k - 1); + assert( pOp->opcode!=OP_Column || pOp->p1!=pLevel->iTabCur ); + assert( pOp->opcode!=OP_Rowid || pOp->p1!=pLevel->iTabCur ); + assert( pOp->opcode!=OP_IfNullRow || pOp->p1!=pLevel->iTabCur ); +#endif pOp = sqlite3VdbeGetOp(v, k); - for(; kp1!=pLevel->iTabCur ) continue; - if( pOp->opcode==OP_Column ){ + pLastOp = pOp + (last - k); + assert( pOp<=pLastOp ); + do{ + if( pOp->p1!=pLevel->iTabCur ){ + /* no-op */ + }else if( pOp->opcode==OP_Column +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + || pOp->opcode==OP_Offset +#endif + ){ int x = pOp->p2; assert( pIdx->pTable==pTab ); +#ifdef SQLITE_ENABLE_OFFSET_SQL_FUNC + if( pOp->opcode==OP_Offset ){ + /* Do not need to translate the column number */ + }else +#endif if( !HasRowid(pTab) ){ Index *pPk = sqlite3PrimaryKeyIndex(pTab); x = pPk->aiColumn[x]; assert( x>=0 ); + }else{ + testcase( x!=sqlite3StorageColumnToTable(pTab,x) ); + x = sqlite3StorageColumnToTable(pTab,x); } - x = sqlite3ColumnOfIndex(pIdx, x); + x = sqlite3TableColumnToIndex(pIdx, x); if( x>=0 ){ pOp->p2 = x; pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); + }else{ + /* Unable to translate the table reference into an index + ** reference. Verify that this is harmless - that the + ** table being referenced really is open. + */ + if( pLoop->wsFlags & WHERE_IDX_ONLY ){ + sqlite3ErrorMsg(pParse, "internal query planner error"); + pParse->rc = SQLITE_INTERNAL; + } } - assert( (pLoop->wsFlags & WHERE_IDX_ONLY)==0 || x>=0 ); }else if( pOp->opcode==OP_Rowid ){ pOp->p1 = pLevel->iIdxCur; pOp->opcode = OP_IdxRowid; + OpcodeRewriteTrace(db, k, pOp); + }else if( pOp->opcode==OP_IfNullRow ){ + pOp->p1 = pLevel->iIdxCur; + OpcodeRewriteTrace(db, k, pOp); } - } +#ifdef SQLITE_DEBUG + k++; +#endif + }while( (++pOp)flags & SQLITE_VdbeAddopTrace ) printf("TRANSLATE complete\n"); +#endif } } + /* The "break" point is here, just past the end of the outer loop. + ** Set it. + */ + sqlite3VdbeResolveLabel(v, pWInfo->iBreak); + /* Final cleanup */ pParse->nQueryLoop = pWInfo->savedNQueryLoop; whereInfoFree(db, pWInfo); + pParse->withinRJSubrtn -= nRJ; return; } /************** End of where.c ***********************************************/ -/************** Begin file parse.c *******************************************/ +/************** Begin file window.c ******************************************/ /* -** 2000-05-29 +** 2018 May 08 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -130716,21 +170687,3127 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ ** May you share freely, never taking more than you give. ** ************************************************************************* -** Driver template for the LEMON parser generator. -** -** The "lemon" program processes an LALR(1) input grammar file, then uses -** this template to construct a parser. The "lemon" program inserts text -** at each "%%" line. Also, any "P-a-r-s-e" identifer prefix (without the -** interstitial "-" characters) contained in this template is changed into -** the value of the %name directive from the grammar. Otherwise, the content -** of this template is copied straight through into the generate parser -** source file. -** -** The following is the concatenation of all %include directives from the -** input grammar file: */ -/* #include */ -/************ Begin %include sections from the grammar ************************/ +/* #include "sqliteInt.h" */ + +#ifndef SQLITE_OMIT_WINDOWFUNC + +/* +** SELECT REWRITING +** +** Any SELECT statement that contains one or more window functions in +** either the select list or ORDER BY clause (the only two places window +** functions may be used) is transformed by function sqlite3WindowRewrite() +** in order to support window function processing. For example, with the +** schema: +** +** CREATE TABLE t1(a, b, c, d, e, f, g); +** +** the statement: +** +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM t1 ORDER BY e; +** +** is transformed to: +** +** SELECT a+1, max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT a, e, c, d, b FROM t1 ORDER BY c, d +** ) ORDER BY e; +** +** The flattening optimization is disabled when processing this transformed +** SELECT statement. This allows the implementation of the window function +** (in this case max()) to process rows sorted in order of (c, d), which +** makes things easier for obvious reasons. More generally: +** +** * FROM, WHERE, GROUP BY and HAVING clauses are all moved to +** the sub-query. +** +** * ORDER BY, LIMIT and OFFSET remain part of the parent query. +** +** * Terminals from each of the expression trees that make up the +** select-list and ORDER BY expressions in the parent query are +** selected by the sub-query. For the purposes of the transformation, +** terminals are column references and aggregate functions. +** +** If there is more than one window function in the SELECT that uses +** the same window declaration (the OVER bit), then a single scan may +** be used to process more than one window function. For example: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY c ORDER BY d) +** FROM t1; +** +** is transformed in the same way as the example above. However: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d), +** min(e) OVER (PARTITION BY a ORDER BY b) +** FROM t1; +** +** Must be transformed to: +** +** SELECT max(b) OVER (PARTITION BY c ORDER BY d) FROM ( +** SELECT e, min(e) OVER (PARTITION BY a ORDER BY b), c, d, b FROM +** SELECT a, e, c, d, b FROM t1 ORDER BY a, b +** ) ORDER BY c, d +** ) ORDER BY e; +** +** so that both min() and max() may process rows in the order defined by +** their respective window declarations. +** +** INTERFACE WITH SELECT.C +** +** When processing the rewritten SELECT statement, code in select.c calls +** sqlite3WhereBegin() to begin iterating through the results of the +** sub-query, which is always implemented as a co-routine. It then calls +** sqlite3WindowCodeStep() to process rows and finish the scan by calling +** sqlite3WhereEnd(). +** +** sqlite3WindowCodeStep() generates VM code so that, for each row returned +** by the sub-query a sub-routine (OP_Gosub) coded by select.c is invoked. +** When the sub-routine is invoked: +** +** * The results of all window-functions for the row are stored +** in the associated Window.regResult registers. +** +** * The required terminal values are stored in the current row of +** temp table Window.iEphCsr. +** +** In some cases, depending on the window frame and the specific window +** functions invoked, sqlite3WindowCodeStep() caches each entire partition +** in a temp table before returning any rows. In other cases it does not. +** This detail is encapsulated within this file, the code generated by +** select.c is the same in either case. +** +** BUILT-IN WINDOW FUNCTIONS +** +** This implementation features the following built-in window functions: +** +** row_number() +** rank() +** dense_rank() +** percent_rank() +** cume_dist() +** ntile(N) +** lead(expr [, offset [, default]]) +** lag(expr [, offset [, default]]) +** first_value(expr) +** last_value(expr) +** nth_value(expr, N) +** +** These are the same built-in window functions supported by Postgres. +** Although the behaviour of aggregate window functions (functions that +** can be used as either aggregates or window functions) allows them to +** be implemented using an API, built-in window functions are much more +** esoteric. Additionally, some window functions (e.g. nth_value()) +** may only be implemented by caching the entire partition in memory. +** As such, some built-in window functions use the same API as aggregate +** window functions and some are implemented directly using VDBE +** instructions. Additionally, for those functions that use the API, the +** window frame is sometimes modified before the SELECT statement is +** rewritten. For example, regardless of the specified window frame, the +** row_number() function always uses: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +** +** See sqlite3WindowUpdate() for details. +** +** As well as some of the built-in window functions, aggregate window +** functions min() and max() are implemented using VDBE instructions if +** the start of the window frame is declared as anything other than +** UNBOUNDED PRECEDING. +*/ + +/* +** Implementation of built-in window function row_number(). Assumes that the +** window frame has been coerced to: +** +** ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void row_numberStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) (*p)++; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void row_numberValueFunc(sqlite3_context *pCtx){ + i64 *p = (i64*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + sqlite3_result_int64(pCtx, (p ? *p : 0)); +} + +/* +** Context object type used by rank(), dense_rank(), percent_rank() and +** cume_dist(). +*/ +struct CallCount { + i64 nValue; + i64 nStep; + i64 nTotal; +}; + +/* +** Implementation of built-in window function dense_rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void dense_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ) p->nStep = 1; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void dense_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nStep ){ + p->nValue++; + p->nStep = 0; + } + sqlite3_result_int64(pCtx, p->nValue); + } +} + +/* +** Implementation of built-in window function nth_value(). This +** implementation is used in "slow mode" only - when the EXCLUDE clause +** is not set to the default value "NO OTHERS". +*/ +struct NthValueCtx { + i64 nStep; + sqlite3_value *pValue; +}; +static void nth_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + i64 iVal; + switch( sqlite3_value_numeric_type(apArg[1]) ){ + case SQLITE_INTEGER: + iVal = sqlite3_value_int64(apArg[1]); + break; + case SQLITE_FLOAT: { + double fVal = sqlite3_value_double(apArg[1]); + if( ((i64)fVal)!=fVal ) goto error_out; + iVal = (i64)fVal; + break; + } + default: + goto error_out; + } + if( iVal<=0 ) goto error_out; + + p->nStep++; + if( iVal==p->nStep ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + return; + + error_out: + sqlite3_result_error( + pCtx, "second argument to nth_value must be a positive integer", -1 + ); +} +static void nth_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } +} +#define nth_valueInvFunc noopStepFunc +#define nth_valueValueFunc noopValueFunc + +static void first_valueStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue==0 ){ + p->pValue = sqlite3_value_dup(apArg[0]); + if( !p->pValue ){ + sqlite3_result_error_nomem(pCtx); + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void first_valueFinalizeFunc(sqlite3_context *pCtx){ + struct NthValueCtx *p; + p = (struct NthValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pValue ){ + sqlite3_result_value(pCtx, p->pValue); + sqlite3_value_free(p->pValue); + p->pValue = 0; + } +} +#define first_valueInvFunc noopStepFunc +#define first_valueValueFunc noopValueFunc + +/* +** Implementation of built-in window function rank(). Assumes that +** the window frame has been set to: +** +** RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW +*/ +static void rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nStep++; + if( p->nValue==0 ){ + p->nValue = p->nStep; + } + } + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); +} +static void rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + sqlite3_result_int64(pCtx, p->nValue); + p->nValue = 0; + } +} + +/* +** Implementation of built-in window function percent_rank(). Assumes that +** the window frame has been set to: +** +** GROUPS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING +*/ +static void percent_rankStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nTotal++; + } +} +static void percent_rankInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; +} +static void percent_rankValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nValue = p->nStep; + if( p->nTotal>1 ){ + double r = (double)p->nValue / (double)(p->nTotal-1); + sqlite3_result_double(pCtx, r); + }else{ + sqlite3_result_double(pCtx, 0.0); + } + } +} +#define percent_rankFinalizeFunc percent_rankValueFunc + +/* +** Implementation of built-in window function cume_dist(). Assumes that +** the window frame has been set to: +** +** GROUPS BETWEEN 1 FOLLOWING AND UNBOUNDED FOLLOWING +*/ +static void cume_distStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + p->nTotal++; + } +} +static void cume_distInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct CallCount *p; + UNUSED_PARAMETER(nArg); assert( nArg==0 ); + UNUSED_PARAMETER(apArg); + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->nStep++; +} +static void cume_distValueFunc(sqlite3_context *pCtx){ + struct CallCount *p; + p = (struct CallCount*)sqlite3_aggregate_context(pCtx, 0); + if( p ){ + double r = (double)(p->nStep) / (double)(p->nTotal); + sqlite3_result_double(pCtx, r); + } +} +#define cume_distFinalizeFunc cume_distValueFunc + +/* +** Context object for ntile() window function. +*/ +struct NtileCtx { + i64 nTotal; /* Total rows in partition */ + i64 nParam; /* Parameter passed to ntile(N) */ + i64 iRow; /* Current row */ +}; + +/* +** Implementation of ntile(). This assumes that the window frame has +** been coerced to: +** +** ROWS CURRENT ROW AND UNBOUNDED FOLLOWING +*/ +static void ntileStepFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p ){ + if( p->nTotal==0 ){ + p->nParam = sqlite3_value_int64(apArg[0]); + if( p->nParam<=0 ){ + sqlite3_result_error( + pCtx, "argument of ntile must be a positive integer", -1 + ); + } + } + p->nTotal++; + } +} +static void ntileInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct NtileCtx *p; + assert( nArg==1 ); UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + p->iRow++; +} +static void ntileValueFunc(sqlite3_context *pCtx){ + struct NtileCtx *p; + p = (struct NtileCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->nParam>0 ){ + int nSize = (p->nTotal / p->nParam); + if( nSize==0 ){ + sqlite3_result_int64(pCtx, p->iRow+1); + }else{ + i64 nLarge = p->nTotal - p->nParam*nSize; + i64 iSmall = nLarge*(nSize+1); + i64 iRow = p->iRow; + + assert( (nLarge*(nSize+1) + (p->nParam-nLarge)*nSize)==p->nTotal ); + + if( iRowpVal); + p->pVal = sqlite3_value_dup(apArg[0]); + if( p->pVal==0 ){ + sqlite3_result_error_nomem(pCtx); + }else{ + p->nVal++; + } + } +} +static void last_valueInvFunc( + sqlite3_context *pCtx, + int nArg, + sqlite3_value **apArg +){ + struct LastValueCtx *p; + UNUSED_PARAMETER(nArg); + UNUSED_PARAMETER(apArg); + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( ALWAYS(p) ){ + p->nVal--; + if( p->nVal==0 ){ + sqlite3_value_free(p->pVal); + p->pVal = 0; + } + } +} +static void last_valueValueFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, 0); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + } +} +static void last_valueFinalizeFunc(sqlite3_context *pCtx){ + struct LastValueCtx *p; + p = (struct LastValueCtx*)sqlite3_aggregate_context(pCtx, sizeof(*p)); + if( p && p->pVal ){ + sqlite3_result_value(pCtx, p->pVal); + sqlite3_value_free(p->pVal); + p->pVal = 0; + } +} + +/* +** Static names for the built-in window function names. These static +** names are used, rather than string literals, so that FuncDef objects +** can be associated with a particular window function by direct +** comparison of the zName pointer. Example: +** +** if( pFuncDef->zName==row_valueName ){ ... } +*/ +static const char row_numberName[] = "row_number"; +static const char dense_rankName[] = "dense_rank"; +static const char rankName[] = "rank"; +static const char percent_rankName[] = "percent_rank"; +static const char cume_distName[] = "cume_dist"; +static const char ntileName[] = "ntile"; +static const char last_valueName[] = "last_value"; +static const char nth_valueName[] = "nth_value"; +static const char first_valueName[] = "first_value"; +static const char leadName[] = "lead"; +static const char lagName[] = "lag"; + +/* +** No-op implementations of xStep() and xFinalize(). Used as place-holders +** for built-in window functions that never call those interfaces. +** +** The noopValueFunc() is called but is expected to do nothing. The +** noopStepFunc() is never called, and so it is marked with NO_TEST to +** let the test coverage routine know not to expect this function to be +** invoked. +*/ +static void noopStepFunc( /*NO_TEST*/ + sqlite3_context *p, /*NO_TEST*/ + int n, /*NO_TEST*/ + sqlite3_value **a /*NO_TEST*/ +){ /*NO_TEST*/ + UNUSED_PARAMETER(p); /*NO_TEST*/ + UNUSED_PARAMETER(n); /*NO_TEST*/ + UNUSED_PARAMETER(a); /*NO_TEST*/ + assert(0); /*NO_TEST*/ +} /*NO_TEST*/ +static void noopValueFunc(sqlite3_context *p){ UNUSED_PARAMETER(p); /*no-op*/ } + +/* Window functions that use all window interfaces: xStep, xFinal, +** xValue, and xInverse */ +#define WINDOWFUNCALL(name,nArg,extra) { \ + nArg, (SQLITE_FUNC_BUILTIN|SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## FinalizeFunc, name ## ValueFunc, \ + name ## InvFunc, name ## Name, {0} \ +} + +/* Window functions that are implemented using bytecode and thus have +** no-op routines for their methods */ +#define WINDOWFUNCNOOP(name,nArg,extra) { \ + nArg, (SQLITE_FUNC_BUILTIN|SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + noopStepFunc, noopValueFunc, noopValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + +/* Window functions that use all window interfaces: xStep, the +** same routine for xFinalize and xValue and which never call +** xInverse. */ +#define WINDOWFUNCX(name,nArg,extra) { \ + nArg, (SQLITE_FUNC_BUILTIN|SQLITE_UTF8|SQLITE_FUNC_WINDOW|extra), 0, 0, \ + name ## StepFunc, name ## ValueFunc, name ## ValueFunc, \ + noopStepFunc, name ## Name, {0} \ +} + + +/* +** Register those built-in window functions that are not also aggregates. +*/ +SQLITE_PRIVATE void sqlite3WindowFunctions(void){ + static FuncDef aWindowFuncs[] = { + WINDOWFUNCX(row_number, 0, 0), + WINDOWFUNCX(dense_rank, 0, 0), + WINDOWFUNCX(rank, 0, 0), + WINDOWFUNCALL(percent_rank, 0, 0), + WINDOWFUNCALL(cume_dist, 0, 0), + WINDOWFUNCALL(ntile, 1, 0), + WINDOWFUNCALL(last_value, 1, 0), + WINDOWFUNCALL(nth_value, 2, 0), + WINDOWFUNCALL(first_value, 1, 0), + WINDOWFUNCNOOP(lead, 1, 0), + WINDOWFUNCNOOP(lead, 2, 0), + WINDOWFUNCNOOP(lead, 3, 0), + WINDOWFUNCNOOP(lag, 1, 0), + WINDOWFUNCNOOP(lag, 2, 0), + WINDOWFUNCNOOP(lag, 3, 0), + }; + sqlite3InsertBuiltinFuncs(aWindowFuncs, ArraySize(aWindowFuncs)); +} + +static Window *windowFind(Parse *pParse, Window *pList, const char *zName){ + Window *p; + for(p=pList; p; p=p->pNextWin){ + if( sqlite3StrICmp(p->zName, zName)==0 ) break; + } + if( p==0 ){ + sqlite3ErrorMsg(pParse, "no such window: %s", zName); + } + return p; +} + +/* +** This function is called immediately after resolving the function name +** for a window function within a SELECT statement. Argument pList is a +** linked list of WINDOW definitions for the current SELECT statement. +** Argument pFunc is the function definition just resolved and pWin +** is the Window object representing the associated OVER clause. This +** function updates the contents of pWin as follows: +** +** * If the OVER clause referred to a named window (as in "max(x) OVER win"), +** search list pList for a matching WINDOW definition, and update pWin +** accordingly. If no such WINDOW clause can be found, leave an error +** in pParse. +** +** * If the function is a built-in window function that requires the +** window to be coerced (see "BUILT-IN WINDOW FUNCTIONS" at the top +** of this file), pWin is updated here. +*/ +SQLITE_PRIVATE void sqlite3WindowUpdate( + Parse *pParse, + Window *pList, /* List of named windows for this SELECT */ + Window *pWin, /* Window frame to update */ + FuncDef *pFunc /* Window function definition */ +){ + if( pWin->zName && pWin->eFrmType==0 ){ + Window *p = windowFind(pParse, pList, pWin->zName); + if( p==0 ) return; + pWin->pPartition = sqlite3ExprListDup(pParse->db, p->pPartition, 0); + pWin->pOrderBy = sqlite3ExprListDup(pParse->db, p->pOrderBy, 0); + pWin->pStart = sqlite3ExprDup(pParse->db, p->pStart, 0); + pWin->pEnd = sqlite3ExprDup(pParse->db, p->pEnd, 0); + pWin->eStart = p->eStart; + pWin->eEnd = p->eEnd; + pWin->eFrmType = p->eFrmType; + pWin->eExclude = p->eExclude; + }else{ + sqlite3WindowChain(pParse, pWin, pList); + } + if( (pWin->eFrmType==TK_RANGE) + && (pWin->pStart || pWin->pEnd) + && (pWin->pOrderBy==0 || pWin->pOrderBy->nExpr!=1) + ){ + sqlite3ErrorMsg(pParse, + "RANGE with offset PRECEDING/FOLLOWING requires one ORDER BY expression" + ); + }else + if( pFunc->funcFlags & SQLITE_FUNC_WINDOW ){ + sqlite3 *db = pParse->db; + if( pWin->pFilter ){ + sqlite3ErrorMsg(pParse, + "FILTER clause may only be used with aggregate window functions" + ); + }else{ + struct WindowUpdate { + const char *zFunc; + int eFrmType; + int eStart; + int eEnd; + } aUp[] = { + { row_numberName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + { dense_rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { rankName, TK_RANGE, TK_UNBOUNDED, TK_CURRENT }, + { percent_rankName, TK_GROUPS, TK_CURRENT, TK_UNBOUNDED }, + { cume_distName, TK_GROUPS, TK_FOLLOWING, TK_UNBOUNDED }, + { ntileName, TK_ROWS, TK_CURRENT, TK_UNBOUNDED }, + { leadName, TK_ROWS, TK_UNBOUNDED, TK_UNBOUNDED }, + { lagName, TK_ROWS, TK_UNBOUNDED, TK_CURRENT }, + }; + int i; + for(i=0; izName==aUp[i].zFunc ){ + sqlite3ExprDelete(db, pWin->pStart); + sqlite3ExprDelete(db, pWin->pEnd); + pWin->pEnd = pWin->pStart = 0; + pWin->eFrmType = aUp[i].eFrmType; + pWin->eStart = aUp[i].eStart; + pWin->eEnd = aUp[i].eEnd; + pWin->eExclude = 0; + if( pWin->eStart==TK_FOLLOWING ){ + pWin->pStart = sqlite3Expr(db, TK_INTEGER, "1"); + } + break; + } + } + } + } + pWin->pWFunc = pFunc; +} + +/* +** Context object passed through sqlite3WalkExprList() to +** selectWindowRewriteExprCb() by selectWindowRewriteEList(). +*/ +typedef struct WindowRewrite WindowRewrite; +struct WindowRewrite { + Window *pWin; + SrcList *pSrc; + ExprList *pSub; + Table *pTab; + Select *pSubSelect; /* Current sub-select, if any */ +}; + +/* +** Callback function used by selectWindowRewriteEList(). If necessary, +** this function appends to the output expression-list and updates +** expression (*ppExpr) in place. +*/ +static int selectWindowRewriteExprCb(Walker *pWalker, Expr *pExpr){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Parse *pParse = pWalker->pParse; + assert( p!=0 ); + assert( p->pWin!=0 ); + + /* If this function is being called from within a scalar sub-select + ** that used by the SELECT statement being processed, only process + ** TK_COLUMN expressions that refer to it (the outer SELECT). Do + ** not process aggregates or window functions at all, as they belong + ** to the scalar sub-select. */ + if( p->pSubSelect ){ + if( pExpr->op!=TK_COLUMN ){ + return WRC_Continue; + }else{ + int nSrc = p->pSrc->nSrc; + int i; + for(i=0; iiTable==p->pSrc->a[i].iCursor ) break; + } + if( i==nSrc ) return WRC_Continue; + } + } + + switch( pExpr->op ){ + + case TK_FUNCTION: + if( !ExprHasProperty(pExpr, EP_WinFunc) ){ + break; + }else{ + Window *pWin; + for(pWin=p->pWin; pWin; pWin=pWin->pNextWin){ + if( pExpr->y.pWin==pWin ){ + assert( pWin->pOwner==pExpr ); + return WRC_Prune; + } + } + } + /* no break */ deliberate_fall_through + + case TK_IF_NULL_ROW: + case TK_AGG_FUNCTION: + case TK_COLUMN: { + int iCol = -1; + if( pParse->db->mallocFailed ) return WRC_Abort; + if( p->pSub ){ + int i; + for(i=0; ipSub->nExpr; i++){ + if( 0==sqlite3ExprCompare(0, p->pSub->a[i].pExpr, pExpr, -1) ){ + iCol = i; + break; + } + } + } + if( iCol<0 ){ + Expr *pDup = sqlite3ExprDup(pParse->db, pExpr, 0); + if( pDup && pDup->op==TK_AGG_FUNCTION ) pDup->op = TK_FUNCTION; + p->pSub = sqlite3ExprListAppend(pParse, p->pSub, pDup); + } + if( p->pSub ){ + int f = pExpr->flags & EP_Collate; + assert( ExprHasProperty(pExpr, EP_Static)==0 ); + ExprSetProperty(pExpr, EP_Static); + sqlite3ExprDelete(pParse->db, pExpr); + ExprClearProperty(pExpr, EP_Static); + memset(pExpr, 0, sizeof(Expr)); + + pExpr->op = TK_COLUMN; + pExpr->iColumn = (iCol<0 ? p->pSub->nExpr-1: iCol); + pExpr->iTable = p->pWin->iEphCsr; + pExpr->y.pTab = p->pTab; + pExpr->flags = f; + } + if( pParse->db->mallocFailed ) return WRC_Abort; + break; + } + + default: /* no-op */ + break; + } + + return WRC_Continue; +} +static int selectWindowRewriteSelectCb(Walker *pWalker, Select *pSelect){ + struct WindowRewrite *p = pWalker->u.pRewrite; + Select *pSave = p->pSubSelect; + if( pSave==pSelect ){ + return WRC_Continue; + }else{ + p->pSubSelect = pSelect; + sqlite3WalkSelect(pWalker, pSelect); + p->pSubSelect = pSave; + } + return WRC_Prune; +} + + +/* +** Iterate through each expression in expression-list pEList. For each: +** +** * TK_COLUMN, +** * aggregate function, or +** * window function with a Window object that is not a member of the +** Window list passed as the second argument (pWin). +** +** Append the node to output expression-list (*ppSub). And replace it +** with a TK_COLUMN that reads the (N-1)th element of table +** pWin->iEphCsr, where N is the number of elements in (*ppSub) after +** appending the new one. +*/ +static void selectWindowRewriteEList( + Parse *pParse, + Window *pWin, + SrcList *pSrc, + ExprList *pEList, /* Rewrite expressions in this list */ + Table *pTab, + ExprList **ppSub /* IN/OUT: Sub-select expression-list */ +){ + Walker sWalker; + WindowRewrite sRewrite; + + assert( pWin!=0 ); + memset(&sWalker, 0, sizeof(Walker)); + memset(&sRewrite, 0, sizeof(WindowRewrite)); + + sRewrite.pSub = *ppSub; + sRewrite.pWin = pWin; + sRewrite.pSrc = pSrc; + sRewrite.pTab = pTab; + + sWalker.pParse = pParse; + sWalker.xExprCallback = selectWindowRewriteExprCb; + sWalker.xSelectCallback = selectWindowRewriteSelectCb; + sWalker.u.pRewrite = &sRewrite; + + (void)sqlite3WalkExprList(&sWalker, pEList); + + *ppSub = sRewrite.pSub; +} + +/* +** Append a copy of each expression in expression-list pAppend to +** expression list pList. Return a pointer to the result list. +*/ +static ExprList *exprListAppendList( + Parse *pParse, /* Parsing context */ + ExprList *pList, /* List to which to append. Might be NULL */ + ExprList *pAppend, /* List of values to append. Might be NULL */ + int bIntToNull +){ + if( pAppend ){ + int i; + int nInit = pList ? pList->nExpr : 0; + for(i=0; inExpr; i++){ + sqlite3 *db = pParse->db; + Expr *pDup = sqlite3ExprDup(db, pAppend->a[i].pExpr, 0); + if( db->mallocFailed ){ + sqlite3ExprDelete(db, pDup); + break; + } + if( bIntToNull ){ + int iDummy; + Expr *pSub; + pSub = sqlite3ExprSkipCollateAndLikely(pDup); + if( sqlite3ExprIsInteger(pSub, &iDummy, 0) ){ + pSub->op = TK_NULL; + pSub->flags &= ~(EP_IntValue|EP_IsTrue|EP_IsFalse); + pSub->u.zToken = 0; + } + } + pList = sqlite3ExprListAppend(pParse, pList, pDup); + if( pList ) pList->a[nInit+i].fg.sortFlags = pAppend->a[i].fg.sortFlags; + } + } + return pList; +} + +/* +** When rewriting a query, if the new subquery in the FROM clause +** contains TK_AGG_FUNCTION nodes that refer to an outer query, +** then we have to increase the Expr->op2 values of those nodes +** due to the extra subquery layer that was added. +** +** See also the incrAggDepth() routine in resolve.c +*/ +static int sqlite3WindowExtraAggFuncDepth(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION + && pExpr->op2>=pWalker->walkerDepth + ){ + pExpr->op2++; + } + return WRC_Continue; +} + +static int disallowAggregatesInOrderByCb(Walker *pWalker, Expr *pExpr){ + if( pExpr->op==TK_AGG_FUNCTION && pExpr->pAggInfo==0 ){ + assert( !ExprHasProperty(pExpr, EP_IntValue) ); + sqlite3ErrorMsg(pWalker->pParse, + "misuse of aggregate: %s()", pExpr->u.zToken); + } + return WRC_Continue; +} + +/* +** If the SELECT statement passed as the second argument does not invoke +** any SQL window functions, this function is a no-op. Otherwise, it +** rewrites the SELECT statement so that window function xStep functions +** are invoked in the correct order as described under "SELECT REWRITING" +** at the top of this file. +*/ +SQLITE_PRIVATE int sqlite3WindowRewrite(Parse *pParse, Select *p){ + int rc = SQLITE_OK; + if( p->pWin + && p->pPrior==0 + && ALWAYS((p->selFlags & SF_WinRewrite)==0) + && ALWAYS(!IN_RENAME_OBJECT) + ){ + Vdbe *v = sqlite3GetVdbe(pParse); + sqlite3 *db = pParse->db; + Select *pSub = 0; /* The subquery */ + SrcList *pSrc = p->pSrc; + Expr *pWhere = p->pWhere; + ExprList *pGroupBy = p->pGroupBy; + Expr *pHaving = p->pHaving; + ExprList *pSort = 0; + + ExprList *pSublist = 0; /* Expression list for sub-query */ + Window *pMWin = p->pWin; /* Main window object */ + Window *pWin; /* Window object iterator */ + Table *pTab; + Walker w; + + u32 selFlags = p->selFlags; + + pTab = sqlite3DbMallocZero(db, sizeof(Table)); + if( pTab==0 ){ + return sqlite3ErrorToParser(db, SQLITE_NOMEM); + } + sqlite3AggInfoPersistWalkerInit(&w, pParse); + sqlite3WalkSelect(&w, p); + if( (p->selFlags & SF_Aggregate)==0 ){ + w.xExprCallback = disallowAggregatesInOrderByCb; + w.xSelectCallback = 0; + sqlite3WalkExprList(&w, p->pOrderBy); + } + + p->pSrc = 0; + p->pWhere = 0; + p->pGroupBy = 0; + p->pHaving = 0; + p->selFlags &= ~SF_Aggregate; + p->selFlags |= SF_WinRewrite; + + /* Create the ORDER BY clause for the sub-select. This is the concatenation + ** of the window PARTITION and ORDER BY clauses. Then, if this makes it + ** redundant, remove the ORDER BY from the parent SELECT. */ + pSort = exprListAppendList(pParse, 0, pMWin->pPartition, 1); + pSort = exprListAppendList(pParse, pSort, pMWin->pOrderBy, 1); + if( pSort && p->pOrderBy && p->pOrderBy->nExpr<=pSort->nExpr ){ + int nSave = pSort->nExpr; + pSort->nExpr = p->pOrderBy->nExpr; + if( sqlite3ExprListCompare(pSort, p->pOrderBy, -1)==0 ){ + sqlite3ExprListDelete(db, p->pOrderBy); + p->pOrderBy = 0; + } + pSort->nExpr = nSave; + } + + /* Assign a cursor number for the ephemeral table used to buffer rows. + ** The OpenEphemeral instruction is coded later, after it is known how + ** many columns the table will have. */ + pMWin->iEphCsr = pParse->nTab++; + pParse->nTab += 3; + + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pEList, pTab, &pSublist); + selectWindowRewriteEList(pParse, pMWin, pSrc, p->pOrderBy, pTab, &pSublist); + pMWin->nBufferCol = (pSublist ? pSublist->nExpr : 0); + + /* Append the PARTITION BY and ORDER BY expressions to the to the + ** sub-select expression list. They are required to figure out where + ** boundaries for partitions and sets of peer rows lie. */ + pSublist = exprListAppendList(pParse, pSublist, pMWin->pPartition, 0); + pSublist = exprListAppendList(pParse, pSublist, pMWin->pOrderBy, 0); + + /* Append the arguments passed to each window function to the + ** sub-select expression list. Also allocate two registers for each + ** window function - one for the accumulator, another for interim + ** results. */ + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + ExprList *pArgs; + assert( ExprUseXList(pWin->pOwner) ); + assert( pWin->pWFunc!=0 ); + pArgs = pWin->pOwner->x.pList; + if( pWin->pWFunc->funcFlags & SQLITE_SUBTYPE ){ + selectWindowRewriteEList(pParse, pMWin, pSrc, pArgs, pTab, &pSublist); + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pWin->bExprArgs = 1; + }else{ + pWin->iArgCol = (pSublist ? pSublist->nExpr : 0); + pSublist = exprListAppendList(pParse, pSublist, pArgs, 0); + } + if( pWin->pFilter ){ + Expr *pFilter = sqlite3ExprDup(db, pWin->pFilter, 0); + pSublist = sqlite3ExprListAppend(pParse, pSublist, pFilter); + } + pWin->regAccum = ++pParse->nMem; + pWin->regResult = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + /* If there is no ORDER BY or PARTITION BY clause, and the window + ** function accepts zero arguments, and there are no other columns + ** selected (e.g. "SELECT row_number() OVER () FROM t1"), it is possible + ** that pSublist is still NULL here. Add a constant expression here to + ** keep everything legal in this case. + */ + if( pSublist==0 ){ + pSublist = sqlite3ExprListAppend(pParse, 0, + sqlite3Expr(db, TK_INTEGER, "0") + ); + } + + pSub = sqlite3SelectNew( + pParse, pSublist, pSrc, pWhere, pGroupBy, pHaving, pSort, 0, 0 + ); + TREETRACE(0x40,pParse,pSub, + ("New window-function subquery in FROM clause of (%u/%p)\n", + p->selId, p)); + p->pSrc = sqlite3SrcListAppend(pParse, 0, 0, 0); + assert( pSub!=0 || p->pSrc==0 ); /* Due to db->mallocFailed test inside + ** of sqlite3DbMallocRawNN() called from + ** sqlite3SrcListAppend() */ + if( p->pSrc==0 ){ + sqlite3SelectDelete(db, pSub); + }else if( sqlite3SrcItemAttachSubquery(pParse, &p->pSrc->a[0], pSub, 0) ){ + Table *pTab2; + p->pSrc->a[0].fg.isCorrelated = 1; + sqlite3SrcListAssignCursors(pParse, p->pSrc); + pSub->selFlags |= SF_Expanded|SF_OrderByReqd; + pTab2 = sqlite3ResultSetOfSelect(pParse, pSub, SQLITE_AFF_NONE); + pSub->selFlags |= (selFlags & SF_Aggregate); + if( pTab2==0 ){ + /* Might actually be some other kind of error, but in that case + ** pParse->nErr will be set, so if SQLITE_NOMEM is set, we will get + ** the correct error message regardless. */ + rc = SQLITE_NOMEM; + }else{ + memcpy(pTab, pTab2, sizeof(Table)); + pTab->tabFlags |= TF_Ephemeral; + p->pSrc->a[0].pSTab = pTab; + pTab = pTab2; + memset(&w, 0, sizeof(w)); + w.xExprCallback = sqlite3WindowExtraAggFuncDepth; + w.xSelectCallback = sqlite3WalkerDepthIncrease; + w.xSelectCallback2 = sqlite3WalkerDepthDecrease; + sqlite3WalkSelect(&w, pSub); + } + } + if( db->mallocFailed ) rc = SQLITE_NOMEM; + + /* Defer deleting the temporary table pTab because if an error occurred, + ** there could still be references to that table embedded in the + ** result-set or ORDER BY clause of the SELECT statement p. */ + sqlite3ParserAddCleanup(pParse, sqlite3DbFree, pTab); + } + + assert( rc==SQLITE_OK || pParse->nErr!=0 ); + return rc; +} + +/* +** Unlink the Window object from the Select to which it is attached, +** if it is attached. +*/ +SQLITE_PRIVATE void sqlite3WindowUnlinkFromSelect(Window *p){ + if( p->ppThis ){ + *p->ppThis = p->pNextWin; + if( p->pNextWin ) p->pNextWin->ppThis = p->ppThis; + p->ppThis = 0; + } +} + +/* +** Free the Window object passed as the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowDelete(sqlite3 *db, Window *p){ + if( p ){ + sqlite3WindowUnlinkFromSelect(p); + sqlite3ExprDelete(db, p->pFilter); + sqlite3ExprListDelete(db, p->pPartition); + sqlite3ExprListDelete(db, p->pOrderBy); + sqlite3ExprDelete(db, p->pEnd); + sqlite3ExprDelete(db, p->pStart); + sqlite3DbFree(db, p->zName); + sqlite3DbFree(db, p->zBase); + sqlite3DbFree(db, p); + } +} + +/* +** Free the linked list of Window objects starting at the second argument. +*/ +SQLITE_PRIVATE void sqlite3WindowListDelete(sqlite3 *db, Window *p){ + while( p ){ + Window *pNext = p->pNextWin; + sqlite3WindowDelete(db, p); + p = pNext; + } +} + +/* +** The argument expression is an PRECEDING or FOLLOWING offset. The +** value should be a non-negative integer. If the value is not a +** constant, change it to NULL. The fact that it is then a non-negative +** integer will be caught later. But it is important not to leave +** variable values in the expression tree. +*/ +static Expr *sqlite3WindowOffsetExpr(Parse *pParse, Expr *pExpr){ + if( 0==sqlite3ExprIsConstant(0,pExpr) ){ + if( IN_RENAME_OBJECT ) sqlite3RenameExprUnmap(pParse, pExpr); + sqlite3ExprDelete(pParse->db, pExpr); + pExpr = sqlite3ExprAlloc(pParse->db, TK_NULL, 0, 0); + } + return pExpr; +} + +/* +** Allocate and return a new Window object describing a Window Definition. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAlloc( + Parse *pParse, /* Parsing context */ + int eType, /* Frame type. TK_RANGE, TK_ROWS, TK_GROUPS, or 0 */ + int eStart, /* Start type: CURRENT, PRECEDING, FOLLOWING, UNBOUNDED */ + Expr *pStart, /* Start window size if TK_PRECEDING or FOLLOWING */ + int eEnd, /* End type: CURRENT, FOLLOWING, TK_UNBOUNDED, PRECEDING */ + Expr *pEnd, /* End window size if TK_FOLLOWING or PRECEDING */ + u8 eExclude /* EXCLUDE clause */ +){ + Window *pWin = 0; + int bImplicitFrame = 0; + + /* Parser assures the following: */ + assert( eType==0 || eType==TK_RANGE || eType==TK_ROWS || eType==TK_GROUPS ); + assert( eStart==TK_CURRENT || eStart==TK_PRECEDING + || eStart==TK_UNBOUNDED || eStart==TK_FOLLOWING ); + assert( eEnd==TK_CURRENT || eEnd==TK_FOLLOWING + || eEnd==TK_UNBOUNDED || eEnd==TK_PRECEDING ); + assert( (eStart==TK_PRECEDING || eStart==TK_FOLLOWING)==(pStart!=0) ); + assert( (eEnd==TK_FOLLOWING || eEnd==TK_PRECEDING)==(pEnd!=0) ); + + if( eType==0 ){ + bImplicitFrame = 1; + eType = TK_RANGE; + } + + /* Additionally, the + ** starting boundary type may not occur earlier in the following list than + ** the ending boundary type: + ** + ** UNBOUNDED PRECEDING + ** PRECEDING + ** CURRENT ROW + ** FOLLOWING + ** UNBOUNDED FOLLOWING + ** + ** The parser ensures that "UNBOUNDED PRECEDING" cannot be used as an ending + ** boundary, and than "UNBOUNDED FOLLOWING" cannot be used as a starting + ** frame boundary. + */ + if( (eStart==TK_CURRENT && eEnd==TK_PRECEDING) + || (eStart==TK_FOLLOWING && (eEnd==TK_PRECEDING || eEnd==TK_CURRENT)) + ){ + sqlite3ErrorMsg(pParse, "unsupported frame specification"); + goto windowAllocErr; + } + + pWin = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( pWin==0 ) goto windowAllocErr; + pWin->eFrmType = eType; + pWin->eStart = eStart; + pWin->eEnd = eEnd; + if( eExclude==0 && OptimizationDisabled(pParse->db, SQLITE_WindowFunc) ){ + eExclude = TK_NO; + } + pWin->eExclude = eExclude; + pWin->bImplicitFrame = bImplicitFrame; + pWin->pEnd = sqlite3WindowOffsetExpr(pParse, pEnd); + pWin->pStart = sqlite3WindowOffsetExpr(pParse, pStart); + return pWin; + +windowAllocErr: + sqlite3ExprDelete(pParse->db, pEnd); + sqlite3ExprDelete(pParse->db, pStart); + return 0; +} + +/* +** Attach PARTITION and ORDER BY clauses pPartition and pOrderBy to window +** pWin. Also, if parameter pBase is not NULL, set pWin->zBase to the +** equivalent nul-terminated string. +*/ +SQLITE_PRIVATE Window *sqlite3WindowAssemble( + Parse *pParse, + Window *pWin, + ExprList *pPartition, + ExprList *pOrderBy, + Token *pBase +){ + if( pWin ){ + pWin->pPartition = pPartition; + pWin->pOrderBy = pOrderBy; + if( pBase ){ + pWin->zBase = sqlite3DbStrNDup(pParse->db, pBase->z, pBase->n); + } + }else{ + sqlite3ExprListDelete(pParse->db, pPartition); + sqlite3ExprListDelete(pParse->db, pOrderBy); + } + return pWin; +} + +/* +** Window *pWin has just been created from a WINDOW clause. Token pBase +** is the base window. Earlier windows from the same WINDOW clause are +** stored in the linked list starting at pWin->pNextWin. This function +** either updates *pWin according to the base specification, or else +** leaves an error in pParse. +*/ +SQLITE_PRIVATE void sqlite3WindowChain(Parse *pParse, Window *pWin, Window *pList){ + if( pWin->zBase ){ + sqlite3 *db = pParse->db; + Window *pExist = windowFind(pParse, pList, pWin->zBase); + if( pExist ){ + const char *zErr = 0; + /* Check for errors */ + if( pWin->pPartition ){ + zErr = "PARTITION clause"; + }else if( pExist->pOrderBy && pWin->pOrderBy ){ + zErr = "ORDER BY clause"; + }else if( pExist->bImplicitFrame==0 ){ + zErr = "frame specification"; + } + if( zErr ){ + sqlite3ErrorMsg(pParse, + "cannot override %s of window: %s", zErr, pWin->zBase + ); + }else{ + pWin->pPartition = sqlite3ExprListDup(db, pExist->pPartition, 0); + if( pExist->pOrderBy ){ + assert( pWin->pOrderBy==0 ); + pWin->pOrderBy = sqlite3ExprListDup(db, pExist->pOrderBy, 0); + } + sqlite3DbFree(db, pWin->zBase); + pWin->zBase = 0; + } + } + } +} + +/* +** Attach window object pWin to expression p. +*/ +SQLITE_PRIVATE void sqlite3WindowAttach(Parse *pParse, Expr *p, Window *pWin){ + if( p ){ + assert( p->op==TK_FUNCTION ); + assert( pWin ); + assert( ExprIsFullSize(p) ); + p->y.pWin = pWin; + ExprSetProperty(p, EP_WinFunc|EP_FullSize); + pWin->pOwner = p; + if( (p->flags & EP_Distinct) && pWin->eFrmType!=TK_FILTER ){ + sqlite3ErrorMsg(pParse, + "DISTINCT is not supported for window functions" + ); + } + }else{ + sqlite3WindowDelete(pParse->db, pWin); + } +} + +/* +** Possibly link window pWin into the list at pSel->pWin (window functions +** to be processed as part of SELECT statement pSel). The window is linked +** in if either (a) there are no other windows already linked to this +** SELECT, or (b) the windows already linked use a compatible window frame. +*/ +SQLITE_PRIVATE void sqlite3WindowLink(Select *pSel, Window *pWin){ + if( pSel ){ + if( 0==pSel->pWin || 0==sqlite3WindowCompare(0, pSel->pWin, pWin, 0) ){ + pWin->pNextWin = pSel->pWin; + if( pSel->pWin ){ + pSel->pWin->ppThis = &pWin->pNextWin; + } + pSel->pWin = pWin; + pWin->ppThis = &pSel->pWin; + }else{ + if( sqlite3ExprListCompare(pWin->pPartition, pSel->pWin->pPartition,-1) ){ + pSel->selFlags |= SF_MultiPart; + } + } + } +} + +/* +** Return 0 if the two window objects are identical, 1 if they are +** different, or 2 if it cannot be determined if the objects are identical +** or not. Identical window objects can be processed in a single scan. +*/ +SQLITE_PRIVATE int sqlite3WindowCompare( + const Parse *pParse, + const Window *p1, + const Window *p2, + int bFilter +){ + int res; + if( NEVER(p1==0) || NEVER(p2==0) ) return 1; + if( p1->eFrmType!=p2->eFrmType ) return 1; + if( p1->eStart!=p2->eStart ) return 1; + if( p1->eEnd!=p2->eEnd ) return 1; + if( p1->eExclude!=p2->eExclude ) return 1; + if( sqlite3ExprCompare(pParse, p1->pStart, p2->pStart, -1) ) return 1; + if( sqlite3ExprCompare(pParse, p1->pEnd, p2->pEnd, -1) ) return 1; + if( (res = sqlite3ExprListCompare(p1->pPartition, p2->pPartition, -1)) ){ + return res; + } + if( (res = sqlite3ExprListCompare(p1->pOrderBy, p2->pOrderBy, -1)) ){ + return res; + } + if( bFilter ){ + if( (res = sqlite3ExprCompare(pParse, p1->pFilter, p2->pFilter, -1)) ){ + return res; + } + } + return 0; +} + + +/* +** This is called by code in select.c before it calls sqlite3WhereBegin() +** to begin iterating through the sub-query results. It is used to allocate +** and initialize registers and cursors used by sqlite3WindowCodeStep(). +*/ +SQLITE_PRIVATE void sqlite3WindowCodeInit(Parse *pParse, Select *pSelect){ + Window *pWin; + int nEphExpr; + Window *pMWin; + Vdbe *v; + + assert( pSelect->pSrc->a[0].fg.isSubquery ); + nEphExpr = pSelect->pSrc->a[0].u4.pSubq->pSelect->pEList->nExpr; + pMWin = pSelect->pWin; + v = sqlite3GetVdbe(pParse); + + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pMWin->iEphCsr, nEphExpr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+1, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+2, pMWin->iEphCsr); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->iEphCsr+3, pMWin->iEphCsr); + + /* Allocate registers to use for PARTITION BY values, if any. Initialize + ** said registers to NULL. */ + if( pMWin->pPartition ){ + int nExpr = pMWin->pPartition->nExpr; + pMWin->regPart = pParse->nMem+1; + pParse->nMem += nExpr; + sqlite3VdbeAddOp3(v, OP_Null, 0, pMWin->regPart, pMWin->regPart+nExpr-1); + } + + pMWin->regOne = ++pParse->nMem; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regOne); + + if( pMWin->eExclude ){ + pMWin->regStartRowid = ++pParse->nMem; + pMWin->regEndRowid = ++pParse->nMem; + pMWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); + sqlite3VdbeAddOp2(v, OP_OpenDup, pMWin->csrApp, pMWin->iEphCsr); + return; + } + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *p = pWin->pWFunc; + if( (p->funcFlags & SQLITE_FUNC_MINMAX) && pWin->eStart!=TK_UNBOUNDED ){ + /* The inline versions of min() and max() require a single ephemeral + ** table and 3 registers. The registers are used as follows: + ** + ** regApp+0: slot to copy min()/max() argument to for MakeRecord + ** regApp+1: integer value used to ensure keys are unique + ** regApp+2: output of MakeRecord + */ + ExprList *pList; + KeyInfo *pKeyInfo; + assert( ExprUseXList(pWin->pOwner) ); + pList = pWin->pOwner->x.pList; + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pList, 0, 0); + pWin->csrApp = pParse->nTab++; + pWin->regApp = pParse->nMem+1; + pParse->nMem += 3; + if( pKeyInfo && pWin->pWFunc->zName[1]=='i' ){ + assert( pKeyInfo->aSortFlags[0]==0 ); + pKeyInfo->aSortFlags[0] = KEYINFO_ORDER_DESC; + } + sqlite3VdbeAddOp2(v, OP_OpenEphemeral, pWin->csrApp, 2); + sqlite3VdbeAppendP4(v, pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + else if( p->zName==nth_valueName || p->zName==first_valueName ){ + /* Allocate two registers at pWin->regApp. These will be used to + ** store the start and end index of the current frame. */ + pWin->regApp = pParse->nMem+1; + pWin->csrApp = pParse->nTab++; + pParse->nMem += 2; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + else if( p->zName==leadName || p->zName==lagName ){ + pWin->csrApp = pParse->nTab++; + sqlite3VdbeAddOp2(v, OP_OpenDup, pWin->csrApp, pMWin->iEphCsr); + } + } +} + +#define WINDOW_STARTING_INT 0 +#define WINDOW_ENDING_INT 1 +#define WINDOW_NTH_VALUE_INT 2 +#define WINDOW_STARTING_NUM 3 +#define WINDOW_ENDING_NUM 4 + +/* +** A "PRECEDING " (eCond==0) or "FOLLOWING " (eCond==1) or the +** value of the second argument to nth_value() (eCond==2) has just been +** evaluated and the result left in register reg. This function generates VM +** code to check that the value is a non-negative integer and throws an +** exception if it is not. +*/ +static void windowCheckValue(Parse *pParse, int reg, int eCond){ + static const char *azErr[] = { + "frame starting offset must be a non-negative integer", + "frame ending offset must be a non-negative integer", + "second argument to nth_value must be a positive integer", + "frame starting offset must be a non-negative number", + "frame ending offset must be a non-negative number", + }; + static int aOp[] = { OP_Ge, OP_Ge, OP_Gt, OP_Ge, OP_Ge }; + Vdbe *v = sqlite3GetVdbe(pParse); + int regZero = sqlite3GetTempReg(pParse); + assert( eCond>=0 && eCond=WINDOW_STARTING_NUM ){ + int regString = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + sqlite3VdbeAddOp3(v, OP_Ge, regString, sqlite3VdbeCurrentAddr(v)+2, reg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC|SQLITE_JUMPIFNULL); + VdbeCoverage(v); + assert( eCond==3 || eCond==4 ); + VdbeCoverageIf(v, eCond==3); + VdbeCoverageIf(v, eCond==4); + }else{ + sqlite3VdbeAddOp2(v, OP_MustBeInt, reg, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + assert( eCond==0 || eCond==1 || eCond==2 ); + VdbeCoverageIf(v, eCond==0); + VdbeCoverageIf(v, eCond==1); + VdbeCoverageIf(v, eCond==2); + } + sqlite3VdbeAddOp3(v, aOp[eCond], regZero, sqlite3VdbeCurrentAddr(v)+2, reg); + sqlite3VdbeChangeP5(v, SQLITE_AFF_NUMERIC); + VdbeCoverageNeverNullIf(v, eCond==0); /* NULL case captured by */ + VdbeCoverageNeverNullIf(v, eCond==1); /* the OP_MustBeInt */ + VdbeCoverageNeverNullIf(v, eCond==2); + VdbeCoverageNeverNullIf(v, eCond==3); /* NULL case caught by */ + VdbeCoverageNeverNullIf(v, eCond==4); /* the OP_Ge */ + sqlite3MayAbort(pParse); + sqlite3VdbeAddOp2(v, OP_Halt, SQLITE_ERROR, OE_Abort); + sqlite3VdbeAppendP4(v, (void*)azErr[eCond], P4_STATIC); + sqlite3ReleaseTempReg(pParse, regZero); +} + +/* +** Return the number of arguments passed to the window-function associated +** with the object passed as the only argument to this function. +*/ +static int windowArgCount(Window *pWin){ + const ExprList *pList; + assert( ExprUseXList(pWin->pOwner) ); + pList = pWin->pOwner->x.pList; + return (pList ? pList->nExpr : 0); +} + +typedef struct WindowCodeArg WindowCodeArg; +typedef struct WindowCsrAndReg WindowCsrAndReg; + +/* +** See comments above struct WindowCodeArg. +*/ +struct WindowCsrAndReg { + int csr; /* Cursor number */ + int reg; /* First in array of peer values */ +}; + +/* +** A single instance of this structure is allocated on the stack by +** sqlite3WindowCodeStep() and a pointer to it passed to the various helper +** routines. This is to reduce the number of arguments required by each +** helper function. +** +** regArg: +** Each window function requires an accumulator register (just as an +** ordinary aggregate function does). This variable is set to the first +** in an array of accumulator registers - one for each window function +** in the WindowCodeArg.pMWin list. +** +** eDelete: +** The window functions implementation sometimes caches the input rows +** that it processes in a temporary table. If it is not zero, this +** variable indicates when rows may be removed from the temp table (in +** order to reduce memory requirements - it would always be safe just +** to leave them there). Possible values for eDelete are: +** +** WINDOW_RETURN_ROW: +** An input row can be discarded after it is returned to the caller. +** +** WINDOW_AGGINVERSE: +** An input row can be discarded after the window functions xInverse() +** callbacks have been invoked in it. +** +** WINDOW_AGGSTEP: +** An input row can be discarded after the window functions xStep() +** callbacks have been invoked in it. +** +** start,current,end +** Consider a window-frame similar to the following: +** +** (ORDER BY a, b GROUPS BETWEEN 2 PRECEDING AND 2 FOLLOWING) +** +** The windows functions implementation caches the input rows in a temp +** table, sorted by "a, b" (it actually populates the cache lazily, and +** aggressively removes rows once they are no longer required, but that's +** a mere detail). It keeps three cursors open on the temp table. One +** (current) that points to the next row to return to the query engine +** once its window function values have been calculated. Another (end) +** points to the next row to call the xStep() method of each window function +** on (so that it is 2 groups ahead of current). And a third (start) that +** points to the next row to call the xInverse() method of each window +** function on. +** +** Each cursor (start, current and end) consists of a VDBE cursor +** (WindowCsrAndReg.csr) and an array of registers (starting at +** WindowCodeArg.reg) that always contains a copy of the peer values +** read from the corresponding cursor. +** +** Depending on the window-frame in question, all three cursors may not +** be required. In this case both WindowCodeArg.csr and reg are set to +** 0. +*/ +struct WindowCodeArg { + Parse *pParse; /* Parse context */ + Window *pMWin; /* First in list of functions being processed */ + Vdbe *pVdbe; /* VDBE object */ + int addrGosub; /* OP_Gosub to this address to return one row */ + int regGosub; /* Register used with OP_Gosub(addrGosub) */ + int regArg; /* First in array of accumulator registers */ + int eDelete; /* See above */ + int regRowid; + + WindowCsrAndReg start; + WindowCsrAndReg current; + WindowCsrAndReg end; +}; + +/* +** Generate VM code to read the window frames peer values from cursor csr into +** an array of registers starting at reg. +*/ +static void windowReadPeerValues( + WindowCodeArg *p, + int csr, + int reg +){ + Window *pMWin = p->pMWin; + ExprList *pOrderBy = pMWin->pOrderBy; + if( pOrderBy ){ + Vdbe *v = sqlite3GetVdbe(p->pParse); + ExprList *pPart = pMWin->pPartition; + int iColOff = pMWin->nBufferCol + (pPart ? pPart->nExpr : 0); + int i; + for(i=0; inExpr; i++){ + sqlite3VdbeAddOp3(v, OP_Column, csr, iColOff+i, reg+i); + } + } +} + +/* +** Generate VM code to invoke either xStep() (if bInverse is 0) or +** xInverse (if bInverse is non-zero) for each window function in the +** linked list starting at pMWin. Or, for built-in window functions +** that do not use the standard function API, generate the required +** inline VM code. +** +** If argument csr is greater than or equal to 0, then argument reg is +** the first register in an array of registers guaranteed to be large +** enough to hold the array of arguments for each function. In this case +** the arguments are extracted from the current row of csr into the +** array of registers before invoking OP_AggStep or OP_AggInverse +** +** Or, if csr is less than zero, then the array of registers at reg is +** already populated with all columns from the current row of the sub-query. +** +** If argument regPartSize is non-zero, then it is a register containing the +** number of rows in the current partition. +*/ +static void windowAggStep( + WindowCodeArg *p, + Window *pMWin, /* Linked list of window functions */ + int csr, /* Read arguments from this cursor */ + int bInverse, /* True to invoke xInverse instead of xStep */ + int reg /* Array of registers */ +){ + Parse *pParse = p->pParse; + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pWFunc; + int regArg; + int nArg = pWin->bExprArgs ? 0 : windowArgCount(pWin); + int i; + + assert( bInverse==0 || pWin->eStart!=TK_UNBOUNDED ); + + /* All OVER clauses in the same window function aggregate step must + ** be the same. */ + assert( pWin==pMWin || sqlite3WindowCompare(pParse,pWin,pMWin,0)!=1 ); + + for(i=0; izName!=nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+i, reg+i); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, pMWin->iEphCsr, pWin->iArgCol+i, reg+i); + } + } + regArg = reg; + + if( pMWin->regStartRowid==0 + && (pFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) + ){ + int addrIsNull = sqlite3VdbeAddOp1(v, OP_IsNull, regArg); + VdbeCoverage(v); + if( bInverse==0 ){ + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1, 1); + sqlite3VdbeAddOp2(v, OP_SCopy, regArg, pWin->regApp); + sqlite3VdbeAddOp3(v, OP_MakeRecord, pWin->regApp, 2, pWin->regApp+2); + sqlite3VdbeAddOp2(v, OP_IdxInsert, pWin->csrApp, pWin->regApp+2); + }else{ + sqlite3VdbeAddOp4Int(v, OP_SeekGE, pWin->csrApp, 0, regArg, 1); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp1(v, OP_Delete, pWin->csrApp); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + } + sqlite3VdbeJumpHere(v, addrIsNull); + }else if( pWin->regApp ){ + assert( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ); + assert( bInverse==0 || bInverse==1 ); + sqlite3VdbeAddOp2(v, OP_AddImm, pWin->regApp+1-bInverse, 1); + }else if( pFunc->xSFunc!=noopStepFunc ){ + int addrIf = 0; + if( pWin->pFilter ){ + int regTmp; + assert( ExprUseXList(pWin->pOwner) ); + assert( pWin->bExprArgs || !nArg ||nArg==pWin->pOwner->x.pList->nExpr ); + assert( pWin->bExprArgs || nArg ||pWin->pOwner->x.pList==0 ); + regTmp = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol+nArg,regTmp); + addrIf = sqlite3VdbeAddOp3(v, OP_IfNot, regTmp, 0, 1); + VdbeCoverage(v); + sqlite3ReleaseTempReg(pParse, regTmp); + } + + if( pWin->bExprArgs ){ + int iOp = sqlite3VdbeCurrentAddr(v); + int iEnd; + + assert( ExprUseXList(pWin->pOwner) ); + nArg = pWin->pOwner->x.pList->nExpr; + regArg = sqlite3GetTempRange(pParse, nArg); + sqlite3ExprCodeExprList(pParse, pWin->pOwner->x.pList, regArg, 0, 0); + + for(iEnd=sqlite3VdbeCurrentAddr(v); iOpopcode==OP_Column && pOp->p1==pMWin->iEphCsr ){ + pOp->p1 = csr; + } + } + } + if( pFunc->funcFlags & SQLITE_FUNC_NEEDCOLL ){ + CollSeq *pColl; + assert( nArg>0 ); + assert( ExprUseXList(pWin->pOwner) ); + pColl = sqlite3ExprNNCollSeq(pParse, pWin->pOwner->x.pList->a[0].pExpr); + sqlite3VdbeAddOp4(v, OP_CollSeq, 0,0,0, (const char*)pColl, P4_COLLSEQ); + } + sqlite3VdbeAddOp3(v, bInverse? OP_AggInverse : OP_AggStep, + bInverse, regArg, pWin->regAccum); + sqlite3VdbeAppendP4(v, pFunc, P4_FUNCDEF); + sqlite3VdbeChangeP5(v, (u8)nArg); + if( pWin->bExprArgs ){ + sqlite3ReleaseTempRange(pParse, regArg, nArg); + } + if( addrIf ) sqlite3VdbeJumpHere(v, addrIf); + } + } +} + +/* +** Values that may be passed as the second argument to windowCodeOp(). +*/ +#define WINDOW_RETURN_ROW 1 +#define WINDOW_AGGINVERSE 2 +#define WINDOW_AGGSTEP 3 + +/* +** Generate VM code to invoke either xValue() (bFin==0) or xFinalize() +** (bFin==1) for each window function in the linked list starting at +** pMWin. Or, for built-in window-functions that do not use the standard +** API, generate the equivalent VM code. +*/ +static void windowAggFinal(WindowCodeArg *p, int bFin){ + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = sqlite3GetVdbe(pParse); + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + if( pMWin->regStartRowid==0 + && (pWin->pWFunc->funcFlags & SQLITE_FUNC_MINMAX) + && (pWin->eStart!=TK_UNBOUNDED) + ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + sqlite3VdbeAddOp1(v, OP_Last, pWin->csrApp); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, pWin->csrApp, 0, pWin->regResult); + sqlite3VdbeJumpHere(v, sqlite3VdbeCurrentAddr(v)-2); + }else if( pWin->regApp ){ + assert( pMWin->regStartRowid==0 ); + }else{ + int nArg = windowArgCount(pWin); + if( bFin ){ + sqlite3VdbeAddOp2(v, OP_AggFinal, pWin->regAccum, nArg); + sqlite3VdbeAppendP4(v, pWin->pWFunc, P4_FUNCDEF); + sqlite3VdbeAddOp2(v, OP_Copy, pWin->regAccum, pWin->regResult); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + }else{ + sqlite3VdbeAddOp3(v, OP_AggValue,pWin->regAccum,nArg,pWin->regResult); + sqlite3VdbeAppendP4(v, pWin->pWFunc, P4_FUNCDEF); + } + } + } +} + +/* +** Generate code to calculate the current values of all window functions in the +** p->pMWin list by doing a full scan of the current window frame. Store the +** results in the Window.regResult registers, ready to return the upper +** layer. +*/ +static void windowFullScan(WindowCodeArg *p){ + Window *pWin; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; + + int regCRowid = 0; /* Current rowid value */ + int regCPeer = 0; /* Current peer values */ + int regRowid = 0; /* AggStep rowid value */ + int regPeer = 0; /* AggStep peer values */ + + int nPeer; + int lblNext; + int lblBrk; + int addrNext; + int csr; + + VdbeModuleComment((v, "windowFullScan begin")); + + assert( pMWin!=0 ); + csr = pMWin->csrApp; + nPeer = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + + lblNext = sqlite3VdbeMakeLabel(pParse); + lblBrk = sqlite3VdbeMakeLabel(pParse); + + regCRowid = sqlite3GetTempReg(pParse); + regRowid = sqlite3GetTempReg(pParse); + if( nPeer ){ + regCPeer = sqlite3GetTempRange(pParse, nPeer); + regPeer = sqlite3GetTempRange(pParse, nPeer); + } + + sqlite3VdbeAddOp2(v, OP_Rowid, pMWin->iEphCsr, regCRowid); + windowReadPeerValues(p, pMWin->iEphCsr, regCPeer); + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + } + + sqlite3VdbeAddOp3(v, OP_SeekGE, csr, lblBrk, pMWin->regStartRowid); + VdbeCoverage(v); + addrNext = sqlite3VdbeCurrentAddr(v); + sqlite3VdbeAddOp2(v, OP_Rowid, csr, regRowid); + sqlite3VdbeAddOp3(v, OP_Gt, pMWin->regEndRowid, lblBrk, regRowid); + VdbeCoverageNeverNull(v); + + if( pMWin->eExclude==TK_CURRENT ){ + sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, lblNext, regRowid); + VdbeCoverageNeverNull(v); + }else if( pMWin->eExclude!=TK_NO ){ + int addr; + int addrEq = 0; + KeyInfo *pKeyInfo = 0; + + if( pMWin->pOrderBy ){ + pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pMWin->pOrderBy, 0, 0); + } + if( pMWin->eExclude==TK_TIES ){ + addrEq = sqlite3VdbeAddOp3(v, OP_Eq, regCRowid, 0, regRowid); + VdbeCoverageNeverNull(v); + } + if( pKeyInfo ){ + windowReadPeerValues(p, csr, regPeer); + sqlite3VdbeAddOp3(v, OP_Compare, regPeer, regCPeer, nPeer); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + addr = sqlite3VdbeCurrentAddr(v)+1; + sqlite3VdbeAddOp3(v, OP_Jump, addr, lblNext, addr); + VdbeCoverageEqNe(v); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblNext); + } + if( addrEq ) sqlite3VdbeJumpHere(v, addrEq); + } + + windowAggStep(p, pMWin, csr, 0, p->regArg); + + sqlite3VdbeResolveLabel(v, lblNext); + sqlite3VdbeAddOp2(v, OP_Next, csr, addrNext); + VdbeCoverage(v); + sqlite3VdbeJumpHere(v, addrNext-1); + sqlite3VdbeJumpHere(v, addrNext+1); + sqlite3ReleaseTempReg(pParse, regRowid); + sqlite3ReleaseTempReg(pParse, regCRowid); + if( nPeer ){ + sqlite3ReleaseTempRange(pParse, regPeer, nPeer); + sqlite3ReleaseTempRange(pParse, regCPeer, nPeer); + } + + windowAggFinal(p, 1); + VdbeModuleComment((v, "windowFullScan end")); +} + +/* +** Invoke the sub-routine at regGosub (generated by code in select.c) to +** return the current row of Window.iEphCsr. If all window functions are +** aggregate window functions that use the standard API, a single +** OP_Gosub instruction is all that this routine generates. Extra VM code +** for per-row processing is only generated for the following built-in window +** functions: +** +** nth_value() +** first_value() +** lag() +** lead() +*/ +static void windowReturnOneRow(WindowCodeArg *p){ + Window *pMWin = p->pMWin; + Vdbe *v = p->pVdbe; + + if( pMWin->regStartRowid ){ + windowFullScan(p); + }else{ + Parse *pParse = p->pParse; + Window *pWin; + + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pWFunc; + assert( ExprUseXList(pWin->pOwner) ); + if( pFunc->zName==nth_valueName + || pFunc->zName==first_valueName + ){ + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + + if( pFunc->zName==nth_valueName ){ + sqlite3VdbeAddOp3(v, OP_Column,pMWin->iEphCsr,pWin->iArgCol+1,tmpReg); + windowCheckValue(pParse, tmpReg, 2); + }else{ + sqlite3VdbeAddOp2(v, OP_Integer, 1, tmpReg); + } + sqlite3VdbeAddOp3(v, OP_Add, tmpReg, pWin->regApp, tmpReg); + sqlite3VdbeAddOp3(v, OP_Gt, pWin->regApp+1, lbl, tmpReg); + VdbeCoverageNeverNull(v); + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, 0, tmpReg); + VdbeCoverageNeverTaken(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + else if( pFunc->zName==leadName || pFunc->zName==lagName ){ + int nArg = pWin->pOwner->x.pList->nExpr; + int csr = pWin->csrApp; + int lbl = sqlite3VdbeMakeLabel(pParse); + int tmpReg = sqlite3GetTempReg(pParse); + int iEph = pMWin->iEphCsr; + + if( nArg<3 ){ + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regResult); + }else{ + sqlite3VdbeAddOp3(v, OP_Column, iEph,pWin->iArgCol+2,pWin->regResult); + } + sqlite3VdbeAddOp2(v, OP_Rowid, iEph, tmpReg); + if( nArg<2 ){ + int val = (pFunc->zName==leadName ? 1 : -1); + sqlite3VdbeAddOp2(v, OP_AddImm, tmpReg, val); + }else{ + int op = (pFunc->zName==leadName ? OP_Add : OP_Subtract); + int tmpReg2 = sqlite3GetTempReg(pParse); + sqlite3VdbeAddOp3(v, OP_Column, iEph, pWin->iArgCol+1, tmpReg2); + sqlite3VdbeAddOp3(v, op, tmpReg2, tmpReg, tmpReg); + sqlite3ReleaseTempReg(pParse, tmpReg2); + } + + sqlite3VdbeAddOp3(v, OP_SeekRowid, csr, lbl, tmpReg); + VdbeCoverage(v); + sqlite3VdbeAddOp3(v, OP_Column, csr, pWin->iArgCol, pWin->regResult); + sqlite3VdbeResolveLabel(v, lbl); + sqlite3ReleaseTempReg(pParse, tmpReg); + } + } + } + sqlite3VdbeAddOp2(v, OP_Gosub, p->regGosub, p->addrGosub); +} + +/* +** Generate code to set the accumulator register for each window function +** in the linked list passed as the second argument to NULL. And perform +** any equivalent initialization required by any built-in window functions +** in the list. +*/ +static int windowInitAccum(Parse *pParse, Window *pMWin){ + Vdbe *v = sqlite3GetVdbe(pParse); + int regArg; + int nArg = 0; + Window *pWin; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pWFunc; + assert( pWin->regAccum ); + sqlite3VdbeAddOp2(v, OP_Null, 0, pWin->regAccum); + nArg = MAX(nArg, windowArgCount(pWin)); + if( pMWin->regStartRowid==0 ){ + if( pFunc->zName==nth_valueName || pFunc->zName==first_valueName ){ + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + + if( (pFunc->funcFlags & SQLITE_FUNC_MINMAX) && pWin->csrApp ){ + assert( pWin->eStart!=TK_UNBOUNDED ); + sqlite3VdbeAddOp1(v, OP_ResetSorter, pWin->csrApp); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pWin->regApp+1); + } + } + } + regArg = pParse->nMem+1; + pParse->nMem += nArg; + return regArg; +} + +/* +** Return true if the current frame should be cached in the ephemeral table, +** even if there are no xInverse() calls required. +*/ +static int windowCacheFrame(Window *pMWin){ + Window *pWin; + if( pMWin->regStartRowid ) return 1; + for(pWin=pMWin; pWin; pWin=pWin->pNextWin){ + FuncDef *pFunc = pWin->pWFunc; + if( (pFunc->zName==nth_valueName) + || (pFunc->zName==first_valueName) + || (pFunc->zName==leadName) + || (pFunc->zName==lagName) + ){ + return 1; + } + } + return 0; +} + +/* +** regOld and regNew are each the first register in an array of size +** pOrderBy->nExpr. This function generates code to compare the two +** arrays of registers using the collation sequences and other comparison +** parameters specified by pOrderBy. +** +** If the two arrays are not equal, the contents of regNew is copied to +** regOld and control falls through. Otherwise, if the contents of the arrays +** are equal, an OP_Goto is executed. The address of the OP_Goto is returned. +*/ +static void windowIfNewPeer( + Parse *pParse, + ExprList *pOrderBy, + int regNew, /* First in array of new values */ + int regOld, /* First in array of old values */ + int addr /* Jump here */ +){ + Vdbe *v = sqlite3GetVdbe(pParse); + if( pOrderBy ){ + int nVal = pOrderBy->nExpr; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pOrderBy, 0, 0); + sqlite3VdbeAddOp3(v, OP_Compare, regOld, regNew, nVal); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, + sqlite3VdbeCurrentAddr(v)+1, addr, sqlite3VdbeCurrentAddr(v)+1 + ); + VdbeCoverageEqNe(v); + sqlite3VdbeAddOp3(v, OP_Copy, regNew, regOld, nVal-1); + }else{ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + } +} + +/* +** This function is called as part of generating VM programs for RANGE +** offset PRECEDING/FOLLOWING frame boundaries. Assuming "ASC" order for +** the ORDER BY term in the window, and that argument op is OP_Ge, it generates +** code equivalent to: +** +** if( csr1.peerVal + regVal >= csr2.peerVal ) goto lbl; +** +** The value of parameter op may also be OP_Gt or OP_Le. In these cases the +** operator in the above pseudo-code is replaced with ">" or "<=", respectively. +** +** If the sort-order for the ORDER BY term in the window is DESC, then the +** comparison is reversed. Instead of adding regVal to csr1.peerVal, it is +** subtracted. And the comparison operator is inverted to - ">=" becomes "<=", +** ">" becomes "<", and so on. So, with DESC sort order, if the argument op +** is OP_Ge, the generated code is equivalent to: +** +** if( csr1.peerVal - regVal <= csr2.peerVal ) goto lbl; +** +** A special type of arithmetic is used such that if csr1.peerVal is not +** a numeric type (real or integer), then the result of the addition +** or subtraction is a a copy of csr1.peerVal. +*/ +static void windowCodeRangeTest( + WindowCodeArg *p, + int op, /* OP_Ge, OP_Gt, or OP_Le */ + int csr1, /* Cursor number for cursor 1 */ + int regVal, /* Register containing non-negative number */ + int csr2, /* Cursor number for cursor 2 */ + int lbl /* Jump destination if condition is true */ +){ + Parse *pParse = p->pParse; + Vdbe *v = sqlite3GetVdbe(pParse); + ExprList *pOrderBy = p->pMWin->pOrderBy; /* ORDER BY clause for window */ + int reg1 = sqlite3GetTempReg(pParse); /* Reg. for csr1.peerVal+regVal */ + int reg2 = sqlite3GetTempReg(pParse); /* Reg. for csr2.peerVal */ + int regString = ++pParse->nMem; /* Reg. for constant value '' */ + int arith = OP_Add; /* OP_Add or OP_Subtract */ + int addrGe; /* Jump destination */ + int addrDone = sqlite3VdbeMakeLabel(pParse); /* Address past OP_Ge */ + CollSeq *pColl; + + /* Read the peer-value from each cursor into a register */ + windowReadPeerValues(p, csr1, reg1); + windowReadPeerValues(p, csr2, reg2); + + assert( op==OP_Ge || op==OP_Gt || op==OP_Le ); + assert( pOrderBy && pOrderBy->nExpr==1 ); + if( pOrderBy->a[0].fg.sortFlags & KEYINFO_ORDER_DESC ){ + switch( op ){ + case OP_Ge: op = OP_Le; break; + case OP_Gt: op = OP_Lt; break; + default: assert( op==OP_Le ); op = OP_Ge; break; + } + arith = OP_Subtract; + } + + VdbeModuleComment((v, "CodeRangeTest: if( R%d %s R%d %s R%d ) goto lbl", + reg1, (arith==OP_Add ? "+" : "-"), regVal, + ((op==OP_Ge) ? ">=" : (op==OP_Le) ? "<=" : (op==OP_Gt) ? ">" : "<"), reg2 + )); + + /* If the BIGNULL flag is set for the ORDER BY, then it is required to + ** consider NULL values to be larger than all other values, instead of + ** the usual smaller. The VDBE opcodes OP_Ge and so on do not handle this + ** (and adding that capability causes a performance regression), so + ** instead if the BIGNULL flag is set then cases where either reg1 or + ** reg2 are NULL are handled separately in the following block. The code + ** generated is equivalent to: + ** + ** if( reg1 IS NULL ){ + ** if( op==OP_Ge ) goto lbl; + ** if( op==OP_Gt && reg2 IS NOT NULL ) goto lbl; + ** if( op==OP_Le && reg2 IS NULL ) goto lbl; + ** }else if( reg2 IS NULL ){ + ** if( op==OP_Le ) goto lbl; + ** } + ** + ** Additionally, if either reg1 or reg2 are NULL but the jump to lbl is + ** not taken, control jumps over the comparison operator coded below this + ** block. */ + if( pOrderBy->a[0].fg.sortFlags & KEYINFO_ORDER_BIGNULL ){ + /* This block runs if reg1 contains a NULL. */ + int addr = sqlite3VdbeAddOp1(v, OP_NotNull, reg1); VdbeCoverage(v); + switch( op ){ + case OP_Ge: + sqlite3VdbeAddOp2(v, OP_Goto, 0, lbl); + break; + case OP_Gt: + sqlite3VdbeAddOp2(v, OP_NotNull, reg2, lbl); + VdbeCoverage(v); + break; + case OP_Le: + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, lbl); + VdbeCoverage(v); + break; + default: assert( op==OP_Lt ); /* no-op */ break; + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrDone); + + /* This block runs if reg1 is not NULL, but reg2 is. */ + sqlite3VdbeJumpHere(v, addr); + sqlite3VdbeAddOp2(v, OP_IsNull, reg2, + (op==OP_Gt || op==OP_Ge) ? addrDone : lbl); + VdbeCoverage(v); + } + + /* Register reg1 currently contains csr1.peerVal (the peer-value from csr1). + ** This block adds (or subtracts for DESC) the numeric value in regVal + ** from it. Or, if reg1 is not numeric (it is a NULL, a text value or a blob), + ** then leave reg1 as it is. In pseudo-code, this is implemented as: + ** + ** if( reg1>='' ) goto addrGe; + ** reg1 = reg1 +/- regVal + ** addrGe: + ** + ** Since all strings and blobs are greater-than-or-equal-to an empty string, + ** the add/subtract is skipped for these, as required. If reg1 is a NULL, + ** then the arithmetic is performed, but since adding or subtracting from + ** NULL is always NULL anyway, this case is handled as required too. */ + sqlite3VdbeAddOp4(v, OP_String8, 0, regString, 0, "", P4_STATIC); + addrGe = sqlite3VdbeAddOp3(v, OP_Ge, regString, 0, reg1); + VdbeCoverage(v); + if( (op==OP_Ge && arith==OP_Add) || (op==OP_Le && arith==OP_Subtract) ){ + sqlite3VdbeAddOp3(v, op, reg2, lbl, reg1); VdbeCoverage(v); + } + sqlite3VdbeAddOp3(v, arith, regVal, reg1, reg1); + sqlite3VdbeJumpHere(v, addrGe); + + /* Compare registers reg2 and reg1, taking the jump if required. Note that + ** control skips over this test if the BIGNULL flag is set and either + ** reg1 or reg2 contain a NULL value. */ + sqlite3VdbeAddOp3(v, op, reg2, lbl, reg1); VdbeCoverage(v); + pColl = sqlite3ExprNNCollSeq(pParse, pOrderBy->a[0].pExpr); + sqlite3VdbeAppendP4(v, (void*)pColl, P4_COLLSEQ); + sqlite3VdbeChangeP5(v, SQLITE_NULLEQ); + sqlite3VdbeResolveLabel(v, addrDone); + + assert( op==OP_Ge || op==OP_Gt || op==OP_Lt || op==OP_Le ); + testcase(op==OP_Ge); VdbeCoverageIf(v, op==OP_Ge); + testcase(op==OP_Lt); VdbeCoverageIf(v, op==OP_Lt); + testcase(op==OP_Le); VdbeCoverageIf(v, op==OP_Le); + testcase(op==OP_Gt); VdbeCoverageIf(v, op==OP_Gt); + sqlite3ReleaseTempReg(pParse, reg1); + sqlite3ReleaseTempReg(pParse, reg2); + + VdbeModuleComment((v, "CodeRangeTest: end")); +} + +/* +** Helper function for sqlite3WindowCodeStep(). Each call to this function +** generates VM code for a single RETURN_ROW, AGGSTEP or AGGINVERSE +** operation. Refer to the header comment for sqlite3WindowCodeStep() for +** details. +*/ +static int windowCodeOp( + WindowCodeArg *p, /* Context object */ + int op, /* WINDOW_RETURN_ROW, AGGSTEP or AGGINVERSE */ + int regCountdown, /* Register for OP_IfPos countdown */ + int jumpOnEof /* Jump here if stepped cursor reaches EOF */ +){ + int csr, reg; + Parse *pParse = p->pParse; + Window *pMWin = p->pMWin; + int ret = 0; + Vdbe *v = p->pVdbe; + int addrContinue = 0; + int bPeer = (pMWin->eFrmType!=TK_ROWS); + + int lblDone = sqlite3VdbeMakeLabel(pParse); + int addrNextRange = 0; + + /* Special case - WINDOW_AGGINVERSE is always a no-op if the frame + ** starts with UNBOUNDED PRECEDING. */ + if( op==WINDOW_AGGINVERSE && pMWin->eStart==TK_UNBOUNDED ){ + assert( regCountdown==0 && jumpOnEof==0 ); + return 0; + } + + if( regCountdown>0 ){ + if( pMWin->eFrmType==TK_RANGE ){ + addrNextRange = sqlite3VdbeCurrentAddr(v); + assert( op==WINDOW_AGGINVERSE || op==WINDOW_AGGSTEP ); + if( op==WINDOW_AGGINVERSE ){ + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeRangeTest( + p, OP_Le, p->current.csr, regCountdown, p->start.csr, lblDone + ); + }else{ + windowCodeRangeTest( + p, OP_Ge, p->start.csr, regCountdown, p->current.csr, lblDone + ); + } + }else{ + windowCodeRangeTest( + p, OP_Gt, p->end.csr, regCountdown, p->current.csr, lblDone + ); + } + }else{ + sqlite3VdbeAddOp3(v, OP_IfPos, regCountdown, lblDone, 1); + VdbeCoverage(v); + } + } + + if( op==WINDOW_RETURN_ROW && pMWin->regStartRowid==0 ){ + windowAggFinal(p, 0); + } + addrContinue = sqlite3VdbeCurrentAddr(v); + + /* If this is a (RANGE BETWEEN a FOLLOWING AND b FOLLOWING) or + ** (RANGE BETWEEN b PRECEDING AND a PRECEDING) frame, ensure the + ** start cursor does not advance past the end cursor within the + ** temporary table. It otherwise might, if (a>b). Also ensure that, + ** if the input cursor is still finding new rows, that the end + ** cursor does not go past it to EOF. */ + if( pMWin->eStart==pMWin->eEnd && regCountdown + && pMWin->eFrmType==TK_RANGE + ){ + int regRowid1 = sqlite3GetTempReg(pParse); + int regRowid2 = sqlite3GetTempReg(pParse); + if( op==WINDOW_AGGINVERSE ){ + sqlite3VdbeAddOp2(v, OP_Rowid, p->start.csr, regRowid1); + sqlite3VdbeAddOp2(v, OP_Rowid, p->end.csr, regRowid2); + sqlite3VdbeAddOp3(v, OP_Ge, regRowid2, lblDone, regRowid1); + VdbeCoverage(v); + }else if( p->regRowid ){ + sqlite3VdbeAddOp2(v, OP_Rowid, p->end.csr, regRowid1); + sqlite3VdbeAddOp3(v, OP_Ge, p->regRowid, lblDone, regRowid1); + VdbeCoverageNeverNull(v); + } + sqlite3ReleaseTempReg(pParse, regRowid1); + sqlite3ReleaseTempReg(pParse, regRowid2); + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ); + } + + switch( op ){ + case WINDOW_RETURN_ROW: + csr = p->current.csr; + reg = p->current.reg; + windowReturnOneRow(p); + break; + + case WINDOW_AGGINVERSE: + csr = p->start.csr; + reg = p->start.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regStartRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 1, p->regArg); + } + break; + + default: + assert( op==WINDOW_AGGSTEP ); + csr = p->end.csr; + reg = p->end.reg; + if( pMWin->regStartRowid ){ + assert( pMWin->regEndRowid ); + sqlite3VdbeAddOp2(v, OP_AddImm, pMWin->regEndRowid, 1); + }else{ + windowAggStep(p, pMWin, csr, 0, p->regArg); + } + break; + } + + if( op==p->eDelete ){ + sqlite3VdbeAddOp1(v, OP_Delete, csr); + sqlite3VdbeChangeP5(v, OPFLAG_SAVEPOSITION); + } + + if( jumpOnEof ){ + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+2); + VdbeCoverage(v); + ret = sqlite3VdbeAddOp0(v, OP_Goto); + }else{ + sqlite3VdbeAddOp2(v, OP_Next, csr, sqlite3VdbeCurrentAddr(v)+1+bPeer); + VdbeCoverage(v); + if( bPeer ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblDone); + } + } + + if( bPeer ){ + int nReg = (pMWin->pOrderBy ? pMWin->pOrderBy->nExpr : 0); + int regTmp = (nReg ? sqlite3GetTempRange(pParse, nReg) : 0); + windowReadPeerValues(p, csr, regTmp); + windowIfNewPeer(pParse, pMWin->pOrderBy, regTmp, reg, addrContinue); + sqlite3ReleaseTempRange(pParse, regTmp, nReg); + } + + if( addrNextRange ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNextRange); + } + sqlite3VdbeResolveLabel(v, lblDone); + return ret; +} + + +/* +** Allocate and return a duplicate of the Window object indicated by the +** third argument. Set the Window.pOwner field of the new object to +** pOwner. +*/ +SQLITE_PRIVATE Window *sqlite3WindowDup(sqlite3 *db, Expr *pOwner, Window *p){ + Window *pNew = 0; + if( ALWAYS(p) ){ + pNew = sqlite3DbMallocZero(db, sizeof(Window)); + if( pNew ){ + pNew->zName = sqlite3DbStrDup(db, p->zName); + pNew->zBase = sqlite3DbStrDup(db, p->zBase); + pNew->pFilter = sqlite3ExprDup(db, p->pFilter, 0); + pNew->pWFunc = p->pWFunc; + pNew->pPartition = sqlite3ExprListDup(db, p->pPartition, 0); + pNew->pOrderBy = sqlite3ExprListDup(db, p->pOrderBy, 0); + pNew->eFrmType = p->eFrmType; + pNew->eEnd = p->eEnd; + pNew->eStart = p->eStart; + pNew->eExclude = p->eExclude; + pNew->regResult = p->regResult; + pNew->regAccum = p->regAccum; + pNew->iArgCol = p->iArgCol; + pNew->iEphCsr = p->iEphCsr; + pNew->bExprArgs = p->bExprArgs; + pNew->pStart = sqlite3ExprDup(db, p->pStart, 0); + pNew->pEnd = sqlite3ExprDup(db, p->pEnd, 0); + pNew->pOwner = pOwner; + pNew->bImplicitFrame = p->bImplicitFrame; + } + } + return pNew; +} + +/* +** Return a copy of the linked list of Window objects passed as the +** second argument. +*/ +SQLITE_PRIVATE Window *sqlite3WindowListDup(sqlite3 *db, Window *p){ + Window *pWin; + Window *pRet = 0; + Window **pp = &pRet; + + for(pWin=p; pWin; pWin=pWin->pNextWin){ + *pp = sqlite3WindowDup(db, 0, pWin); + if( *pp==0 ) break; + pp = &((*pp)->pNextWin); + } + + return pRet; +} + +/* +** Return true if it can be determined at compile time that expression +** pExpr evaluates to a value that, when cast to an integer, is greater +** than zero. False otherwise. +** +** If an OOM error occurs, this function sets the Parse.db.mallocFailed +** flag and returns zero. +*/ +static int windowExprGtZero(Parse *pParse, Expr *pExpr){ + int ret = 0; + sqlite3 *db = pParse->db; + sqlite3_value *pVal = 0; + sqlite3ValueFromExpr(db, pExpr, db->enc, SQLITE_AFF_NUMERIC, &pVal); + if( pVal && sqlite3_value_int(pVal)>0 ){ + ret = 1; + } + sqlite3ValueFree(pVal); + return ret; +} + +/* +** sqlite3WhereBegin() has already been called for the SELECT statement +** passed as the second argument when this function is invoked. It generates +** code to populate the Window.regResult register for each window function +** and invoke the sub-routine at instruction addrGosub once for each row. +** sqlite3WhereEnd() is always called before returning. +** +** This function handles several different types of window frames, which +** require slightly different processing. The following pseudo code is +** used to implement window frames of the form: +** +** ROWS BETWEEN PRECEDING AND FOLLOWING +** +** Other window frame types use variants of the following: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** +** if( first row of partition ){ +** // Rewind three cursors, all open on the eph table. +** Rewind(csrEnd); +** Rewind(csrStart); +** Rewind(csrCurrent); +** +** regEnd = // FOLLOWING expression +** regStart = // PRECEDING expression +** }else{ +** // First time this branch is taken, the eph table contains two +** // rows. The first row in the partition, which all three cursors +** // currently point to, and the following row. +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** if( (regStart--)<=0 ){ +** AggInverse(csrStart) +** Next(csrStart) +** } +** } +** +** The pseudo-code above uses the following shorthand: +** +** AGGSTEP: invoke the aggregate xStep() function for each window function +** with arguments read from the current row of cursor csrEnd, then +** step cursor csrEnd forward one row (i.e. sqlite3BtreeNext()). +** +** RETURN_ROW: return a row to the caller based on the contents of the +** current row of csrCurrent and the current state of all +** aggregates. Then step cursor csrCurrent forward one row. +** +** AGGINVERSE: invoke the aggregate xInverse() function for each window +** functions with arguments read from the current row of cursor +** csrStart. Then step csrStart forward one row. +** +** There are two other ROWS window frames that are handled significantly +** differently from the above - "BETWEEN PRECEDING AND PRECEDING" +** and "BETWEEN FOLLOWING AND FOLLOWING". These are special +** cases because they change the order in which the three cursors (csrStart, +** csrCurrent and csrEnd) iterate through the ephemeral table. Cases that +** use UNBOUNDED or CURRENT ROW are much simpler variations on one of these +** three. +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** if( (regEnd--)<=0 ){ +** AGGSTEP +** } +** RETURN_ROW +** +** +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = regEnd - +** }else{ +** AGGSTEP +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regEnd--)<=0 ){ +** RETURN_ROW +** if( eof ) break; +** } +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** For the most part, the patterns above are adapted to support UNBOUNDED by +** assuming that it is equivalent to "infinity PRECEDING/FOLLOWING" and +** CURRENT ROW by assuming that it is equivalent to "0 PRECEDING/FOLLOWING". +** This is optimized of course - branches that will never be taken and +** conditions that are always true are omitted from the VM code. The only +** exceptional case is: +** +** ROWS BETWEEN FOLLOWING AND UNBOUNDED FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regStart = +** }else{ +** AGGSTEP +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** if( (regStart--)<=0 ){ +** AGGINVERSE +** if( eof ) break +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** Also requiring special handling are the cases: +** +** ROWS BETWEEN PRECEDING AND PRECEDING +** ROWS BETWEEN FOLLOWING AND FOLLOWING +** +** when (expr1 < expr2). This is detected at runtime, not by this function. +** To handle this case, the pseudo-code programs depicted above are modified +** slightly to be: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** if( regEnd < regStart ){ +** RETURN_ROW +** delete eph table contents +** continue +** } +** ... +** +** The new "continue" statement in the above jumps to the next iteration +** of the outer loop - the one started by sqlite3WhereBegin(). +** +** The various GROUPS cases are implemented using the same patterns as +** ROWS. The VM code is modified slightly so that: +** +** 1. The else branch in the main loop is only taken if the row just +** added to the ephemeral table is the start of a new group. In +** other words, it becomes: +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else if( new group ){ +** ... +** } +** } +** +** 2. Instead of processing a single row, each RETURN_ROW, AGGSTEP or +** AGGINVERSE step processes the current row of the relevant cursor and +** all subsequent rows belonging to the same group. +** +** RANGE window frames are a little different again. As for GROUPS, the +** main loop runs once per group only. And RETURN_ROW, AGGSTEP and AGGINVERSE +** deal in groups instead of rows. As for ROWS and GROUPS, there are three +** basic cases: +** +** RANGE BETWEEN PRECEDING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** RETURN_ROW +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** RETURN ROW +** if( csrCurrent is EOF ) break; +** while( csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** } +** } +** +** In the above notation, "csr.key" means the current value of the ORDER BY +** expression (there is only ever 1 for a RANGE that uses an FOLLOWING +** or PRECEDING AND PRECEDING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** flush: +** while( (csrEnd.key + regEnd) <= csrCurrent.key ){ +** AGGSTEP +** } +** while( (csrStart.key + regStart) < csrCurrent.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** +** RANGE BETWEEN FOLLOWING AND FOLLOWING +** +** ... loop started by sqlite3WhereBegin() ... +** if( new partition ){ +** Gosub flush +** } +** Insert new row into eph table. +** if( first row of partition ){ +** Rewind(csrEnd) ; Rewind(csrStart) ; Rewind(csrCurrent) +** regEnd = +** regStart = +** }else{ +** AGGSTEP +** while( (csrCurrent.key + regEnd) < csrEnd.key ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** } +** RETURN_ROW +** } +** } +** } +** flush: +** AGGSTEP +** while( 1 ){ +** while( (csrCurrent.key + regStart) > csrStart.key ){ +** AGGINVERSE +** if( eof ) break "while( 1 )" loop. +** } +** RETURN_ROW +** } +** while( !eof csrCurrent ){ +** RETURN_ROW +** } +** +** The text above leaves out many details. Refer to the code and comments +** below for a more complete picture. +*/ +SQLITE_PRIVATE void sqlite3WindowCodeStep( + Parse *pParse, /* Parse context */ + Select *p, /* Rewritten SELECT statement */ + WhereInfo *pWInfo, /* Context returned by sqlite3WhereBegin() */ + int regGosub, /* Register for OP_Gosub */ + int addrGosub /* OP_Gosub here to return each row */ +){ + Window *pMWin = p->pWin; + ExprList *pOrderBy = pMWin->pOrderBy; + Vdbe *v = sqlite3GetVdbe(pParse); + int csrWrite; /* Cursor used to write to eph. table */ + int csrInput = p->pSrc->a[0].iCursor; /* Cursor of sub-select */ + int nInput = p->pSrc->a[0].pSTab->nCol; /* Number of cols returned by sub */ + int iInput; /* To iterate through sub cols */ + int addrNe; /* Address of OP_Ne */ + int addrGosubFlush = 0; /* Address of OP_Gosub to flush: */ + int addrInteger = 0; /* Address of OP_Integer */ + int addrEmpty; /* Address of OP_Rewind in flush: */ + int regNew; /* Array of registers holding new input row */ + int regRecord; /* regNew array in record form */ + int regNewPeer = 0; /* Peer values for new row (part of regNew) */ + int regPeer = 0; /* Peer values for current row */ + int regFlushPart = 0; /* Register for "Gosub flush_partition" */ + WindowCodeArg s; /* Context object for sub-routines */ + int lblWhereEnd; /* Label just before sqlite3WhereEnd() code */ + int regStart = 0; /* Value of PRECEDING */ + int regEnd = 0; /* Value of FOLLOWING */ + + assert( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_CURRENT + || pMWin->eStart==TK_FOLLOWING || pMWin->eStart==TK_UNBOUNDED + ); + assert( pMWin->eEnd==TK_FOLLOWING || pMWin->eEnd==TK_CURRENT + || pMWin->eEnd==TK_UNBOUNDED || pMWin->eEnd==TK_PRECEDING + ); + assert( pMWin->eExclude==0 || pMWin->eExclude==TK_CURRENT + || pMWin->eExclude==TK_GROUP || pMWin->eExclude==TK_TIES + || pMWin->eExclude==TK_NO + ); + + lblWhereEnd = sqlite3VdbeMakeLabel(pParse); + + /* Fill in the context object */ + memset(&s, 0, sizeof(WindowCodeArg)); + s.pParse = pParse; + s.pMWin = pMWin; + s.pVdbe = v; + s.regGosub = regGosub; + s.addrGosub = addrGosub; + s.current.csr = pMWin->iEphCsr; + csrWrite = s.current.csr+1; + s.start.csr = s.current.csr+2; + s.end.csr = s.current.csr+3; + + /* Figure out when rows may be deleted from the ephemeral table. There + ** are four options - they may never be deleted (eDelete==0), they may + ** be deleted as soon as they are no longer part of the window frame + ** (eDelete==WINDOW_AGGINVERSE), they may be deleted as after the row + ** has been returned to the caller (WINDOW_RETURN_ROW), or they may + ** be deleted after they enter the frame (WINDOW_AGGSTEP). */ + switch( pMWin->eStart ){ + case TK_FOLLOWING: + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pStart) + ){ + s.eDelete = WINDOW_RETURN_ROW; + } + break; + case TK_UNBOUNDED: + if( windowCacheFrame(pMWin)==0 ){ + if( pMWin->eEnd==TK_PRECEDING ){ + if( pMWin->eFrmType!=TK_RANGE + && windowExprGtZero(pParse, pMWin->pEnd) + ){ + s.eDelete = WINDOW_AGGSTEP; + } + }else{ + s.eDelete = WINDOW_RETURN_ROW; + } + } + break; + default: + s.eDelete = WINDOW_AGGINVERSE; + break; + } + + /* Allocate registers for the array of values from the sub-query, the + ** same values in record form, and the rowid used to insert said record + ** into the ephemeral table. */ + regNew = pParse->nMem+1; + pParse->nMem += nInput; + regRecord = ++pParse->nMem; + s.regRowid = ++pParse->nMem; + + /* If the window frame contains an " PRECEDING" or " FOLLOWING" + ** clause, allocate registers to store the results of evaluating each + ** . */ + if( pMWin->eStart==TK_PRECEDING || pMWin->eStart==TK_FOLLOWING ){ + regStart = ++pParse->nMem; + } + if( pMWin->eEnd==TK_PRECEDING || pMWin->eEnd==TK_FOLLOWING ){ + regEnd = ++pParse->nMem; + } + + /* If this is not a "ROWS BETWEEN ..." frame, then allocate arrays of + ** registers to store copies of the ORDER BY expressions (peer values) + ** for the main loop, and for each cursor (start, current and end). */ + if( pMWin->eFrmType!=TK_ROWS ){ + int nPeer = (pOrderBy ? pOrderBy->nExpr : 0); + regNewPeer = regNew + pMWin->nBufferCol; + if( pMWin->pPartition ) regNewPeer += pMWin->pPartition->nExpr; + regPeer = pParse->nMem+1; pParse->nMem += nPeer; + s.start.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.current.reg = pParse->nMem+1; pParse->nMem += nPeer; + s.end.reg = pParse->nMem+1; pParse->nMem += nPeer; + } + + /* Load the column values for the row returned by the sub-select + ** into an array of registers starting at regNew. Assemble them into + ** a record in register regRecord. */ + for(iInput=0; iInputpPartition ){ + int addr; + ExprList *pPart = pMWin->pPartition; + int nPart = pPart->nExpr; + int regNewPart = regNew + pMWin->nBufferCol; + KeyInfo *pKeyInfo = sqlite3KeyInfoFromExprList(pParse, pPart, 0, 0); + + regFlushPart = ++pParse->nMem; + addr = sqlite3VdbeAddOp3(v, OP_Compare, regNewPart, pMWin->regPart, nPart); + sqlite3VdbeAppendP4(v, (void*)pKeyInfo, P4_KEYINFO); + sqlite3VdbeAddOp3(v, OP_Jump, addr+2, addr+4, addr+2); + VdbeCoverageEqNe(v); + addrGosubFlush = sqlite3VdbeAddOp1(v, OP_Gosub, regFlushPart); + VdbeComment((v, "call flush_partition")); + sqlite3VdbeAddOp3(v, OP_Copy, regNewPart, pMWin->regPart, nPart-1); + } + + /* Insert the new row into the ephemeral table */ + sqlite3VdbeAddOp2(v, OP_NewRowid, csrWrite, s.regRowid); + sqlite3VdbeAddOp3(v, OP_Insert, csrWrite, regRecord, s.regRowid); + addrNe = sqlite3VdbeAddOp3(v, OP_Ne, pMWin->regOne, 0, s.regRowid); + VdbeCoverageNeverNull(v); + + /* This block is run for the first row of each partition */ + s.regArg = windowInitAccum(pParse, pMWin); + + if( regStart ){ + sqlite3ExprCode(pParse, pMWin->pStart, regStart); + windowCheckValue(pParse, regStart, 0 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + if( regEnd ){ + sqlite3ExprCode(pParse, pMWin->pEnd, regEnd); + windowCheckValue(pParse, regEnd, 1 + (pMWin->eFrmType==TK_RANGE?3:0)); + } + + if( pMWin->eFrmType!=TK_RANGE && pMWin->eStart==pMWin->eEnd && regStart ){ + int op = ((pMWin->eStart==TK_FOLLOWING) ? OP_Ge : OP_Le); + int addrGe = sqlite3VdbeAddOp3(v, op, regStart, 0, regEnd); + VdbeCoverageNeverNullIf(v, op==OP_Ge); /* NeverNull because bound */ + VdbeCoverageNeverNullIf(v, op==OP_Le); /* values previously checked */ + windowAggFinal(&s, 0); + sqlite3VdbeAddOp1(v, OP_Rewind, s.current.csr); + windowReturnOneRow(&s); + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + sqlite3VdbeJumpHere(v, addrGe); + } + if( pMWin->eStart==TK_FOLLOWING && pMWin->eFrmType!=TK_RANGE && regEnd ){ + assert( pMWin->eEnd==TK_FOLLOWING ); + sqlite3VdbeAddOp3(v, OP_Subtract, regStart, regEnd, regStart); + } + + if( pMWin->eStart!=TK_UNBOUNDED ){ + sqlite3VdbeAddOp1(v, OP_Rewind, s.start.csr); + } + sqlite3VdbeAddOp1(v, OP_Rewind, s.current.csr); + sqlite3VdbeAddOp1(v, OP_Rewind, s.end.csr); + if( regPeer && pOrderBy ){ + sqlite3VdbeAddOp3(v, OP_Copy, regNewPeer, regPeer, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.start.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.current.reg, pOrderBy->nExpr-1); + sqlite3VdbeAddOp3(v, OP_Copy, regPeer, s.end.reg, pOrderBy->nExpr-1); + } + + sqlite3VdbeAddOp2(v, OP_Goto, 0, lblWhereEnd); + + sqlite3VdbeJumpHere(v, addrNe); + + /* Beginning of the block executed for the second and subsequent rows. */ + if( regPeer ){ + windowIfNewPeer(pParse, pOrderBy, regNewPeer, regPeer, lblWhereEnd); + } + if( pMWin->eStart==TK_FOLLOWING ){ + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = sqlite3VdbeMakeLabel(pParse); + int addrNext = sqlite3VdbeCurrentAddr(v); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrNext); + sqlite3VdbeResolveLabel(v, lbl); + }else{ + windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + } + } + }else + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + if( !bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + }else{ + int addr = 0; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eEnd!=TK_UNBOUNDED ){ + if( pMWin->eFrmType==TK_RANGE ){ + int lbl = 0; + addr = sqlite3VdbeCurrentAddr(v); + if( regEnd ){ + lbl = sqlite3VdbeMakeLabel(pParse); + windowCodeRangeTest(&s, OP_Ge, s.current.csr, regEnd, s.end.csr, lbl); + } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ){ + sqlite3VdbeAddOp2(v, OP_Goto, 0, addr); + sqlite3VdbeResolveLabel(v, lbl); + } + }else{ + if( regEnd ){ + addr = sqlite3VdbeAddOp3(v, OP_IfPos, regEnd, 0, 1); + VdbeCoverage(v); + } + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + if( regEnd ) sqlite3VdbeJumpHere(v, addr); + } + } + } + + /* End of the main input loop */ + sqlite3VdbeResolveLabel(v, lblWhereEnd); + sqlite3WhereEnd(pWInfo); + + /* Fall through */ + if( pMWin->pPartition ){ + addrInteger = sqlite3VdbeAddOp2(v, OP_Integer, 0, regFlushPart); + sqlite3VdbeJumpHere(v, addrGosubFlush); + } + + s.regRowid = 0; + addrEmpty = sqlite3VdbeAddOp1(v, OP_Rewind, csrWrite); + VdbeCoverage(v); + if( pMWin->eEnd==TK_PRECEDING ){ + int bRPS = (pMWin->eStart==TK_PRECEDING && pMWin->eFrmType==TK_RANGE); + windowCodeOp(&s, WINDOW_AGGSTEP, regEnd, 0); + if( bRPS ) windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 0); + }else if( pMWin->eStart==TK_FOLLOWING ){ + int addrStart; + int addrBreak1; + int addrBreak2; + int addrBreak3; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + if( pMWin->eFrmType==TK_RANGE ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + }else + if( pMWin->eEnd==TK_UNBOUNDED ){ + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regStart, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, 0, 1); + }else{ + assert( pMWin->eEnd==TK_FOLLOWING ); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak1 = windowCodeOp(&s, WINDOW_RETURN_ROW, regEnd, 1); + addrBreak2 = windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 1); + } + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak2); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak3 = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak1); + sqlite3VdbeJumpHere(v, addrBreak3); + }else{ + int addrBreak; + int addrStart; + windowCodeOp(&s, WINDOW_AGGSTEP, 0, 0); + addrStart = sqlite3VdbeCurrentAddr(v); + addrBreak = windowCodeOp(&s, WINDOW_RETURN_ROW, 0, 1); + windowCodeOp(&s, WINDOW_AGGINVERSE, regStart, 0); + sqlite3VdbeAddOp2(v, OP_Goto, 0, addrStart); + sqlite3VdbeJumpHere(v, addrBreak); + } + sqlite3VdbeJumpHere(v, addrEmpty); + + sqlite3VdbeAddOp1(v, OP_ResetSorter, s.current.csr); + if( pMWin->pPartition ){ + if( pMWin->regStartRowid ){ + sqlite3VdbeAddOp2(v, OP_Integer, 1, pMWin->regStartRowid); + sqlite3VdbeAddOp2(v, OP_Integer, 0, pMWin->regEndRowid); + } + sqlite3VdbeChangeP1(v, addrInteger, sqlite3VdbeCurrentAddr(v)); + sqlite3VdbeAddOp1(v, OP_Return, regFlushPart); + } +} + +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/************** End of window.c **********************************************/ +/************** Begin file parse.c *******************************************/ +/* This file is automatically generated by Lemon from input grammar +** source file "parse.y". +*/ +/* +** 2001-09-15 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** This file contains SQLite's SQL parser. +** +** The canonical source code to this file ("parse.y") is a Lemon grammar +** file that specifies the input grammar and actions to take while parsing. +** That input file is processed by Lemon to generate a C-language +** implementation of a parser for the given grammar. You might be reading +** this comment as part of the translated C-code. Edits should be made +** to the original parse.y sources. +*/ /* #include "sqliteInt.h" */ @@ -130751,30 +173828,25 @@ SQLITE_PRIVATE void sqlite3WhereEnd(WhereInfo *pWInfo){ */ #define YYPARSEFREENEVERNULL 1 +/* +** In the amalgamation, the parse.c file generated by lemon and the +** tokenize.c file are concatenated. In that case, sqlite3RunParser() +** has access to the the size of the yyParser object and so the parser +** engine can be allocated from stack. In that case, only the +** sqlite3ParserInit() and sqlite3ParserFinalize() routines are invoked +** and the sqlite3ParserAlloc() and sqlite3ParserFree() routines can be +** omitted. +*/ +#ifdef SQLITE_AMALGAMATION +# define sqlite3Parser_ENGINEALWAYSONSTACK 1 +#endif + /* ** Alternative datatype for the argument to the malloc() routine passed ** into sqlite3ParserAlloc(). The default is size_t. */ #define YYMALLOCARGTYPE u64 -/* -** An instance of this structure holds information about the -** LIMIT clause of a SELECT statement. -*/ -struct LimitVal { - Expr *pLimit; /* The LIMIT expression. NULL if there is no limit */ - Expr *pOffset; /* The OFFSET expression. NULL if there is none */ -}; - -/* -** An instance of this structure is used to store the LIKE, -** GLOB, NOT LIKE, and NOT GLOB operators. -*/ -struct LikeOp { - Token eOperator; /* "like" or "glob" or "regexp" */ - int bNot; /* True if the NOT keyword is present */ -}; - /* ** An instance of the following structure describes the event of a ** TRIGGER. "a" is the event type, one of TK_UPDATE, TK_INSERT, @@ -130786,20 +173858,39 @@ struct LikeOp { */ struct TrigEvent { int a; IdList * b; }; -/* -** An instance of this structure holds the ATTACH key and the key type. -*/ -struct AttachKey { int type; Token key; }; +struct FrameBound { int eType; Expr *pExpr; }; /* ** Disable lookaside memory allocation for objects that might be ** shared across database connections. */ static void disableLookaside(Parse *pParse){ + sqlite3 *db = pParse->db; pParse->disableLookaside++; - pParse->db->lookaside.bDisable++; + DisableLookaside; } +#if !defined(SQLITE_ENABLE_UPDATE_DELETE_LIMIT) \ + && defined(SQLITE_UDL_CAPABLE_PARSER) +/* +** Issue an error message if an ORDER BY or LIMIT clause occurs on an +** UPDATE or DELETE statement. +*/ +static void updateDeleteLimitError( + Parse *pParse, + ExprList *pOrderBy, + Expr *pLimit +){ + if( pOrderBy ){ + sqlite3ErrorMsg(pParse, "syntax error near \"ORDER BY\""); + }else{ + sqlite3ErrorMsg(pParse, "syntax error near \"LIMIT\""); + } + sqlite3ExprListDelete(pParse->db, pOrderBy); + sqlite3ExprDelete(pParse->db, pLimit); +} +#endif /* SQLITE_ENABLE_UPDATE_DELETE_LIMIT */ + /* ** For a compound SELECT statement, make sure p->pPrior->pNext==p for @@ -130807,100 +173898,101 @@ static void disableLookaside(Parse *pParse){ ** SQLITE_LIMIT_COMPOUND_SELECT. */ static void parserDoubleLinkSelect(Parse *pParse, Select *p){ + assert( p!=0 ); if( p->pPrior ){ - Select *pNext = 0, *pLoop; - int mxSelect, cnt = 0; - for(pLoop=p; pLoop; pNext=pLoop, pLoop=pLoop->pPrior, cnt++){ + Select *pNext = 0, *pLoop = p; + int mxSelect, cnt = 1; + while(1){ pLoop->pNext = pNext; pLoop->selFlags |= SF_Compound; + pNext = pLoop; + pLoop = pLoop->pPrior; + if( pLoop==0 ) break; + cnt++; + if( pLoop->pOrderBy || pLoop->pLimit ){ + sqlite3ErrorMsg(pParse,"%s clause should come after %s not before", + pLoop->pOrderBy!=0 ? "ORDER BY" : "LIMIT", + sqlite3SelectOpName(pNext->op)); + break; + } } - if( (p->selFlags & SF_MultiValue)==0 && - (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 && - cnt>mxSelect + if( (p->selFlags & (SF_MultiValue|SF_Values))==0 + && (mxSelect = pParse->db->aLimit[SQLITE_LIMIT_COMPOUND_SELECT])>0 + && cnt>mxSelect ){ sqlite3ErrorMsg(pParse, "too many terms in compound SELECT"); } } } - /* This is a utility routine used to set the ExprSpan.zStart and - ** ExprSpan.zEnd values of pOut so that the span covers the complete - ** range of text beginning with pStart and going to the end of pEnd. + /* Attach a With object describing the WITH clause to a Select + ** object describing the query for which the WITH clause is a prefix. */ - static void spanSet(ExprSpan *pOut, Token *pStart, Token *pEnd){ - pOut->zStart = pStart->z; - pOut->zEnd = &pEnd->z[pEnd->n]; - } - - /* Construct a new Expr object from a single identifier. Use the - ** new Expr to populate pOut. Set the span of pOut to be the identifier - ** that created the expression. - */ - static void spanExpr(ExprSpan *pOut, Parse *pParse, int op, Token t){ - pOut->pExpr = sqlite3PExpr(pParse, op, 0, 0, &t); - pOut->zStart = t.z; - pOut->zEnd = &t.z[t.n]; - } - - /* This routine constructs a binary expression node out of two ExprSpan - ** objects and uses the result to populate a new ExprSpan object. - */ - static void spanBinaryExpr( - Parse *pParse, /* The parsing context. Errors accumulate here */ - int op, /* The binary operation */ - ExprSpan *pLeft, /* The left operand, and output */ - ExprSpan *pRight /* The right operand */ - ){ - pLeft->pExpr = sqlite3PExpr(pParse, op, pLeft->pExpr, pRight->pExpr, 0); - pLeft->zEnd = pRight->zEnd; - } - - /* If doNot is true, then add a TK_NOT Expr-node wrapper around the - ** outside of *ppExpr. - */ - static void exprNot(Parse *pParse, int doNot, ExprSpan *pSpan){ - if( doNot ){ - pSpan->pExpr = sqlite3PExpr(pParse, TK_NOT, pSpan->pExpr, 0, 0); + static Select *attachWithToSelect(Parse *pParse, Select *pSelect, With *pWith){ + if( pSelect ){ + pSelect->pWith = pWith; + parserDoubleLinkSelect(pParse, pSelect); + }else{ + sqlite3WithDelete(pParse->db, pWith); } + return pSelect; } - /* Construct an expression node for a unary postfix operator + /* Memory allocator for parser stack resizing. This is a thin wrapper around + ** sqlite3_realloc() that includes a call to sqlite3FaultSim() to facilitate + ** testing. */ - static void spanUnaryPostfix( - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand, and output */ - Token *pPostOp /* The operand token for setting the span */ - ){ - pOperand->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOperand->zEnd = &pPostOp->z[pPostOp->n]; - } + static void *parserStackRealloc(void *pOld, sqlite3_uint64 newSize){ + return sqlite3FaultSim(700) ? 0 : sqlite3_realloc(pOld, newSize); + } + + + /* Construct a new Expr object from a single token */ + static Expr *tokenExpr(Parse *pParse, int op, Token t){ + Expr *p = sqlite3DbMallocRawNN(pParse->db, sizeof(Expr)+t.n+1); + if( p ){ + /* memset(p, 0, sizeof(Expr)); */ + p->op = (u8)op; + p->affExpr = 0; + p->flags = EP_Leaf; + ExprClearVVAProperties(p); + /* p->iAgg = -1; // Not required */ + p->pLeft = p->pRight = 0; + p->pAggInfo = 0; + memset(&p->x, 0, sizeof(p->x)); + memset(&p->y, 0, sizeof(p->y)); + p->op2 = 0; + p->iTable = 0; + p->iColumn = 0; + p->u.zToken = (char*)&p[1]; + memcpy(p->u.zToken, t.z, t.n); + p->u.zToken[t.n] = 0; + p->w.iOfst = (int)(t.z - pParse->zTail); + if( sqlite3Isquote(p->u.zToken[0]) ){ + sqlite3DequoteExpr(p); + } +#if SQLITE_MAX_EXPR_DEPTH>0 + p->nHeight = 1; +#endif + if( IN_RENAME_OBJECT ){ + return (Expr*)sqlite3RenameTokenMap(pParse, (void*)p, &t); + } + } + return p; + } + /* A routine to convert a binary TK_IS or TK_ISNOT expression into a ** unary TK_ISNULL or TK_NOTNULL expression. */ static void binaryToUnaryIfNull(Parse *pParse, Expr *pY, Expr *pA, int op){ sqlite3 *db = pParse->db; - if( pA && pY && pY->op==TK_NULL ){ + if( pA && pY && pY->op==TK_NULL && !IN_RENAME_OBJECT ){ pA->op = (u8)op; sqlite3ExprDelete(db, pA->pRight); pA->pRight = 0; } } - /* Construct an expression node for a unary prefix operator - */ - static void spanUnaryPrefix( - ExprSpan *pOut, /* Write the new expression node here */ - Parse *pParse, /* Parsing context to record errors */ - int op, /* The operator */ - ExprSpan *pOperand, /* The operand */ - Token *pPreOp /* The operand token for setting the span */ - ){ - pOut->zStart = pPreOp->z; - pOut->pExpr = sqlite3PExpr(pParse, op, pOperand->pExpr, 0, 0); - pOut->zEnd = pOperand->zEnd; - } - /* Add a single new term to an ExprList that is used to store a ** list of identifiers. Report an error if the ID list contains ** a COLLATE clause or an ASC or DESC keyword, except ignore the @@ -130923,12 +174015,201 @@ static void disableLookaside(Parse *pParse){ sqlite3ExprListSetName(pParse, p, pIdToken, 1); return p; } + +#if TK_SPAN>255 +# error too many tokens in the grammar +#endif /**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +#ifndef TK_SEMI +#define TK_SEMI 1 +#define TK_EXPLAIN 2 +#define TK_QUERY 3 +#define TK_PLAN 4 +#define TK_BEGIN 5 +#define TK_TRANSACTION 6 +#define TK_DEFERRED 7 +#define TK_IMMEDIATE 8 +#define TK_EXCLUSIVE 9 +#define TK_COMMIT 10 +#define TK_END 11 +#define TK_ROLLBACK 12 +#define TK_SAVEPOINT 13 +#define TK_RELEASE 14 +#define TK_TO 15 +#define TK_TABLE 16 +#define TK_CREATE 17 +#define TK_IF 18 +#define TK_NOT 19 +#define TK_EXISTS 20 +#define TK_TEMP 21 +#define TK_LP 22 +#define TK_RP 23 +#define TK_AS 24 +#define TK_COMMA 25 +#define TK_WITHOUT 26 +#define TK_ABORT 27 +#define TK_ACTION 28 +#define TK_AFTER 29 +#define TK_ANALYZE 30 +#define TK_ASC 31 +#define TK_ATTACH 32 +#define TK_BEFORE 33 +#define TK_BY 34 +#define TK_CASCADE 35 +#define TK_CAST 36 +#define TK_CONFLICT 37 +#define TK_DATABASE 38 +#define TK_DESC 39 +#define TK_DETACH 40 +#define TK_EACH 41 +#define TK_FAIL 42 +#define TK_OR 43 +#define TK_AND 44 +#define TK_IS 45 +#define TK_ISNOT 46 +#define TK_MATCH 47 +#define TK_LIKE_KW 48 +#define TK_BETWEEN 49 +#define TK_IN 50 +#define TK_ISNULL 51 +#define TK_NOTNULL 52 +#define TK_NE 53 +#define TK_EQ 54 +#define TK_GT 55 +#define TK_LE 56 +#define TK_LT 57 +#define TK_GE 58 +#define TK_ESCAPE 59 +#define TK_ID 60 +#define TK_COLUMNKW 61 +#define TK_DO 62 +#define TK_FOR 63 +#define TK_IGNORE 64 +#define TK_INITIALLY 65 +#define TK_INSTEAD 66 +#define TK_NO 67 +#define TK_KEY 68 +#define TK_OF 69 +#define TK_OFFSET 70 +#define TK_PRAGMA 71 +#define TK_RAISE 72 +#define TK_RECURSIVE 73 +#define TK_REPLACE 74 +#define TK_RESTRICT 75 +#define TK_ROW 76 +#define TK_ROWS 77 +#define TK_TRIGGER 78 +#define TK_VACUUM 79 +#define TK_VIEW 80 +#define TK_VIRTUAL 81 +#define TK_WITH 82 +#define TK_NULLS 83 +#define TK_FIRST 84 +#define TK_LAST 85 +#define TK_CURRENT 86 +#define TK_FOLLOWING 87 +#define TK_PARTITION 88 +#define TK_PRECEDING 89 +#define TK_RANGE 90 +#define TK_UNBOUNDED 91 +#define TK_EXCLUDE 92 +#define TK_GROUPS 93 +#define TK_OTHERS 94 +#define TK_TIES 95 +#define TK_GENERATED 96 +#define TK_ALWAYS 97 +#define TK_MATERIALIZED 98 +#define TK_REINDEX 99 +#define TK_RENAME 100 +#define TK_CTIME_KW 101 +#define TK_ANY 102 +#define TK_BITAND 103 +#define TK_BITOR 104 +#define TK_LSHIFT 105 +#define TK_RSHIFT 106 +#define TK_PLUS 107 +#define TK_MINUS 108 +#define TK_STAR 109 +#define TK_SLASH 110 +#define TK_REM 111 +#define TK_CONCAT 112 +#define TK_PTR 113 +#define TK_COLLATE 114 +#define TK_BITNOT 115 +#define TK_ON 116 +#define TK_INDEXED 117 +#define TK_STRING 118 +#define TK_JOIN_KW 119 +#define TK_CONSTRAINT 120 +#define TK_DEFAULT 121 +#define TK_NULL 122 +#define TK_PRIMARY 123 +#define TK_UNIQUE 124 +#define TK_CHECK 125 +#define TK_REFERENCES 126 +#define TK_AUTOINCR 127 +#define TK_INSERT 128 +#define TK_DELETE 129 +#define TK_UPDATE 130 +#define TK_SET 131 +#define TK_DEFERRABLE 132 +#define TK_FOREIGN 133 +#define TK_DROP 134 +#define TK_UNION 135 +#define TK_ALL 136 +#define TK_EXCEPT 137 +#define TK_INTERSECT 138 +#define TK_SELECT 139 +#define TK_VALUES 140 +#define TK_DISTINCT 141 +#define TK_DOT 142 +#define TK_FROM 143 +#define TK_JOIN 144 +#define TK_USING 145 +#define TK_ORDER 146 +#define TK_GROUP 147 +#define TK_HAVING 148 +#define TK_LIMIT 149 +#define TK_WHERE 150 +#define TK_RETURNING 151 +#define TK_INTO 152 +#define TK_NOTHING 153 +#define TK_FLOAT 154 +#define TK_BLOB 155 +#define TK_INTEGER 156 +#define TK_VARIABLE 157 +#define TK_CASE 158 +#define TK_WHEN 159 +#define TK_THEN 160 +#define TK_ELSE 161 +#define TK_INDEX 162 +#define TK_ALTER 163 +#define TK_ADD 164 +#define TK_WINDOW 165 +#define TK_OVER 166 +#define TK_FILTER 167 +#define TK_COLUMN 168 +#define TK_AGG_FUNCTION 169 +#define TK_AGG_COLUMN 170 +#define TK_TRUEFALSE 171 +#define TK_FUNCTION 172 +#define TK_UPLUS 173 +#define TK_UMINUS 174 +#define TK_TRUTH 175 +#define TK_REGISTER 176 +#define TK_VECTOR 177 +#define TK_SELECT_COLUMN 178 +#define TK_IF_NULL_ROW 179 +#define TK_ASTERISK 180 +#define TK_SPAN 181 +#define TK_ERROR 182 +#define TK_QNUMBER 183 +#define TK_SPACE 184 +#define TK_ILLEGAL 185 +#endif +/**************** End token definitions ***************************************/ /* The next sections is a series of control #defines. ** various aspects of the generated parser. @@ -130953,7 +174234,7 @@ static void disableLookaside(Parse *pParse){ ** the minor type might be the name of the identifier. ** Each non-terminal can have a different minor type. ** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal +** This macros defines the minor type for terminal ** symbols. ** YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of @@ -130963,65 +174244,93 @@ static void disableLookaside(Parse *pParse){ ** zero the stack is dynamically sized using realloc() ** sqlite3ParserARG_SDECL A static variable declaration for the %extra_argument ** sqlite3ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter ** sqlite3ParserARG_STORE Code to store %extra_argument into yypParser ** sqlite3ParserARG_FETCH Code to extract %extra_argument from yypParser +** sqlite3ParserCTX_* As sqlite3ParserARG_ except for %extra_context +** YYREALLOC Name of the realloc() function to use +** YYFREE Name of the free() function to use +** YYDYNSTACK True if stack space should be extended on heap ** YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** YYNSTATE the combined number of states. ** YYNRULE the number of rules in the grammar +** YYNTOKEN Number of terminal symbols ** YY_MAX_SHIFT Maximum value for shift actions ** YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions ** YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** YY_MIN_REDUCE Maximum value for reduce actions ** YY_ERROR_ACTION The yy_action[] code for syntax error ** YY_ACCEPT_ACTION The yy_action[] code for accept ** YY_NO_ACTION The yy_action[] code for no-op +** YY_MIN_REDUCE Minimum value for reduce actions +** YY_MAX_REDUCE Maximum value for reduce actions +** YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 #endif /************* Begin control #defines *****************************************/ -#define YYCODETYPE unsigned char -#define YYNOCODE 251 +#define YYCODETYPE unsigned short int +#define YYNOCODE 322 #define YYACTIONTYPE unsigned short int -#define YYWILDCARD 96 +#define YYWILDCARD 102 #define sqlite3ParserTOKENTYPE Token typedef union { int yyinit; sqlite3ParserTOKENTYPE yy0; - struct LimitVal yy64; - Expr* yy122; - Select* yy159; - IdList* yy180; - struct {int value; int mask;} yy207; - struct LikeOp yy318; - TriggerStep* yy327; - With* yy331; - ExprSpan yy342; - SrcList* yy347; - int yy392; - struct TrigEvent yy410; - ExprList* yy442; + ExprList* yy14; + With* yy59; + Cte* yy67; + Upsert* yy122; + IdList* yy132; + int yy144; + const char* yy168; + SrcList* yy203; + Window* yy211; + OnOrUsing yy269; + struct TrigEvent yy286; + struct {int value; int mask;} yy383; + u32 yy391; + TriggerStep* yy427; + Expr* yy454; + u8 yy462; + struct FrameBound yy509; + Select* yy555; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 #endif -#define sqlite3ParserARG_SDECL Parse *pParse; -#define sqlite3ParserARG_PDECL ,Parse *pParse -#define sqlite3ParserARG_FETCH Parse *pParse = yypParser->pParse -#define sqlite3ParserARG_STORE yypParser->pParse = pParse +#define sqlite3ParserARG_SDECL +#define sqlite3ParserARG_PDECL +#define sqlite3ParserARG_PARAM +#define sqlite3ParserARG_FETCH +#define sqlite3ParserARG_STORE +#define YYREALLOC parserStackRealloc +#define YYFREE sqlite3_free +#define YYDYNSTACK 1 +#define sqlite3ParserCTX_SDECL Parse *pParse; +#define sqlite3ParserCTX_PDECL ,Parse *pParse +#define sqlite3ParserCTX_PARAM ,pParse +#define sqlite3ParserCTX_FETCH Parse *pParse=yypParser->pParse; +#define sqlite3ParserCTX_STORE yypParser->pParse=pParse; #define YYFALLBACK 1 -#define YYNSTATE 440 -#define YYNRULE 326 -#define YY_MAX_SHIFT 439 -#define YY_MIN_SHIFTREDUCE 649 -#define YY_MAX_SHIFTREDUCE 974 -#define YY_MIN_REDUCE 975 -#define YY_MAX_REDUCE 1300 -#define YY_ERROR_ACTION 1301 -#define YY_ACCEPT_ACTION 1302 -#define YY_NO_ACTION 1303 +#define YYNSTATE 583 +#define YYNRULE 409 +#define YYNRULE_WITH_ACTION 344 +#define YYNTOKEN 186 +#define YY_MAX_SHIFT 582 +#define YY_MIN_SHIFTREDUCE 845 +#define YY_MAX_SHIFTREDUCE 1253 +#define YY_ERROR_ACTION 1254 +#define YY_ACCEPT_ACTION 1255 +#define YY_NO_ACTION 1256 +#define YY_MIN_REDUCE 1257 +#define YY_MAX_REDUCE 1665 +#define YY_MIN_DSTRCTR 205 +#define YY_MAX_DSTRCTR 319 /************* End control #defines *******************************************/ +#define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) /* Define the yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -131035,11 +174344,27 @@ typedef union { # define yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if YYSTACKDEPTH<=0 || YYDYNSTACK +# define YYGROWABLESTACK 1 +#else +# define YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if YYSTACKDEPTH<=0 +# undef YYSTACKDEPTH +# define YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement ** functions that take a state number and lookahead value and return an -** action integer. +** action integer. ** ** Suppose the action integer is N. Then the action is determined as ** follows @@ -131050,9 +174375,6 @@ typedef union { ** N between YY_MIN_SHIFTREDUCE Shift to an arbitrary state then ** and YY_MAX_SHIFTREDUCE reduce by rule N-YY_MIN_SHIFTREDUCE. ** -** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE -** and YY_MAX_REDUCE - ** N == YY_ERROR_ACTION A syntax error has occurred. ** ** N == YY_ACCEPT_ACTION The parser accepts its input. @@ -131060,21 +174382,22 @@ typedef union { ** N == YY_NO_ACTION No such action. Denotes unused ** slots in the yy_action[] table. ** +** N between YY_MIN_REDUCE Reduce by rule N-YY_MIN_REDUCE +** and YY_MAX_REDUCE +** ** The action table is constructed as a single large table named yy_action[]. -** Given state S and lookahead X, the action is computed as +** Given state S and lookahead X, the action is computed as either: ** -** yy_action[ yy_shift_ofst[S] + X ] +** (A) N = yy_action[ yy_shift_ofst[S] + X ] +** (B) N = yy_default[S] ** -** If the index value yy_shift_ofst[S]+X is out of range or if the value -** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X or if yy_shift_ofst[S] -** is equal to YY_SHIFT_USE_DFLT, it means that the action is not in the table -** and that yy_default[S] should be used instead. +** The (A) formula is preferred. The B formula is used instead if +** yy_lookahead[yy_shift_ofst[S]+X] is not equal to X. ** -** The formula above is for computing the action when the lookahead is +** The formulas above are for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the yy_reduce_ofst[] array is used in place of -** the yy_shift_ofst[] array and YY_REDUCE_USE_DFLT is used in place of -** YY_SHIFT_USE_DFLT. +** the yy_shift_ofst[] array. ** ** The following are the tables generated in this section: ** @@ -131088,452 +174411,649 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (1501) +#define YY_ACTTAB_COUNT (2207) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 315, 810, 339, 804, 5, 194, 194, 798, 92, 93, - /* 10 */ 83, 819, 819, 831, 834, 823, 823, 90, 90, 91, - /* 20 */ 91, 91, 91, 290, 89, 89, 89, 89, 88, 88, - /* 30 */ 87, 87, 87, 86, 339, 315, 952, 952, 803, 803, - /* 40 */ 803, 922, 342, 92, 93, 83, 819, 819, 831, 834, - /* 50 */ 823, 823, 90, 90, 91, 91, 91, 91, 123, 89, - /* 60 */ 89, 89, 89, 88, 88, 87, 87, 87, 86, 339, - /* 70 */ 88, 88, 87, 87, 87, 86, 339, 772, 952, 952, - /* 80 */ 315, 87, 87, 87, 86, 339, 773, 68, 92, 93, - /* 90 */ 83, 819, 819, 831, 834, 823, 823, 90, 90, 91, - /* 100 */ 91, 91, 91, 434, 89, 89, 89, 89, 88, 88, - /* 110 */ 87, 87, 87, 86, 339, 1302, 146, 921, 2, 315, - /* 120 */ 427, 24, 679, 953, 48, 86, 339, 92, 93, 83, - /* 130 */ 819, 819, 831, 834, 823, 823, 90, 90, 91, 91, - /* 140 */ 91, 91, 94, 89, 89, 89, 89, 88, 88, 87, - /* 150 */ 87, 87, 86, 339, 933, 933, 315, 259, 412, 398, - /* 160 */ 396, 57, 733, 733, 92, 93, 83, 819, 819, 831, - /* 170 */ 834, 823, 823, 90, 90, 91, 91, 91, 91, 56, - /* 180 */ 89, 89, 89, 89, 88, 88, 87, 87, 87, 86, - /* 190 */ 339, 315, 1245, 922, 342, 268, 934, 935, 241, 92, - /* 200 */ 93, 83, 819, 819, 831, 834, 823, 823, 90, 90, - /* 210 */ 91, 91, 91, 91, 291, 89, 89, 89, 89, 88, - /* 220 */ 88, 87, 87, 87, 86, 339, 315, 913, 1295, 682, - /* 230 */ 687, 1295, 233, 397, 92, 93, 83, 819, 819, 831, - /* 240 */ 834, 823, 823, 90, 90, 91, 91, 91, 91, 326, - /* 250 */ 89, 89, 89, 89, 88, 88, 87, 87, 87, 86, - /* 260 */ 339, 315, 85, 82, 168, 680, 431, 938, 939, 92, - /* 270 */ 93, 83, 819, 819, 831, 834, 823, 823, 90, 90, - /* 280 */ 91, 91, 91, 91, 291, 89, 89, 89, 89, 88, - /* 290 */ 88, 87, 87, 87, 86, 339, 315, 319, 913, 1296, - /* 300 */ 797, 911, 1296, 681, 92, 93, 83, 819, 819, 831, - /* 310 */ 834, 823, 823, 90, 90, 91, 91, 91, 91, 335, - /* 320 */ 89, 89, 89, 89, 88, 88, 87, 87, 87, 86, - /* 330 */ 339, 315, 876, 876, 373, 85, 82, 168, 944, 92, - /* 340 */ 93, 83, 819, 819, 831, 834, 823, 823, 90, 90, - /* 350 */ 91, 91, 91, 91, 896, 89, 89, 89, 89, 88, - /* 360 */ 88, 87, 87, 87, 86, 339, 315, 370, 307, 973, - /* 370 */ 367, 1, 911, 433, 92, 93, 83, 819, 819, 831, - /* 380 */ 834, 823, 823, 90, 90, 91, 91, 91, 91, 189, - /* 390 */ 89, 89, 89, 89, 88, 88, 87, 87, 87, 86, - /* 400 */ 339, 315, 720, 948, 933, 933, 149, 718, 948, 92, - /* 410 */ 93, 83, 819, 819, 831, 834, 823, 823, 90, 90, - /* 420 */ 91, 91, 91, 91, 434, 89, 89, 89, 89, 88, - /* 430 */ 88, 87, 87, 87, 86, 339, 338, 938, 939, 947, - /* 440 */ 694, 940, 974, 315, 953, 48, 934, 935, 715, 689, - /* 450 */ 71, 92, 93, 83, 819, 819, 831, 834, 823, 823, - /* 460 */ 90, 90, 91, 91, 91, 91, 320, 89, 89, 89, - /* 470 */ 89, 88, 88, 87, 87, 87, 86, 339, 315, 412, - /* 480 */ 403, 820, 820, 832, 835, 74, 92, 81, 83, 819, - /* 490 */ 819, 831, 834, 823, 823, 90, 90, 91, 91, 91, - /* 500 */ 91, 698, 89, 89, 89, 89, 88, 88, 87, 87, - /* 510 */ 87, 86, 339, 315, 259, 654, 655, 656, 393, 111, - /* 520 */ 331, 153, 93, 83, 819, 819, 831, 834, 823, 823, - /* 530 */ 90, 90, 91, 91, 91, 91, 434, 89, 89, 89, - /* 540 */ 89, 88, 88, 87, 87, 87, 86, 339, 315, 188, - /* 550 */ 187, 186, 824, 937, 328, 219, 953, 48, 83, 819, - /* 560 */ 819, 831, 834, 823, 823, 90, 90, 91, 91, 91, - /* 570 */ 91, 956, 89, 89, 89, 89, 88, 88, 87, 87, - /* 580 */ 87, 86, 339, 79, 429, 738, 3, 1174, 955, 348, - /* 590 */ 737, 332, 792, 933, 933, 937, 79, 429, 730, 3, - /* 600 */ 203, 160, 278, 391, 273, 390, 190, 892, 434, 400, - /* 610 */ 741, 76, 77, 271, 287, 253, 353, 242, 78, 340, - /* 620 */ 340, 85, 82, 168, 76, 77, 233, 397, 953, 48, - /* 630 */ 432, 78, 340, 340, 277, 934, 935, 185, 439, 651, - /* 640 */ 388, 385, 384, 432, 234, 276, 107, 418, 349, 337, - /* 650 */ 336, 383, 893, 728, 215, 949, 123, 971, 308, 810, - /* 660 */ 418, 436, 435, 412, 394, 798, 400, 873, 894, 123, - /* 670 */ 721, 872, 810, 889, 436, 435, 215, 949, 798, 351, - /* 680 */ 722, 697, 380, 434, 771, 371, 22, 434, 400, 79, - /* 690 */ 429, 232, 3, 189, 413, 870, 803, 803, 803, 805, - /* 700 */ 18, 54, 148, 953, 48, 956, 113, 953, 9, 803, - /* 710 */ 803, 803, 805, 18, 310, 123, 748, 76, 77, 742, - /* 720 */ 123, 325, 955, 866, 78, 340, 340, 113, 350, 359, - /* 730 */ 85, 82, 168, 343, 960, 960, 432, 770, 412, 414, - /* 740 */ 407, 23, 1240, 1240, 79, 429, 357, 3, 166, 91, - /* 750 */ 91, 91, 91, 418, 89, 89, 89, 89, 88, 88, - /* 760 */ 87, 87, 87, 86, 339, 810, 434, 436, 435, 792, - /* 770 */ 320, 798, 76, 77, 789, 271, 123, 434, 360, 78, - /* 780 */ 340, 340, 864, 85, 82, 168, 953, 9, 395, 743, - /* 790 */ 360, 432, 253, 358, 252, 933, 933, 953, 30, 889, - /* 800 */ 327, 216, 803, 803, 803, 805, 18, 113, 418, 89, - /* 810 */ 89, 89, 89, 88, 88, 87, 87, 87, 86, 339, - /* 820 */ 810, 113, 436, 435, 792, 185, 798, 288, 388, 385, - /* 830 */ 384, 123, 113, 920, 2, 796, 696, 934, 935, 383, - /* 840 */ 69, 429, 434, 3, 218, 110, 738, 253, 358, 252, - /* 850 */ 434, 737, 933, 933, 892, 359, 222, 803, 803, 803, - /* 860 */ 805, 18, 953, 47, 933, 933, 933, 933, 76, 77, - /* 870 */ 953, 9, 366, 904, 217, 78, 340, 340, 677, 305, - /* 880 */ 304, 303, 206, 301, 224, 259, 664, 432, 337, 336, - /* 890 */ 434, 228, 247, 144, 934, 935, 933, 933, 667, 893, - /* 900 */ 324, 1259, 96, 434, 418, 796, 934, 935, 934, 935, - /* 910 */ 953, 48, 401, 148, 289, 894, 810, 417, 436, 435, - /* 920 */ 677, 759, 798, 953, 9, 314, 220, 162, 161, 170, - /* 930 */ 402, 239, 953, 8, 194, 683, 683, 410, 934, 935, - /* 940 */ 238, 959, 933, 933, 225, 408, 945, 365, 957, 212, - /* 950 */ 958, 172, 757, 803, 803, 803, 805, 18, 173, 365, - /* 960 */ 176, 123, 171, 113, 244, 952, 246, 434, 356, 796, - /* 970 */ 372, 365, 236, 960, 960, 810, 290, 804, 191, 165, - /* 980 */ 852, 798, 259, 316, 934, 935, 237, 953, 34, 404, - /* 990 */ 91, 91, 91, 91, 84, 89, 89, 89, 89, 88, - /* 1000 */ 88, 87, 87, 87, 86, 339, 701, 952, 434, 240, - /* 1010 */ 347, 758, 803, 803, 803, 434, 245, 1179, 434, 389, - /* 1020 */ 434, 376, 434, 895, 167, 434, 405, 702, 953, 35, - /* 1030 */ 673, 321, 221, 434, 333, 953, 11, 434, 953, 26, - /* 1040 */ 953, 36, 953, 37, 251, 953, 38, 434, 259, 434, - /* 1050 */ 757, 434, 329, 953, 27, 434, 223, 953, 28, 434, - /* 1060 */ 690, 434, 67, 434, 65, 434, 862, 953, 39, 953, - /* 1070 */ 40, 953, 41, 423, 434, 953, 10, 434, 772, 953, - /* 1080 */ 42, 953, 98, 953, 43, 953, 44, 773, 434, 346, - /* 1090 */ 434, 75, 434, 73, 953, 31, 434, 953, 45, 434, - /* 1100 */ 259, 434, 690, 434, 757, 434, 887, 434, 953, 46, - /* 1110 */ 953, 32, 953, 115, 434, 266, 953, 116, 951, 953, - /* 1120 */ 117, 953, 52, 953, 33, 953, 99, 953, 49, 726, - /* 1130 */ 434, 909, 434, 19, 953, 100, 434, 344, 434, 113, - /* 1140 */ 434, 258, 692, 434, 259, 434, 670, 434, 20, 434, - /* 1150 */ 953, 101, 953, 97, 434, 259, 953, 114, 953, 112, - /* 1160 */ 953, 105, 113, 953, 104, 953, 102, 953, 103, 953, - /* 1170 */ 51, 434, 148, 434, 953, 53, 167, 434, 259, 113, - /* 1180 */ 300, 307, 912, 363, 311, 860, 248, 261, 209, 264, - /* 1190 */ 416, 953, 50, 953, 25, 420, 727, 953, 29, 430, - /* 1200 */ 321, 424, 757, 428, 322, 124, 1269, 214, 165, 710, - /* 1210 */ 859, 908, 806, 794, 309, 158, 193, 361, 254, 723, - /* 1220 */ 364, 67, 381, 269, 735, 199, 67, 70, 113, 700, - /* 1230 */ 699, 707, 708, 884, 113, 766, 113, 855, 193, 883, - /* 1240 */ 199, 869, 869, 675, 868, 868, 109, 368, 255, 260, - /* 1250 */ 263, 280, 859, 265, 806, 974, 267, 711, 695, 272, - /* 1260 */ 764, 282, 795, 284, 150, 744, 755, 415, 292, 293, - /* 1270 */ 802, 678, 672, 661, 660, 662, 927, 6, 306, 386, - /* 1280 */ 352, 786, 243, 250, 886, 362, 163, 286, 419, 298, - /* 1290 */ 930, 159, 968, 196, 126, 903, 901, 965, 55, 58, - /* 1300 */ 323, 275, 857, 136, 147, 694, 856, 121, 65, 354, - /* 1310 */ 355, 379, 175, 61, 151, 369, 180, 871, 375, 129, - /* 1320 */ 257, 756, 210, 181, 145, 131, 132, 377, 262, 663, - /* 1330 */ 133, 134, 139, 783, 791, 182, 392, 183, 312, 330, - /* 1340 */ 714, 888, 713, 851, 692, 195, 712, 406, 686, 705, - /* 1350 */ 313, 685, 64, 839, 274, 72, 684, 334, 942, 95, - /* 1360 */ 752, 279, 281, 704, 753, 751, 422, 283, 411, 750, - /* 1370 */ 426, 66, 204, 409, 21, 285, 928, 669, 437, 205, - /* 1380 */ 207, 208, 438, 658, 657, 652, 118, 108, 119, 226, - /* 1390 */ 650, 341, 157, 235, 169, 345, 106, 734, 790, 296, - /* 1400 */ 294, 295, 120, 297, 867, 865, 127, 128, 130, 724, - /* 1410 */ 229, 174, 249, 882, 137, 230, 138, 135, 885, 231, - /* 1420 */ 59, 60, 177, 881, 7, 178, 12, 179, 256, 874, - /* 1430 */ 140, 193, 962, 374, 141, 152, 666, 378, 276, 184, - /* 1440 */ 270, 122, 142, 382, 387, 62, 13, 14, 703, 63, - /* 1450 */ 125, 317, 318, 227, 809, 808, 837, 732, 15, 164, - /* 1460 */ 736, 4, 765, 211, 399, 213, 192, 143, 760, 70, - /* 1470 */ 67, 16, 17, 838, 836, 891, 841, 890, 198, 197, - /* 1480 */ 917, 154, 421, 923, 918, 155, 200, 977, 425, 840, - /* 1490 */ 156, 201, 807, 676, 80, 302, 299, 977, 202, 1261, - /* 1500 */ 1260, + /* 0 */ 130, 127, 234, 282, 282, 1328, 576, 1307, 460, 289, + /* 10 */ 289, 576, 1622, 381, 576, 1328, 573, 576, 562, 413, + /* 20 */ 1300, 1542, 573, 481, 562, 524, 460, 459, 558, 82, + /* 30 */ 82, 983, 294, 375, 51, 51, 498, 61, 61, 984, + /* 40 */ 82, 82, 1577, 137, 138, 91, 7, 1228, 1228, 1063, + /* 50 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 413, + /* 60 */ 288, 288, 182, 288, 288, 481, 536, 288, 288, 130, + /* 70 */ 127, 234, 432, 573, 525, 562, 573, 557, 562, 1290, + /* 80 */ 573, 421, 562, 137, 138, 91, 559, 1228, 1228, 1063, + /* 90 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 296, + /* 100 */ 460, 398, 1249, 134, 134, 134, 134, 133, 133, 132, + /* 110 */ 132, 132, 131, 128, 451, 44, 1050, 1050, 1064, 1067, + /* 120 */ 1255, 1, 1, 582, 2, 1259, 581, 1174, 1259, 1174, + /* 130 */ 321, 413, 155, 321, 1584, 155, 379, 112, 498, 1341, + /* 140 */ 456, 299, 1341, 134, 134, 134, 134, 133, 133, 132, + /* 150 */ 132, 132, 131, 128, 451, 137, 138, 91, 1105, 1228, + /* 160 */ 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, + /* 170 */ 136, 1204, 320, 567, 288, 288, 283, 288, 288, 523, + /* 180 */ 523, 1250, 139, 1541, 7, 214, 503, 573, 1169, 562, + /* 190 */ 573, 1054, 562, 136, 136, 136, 136, 129, 401, 547, + /* 200 */ 487, 1169, 245, 1568, 1169, 245, 133, 133, 132, 132, + /* 210 */ 132, 131, 128, 451, 261, 134, 134, 134, 134, 133, + /* 220 */ 133, 132, 132, 132, 131, 128, 451, 451, 1204, 1205, + /* 230 */ 1204, 130, 127, 234, 455, 413, 182, 455, 130, 127, + /* 240 */ 234, 134, 134, 134, 134, 133, 133, 132, 132, 132, + /* 250 */ 131, 128, 451, 136, 136, 136, 136, 538, 576, 137, + /* 260 */ 138, 91, 261, 1228, 1228, 1063, 1066, 1053, 1053, 135, + /* 270 */ 135, 136, 136, 136, 136, 44, 472, 346, 1204, 472, + /* 280 */ 346, 51, 51, 418, 93, 157, 134, 134, 134, 134, + /* 290 */ 133, 133, 132, 132, 132, 131, 128, 451, 166, 363, + /* 300 */ 298, 134, 134, 134, 134, 133, 133, 132, 132, 132, + /* 310 */ 131, 128, 451, 1293, 461, 1570, 423, 377, 275, 134, + /* 320 */ 134, 134, 134, 133, 133, 132, 132, 132, 131, 128, + /* 330 */ 451, 418, 320, 567, 1292, 1204, 1205, 1204, 257, 413, + /* 340 */ 483, 511, 508, 507, 94, 132, 132, 132, 131, 128, + /* 350 */ 451, 506, 1204, 548, 548, 388, 576, 384, 7, 413, + /* 360 */ 550, 229, 522, 137, 138, 91, 530, 1228, 1228, 1063, + /* 370 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 51, + /* 380 */ 51, 1582, 380, 137, 138, 91, 331, 1228, 1228, 1063, + /* 390 */ 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, 320, + /* 400 */ 567, 288, 288, 320, 567, 1602, 582, 2, 1259, 1204, + /* 410 */ 1205, 1204, 1628, 321, 573, 155, 562, 576, 1511, 264, + /* 420 */ 231, 520, 1341, 134, 134, 134, 134, 133, 133, 132, + /* 430 */ 132, 132, 131, 128, 451, 519, 1511, 1513, 1333, 1333, + /* 440 */ 82, 82, 498, 134, 134, 134, 134, 133, 133, 132, + /* 450 */ 132, 132, 131, 128, 451, 1435, 257, 288, 288, 511, + /* 460 */ 508, 507, 944, 1568, 413, 1019, 1204, 943, 360, 506, + /* 470 */ 573, 1598, 562, 44, 575, 551, 551, 557, 1107, 1582, + /* 480 */ 544, 576, 1107, 40, 417, 245, 531, 1505, 137, 138, + /* 490 */ 91, 219, 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, + /* 500 */ 136, 136, 136, 136, 81, 81, 1281, 1204, 413, 553, + /* 510 */ 1511, 48, 512, 448, 447, 493, 578, 455, 578, 344, + /* 520 */ 45, 1204, 1233, 1204, 1205, 1204, 428, 1235, 158, 882, + /* 530 */ 320, 567, 137, 138, 91, 1234, 1228, 1228, 1063, 1066, + /* 540 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 134, 134, + /* 550 */ 134, 134, 133, 133, 132, 132, 132, 131, 128, 451, + /* 560 */ 1236, 576, 1236, 329, 1204, 1205, 1204, 387, 492, 403, + /* 570 */ 1040, 382, 489, 123, 568, 1569, 4, 377, 1204, 1205, + /* 580 */ 1204, 570, 570, 570, 82, 82, 882, 1029, 1331, 1331, + /* 590 */ 571, 1028, 134, 134, 134, 134, 133, 133, 132, 132, + /* 600 */ 132, 131, 128, 451, 288, 288, 1281, 1204, 576, 423, + /* 610 */ 576, 1568, 413, 423, 452, 378, 886, 573, 1279, 562, + /* 620 */ 46, 557, 532, 1028, 1028, 1030, 565, 130, 127, 234, + /* 630 */ 556, 82, 82, 82, 82, 479, 137, 138, 91, 462, + /* 640 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, + /* 650 */ 136, 136, 1188, 487, 1506, 1040, 413, 6, 1204, 50, + /* 660 */ 879, 121, 121, 948, 1204, 1205, 1204, 358, 557, 122, + /* 670 */ 316, 452, 577, 452, 535, 1204, 1028, 439, 303, 212, + /* 680 */ 137, 138, 91, 213, 1228, 1228, 1063, 1066, 1053, 1053, + /* 690 */ 135, 135, 136, 136, 136, 136, 134, 134, 134, 134, + /* 700 */ 133, 133, 132, 132, 132, 131, 128, 451, 1028, 1028, + /* 710 */ 1030, 1031, 35, 288, 288, 1204, 1205, 1204, 1040, 1339, + /* 720 */ 533, 123, 568, 1569, 4, 377, 573, 1019, 562, 353, + /* 730 */ 1277, 356, 1204, 1205, 1204, 1029, 488, 1188, 571, 1028, + /* 740 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 750 */ 128, 451, 576, 343, 288, 288, 449, 449, 449, 971, + /* 760 */ 413, 1627, 452, 911, 1187, 288, 288, 573, 464, 562, + /* 770 */ 238, 1028, 1028, 1030, 565, 82, 82, 498, 573, 411, + /* 780 */ 562, 344, 467, 332, 137, 138, 91, 197, 1228, 1228, + /* 790 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, + /* 800 */ 1188, 528, 1169, 1040, 413, 1110, 1110, 495, 1041, 121, + /* 810 */ 121, 1204, 317, 540, 862, 1169, 1244, 122, 1169, 452, + /* 820 */ 577, 452, 1340, 198, 1028, 1204, 481, 526, 137, 138, + /* 830 */ 91, 560, 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, + /* 840 */ 136, 136, 136, 136, 134, 134, 134, 134, 133, 133, + /* 850 */ 132, 132, 132, 131, 128, 451, 1028, 1028, 1030, 1031, + /* 860 */ 35, 1204, 288, 288, 1204, 477, 288, 288, 1204, 1205, + /* 870 */ 1204, 539, 481, 437, 470, 573, 1451, 562, 364, 573, + /* 880 */ 1153, 562, 1204, 1205, 1204, 1188, 5, 576, 134, 134, + /* 890 */ 134, 134, 133, 133, 132, 132, 132, 131, 128, 451, + /* 900 */ 221, 214, 302, 96, 1149, 1657, 232, 1657, 413, 392, + /* 910 */ 19, 19, 1024, 949, 406, 373, 1595, 1085, 1204, 1205, + /* 920 */ 1204, 1204, 1205, 1204, 1204, 426, 1149, 1658, 413, 1658, + /* 930 */ 1659, 399, 137, 138, 91, 3, 1228, 1228, 1063, 1066, + /* 940 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 304, 1311, + /* 950 */ 514, 1204, 137, 138, 91, 1498, 1228, 1228, 1063, 1066, + /* 960 */ 1053, 1053, 135, 135, 136, 136, 136, 136, 434, 131, + /* 970 */ 128, 451, 375, 1204, 274, 291, 372, 517, 367, 516, + /* 980 */ 262, 1204, 1205, 1204, 1147, 227, 363, 448, 447, 1435, + /* 990 */ 1568, 1310, 134, 134, 134, 134, 133, 133, 132, 132, + /* 1000 */ 132, 131, 128, 451, 1568, 576, 1147, 487, 1204, 1205, + /* 1010 */ 1204, 442, 134, 134, 134, 134, 133, 133, 132, 132, + /* 1020 */ 132, 131, 128, 451, 386, 576, 485, 576, 19, 19, + /* 1030 */ 1204, 1205, 1204, 1345, 1236, 970, 1236, 574, 47, 936, + /* 1040 */ 936, 473, 413, 431, 1552, 573, 1125, 562, 19, 19, + /* 1050 */ 19, 19, 49, 336, 850, 851, 852, 111, 1368, 315, + /* 1060 */ 429, 576, 413, 433, 341, 306, 137, 138, 91, 115, + /* 1070 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, + /* 1080 */ 136, 136, 576, 1309, 82, 82, 137, 138, 91, 529, + /* 1090 */ 1228, 1228, 1063, 1066, 1053, 1053, 135, 135, 136, 136, + /* 1100 */ 136, 136, 1569, 222, 377, 19, 19, 305, 1126, 1169, + /* 1110 */ 398, 1148, 22, 22, 498, 333, 1569, 335, 377, 576, + /* 1120 */ 438, 445, 1169, 1127, 486, 1169, 134, 134, 134, 134, + /* 1130 */ 133, 133, 132, 132, 132, 131, 128, 451, 1128, 576, + /* 1140 */ 902, 576, 145, 145, 6, 576, 134, 134, 134, 134, + /* 1150 */ 133, 133, 132, 132, 132, 131, 128, 451, 214, 1336, + /* 1160 */ 922, 576, 19, 19, 19, 19, 1282, 419, 19, 19, + /* 1170 */ 923, 412, 515, 141, 576, 1169, 413, 206, 465, 207, + /* 1180 */ 903, 215, 1575, 552, 147, 147, 7, 227, 1169, 411, + /* 1190 */ 1250, 1169, 120, 307, 117, 307, 413, 66, 66, 334, + /* 1200 */ 137, 138, 91, 119, 1228, 1228, 1063, 1066, 1053, 1053, + /* 1210 */ 135, 135, 136, 136, 136, 136, 413, 285, 209, 969, + /* 1220 */ 137, 138, 91, 471, 1228, 1228, 1063, 1066, 1053, 1053, + /* 1230 */ 135, 135, 136, 136, 136, 136, 435, 10, 1450, 267, + /* 1240 */ 137, 126, 91, 1435, 1228, 1228, 1063, 1066, 1053, 1053, + /* 1250 */ 135, 135, 136, 136, 136, 136, 1435, 1435, 410, 409, + /* 1260 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 1270 */ 128, 451, 576, 969, 576, 1224, 498, 373, 1595, 1554, + /* 1280 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 1290 */ 128, 451, 532, 457, 576, 82, 82, 82, 82, 111, + /* 1300 */ 134, 134, 134, 134, 133, 133, 132, 132, 132, 131, + /* 1310 */ 128, 451, 109, 233, 430, 1576, 546, 67, 67, 7, + /* 1320 */ 413, 351, 550, 1550, 260, 259, 258, 494, 443, 569, + /* 1330 */ 419, 983, 446, 1224, 450, 545, 1207, 576, 969, 984, + /* 1340 */ 413, 475, 1449, 1574, 1180, 138, 91, 7, 1228, 1228, + /* 1350 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, + /* 1360 */ 21, 21, 267, 576, 300, 1126, 91, 233, 1228, 1228, + /* 1370 */ 1063, 1066, 1053, 1053, 135, 135, 136, 136, 136, 136, + /* 1380 */ 1127, 373, 1595, 161, 1573, 16, 53, 53, 7, 108, + /* 1390 */ 533, 38, 969, 125, 1207, 1128, 1180, 576, 1224, 123, + /* 1400 */ 568, 893, 4, 324, 134, 134, 134, 134, 133, 133, + /* 1410 */ 132, 132, 132, 131, 128, 451, 571, 564, 534, 576, + /* 1420 */ 68, 68, 576, 39, 134, 134, 134, 134, 133, 133, + /* 1430 */ 132, 132, 132, 131, 128, 451, 576, 160, 1571, 1223, + /* 1440 */ 452, 576, 54, 54, 576, 69, 69, 576, 1366, 576, + /* 1450 */ 420, 184, 565, 463, 297, 576, 1224, 463, 297, 70, + /* 1460 */ 70, 576, 44, 474, 71, 71, 576, 72, 72, 576, + /* 1470 */ 73, 73, 55, 55, 411, 874, 242, 576, 56, 56, + /* 1480 */ 576, 1040, 576, 478, 57, 57, 576, 121, 121, 59, + /* 1490 */ 59, 23, 60, 60, 411, 122, 319, 452, 577, 452, + /* 1500 */ 74, 74, 1028, 75, 75, 76, 76, 411, 290, 20, + /* 1510 */ 20, 108, 287, 231, 553, 123, 568, 325, 4, 320, + /* 1520 */ 567, 97, 218, 944, 1144, 328, 400, 576, 943, 576, + /* 1530 */ 1380, 424, 571, 874, 1028, 1028, 1030, 1031, 35, 293, + /* 1540 */ 534, 576, 1104, 576, 1104, 9, 576, 342, 576, 111, + /* 1550 */ 77, 77, 143, 143, 576, 205, 452, 222, 1379, 889, + /* 1560 */ 576, 901, 900, 1188, 144, 144, 78, 78, 565, 62, + /* 1570 */ 62, 79, 79, 323, 1021, 576, 266, 63, 63, 908, + /* 1580 */ 909, 1589, 542, 80, 80, 576, 371, 541, 123, 568, + /* 1590 */ 480, 4, 266, 482, 244, 266, 370, 1040, 64, 64, + /* 1600 */ 576, 466, 576, 121, 121, 571, 1557, 576, 170, 170, + /* 1610 */ 576, 122, 576, 452, 577, 452, 576, 889, 1028, 576, + /* 1620 */ 165, 576, 111, 171, 171, 87, 87, 337, 1616, 452, + /* 1630 */ 65, 65, 1530, 83, 83, 146, 146, 986, 987, 84, + /* 1640 */ 84, 565, 168, 168, 148, 148, 1092, 347, 1032, 111, + /* 1650 */ 1028, 1028, 1030, 1031, 35, 542, 1103, 576, 1103, 576, + /* 1660 */ 543, 123, 568, 504, 4, 263, 576, 361, 1529, 111, + /* 1670 */ 1040, 1088, 576, 263, 576, 490, 121, 121, 571, 1188, + /* 1680 */ 142, 142, 169, 169, 122, 576, 452, 577, 452, 162, + /* 1690 */ 162, 1028, 576, 563, 576, 152, 152, 151, 151, 348, + /* 1700 */ 1376, 974, 452, 266, 1092, 942, 1032, 125, 149, 149, + /* 1710 */ 939, 576, 125, 576, 565, 150, 150, 86, 86, 872, + /* 1720 */ 352, 159, 576, 1028, 1028, 1030, 1031, 35, 542, 941, + /* 1730 */ 576, 125, 355, 541, 88, 88, 85, 85, 357, 359, + /* 1740 */ 1324, 1308, 366, 1040, 376, 52, 52, 499, 1389, 121, + /* 1750 */ 121, 1434, 1188, 58, 58, 1362, 1374, 122, 1439, 452, + /* 1760 */ 577, 452, 1289, 167, 1028, 1280, 280, 1268, 1267, 1269, + /* 1770 */ 1609, 1359, 312, 313, 12, 314, 397, 1421, 224, 1416, + /* 1780 */ 295, 237, 1409, 339, 340, 1426, 301, 345, 484, 228, + /* 1790 */ 1371, 1307, 1372, 1370, 1425, 404, 1028, 1028, 1030, 1031, + /* 1800 */ 35, 1601, 1192, 454, 509, 369, 292, 1502, 210, 1501, + /* 1810 */ 1369, 396, 396, 395, 277, 393, 211, 566, 859, 1612, + /* 1820 */ 1244, 123, 568, 391, 4, 1188, 223, 270, 1549, 1547, + /* 1830 */ 1241, 239, 186, 327, 422, 96, 195, 220, 571, 235, + /* 1840 */ 180, 326, 188, 468, 190, 1507, 191, 192, 92, 193, + /* 1850 */ 469, 95, 1422, 13, 502, 247, 1430, 109, 199, 402, + /* 1860 */ 476, 405, 452, 1496, 1428, 1427, 14, 491, 251, 102, + /* 1870 */ 497, 1518, 241, 281, 565, 253, 203, 354, 500, 254, + /* 1880 */ 175, 1270, 407, 43, 350, 518, 1327, 436, 255, 1326, + /* 1890 */ 1325, 1318, 104, 893, 1626, 229, 408, 440, 1625, 441, + /* 1900 */ 240, 310, 1296, 1040, 311, 1317, 527, 1594, 1297, 121, + /* 1910 */ 121, 368, 1295, 1624, 268, 269, 1580, 122, 1579, 452, + /* 1920 */ 577, 452, 374, 444, 1028, 1394, 1393, 140, 553, 90, + /* 1930 */ 568, 11, 4, 1483, 383, 414, 385, 110, 116, 216, + /* 1940 */ 320, 567, 1350, 555, 42, 318, 571, 537, 1349, 389, + /* 1950 */ 390, 579, 1198, 276, 279, 278, 1028, 1028, 1030, 1031, + /* 1960 */ 35, 580, 415, 1265, 458, 1260, 416, 185, 1534, 172, + /* 1970 */ 452, 1535, 173, 156, 308, 846, 1533, 1532, 453, 217, + /* 1980 */ 225, 89, 565, 174, 322, 1188, 226, 236, 1102, 154, + /* 1990 */ 1100, 330, 176, 187, 1223, 189, 925, 338, 243, 1116, + /* 2000 */ 246, 194, 177, 178, 425, 427, 98, 99, 196, 100, + /* 2010 */ 101, 1040, 179, 1119, 248, 1115, 249, 121, 121, 24, + /* 2020 */ 163, 250, 349, 1108, 266, 122, 1238, 452, 577, 452, + /* 2030 */ 1192, 454, 1028, 200, 292, 496, 252, 201, 861, 396, + /* 2040 */ 396, 395, 277, 393, 15, 501, 859, 370, 292, 256, + /* 2050 */ 202, 554, 505, 396, 396, 395, 277, 393, 103, 239, + /* 2060 */ 859, 327, 25, 26, 1028, 1028, 1030, 1031, 35, 326, + /* 2070 */ 362, 510, 891, 239, 365, 327, 513, 904, 105, 309, + /* 2080 */ 164, 181, 27, 326, 106, 521, 107, 1185, 1069, 1155, + /* 2090 */ 17, 1154, 284, 1188, 286, 978, 265, 204, 125, 1171, + /* 2100 */ 241, 230, 972, 1175, 28, 1160, 29, 1179, 175, 1173, + /* 2110 */ 30, 43, 31, 1178, 241, 32, 41, 549, 8, 33, + /* 2120 */ 208, 111, 175, 1083, 1070, 43, 113, 1068, 240, 114, + /* 2130 */ 1072, 34, 1073, 561, 1124, 118, 271, 36, 18, 1194, + /* 2140 */ 1033, 873, 240, 935, 124, 37, 272, 273, 1617, 572, + /* 2150 */ 183, 153, 394, 1193, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2160 */ 1256, 1256, 1256, 414, 1256, 1256, 1256, 1256, 320, 567, + /* 2170 */ 1256, 1256, 1256, 1256, 1256, 1256, 1256, 414, 1256, 1256, + /* 2180 */ 1256, 1256, 320, 567, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2190 */ 1256, 1256, 458, 1256, 1256, 1256, 1256, 1256, 1256, 1256, + /* 2200 */ 1256, 1256, 1256, 1256, 1256, 1256, 458, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 19, 95, 53, 97, 22, 24, 24, 101, 27, 28, - /* 10 */ 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - /* 20 */ 39, 40, 41, 152, 43, 44, 45, 46, 47, 48, - /* 30 */ 49, 50, 51, 52, 53, 19, 55, 55, 132, 133, - /* 40 */ 134, 1, 2, 27, 28, 29, 30, 31, 32, 33, - /* 50 */ 34, 35, 36, 37, 38, 39, 40, 41, 92, 43, - /* 60 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 70 */ 47, 48, 49, 50, 51, 52, 53, 61, 97, 97, - /* 80 */ 19, 49, 50, 51, 52, 53, 70, 26, 27, 28, - /* 90 */ 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, - /* 100 */ 39, 40, 41, 152, 43, 44, 45, 46, 47, 48, - /* 110 */ 49, 50, 51, 52, 53, 144, 145, 146, 147, 19, - /* 120 */ 249, 22, 172, 172, 173, 52, 53, 27, 28, 29, - /* 130 */ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, - /* 140 */ 40, 41, 81, 43, 44, 45, 46, 47, 48, 49, - /* 150 */ 50, 51, 52, 53, 55, 56, 19, 152, 207, 208, - /* 160 */ 115, 24, 117, 118, 27, 28, 29, 30, 31, 32, - /* 170 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 79, - /* 180 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 190 */ 53, 19, 0, 1, 2, 23, 97, 98, 193, 27, - /* 200 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 210 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, - /* 220 */ 48, 49, 50, 51, 52, 53, 19, 22, 23, 172, - /* 230 */ 23, 26, 119, 120, 27, 28, 29, 30, 31, 32, - /* 240 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 187, - /* 250 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 260 */ 53, 19, 221, 222, 223, 23, 168, 169, 170, 27, - /* 270 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 280 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, - /* 290 */ 48, 49, 50, 51, 52, 53, 19, 157, 22, 23, - /* 300 */ 23, 96, 26, 172, 27, 28, 29, 30, 31, 32, - /* 310 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 187, - /* 320 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 330 */ 53, 19, 108, 109, 110, 221, 222, 223, 185, 27, - /* 340 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 350 */ 38, 39, 40, 41, 240, 43, 44, 45, 46, 47, - /* 360 */ 48, 49, 50, 51, 52, 53, 19, 227, 22, 23, - /* 370 */ 230, 22, 96, 152, 27, 28, 29, 30, 31, 32, - /* 380 */ 33, 34, 35, 36, 37, 38, 39, 40, 41, 30, - /* 390 */ 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, - /* 400 */ 53, 19, 190, 191, 55, 56, 24, 190, 191, 27, - /* 410 */ 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, - /* 420 */ 38, 39, 40, 41, 152, 43, 44, 45, 46, 47, - /* 430 */ 48, 49, 50, 51, 52, 53, 168, 169, 170, 179, - /* 440 */ 180, 171, 96, 19, 172, 173, 97, 98, 188, 179, - /* 450 */ 138, 27, 28, 29, 30, 31, 32, 33, 34, 35, - /* 460 */ 36, 37, 38, 39, 40, 41, 107, 43, 44, 45, - /* 470 */ 46, 47, 48, 49, 50, 51, 52, 53, 19, 207, - /* 480 */ 208, 30, 31, 32, 33, 138, 27, 28, 29, 30, - /* 490 */ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - /* 500 */ 41, 181, 43, 44, 45, 46, 47, 48, 49, 50, - /* 510 */ 51, 52, 53, 19, 152, 7, 8, 9, 49, 22, - /* 520 */ 19, 24, 28, 29, 30, 31, 32, 33, 34, 35, - /* 530 */ 36, 37, 38, 39, 40, 41, 152, 43, 44, 45, - /* 540 */ 46, 47, 48, 49, 50, 51, 52, 53, 19, 108, - /* 550 */ 109, 110, 101, 55, 53, 193, 172, 173, 29, 30, - /* 560 */ 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - /* 570 */ 41, 152, 43, 44, 45, 46, 47, 48, 49, 50, - /* 580 */ 51, 52, 53, 19, 20, 116, 22, 23, 169, 170, - /* 590 */ 121, 207, 85, 55, 56, 97, 19, 20, 195, 22, - /* 600 */ 99, 100, 101, 102, 103, 104, 105, 12, 152, 206, - /* 610 */ 210, 47, 48, 112, 152, 108, 109, 110, 54, 55, - /* 620 */ 56, 221, 222, 223, 47, 48, 119, 120, 172, 173, - /* 630 */ 66, 54, 55, 56, 101, 97, 98, 99, 148, 149, - /* 640 */ 102, 103, 104, 66, 154, 112, 156, 83, 229, 47, - /* 650 */ 48, 113, 57, 163, 194, 195, 92, 246, 247, 95, - /* 660 */ 83, 97, 98, 207, 208, 101, 206, 59, 73, 92, - /* 670 */ 75, 63, 95, 163, 97, 98, 194, 195, 101, 219, - /* 680 */ 85, 181, 19, 152, 175, 77, 196, 152, 206, 19, - /* 690 */ 20, 199, 22, 30, 163, 11, 132, 133, 134, 135, - /* 700 */ 136, 209, 152, 172, 173, 152, 196, 172, 173, 132, - /* 710 */ 133, 134, 135, 136, 164, 92, 213, 47, 48, 49, - /* 720 */ 92, 186, 169, 170, 54, 55, 56, 196, 100, 219, - /* 730 */ 221, 222, 223, 243, 132, 133, 66, 175, 207, 208, - /* 740 */ 152, 231, 119, 120, 19, 20, 236, 22, 152, 38, - /* 750 */ 39, 40, 41, 83, 43, 44, 45, 46, 47, 48, - /* 760 */ 49, 50, 51, 52, 53, 95, 152, 97, 98, 85, - /* 770 */ 107, 101, 47, 48, 163, 112, 92, 152, 152, 54, - /* 780 */ 55, 56, 229, 221, 222, 223, 172, 173, 163, 49, - /* 790 */ 152, 66, 108, 109, 110, 55, 56, 172, 173, 163, - /* 800 */ 186, 22, 132, 133, 134, 135, 136, 196, 83, 43, - /* 810 */ 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, - /* 820 */ 95, 196, 97, 98, 85, 99, 101, 152, 102, 103, - /* 830 */ 104, 92, 196, 146, 147, 152, 181, 97, 98, 113, - /* 840 */ 19, 20, 152, 22, 218, 22, 116, 108, 109, 110, - /* 850 */ 152, 121, 55, 56, 12, 219, 218, 132, 133, 134, - /* 860 */ 135, 136, 172, 173, 55, 56, 55, 56, 47, 48, - /* 870 */ 172, 173, 236, 152, 5, 54, 55, 56, 55, 10, - /* 880 */ 11, 12, 13, 14, 186, 152, 17, 66, 47, 48, - /* 890 */ 152, 210, 16, 84, 97, 98, 55, 56, 21, 57, - /* 900 */ 217, 122, 22, 152, 83, 152, 97, 98, 97, 98, - /* 910 */ 172, 173, 152, 152, 224, 73, 95, 75, 97, 98, - /* 920 */ 97, 124, 101, 172, 173, 164, 193, 47, 48, 60, - /* 930 */ 163, 62, 172, 173, 24, 55, 56, 186, 97, 98, - /* 940 */ 71, 100, 55, 56, 183, 207, 185, 152, 107, 23, - /* 950 */ 109, 82, 26, 132, 133, 134, 135, 136, 89, 152, - /* 960 */ 26, 92, 93, 196, 88, 55, 90, 152, 91, 152, - /* 970 */ 217, 152, 152, 132, 133, 95, 152, 97, 211, 212, - /* 980 */ 103, 101, 152, 114, 97, 98, 152, 172, 173, 19, - /* 990 */ 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, - /* 1000 */ 48, 49, 50, 51, 52, 53, 65, 97, 152, 152, - /* 1010 */ 141, 124, 132, 133, 134, 152, 140, 140, 152, 78, - /* 1020 */ 152, 233, 152, 193, 98, 152, 56, 86, 172, 173, - /* 1030 */ 166, 167, 237, 152, 217, 172, 173, 152, 172, 173, - /* 1040 */ 172, 173, 172, 173, 237, 172, 173, 152, 152, 152, - /* 1050 */ 124, 152, 111, 172, 173, 152, 237, 172, 173, 152, - /* 1060 */ 55, 152, 26, 152, 130, 152, 152, 172, 173, 172, - /* 1070 */ 173, 172, 173, 249, 152, 172, 173, 152, 61, 172, - /* 1080 */ 173, 172, 173, 172, 173, 172, 173, 70, 152, 193, - /* 1090 */ 152, 137, 152, 139, 172, 173, 152, 172, 173, 152, - /* 1100 */ 152, 152, 97, 152, 26, 152, 163, 152, 172, 173, - /* 1110 */ 172, 173, 172, 173, 152, 16, 172, 173, 26, 172, - /* 1120 */ 173, 172, 173, 172, 173, 172, 173, 172, 173, 163, - /* 1130 */ 152, 152, 152, 22, 172, 173, 152, 241, 152, 196, - /* 1140 */ 152, 193, 106, 152, 152, 152, 163, 152, 37, 152, - /* 1150 */ 172, 173, 172, 173, 152, 152, 172, 173, 172, 173, - /* 1160 */ 172, 173, 196, 172, 173, 172, 173, 172, 173, 172, - /* 1170 */ 173, 152, 152, 152, 172, 173, 98, 152, 152, 196, - /* 1180 */ 160, 22, 23, 19, 164, 193, 152, 88, 232, 90, - /* 1190 */ 191, 172, 173, 172, 173, 163, 193, 172, 173, 166, - /* 1200 */ 167, 163, 124, 163, 244, 245, 23, 211, 212, 26, - /* 1210 */ 55, 23, 55, 23, 26, 123, 26, 152, 23, 193, - /* 1220 */ 56, 26, 23, 23, 23, 26, 26, 26, 196, 100, - /* 1230 */ 101, 7, 8, 152, 196, 23, 196, 23, 26, 152, - /* 1240 */ 26, 132, 133, 23, 132, 133, 26, 152, 152, 152, - /* 1250 */ 152, 210, 97, 152, 97, 96, 152, 152, 152, 152, - /* 1260 */ 152, 210, 152, 210, 197, 152, 152, 152, 152, 152, - /* 1270 */ 152, 152, 152, 152, 152, 152, 152, 198, 150, 176, - /* 1280 */ 214, 201, 214, 238, 201, 238, 184, 214, 226, 200, - /* 1290 */ 155, 198, 67, 122, 242, 159, 159, 69, 239, 239, - /* 1300 */ 159, 175, 175, 22, 220, 180, 175, 27, 130, 18, - /* 1310 */ 159, 18, 158, 137, 220, 159, 158, 235, 74, 189, - /* 1320 */ 234, 159, 159, 158, 22, 192, 192, 177, 159, 159, - /* 1330 */ 192, 192, 189, 201, 189, 158, 107, 158, 177, 76, - /* 1340 */ 174, 201, 174, 201, 106, 159, 174, 125, 174, 182, - /* 1350 */ 177, 176, 107, 159, 174, 137, 174, 53, 174, 129, - /* 1360 */ 216, 215, 215, 182, 216, 216, 177, 215, 126, 216, - /* 1370 */ 177, 128, 25, 127, 26, 215, 13, 162, 161, 153, - /* 1380 */ 153, 6, 151, 151, 151, 151, 165, 178, 165, 178, - /* 1390 */ 4, 3, 22, 142, 15, 94, 16, 205, 120, 202, - /* 1400 */ 204, 203, 165, 201, 23, 23, 131, 111, 123, 20, - /* 1410 */ 225, 125, 16, 1, 131, 228, 111, 123, 56, 228, - /* 1420 */ 37, 37, 64, 1, 5, 122, 22, 107, 140, 80, - /* 1430 */ 80, 26, 87, 72, 107, 24, 20, 19, 112, 105, - /* 1440 */ 23, 68, 22, 79, 79, 22, 22, 22, 58, 22, - /* 1450 */ 245, 248, 248, 79, 23, 23, 23, 116, 22, 122, - /* 1460 */ 23, 22, 56, 23, 26, 23, 64, 22, 124, 26, - /* 1470 */ 26, 64, 64, 23, 23, 23, 11, 23, 22, 26, - /* 1480 */ 23, 22, 24, 1, 23, 22, 26, 250, 24, 23, - /* 1490 */ 22, 122, 23, 23, 22, 15, 23, 250, 122, 122, - /* 1500 */ 122, + /* 0 */ 276, 277, 278, 240, 241, 224, 194, 226, 194, 240, + /* 10 */ 241, 194, 216, 220, 194, 234, 253, 194, 255, 19, + /* 20 */ 224, 297, 253, 194, 255, 205, 212, 213, 205, 217, + /* 30 */ 218, 31, 205, 194, 217, 218, 194, 217, 218, 39, + /* 40 */ 217, 218, 312, 43, 44, 45, 316, 47, 48, 49, + /* 50 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 19, + /* 60 */ 240, 241, 194, 240, 241, 194, 254, 240, 241, 276, + /* 70 */ 277, 278, 233, 253, 254, 255, 253, 254, 255, 217, + /* 80 */ 253, 239, 255, 43, 44, 45, 263, 47, 48, 49, + /* 90 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 270, + /* 100 */ 286, 22, 23, 103, 104, 105, 106, 107, 108, 109, + /* 110 */ 110, 111, 112, 113, 114, 82, 47, 48, 49, 50, + /* 120 */ 186, 187, 188, 189, 190, 191, 189, 87, 191, 89, + /* 130 */ 196, 19, 198, 196, 317, 198, 319, 25, 194, 205, + /* 140 */ 298, 270, 205, 103, 104, 105, 106, 107, 108, 109, + /* 150 */ 110, 111, 112, 113, 114, 43, 44, 45, 11, 47, + /* 160 */ 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, + /* 170 */ 58, 60, 139, 140, 240, 241, 214, 240, 241, 311, + /* 180 */ 312, 102, 70, 239, 316, 194, 19, 253, 77, 255, + /* 190 */ 253, 122, 255, 55, 56, 57, 58, 59, 207, 88, + /* 200 */ 194, 90, 268, 194, 93, 268, 107, 108, 109, 110, + /* 210 */ 111, 112, 113, 114, 47, 103, 104, 105, 106, 107, + /* 220 */ 108, 109, 110, 111, 112, 113, 114, 114, 117, 118, + /* 230 */ 119, 276, 277, 278, 300, 19, 194, 300, 276, 277, + /* 240 */ 278, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 250 */ 112, 113, 114, 55, 56, 57, 58, 146, 194, 43, + /* 260 */ 44, 45, 47, 47, 48, 49, 50, 51, 52, 53, + /* 270 */ 54, 55, 56, 57, 58, 82, 129, 130, 60, 129, + /* 280 */ 130, 217, 218, 116, 68, 25, 103, 104, 105, 106, + /* 290 */ 107, 108, 109, 110, 111, 112, 113, 114, 23, 132, + /* 300 */ 294, 103, 104, 105, 106, 107, 108, 109, 110, 111, + /* 310 */ 112, 113, 114, 217, 121, 306, 194, 308, 26, 103, + /* 320 */ 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, + /* 330 */ 114, 116, 139, 140, 217, 117, 118, 119, 120, 19, + /* 340 */ 194, 123, 124, 125, 24, 109, 110, 111, 112, 113, + /* 350 */ 114, 133, 60, 311, 312, 250, 194, 252, 316, 19, + /* 360 */ 194, 166, 167, 43, 44, 45, 205, 47, 48, 49, + /* 370 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 217, + /* 380 */ 218, 317, 318, 43, 44, 45, 264, 47, 48, 49, + /* 390 */ 50, 51, 52, 53, 54, 55, 56, 57, 58, 139, + /* 400 */ 140, 240, 241, 139, 140, 188, 189, 190, 191, 117, + /* 410 */ 118, 119, 231, 196, 253, 198, 255, 194, 194, 258, + /* 420 */ 259, 146, 205, 103, 104, 105, 106, 107, 108, 109, + /* 430 */ 110, 111, 112, 113, 114, 109, 212, 213, 236, 237, + /* 440 */ 217, 218, 194, 103, 104, 105, 106, 107, 108, 109, + /* 450 */ 110, 111, 112, 113, 114, 194, 120, 240, 241, 123, + /* 460 */ 124, 125, 136, 194, 19, 74, 60, 141, 23, 133, + /* 470 */ 253, 194, 255, 82, 194, 309, 310, 254, 29, 317, + /* 480 */ 318, 194, 33, 22, 199, 268, 263, 239, 43, 44, + /* 490 */ 45, 151, 47, 48, 49, 50, 51, 52, 53, 54, + /* 500 */ 55, 56, 57, 58, 217, 218, 194, 60, 19, 146, + /* 510 */ 286, 242, 23, 107, 108, 66, 204, 300, 206, 128, + /* 520 */ 73, 60, 116, 117, 118, 119, 265, 121, 165, 60, + /* 530 */ 139, 140, 43, 44, 45, 129, 47, 48, 49, 50, + /* 540 */ 51, 52, 53, 54, 55, 56, 57, 58, 103, 104, + /* 550 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 560 */ 154, 194, 156, 194, 117, 118, 119, 280, 283, 205, + /* 570 */ 101, 220, 287, 19, 20, 306, 22, 308, 117, 118, + /* 580 */ 119, 211, 212, 213, 217, 218, 117, 118, 236, 237, + /* 590 */ 36, 122, 103, 104, 105, 106, 107, 108, 109, 110, + /* 600 */ 111, 112, 113, 114, 240, 241, 194, 60, 194, 194, + /* 610 */ 194, 194, 19, 194, 60, 194, 23, 253, 206, 255, + /* 620 */ 73, 254, 19, 154, 155, 156, 72, 276, 277, 278, + /* 630 */ 263, 217, 218, 217, 218, 271, 43, 44, 45, 271, + /* 640 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 650 */ 57, 58, 183, 194, 285, 101, 19, 214, 60, 242, + /* 660 */ 23, 107, 108, 109, 117, 118, 119, 16, 254, 115, + /* 670 */ 254, 117, 118, 119, 194, 60, 122, 263, 205, 264, + /* 680 */ 43, 44, 45, 264, 47, 48, 49, 50, 51, 52, + /* 690 */ 53, 54, 55, 56, 57, 58, 103, 104, 105, 106, + /* 700 */ 107, 108, 109, 110, 111, 112, 113, 114, 154, 155, + /* 710 */ 156, 157, 158, 240, 241, 117, 118, 119, 101, 205, + /* 720 */ 117, 19, 20, 306, 22, 308, 253, 74, 255, 78, + /* 730 */ 205, 80, 117, 118, 119, 118, 293, 183, 36, 122, + /* 740 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 750 */ 113, 114, 194, 294, 240, 241, 211, 212, 213, 144, + /* 760 */ 19, 23, 60, 25, 23, 240, 241, 253, 245, 255, + /* 770 */ 15, 154, 155, 156, 72, 217, 218, 194, 253, 256, + /* 780 */ 255, 128, 129, 130, 43, 44, 45, 22, 47, 48, + /* 790 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 800 */ 183, 19, 77, 101, 19, 128, 129, 130, 23, 107, + /* 810 */ 108, 60, 254, 88, 21, 90, 61, 115, 93, 117, + /* 820 */ 118, 119, 239, 22, 122, 60, 194, 205, 43, 44, + /* 830 */ 45, 205, 47, 48, 49, 50, 51, 52, 53, 54, + /* 840 */ 55, 56, 57, 58, 103, 104, 105, 106, 107, 108, + /* 850 */ 109, 110, 111, 112, 113, 114, 154, 155, 156, 157, + /* 860 */ 158, 60, 240, 241, 60, 116, 240, 241, 117, 118, + /* 870 */ 119, 146, 194, 19, 81, 253, 275, 255, 24, 253, + /* 880 */ 98, 255, 117, 118, 119, 183, 22, 194, 103, 104, + /* 890 */ 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, + /* 900 */ 151, 194, 270, 152, 22, 23, 194, 25, 19, 202, + /* 910 */ 217, 218, 23, 109, 207, 314, 315, 124, 117, 118, + /* 920 */ 119, 117, 118, 119, 60, 232, 22, 23, 19, 25, + /* 930 */ 303, 304, 43, 44, 45, 22, 47, 48, 49, 50, + /* 940 */ 51, 52, 53, 54, 55, 56, 57, 58, 270, 227, + /* 950 */ 96, 60, 43, 44, 45, 162, 47, 48, 49, 50, + /* 960 */ 51, 52, 53, 54, 55, 56, 57, 58, 114, 112, + /* 970 */ 113, 114, 194, 60, 120, 121, 122, 123, 124, 125, + /* 980 */ 126, 117, 118, 119, 102, 25, 132, 107, 108, 194, + /* 990 */ 194, 227, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1000 */ 111, 112, 113, 114, 194, 194, 102, 194, 117, 118, + /* 1010 */ 119, 233, 103, 104, 105, 106, 107, 108, 109, 110, + /* 1020 */ 111, 112, 113, 114, 194, 194, 19, 194, 217, 218, + /* 1030 */ 117, 118, 119, 241, 154, 144, 156, 135, 242, 137, + /* 1040 */ 138, 130, 19, 232, 194, 253, 23, 255, 217, 218, + /* 1050 */ 217, 218, 242, 16, 7, 8, 9, 25, 261, 262, + /* 1060 */ 265, 194, 19, 232, 153, 232, 43, 44, 45, 160, + /* 1070 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1080 */ 57, 58, 194, 227, 217, 218, 43, 44, 45, 194, + /* 1090 */ 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, + /* 1100 */ 57, 58, 306, 143, 308, 217, 218, 294, 12, 77, + /* 1110 */ 22, 23, 217, 218, 194, 78, 306, 80, 308, 194, + /* 1120 */ 232, 254, 90, 27, 117, 93, 103, 104, 105, 106, + /* 1130 */ 107, 108, 109, 110, 111, 112, 113, 114, 42, 194, + /* 1140 */ 35, 194, 217, 218, 214, 194, 103, 104, 105, 106, + /* 1150 */ 107, 108, 109, 110, 111, 112, 113, 114, 194, 239, + /* 1160 */ 64, 194, 217, 218, 217, 218, 209, 210, 217, 218, + /* 1170 */ 74, 207, 67, 22, 194, 77, 19, 232, 245, 232, + /* 1180 */ 75, 24, 312, 232, 217, 218, 316, 25, 90, 256, + /* 1190 */ 102, 93, 159, 229, 161, 231, 19, 217, 218, 162, + /* 1200 */ 43, 44, 45, 160, 47, 48, 49, 50, 51, 52, + /* 1210 */ 53, 54, 55, 56, 57, 58, 19, 23, 288, 25, + /* 1220 */ 43, 44, 45, 293, 47, 48, 49, 50, 51, 52, + /* 1230 */ 53, 54, 55, 56, 57, 58, 131, 22, 275, 24, + /* 1240 */ 43, 44, 45, 194, 47, 48, 49, 50, 51, 52, + /* 1250 */ 53, 54, 55, 56, 57, 58, 194, 194, 107, 108, + /* 1260 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1270 */ 113, 114, 194, 25, 194, 60, 194, 314, 315, 194, + /* 1280 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1290 */ 113, 114, 19, 194, 194, 217, 218, 217, 218, 25, + /* 1300 */ 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, + /* 1310 */ 113, 114, 150, 119, 265, 312, 67, 217, 218, 316, + /* 1320 */ 19, 239, 194, 194, 128, 129, 130, 265, 265, 209, + /* 1330 */ 210, 31, 254, 118, 254, 86, 60, 194, 144, 39, + /* 1340 */ 19, 130, 275, 312, 95, 44, 45, 316, 47, 48, + /* 1350 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 1360 */ 217, 218, 24, 194, 153, 12, 45, 119, 47, 48, + /* 1370 */ 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, + /* 1380 */ 27, 314, 315, 22, 312, 24, 217, 218, 316, 116, + /* 1390 */ 117, 22, 144, 25, 118, 42, 147, 194, 60, 19, + /* 1400 */ 20, 127, 22, 194, 103, 104, 105, 106, 107, 108, + /* 1410 */ 109, 110, 111, 112, 113, 114, 36, 64, 145, 194, + /* 1420 */ 217, 218, 194, 54, 103, 104, 105, 106, 107, 108, + /* 1430 */ 109, 110, 111, 112, 113, 114, 194, 22, 310, 25, + /* 1440 */ 60, 194, 217, 218, 194, 217, 218, 194, 260, 194, + /* 1450 */ 301, 302, 72, 262, 262, 194, 118, 266, 266, 217, + /* 1460 */ 218, 194, 82, 245, 217, 218, 194, 217, 218, 194, + /* 1470 */ 217, 218, 217, 218, 256, 60, 24, 194, 217, 218, + /* 1480 */ 194, 101, 194, 245, 217, 218, 194, 107, 108, 217, + /* 1490 */ 218, 22, 217, 218, 256, 115, 245, 117, 118, 119, + /* 1500 */ 217, 218, 122, 217, 218, 217, 218, 256, 22, 217, + /* 1510 */ 218, 116, 258, 259, 146, 19, 20, 194, 22, 139, + /* 1520 */ 140, 150, 151, 136, 23, 194, 25, 194, 141, 194, + /* 1530 */ 194, 62, 36, 118, 154, 155, 156, 157, 158, 100, + /* 1540 */ 145, 194, 154, 194, 156, 49, 194, 23, 194, 25, + /* 1550 */ 217, 218, 217, 218, 194, 257, 60, 143, 194, 60, + /* 1560 */ 194, 121, 122, 183, 217, 218, 217, 218, 72, 217, + /* 1570 */ 218, 217, 218, 134, 23, 194, 25, 217, 218, 7, + /* 1580 */ 8, 321, 86, 217, 218, 194, 122, 91, 19, 20, + /* 1590 */ 23, 22, 25, 23, 142, 25, 132, 101, 217, 218, + /* 1600 */ 194, 194, 194, 107, 108, 36, 194, 194, 217, 218, + /* 1610 */ 194, 115, 194, 117, 118, 119, 194, 118, 122, 194, + /* 1620 */ 23, 194, 25, 217, 218, 217, 218, 194, 142, 60, + /* 1630 */ 217, 218, 194, 217, 218, 217, 218, 84, 85, 217, + /* 1640 */ 218, 72, 217, 218, 217, 218, 60, 23, 60, 25, + /* 1650 */ 154, 155, 156, 157, 158, 86, 154, 194, 156, 194, + /* 1660 */ 91, 19, 20, 23, 22, 25, 194, 23, 194, 25, + /* 1670 */ 101, 23, 194, 25, 194, 194, 107, 108, 36, 183, + /* 1680 */ 217, 218, 217, 218, 115, 194, 117, 118, 119, 217, + /* 1690 */ 218, 122, 194, 237, 194, 217, 218, 217, 218, 194, + /* 1700 */ 194, 23, 60, 25, 118, 23, 118, 25, 217, 218, + /* 1710 */ 23, 194, 25, 194, 72, 217, 218, 217, 218, 23, + /* 1720 */ 194, 25, 194, 154, 155, 156, 157, 158, 86, 23, + /* 1730 */ 194, 25, 194, 91, 217, 218, 217, 218, 194, 194, + /* 1740 */ 194, 194, 194, 101, 194, 217, 218, 290, 194, 107, + /* 1750 */ 108, 194, 183, 217, 218, 194, 194, 115, 194, 117, + /* 1760 */ 118, 119, 194, 243, 122, 194, 289, 194, 194, 194, + /* 1770 */ 194, 257, 257, 257, 244, 257, 192, 273, 215, 269, + /* 1780 */ 246, 299, 269, 295, 247, 273, 247, 246, 295, 230, + /* 1790 */ 261, 226, 261, 261, 273, 273, 154, 155, 156, 157, + /* 1800 */ 158, 0, 1, 2, 221, 220, 5, 220, 250, 220, + /* 1810 */ 261, 10, 11, 12, 13, 14, 250, 282, 17, 197, + /* 1820 */ 61, 19, 20, 246, 22, 183, 244, 142, 201, 201, + /* 1830 */ 38, 30, 299, 32, 201, 152, 22, 151, 36, 299, + /* 1840 */ 43, 40, 235, 18, 238, 285, 238, 238, 296, 238, + /* 1850 */ 201, 296, 274, 272, 18, 200, 235, 150, 235, 247, + /* 1860 */ 247, 247, 60, 247, 274, 274, 272, 201, 200, 159, + /* 1870 */ 63, 292, 71, 201, 72, 200, 22, 201, 222, 200, + /* 1880 */ 79, 201, 222, 82, 291, 116, 219, 65, 200, 219, + /* 1890 */ 219, 228, 22, 127, 225, 166, 222, 24, 225, 114, + /* 1900 */ 99, 284, 221, 101, 284, 228, 307, 315, 219, 107, + /* 1910 */ 108, 219, 219, 219, 201, 92, 320, 115, 320, 117, + /* 1920 */ 118, 119, 222, 83, 122, 267, 267, 149, 146, 19, + /* 1930 */ 20, 22, 22, 279, 250, 134, 201, 148, 159, 249, + /* 1940 */ 139, 140, 251, 141, 25, 281, 36, 147, 251, 248, + /* 1950 */ 247, 203, 13, 195, 6, 195, 154, 155, 156, 157, + /* 1960 */ 158, 193, 305, 193, 163, 193, 305, 302, 214, 208, + /* 1970 */ 60, 214, 208, 223, 223, 4, 214, 214, 3, 22, + /* 1980 */ 215, 214, 72, 208, 164, 183, 215, 15, 23, 16, + /* 1990 */ 23, 140, 131, 152, 25, 143, 20, 16, 24, 1, + /* 2000 */ 145, 143, 131, 131, 62, 37, 54, 54, 152, 54, + /* 2010 */ 54, 101, 131, 117, 34, 1, 142, 107, 108, 22, + /* 2020 */ 5, 116, 162, 69, 25, 115, 76, 117, 118, 119, + /* 2030 */ 1, 2, 122, 69, 5, 41, 142, 116, 20, 10, + /* 2040 */ 11, 12, 13, 14, 24, 19, 17, 132, 5, 126, + /* 2050 */ 22, 141, 68, 10, 11, 12, 13, 14, 22, 30, + /* 2060 */ 17, 32, 22, 22, 154, 155, 156, 157, 158, 40, + /* 2070 */ 23, 68, 60, 30, 24, 32, 97, 28, 22, 68, + /* 2080 */ 23, 37, 34, 40, 150, 22, 25, 23, 23, 23, + /* 2090 */ 22, 98, 23, 183, 23, 117, 34, 22, 25, 89, + /* 2100 */ 71, 142, 144, 76, 34, 23, 34, 76, 79, 87, + /* 2110 */ 34, 82, 34, 94, 71, 34, 22, 24, 44, 34, + /* 2120 */ 25, 25, 79, 23, 23, 82, 143, 23, 99, 143, + /* 2130 */ 23, 22, 11, 25, 23, 25, 22, 22, 22, 1, + /* 2140 */ 23, 23, 99, 136, 22, 22, 142, 142, 142, 25, + /* 2150 */ 25, 23, 15, 1, 322, 322, 322, 322, 322, 322, + /* 2160 */ 322, 322, 322, 134, 322, 322, 322, 322, 139, 140, + /* 2170 */ 322, 322, 322, 322, 322, 322, 322, 134, 322, 322, + /* 2180 */ 322, 322, 139, 140, 322, 322, 322, 322, 322, 322, + /* 2190 */ 322, 322, 163, 322, 322, 322, 322, 322, 322, 322, + /* 2200 */ 322, 322, 322, 322, 322, 322, 163, 322, 322, 322, + /* 2210 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2220 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2230 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2240 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2250 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2260 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2270 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2280 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2290 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2300 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2310 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2320 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2330 */ 322, 322, 322, 322, 322, 322, 322, 322, 322, 322, + /* 2340 */ 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + /* 2350 */ 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + /* 2360 */ 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + /* 2370 */ 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + /* 2380 */ 186, 186, 186, 186, 186, 186, 186, 186, 186, 186, + /* 2390 */ 186, 186, 186, }; -#define YY_SHIFT_USE_DFLT (-95) -#define YY_SHIFT_COUNT (439) -#define YY_SHIFT_MIN (-94) -#define YY_SHIFT_MAX (1482) -static const short yy_shift_ofst[] = { - /* 0 */ 40, 564, 869, 577, 725, 725, 725, 739, -19, 16, - /* 10 */ 16, 100, 725, 725, 725, 725, 725, 725, 725, 841, - /* 20 */ 841, 538, 507, 684, 623, 61, 137, 172, 207, 242, - /* 30 */ 277, 312, 347, 382, 424, 424, 424, 424, 424, 424, - /* 40 */ 424, 424, 424, 424, 424, 424, 424, 424, 424, 459, - /* 50 */ 424, 494, 529, 529, 670, 725, 725, 725, 725, 725, - /* 60 */ 725, 725, 725, 725, 725, 725, 725, 725, 725, 725, - /* 70 */ 725, 725, 725, 725, 725, 725, 725, 725, 725, 725, - /* 80 */ 725, 725, 725, 821, 725, 725, 725, 725, 725, 725, - /* 90 */ 725, 725, 725, 725, 725, 725, 725, 952, 711, 711, - /* 100 */ 711, 711, 711, 766, 23, 32, 811, 877, 663, 602, - /* 110 */ 602, 811, 73, 113, -51, -95, -95, -95, 501, 501, - /* 120 */ 501, 595, 595, 809, 205, 276, 811, 811, 811, 811, - /* 130 */ 811, 811, 811, 811, 811, 811, 811, 811, 811, 811, - /* 140 */ 811, 811, 811, 811, 811, 811, 192, 628, 498, 498, - /* 150 */ 113, -34, -34, -34, -34, -34, -34, -95, -95, -95, - /* 160 */ 880, -94, -94, 726, 740, 99, 797, 887, 349, 811, - /* 170 */ 811, 811, 811, 811, 811, 811, 811, 811, 811, 811, - /* 180 */ 811, 811, 811, 811, 811, 811, 941, 941, 941, 811, - /* 190 */ 811, 926, 811, 811, 811, -18, 811, 811, 842, 811, - /* 200 */ 811, 811, 811, 811, 811, 811, 811, 811, 811, 224, - /* 210 */ 608, 910, 910, 910, 1078, 45, 469, 508, 934, 970, - /* 220 */ 970, 1164, 934, 1164, 1036, 1183, 359, 1017, 970, 954, - /* 230 */ 1017, 1017, 1092, 730, 497, 1225, 1171, 1171, 1228, 1228, - /* 240 */ 1171, 1281, 1280, 1178, 1291, 1291, 1291, 1291, 1171, 1293, - /* 250 */ 1178, 1281, 1280, 1280, 1178, 1171, 1293, 1176, 1244, 1171, - /* 260 */ 1171, 1293, 1302, 1171, 1293, 1171, 1293, 1302, 1229, 1229, - /* 270 */ 1229, 1263, 1302, 1229, 1238, 1229, 1263, 1229, 1229, 1222, - /* 280 */ 1245, 1222, 1245, 1222, 1245, 1222, 1245, 1171, 1171, 1218, - /* 290 */ 1302, 1304, 1304, 1302, 1230, 1242, 1243, 1246, 1178, 1347, - /* 300 */ 1348, 1363, 1363, 1375, 1375, 1375, 1375, -95, -95, -95, - /* 310 */ -95, -95, -95, -95, -95, 451, 876, 346, 1159, 1099, - /* 320 */ 441, 823, 1188, 1111, 1190, 1195, 1199, 1200, 1005, 1129, - /* 330 */ 1224, 533, 1201, 1212, 1155, 1214, 1109, 1112, 1220, 1157, - /* 340 */ 779, 1386, 1388, 1370, 1251, 1379, 1301, 1380, 1381, 1382, - /* 350 */ 1278, 1275, 1296, 1285, 1389, 1286, 1396, 1412, 1294, 1283, - /* 360 */ 1383, 1384, 1305, 1362, 1358, 1303, 1422, 1419, 1404, 1320, - /* 370 */ 1288, 1349, 1405, 1350, 1345, 1361, 1327, 1411, 1416, 1418, - /* 380 */ 1326, 1334, 1420, 1364, 1423, 1424, 1417, 1425, 1365, 1390, - /* 390 */ 1427, 1374, 1373, 1431, 1432, 1433, 1341, 1436, 1437, 1439, - /* 400 */ 1438, 1337, 1440, 1442, 1406, 1402, 1445, 1344, 1443, 1407, - /* 410 */ 1444, 1408, 1443, 1450, 1451, 1452, 1453, 1454, 1456, 1465, - /* 420 */ 1457, 1459, 1458, 1460, 1461, 1463, 1464, 1460, 1466, 1468, - /* 430 */ 1469, 1470, 1472, 1369, 1376, 1377, 1378, 1473, 1480, 1482, +#define YY_SHIFT_COUNT (582) +#define YY_SHIFT_MIN (0) +#define YY_SHIFT_MAX (2152) +static const unsigned short int yy_shift_ofst[] = { + /* 0 */ 2029, 1801, 2043, 1380, 1380, 33, 391, 1496, 1569, 1642, + /* 10 */ 702, 702, 702, 193, 33, 33, 33, 33, 33, 0, + /* 20 */ 0, 216, 1177, 702, 702, 702, 702, 702, 702, 702, + /* 30 */ 702, 702, 702, 702, 702, 702, 702, 702, 406, 406, + /* 40 */ 111, 111, 218, 447, 547, 598, 598, 260, 260, 260, + /* 50 */ 260, 40, 112, 320, 340, 445, 489, 593, 637, 741, + /* 60 */ 785, 889, 909, 1023, 1043, 1157, 1177, 1177, 1177, 1177, + /* 70 */ 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, 1177, + /* 80 */ 1177, 1177, 1177, 1177, 1197, 1177, 1301, 1321, 1321, 554, + /* 90 */ 1802, 1910, 702, 702, 702, 702, 702, 702, 702, 702, + /* 100 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 110 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 120 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 130 */ 702, 702, 702, 702, 702, 702, 702, 702, 702, 702, + /* 140 */ 702, 702, 138, 198, 198, 198, 198, 198, 198, 198, + /* 150 */ 183, 99, 236, 292, 598, 793, 167, 598, 598, 880, + /* 160 */ 880, 598, 857, 150, 195, 195, 195, 264, 113, 113, + /* 170 */ 2207, 2207, 854, 854, 854, 751, 765, 765, 765, 765, + /* 180 */ 1096, 1096, 725, 292, 882, 904, 598, 598, 598, 598, + /* 190 */ 598, 598, 598, 598, 598, 598, 598, 598, 598, 598, + /* 200 */ 598, 598, 598, 598, 598, 1273, 1032, 1032, 598, 147, + /* 210 */ 1098, 1098, 603, 603, 1276, 1276, 363, 2207, 2207, 2207, + /* 220 */ 2207, 2207, 2207, 2207, 469, 617, 617, 801, 336, 461, + /* 230 */ 804, 864, 615, 891, 913, 598, 598, 598, 598, 598, + /* 240 */ 598, 598, 598, 598, 598, 653, 598, 598, 598, 598, + /* 250 */ 598, 598, 598, 598, 598, 598, 598, 598, 1105, 1105, + /* 260 */ 1105, 598, 598, 598, 1194, 598, 598, 598, 1215, 1249, + /* 270 */ 598, 1353, 598, 598, 598, 598, 598, 598, 598, 598, + /* 280 */ 677, 449, 902, 1338, 1338, 1338, 1338, 1248, 902, 902, + /* 290 */ 326, 1151, 1047, 755, 749, 1371, 960, 1371, 1007, 1162, + /* 300 */ 749, 749, 1162, 749, 960, 1007, 1274, 738, 215, 1300, + /* 310 */ 1300, 1300, 1395, 1395, 1395, 1395, 1368, 1368, 1033, 1414, + /* 320 */ 1387, 1361, 1759, 1759, 1685, 1685, 1792, 1792, 1685, 1683, + /* 330 */ 1686, 1814, 1797, 1825, 1825, 1825, 1825, 1685, 1836, 1707, + /* 340 */ 1686, 1686, 1707, 1814, 1797, 1707, 1797, 1707, 1685, 1836, + /* 350 */ 1710, 1807, 1685, 1836, 1854, 1685, 1836, 1685, 1836, 1854, + /* 360 */ 1769, 1769, 1769, 1822, 1870, 1870, 1854, 1769, 1766, 1769, + /* 370 */ 1822, 1769, 1769, 1729, 1873, 1785, 1785, 1854, 1685, 1823, + /* 380 */ 1823, 1840, 1840, 1778, 1782, 1909, 1685, 1779, 1778, 1789, + /* 390 */ 1800, 1707, 1919, 1939, 1939, 1948, 1948, 1948, 2207, 2207, + /* 400 */ 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, 2207, + /* 410 */ 2207, 2207, 2207, 69, 1037, 79, 1088, 651, 1196, 1415, + /* 420 */ 1501, 1439, 1369, 1452, 911, 1211, 1524, 1469, 1551, 1567, + /* 430 */ 1570, 1624, 1640, 1644, 1499, 1440, 1572, 1464, 1597, 275, + /* 440 */ 782, 1586, 1648, 1678, 1553, 1682, 1687, 1388, 1502, 1696, + /* 450 */ 1706, 1588, 1486, 1971, 1975, 1957, 1820, 1972, 1973, 1965, + /* 460 */ 1967, 1851, 1841, 1861, 1969, 1969, 1974, 1852, 1976, 1855, + /* 470 */ 1981, 1998, 1858, 1871, 1969, 1872, 1942, 1968, 1969, 1856, + /* 480 */ 1952, 1953, 1955, 1956, 1881, 1896, 1980, 1874, 2014, 2015, + /* 490 */ 1997, 1905, 1860, 1954, 1999, 1964, 1950, 1994, 1894, 1921, + /* 500 */ 2020, 2018, 2026, 1915, 1923, 2028, 1984, 2036, 2040, 2047, + /* 510 */ 2041, 2003, 2012, 2050, 1979, 2049, 2056, 2011, 2044, 2057, + /* 520 */ 2048, 1934, 2063, 2064, 2065, 2061, 2066, 2068, 1993, 1959, + /* 530 */ 2069, 2071, 1978, 2062, 2075, 1958, 2073, 2070, 2072, 2076, + /* 540 */ 2078, 2010, 2027, 2022, 2074, 2031, 2019, 2081, 2082, 2094, + /* 550 */ 2093, 2095, 2096, 2085, 1983, 1986, 2100, 2073, 2101, 2104, + /* 560 */ 2107, 2109, 2108, 2110, 2111, 2114, 2121, 2115, 2116, 2117, + /* 570 */ 2118, 2122, 2123, 2124, 2007, 2004, 2005, 2006, 2125, 2128, + /* 580 */ 2137, 2138, 2152, }; -#define YY_REDUCE_USE_DFLT (-130) -#define YY_REDUCE_COUNT (314) -#define YY_REDUCE_MIN (-129) -#define YY_REDUCE_MAX (1237) +#define YY_REDUCE_COUNT (412) +#define YY_REDUCE_MIN (-276) +#define YY_REDUCE_MAX (1775) static const short yy_reduce_ofst[] = { - /* 0 */ -29, 531, 490, 625, -49, 272, 456, 510, 400, 509, - /* 10 */ 562, 114, 535, 614, 698, 384, 738, 751, 690, 419, - /* 20 */ 553, 761, 460, 636, 767, 41, 41, 41, 41, 41, - /* 30 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - /* 40 */ 41, 41, 41, 41, 41, 41, 41, 41, 41, 41, - /* 50 */ 41, 41, 41, 41, 760, 815, 856, 863, 866, 868, - /* 60 */ 870, 873, 881, 885, 895, 897, 899, 903, 907, 909, - /* 70 */ 911, 913, 922, 925, 936, 938, 940, 944, 947, 949, - /* 80 */ 951, 953, 955, 962, 978, 980, 984, 986, 988, 991, - /* 90 */ 993, 995, 997, 1002, 1019, 1021, 1025, 41, 41, 41, - /* 100 */ 41, 41, 41, 41, 41, 41, 896, 140, 260, 98, - /* 110 */ 268, 1020, 41, 482, 41, 41, 41, 41, 270, 270, - /* 120 */ 270, 212, 217, -129, 411, 411, 550, 5, 626, 362, - /* 130 */ 733, 830, 992, 1003, 1026, 795, 683, 807, 638, 819, - /* 140 */ 753, 948, 62, 817, 824, 132, 687, 611, 864, 1033, - /* 150 */ 403, 943, 966, 983, 1032, 1038, 1040, 960, 996, 492, - /* 160 */ -50, 57, 131, 153, 221, 462, 588, 596, 675, 721, - /* 170 */ 820, 834, 857, 914, 979, 1034, 1065, 1081, 1087, 1095, - /* 180 */ 1096, 1097, 1098, 1101, 1104, 1105, 320, 500, 655, 1106, - /* 190 */ 1107, 503, 1108, 1110, 1113, 681, 1114, 1115, 999, 1116, - /* 200 */ 1117, 1118, 221, 1119, 1120, 1121, 1122, 1123, 1124, 788, - /* 210 */ 956, 1041, 1051, 1053, 503, 1067, 1079, 1128, 1080, 1066, - /* 220 */ 1068, 1045, 1083, 1047, 1103, 1102, 1125, 1126, 1073, 1062, - /* 230 */ 1127, 1131, 1089, 1093, 1135, 1052, 1136, 1137, 1059, 1060, - /* 240 */ 1141, 1084, 1130, 1132, 1133, 1134, 1138, 1139, 1151, 1154, - /* 250 */ 1140, 1094, 1143, 1145, 1142, 1156, 1158, 1082, 1086, 1162, - /* 260 */ 1163, 1165, 1150, 1169, 1177, 1170, 1179, 1161, 1166, 1168, - /* 270 */ 1172, 1167, 1173, 1174, 1175, 1180, 1181, 1182, 1184, 1144, - /* 280 */ 1146, 1148, 1147, 1149, 1152, 1153, 1160, 1186, 1194, 1185, - /* 290 */ 1189, 1187, 1191, 1193, 1192, 1196, 1198, 1197, 1202, 1215, - /* 300 */ 1217, 1226, 1227, 1231, 1232, 1233, 1234, 1203, 1204, 1205, - /* 310 */ 1221, 1223, 1209, 1211, 1237, + /* 0 */ -66, 217, -63, -177, -180, 161, 364, 64, -183, 162, + /* 10 */ 223, 367, 414, -173, 473, 514, 525, 622, 626, -207, + /* 20 */ 351, -276, -38, 693, 811, 831, 833, 888, -188, 945, + /* 30 */ 947, 416, 558, 951, 867, 287, 1078, 1080, -186, 224, + /* 40 */ -132, 42, 964, 269, 417, 796, 810, -237, -231, -237, + /* 50 */ -231, -45, -45, -45, -45, -45, -45, -45, -45, -45, + /* 60 */ -45, -45, -45, -45, -45, -45, -45, -45, -45, -45, + /* 70 */ -45, -45, -45, -45, -45, -45, -45, -45, -45, -45, + /* 80 */ -45, -45, -45, -45, -45, -45, -45, -45, -45, 895, + /* 90 */ 925, 967, 980, 1100, 1143, 1169, 1203, 1225, 1228, 1242, + /* 100 */ 1247, 1250, 1253, 1255, 1261, 1267, 1272, 1275, 1283, 1286, + /* 110 */ 1288, 1292, 1333, 1335, 1347, 1349, 1352, 1354, 1360, 1366, + /* 120 */ 1381, 1391, 1406, 1408, 1413, 1416, 1418, 1422, 1425, 1427, + /* 130 */ 1463, 1465, 1472, 1478, 1480, 1491, 1498, 1500, 1517, 1519, + /* 140 */ 1528, 1536, -45, -45, -45, -45, -45, -45, -45, -45, + /* 150 */ -45, -45, -45, 312, -158, 285, -219, 9, 166, 370, + /* 160 */ 545, 707, -45, 930, 601, 963, 1067, 792, -45, -45, + /* 170 */ -45, -45, -204, -204, -204, 369, -171, -129, 632, 678, + /* 180 */ 202, 352, -270, 412, 627, 627, -9, 122, 415, 419, + /* 190 */ -56, 248, 583, 920, 6, 261, 459, 795, 1049, 813, + /* 200 */ 1062, 1082, -161, 778, 1063, 797, 870, 1003, 1128, 443, + /* 210 */ 1031, 1072, 1191, 1192, 957, 1120, 105, 1149, 523, 933, + /* 220 */ 1218, 1238, 1254, 1251, -138, 96, 117, 146, 181, 277, + /* 230 */ 280, 421, 480, 712, 830, 850, 1085, 1099, 1129, 1209, + /* 240 */ 1323, 1331, 1336, 1364, 1407, 368, 1412, 1433, 1438, 1474, + /* 250 */ 1481, 1505, 1506, 1526, 1538, 1544, 1545, 1546, 722, 764, + /* 260 */ 856, 1547, 1548, 1550, 1188, 1554, 1557, 1561, 1298, 1260, + /* 270 */ 1562, 1456, 1564, 280, 1568, 1571, 1573, 1574, 1575, 1576, + /* 280 */ 1457, 1477, 1520, 1514, 1515, 1516, 1518, 1188, 1520, 1520, + /* 290 */ 1530, 1563, 1584, 1482, 1504, 1510, 1534, 1513, 1488, 1537, + /* 300 */ 1512, 1521, 1539, 1522, 1541, 1493, 1583, 1559, 1565, 1585, + /* 310 */ 1587, 1589, 1529, 1531, 1532, 1549, 1558, 1566, 1535, 1577, + /* 320 */ 1582, 1622, 1533, 1540, 1627, 1628, 1552, 1555, 1633, 1560, + /* 330 */ 1578, 1581, 1607, 1606, 1608, 1609, 1611, 1649, 1655, 1612, + /* 340 */ 1590, 1591, 1613, 1594, 1621, 1614, 1623, 1616, 1666, 1668, + /* 350 */ 1579, 1593, 1672, 1675, 1656, 1676, 1679, 1680, 1688, 1660, + /* 360 */ 1667, 1670, 1671, 1663, 1669, 1673, 1674, 1689, 1681, 1692, + /* 370 */ 1677, 1693, 1694, 1592, 1599, 1617, 1620, 1700, 1713, 1596, + /* 380 */ 1598, 1658, 1659, 1691, 1684, 1654, 1735, 1664, 1697, 1690, + /* 390 */ 1701, 1703, 1748, 1758, 1760, 1768, 1770, 1772, 1657, 1661, + /* 400 */ 1665, 1761, 1754, 1757, 1762, 1763, 1764, 1750, 1751, 1765, + /* 410 */ 1771, 1767, 1775, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1250, 1240, 1240, 1240, 1174, 1174, 1174, 1240, 1071, 1100, - /* 10 */ 1100, 1224, 1301, 1301, 1301, 1301, 1301, 1301, 1173, 1301, - /* 20 */ 1301, 1301, 1301, 1240, 1075, 1106, 1301, 1301, 1301, 1301, - /* 30 */ 1301, 1301, 1301, 1301, 1223, 1225, 1114, 1113, 1206, 1087, - /* 40 */ 1111, 1104, 1108, 1175, 1169, 1170, 1168, 1172, 1176, 1301, - /* 50 */ 1107, 1138, 1153, 1137, 1301, 1301, 1301, 1301, 1301, 1301, - /* 60 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 70 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 80 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 90 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1147, 1152, 1159, - /* 100 */ 1151, 1148, 1140, 1139, 1141, 1142, 1301, 994, 1042, 1301, - /* 110 */ 1301, 1301, 1143, 1301, 1144, 1156, 1155, 1154, 1231, 1258, - /* 120 */ 1257, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 130 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 140 */ 1301, 1301, 1301, 1301, 1301, 1301, 1250, 1240, 1000, 1000, - /* 150 */ 1301, 1240, 1240, 1240, 1240, 1240, 1240, 1236, 1075, 1066, - /* 160 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 170 */ 1228, 1226, 1301, 1187, 1301, 1301, 1301, 1301, 1301, 1301, - /* 180 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 190 */ 1301, 1301, 1301, 1301, 1301, 1071, 1301, 1301, 1301, 1301, - /* 200 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1252, 1301, - /* 210 */ 1201, 1071, 1071, 1071, 1073, 1055, 1065, 979, 1110, 1089, - /* 220 */ 1089, 1290, 1110, 1290, 1017, 1272, 1014, 1100, 1089, 1171, - /* 230 */ 1100, 1100, 1072, 1065, 1301, 1293, 1080, 1080, 1292, 1292, - /* 240 */ 1080, 1119, 1045, 1110, 1051, 1051, 1051, 1051, 1080, 991, - /* 250 */ 1110, 1119, 1045, 1045, 1110, 1080, 991, 1205, 1287, 1080, - /* 260 */ 1080, 991, 1180, 1080, 991, 1080, 991, 1180, 1043, 1043, - /* 270 */ 1043, 1032, 1180, 1043, 1017, 1043, 1032, 1043, 1043, 1093, - /* 280 */ 1088, 1093, 1088, 1093, 1088, 1093, 1088, 1080, 1080, 1301, - /* 290 */ 1180, 1184, 1184, 1180, 1105, 1094, 1103, 1101, 1110, 997, - /* 300 */ 1035, 1255, 1255, 1251, 1251, 1251, 1251, 1298, 1298, 1236, - /* 310 */ 1267, 1267, 1019, 1019, 1267, 1301, 1301, 1301, 1301, 1301, - /* 320 */ 1301, 1262, 1301, 1189, 1301, 1301, 1301, 1301, 1301, 1301, - /* 330 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 340 */ 1125, 1301, 975, 1233, 1301, 1301, 1232, 1301, 1301, 1301, - /* 350 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 360 */ 1301, 1301, 1301, 1301, 1301, 1289, 1301, 1301, 1301, 1301, - /* 370 */ 1301, 1301, 1204, 1203, 1301, 1301, 1301, 1301, 1301, 1301, - /* 380 */ 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 390 */ 1301, 1301, 1301, 1301, 1301, 1301, 1057, 1301, 1301, 1301, - /* 400 */ 1276, 1301, 1301, 1301, 1301, 1301, 1301, 1301, 1102, 1301, - /* 410 */ 1095, 1301, 1280, 1301, 1301, 1301, 1301, 1301, 1301, 1301, - /* 420 */ 1301, 1301, 1301, 1242, 1301, 1301, 1301, 1241, 1301, 1301, - /* 430 */ 1301, 1301, 1301, 1127, 1301, 1126, 1130, 1301, 985, 1301, + /* 0 */ 1663, 1663, 1663, 1491, 1254, 1367, 1254, 1254, 1254, 1254, + /* 10 */ 1491, 1491, 1491, 1254, 1254, 1254, 1254, 1254, 1254, 1397, + /* 20 */ 1397, 1544, 1287, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 30 */ 1254, 1254, 1254, 1254, 1254, 1490, 1254, 1254, 1254, 1254, + /* 40 */ 1578, 1578, 1254, 1254, 1254, 1254, 1254, 1563, 1562, 1254, + /* 50 */ 1254, 1254, 1406, 1254, 1413, 1254, 1254, 1254, 1254, 1254, + /* 60 */ 1492, 1493, 1254, 1254, 1254, 1254, 1543, 1545, 1508, 1420, + /* 70 */ 1419, 1418, 1417, 1526, 1385, 1411, 1404, 1408, 1487, 1488, + /* 80 */ 1486, 1641, 1493, 1492, 1254, 1407, 1455, 1471, 1454, 1254, + /* 90 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 100 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 110 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 120 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 130 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 140 */ 1254, 1254, 1463, 1470, 1469, 1468, 1477, 1467, 1464, 1457, + /* 150 */ 1456, 1458, 1459, 1278, 1254, 1275, 1329, 1254, 1254, 1254, + /* 160 */ 1254, 1254, 1460, 1287, 1448, 1447, 1446, 1254, 1474, 1461, + /* 170 */ 1473, 1472, 1551, 1615, 1614, 1509, 1254, 1254, 1254, 1254, + /* 180 */ 1254, 1254, 1578, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 190 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 200 */ 1254, 1254, 1254, 1254, 1254, 1387, 1578, 1578, 1254, 1287, + /* 210 */ 1578, 1578, 1388, 1388, 1283, 1283, 1391, 1558, 1358, 1358, + /* 220 */ 1358, 1358, 1367, 1358, 1254, 1254, 1254, 1254, 1254, 1254, + /* 230 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1548, + /* 240 */ 1546, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 250 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 260 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1363, 1254, + /* 270 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1608, + /* 280 */ 1254, 1521, 1343, 1363, 1363, 1363, 1363, 1365, 1344, 1342, + /* 290 */ 1357, 1288, 1261, 1655, 1423, 1412, 1364, 1412, 1652, 1410, + /* 300 */ 1423, 1423, 1410, 1423, 1364, 1652, 1304, 1630, 1299, 1397, + /* 310 */ 1397, 1397, 1387, 1387, 1387, 1387, 1391, 1391, 1489, 1364, + /* 320 */ 1357, 1254, 1655, 1655, 1373, 1373, 1654, 1654, 1373, 1509, + /* 330 */ 1638, 1432, 1332, 1338, 1338, 1338, 1338, 1373, 1272, 1410, + /* 340 */ 1638, 1638, 1410, 1432, 1332, 1410, 1332, 1410, 1373, 1272, + /* 350 */ 1525, 1649, 1373, 1272, 1499, 1373, 1272, 1373, 1272, 1499, + /* 360 */ 1330, 1330, 1330, 1319, 1254, 1254, 1499, 1330, 1304, 1330, + /* 370 */ 1319, 1330, 1330, 1596, 1254, 1503, 1503, 1499, 1373, 1588, + /* 380 */ 1588, 1400, 1400, 1405, 1391, 1494, 1373, 1254, 1405, 1403, + /* 390 */ 1401, 1410, 1322, 1611, 1611, 1607, 1607, 1607, 1660, 1660, + /* 400 */ 1558, 1623, 1287, 1287, 1287, 1287, 1623, 1306, 1306, 1288, + /* 410 */ 1288, 1287, 1623, 1254, 1254, 1254, 1254, 1254, 1254, 1618, + /* 420 */ 1254, 1553, 1510, 1377, 1254, 1254, 1254, 1254, 1254, 1254, + /* 430 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 440 */ 1564, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 450 */ 1254, 1254, 1437, 1254, 1257, 1555, 1254, 1254, 1254, 1254, + /* 460 */ 1254, 1254, 1254, 1254, 1414, 1415, 1378, 1254, 1254, 1254, + /* 470 */ 1254, 1254, 1254, 1254, 1429, 1254, 1254, 1254, 1424, 1254, + /* 480 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1651, 1254, 1254, + /* 490 */ 1254, 1254, 1254, 1254, 1524, 1523, 1254, 1254, 1375, 1254, + /* 500 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 510 */ 1254, 1254, 1302, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 520 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 530 */ 1254, 1254, 1254, 1254, 1254, 1254, 1402, 1254, 1254, 1254, + /* 540 */ 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 550 */ 1254, 1593, 1392, 1254, 1254, 1254, 1254, 1642, 1254, 1254, + /* 560 */ 1254, 1254, 1352, 1254, 1254, 1254, 1254, 1254, 1254, 1254, + /* 570 */ 1254, 1254, 1254, 1634, 1346, 1438, 1254, 1441, 1276, 1254, + /* 580 */ 1266, 1254, 1254, }; /********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens (terminal symbols) into fallback tokens. +/* The next table maps tokens (terminal symbols) into fallback tokens. ** If a construct like the following: -** +** ** %fallback ID X Y Z. ** ** appears in the grammar, then ID becomes a fallback token for X, Y, @@ -131549,36 +175069,53 @@ static const YYACTIONTYPE yy_default[] = { static const YYCODETYPE yyFallback[] = { 0, /* $ => nothing */ 0, /* SEMI => nothing */ - 55, /* EXPLAIN => ID */ - 55, /* QUERY => ID */ - 55, /* PLAN => ID */ - 55, /* BEGIN => ID */ + 60, /* EXPLAIN => ID */ + 60, /* QUERY => ID */ + 60, /* PLAN => ID */ + 60, /* BEGIN => ID */ 0, /* TRANSACTION => nothing */ - 55, /* DEFERRED => ID */ - 55, /* IMMEDIATE => ID */ - 55, /* EXCLUSIVE => ID */ + 60, /* DEFERRED => ID */ + 60, /* IMMEDIATE => ID */ + 60, /* EXCLUSIVE => ID */ 0, /* COMMIT => nothing */ - 55, /* END => ID */ - 55, /* ROLLBACK => ID */ - 55, /* SAVEPOINT => ID */ - 55, /* RELEASE => ID */ + 60, /* END => ID */ + 60, /* ROLLBACK => ID */ + 60, /* SAVEPOINT => ID */ + 60, /* RELEASE => ID */ 0, /* TO => nothing */ 0, /* TABLE => nothing */ 0, /* CREATE => nothing */ - 55, /* IF => ID */ + 60, /* IF => ID */ 0, /* NOT => nothing */ 0, /* EXISTS => nothing */ - 55, /* TEMP => ID */ + 60, /* TEMP => ID */ 0, /* LP => nothing */ 0, /* RP => nothing */ 0, /* AS => nothing */ - 55, /* WITHOUT => ID */ 0, /* COMMA => nothing */ + 60, /* WITHOUT => ID */ + 60, /* ABORT => ID */ + 60, /* ACTION => ID */ + 60, /* AFTER => ID */ + 60, /* ANALYZE => ID */ + 60, /* ASC => ID */ + 60, /* ATTACH => ID */ + 60, /* BEFORE => ID */ + 60, /* BY => ID */ + 60, /* CASCADE => ID */ + 60, /* CAST => ID */ + 60, /* CONFLICT => ID */ + 60, /* DATABASE => ID */ + 60, /* DESC => ID */ + 60, /* DETACH => ID */ + 60, /* EACH => ID */ + 60, /* FAIL => ID */ 0, /* OR => nothing */ 0, /* AND => nothing */ 0, /* IS => nothing */ - 55, /* MATCH => ID */ - 55, /* LIKE_KW => ID */ + 0, /* ISNOT => nothing */ + 60, /* MATCH => ID */ + 60, /* LIKE_KW => ID */ 0, /* BETWEEN => nothing */ 0, /* IN => nothing */ 0, /* ISNULL => nothing */ @@ -131590,6 +175127,49 @@ static const YYCODETYPE yyFallback[] = { 0, /* LT => nothing */ 0, /* GE => nothing */ 0, /* ESCAPE => nothing */ + 0, /* ID => nothing */ + 60, /* COLUMNKW => ID */ + 60, /* DO => ID */ + 60, /* FOR => ID */ + 60, /* IGNORE => ID */ + 60, /* INITIALLY => ID */ + 60, /* INSTEAD => ID */ + 60, /* NO => ID */ + 60, /* KEY => ID */ + 60, /* OF => ID */ + 60, /* OFFSET => ID */ + 60, /* PRAGMA => ID */ + 60, /* RAISE => ID */ + 60, /* RECURSIVE => ID */ + 60, /* REPLACE => ID */ + 60, /* RESTRICT => ID */ + 60, /* ROW => ID */ + 60, /* ROWS => ID */ + 60, /* TRIGGER => ID */ + 60, /* VACUUM => ID */ + 60, /* VIEW => ID */ + 60, /* VIRTUAL => ID */ + 60, /* WITH => ID */ + 60, /* NULLS => ID */ + 60, /* FIRST => ID */ + 60, /* LAST => ID */ + 60, /* CURRENT => ID */ + 60, /* FOLLOWING => ID */ + 60, /* PARTITION => ID */ + 60, /* PRECEDING => ID */ + 60, /* RANGE => ID */ + 60, /* UNBOUNDED => ID */ + 60, /* EXCLUDE => ID */ + 60, /* GROUPS => ID */ + 60, /* OTHERS => ID */ + 60, /* TIES => ID */ + 60, /* GENERATED => ID */ + 60, /* ALWAYS => ID */ + 60, /* MATERIALIZED => ID */ + 60, /* REINDEX => ID */ + 60, /* RENAME => ID */ + 60, /* CTIME_KW => ID */ + 0, /* ANY => nothing */ 0, /* BITAND => nothing */ 0, /* BITOR => nothing */ 0, /* LSHIFT => nothing */ @@ -131600,49 +175180,79 @@ static const YYCODETYPE yyFallback[] = { 0, /* SLASH => nothing */ 0, /* REM => nothing */ 0, /* CONCAT => nothing */ + 0, /* PTR => nothing */ 0, /* COLLATE => nothing */ 0, /* BITNOT => nothing */ - 0, /* ID => nothing */ + 0, /* ON => nothing */ 0, /* INDEXED => nothing */ - 55, /* ABORT => ID */ - 55, /* ACTION => ID */ - 55, /* AFTER => ID */ - 55, /* ANALYZE => ID */ - 55, /* ASC => ID */ - 55, /* ATTACH => ID */ - 55, /* BEFORE => ID */ - 55, /* BY => ID */ - 55, /* CASCADE => ID */ - 55, /* CAST => ID */ - 55, /* COLUMNKW => ID */ - 55, /* CONFLICT => ID */ - 55, /* DATABASE => ID */ - 55, /* DESC => ID */ - 55, /* DETACH => ID */ - 55, /* EACH => ID */ - 55, /* FAIL => ID */ - 55, /* FOR => ID */ - 55, /* IGNORE => ID */ - 55, /* INITIALLY => ID */ - 55, /* INSTEAD => ID */ - 55, /* NO => ID */ - 55, /* KEY => ID */ - 55, /* OF => ID */ - 55, /* OFFSET => ID */ - 55, /* PRAGMA => ID */ - 55, /* RAISE => ID */ - 55, /* RECURSIVE => ID */ - 55, /* REPLACE => ID */ - 55, /* RESTRICT => ID */ - 55, /* ROW => ID */ - 55, /* TRIGGER => ID */ - 55, /* VACUUM => ID */ - 55, /* VIEW => ID */ - 55, /* VIRTUAL => ID */ - 55, /* WITH => ID */ - 55, /* REINDEX => ID */ - 55, /* RENAME => ID */ - 55, /* CTIME_KW => ID */ + 0, /* STRING => nothing */ + 0, /* JOIN_KW => nothing */ + 0, /* CONSTRAINT => nothing */ + 0, /* DEFAULT => nothing */ + 0, /* NULL => nothing */ + 0, /* PRIMARY => nothing */ + 0, /* UNIQUE => nothing */ + 0, /* CHECK => nothing */ + 0, /* REFERENCES => nothing */ + 0, /* AUTOINCR => nothing */ + 0, /* INSERT => nothing */ + 0, /* DELETE => nothing */ + 0, /* UPDATE => nothing */ + 0, /* SET => nothing */ + 0, /* DEFERRABLE => nothing */ + 0, /* FOREIGN => nothing */ + 0, /* DROP => nothing */ + 0, /* UNION => nothing */ + 0, /* ALL => nothing */ + 0, /* EXCEPT => nothing */ + 0, /* INTERSECT => nothing */ + 0, /* SELECT => nothing */ + 0, /* VALUES => nothing */ + 0, /* DISTINCT => nothing */ + 0, /* DOT => nothing */ + 0, /* FROM => nothing */ + 0, /* JOIN => nothing */ + 0, /* USING => nothing */ + 0, /* ORDER => nothing */ + 0, /* GROUP => nothing */ + 0, /* HAVING => nothing */ + 0, /* LIMIT => nothing */ + 0, /* WHERE => nothing */ + 0, /* RETURNING => nothing */ + 0, /* INTO => nothing */ + 0, /* NOTHING => nothing */ + 0, /* FLOAT => nothing */ + 0, /* BLOB => nothing */ + 0, /* INTEGER => nothing */ + 0, /* VARIABLE => nothing */ + 0, /* CASE => nothing */ + 0, /* WHEN => nothing */ + 0, /* THEN => nothing */ + 0, /* ELSE => nothing */ + 0, /* INDEX => nothing */ + 0, /* ALTER => nothing */ + 0, /* ADD => nothing */ + 0, /* WINDOW => nothing */ + 0, /* OVER => nothing */ + 0, /* FILTER => nothing */ + 0, /* COLUMN => nothing */ + 0, /* AGG_FUNCTION => nothing */ + 0, /* AGG_COLUMN => nothing */ + 0, /* TRUEFALSE => nothing */ + 0, /* FUNCTION => nothing */ + 0, /* UPLUS => nothing */ + 0, /* UMINUS => nothing */ + 0, /* TRUTH => nothing */ + 0, /* REGISTER => nothing */ + 0, /* VECTOR => nothing */ + 0, /* SELECT_COLUMN => nothing */ + 0, /* IF_NULL_ROW => nothing */ + 0, /* ASTERISK => nothing */ + 0, /* SPAN => nothing */ + 0, /* ERROR => nothing */ + 0, /* QNUMBER => nothing */ + 0, /* SPACE => nothing */ + 0, /* ILLEGAL => nothing */ }; #endif /* YYFALLBACK */ @@ -131674,23 +175284,22 @@ typedef struct yyStackEntry yyStackEntry; /* The state of the parser is completely contained in an instance of ** the following structure */ struct yyParser { - int yyidx; /* Index of top element in stack */ + yyStackEntry *yytos; /* Pointer to top element of the stack */ #ifdef YYTRACKMAXSTACKDEPTH - int yyidxMax; /* Maximum value of yyidx */ + int yyhwm; /* High-water mark of the stack */ #endif #ifndef YYNOERRORRECOVERY int yyerrcnt; /* Shifts left before out of the error */ #endif sqlite3ParserARG_SDECL /* A place to hold %extra_argument */ -#if YYSTACKDEPTH<=0 - int yystksz; /* Current side of the stack */ - yyStackEntry *yystack; /* The parser's stack */ -#else - yyStackEntry yystack[YYSTACKDEPTH]; /* The parser's stack */ -#endif + sqlite3ParserCTX_SDECL /* A place to hold %extra_context */ + yyStackEntry *yystackEnd; /* Last entry in the stack */ + yyStackEntry *yystack; /* The parser stack */ + yyStackEntry yystk0[YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct yyParser yyParser; +/* #include */ #ifndef NDEBUG /* #include */ static FILE *yyTraceFILE = 0; @@ -131698,10 +175307,10 @@ static char *yyTracePrompt = 0; #endif /* NDEBUG */ #ifndef NDEBUG -/* +/* ** Turn parser tracing on by giving a stream to which to write the trace ** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL +** by making either argument NULL ** ** Inputs: **
              @@ -131723,75 +175332,334 @@ SQLITE_PRIVATE void sqlite3ParserTrace(FILE *TraceFILE, char *zTracePrompt){ } #endif /* NDEBUG */ -#ifndef NDEBUG +#if defined(YYCOVERAGE) || !defined(NDEBUG) /* For tracing shifts, the names of all terminals and nonterminals ** are required. The following table supplies these names */ -static const char *const yyTokenName[] = { - "$", "SEMI", "EXPLAIN", "QUERY", - "PLAN", "BEGIN", "TRANSACTION", "DEFERRED", - "IMMEDIATE", "EXCLUSIVE", "COMMIT", "END", - "ROLLBACK", "SAVEPOINT", "RELEASE", "TO", - "TABLE", "CREATE", "IF", "NOT", - "EXISTS", "TEMP", "LP", "RP", - "AS", "WITHOUT", "COMMA", "OR", - "AND", "IS", "MATCH", "LIKE_KW", - "BETWEEN", "IN", "ISNULL", "NOTNULL", - "NE", "EQ", "GT", "LE", - "LT", "GE", "ESCAPE", "BITAND", - "BITOR", "LSHIFT", "RSHIFT", "PLUS", - "MINUS", "STAR", "SLASH", "REM", - "CONCAT", "COLLATE", "BITNOT", "ID", - "INDEXED", "ABORT", "ACTION", "AFTER", - "ANALYZE", "ASC", "ATTACH", "BEFORE", - "BY", "CASCADE", "CAST", "COLUMNKW", - "CONFLICT", "DATABASE", "DESC", "DETACH", - "EACH", "FAIL", "FOR", "IGNORE", - "INITIALLY", "INSTEAD", "NO", "KEY", - "OF", "OFFSET", "PRAGMA", "RAISE", - "RECURSIVE", "REPLACE", "RESTRICT", "ROW", - "TRIGGER", "VACUUM", "VIEW", "VIRTUAL", - "WITH", "REINDEX", "RENAME", "CTIME_KW", - "ANY", "STRING", "JOIN_KW", "CONSTRAINT", - "DEFAULT", "NULL", "PRIMARY", "UNIQUE", - "CHECK", "REFERENCES", "AUTOINCR", "ON", - "INSERT", "DELETE", "UPDATE", "SET", - "DEFERRABLE", "FOREIGN", "DROP", "UNION", - "ALL", "EXCEPT", "INTERSECT", "SELECT", - "VALUES", "DISTINCT", "DOT", "FROM", - "JOIN", "USING", "ORDER", "GROUP", - "HAVING", "LIMIT", "WHERE", "INTO", - "INTEGER", "FLOAT", "BLOB", "VARIABLE", - "CASE", "WHEN", "THEN", "ELSE", - "INDEX", "ALTER", "ADD", "error", - "input", "cmdlist", "ecmd", "explain", - "cmdx", "cmd", "transtype", "trans_opt", - "nm", "savepoint_opt", "create_table", "create_table_args", - "createkw", "temp", "ifnotexists", "dbnm", - "columnlist", "conslist_opt", "table_options", "select", - "columnname", "carglist", "typetoken", "typename", - "signed", "plus_num", "minus_num", "ccons", - "term", "expr", "onconf", "sortorder", - "autoinc", "eidlist_opt", "refargs", "defer_subclause", - "refarg", "refact", "init_deferred_pred_opt", "conslist", - "tconscomma", "tcons", "sortlist", "eidlist", - "defer_subclause_opt", "orconf", "resolvetype", "raisetype", - "ifexists", "fullname", "selectnowith", "oneselect", - "with", "multiselect_op", "distinct", "selcollist", - "from", "where_opt", "groupby_opt", "having_opt", - "orderby_opt", "limit_opt", "values", "nexprlist", - "exprlist", "sclp", "as", "seltablist", - "stl_prefix", "joinop", "indexed_opt", "on_opt", - "using_opt", "idlist", "setlist", "insert_cmd", - "idlist_opt", "likeop", "between_op", "in_op", - "case_operand", "case_exprlist", "case_else", "uniqueflag", - "collate", "nmnum", "trigger_decl", "trigger_cmd_list", - "trigger_time", "trigger_event", "foreach_clause", "when_clause", - "trigger_cmd", "trnm", "tridxby", "database_kw_opt", - "key_opt", "add_column_fullname", "kwcolumn_opt", "create_vtab", - "vtabarglist", "vtabarg", "vtabargtoken", "lp", - "anylist", "wqlist", +static const char *const yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "SEMI", + /* 2 */ "EXPLAIN", + /* 3 */ "QUERY", + /* 4 */ "PLAN", + /* 5 */ "BEGIN", + /* 6 */ "TRANSACTION", + /* 7 */ "DEFERRED", + /* 8 */ "IMMEDIATE", + /* 9 */ "EXCLUSIVE", + /* 10 */ "COMMIT", + /* 11 */ "END", + /* 12 */ "ROLLBACK", + /* 13 */ "SAVEPOINT", + /* 14 */ "RELEASE", + /* 15 */ "TO", + /* 16 */ "TABLE", + /* 17 */ "CREATE", + /* 18 */ "IF", + /* 19 */ "NOT", + /* 20 */ "EXISTS", + /* 21 */ "TEMP", + /* 22 */ "LP", + /* 23 */ "RP", + /* 24 */ "AS", + /* 25 */ "COMMA", + /* 26 */ "WITHOUT", + /* 27 */ "ABORT", + /* 28 */ "ACTION", + /* 29 */ "AFTER", + /* 30 */ "ANALYZE", + /* 31 */ "ASC", + /* 32 */ "ATTACH", + /* 33 */ "BEFORE", + /* 34 */ "BY", + /* 35 */ "CASCADE", + /* 36 */ "CAST", + /* 37 */ "CONFLICT", + /* 38 */ "DATABASE", + /* 39 */ "DESC", + /* 40 */ "DETACH", + /* 41 */ "EACH", + /* 42 */ "FAIL", + /* 43 */ "OR", + /* 44 */ "AND", + /* 45 */ "IS", + /* 46 */ "ISNOT", + /* 47 */ "MATCH", + /* 48 */ "LIKE_KW", + /* 49 */ "BETWEEN", + /* 50 */ "IN", + /* 51 */ "ISNULL", + /* 52 */ "NOTNULL", + /* 53 */ "NE", + /* 54 */ "EQ", + /* 55 */ "GT", + /* 56 */ "LE", + /* 57 */ "LT", + /* 58 */ "GE", + /* 59 */ "ESCAPE", + /* 60 */ "ID", + /* 61 */ "COLUMNKW", + /* 62 */ "DO", + /* 63 */ "FOR", + /* 64 */ "IGNORE", + /* 65 */ "INITIALLY", + /* 66 */ "INSTEAD", + /* 67 */ "NO", + /* 68 */ "KEY", + /* 69 */ "OF", + /* 70 */ "OFFSET", + /* 71 */ "PRAGMA", + /* 72 */ "RAISE", + /* 73 */ "RECURSIVE", + /* 74 */ "REPLACE", + /* 75 */ "RESTRICT", + /* 76 */ "ROW", + /* 77 */ "ROWS", + /* 78 */ "TRIGGER", + /* 79 */ "VACUUM", + /* 80 */ "VIEW", + /* 81 */ "VIRTUAL", + /* 82 */ "WITH", + /* 83 */ "NULLS", + /* 84 */ "FIRST", + /* 85 */ "LAST", + /* 86 */ "CURRENT", + /* 87 */ "FOLLOWING", + /* 88 */ "PARTITION", + /* 89 */ "PRECEDING", + /* 90 */ "RANGE", + /* 91 */ "UNBOUNDED", + /* 92 */ "EXCLUDE", + /* 93 */ "GROUPS", + /* 94 */ "OTHERS", + /* 95 */ "TIES", + /* 96 */ "GENERATED", + /* 97 */ "ALWAYS", + /* 98 */ "MATERIALIZED", + /* 99 */ "REINDEX", + /* 100 */ "RENAME", + /* 101 */ "CTIME_KW", + /* 102 */ "ANY", + /* 103 */ "BITAND", + /* 104 */ "BITOR", + /* 105 */ "LSHIFT", + /* 106 */ "RSHIFT", + /* 107 */ "PLUS", + /* 108 */ "MINUS", + /* 109 */ "STAR", + /* 110 */ "SLASH", + /* 111 */ "REM", + /* 112 */ "CONCAT", + /* 113 */ "PTR", + /* 114 */ "COLLATE", + /* 115 */ "BITNOT", + /* 116 */ "ON", + /* 117 */ "INDEXED", + /* 118 */ "STRING", + /* 119 */ "JOIN_KW", + /* 120 */ "CONSTRAINT", + /* 121 */ "DEFAULT", + /* 122 */ "NULL", + /* 123 */ "PRIMARY", + /* 124 */ "UNIQUE", + /* 125 */ "CHECK", + /* 126 */ "REFERENCES", + /* 127 */ "AUTOINCR", + /* 128 */ "INSERT", + /* 129 */ "DELETE", + /* 130 */ "UPDATE", + /* 131 */ "SET", + /* 132 */ "DEFERRABLE", + /* 133 */ "FOREIGN", + /* 134 */ "DROP", + /* 135 */ "UNION", + /* 136 */ "ALL", + /* 137 */ "EXCEPT", + /* 138 */ "INTERSECT", + /* 139 */ "SELECT", + /* 140 */ "VALUES", + /* 141 */ "DISTINCT", + /* 142 */ "DOT", + /* 143 */ "FROM", + /* 144 */ "JOIN", + /* 145 */ "USING", + /* 146 */ "ORDER", + /* 147 */ "GROUP", + /* 148 */ "HAVING", + /* 149 */ "LIMIT", + /* 150 */ "WHERE", + /* 151 */ "RETURNING", + /* 152 */ "INTO", + /* 153 */ "NOTHING", + /* 154 */ "FLOAT", + /* 155 */ "BLOB", + /* 156 */ "INTEGER", + /* 157 */ "VARIABLE", + /* 158 */ "CASE", + /* 159 */ "WHEN", + /* 160 */ "THEN", + /* 161 */ "ELSE", + /* 162 */ "INDEX", + /* 163 */ "ALTER", + /* 164 */ "ADD", + /* 165 */ "WINDOW", + /* 166 */ "OVER", + /* 167 */ "FILTER", + /* 168 */ "COLUMN", + /* 169 */ "AGG_FUNCTION", + /* 170 */ "AGG_COLUMN", + /* 171 */ "TRUEFALSE", + /* 172 */ "FUNCTION", + /* 173 */ "UPLUS", + /* 174 */ "UMINUS", + /* 175 */ "TRUTH", + /* 176 */ "REGISTER", + /* 177 */ "VECTOR", + /* 178 */ "SELECT_COLUMN", + /* 179 */ "IF_NULL_ROW", + /* 180 */ "ASTERISK", + /* 181 */ "SPAN", + /* 182 */ "ERROR", + /* 183 */ "QNUMBER", + /* 184 */ "SPACE", + /* 185 */ "ILLEGAL", + /* 186 */ "input", + /* 187 */ "cmdlist", + /* 188 */ "ecmd", + /* 189 */ "cmdx", + /* 190 */ "explain", + /* 191 */ "cmd", + /* 192 */ "transtype", + /* 193 */ "trans_opt", + /* 194 */ "nm", + /* 195 */ "savepoint_opt", + /* 196 */ "create_table", + /* 197 */ "create_table_args", + /* 198 */ "createkw", + /* 199 */ "temp", + /* 200 */ "ifnotexists", + /* 201 */ "dbnm", + /* 202 */ "columnlist", + /* 203 */ "conslist_opt", + /* 204 */ "table_option_set", + /* 205 */ "select", + /* 206 */ "table_option", + /* 207 */ "columnname", + /* 208 */ "carglist", + /* 209 */ "typetoken", + /* 210 */ "typename", + /* 211 */ "signed", + /* 212 */ "plus_num", + /* 213 */ "minus_num", + /* 214 */ "scanpt", + /* 215 */ "scantok", + /* 216 */ "ccons", + /* 217 */ "term", + /* 218 */ "expr", + /* 219 */ "onconf", + /* 220 */ "sortorder", + /* 221 */ "autoinc", + /* 222 */ "eidlist_opt", + /* 223 */ "refargs", + /* 224 */ "defer_subclause", + /* 225 */ "generated", + /* 226 */ "refarg", + /* 227 */ "refact", + /* 228 */ "init_deferred_pred_opt", + /* 229 */ "conslist", + /* 230 */ "tconscomma", + /* 231 */ "tcons", + /* 232 */ "sortlist", + /* 233 */ "eidlist", + /* 234 */ "defer_subclause_opt", + /* 235 */ "orconf", + /* 236 */ "resolvetype", + /* 237 */ "raisetype", + /* 238 */ "ifexists", + /* 239 */ "fullname", + /* 240 */ "selectnowith", + /* 241 */ "oneselect", + /* 242 */ "wqlist", + /* 243 */ "multiselect_op", + /* 244 */ "distinct", + /* 245 */ "selcollist", + /* 246 */ "from", + /* 247 */ "where_opt", + /* 248 */ "groupby_opt", + /* 249 */ "having_opt", + /* 250 */ "orderby_opt", + /* 251 */ "limit_opt", + /* 252 */ "window_clause", + /* 253 */ "values", + /* 254 */ "nexprlist", + /* 255 */ "mvalues", + /* 256 */ "sclp", + /* 257 */ "as", + /* 258 */ "seltablist", + /* 259 */ "stl_prefix", + /* 260 */ "joinop", + /* 261 */ "on_using", + /* 262 */ "indexed_by", + /* 263 */ "exprlist", + /* 264 */ "xfullname", + /* 265 */ "idlist", + /* 266 */ "indexed_opt", + /* 267 */ "nulls", + /* 268 */ "with", + /* 269 */ "where_opt_ret", + /* 270 */ "setlist", + /* 271 */ "insert_cmd", + /* 272 */ "idlist_opt", + /* 273 */ "upsert", + /* 274 */ "returning", + /* 275 */ "filter_over", + /* 276 */ "likeop", + /* 277 */ "between_op", + /* 278 */ "in_op", + /* 279 */ "paren_exprlist", + /* 280 */ "case_operand", + /* 281 */ "case_exprlist", + /* 282 */ "case_else", + /* 283 */ "uniqueflag", + /* 284 */ "collate", + /* 285 */ "vinto", + /* 286 */ "nmnum", + /* 287 */ "trigger_decl", + /* 288 */ "trigger_cmd_list", + /* 289 */ "trigger_time", + /* 290 */ "trigger_event", + /* 291 */ "foreach_clause", + /* 292 */ "when_clause", + /* 293 */ "trigger_cmd", + /* 294 */ "trnm", + /* 295 */ "tridxby", + /* 296 */ "database_kw_opt", + /* 297 */ "key_opt", + /* 298 */ "add_column_fullname", + /* 299 */ "kwcolumn_opt", + /* 300 */ "create_vtab", + /* 301 */ "vtabarglist", + /* 302 */ "vtabarg", + /* 303 */ "vtabargtoken", + /* 304 */ "lp", + /* 305 */ "anylist", + /* 306 */ "wqitem", + /* 307 */ "wqas", + /* 308 */ "withnm", + /* 309 */ "windowdefn_list", + /* 310 */ "windowdefn", + /* 311 */ "window", + /* 312 */ "frame_opt", + /* 313 */ "part_opt", + /* 314 */ "filter_clause", + /* 315 */ "over_clause", + /* 316 */ "range_or_rows", + /* 317 */ "frame_bound", + /* 318 */ "frame_bound_s", + /* 319 */ "frame_bound_e", + /* 320 */ "frame_exclude_opt", + /* 321 */ "frame_exclude", }; -#endif /* NDEBUG */ +#endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ #ifndef NDEBUG /* For tracing reduce actions, the names of all rules are required. @@ -131805,349 +175673,450 @@ static const char *const yyRuleName[] = { /* 5 */ "transtype ::= DEFERRED", /* 6 */ "transtype ::= IMMEDIATE", /* 7 */ "transtype ::= EXCLUSIVE", - /* 8 */ "cmd ::= COMMIT trans_opt", - /* 9 */ "cmd ::= END trans_opt", - /* 10 */ "cmd ::= ROLLBACK trans_opt", - /* 11 */ "cmd ::= SAVEPOINT nm", - /* 12 */ "cmd ::= RELEASE savepoint_opt nm", - /* 13 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", - /* 14 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", - /* 15 */ "createkw ::= CREATE", - /* 16 */ "ifnotexists ::=", - /* 17 */ "ifnotexists ::= IF NOT EXISTS", - /* 18 */ "temp ::= TEMP", - /* 19 */ "temp ::=", - /* 20 */ "create_table_args ::= LP columnlist conslist_opt RP table_options", - /* 21 */ "create_table_args ::= AS select", - /* 22 */ "table_options ::=", - /* 23 */ "table_options ::= WITHOUT nm", - /* 24 */ "columnname ::= nm typetoken", - /* 25 */ "typetoken ::=", - /* 26 */ "typetoken ::= typename LP signed RP", - /* 27 */ "typetoken ::= typename LP signed COMMA signed RP", - /* 28 */ "typename ::= typename ID|STRING", - /* 29 */ "ccons ::= CONSTRAINT nm", - /* 30 */ "ccons ::= DEFAULT term", - /* 31 */ "ccons ::= DEFAULT LP expr RP", - /* 32 */ "ccons ::= DEFAULT PLUS term", - /* 33 */ "ccons ::= DEFAULT MINUS term", - /* 34 */ "ccons ::= DEFAULT ID|INDEXED", - /* 35 */ "ccons ::= NOT NULL onconf", - /* 36 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", - /* 37 */ "ccons ::= UNIQUE onconf", - /* 38 */ "ccons ::= CHECK LP expr RP", - /* 39 */ "ccons ::= REFERENCES nm eidlist_opt refargs", - /* 40 */ "ccons ::= defer_subclause", - /* 41 */ "ccons ::= COLLATE ID|STRING", - /* 42 */ "autoinc ::=", - /* 43 */ "autoinc ::= AUTOINCR", - /* 44 */ "refargs ::=", - /* 45 */ "refargs ::= refargs refarg", - /* 46 */ "refarg ::= MATCH nm", - /* 47 */ "refarg ::= ON INSERT refact", - /* 48 */ "refarg ::= ON DELETE refact", - /* 49 */ "refarg ::= ON UPDATE refact", - /* 50 */ "refact ::= SET NULL", - /* 51 */ "refact ::= SET DEFAULT", - /* 52 */ "refact ::= CASCADE", - /* 53 */ "refact ::= RESTRICT", - /* 54 */ "refact ::= NO ACTION", - /* 55 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", - /* 56 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", - /* 57 */ "init_deferred_pred_opt ::=", - /* 58 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", - /* 59 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", - /* 60 */ "conslist_opt ::=", - /* 61 */ "tconscomma ::= COMMA", - /* 62 */ "tcons ::= CONSTRAINT nm", - /* 63 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", - /* 64 */ "tcons ::= UNIQUE LP sortlist RP onconf", - /* 65 */ "tcons ::= CHECK LP expr RP onconf", - /* 66 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", - /* 67 */ "defer_subclause_opt ::=", - /* 68 */ "onconf ::=", - /* 69 */ "onconf ::= ON CONFLICT resolvetype", - /* 70 */ "orconf ::=", - /* 71 */ "orconf ::= OR resolvetype", - /* 72 */ "resolvetype ::= IGNORE", - /* 73 */ "resolvetype ::= REPLACE", - /* 74 */ "cmd ::= DROP TABLE ifexists fullname", - /* 75 */ "ifexists ::= IF EXISTS", - /* 76 */ "ifexists ::=", - /* 77 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", - /* 78 */ "cmd ::= DROP VIEW ifexists fullname", - /* 79 */ "cmd ::= select", - /* 80 */ "select ::= with selectnowith", - /* 81 */ "selectnowith ::= selectnowith multiselect_op oneselect", - /* 82 */ "multiselect_op ::= UNION", - /* 83 */ "multiselect_op ::= UNION ALL", - /* 84 */ "multiselect_op ::= EXCEPT|INTERSECT", - /* 85 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", - /* 86 */ "values ::= VALUES LP nexprlist RP", - /* 87 */ "values ::= values COMMA LP exprlist RP", - /* 88 */ "distinct ::= DISTINCT", - /* 89 */ "distinct ::= ALL", - /* 90 */ "distinct ::=", - /* 91 */ "sclp ::=", - /* 92 */ "selcollist ::= sclp expr as", - /* 93 */ "selcollist ::= sclp STAR", - /* 94 */ "selcollist ::= sclp nm DOT STAR", - /* 95 */ "as ::= AS nm", - /* 96 */ "as ::=", - /* 97 */ "from ::=", - /* 98 */ "from ::= FROM seltablist", - /* 99 */ "stl_prefix ::= seltablist joinop", - /* 100 */ "stl_prefix ::=", - /* 101 */ "seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt", - /* 102 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt", - /* 103 */ "seltablist ::= stl_prefix LP select RP as on_opt using_opt", - /* 104 */ "seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt", - /* 105 */ "dbnm ::=", - /* 106 */ "dbnm ::= DOT nm", - /* 107 */ "fullname ::= nm dbnm", - /* 108 */ "joinop ::= COMMA|JOIN", - /* 109 */ "joinop ::= JOIN_KW JOIN", - /* 110 */ "joinop ::= JOIN_KW nm JOIN", - /* 111 */ "joinop ::= JOIN_KW nm nm JOIN", - /* 112 */ "on_opt ::= ON expr", - /* 113 */ "on_opt ::=", - /* 114 */ "indexed_opt ::=", - /* 115 */ "indexed_opt ::= INDEXED BY nm", - /* 116 */ "indexed_opt ::= NOT INDEXED", - /* 117 */ "using_opt ::= USING LP idlist RP", - /* 118 */ "using_opt ::=", - /* 119 */ "orderby_opt ::=", - /* 120 */ "orderby_opt ::= ORDER BY sortlist", - /* 121 */ "sortlist ::= sortlist COMMA expr sortorder", - /* 122 */ "sortlist ::= expr sortorder", - /* 123 */ "sortorder ::= ASC", - /* 124 */ "sortorder ::= DESC", - /* 125 */ "sortorder ::=", - /* 126 */ "groupby_opt ::=", - /* 127 */ "groupby_opt ::= GROUP BY nexprlist", - /* 128 */ "having_opt ::=", - /* 129 */ "having_opt ::= HAVING expr", - /* 130 */ "limit_opt ::=", - /* 131 */ "limit_opt ::= LIMIT expr", - /* 132 */ "limit_opt ::= LIMIT expr OFFSET expr", - /* 133 */ "limit_opt ::= LIMIT expr COMMA expr", - /* 134 */ "cmd ::= with DELETE FROM fullname indexed_opt where_opt", - /* 135 */ "where_opt ::=", - /* 136 */ "where_opt ::= WHERE expr", - /* 137 */ "cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt", - /* 138 */ "setlist ::= setlist COMMA nm EQ expr", - /* 139 */ "setlist ::= nm EQ expr", - /* 140 */ "cmd ::= with insert_cmd INTO fullname idlist_opt select", - /* 141 */ "cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES", - /* 142 */ "insert_cmd ::= INSERT orconf", - /* 143 */ "insert_cmd ::= REPLACE", - /* 144 */ "idlist_opt ::=", - /* 145 */ "idlist_opt ::= LP idlist RP", - /* 146 */ "idlist ::= idlist COMMA nm", - /* 147 */ "idlist ::= nm", - /* 148 */ "expr ::= LP expr RP", - /* 149 */ "term ::= NULL", - /* 150 */ "expr ::= ID|INDEXED", - /* 151 */ "expr ::= JOIN_KW", - /* 152 */ "expr ::= nm DOT nm", - /* 153 */ "expr ::= nm DOT nm DOT nm", - /* 154 */ "term ::= INTEGER|FLOAT|BLOB", - /* 155 */ "term ::= STRING", - /* 156 */ "expr ::= VARIABLE", - /* 157 */ "expr ::= expr COLLATE ID|STRING", - /* 158 */ "expr ::= CAST LP expr AS typetoken RP", - /* 159 */ "expr ::= ID|INDEXED LP distinct exprlist RP", - /* 160 */ "expr ::= ID|INDEXED LP STAR RP", - /* 161 */ "term ::= CTIME_KW", - /* 162 */ "expr ::= expr AND expr", - /* 163 */ "expr ::= expr OR expr", - /* 164 */ "expr ::= expr LT|GT|GE|LE expr", - /* 165 */ "expr ::= expr EQ|NE expr", - /* 166 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", - /* 167 */ "expr ::= expr PLUS|MINUS expr", - /* 168 */ "expr ::= expr STAR|SLASH|REM expr", - /* 169 */ "expr ::= expr CONCAT expr", - /* 170 */ "likeop ::= LIKE_KW|MATCH", - /* 171 */ "likeop ::= NOT LIKE_KW|MATCH", - /* 172 */ "expr ::= expr likeop expr", - /* 173 */ "expr ::= expr likeop expr ESCAPE expr", - /* 174 */ "expr ::= expr ISNULL|NOTNULL", - /* 175 */ "expr ::= expr NOT NULL", - /* 176 */ "expr ::= expr IS expr", - /* 177 */ "expr ::= expr IS NOT expr", - /* 178 */ "expr ::= NOT expr", - /* 179 */ "expr ::= BITNOT expr", - /* 180 */ "expr ::= MINUS expr", - /* 181 */ "expr ::= PLUS expr", - /* 182 */ "between_op ::= BETWEEN", - /* 183 */ "between_op ::= NOT BETWEEN", - /* 184 */ "expr ::= expr between_op expr AND expr", - /* 185 */ "in_op ::= IN", - /* 186 */ "in_op ::= NOT IN", - /* 187 */ "expr ::= expr in_op LP exprlist RP", - /* 188 */ "expr ::= LP select RP", - /* 189 */ "expr ::= expr in_op LP select RP", - /* 190 */ "expr ::= expr in_op nm dbnm", - /* 191 */ "expr ::= EXISTS LP select RP", - /* 192 */ "expr ::= CASE case_operand case_exprlist case_else END", - /* 193 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", - /* 194 */ "case_exprlist ::= WHEN expr THEN expr", - /* 195 */ "case_else ::= ELSE expr", - /* 196 */ "case_else ::=", - /* 197 */ "case_operand ::= expr", - /* 198 */ "case_operand ::=", - /* 199 */ "exprlist ::=", - /* 200 */ "nexprlist ::= nexprlist COMMA expr", - /* 201 */ "nexprlist ::= expr", - /* 202 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", - /* 203 */ "uniqueflag ::= UNIQUE", - /* 204 */ "uniqueflag ::=", - /* 205 */ "eidlist_opt ::=", - /* 206 */ "eidlist_opt ::= LP eidlist RP", - /* 207 */ "eidlist ::= eidlist COMMA nm collate sortorder", - /* 208 */ "eidlist ::= nm collate sortorder", - /* 209 */ "collate ::=", - /* 210 */ "collate ::= COLLATE ID|STRING", - /* 211 */ "cmd ::= DROP INDEX ifexists fullname", - /* 212 */ "cmd ::= VACUUM", - /* 213 */ "cmd ::= VACUUM nm", - /* 214 */ "cmd ::= PRAGMA nm dbnm", - /* 215 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", - /* 216 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", - /* 217 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", - /* 218 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", - /* 219 */ "plus_num ::= PLUS INTEGER|FLOAT", - /* 220 */ "minus_num ::= MINUS INTEGER|FLOAT", - /* 221 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", - /* 222 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", - /* 223 */ "trigger_time ::= BEFORE", - /* 224 */ "trigger_time ::= AFTER", - /* 225 */ "trigger_time ::= INSTEAD OF", - /* 226 */ "trigger_time ::=", - /* 227 */ "trigger_event ::= DELETE|INSERT", - /* 228 */ "trigger_event ::= UPDATE", - /* 229 */ "trigger_event ::= UPDATE OF idlist", - /* 230 */ "when_clause ::=", - /* 231 */ "when_clause ::= WHEN expr", - /* 232 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", - /* 233 */ "trigger_cmd_list ::= trigger_cmd SEMI", - /* 234 */ "trnm ::= nm DOT nm", - /* 235 */ "tridxby ::= INDEXED BY nm", - /* 236 */ "tridxby ::= NOT INDEXED", - /* 237 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt", - /* 238 */ "trigger_cmd ::= insert_cmd INTO trnm idlist_opt select", - /* 239 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt", - /* 240 */ "trigger_cmd ::= select", - /* 241 */ "expr ::= RAISE LP IGNORE RP", - /* 242 */ "expr ::= RAISE LP raisetype COMMA nm RP", - /* 243 */ "raisetype ::= ROLLBACK", - /* 244 */ "raisetype ::= ABORT", - /* 245 */ "raisetype ::= FAIL", - /* 246 */ "cmd ::= DROP TRIGGER ifexists fullname", - /* 247 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", - /* 248 */ "cmd ::= DETACH database_kw_opt expr", - /* 249 */ "key_opt ::=", - /* 250 */ "key_opt ::= KEY expr", - /* 251 */ "cmd ::= REINDEX", - /* 252 */ "cmd ::= REINDEX nm dbnm", - /* 253 */ "cmd ::= ANALYZE", - /* 254 */ "cmd ::= ANALYZE nm dbnm", - /* 255 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", - /* 256 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", - /* 257 */ "add_column_fullname ::= fullname", - /* 258 */ "cmd ::= create_vtab", - /* 259 */ "cmd ::= create_vtab LP vtabarglist RP", - /* 260 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", - /* 261 */ "vtabarg ::=", - /* 262 */ "vtabargtoken ::= ANY", - /* 263 */ "vtabargtoken ::= lp anylist RP", - /* 264 */ "lp ::= LP", - /* 265 */ "with ::=", - /* 266 */ "with ::= WITH wqlist", - /* 267 */ "with ::= WITH RECURSIVE wqlist", - /* 268 */ "wqlist ::= nm eidlist_opt AS LP select RP", - /* 269 */ "wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP", - /* 270 */ "input ::= cmdlist", - /* 271 */ "cmdlist ::= cmdlist ecmd", - /* 272 */ "cmdlist ::= ecmd", - /* 273 */ "ecmd ::= SEMI", - /* 274 */ "ecmd ::= explain cmdx SEMI", - /* 275 */ "explain ::=", - /* 276 */ "trans_opt ::=", - /* 277 */ "trans_opt ::= TRANSACTION", - /* 278 */ "trans_opt ::= TRANSACTION nm", - /* 279 */ "savepoint_opt ::= SAVEPOINT", - /* 280 */ "savepoint_opt ::=", - /* 281 */ "cmd ::= create_table create_table_args", - /* 282 */ "columnlist ::= columnlist COMMA columnname carglist", - /* 283 */ "columnlist ::= columnname carglist", - /* 284 */ "nm ::= ID|INDEXED", - /* 285 */ "nm ::= STRING", - /* 286 */ "nm ::= JOIN_KW", - /* 287 */ "typetoken ::= typename", - /* 288 */ "typename ::= ID|STRING", - /* 289 */ "signed ::= plus_num", - /* 290 */ "signed ::= minus_num", - /* 291 */ "carglist ::= carglist ccons", - /* 292 */ "carglist ::=", - /* 293 */ "ccons ::= NULL onconf", - /* 294 */ "conslist_opt ::= COMMA conslist", - /* 295 */ "conslist ::= conslist tconscomma tcons", - /* 296 */ "conslist ::= tcons", - /* 297 */ "tconscomma ::=", - /* 298 */ "defer_subclause_opt ::= defer_subclause", - /* 299 */ "resolvetype ::= raisetype", - /* 300 */ "selectnowith ::= oneselect", - /* 301 */ "oneselect ::= values", - /* 302 */ "sclp ::= selcollist COMMA", - /* 303 */ "as ::= ID|STRING", - /* 304 */ "expr ::= term", - /* 305 */ "exprlist ::= nexprlist", - /* 306 */ "nmnum ::= plus_num", - /* 307 */ "nmnum ::= nm", - /* 308 */ "nmnum ::= ON", - /* 309 */ "nmnum ::= DELETE", - /* 310 */ "nmnum ::= DEFAULT", - /* 311 */ "plus_num ::= INTEGER|FLOAT", - /* 312 */ "foreach_clause ::=", - /* 313 */ "foreach_clause ::= FOR EACH ROW", - /* 314 */ "trnm ::= nm", - /* 315 */ "tridxby ::=", - /* 316 */ "database_kw_opt ::= DATABASE", - /* 317 */ "database_kw_opt ::=", - /* 318 */ "kwcolumn_opt ::=", - /* 319 */ "kwcolumn_opt ::= COLUMNKW", - /* 320 */ "vtabarglist ::= vtabarg", - /* 321 */ "vtabarglist ::= vtabarglist COMMA vtabarg", - /* 322 */ "vtabarg ::= vtabarg vtabargtoken", - /* 323 */ "anylist ::=", - /* 324 */ "anylist ::= anylist LP anylist RP", - /* 325 */ "anylist ::= anylist ANY", + /* 8 */ "cmd ::= COMMIT|END trans_opt", + /* 9 */ "cmd ::= ROLLBACK trans_opt", + /* 10 */ "cmd ::= SAVEPOINT nm", + /* 11 */ "cmd ::= RELEASE savepoint_opt nm", + /* 12 */ "cmd ::= ROLLBACK trans_opt TO savepoint_opt nm", + /* 13 */ "create_table ::= createkw temp TABLE ifnotexists nm dbnm", + /* 14 */ "createkw ::= CREATE", + /* 15 */ "ifnotexists ::=", + /* 16 */ "ifnotexists ::= IF NOT EXISTS", + /* 17 */ "temp ::= TEMP", + /* 18 */ "temp ::=", + /* 19 */ "create_table_args ::= LP columnlist conslist_opt RP table_option_set", + /* 20 */ "create_table_args ::= AS select", + /* 21 */ "table_option_set ::=", + /* 22 */ "table_option_set ::= table_option_set COMMA table_option", + /* 23 */ "table_option ::= WITHOUT nm", + /* 24 */ "table_option ::= nm", + /* 25 */ "columnname ::= nm typetoken", + /* 26 */ "typetoken ::=", + /* 27 */ "typetoken ::= typename LP signed RP", + /* 28 */ "typetoken ::= typename LP signed COMMA signed RP", + /* 29 */ "typename ::= typename ID|STRING", + /* 30 */ "scanpt ::=", + /* 31 */ "scantok ::=", + /* 32 */ "ccons ::= CONSTRAINT nm", + /* 33 */ "ccons ::= DEFAULT scantok term", + /* 34 */ "ccons ::= DEFAULT LP expr RP", + /* 35 */ "ccons ::= DEFAULT PLUS scantok term", + /* 36 */ "ccons ::= DEFAULT MINUS scantok term", + /* 37 */ "ccons ::= DEFAULT scantok ID|INDEXED", + /* 38 */ "ccons ::= NOT NULL onconf", + /* 39 */ "ccons ::= PRIMARY KEY sortorder onconf autoinc", + /* 40 */ "ccons ::= UNIQUE onconf", + /* 41 */ "ccons ::= CHECK LP expr RP", + /* 42 */ "ccons ::= REFERENCES nm eidlist_opt refargs", + /* 43 */ "ccons ::= defer_subclause", + /* 44 */ "ccons ::= COLLATE ID|STRING", + /* 45 */ "generated ::= LP expr RP", + /* 46 */ "generated ::= LP expr RP ID", + /* 47 */ "autoinc ::=", + /* 48 */ "autoinc ::= AUTOINCR", + /* 49 */ "refargs ::=", + /* 50 */ "refargs ::= refargs refarg", + /* 51 */ "refarg ::= MATCH nm", + /* 52 */ "refarg ::= ON INSERT refact", + /* 53 */ "refarg ::= ON DELETE refact", + /* 54 */ "refarg ::= ON UPDATE refact", + /* 55 */ "refact ::= SET NULL", + /* 56 */ "refact ::= SET DEFAULT", + /* 57 */ "refact ::= CASCADE", + /* 58 */ "refact ::= RESTRICT", + /* 59 */ "refact ::= NO ACTION", + /* 60 */ "defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt", + /* 61 */ "defer_subclause ::= DEFERRABLE init_deferred_pred_opt", + /* 62 */ "init_deferred_pred_opt ::=", + /* 63 */ "init_deferred_pred_opt ::= INITIALLY DEFERRED", + /* 64 */ "init_deferred_pred_opt ::= INITIALLY IMMEDIATE", + /* 65 */ "conslist_opt ::=", + /* 66 */ "tconscomma ::= COMMA", + /* 67 */ "tcons ::= CONSTRAINT nm", + /* 68 */ "tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf", + /* 69 */ "tcons ::= UNIQUE LP sortlist RP onconf", + /* 70 */ "tcons ::= CHECK LP expr RP onconf", + /* 71 */ "tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt", + /* 72 */ "defer_subclause_opt ::=", + /* 73 */ "onconf ::=", + /* 74 */ "onconf ::= ON CONFLICT resolvetype", + /* 75 */ "orconf ::=", + /* 76 */ "orconf ::= OR resolvetype", + /* 77 */ "resolvetype ::= IGNORE", + /* 78 */ "resolvetype ::= REPLACE", + /* 79 */ "cmd ::= DROP TABLE ifexists fullname", + /* 80 */ "ifexists ::= IF EXISTS", + /* 81 */ "ifexists ::=", + /* 82 */ "cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select", + /* 83 */ "cmd ::= DROP VIEW ifexists fullname", + /* 84 */ "cmd ::= select", + /* 85 */ "select ::= WITH wqlist selectnowith", + /* 86 */ "select ::= WITH RECURSIVE wqlist selectnowith", + /* 87 */ "select ::= selectnowith", + /* 88 */ "selectnowith ::= selectnowith multiselect_op oneselect", + /* 89 */ "multiselect_op ::= UNION", + /* 90 */ "multiselect_op ::= UNION ALL", + /* 91 */ "multiselect_op ::= EXCEPT|INTERSECT", + /* 92 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt", + /* 93 */ "oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt", + /* 94 */ "values ::= VALUES LP nexprlist RP", + /* 95 */ "oneselect ::= mvalues", + /* 96 */ "mvalues ::= values COMMA LP nexprlist RP", + /* 97 */ "mvalues ::= mvalues COMMA LP nexprlist RP", + /* 98 */ "distinct ::= DISTINCT", + /* 99 */ "distinct ::= ALL", + /* 100 */ "distinct ::=", + /* 101 */ "sclp ::=", + /* 102 */ "selcollist ::= sclp scanpt expr scanpt as", + /* 103 */ "selcollist ::= sclp scanpt STAR", + /* 104 */ "selcollist ::= sclp scanpt nm DOT STAR", + /* 105 */ "as ::= AS nm", + /* 106 */ "as ::=", + /* 107 */ "from ::=", + /* 108 */ "from ::= FROM seltablist", + /* 109 */ "stl_prefix ::= seltablist joinop", + /* 110 */ "stl_prefix ::=", + /* 111 */ "seltablist ::= stl_prefix nm dbnm as on_using", + /* 112 */ "seltablist ::= stl_prefix nm dbnm as indexed_by on_using", + /* 113 */ "seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using", + /* 114 */ "seltablist ::= stl_prefix LP select RP as on_using", + /* 115 */ "seltablist ::= stl_prefix LP seltablist RP as on_using", + /* 116 */ "dbnm ::=", + /* 117 */ "dbnm ::= DOT nm", + /* 118 */ "fullname ::= nm", + /* 119 */ "fullname ::= nm DOT nm", + /* 120 */ "xfullname ::= nm", + /* 121 */ "xfullname ::= nm DOT nm", + /* 122 */ "xfullname ::= nm DOT nm AS nm", + /* 123 */ "xfullname ::= nm AS nm", + /* 124 */ "joinop ::= COMMA|JOIN", + /* 125 */ "joinop ::= JOIN_KW JOIN", + /* 126 */ "joinop ::= JOIN_KW nm JOIN", + /* 127 */ "joinop ::= JOIN_KW nm nm JOIN", + /* 128 */ "on_using ::= ON expr", + /* 129 */ "on_using ::= USING LP idlist RP", + /* 130 */ "on_using ::=", + /* 131 */ "indexed_opt ::=", + /* 132 */ "indexed_by ::= INDEXED BY nm", + /* 133 */ "indexed_by ::= NOT INDEXED", + /* 134 */ "orderby_opt ::=", + /* 135 */ "orderby_opt ::= ORDER BY sortlist", + /* 136 */ "sortlist ::= sortlist COMMA expr sortorder nulls", + /* 137 */ "sortlist ::= expr sortorder nulls", + /* 138 */ "sortorder ::= ASC", + /* 139 */ "sortorder ::= DESC", + /* 140 */ "sortorder ::=", + /* 141 */ "nulls ::= NULLS FIRST", + /* 142 */ "nulls ::= NULLS LAST", + /* 143 */ "nulls ::=", + /* 144 */ "groupby_opt ::=", + /* 145 */ "groupby_opt ::= GROUP BY nexprlist", + /* 146 */ "having_opt ::=", + /* 147 */ "having_opt ::= HAVING expr", + /* 148 */ "limit_opt ::=", + /* 149 */ "limit_opt ::= LIMIT expr", + /* 150 */ "limit_opt ::= LIMIT expr OFFSET expr", + /* 151 */ "limit_opt ::= LIMIT expr COMMA expr", + /* 152 */ "cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret", + /* 153 */ "where_opt ::=", + /* 154 */ "where_opt ::= WHERE expr", + /* 155 */ "where_opt_ret ::=", + /* 156 */ "where_opt_ret ::= WHERE expr", + /* 157 */ "where_opt_ret ::= RETURNING selcollist", + /* 158 */ "where_opt_ret ::= WHERE expr RETURNING selcollist", + /* 159 */ "cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret", + /* 160 */ "setlist ::= setlist COMMA nm EQ expr", + /* 161 */ "setlist ::= setlist COMMA LP idlist RP EQ expr", + /* 162 */ "setlist ::= nm EQ expr", + /* 163 */ "setlist ::= LP idlist RP EQ expr", + /* 164 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert", + /* 165 */ "cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning", + /* 166 */ "upsert ::=", + /* 167 */ "upsert ::= RETURNING selcollist", + /* 168 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert", + /* 169 */ "upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert", + /* 170 */ "upsert ::= ON CONFLICT DO NOTHING returning", + /* 171 */ "upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning", + /* 172 */ "returning ::= RETURNING selcollist", + /* 173 */ "insert_cmd ::= INSERT orconf", + /* 174 */ "insert_cmd ::= REPLACE", + /* 175 */ "idlist_opt ::=", + /* 176 */ "idlist_opt ::= LP idlist RP", + /* 177 */ "idlist ::= idlist COMMA nm", + /* 178 */ "idlist ::= nm", + /* 179 */ "expr ::= LP expr RP", + /* 180 */ "expr ::= ID|INDEXED|JOIN_KW", + /* 181 */ "expr ::= nm DOT nm", + /* 182 */ "expr ::= nm DOT nm DOT nm", + /* 183 */ "term ::= NULL|FLOAT|BLOB", + /* 184 */ "term ::= STRING", + /* 185 */ "term ::= INTEGER", + /* 186 */ "expr ::= VARIABLE", + /* 187 */ "expr ::= expr COLLATE ID|STRING", + /* 188 */ "expr ::= CAST LP expr AS typetoken RP", + /* 189 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP", + /* 190 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP", + /* 191 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP", + /* 192 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over", + /* 193 */ "expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over", + /* 194 */ "expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over", + /* 195 */ "term ::= CTIME_KW", + /* 196 */ "expr ::= LP nexprlist COMMA expr RP", + /* 197 */ "expr ::= expr AND expr", + /* 198 */ "expr ::= expr OR expr", + /* 199 */ "expr ::= expr LT|GT|GE|LE expr", + /* 200 */ "expr ::= expr EQ|NE expr", + /* 201 */ "expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr", + /* 202 */ "expr ::= expr PLUS|MINUS expr", + /* 203 */ "expr ::= expr STAR|SLASH|REM expr", + /* 204 */ "expr ::= expr CONCAT expr", + /* 205 */ "likeop ::= NOT LIKE_KW|MATCH", + /* 206 */ "expr ::= expr likeop expr", + /* 207 */ "expr ::= expr likeop expr ESCAPE expr", + /* 208 */ "expr ::= expr ISNULL|NOTNULL", + /* 209 */ "expr ::= expr NOT NULL", + /* 210 */ "expr ::= expr IS expr", + /* 211 */ "expr ::= expr IS NOT expr", + /* 212 */ "expr ::= expr IS NOT DISTINCT FROM expr", + /* 213 */ "expr ::= expr IS DISTINCT FROM expr", + /* 214 */ "expr ::= NOT expr", + /* 215 */ "expr ::= BITNOT expr", + /* 216 */ "expr ::= PLUS|MINUS expr", + /* 217 */ "expr ::= expr PTR expr", + /* 218 */ "between_op ::= BETWEEN", + /* 219 */ "between_op ::= NOT BETWEEN", + /* 220 */ "expr ::= expr between_op expr AND expr", + /* 221 */ "in_op ::= IN", + /* 222 */ "in_op ::= NOT IN", + /* 223 */ "expr ::= expr in_op LP exprlist RP", + /* 224 */ "expr ::= LP select RP", + /* 225 */ "expr ::= expr in_op LP select RP", + /* 226 */ "expr ::= expr in_op nm dbnm paren_exprlist", + /* 227 */ "expr ::= EXISTS LP select RP", + /* 228 */ "expr ::= CASE case_operand case_exprlist case_else END", + /* 229 */ "case_exprlist ::= case_exprlist WHEN expr THEN expr", + /* 230 */ "case_exprlist ::= WHEN expr THEN expr", + /* 231 */ "case_else ::= ELSE expr", + /* 232 */ "case_else ::=", + /* 233 */ "case_operand ::=", + /* 234 */ "exprlist ::=", + /* 235 */ "nexprlist ::= nexprlist COMMA expr", + /* 236 */ "nexprlist ::= expr", + /* 237 */ "paren_exprlist ::=", + /* 238 */ "paren_exprlist ::= LP exprlist RP", + /* 239 */ "cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt", + /* 240 */ "uniqueflag ::= UNIQUE", + /* 241 */ "uniqueflag ::=", + /* 242 */ "eidlist_opt ::=", + /* 243 */ "eidlist_opt ::= LP eidlist RP", + /* 244 */ "eidlist ::= eidlist COMMA nm collate sortorder", + /* 245 */ "eidlist ::= nm collate sortorder", + /* 246 */ "collate ::=", + /* 247 */ "collate ::= COLLATE ID|STRING", + /* 248 */ "cmd ::= DROP INDEX ifexists fullname", + /* 249 */ "cmd ::= VACUUM vinto", + /* 250 */ "cmd ::= VACUUM nm vinto", + /* 251 */ "vinto ::= INTO expr", + /* 252 */ "vinto ::=", + /* 253 */ "cmd ::= PRAGMA nm dbnm", + /* 254 */ "cmd ::= PRAGMA nm dbnm EQ nmnum", + /* 255 */ "cmd ::= PRAGMA nm dbnm LP nmnum RP", + /* 256 */ "cmd ::= PRAGMA nm dbnm EQ minus_num", + /* 257 */ "cmd ::= PRAGMA nm dbnm LP minus_num RP", + /* 258 */ "plus_num ::= PLUS INTEGER|FLOAT", + /* 259 */ "minus_num ::= MINUS INTEGER|FLOAT", + /* 260 */ "cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END", + /* 261 */ "trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause", + /* 262 */ "trigger_time ::= BEFORE|AFTER", + /* 263 */ "trigger_time ::= INSTEAD OF", + /* 264 */ "trigger_time ::=", + /* 265 */ "trigger_event ::= DELETE|INSERT", + /* 266 */ "trigger_event ::= UPDATE", + /* 267 */ "trigger_event ::= UPDATE OF idlist", + /* 268 */ "when_clause ::=", + /* 269 */ "when_clause ::= WHEN expr", + /* 270 */ "trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI", + /* 271 */ "trigger_cmd_list ::= trigger_cmd SEMI", + /* 272 */ "trnm ::= nm DOT nm", + /* 273 */ "tridxby ::= INDEXED BY nm", + /* 274 */ "tridxby ::= NOT INDEXED", + /* 275 */ "trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt", + /* 276 */ "trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt", + /* 277 */ "trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt", + /* 278 */ "trigger_cmd ::= scanpt select scanpt", + /* 279 */ "expr ::= RAISE LP IGNORE RP", + /* 280 */ "expr ::= RAISE LP raisetype COMMA expr RP", + /* 281 */ "raisetype ::= ROLLBACK", + /* 282 */ "raisetype ::= ABORT", + /* 283 */ "raisetype ::= FAIL", + /* 284 */ "cmd ::= DROP TRIGGER ifexists fullname", + /* 285 */ "cmd ::= ATTACH database_kw_opt expr AS expr key_opt", + /* 286 */ "cmd ::= DETACH database_kw_opt expr", + /* 287 */ "key_opt ::=", + /* 288 */ "key_opt ::= KEY expr", + /* 289 */ "cmd ::= REINDEX", + /* 290 */ "cmd ::= REINDEX nm dbnm", + /* 291 */ "cmd ::= ANALYZE", + /* 292 */ "cmd ::= ANALYZE nm dbnm", + /* 293 */ "cmd ::= ALTER TABLE fullname RENAME TO nm", + /* 294 */ "cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist", + /* 295 */ "cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm", + /* 296 */ "add_column_fullname ::= fullname", + /* 297 */ "cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm", + /* 298 */ "cmd ::= create_vtab", + /* 299 */ "cmd ::= create_vtab LP vtabarglist RP", + /* 300 */ "create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm", + /* 301 */ "vtabarg ::=", + /* 302 */ "vtabargtoken ::= ANY", + /* 303 */ "vtabargtoken ::= lp anylist RP", + /* 304 */ "lp ::= LP", + /* 305 */ "with ::= WITH wqlist", + /* 306 */ "with ::= WITH RECURSIVE wqlist", + /* 307 */ "wqas ::= AS", + /* 308 */ "wqas ::= AS MATERIALIZED", + /* 309 */ "wqas ::= AS NOT MATERIALIZED", + /* 310 */ "wqitem ::= withnm eidlist_opt wqas LP select RP", + /* 311 */ "withnm ::= nm", + /* 312 */ "wqlist ::= wqitem", + /* 313 */ "wqlist ::= wqlist COMMA wqitem", + /* 314 */ "windowdefn_list ::= windowdefn_list COMMA windowdefn", + /* 315 */ "windowdefn ::= nm AS LP window RP", + /* 316 */ "window ::= PARTITION BY nexprlist orderby_opt frame_opt", + /* 317 */ "window ::= nm PARTITION BY nexprlist orderby_opt frame_opt", + /* 318 */ "window ::= ORDER BY sortlist frame_opt", + /* 319 */ "window ::= nm ORDER BY sortlist frame_opt", + /* 320 */ "window ::= nm frame_opt", + /* 321 */ "frame_opt ::=", + /* 322 */ "frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt", + /* 323 */ "frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt", + /* 324 */ "range_or_rows ::= RANGE|ROWS|GROUPS", + /* 325 */ "frame_bound_s ::= frame_bound", + /* 326 */ "frame_bound_s ::= UNBOUNDED PRECEDING", + /* 327 */ "frame_bound_e ::= frame_bound", + /* 328 */ "frame_bound_e ::= UNBOUNDED FOLLOWING", + /* 329 */ "frame_bound ::= expr PRECEDING|FOLLOWING", + /* 330 */ "frame_bound ::= CURRENT ROW", + /* 331 */ "frame_exclude_opt ::=", + /* 332 */ "frame_exclude_opt ::= EXCLUDE frame_exclude", + /* 333 */ "frame_exclude ::= NO OTHERS", + /* 334 */ "frame_exclude ::= CURRENT ROW", + /* 335 */ "frame_exclude ::= GROUP|TIES", + /* 336 */ "window_clause ::= WINDOW windowdefn_list", + /* 337 */ "filter_over ::= filter_clause over_clause", + /* 338 */ "filter_over ::= over_clause", + /* 339 */ "filter_over ::= filter_clause", + /* 340 */ "over_clause ::= OVER LP window RP", + /* 341 */ "over_clause ::= OVER nm", + /* 342 */ "filter_clause ::= FILTER LP WHERE expr RP", + /* 343 */ "term ::= QNUMBER", + /* 344 */ "input ::= cmdlist", + /* 345 */ "cmdlist ::= cmdlist ecmd", + /* 346 */ "cmdlist ::= ecmd", + /* 347 */ "ecmd ::= SEMI", + /* 348 */ "ecmd ::= cmdx SEMI", + /* 349 */ "ecmd ::= explain cmdx SEMI", + /* 350 */ "trans_opt ::=", + /* 351 */ "trans_opt ::= TRANSACTION", + /* 352 */ "trans_opt ::= TRANSACTION nm", + /* 353 */ "savepoint_opt ::= SAVEPOINT", + /* 354 */ "savepoint_opt ::=", + /* 355 */ "cmd ::= create_table create_table_args", + /* 356 */ "table_option_set ::= table_option", + /* 357 */ "columnlist ::= columnlist COMMA columnname carglist", + /* 358 */ "columnlist ::= columnname carglist", + /* 359 */ "nm ::= ID|INDEXED|JOIN_KW", + /* 360 */ "nm ::= STRING", + /* 361 */ "typetoken ::= typename", + /* 362 */ "typename ::= ID|STRING", + /* 363 */ "signed ::= plus_num", + /* 364 */ "signed ::= minus_num", + /* 365 */ "carglist ::= carglist ccons", + /* 366 */ "carglist ::=", + /* 367 */ "ccons ::= NULL onconf", + /* 368 */ "ccons ::= GENERATED ALWAYS AS generated", + /* 369 */ "ccons ::= AS generated", + /* 370 */ "conslist_opt ::= COMMA conslist", + /* 371 */ "conslist ::= conslist tconscomma tcons", + /* 372 */ "conslist ::= tcons", + /* 373 */ "tconscomma ::=", + /* 374 */ "defer_subclause_opt ::= defer_subclause", + /* 375 */ "resolvetype ::= raisetype", + /* 376 */ "selectnowith ::= oneselect", + /* 377 */ "oneselect ::= values", + /* 378 */ "sclp ::= selcollist COMMA", + /* 379 */ "as ::= ID|STRING", + /* 380 */ "indexed_opt ::= indexed_by", + /* 381 */ "returning ::=", + /* 382 */ "expr ::= term", + /* 383 */ "likeop ::= LIKE_KW|MATCH", + /* 384 */ "case_operand ::= expr", + /* 385 */ "exprlist ::= nexprlist", + /* 386 */ "nmnum ::= plus_num", + /* 387 */ "nmnum ::= nm", + /* 388 */ "nmnum ::= ON", + /* 389 */ "nmnum ::= DELETE", + /* 390 */ "nmnum ::= DEFAULT", + /* 391 */ "plus_num ::= INTEGER|FLOAT", + /* 392 */ "foreach_clause ::=", + /* 393 */ "foreach_clause ::= FOR EACH ROW", + /* 394 */ "trnm ::= nm", + /* 395 */ "tridxby ::=", + /* 396 */ "database_kw_opt ::= DATABASE", + /* 397 */ "database_kw_opt ::=", + /* 398 */ "kwcolumn_opt ::=", + /* 399 */ "kwcolumn_opt ::= COLUMNKW", + /* 400 */ "vtabarglist ::= vtabarg", + /* 401 */ "vtabarglist ::= vtabarglist COMMA vtabarg", + /* 402 */ "vtabarg ::= vtabarg vtabargtoken", + /* 403 */ "anylist ::=", + /* 404 */ "anylist ::= anylist LP anylist RP", + /* 405 */ "anylist ::= anylist ANY", + /* 406 */ "with ::=", + /* 407 */ "windowdefn_list ::= windowdefn", + /* 408 */ "window ::= frame_opt", }; #endif /* NDEBUG */ -#if YYSTACKDEPTH<=0 +#if YYGROWABLESTACK /* -** Try to increase the size of the parser stack. +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. */ -static void yyGrowStack(yyParser *p){ +static int yyGrowStack(yyParser *p){ + int oldSize = 1 + (int)(p->yystackEnd - p->yystack); int newSize; + int idx; yyStackEntry *pNew; - newSize = p->yystksz*2 + 100; - pNew = realloc(p->yystack, newSize*sizeof(pNew[0])); - if( pNew ){ - p->yystack = pNew; - p->yystksz = newSize; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sStack grows to %d entries!\n", - yyTracePrompt, p->yystksz); - } -#endif + newSize = oldSize*2 + 100; + idx = (int)(p->yytos - p->yystack); + if( p->yystack==p->yystk0 ){ + pNew = YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->yystack, oldSize*sizeof(pNew[0])); + }else{ + pNew = YYREALLOC(p->yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } + p->yystack = pNew; + p->yytos = &p->yystack[idx]; +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sStack grows from %d to %d entries.\n", + yyTracePrompt, oldSize, newSize); + } +#endif + p->yystackEnd = &p->yystack[newSize-1]; + return 0; } +#endif /* YYGROWABLESTACK */ + +#if !YYGROWABLESTACK +/* For builds that do no have a growable stack, yyGrowStack always +** returns an error. +*/ +# define yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -132159,7 +176128,26 @@ static void yyGrowStack(yyParser *p){ # define YYMALLOCARGTYPE size_t #endif -/* +/* Initialize a new parser that has already been allocated. +*/ +SQLITE_PRIVATE void sqlite3ParserInit(void *yypRawParser sqlite3ParserCTX_PDECL){ + yyParser *yypParser = (yyParser*)yypRawParser; + sqlite3ParserCTX_STORE +#ifdef YYTRACKMAXSTACKDEPTH + yypParser->yyhwm = 0; +#endif + yypParser->yystack = yypParser->yystk0; + yypParser->yystackEnd = &yypParser->yystack[YYSTACKDEPTH-1]; +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + yypParser->yytos = yypParser->yystack; + yypParser->yystack[0].stateno = 0; + yypParser->yystack[0].major = 0; +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like ** malloc. @@ -132171,27 +176159,22 @@ static void yyGrowStack(yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to sqlite3Parser and sqlite3ParserFree. */ -SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE)){ - yyParser *pParser; - pParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); - if( pParser ){ - pParser->yyidx = -1; -#ifdef YYTRACKMAXSTACKDEPTH - pParser->yyidxMax = 0; -#endif -#if YYSTACKDEPTH<=0 - pParser->yystack = NULL; - pParser->yystksz = 0; - yyGrowStack(pParser); -#endif +SQLITE_PRIVATE void *sqlite3ParserAlloc(void *(*mallocProc)(YYMALLOCARGTYPE) sqlite3ParserCTX_PDECL){ + yyParser *yypParser; + yypParser = (yyParser*)(*mallocProc)( (YYMALLOCARGTYPE)sizeof(yyParser) ); + if( yypParser ){ + sqlite3ParserCTX_STORE + sqlite3ParserInit(yypParser sqlite3ParserCTX_PARAM); } - return pParser; + return (void*)yypParser; } +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ + /* The following function deletes the "minor type" or semantic value ** associated with a symbol. The symbol can be either a terminal ** or nonterminal. "yymajor" is the symbol code, and "yypminor" is -** a pointer to the value to be deleted. The code used to do the +** a pointer to the value to be deleted. The code used to do the ** deletions is derived from the %destructor and/or %token_destructor ** directives of the input grammar. */ @@ -132200,12 +176183,13 @@ static void yy_destructor( YYCODETYPE yymajor, /* Type code for object to destroy */ YYMINORTYPE *yypminor /* The object to be destroyed */ ){ - sqlite3ParserARG_FETCH; + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH switch( yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is + ** reduce or during error processing or when a parser is ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those @@ -132213,76 +176197,98 @@ static void yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 163: /* select */ - case 194: /* selectnowith */ - case 195: /* oneselect */ - case 206: /* values */ + case 205: /* select */ + case 240: /* selectnowith */ + case 241: /* oneselect */ + case 253: /* values */ + case 255: /* mvalues */ { -sqlite3SelectDelete(pParse->db, (yypminor->yy159)); +sqlite3SelectDelete(pParse->db, (yypminor->yy555)); } break; - case 172: /* term */ - case 173: /* expr */ + case 217: /* term */ + case 218: /* expr */ + case 247: /* where_opt */ + case 249: /* having_opt */ + case 269: /* where_opt_ret */ + case 280: /* case_operand */ + case 282: /* case_else */ + case 285: /* vinto */ + case 292: /* when_clause */ + case 297: /* key_opt */ + case 314: /* filter_clause */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy342).pExpr); +sqlite3ExprDelete(pParse->db, (yypminor->yy454)); } break; - case 177: /* eidlist_opt */ - case 186: /* sortlist */ - case 187: /* eidlist */ - case 199: /* selcollist */ - case 202: /* groupby_opt */ - case 204: /* orderby_opt */ - case 207: /* nexprlist */ - case 208: /* exprlist */ - case 209: /* sclp */ - case 218: /* setlist */ - case 225: /* case_exprlist */ + case 222: /* eidlist_opt */ + case 232: /* sortlist */ + case 233: /* eidlist */ + case 245: /* selcollist */ + case 248: /* groupby_opt */ + case 250: /* orderby_opt */ + case 254: /* nexprlist */ + case 256: /* sclp */ + case 263: /* exprlist */ + case 270: /* setlist */ + case 279: /* paren_exprlist */ + case 281: /* case_exprlist */ + case 313: /* part_opt */ { -sqlite3ExprListDelete(pParse->db, (yypminor->yy442)); +sqlite3ExprListDelete(pParse->db, (yypminor->yy14)); } break; - case 193: /* fullname */ - case 200: /* from */ - case 211: /* seltablist */ - case 212: /* stl_prefix */ + case 239: /* fullname */ + case 246: /* from */ + case 258: /* seltablist */ + case 259: /* stl_prefix */ + case 264: /* xfullname */ { -sqlite3SrcListDelete(pParse->db, (yypminor->yy347)); +sqlite3SrcListDelete(pParse->db, (yypminor->yy203)); } break; - case 196: /* with */ - case 249: /* wqlist */ + case 242: /* wqlist */ { -sqlite3WithDelete(pParse->db, (yypminor->yy331)); +sqlite3WithDelete(pParse->db, (yypminor->yy59)); } break; - case 201: /* where_opt */ - case 203: /* having_opt */ - case 215: /* on_opt */ - case 224: /* case_operand */ - case 226: /* case_else */ - case 235: /* when_clause */ - case 240: /* key_opt */ + case 252: /* window_clause */ + case 309: /* windowdefn_list */ { -sqlite3ExprDelete(pParse->db, (yypminor->yy122)); +sqlite3WindowListDelete(pParse->db, (yypminor->yy211)); } break; - case 216: /* using_opt */ - case 217: /* idlist */ - case 220: /* idlist_opt */ + case 265: /* idlist */ + case 272: /* idlist_opt */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy180)); +sqlite3IdListDelete(pParse->db, (yypminor->yy132)); } break; - case 231: /* trigger_cmd_list */ - case 236: /* trigger_cmd */ + case 275: /* filter_over */ + case 310: /* windowdefn */ + case 311: /* window */ + case 312: /* frame_opt */ + case 315: /* over_clause */ { -sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy327)); +sqlite3WindowDelete(pParse->db, (yypminor->yy211)); } break; - case 233: /* trigger_event */ + case 288: /* trigger_cmd_list */ + case 293: /* trigger_cmd */ { -sqlite3IdListDelete(pParse->db, (yypminor->yy410).b); +sqlite3DeleteTriggerStep(pParse->db, (yypminor->yy427)); +} + break; + case 290: /* trigger_event */ +{ +sqlite3IdListDelete(pParse->db, (yypminor->yy286).b); +} + break; + case 317: /* frame_bound */ + case 318: /* frame_bound_s */ + case 319: /* frame_bound_e */ +{ +sqlite3ExprDelete(pParse->db, (yypminor->yy509).pExpr); } break; /********* End destructor definitions *****************************************/ @@ -132298,8 +176304,9 @@ sqlite3IdListDelete(pParse->db, (yypminor->yy410).b); */ static void yy_pop_parser_stack(yyParser *pParser){ yyStackEntry *yytos; - assert( pParser->yyidx>=0 ); - yytos = &pParser->yystack[pParser->yyidx--]; + assert( pParser->yytos!=0 ); + assert( pParser->yytos > pParser->yystack ); + yytos = pParser->yytos--; #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sPopping %s\n", @@ -132310,7 +176317,36 @@ static void yy_pop_parser_stack(yyParser *pParser){ yy_destructor(pParser, yytos->major, &yytos->minor); } -/* +/* +** Clear all secondary memory allocations from the parser +*/ +SQLITE_PRIVATE void sqlite3ParserFinalize(void *p){ + yyParser *pParser = (yyParser*)p; + + /* In-lined version of calling yy_pop_parser_stack() for each + ** element left in the stack */ + yyStackEntry *yytos = pParser->yytos; + while( yytos>pParser->yystack ){ +#ifndef NDEBUG + if( yyTraceFILE ){ + fprintf(yyTraceFILE,"%sPopping %s\n", + yyTracePrompt, + yyTokenName[yytos->major]); + } +#endif + if( yytos->major>=YY_MIN_DSTRCTR ){ + yy_destructor(pParser, yytos->major, &yytos->minor); + } + yytos--; + } + +#if YYGROWABLESTACK + if( pParser->yystack!=pParser->yystk0 ) YYFREE(pParser->yystack); +#endif +} + +#ifndef sqlite3Parser_ENGINEALWAYSONSTACK +/* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. ** @@ -132322,16 +176358,13 @@ SQLITE_PRIVATE void sqlite3ParserFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ - yyParser *pParser = (yyParser*)p; #ifndef YYPARSEFREENEVERNULL - if( pParser==0 ) return; + if( p==0 ) return; #endif - while( pParser->yyidx>=0 ) yy_pop_parser_stack(pParser); -#if YYSTACKDEPTH<=0 - free(pParser->yystack); -#endif - (*freeProc)((void*)pParser); + sqlite3ParserFinalize(p); + (*freeProc)(p); } +#endif /* sqlite3Parser_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. @@ -132339,7 +176372,44 @@ SQLITE_PRIVATE void sqlite3ParserFree( #ifdef YYTRACKMAXSTACKDEPTH SQLITE_PRIVATE int sqlite3ParserStackPeak(void *p){ yyParser *pParser = (yyParser*)p; - return pParser->yyidxMax; + return pParser->yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(YYCOVERAGE) +static unsigned char yycoverage[YYNSTATE][YYNTOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(YYCOVERAGE) +SQLITE_PRIVATE int sqlite3ParserCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenoyystack[pParser->yyidx].stateno; - - if( stateno>=YY_MIN_REDUCE ) return stateno; + + if( stateno>YY_MAX_SHIFT ) return stateno; assert( stateno <= YY_SHIFT_COUNT ); +#if defined(YYCOVERAGE) + yycoverage[stateno][iLookAhead] = 1; +#endif do{ i = yy_shift_ofst[stateno]; - if( i==YY_SHIFT_USE_DFLT ) return yy_default[stateno]; + assert( i>=0 ); + assert( i<=YY_ACTTAB_COUNT ); + assert( i+YYNTOKEN<=(int)YY_NLOOKAHEAD ); assert( iLookAhead!=YYNOCODE ); + assert( iLookAhead < YYNTOKEN ); i += iLookAhead; - if( i<0 || i>=YY_ACTTAB_COUNT || yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ + assert( i<(int)YY_NLOOKAHEAD ); + if( yy_lookahead[i]!=iLookAhead ){ #ifdef YYFALLBACK - YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); - } -#endif - assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sFALLBACK %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], yyTokenName[iFallback]); } +#endif + assert( yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } #endif #ifdef YYWILDCARD - { - int j = i - iLookAhead + YYWILDCARD; - if( -#if YY_SHIFT_MIN+YYWILDCARD<0 - j>=0 && -#endif -#if YY_SHIFT_MAX+YYWILDCARD>=YY_ACTTAB_COUNT - j0 ){ #ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", - yyTracePrompt, yyTokenName[iLookAhead], - yyTokenName[YYWILDCARD]); - } -#endif /* NDEBUG */ - return yy_action[j]; + if( yyTraceFILE ){ + fprintf(yyTraceFILE, "%sWILDCARD %s => %s\n", + yyTracePrompt, yyTokenName[iLookAhead], + yyTokenName[YYWILDCARD]); } +#endif /* NDEBUG */ + return yy_action[j]; } -#endif /* YYWILDCARD */ } +#endif /* YYWILDCARD */ return yy_default[stateno]; }else{ + assert( i>=0 && i<(int)(sizeof(yy_action)/sizeof(yy_action[0])) ); return yy_action[i]; } }while(1); @@ -132413,8 +176482,8 @@ static unsigned int yy_find_shift_action( ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. */ -static int yy_find_reduce_action( - int stateno, /* Current state number */ +static YYACTIONTYPE yy_find_reduce_action( + YYACTIONTYPE stateno, /* Current state number */ YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; @@ -132426,7 +176495,6 @@ static int yy_find_reduce_action( assert( stateno<=YY_REDUCE_COUNT ); #endif i = yy_reduce_ofst[stateno]; - assert( i!=YY_REDUCE_USE_DFLT ); assert( iLookAhead!=YYNOCODE ); i += iLookAhead; #ifdef YYERRORSYMBOL @@ -132444,41 +176512,43 @@ static int yy_find_reduce_action( ** The following routine is called if the stack overflows. */ static void yyStackOverflow(yyParser *yypParser){ - sqlite3ParserARG_FETCH; - yypParser->yyidx--; + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sStack Overflow!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ - sqlite3ErrorMsg(pParse, "parser stack overflow"); + sqlite3OomFault(pParse->db); /******** End %stack_overflow code ********************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument var */ + sqlite3ParserCTX_STORE } /* ** Print tracing information for a SHIFT action */ #ifndef NDEBUG -static void yyTraceShift(yyParser *yypParser, int yyNewState){ +static void yyTraceShift(yyParser *yypParser, int yyNewState, const char *zTag){ if( yyTraceFILE ){ if( yyNewStateyystack[yypParser->yyidx].major], + fprintf(yyTraceFILE,"%s%s '%s', go to state %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], yyNewState); }else{ - fprintf(yyTraceFILE,"%sShift '%s'\n", - yyTracePrompt,yyTokenName[yypParser->yystack[yypParser->yyidx].major]); + fprintf(yyTraceFILE,"%s%s '%s', pending reduce %d\n", + yyTracePrompt, zTag, yyTokenName[yypParser->yytos->major], + yyNewState - YY_MIN_REDUCE); } } } #else -# define yyTraceShift(X,Y) +# define yyTraceShift(X,Y,Z) #endif /* @@ -132486,371 +176556,863 @@ static void yyTraceShift(yyParser *yypParser, int yyNewState){ */ static void yy_shift( yyParser *yypParser, /* The parser to be shifted */ - int yyNewState, /* The new state to shift in */ - int yyMajor, /* The major token to shift in */ + YYACTIONTYPE yyNewState, /* The new state to shift in */ + YYCODETYPE yyMajor, /* The major token to shift in */ sqlite3ParserTOKENTYPE yyMinor /* The minor token to shift in */ ){ yyStackEntry *yytos; - yypParser->yyidx++; + yypParser->yytos++; #ifdef YYTRACKMAXSTACKDEPTH - if( yypParser->yyidx>yypParser->yyidxMax ){ - yypParser->yyidxMax = yypParser->yyidx; + if( (int)(yypParser->yytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == (int)(yypParser->yytos - yypParser->yystack) ); } #endif -#if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH ){ - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yyidx>=yypParser->yystksz ){ - yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz ){ + yytos = yypParser->yytos; + if( yytos>yypParser->yystackEnd ){ + if( yyGrowStack(yypParser) ){ + yypParser->yytos--; yyStackOverflow(yypParser); return; } + yytos = yypParser->yytos; + assert( yytos <= yypParser->yystackEnd ); } -#endif - yytos = &yypParser->yystack[yypParser->yyidx]; - yytos->stateno = (YYACTIONTYPE)yyNewState; - yytos->major = (YYCODETYPE)yyMajor; + if( yyNewState > YY_MAX_SHIFT ){ + yyNewState += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; + } + yytos->stateno = yyNewState; + yytos->major = yyMajor; yytos->minor.yy0 = yyMinor; - yyTraceShift(yypParser, yyNewState); + yyTraceShift(yypParser, yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - unsigned char nrhs; /* Number of right-hand side symbols in the rule */ -} yyRuleInfo[] = { - { 147, 1 }, - { 147, 3 }, - { 148, 1 }, - { 149, 3 }, - { 150, 0 }, - { 150, 1 }, - { 150, 1 }, - { 150, 1 }, - { 149, 2 }, - { 149, 2 }, - { 149, 2 }, - { 149, 2 }, - { 149, 3 }, - { 149, 5 }, - { 154, 6 }, - { 156, 1 }, - { 158, 0 }, - { 158, 3 }, - { 157, 1 }, - { 157, 0 }, - { 155, 5 }, - { 155, 2 }, - { 162, 0 }, - { 162, 2 }, - { 164, 2 }, - { 166, 0 }, - { 166, 4 }, - { 166, 6 }, - { 167, 2 }, - { 171, 2 }, - { 171, 2 }, - { 171, 4 }, - { 171, 3 }, - { 171, 3 }, - { 171, 2 }, - { 171, 3 }, - { 171, 5 }, - { 171, 2 }, - { 171, 4 }, - { 171, 4 }, - { 171, 1 }, - { 171, 2 }, - { 176, 0 }, - { 176, 1 }, - { 178, 0 }, - { 178, 2 }, - { 180, 2 }, - { 180, 3 }, - { 180, 3 }, - { 180, 3 }, - { 181, 2 }, - { 181, 2 }, - { 181, 1 }, - { 181, 1 }, - { 181, 2 }, - { 179, 3 }, - { 179, 2 }, - { 182, 0 }, - { 182, 2 }, - { 182, 2 }, - { 161, 0 }, - { 184, 1 }, - { 185, 2 }, - { 185, 7 }, - { 185, 5 }, - { 185, 5 }, - { 185, 10 }, - { 188, 0 }, - { 174, 0 }, - { 174, 3 }, - { 189, 0 }, - { 189, 2 }, - { 190, 1 }, - { 190, 1 }, - { 149, 4 }, - { 192, 2 }, - { 192, 0 }, - { 149, 9 }, - { 149, 4 }, - { 149, 1 }, - { 163, 2 }, - { 194, 3 }, - { 197, 1 }, - { 197, 2 }, - { 197, 1 }, - { 195, 9 }, - { 206, 4 }, - { 206, 5 }, - { 198, 1 }, - { 198, 1 }, - { 198, 0 }, - { 209, 0 }, - { 199, 3 }, - { 199, 2 }, - { 199, 4 }, - { 210, 2 }, - { 210, 0 }, - { 200, 0 }, - { 200, 2 }, - { 212, 2 }, - { 212, 0 }, - { 211, 7 }, - { 211, 9 }, - { 211, 7 }, - { 211, 7 }, - { 159, 0 }, - { 159, 2 }, - { 193, 2 }, - { 213, 1 }, - { 213, 2 }, - { 213, 3 }, - { 213, 4 }, - { 215, 2 }, - { 215, 0 }, - { 214, 0 }, - { 214, 3 }, - { 214, 2 }, - { 216, 4 }, - { 216, 0 }, - { 204, 0 }, - { 204, 3 }, - { 186, 4 }, - { 186, 2 }, - { 175, 1 }, - { 175, 1 }, - { 175, 0 }, - { 202, 0 }, - { 202, 3 }, - { 203, 0 }, - { 203, 2 }, - { 205, 0 }, - { 205, 2 }, - { 205, 4 }, - { 205, 4 }, - { 149, 6 }, - { 201, 0 }, - { 201, 2 }, - { 149, 8 }, - { 218, 5 }, - { 218, 3 }, - { 149, 6 }, - { 149, 7 }, - { 219, 2 }, - { 219, 1 }, - { 220, 0 }, - { 220, 3 }, - { 217, 3 }, - { 217, 1 }, - { 173, 3 }, - { 172, 1 }, - { 173, 1 }, - { 173, 1 }, - { 173, 3 }, - { 173, 5 }, - { 172, 1 }, - { 172, 1 }, - { 173, 1 }, - { 173, 3 }, - { 173, 6 }, - { 173, 5 }, - { 173, 4 }, - { 172, 1 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 173, 3 }, - { 221, 1 }, - { 221, 2 }, - { 173, 3 }, - { 173, 5 }, - { 173, 2 }, - { 173, 3 }, - { 173, 3 }, - { 173, 4 }, - { 173, 2 }, - { 173, 2 }, - { 173, 2 }, - { 173, 2 }, - { 222, 1 }, - { 222, 2 }, - { 173, 5 }, - { 223, 1 }, - { 223, 2 }, - { 173, 5 }, - { 173, 3 }, - { 173, 5 }, - { 173, 4 }, - { 173, 4 }, - { 173, 5 }, - { 225, 5 }, - { 225, 4 }, - { 226, 2 }, - { 226, 0 }, - { 224, 1 }, - { 224, 0 }, - { 208, 0 }, - { 207, 3 }, - { 207, 1 }, - { 149, 12 }, - { 227, 1 }, - { 227, 0 }, - { 177, 0 }, - { 177, 3 }, - { 187, 5 }, - { 187, 3 }, - { 228, 0 }, - { 228, 2 }, - { 149, 4 }, - { 149, 1 }, - { 149, 2 }, - { 149, 3 }, - { 149, 5 }, - { 149, 6 }, - { 149, 5 }, - { 149, 6 }, - { 169, 2 }, - { 170, 2 }, - { 149, 5 }, - { 230, 11 }, - { 232, 1 }, - { 232, 1 }, - { 232, 2 }, - { 232, 0 }, - { 233, 1 }, - { 233, 1 }, - { 233, 3 }, - { 235, 0 }, - { 235, 2 }, - { 231, 3 }, - { 231, 2 }, - { 237, 3 }, - { 238, 3 }, - { 238, 2 }, - { 236, 7 }, - { 236, 5 }, - { 236, 5 }, - { 236, 1 }, - { 173, 4 }, - { 173, 6 }, - { 191, 1 }, - { 191, 1 }, - { 191, 1 }, - { 149, 4 }, - { 149, 6 }, - { 149, 3 }, - { 240, 0 }, - { 240, 2 }, - { 149, 1 }, - { 149, 3 }, - { 149, 1 }, - { 149, 3 }, - { 149, 6 }, - { 149, 7 }, - { 241, 1 }, - { 149, 1 }, - { 149, 4 }, - { 243, 8 }, - { 245, 0 }, - { 246, 1 }, - { 246, 3 }, - { 247, 1 }, - { 196, 0 }, - { 196, 2 }, - { 196, 3 }, - { 249, 6 }, - { 249, 8 }, - { 144, 1 }, - { 145, 2 }, - { 145, 1 }, - { 146, 1 }, - { 146, 3 }, - { 147, 0 }, - { 151, 0 }, - { 151, 1 }, - { 151, 2 }, - { 153, 1 }, - { 153, 0 }, - { 149, 2 }, - { 160, 4 }, - { 160, 2 }, - { 152, 1 }, - { 152, 1 }, - { 152, 1 }, - { 166, 1 }, - { 167, 1 }, - { 168, 1 }, - { 168, 1 }, - { 165, 2 }, - { 165, 0 }, - { 171, 2 }, - { 161, 2 }, - { 183, 3 }, - { 183, 1 }, - { 184, 0 }, - { 188, 1 }, - { 190, 1 }, - { 194, 1 }, - { 195, 1 }, - { 209, 2 }, - { 210, 1 }, - { 173, 1 }, - { 208, 1 }, - { 229, 1 }, - { 229, 1 }, - { 229, 1 }, - { 229, 1 }, - { 229, 1 }, - { 169, 1 }, - { 234, 0 }, - { 234, 3 }, - { 237, 1 }, - { 238, 0 }, - { 239, 1 }, - { 239, 0 }, - { 242, 0 }, - { 242, 1 }, - { 244, 1 }, - { 244, 3 }, - { 245, 2 }, - { 248, 0 }, - { 248, 4 }, - { 248, 2 }, +/* For rule J, yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const YYCODETYPE yyRuleInfoLhs[] = { + 190, /* (0) explain ::= EXPLAIN */ + 190, /* (1) explain ::= EXPLAIN QUERY PLAN */ + 189, /* (2) cmdx ::= cmd */ + 191, /* (3) cmd ::= BEGIN transtype trans_opt */ + 192, /* (4) transtype ::= */ + 192, /* (5) transtype ::= DEFERRED */ + 192, /* (6) transtype ::= IMMEDIATE */ + 192, /* (7) transtype ::= EXCLUSIVE */ + 191, /* (8) cmd ::= COMMIT|END trans_opt */ + 191, /* (9) cmd ::= ROLLBACK trans_opt */ + 191, /* (10) cmd ::= SAVEPOINT nm */ + 191, /* (11) cmd ::= RELEASE savepoint_opt nm */ + 191, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + 196, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + 198, /* (14) createkw ::= CREATE */ + 200, /* (15) ifnotexists ::= */ + 200, /* (16) ifnotexists ::= IF NOT EXISTS */ + 199, /* (17) temp ::= TEMP */ + 199, /* (18) temp ::= */ + 197, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + 197, /* (20) create_table_args ::= AS select */ + 204, /* (21) table_option_set ::= */ + 204, /* (22) table_option_set ::= table_option_set COMMA table_option */ + 206, /* (23) table_option ::= WITHOUT nm */ + 206, /* (24) table_option ::= nm */ + 207, /* (25) columnname ::= nm typetoken */ + 209, /* (26) typetoken ::= */ + 209, /* (27) typetoken ::= typename LP signed RP */ + 209, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + 210, /* (29) typename ::= typename ID|STRING */ + 214, /* (30) scanpt ::= */ + 215, /* (31) scantok ::= */ + 216, /* (32) ccons ::= CONSTRAINT nm */ + 216, /* (33) ccons ::= DEFAULT scantok term */ + 216, /* (34) ccons ::= DEFAULT LP expr RP */ + 216, /* (35) ccons ::= DEFAULT PLUS scantok term */ + 216, /* (36) ccons ::= DEFAULT MINUS scantok term */ + 216, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + 216, /* (38) ccons ::= NOT NULL onconf */ + 216, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + 216, /* (40) ccons ::= UNIQUE onconf */ + 216, /* (41) ccons ::= CHECK LP expr RP */ + 216, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + 216, /* (43) ccons ::= defer_subclause */ + 216, /* (44) ccons ::= COLLATE ID|STRING */ + 225, /* (45) generated ::= LP expr RP */ + 225, /* (46) generated ::= LP expr RP ID */ + 221, /* (47) autoinc ::= */ + 221, /* (48) autoinc ::= AUTOINCR */ + 223, /* (49) refargs ::= */ + 223, /* (50) refargs ::= refargs refarg */ + 226, /* (51) refarg ::= MATCH nm */ + 226, /* (52) refarg ::= ON INSERT refact */ + 226, /* (53) refarg ::= ON DELETE refact */ + 226, /* (54) refarg ::= ON UPDATE refact */ + 227, /* (55) refact ::= SET NULL */ + 227, /* (56) refact ::= SET DEFAULT */ + 227, /* (57) refact ::= CASCADE */ + 227, /* (58) refact ::= RESTRICT */ + 227, /* (59) refact ::= NO ACTION */ + 224, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + 224, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 228, /* (62) init_deferred_pred_opt ::= */ + 228, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + 228, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 203, /* (65) conslist_opt ::= */ + 230, /* (66) tconscomma ::= COMMA */ + 231, /* (67) tcons ::= CONSTRAINT nm */ + 231, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + 231, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + 231, /* (70) tcons ::= CHECK LP expr RP onconf */ + 231, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 234, /* (72) defer_subclause_opt ::= */ + 219, /* (73) onconf ::= */ + 219, /* (74) onconf ::= ON CONFLICT resolvetype */ + 235, /* (75) orconf ::= */ + 235, /* (76) orconf ::= OR resolvetype */ + 236, /* (77) resolvetype ::= IGNORE */ + 236, /* (78) resolvetype ::= REPLACE */ + 191, /* (79) cmd ::= DROP TABLE ifexists fullname */ + 238, /* (80) ifexists ::= IF EXISTS */ + 238, /* (81) ifexists ::= */ + 191, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + 191, /* (83) cmd ::= DROP VIEW ifexists fullname */ + 191, /* (84) cmd ::= select */ + 205, /* (85) select ::= WITH wqlist selectnowith */ + 205, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + 205, /* (87) select ::= selectnowith */ + 240, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + 243, /* (89) multiselect_op ::= UNION */ + 243, /* (90) multiselect_op ::= UNION ALL */ + 243, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + 241, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + 241, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + 253, /* (94) values ::= VALUES LP nexprlist RP */ + 241, /* (95) oneselect ::= mvalues */ + 255, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + 255, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + 244, /* (98) distinct ::= DISTINCT */ + 244, /* (99) distinct ::= ALL */ + 244, /* (100) distinct ::= */ + 256, /* (101) sclp ::= */ + 245, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + 245, /* (103) selcollist ::= sclp scanpt STAR */ + 245, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + 257, /* (105) as ::= AS nm */ + 257, /* (106) as ::= */ + 246, /* (107) from ::= */ + 246, /* (108) from ::= FROM seltablist */ + 259, /* (109) stl_prefix ::= seltablist joinop */ + 259, /* (110) stl_prefix ::= */ + 258, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + 258, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + 258, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + 258, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + 258, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 201, /* (116) dbnm ::= */ + 201, /* (117) dbnm ::= DOT nm */ + 239, /* (118) fullname ::= nm */ + 239, /* (119) fullname ::= nm DOT nm */ + 264, /* (120) xfullname ::= nm */ + 264, /* (121) xfullname ::= nm DOT nm */ + 264, /* (122) xfullname ::= nm DOT nm AS nm */ + 264, /* (123) xfullname ::= nm AS nm */ + 260, /* (124) joinop ::= COMMA|JOIN */ + 260, /* (125) joinop ::= JOIN_KW JOIN */ + 260, /* (126) joinop ::= JOIN_KW nm JOIN */ + 260, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + 261, /* (128) on_using ::= ON expr */ + 261, /* (129) on_using ::= USING LP idlist RP */ + 261, /* (130) on_using ::= */ + 266, /* (131) indexed_opt ::= */ + 262, /* (132) indexed_by ::= INDEXED BY nm */ + 262, /* (133) indexed_by ::= NOT INDEXED */ + 250, /* (134) orderby_opt ::= */ + 250, /* (135) orderby_opt ::= ORDER BY sortlist */ + 232, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + 232, /* (137) sortlist ::= expr sortorder nulls */ + 220, /* (138) sortorder ::= ASC */ + 220, /* (139) sortorder ::= DESC */ + 220, /* (140) sortorder ::= */ + 267, /* (141) nulls ::= NULLS FIRST */ + 267, /* (142) nulls ::= NULLS LAST */ + 267, /* (143) nulls ::= */ + 248, /* (144) groupby_opt ::= */ + 248, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 249, /* (146) having_opt ::= */ + 249, /* (147) having_opt ::= HAVING expr */ + 251, /* (148) limit_opt ::= */ + 251, /* (149) limit_opt ::= LIMIT expr */ + 251, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + 251, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + 191, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 247, /* (153) where_opt ::= */ + 247, /* (154) where_opt ::= WHERE expr */ + 269, /* (155) where_opt_ret ::= */ + 269, /* (156) where_opt_ret ::= WHERE expr */ + 269, /* (157) where_opt_ret ::= RETURNING selcollist */ + 269, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + 191, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + 270, /* (160) setlist ::= setlist COMMA nm EQ expr */ + 270, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + 270, /* (162) setlist ::= nm EQ expr */ + 270, /* (163) setlist ::= LP idlist RP EQ expr */ + 191, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + 191, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 273, /* (166) upsert ::= */ + 273, /* (167) upsert ::= RETURNING selcollist */ + 273, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + 273, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + 273, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + 273, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + 274, /* (172) returning ::= RETURNING selcollist */ + 271, /* (173) insert_cmd ::= INSERT orconf */ + 271, /* (174) insert_cmd ::= REPLACE */ + 272, /* (175) idlist_opt ::= */ + 272, /* (176) idlist_opt ::= LP idlist RP */ + 265, /* (177) idlist ::= idlist COMMA nm */ + 265, /* (178) idlist ::= nm */ + 218, /* (179) expr ::= LP expr RP */ + 218, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + 218, /* (181) expr ::= nm DOT nm */ + 218, /* (182) expr ::= nm DOT nm DOT nm */ + 217, /* (183) term ::= NULL|FLOAT|BLOB */ + 217, /* (184) term ::= STRING */ + 217, /* (185) term ::= INTEGER */ + 218, /* (186) expr ::= VARIABLE */ + 218, /* (187) expr ::= expr COLLATE ID|STRING */ + 218, /* (188) expr ::= CAST LP expr AS typetoken RP */ + 218, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + 218, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + 218, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + 218, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + 218, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + 218, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + 217, /* (195) term ::= CTIME_KW */ + 218, /* (196) expr ::= LP nexprlist COMMA expr RP */ + 218, /* (197) expr ::= expr AND expr */ + 218, /* (198) expr ::= expr OR expr */ + 218, /* (199) expr ::= expr LT|GT|GE|LE expr */ + 218, /* (200) expr ::= expr EQ|NE expr */ + 218, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + 218, /* (202) expr ::= expr PLUS|MINUS expr */ + 218, /* (203) expr ::= expr STAR|SLASH|REM expr */ + 218, /* (204) expr ::= expr CONCAT expr */ + 276, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + 218, /* (206) expr ::= expr likeop expr */ + 218, /* (207) expr ::= expr likeop expr ESCAPE expr */ + 218, /* (208) expr ::= expr ISNULL|NOTNULL */ + 218, /* (209) expr ::= expr NOT NULL */ + 218, /* (210) expr ::= expr IS expr */ + 218, /* (211) expr ::= expr IS NOT expr */ + 218, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + 218, /* (213) expr ::= expr IS DISTINCT FROM expr */ + 218, /* (214) expr ::= NOT expr */ + 218, /* (215) expr ::= BITNOT expr */ + 218, /* (216) expr ::= PLUS|MINUS expr */ + 218, /* (217) expr ::= expr PTR expr */ + 277, /* (218) between_op ::= BETWEEN */ + 277, /* (219) between_op ::= NOT BETWEEN */ + 218, /* (220) expr ::= expr between_op expr AND expr */ + 278, /* (221) in_op ::= IN */ + 278, /* (222) in_op ::= NOT IN */ + 218, /* (223) expr ::= expr in_op LP exprlist RP */ + 218, /* (224) expr ::= LP select RP */ + 218, /* (225) expr ::= expr in_op LP select RP */ + 218, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + 218, /* (227) expr ::= EXISTS LP select RP */ + 218, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + 281, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + 281, /* (230) case_exprlist ::= WHEN expr THEN expr */ + 282, /* (231) case_else ::= ELSE expr */ + 282, /* (232) case_else ::= */ + 280, /* (233) case_operand ::= */ + 263, /* (234) exprlist ::= */ + 254, /* (235) nexprlist ::= nexprlist COMMA expr */ + 254, /* (236) nexprlist ::= expr */ + 279, /* (237) paren_exprlist ::= */ + 279, /* (238) paren_exprlist ::= LP exprlist RP */ + 191, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + 283, /* (240) uniqueflag ::= UNIQUE */ + 283, /* (241) uniqueflag ::= */ + 222, /* (242) eidlist_opt ::= */ + 222, /* (243) eidlist_opt ::= LP eidlist RP */ + 233, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + 233, /* (245) eidlist ::= nm collate sortorder */ + 284, /* (246) collate ::= */ + 284, /* (247) collate ::= COLLATE ID|STRING */ + 191, /* (248) cmd ::= DROP INDEX ifexists fullname */ + 191, /* (249) cmd ::= VACUUM vinto */ + 191, /* (250) cmd ::= VACUUM nm vinto */ + 285, /* (251) vinto ::= INTO expr */ + 285, /* (252) vinto ::= */ + 191, /* (253) cmd ::= PRAGMA nm dbnm */ + 191, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + 191, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + 191, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + 191, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + 212, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + 213, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + 191, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + 287, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + 289, /* (262) trigger_time ::= BEFORE|AFTER */ + 289, /* (263) trigger_time ::= INSTEAD OF */ + 289, /* (264) trigger_time ::= */ + 290, /* (265) trigger_event ::= DELETE|INSERT */ + 290, /* (266) trigger_event ::= UPDATE */ + 290, /* (267) trigger_event ::= UPDATE OF idlist */ + 292, /* (268) when_clause ::= */ + 292, /* (269) when_clause ::= WHEN expr */ + 288, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + 288, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + 294, /* (272) trnm ::= nm DOT nm */ + 295, /* (273) tridxby ::= INDEXED BY nm */ + 295, /* (274) tridxby ::= NOT INDEXED */ + 293, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + 293, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + 293, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + 293, /* (278) trigger_cmd ::= scanpt select scanpt */ + 218, /* (279) expr ::= RAISE LP IGNORE RP */ + 218, /* (280) expr ::= RAISE LP raisetype COMMA expr RP */ + 237, /* (281) raisetype ::= ROLLBACK */ + 237, /* (282) raisetype ::= ABORT */ + 237, /* (283) raisetype ::= FAIL */ + 191, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + 191, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + 191, /* (286) cmd ::= DETACH database_kw_opt expr */ + 297, /* (287) key_opt ::= */ + 297, /* (288) key_opt ::= KEY expr */ + 191, /* (289) cmd ::= REINDEX */ + 191, /* (290) cmd ::= REINDEX nm dbnm */ + 191, /* (291) cmd ::= ANALYZE */ + 191, /* (292) cmd ::= ANALYZE nm dbnm */ + 191, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + 191, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + 191, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + 298, /* (296) add_column_fullname ::= fullname */ + 191, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + 191, /* (298) cmd ::= create_vtab */ + 191, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + 300, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 302, /* (301) vtabarg ::= */ + 303, /* (302) vtabargtoken ::= ANY */ + 303, /* (303) vtabargtoken ::= lp anylist RP */ + 304, /* (304) lp ::= LP */ + 268, /* (305) with ::= WITH wqlist */ + 268, /* (306) with ::= WITH RECURSIVE wqlist */ + 307, /* (307) wqas ::= AS */ + 307, /* (308) wqas ::= AS MATERIALIZED */ + 307, /* (309) wqas ::= AS NOT MATERIALIZED */ + 306, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + 308, /* (311) withnm ::= nm */ + 242, /* (312) wqlist ::= wqitem */ + 242, /* (313) wqlist ::= wqlist COMMA wqitem */ + 309, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + 310, /* (315) windowdefn ::= nm AS LP window RP */ + 311, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + 311, /* (318) window ::= ORDER BY sortlist frame_opt */ + 311, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + 311, /* (320) window ::= nm frame_opt */ + 312, /* (321) frame_opt ::= */ + 312, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + 312, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + 316, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + 318, /* (325) frame_bound_s ::= frame_bound */ + 318, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + 319, /* (327) frame_bound_e ::= frame_bound */ + 319, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + 317, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + 317, /* (330) frame_bound ::= CURRENT ROW */ + 320, /* (331) frame_exclude_opt ::= */ + 320, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + 321, /* (333) frame_exclude ::= NO OTHERS */ + 321, /* (334) frame_exclude ::= CURRENT ROW */ + 321, /* (335) frame_exclude ::= GROUP|TIES */ + 252, /* (336) window_clause ::= WINDOW windowdefn_list */ + 275, /* (337) filter_over ::= filter_clause over_clause */ + 275, /* (338) filter_over ::= over_clause */ + 275, /* (339) filter_over ::= filter_clause */ + 315, /* (340) over_clause ::= OVER LP window RP */ + 315, /* (341) over_clause ::= OVER nm */ + 314, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + 217, /* (343) term ::= QNUMBER */ + 186, /* (344) input ::= cmdlist */ + 187, /* (345) cmdlist ::= cmdlist ecmd */ + 187, /* (346) cmdlist ::= ecmd */ + 188, /* (347) ecmd ::= SEMI */ + 188, /* (348) ecmd ::= cmdx SEMI */ + 188, /* (349) ecmd ::= explain cmdx SEMI */ + 193, /* (350) trans_opt ::= */ + 193, /* (351) trans_opt ::= TRANSACTION */ + 193, /* (352) trans_opt ::= TRANSACTION nm */ + 195, /* (353) savepoint_opt ::= SAVEPOINT */ + 195, /* (354) savepoint_opt ::= */ + 191, /* (355) cmd ::= create_table create_table_args */ + 204, /* (356) table_option_set ::= table_option */ + 202, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + 202, /* (358) columnlist ::= columnname carglist */ + 194, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + 194, /* (360) nm ::= STRING */ + 209, /* (361) typetoken ::= typename */ + 210, /* (362) typename ::= ID|STRING */ + 211, /* (363) signed ::= plus_num */ + 211, /* (364) signed ::= minus_num */ + 208, /* (365) carglist ::= carglist ccons */ + 208, /* (366) carglist ::= */ + 216, /* (367) ccons ::= NULL onconf */ + 216, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + 216, /* (369) ccons ::= AS generated */ + 203, /* (370) conslist_opt ::= COMMA conslist */ + 229, /* (371) conslist ::= conslist tconscomma tcons */ + 229, /* (372) conslist ::= tcons */ + 230, /* (373) tconscomma ::= */ + 234, /* (374) defer_subclause_opt ::= defer_subclause */ + 236, /* (375) resolvetype ::= raisetype */ + 240, /* (376) selectnowith ::= oneselect */ + 241, /* (377) oneselect ::= values */ + 256, /* (378) sclp ::= selcollist COMMA */ + 257, /* (379) as ::= ID|STRING */ + 266, /* (380) indexed_opt ::= indexed_by */ + 274, /* (381) returning ::= */ + 218, /* (382) expr ::= term */ + 276, /* (383) likeop ::= LIKE_KW|MATCH */ + 280, /* (384) case_operand ::= expr */ + 263, /* (385) exprlist ::= nexprlist */ + 286, /* (386) nmnum ::= plus_num */ + 286, /* (387) nmnum ::= nm */ + 286, /* (388) nmnum ::= ON */ + 286, /* (389) nmnum ::= DELETE */ + 286, /* (390) nmnum ::= DEFAULT */ + 212, /* (391) plus_num ::= INTEGER|FLOAT */ + 291, /* (392) foreach_clause ::= */ + 291, /* (393) foreach_clause ::= FOR EACH ROW */ + 294, /* (394) trnm ::= nm */ + 295, /* (395) tridxby ::= */ + 296, /* (396) database_kw_opt ::= DATABASE */ + 296, /* (397) database_kw_opt ::= */ + 299, /* (398) kwcolumn_opt ::= */ + 299, /* (399) kwcolumn_opt ::= COLUMNKW */ + 301, /* (400) vtabarglist ::= vtabarg */ + 301, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + 302, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 305, /* (403) anylist ::= */ + 305, /* (404) anylist ::= anylist LP anylist RP */ + 305, /* (405) anylist ::= anylist ANY */ + 268, /* (406) with ::= */ + 309, /* (407) windowdefn_list ::= windowdefn */ + 311, /* (408) window ::= frame_opt */ +}; + +/* For rule J, yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char yyRuleInfoNRhs[] = { + -1, /* (0) explain ::= EXPLAIN */ + -3, /* (1) explain ::= EXPLAIN QUERY PLAN */ + -1, /* (2) cmdx ::= cmd */ + -3, /* (3) cmd ::= BEGIN transtype trans_opt */ + 0, /* (4) transtype ::= */ + -1, /* (5) transtype ::= DEFERRED */ + -1, /* (6) transtype ::= IMMEDIATE */ + -1, /* (7) transtype ::= EXCLUSIVE */ + -2, /* (8) cmd ::= COMMIT|END trans_opt */ + -2, /* (9) cmd ::= ROLLBACK trans_opt */ + -2, /* (10) cmd ::= SAVEPOINT nm */ + -3, /* (11) cmd ::= RELEASE savepoint_opt nm */ + -5, /* (12) cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + -6, /* (13) create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + -1, /* (14) createkw ::= CREATE */ + 0, /* (15) ifnotexists ::= */ + -3, /* (16) ifnotexists ::= IF NOT EXISTS */ + -1, /* (17) temp ::= TEMP */ + 0, /* (18) temp ::= */ + -5, /* (19) create_table_args ::= LP columnlist conslist_opt RP table_option_set */ + -2, /* (20) create_table_args ::= AS select */ + 0, /* (21) table_option_set ::= */ + -3, /* (22) table_option_set ::= table_option_set COMMA table_option */ + -2, /* (23) table_option ::= WITHOUT nm */ + -1, /* (24) table_option ::= nm */ + -2, /* (25) columnname ::= nm typetoken */ + 0, /* (26) typetoken ::= */ + -4, /* (27) typetoken ::= typename LP signed RP */ + -6, /* (28) typetoken ::= typename LP signed COMMA signed RP */ + -2, /* (29) typename ::= typename ID|STRING */ + 0, /* (30) scanpt ::= */ + 0, /* (31) scantok ::= */ + -2, /* (32) ccons ::= CONSTRAINT nm */ + -3, /* (33) ccons ::= DEFAULT scantok term */ + -4, /* (34) ccons ::= DEFAULT LP expr RP */ + -4, /* (35) ccons ::= DEFAULT PLUS scantok term */ + -4, /* (36) ccons ::= DEFAULT MINUS scantok term */ + -3, /* (37) ccons ::= DEFAULT scantok ID|INDEXED */ + -3, /* (38) ccons ::= NOT NULL onconf */ + -5, /* (39) ccons ::= PRIMARY KEY sortorder onconf autoinc */ + -2, /* (40) ccons ::= UNIQUE onconf */ + -4, /* (41) ccons ::= CHECK LP expr RP */ + -4, /* (42) ccons ::= REFERENCES nm eidlist_opt refargs */ + -1, /* (43) ccons ::= defer_subclause */ + -2, /* (44) ccons ::= COLLATE ID|STRING */ + -3, /* (45) generated ::= LP expr RP */ + -4, /* (46) generated ::= LP expr RP ID */ + 0, /* (47) autoinc ::= */ + -1, /* (48) autoinc ::= AUTOINCR */ + 0, /* (49) refargs ::= */ + -2, /* (50) refargs ::= refargs refarg */ + -2, /* (51) refarg ::= MATCH nm */ + -3, /* (52) refarg ::= ON INSERT refact */ + -3, /* (53) refarg ::= ON DELETE refact */ + -3, /* (54) refarg ::= ON UPDATE refact */ + -2, /* (55) refact ::= SET NULL */ + -2, /* (56) refact ::= SET DEFAULT */ + -1, /* (57) refact ::= CASCADE */ + -1, /* (58) refact ::= RESTRICT */ + -2, /* (59) refact ::= NO ACTION */ + -3, /* (60) defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ + -2, /* (61) defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + 0, /* (62) init_deferred_pred_opt ::= */ + -2, /* (63) init_deferred_pred_opt ::= INITIALLY DEFERRED */ + -2, /* (64) init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ + 0, /* (65) conslist_opt ::= */ + -1, /* (66) tconscomma ::= COMMA */ + -2, /* (67) tcons ::= CONSTRAINT nm */ + -7, /* (68) tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ + -5, /* (69) tcons ::= UNIQUE LP sortlist RP onconf */ + -5, /* (70) tcons ::= CHECK LP expr RP onconf */ + -10, /* (71) tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + 0, /* (72) defer_subclause_opt ::= */ + 0, /* (73) onconf ::= */ + -3, /* (74) onconf ::= ON CONFLICT resolvetype */ + 0, /* (75) orconf ::= */ + -2, /* (76) orconf ::= OR resolvetype */ + -1, /* (77) resolvetype ::= IGNORE */ + -1, /* (78) resolvetype ::= REPLACE */ + -4, /* (79) cmd ::= DROP TABLE ifexists fullname */ + -2, /* (80) ifexists ::= IF EXISTS */ + 0, /* (81) ifexists ::= */ + -9, /* (82) cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + -4, /* (83) cmd ::= DROP VIEW ifexists fullname */ + -1, /* (84) cmd ::= select */ + -3, /* (85) select ::= WITH wqlist selectnowith */ + -4, /* (86) select ::= WITH RECURSIVE wqlist selectnowith */ + -1, /* (87) select ::= selectnowith */ + -3, /* (88) selectnowith ::= selectnowith multiselect_op oneselect */ + -1, /* (89) multiselect_op ::= UNION */ + -2, /* (90) multiselect_op ::= UNION ALL */ + -1, /* (91) multiselect_op ::= EXCEPT|INTERSECT */ + -9, /* (92) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + -10, /* (93) oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ + -4, /* (94) values ::= VALUES LP nexprlist RP */ + -1, /* (95) oneselect ::= mvalues */ + -5, /* (96) mvalues ::= values COMMA LP nexprlist RP */ + -5, /* (97) mvalues ::= mvalues COMMA LP nexprlist RP */ + -1, /* (98) distinct ::= DISTINCT */ + -1, /* (99) distinct ::= ALL */ + 0, /* (100) distinct ::= */ + 0, /* (101) sclp ::= */ + -5, /* (102) selcollist ::= sclp scanpt expr scanpt as */ + -3, /* (103) selcollist ::= sclp scanpt STAR */ + -5, /* (104) selcollist ::= sclp scanpt nm DOT STAR */ + -2, /* (105) as ::= AS nm */ + 0, /* (106) as ::= */ + 0, /* (107) from ::= */ + -2, /* (108) from ::= FROM seltablist */ + -2, /* (109) stl_prefix ::= seltablist joinop */ + 0, /* (110) stl_prefix ::= */ + -5, /* (111) seltablist ::= stl_prefix nm dbnm as on_using */ + -6, /* (112) seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ + -8, /* (113) seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ + -6, /* (114) seltablist ::= stl_prefix LP select RP as on_using */ + -6, /* (115) seltablist ::= stl_prefix LP seltablist RP as on_using */ + 0, /* (116) dbnm ::= */ + -2, /* (117) dbnm ::= DOT nm */ + -1, /* (118) fullname ::= nm */ + -3, /* (119) fullname ::= nm DOT nm */ + -1, /* (120) xfullname ::= nm */ + -3, /* (121) xfullname ::= nm DOT nm */ + -5, /* (122) xfullname ::= nm DOT nm AS nm */ + -3, /* (123) xfullname ::= nm AS nm */ + -1, /* (124) joinop ::= COMMA|JOIN */ + -2, /* (125) joinop ::= JOIN_KW JOIN */ + -3, /* (126) joinop ::= JOIN_KW nm JOIN */ + -4, /* (127) joinop ::= JOIN_KW nm nm JOIN */ + -2, /* (128) on_using ::= ON expr */ + -4, /* (129) on_using ::= USING LP idlist RP */ + 0, /* (130) on_using ::= */ + 0, /* (131) indexed_opt ::= */ + -3, /* (132) indexed_by ::= INDEXED BY nm */ + -2, /* (133) indexed_by ::= NOT INDEXED */ + 0, /* (134) orderby_opt ::= */ + -3, /* (135) orderby_opt ::= ORDER BY sortlist */ + -5, /* (136) sortlist ::= sortlist COMMA expr sortorder nulls */ + -3, /* (137) sortlist ::= expr sortorder nulls */ + -1, /* (138) sortorder ::= ASC */ + -1, /* (139) sortorder ::= DESC */ + 0, /* (140) sortorder ::= */ + -2, /* (141) nulls ::= NULLS FIRST */ + -2, /* (142) nulls ::= NULLS LAST */ + 0, /* (143) nulls ::= */ + 0, /* (144) groupby_opt ::= */ + -3, /* (145) groupby_opt ::= GROUP BY nexprlist */ + 0, /* (146) having_opt ::= */ + -2, /* (147) having_opt ::= HAVING expr */ + 0, /* (148) limit_opt ::= */ + -2, /* (149) limit_opt ::= LIMIT expr */ + -4, /* (150) limit_opt ::= LIMIT expr OFFSET expr */ + -4, /* (151) limit_opt ::= LIMIT expr COMMA expr */ + -6, /* (152) cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ + 0, /* (153) where_opt ::= */ + -2, /* (154) where_opt ::= WHERE expr */ + 0, /* (155) where_opt_ret ::= */ + -2, /* (156) where_opt_ret ::= WHERE expr */ + -2, /* (157) where_opt_ret ::= RETURNING selcollist */ + -4, /* (158) where_opt_ret ::= WHERE expr RETURNING selcollist */ + -9, /* (159) cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ + -5, /* (160) setlist ::= setlist COMMA nm EQ expr */ + -7, /* (161) setlist ::= setlist COMMA LP idlist RP EQ expr */ + -3, /* (162) setlist ::= nm EQ expr */ + -5, /* (163) setlist ::= LP idlist RP EQ expr */ + -7, /* (164) cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ + -8, /* (165) cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ + 0, /* (166) upsert ::= */ + -2, /* (167) upsert ::= RETURNING selcollist */ + -12, /* (168) upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ + -9, /* (169) upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ + -5, /* (170) upsert ::= ON CONFLICT DO NOTHING returning */ + -8, /* (171) upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ + -2, /* (172) returning ::= RETURNING selcollist */ + -2, /* (173) insert_cmd ::= INSERT orconf */ + -1, /* (174) insert_cmd ::= REPLACE */ + 0, /* (175) idlist_opt ::= */ + -3, /* (176) idlist_opt ::= LP idlist RP */ + -3, /* (177) idlist ::= idlist COMMA nm */ + -1, /* (178) idlist ::= nm */ + -3, /* (179) expr ::= LP expr RP */ + -1, /* (180) expr ::= ID|INDEXED|JOIN_KW */ + -3, /* (181) expr ::= nm DOT nm */ + -5, /* (182) expr ::= nm DOT nm DOT nm */ + -1, /* (183) term ::= NULL|FLOAT|BLOB */ + -1, /* (184) term ::= STRING */ + -1, /* (185) term ::= INTEGER */ + -1, /* (186) expr ::= VARIABLE */ + -3, /* (187) expr ::= expr COLLATE ID|STRING */ + -6, /* (188) expr ::= CAST LP expr AS typetoken RP */ + -5, /* (189) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ + -8, /* (190) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ + -4, /* (191) expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ + -6, /* (192) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ + -9, /* (193) expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ + -5, /* (194) expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ + -1, /* (195) term ::= CTIME_KW */ + -5, /* (196) expr ::= LP nexprlist COMMA expr RP */ + -3, /* (197) expr ::= expr AND expr */ + -3, /* (198) expr ::= expr OR expr */ + -3, /* (199) expr ::= expr LT|GT|GE|LE expr */ + -3, /* (200) expr ::= expr EQ|NE expr */ + -3, /* (201) expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ + -3, /* (202) expr ::= expr PLUS|MINUS expr */ + -3, /* (203) expr ::= expr STAR|SLASH|REM expr */ + -3, /* (204) expr ::= expr CONCAT expr */ + -2, /* (205) likeop ::= NOT LIKE_KW|MATCH */ + -3, /* (206) expr ::= expr likeop expr */ + -5, /* (207) expr ::= expr likeop expr ESCAPE expr */ + -2, /* (208) expr ::= expr ISNULL|NOTNULL */ + -3, /* (209) expr ::= expr NOT NULL */ + -3, /* (210) expr ::= expr IS expr */ + -4, /* (211) expr ::= expr IS NOT expr */ + -6, /* (212) expr ::= expr IS NOT DISTINCT FROM expr */ + -5, /* (213) expr ::= expr IS DISTINCT FROM expr */ + -2, /* (214) expr ::= NOT expr */ + -2, /* (215) expr ::= BITNOT expr */ + -2, /* (216) expr ::= PLUS|MINUS expr */ + -3, /* (217) expr ::= expr PTR expr */ + -1, /* (218) between_op ::= BETWEEN */ + -2, /* (219) between_op ::= NOT BETWEEN */ + -5, /* (220) expr ::= expr between_op expr AND expr */ + -1, /* (221) in_op ::= IN */ + -2, /* (222) in_op ::= NOT IN */ + -5, /* (223) expr ::= expr in_op LP exprlist RP */ + -3, /* (224) expr ::= LP select RP */ + -5, /* (225) expr ::= expr in_op LP select RP */ + -5, /* (226) expr ::= expr in_op nm dbnm paren_exprlist */ + -4, /* (227) expr ::= EXISTS LP select RP */ + -5, /* (228) expr ::= CASE case_operand case_exprlist case_else END */ + -5, /* (229) case_exprlist ::= case_exprlist WHEN expr THEN expr */ + -4, /* (230) case_exprlist ::= WHEN expr THEN expr */ + -2, /* (231) case_else ::= ELSE expr */ + 0, /* (232) case_else ::= */ + 0, /* (233) case_operand ::= */ + 0, /* (234) exprlist ::= */ + -3, /* (235) nexprlist ::= nexprlist COMMA expr */ + -1, /* (236) nexprlist ::= expr */ + 0, /* (237) paren_exprlist ::= */ + -3, /* (238) paren_exprlist ::= LP exprlist RP */ + -12, /* (239) cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + -1, /* (240) uniqueflag ::= UNIQUE */ + 0, /* (241) uniqueflag ::= */ + 0, /* (242) eidlist_opt ::= */ + -3, /* (243) eidlist_opt ::= LP eidlist RP */ + -5, /* (244) eidlist ::= eidlist COMMA nm collate sortorder */ + -3, /* (245) eidlist ::= nm collate sortorder */ + 0, /* (246) collate ::= */ + -2, /* (247) collate ::= COLLATE ID|STRING */ + -4, /* (248) cmd ::= DROP INDEX ifexists fullname */ + -2, /* (249) cmd ::= VACUUM vinto */ + -3, /* (250) cmd ::= VACUUM nm vinto */ + -2, /* (251) vinto ::= INTO expr */ + 0, /* (252) vinto ::= */ + -3, /* (253) cmd ::= PRAGMA nm dbnm */ + -5, /* (254) cmd ::= PRAGMA nm dbnm EQ nmnum */ + -6, /* (255) cmd ::= PRAGMA nm dbnm LP nmnum RP */ + -5, /* (256) cmd ::= PRAGMA nm dbnm EQ minus_num */ + -6, /* (257) cmd ::= PRAGMA nm dbnm LP minus_num RP */ + -2, /* (258) plus_num ::= PLUS INTEGER|FLOAT */ + -2, /* (259) minus_num ::= MINUS INTEGER|FLOAT */ + -5, /* (260) cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + -11, /* (261) trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + -1, /* (262) trigger_time ::= BEFORE|AFTER */ + -2, /* (263) trigger_time ::= INSTEAD OF */ + 0, /* (264) trigger_time ::= */ + -1, /* (265) trigger_event ::= DELETE|INSERT */ + -1, /* (266) trigger_event ::= UPDATE */ + -3, /* (267) trigger_event ::= UPDATE OF idlist */ + 0, /* (268) when_clause ::= */ + -2, /* (269) when_clause ::= WHEN expr */ + -3, /* (270) trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + -2, /* (271) trigger_cmd_list ::= trigger_cmd SEMI */ + -3, /* (272) trnm ::= nm DOT nm */ + -3, /* (273) tridxby ::= INDEXED BY nm */ + -2, /* (274) tridxby ::= NOT INDEXED */ + -9, /* (275) trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ + -8, /* (276) trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ + -6, /* (277) trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ + -3, /* (278) trigger_cmd ::= scanpt select scanpt */ + -4, /* (279) expr ::= RAISE LP IGNORE RP */ + -6, /* (280) expr ::= RAISE LP raisetype COMMA expr RP */ + -1, /* (281) raisetype ::= ROLLBACK */ + -1, /* (282) raisetype ::= ABORT */ + -1, /* (283) raisetype ::= FAIL */ + -4, /* (284) cmd ::= DROP TRIGGER ifexists fullname */ + -6, /* (285) cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + -3, /* (286) cmd ::= DETACH database_kw_opt expr */ + 0, /* (287) key_opt ::= */ + -2, /* (288) key_opt ::= KEY expr */ + -1, /* (289) cmd ::= REINDEX */ + -3, /* (290) cmd ::= REINDEX nm dbnm */ + -1, /* (291) cmd ::= ANALYZE */ + -3, /* (292) cmd ::= ANALYZE nm dbnm */ + -6, /* (293) cmd ::= ALTER TABLE fullname RENAME TO nm */ + -7, /* (294) cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + -6, /* (295) cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ + -1, /* (296) add_column_fullname ::= fullname */ + -8, /* (297) cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ + -1, /* (298) cmd ::= create_vtab */ + -4, /* (299) cmd ::= create_vtab LP vtabarglist RP */ + -8, /* (300) create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + 0, /* (301) vtabarg ::= */ + -1, /* (302) vtabargtoken ::= ANY */ + -3, /* (303) vtabargtoken ::= lp anylist RP */ + -1, /* (304) lp ::= LP */ + -2, /* (305) with ::= WITH wqlist */ + -3, /* (306) with ::= WITH RECURSIVE wqlist */ + -1, /* (307) wqas ::= AS */ + -2, /* (308) wqas ::= AS MATERIALIZED */ + -3, /* (309) wqas ::= AS NOT MATERIALIZED */ + -6, /* (310) wqitem ::= withnm eidlist_opt wqas LP select RP */ + -1, /* (311) withnm ::= nm */ + -1, /* (312) wqlist ::= wqitem */ + -3, /* (313) wqlist ::= wqlist COMMA wqitem */ + -3, /* (314) windowdefn_list ::= windowdefn_list COMMA windowdefn */ + -5, /* (315) windowdefn ::= nm AS LP window RP */ + -5, /* (316) window ::= PARTITION BY nexprlist orderby_opt frame_opt */ + -6, /* (317) window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ + -4, /* (318) window ::= ORDER BY sortlist frame_opt */ + -5, /* (319) window ::= nm ORDER BY sortlist frame_opt */ + -2, /* (320) window ::= nm frame_opt */ + 0, /* (321) frame_opt ::= */ + -3, /* (322) frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ + -6, /* (323) frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ + -1, /* (324) range_or_rows ::= RANGE|ROWS|GROUPS */ + -1, /* (325) frame_bound_s ::= frame_bound */ + -2, /* (326) frame_bound_s ::= UNBOUNDED PRECEDING */ + -1, /* (327) frame_bound_e ::= frame_bound */ + -2, /* (328) frame_bound_e ::= UNBOUNDED FOLLOWING */ + -2, /* (329) frame_bound ::= expr PRECEDING|FOLLOWING */ + -2, /* (330) frame_bound ::= CURRENT ROW */ + 0, /* (331) frame_exclude_opt ::= */ + -2, /* (332) frame_exclude_opt ::= EXCLUDE frame_exclude */ + -2, /* (333) frame_exclude ::= NO OTHERS */ + -2, /* (334) frame_exclude ::= CURRENT ROW */ + -1, /* (335) frame_exclude ::= GROUP|TIES */ + -2, /* (336) window_clause ::= WINDOW windowdefn_list */ + -2, /* (337) filter_over ::= filter_clause over_clause */ + -1, /* (338) filter_over ::= over_clause */ + -1, /* (339) filter_over ::= filter_clause */ + -4, /* (340) over_clause ::= OVER LP window RP */ + -2, /* (341) over_clause ::= OVER nm */ + -5, /* (342) filter_clause ::= FILTER LP WHERE expr RP */ + -1, /* (343) term ::= QNUMBER */ + -1, /* (344) input ::= cmdlist */ + -2, /* (345) cmdlist ::= cmdlist ecmd */ + -1, /* (346) cmdlist ::= ecmd */ + -1, /* (347) ecmd ::= SEMI */ + -2, /* (348) ecmd ::= cmdx SEMI */ + -3, /* (349) ecmd ::= explain cmdx SEMI */ + 0, /* (350) trans_opt ::= */ + -1, /* (351) trans_opt ::= TRANSACTION */ + -2, /* (352) trans_opt ::= TRANSACTION nm */ + -1, /* (353) savepoint_opt ::= SAVEPOINT */ + 0, /* (354) savepoint_opt ::= */ + -2, /* (355) cmd ::= create_table create_table_args */ + -1, /* (356) table_option_set ::= table_option */ + -4, /* (357) columnlist ::= columnlist COMMA columnname carglist */ + -2, /* (358) columnlist ::= columnname carglist */ + -1, /* (359) nm ::= ID|INDEXED|JOIN_KW */ + -1, /* (360) nm ::= STRING */ + -1, /* (361) typetoken ::= typename */ + -1, /* (362) typename ::= ID|STRING */ + -1, /* (363) signed ::= plus_num */ + -1, /* (364) signed ::= minus_num */ + -2, /* (365) carglist ::= carglist ccons */ + 0, /* (366) carglist ::= */ + -2, /* (367) ccons ::= NULL onconf */ + -4, /* (368) ccons ::= GENERATED ALWAYS AS generated */ + -2, /* (369) ccons ::= AS generated */ + -2, /* (370) conslist_opt ::= COMMA conslist */ + -3, /* (371) conslist ::= conslist tconscomma tcons */ + -1, /* (372) conslist ::= tcons */ + 0, /* (373) tconscomma ::= */ + -1, /* (374) defer_subclause_opt ::= defer_subclause */ + -1, /* (375) resolvetype ::= raisetype */ + -1, /* (376) selectnowith ::= oneselect */ + -1, /* (377) oneselect ::= values */ + -2, /* (378) sclp ::= selcollist COMMA */ + -1, /* (379) as ::= ID|STRING */ + -1, /* (380) indexed_opt ::= indexed_by */ + 0, /* (381) returning ::= */ + -1, /* (382) expr ::= term */ + -1, /* (383) likeop ::= LIKE_KW|MATCH */ + -1, /* (384) case_operand ::= expr */ + -1, /* (385) exprlist ::= nexprlist */ + -1, /* (386) nmnum ::= plus_num */ + -1, /* (387) nmnum ::= nm */ + -1, /* (388) nmnum ::= ON */ + -1, /* (389) nmnum ::= DELETE */ + -1, /* (390) nmnum ::= DEFAULT */ + -1, /* (391) plus_num ::= INTEGER|FLOAT */ + 0, /* (392) foreach_clause ::= */ + -3, /* (393) foreach_clause ::= FOR EACH ROW */ + -1, /* (394) trnm ::= nm */ + 0, /* (395) tridxby ::= */ + -1, /* (396) database_kw_opt ::= DATABASE */ + 0, /* (397) database_kw_opt ::= */ + 0, /* (398) kwcolumn_opt ::= */ + -1, /* (399) kwcolumn_opt ::= COLUMNKW */ + -1, /* (400) vtabarglist ::= vtabarg */ + -3, /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ + -2, /* (402) vtabarg ::= vtabarg vtabargtoken */ + 0, /* (403) anylist ::= */ + -4, /* (404) anylist ::= anylist LP anylist RP */ + -2, /* (405) anylist ::= anylist ANY */ + 0, /* (406) with ::= */ + -1, /* (407) windowdefn_list ::= windowdefn */ + -1, /* (408) window ::= frame_opt */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -132858,49 +177420,28 @@ static void yy_accept(yyParser*); /* Forward Declaration */ /* ** Perform a reduce action and the shift that must immediately ** follow the reduce. +** +** The yyLookahead and yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The yyLookahead will be YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. */ -static void yy_reduce( +static YYACTIONTYPE yy_reduce( yyParser *yypParser, /* The parser */ - unsigned int yyruleno /* Number of the rule by which to reduce */ + unsigned int yyruleno, /* Number of the rule by which to reduce */ + int yyLookahead, /* Lookahead token, or YYNOCODE if none */ + sqlite3ParserTOKENTYPE yyLookaheadToken /* Value of the lookahead token */ + sqlite3ParserCTX_PDECL /* %extra_context */ ){ int yygoto; /* The next state */ - int yyact; /* The next action */ + YYACTIONTYPE yyact; /* The next action */ yyStackEntry *yymsp; /* The top of the parser's stack */ int yysize; /* Amount to pop the stack */ - sqlite3ParserARG_FETCH; - yymsp = &yypParser->yystack[yypParser->yyidx]; -#ifndef NDEBUG - if( yyTraceFILE && yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ){ - yysize = yyRuleInfo[yyruleno].nrhs; - fprintf(yyTraceFILE, "%sReduce [%s], go to state %d.\n", yyTracePrompt, - yyRuleName[yyruleno], yymsp[-yysize].stateno); - } -#endif /* NDEBUG */ - - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( yyRuleInfo[yyruleno].nrhs==0 ){ -#ifdef YYTRACKMAXSTACKDEPTH - if( yypParser->yyidx>yypParser->yyidxMax ){ - yypParser->yyidxMax = yypParser->yyidx; - } -#endif -#if YYSTACKDEPTH>0 - if( yypParser->yyidx>=YYSTACKDEPTH-1 ){ - yyStackOverflow(yypParser); - return; - } -#else - if( yypParser->yyidx>=yypParser->yystksz-1 ){ - yyGrowStack(yypParser); - if( yypParser->yyidx>=yypParser->yystksz-1 ){ - yyStackOverflow(yypParser); - return; - } - } -#endif - } + sqlite3ParserARG_FETCH + (void)yyLookahead; + (void)yyLookaheadToken; + yymsp = yypParser->yytos; switch( yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -132914,770 +177455,972 @@ static void yy_reduce( /********** Begin reduce actions **********************************************/ YYMINORTYPE yylhsminor; case 0: /* explain ::= EXPLAIN */ -{ pParse->explain = 1; } +{ if( pParse->pReprepare==0 ) pParse->explain = 1; } break; case 1: /* explain ::= EXPLAIN QUERY PLAN */ -{ pParse->explain = 2; } +{ if( pParse->pReprepare==0 ) pParse->explain = 2; } break; case 2: /* cmdx ::= cmd */ { sqlite3FinishCoding(pParse); } break; case 3: /* cmd ::= BEGIN transtype trans_opt */ -{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy392);} +{sqlite3BeginTransaction(pParse, yymsp[-1].minor.yy144);} break; case 4: /* transtype ::= */ -{yymsp[1].minor.yy392 = TK_DEFERRED;} +{yymsp[1].minor.yy144 = TK_DEFERRED;} break; case 5: /* transtype ::= DEFERRED */ case 6: /* transtype ::= IMMEDIATE */ yytestcase(yyruleno==6); case 7: /* transtype ::= EXCLUSIVE */ yytestcase(yyruleno==7); -{yymsp[0].minor.yy392 = yymsp[0].major; /*A-overwrites-X*/} + case 324: /* range_or_rows ::= RANGE|ROWS|GROUPS */ yytestcase(yyruleno==324); +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/} break; - case 8: /* cmd ::= COMMIT trans_opt */ - case 9: /* cmd ::= END trans_opt */ yytestcase(yyruleno==9); -{sqlite3CommitTransaction(pParse);} + case 8: /* cmd ::= COMMIT|END trans_opt */ + case 9: /* cmd ::= ROLLBACK trans_opt */ yytestcase(yyruleno==9); +{sqlite3EndTransaction(pParse,yymsp[-1].major);} break; - case 10: /* cmd ::= ROLLBACK trans_opt */ -{sqlite3RollbackTransaction(pParse);} - break; - case 11: /* cmd ::= SAVEPOINT nm */ + case 10: /* cmd ::= SAVEPOINT nm */ { sqlite3Savepoint(pParse, SAVEPOINT_BEGIN, &yymsp[0].minor.yy0); } break; - case 12: /* cmd ::= RELEASE savepoint_opt nm */ + case 11: /* cmd ::= RELEASE savepoint_opt nm */ { sqlite3Savepoint(pParse, SAVEPOINT_RELEASE, &yymsp[0].minor.yy0); } break; - case 13: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ + case 12: /* cmd ::= ROLLBACK trans_opt TO savepoint_opt nm */ { sqlite3Savepoint(pParse, SAVEPOINT_ROLLBACK, &yymsp[0].minor.yy0); } break; - case 14: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ + case 13: /* create_table ::= createkw temp TABLE ifnotexists nm dbnm */ { - sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy392,0,0,yymsp[-2].minor.yy392); + sqlite3StartTable(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,yymsp[-4].minor.yy144,0,0,yymsp[-2].minor.yy144); } break; - case 15: /* createkw ::= CREATE */ + case 14: /* createkw ::= CREATE */ {disableLookaside(pParse);} break; - case 16: /* ifnotexists ::= */ - case 19: /* temp ::= */ yytestcase(yyruleno==19); - case 22: /* table_options ::= */ yytestcase(yyruleno==22); - case 42: /* autoinc ::= */ yytestcase(yyruleno==42); - case 57: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==57); - case 67: /* defer_subclause_opt ::= */ yytestcase(yyruleno==67); - case 76: /* ifexists ::= */ yytestcase(yyruleno==76); - case 90: /* distinct ::= */ yytestcase(yyruleno==90); - case 209: /* collate ::= */ yytestcase(yyruleno==209); -{yymsp[1].minor.yy392 = 0;} + case 15: /* ifnotexists ::= */ + case 18: /* temp ::= */ yytestcase(yyruleno==18); + case 47: /* autoinc ::= */ yytestcase(yyruleno==47); + case 62: /* init_deferred_pred_opt ::= */ yytestcase(yyruleno==62); + case 72: /* defer_subclause_opt ::= */ yytestcase(yyruleno==72); + case 81: /* ifexists ::= */ yytestcase(yyruleno==81); + case 100: /* distinct ::= */ yytestcase(yyruleno==100); + case 246: /* collate ::= */ yytestcase(yyruleno==246); +{yymsp[1].minor.yy144 = 0;} break; - case 17: /* ifnotexists ::= IF NOT EXISTS */ -{yymsp[-2].minor.yy392 = 1;} + case 16: /* ifnotexists ::= IF NOT EXISTS */ +{yymsp[-2].minor.yy144 = 1;} break; - case 18: /* temp ::= TEMP */ - case 43: /* autoinc ::= AUTOINCR */ yytestcase(yyruleno==43); -{yymsp[0].minor.yy392 = 1;} + case 17: /* temp ::= TEMP */ +{yymsp[0].minor.yy144 = pParse->db->init.busy==0;} break; - case 20: /* create_table_args ::= LP columnlist conslist_opt RP table_options */ + case 19: /* create_table_args ::= LP columnlist conslist_opt RP table_option_set */ { - sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy392,0); + sqlite3EndTable(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,yymsp[0].minor.yy391,0); } break; - case 21: /* create_table_args ::= AS select */ + case 20: /* create_table_args ::= AS select */ { - sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy159); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy159); + sqlite3EndTable(pParse,0,0,0,yymsp[0].minor.yy555); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; - case 23: /* table_options ::= WITHOUT nm */ + case 21: /* table_option_set ::= */ +{yymsp[1].minor.yy391 = 0;} + break; + case 22: /* table_option_set ::= table_option_set COMMA table_option */ +{yylhsminor.yy391 = yymsp[-2].minor.yy391|yymsp[0].minor.yy391;} + yymsp[-2].minor.yy391 = yylhsminor.yy391; + break; + case 23: /* table_option ::= WITHOUT nm */ { if( yymsp[0].minor.yy0.n==5 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"rowid",5)==0 ){ - yymsp[-1].minor.yy392 = TF_WithoutRowid | TF_NoVisibleRowid; + yymsp[-1].minor.yy391 = TF_WithoutRowid | TF_NoVisibleRowid; }else{ - yymsp[-1].minor.yy392 = 0; + yymsp[-1].minor.yy391 = 0; sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); } } break; - case 24: /* columnname ::= nm typetoken */ -{sqlite3AddColumn(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0);} + case 24: /* table_option ::= nm */ +{ + if( yymsp[0].minor.yy0.n==6 && sqlite3_strnicmp(yymsp[0].minor.yy0.z,"strict",6)==0 ){ + yylhsminor.yy391 = TF_Strict; + }else{ + yylhsminor.yy391 = 0; + sqlite3ErrorMsg(pParse, "unknown table option: %.*s", yymsp[0].minor.yy0.n, yymsp[0].minor.yy0.z); + } +} + yymsp[0].minor.yy391 = yylhsminor.yy391; break; - case 25: /* typetoken ::= */ - case 60: /* conslist_opt ::= */ yytestcase(yyruleno==60); - case 96: /* as ::= */ yytestcase(yyruleno==96); + case 25: /* columnname ::= nm typetoken */ +{sqlite3AddColumn(pParse,yymsp[-1].minor.yy0,yymsp[0].minor.yy0);} + break; + case 26: /* typetoken ::= */ + case 65: /* conslist_opt ::= */ yytestcase(yyruleno==65); + case 106: /* as ::= */ yytestcase(yyruleno==106); {yymsp[1].minor.yy0.n = 0; yymsp[1].minor.yy0.z = 0;} break; - case 26: /* typetoken ::= typename LP signed RP */ + case 27: /* typetoken ::= typename LP signed RP */ { yymsp[-3].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-3].minor.yy0.z); } break; - case 27: /* typetoken ::= typename LP signed COMMA signed RP */ + case 28: /* typetoken ::= typename LP signed COMMA signed RP */ { yymsp[-5].minor.yy0.n = (int)(&yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] - yymsp[-5].minor.yy0.z); } break; - case 28: /* typename ::= typename ID|STRING */ + case 29: /* typename ::= typename ID|STRING */ {yymsp[-1].minor.yy0.n=yymsp[0].minor.yy0.n+(int)(yymsp[0].minor.yy0.z-yymsp[-1].minor.yy0.z);} break; - case 29: /* ccons ::= CONSTRAINT nm */ - case 62: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==62); + case 30: /* scanpt ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy168 = yyLookaheadToken.z; +} + break; + case 31: /* scantok ::= */ +{ + assert( yyLookahead!=YYNOCODE ); + yymsp[1].minor.yy0 = yyLookaheadToken; +} + break; + case 32: /* ccons ::= CONSTRAINT nm */ + case 67: /* tcons ::= CONSTRAINT nm */ yytestcase(yyruleno==67); {pParse->constraintName = yymsp[0].minor.yy0;} break; - case 30: /* ccons ::= DEFAULT term */ - case 32: /* ccons ::= DEFAULT PLUS term */ yytestcase(yyruleno==32); -{sqlite3AddDefaultValue(pParse,&yymsp[0].minor.yy342);} + case 33: /* ccons ::= DEFAULT scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-1].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} break; - case 31: /* ccons ::= DEFAULT LP expr RP */ -{sqlite3AddDefaultValue(pParse,&yymsp[-1].minor.yy342);} + case 34: /* ccons ::= DEFAULT LP expr RP */ +{sqlite3AddDefaultValue(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z+1,yymsp[0].minor.yy0.z);} break; - case 33: /* ccons ::= DEFAULT MINUS term */ + case 35: /* ccons ::= DEFAULT PLUS scantok term */ +{sqlite3AddDefaultValue(pParse,yymsp[0].minor.yy454,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]);} + break; + case 36: /* ccons ::= DEFAULT MINUS scantok term */ { - ExprSpan v; - v.pExpr = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy342.pExpr, 0, 0); - v.zStart = yymsp[-1].minor.yy0.z; - v.zEnd = yymsp[0].minor.yy342.zEnd; - sqlite3AddDefaultValue(pParse,&v); + Expr *p = sqlite3PExpr(pParse, TK_UMINUS, yymsp[0].minor.yy454, 0); + sqlite3AddDefaultValue(pParse,p,yymsp[-2].minor.yy0.z,&yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]); } break; - case 34: /* ccons ::= DEFAULT ID|INDEXED */ + case 37: /* ccons ::= DEFAULT scantok ID|INDEXED */ { - ExprSpan v; - spanExpr(&v, pParse, TK_STRING, yymsp[0].minor.yy0); - sqlite3AddDefaultValue(pParse,&v); + Expr *p = tokenExpr(pParse, TK_STRING, yymsp[0].minor.yy0); + if( p ){ + sqlite3ExprIdToTrueFalse(p); + testcase( p->op==TK_TRUEFALSE && sqlite3ExprTruthValue(p) ); + } + sqlite3AddDefaultValue(pParse,p,yymsp[0].minor.yy0.z,yymsp[0].minor.yy0.z+yymsp[0].minor.yy0.n); } break; - case 35: /* ccons ::= NOT NULL onconf */ -{sqlite3AddNotNull(pParse, yymsp[0].minor.yy392);} + case 38: /* ccons ::= NOT NULL onconf */ +{sqlite3AddNotNull(pParse, yymsp[0].minor.yy144);} break; - case 36: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ -{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy392,yymsp[0].minor.yy392,yymsp[-2].minor.yy392);} + case 39: /* ccons ::= PRIMARY KEY sortorder onconf autoinc */ +{sqlite3AddPrimaryKey(pParse,0,yymsp[-1].minor.yy144,yymsp[0].minor.yy144,yymsp[-2].minor.yy144);} break; - case 37: /* ccons ::= UNIQUE onconf */ -{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy392,0,0,0,0);} + case 40: /* ccons ::= UNIQUE onconf */ +{sqlite3CreateIndex(pParse,0,0,0,0,yymsp[0].minor.yy144,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} break; - case 38: /* ccons ::= CHECK LP expr RP */ -{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy342.pExpr);} + case 41: /* ccons ::= CHECK LP expr RP */ +{sqlite3AddCheckConstraint(pParse,yymsp[-1].minor.yy454,yymsp[-2].minor.yy0.z,yymsp[0].minor.yy0.z);} break; - case 39: /* ccons ::= REFERENCES nm eidlist_opt refargs */ -{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy442,yymsp[0].minor.yy392);} + case 42: /* ccons ::= REFERENCES nm eidlist_opt refargs */ +{sqlite3CreateForeignKey(pParse,0,&yymsp[-2].minor.yy0,yymsp[-1].minor.yy14,yymsp[0].minor.yy144);} break; - case 40: /* ccons ::= defer_subclause */ -{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy392);} + case 43: /* ccons ::= defer_subclause */ +{sqlite3DeferForeignKey(pParse,yymsp[0].minor.yy144);} break; - case 41: /* ccons ::= COLLATE ID|STRING */ + case 44: /* ccons ::= COLLATE ID|STRING */ {sqlite3AddCollateType(pParse, &yymsp[0].minor.yy0);} break; - case 44: /* refargs ::= */ -{ yymsp[1].minor.yy392 = OE_None*0x0101; /* EV: R-19803-45884 */} + case 45: /* generated ::= LP expr RP */ +{sqlite3AddGenerated(pParse,yymsp[-1].minor.yy454,0);} break; - case 45: /* refargs ::= refargs refarg */ -{ yymsp[-1].minor.yy392 = (yymsp[-1].minor.yy392 & ~yymsp[0].minor.yy207.mask) | yymsp[0].minor.yy207.value; } + case 46: /* generated ::= LP expr RP ID */ +{sqlite3AddGenerated(pParse,yymsp[-2].minor.yy454,&yymsp[0].minor.yy0);} break; - case 46: /* refarg ::= MATCH nm */ -{ yymsp[-1].minor.yy207.value = 0; yymsp[-1].minor.yy207.mask = 0x000000; } + case 48: /* autoinc ::= AUTOINCR */ +{yymsp[0].minor.yy144 = 1;} break; - case 47: /* refarg ::= ON INSERT refact */ -{ yymsp[-2].minor.yy207.value = 0; yymsp[-2].minor.yy207.mask = 0x000000; } + case 49: /* refargs ::= */ +{ yymsp[1].minor.yy144 = OE_None*0x0101; /* EV: R-19803-45884 */} break; - case 48: /* refarg ::= ON DELETE refact */ -{ yymsp[-2].minor.yy207.value = yymsp[0].minor.yy392; yymsp[-2].minor.yy207.mask = 0x0000ff; } + case 50: /* refargs ::= refargs refarg */ +{ yymsp[-1].minor.yy144 = (yymsp[-1].minor.yy144 & ~yymsp[0].minor.yy383.mask) | yymsp[0].minor.yy383.value; } break; - case 49: /* refarg ::= ON UPDATE refact */ -{ yymsp[-2].minor.yy207.value = yymsp[0].minor.yy392<<8; yymsp[-2].minor.yy207.mask = 0x00ff00; } + case 51: /* refarg ::= MATCH nm */ +{ yymsp[-1].minor.yy383.value = 0; yymsp[-1].minor.yy383.mask = 0x000000; } break; - case 50: /* refact ::= SET NULL */ -{ yymsp[-1].minor.yy392 = OE_SetNull; /* EV: R-33326-45252 */} + case 52: /* refarg ::= ON INSERT refact */ +{ yymsp[-2].minor.yy383.value = 0; yymsp[-2].minor.yy383.mask = 0x000000; } break; - case 51: /* refact ::= SET DEFAULT */ -{ yymsp[-1].minor.yy392 = OE_SetDflt; /* EV: R-33326-45252 */} + case 53: /* refarg ::= ON DELETE refact */ +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144; yymsp[-2].minor.yy383.mask = 0x0000ff; } break; - case 52: /* refact ::= CASCADE */ -{ yymsp[0].minor.yy392 = OE_Cascade; /* EV: R-33326-45252 */} + case 54: /* refarg ::= ON UPDATE refact */ +{ yymsp[-2].minor.yy383.value = yymsp[0].minor.yy144<<8; yymsp[-2].minor.yy383.mask = 0x00ff00; } break; - case 53: /* refact ::= RESTRICT */ -{ yymsp[0].minor.yy392 = OE_Restrict; /* EV: R-33326-45252 */} + case 55: /* refact ::= SET NULL */ +{ yymsp[-1].minor.yy144 = OE_SetNull; /* EV: R-33326-45252 */} break; - case 54: /* refact ::= NO ACTION */ -{ yymsp[-1].minor.yy392 = OE_None; /* EV: R-33326-45252 */} + case 56: /* refact ::= SET DEFAULT */ +{ yymsp[-1].minor.yy144 = OE_SetDflt; /* EV: R-33326-45252 */} break; - case 55: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ -{yymsp[-2].minor.yy392 = 0;} + case 57: /* refact ::= CASCADE */ +{ yymsp[0].minor.yy144 = OE_Cascade; /* EV: R-33326-45252 */} break; - case 56: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ - case 71: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==71); - case 142: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==142); -{yymsp[-1].minor.yy392 = yymsp[0].minor.yy392;} + case 58: /* refact ::= RESTRICT */ +{ yymsp[0].minor.yy144 = OE_Restrict; /* EV: R-33326-45252 */} break; - case 58: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ - case 75: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==75); - case 183: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==183); - case 186: /* in_op ::= NOT IN */ yytestcase(yyruleno==186); - case 210: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==210); -{yymsp[-1].minor.yy392 = 1;} + case 59: /* refact ::= NO ACTION */ +{ yymsp[-1].minor.yy144 = OE_None; /* EV: R-33326-45252 */} break; - case 59: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ -{yymsp[-1].minor.yy392 = 0;} + case 60: /* defer_subclause ::= NOT DEFERRABLE init_deferred_pred_opt */ +{yymsp[-2].minor.yy144 = 0;} break; - case 61: /* tconscomma ::= COMMA */ + case 61: /* defer_subclause ::= DEFERRABLE init_deferred_pred_opt */ + case 76: /* orconf ::= OR resolvetype */ yytestcase(yyruleno==76); + case 173: /* insert_cmd ::= INSERT orconf */ yytestcase(yyruleno==173); +{yymsp[-1].minor.yy144 = yymsp[0].minor.yy144;} + break; + case 63: /* init_deferred_pred_opt ::= INITIALLY DEFERRED */ + case 80: /* ifexists ::= IF EXISTS */ yytestcase(yyruleno==80); + case 219: /* between_op ::= NOT BETWEEN */ yytestcase(yyruleno==219); + case 222: /* in_op ::= NOT IN */ yytestcase(yyruleno==222); + case 247: /* collate ::= COLLATE ID|STRING */ yytestcase(yyruleno==247); +{yymsp[-1].minor.yy144 = 1;} + break; + case 64: /* init_deferred_pred_opt ::= INITIALLY IMMEDIATE */ +{yymsp[-1].minor.yy144 = 0;} + break; + case 66: /* tconscomma ::= COMMA */ {pParse->constraintName.n = 0;} break; - case 63: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ -{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy442,yymsp[0].minor.yy392,yymsp[-2].minor.yy392,0);} + case 68: /* tcons ::= PRIMARY KEY LP sortlist autoinc RP onconf */ +{sqlite3AddPrimaryKey(pParse,yymsp[-3].minor.yy14,yymsp[0].minor.yy144,yymsp[-2].minor.yy144,0);} break; - case 64: /* tcons ::= UNIQUE LP sortlist RP onconf */ -{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy442,yymsp[0].minor.yy392,0,0,0,0);} + case 69: /* tcons ::= UNIQUE LP sortlist RP onconf */ +{sqlite3CreateIndex(pParse,0,0,0,yymsp[-2].minor.yy14,yymsp[0].minor.yy144,0,0,0,0, + SQLITE_IDXTYPE_UNIQUE);} break; - case 65: /* tcons ::= CHECK LP expr RP onconf */ -{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy342.pExpr);} + case 70: /* tcons ::= CHECK LP expr RP onconf */ +{sqlite3AddCheckConstraint(pParse,yymsp[-2].minor.yy454,yymsp[-3].minor.yy0.z,yymsp[-1].minor.yy0.z);} break; - case 66: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ + case 71: /* tcons ::= FOREIGN KEY LP eidlist RP REFERENCES nm eidlist_opt refargs defer_subclause_opt */ { - sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy442, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy442, yymsp[-1].minor.yy392); - sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy392); + sqlite3CreateForeignKey(pParse, yymsp[-6].minor.yy14, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[-1].minor.yy144); + sqlite3DeferForeignKey(pParse, yymsp[0].minor.yy144); } break; - case 68: /* onconf ::= */ - case 70: /* orconf ::= */ yytestcase(yyruleno==70); -{yymsp[1].minor.yy392 = OE_Default;} + case 73: /* onconf ::= */ + case 75: /* orconf ::= */ yytestcase(yyruleno==75); +{yymsp[1].minor.yy144 = OE_Default;} break; - case 69: /* onconf ::= ON CONFLICT resolvetype */ -{yymsp[-2].minor.yy392 = yymsp[0].minor.yy392;} + case 74: /* onconf ::= ON CONFLICT resolvetype */ +{yymsp[-2].minor.yy144 = yymsp[0].minor.yy144;} break; - case 72: /* resolvetype ::= IGNORE */ -{yymsp[0].minor.yy392 = OE_Ignore;} + case 77: /* resolvetype ::= IGNORE */ +{yymsp[0].minor.yy144 = OE_Ignore;} break; - case 73: /* resolvetype ::= REPLACE */ - case 143: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==143); -{yymsp[0].minor.yy392 = OE_Replace;} + case 78: /* resolvetype ::= REPLACE */ + case 174: /* insert_cmd ::= REPLACE */ yytestcase(yyruleno==174); +{yymsp[0].minor.yy144 = OE_Replace;} break; - case 74: /* cmd ::= DROP TABLE ifexists fullname */ + case 79: /* cmd ::= DROP TABLE ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy347, 0, yymsp[-1].minor.yy392); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 0, yymsp[-1].minor.yy144); } break; - case 77: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ + case 82: /* cmd ::= createkw temp VIEW ifnotexists nm dbnm eidlist_opt AS select */ { - sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy442, yymsp[0].minor.yy159, yymsp[-7].minor.yy392, yymsp[-5].minor.yy392); + sqlite3CreateView(pParse, &yymsp[-8].minor.yy0, &yymsp[-4].minor.yy0, &yymsp[-3].minor.yy0, yymsp[-2].minor.yy14, yymsp[0].minor.yy555, yymsp[-7].minor.yy144, yymsp[-5].minor.yy144); } break; - case 78: /* cmd ::= DROP VIEW ifexists fullname */ + case 83: /* cmd ::= DROP VIEW ifexists fullname */ { - sqlite3DropTable(pParse, yymsp[0].minor.yy347, 1, yymsp[-1].minor.yy392); + sqlite3DropTable(pParse, yymsp[0].minor.yy203, 1, yymsp[-1].minor.yy144); } break; - case 79: /* cmd ::= select */ + case 84: /* cmd ::= select */ { - SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0}; - sqlite3Select(pParse, yymsp[0].minor.yy159, &dest); - sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy159); + SelectDest dest = {SRT_Output, 0, 0, 0, 0, 0, 0}; + sqlite3Select(pParse, yymsp[0].minor.yy555, &dest); + sqlite3SelectDelete(pParse->db, yymsp[0].minor.yy555); } break; - case 80: /* select ::= with selectnowith */ + case 85: /* select ::= WITH wqlist selectnowith */ +{yymsp[-2].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} + break; + case 86: /* select ::= WITH RECURSIVE wqlist selectnowith */ +{yymsp[-3].minor.yy555 = attachWithToSelect(pParse,yymsp[0].minor.yy555,yymsp[-1].minor.yy59);} + break; + case 87: /* select ::= selectnowith */ { - Select *p = yymsp[0].minor.yy159; + Select *p = yymsp[0].minor.yy555; if( p ){ - p->pWith = yymsp[-1].minor.yy331; parserDoubleLinkSelect(pParse, p); - }else{ - sqlite3WithDelete(pParse->db, yymsp[-1].minor.yy331); } - yymsp[-1].minor.yy159 = p; /*A-overwrites-W*/ } break; - case 81: /* selectnowith ::= selectnowith multiselect_op oneselect */ + case 88: /* selectnowith ::= selectnowith multiselect_op oneselect */ { - Select *pRhs = yymsp[0].minor.yy159; - Select *pLhs = yymsp[-2].minor.yy159; + Select *pRhs = yymsp[0].minor.yy555; + Select *pLhs = yymsp[-2].minor.yy555; if( pRhs && pRhs->pPrior ){ SrcList *pFrom; Token x; x.n = 0; parserDoubleLinkSelect(pParse, pRhs); - pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0,0); - pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0,0); + pFrom = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&x,pRhs,0); + pRhs = sqlite3SelectNew(pParse,0,pFrom,0,0,0,0,0,0); } if( pRhs ){ - pRhs->op = (u8)yymsp[-1].minor.yy392; + pRhs->op = (u8)yymsp[-1].minor.yy144; pRhs->pPrior = pLhs; if( ALWAYS(pLhs) ) pLhs->selFlags &= ~SF_MultiValue; pRhs->selFlags &= ~SF_MultiValue; - if( yymsp[-1].minor.yy392!=TK_ALL ) pParse->hasCompound = 1; + if( yymsp[-1].minor.yy144!=TK_ALL ) pParse->hasCompound = 1; }else{ sqlite3SelectDelete(pParse->db, pLhs); } - yymsp[-2].minor.yy159 = pRhs; + yymsp[-2].minor.yy555 = pRhs; } break; - case 82: /* multiselect_op ::= UNION */ - case 84: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==84); -{yymsp[0].minor.yy392 = yymsp[0].major; /*A-overwrites-OP*/} + case 89: /* multiselect_op ::= UNION */ + case 91: /* multiselect_op ::= EXCEPT|INTERSECT */ yytestcase(yyruleno==91); +{yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-OP*/} break; - case 83: /* multiselect_op ::= UNION ALL */ -{yymsp[-1].minor.yy392 = TK_ALL;} + case 90: /* multiselect_op ::= UNION ALL */ +{yymsp[-1].minor.yy144 = TK_ALL;} break; - case 85: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ + case 92: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt orderby_opt limit_opt */ { -#if SELECTTRACE_ENABLED - Token s = yymsp[-8].minor.yy0; /*A-overwrites-S*/ -#endif - yymsp[-8].minor.yy159 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy442,yymsp[-5].minor.yy347,yymsp[-4].minor.yy122,yymsp[-3].minor.yy442,yymsp[-2].minor.yy122,yymsp[-1].minor.yy442,yymsp[-7].minor.yy392,yymsp[0].minor.yy64.pLimit,yymsp[0].minor.yy64.pOffset); -#if SELECTTRACE_ENABLED - /* Populate the Select.zSelName[] string that is used to help with - ** query planner debugging, to differentiate between multiple Select - ** objects in a complex query. - ** - ** If the SELECT keyword is immediately followed by a C-style comment - ** then extract the first few alphanumeric characters from within that - ** comment to be the zSelName value. Otherwise, the label is #N where - ** is an integer that is incremented with each SELECT statement seen. - */ - if( yymsp[-8].minor.yy159!=0 ){ - const char *z = s.z+6; - int i; - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy159->zSelName), yymsp[-8].minor.yy159->zSelName, "#%d", - ++pParse->nSelect); - while( z[0]==' ' ) z++; - if( z[0]=='/' && z[1]=='*' ){ - z += 2; - while( z[0]==' ' ) z++; - for(i=0; sqlite3Isalnum(z[i]); i++){} - sqlite3_snprintf(sizeof(yymsp[-8].minor.yy159->zSelName), yymsp[-8].minor.yy159->zSelName, "%.*s", i, z); - } - } -#endif /* SELECTRACE_ENABLED */ + yymsp[-8].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-6].minor.yy14,yymsp[-5].minor.yy203,yymsp[-4].minor.yy454,yymsp[-3].minor.yy14,yymsp[-2].minor.yy454,yymsp[-1].minor.yy14,yymsp[-7].minor.yy144,yymsp[0].minor.yy454); } break; - case 86: /* values ::= VALUES LP nexprlist RP */ + case 93: /* oneselect ::= SELECT distinct selcollist from where_opt groupby_opt having_opt window_clause orderby_opt limit_opt */ { - yymsp[-3].minor.yy159 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy442,0,0,0,0,0,SF_Values,0,0); -} - break; - case 87: /* values ::= values COMMA LP exprlist RP */ -{ - Select *pRight, *pLeft = yymsp[-4].minor.yy159; - pRight = sqlite3SelectNew(pParse,yymsp[-1].minor.yy442,0,0,0,0,0,SF_Values|SF_MultiValue,0,0); - if( ALWAYS(pLeft) ) pLeft->selFlags &= ~SF_MultiValue; - if( pRight ){ - pRight->op = TK_ALL; - pRight->pPrior = pLeft; - yymsp[-4].minor.yy159 = pRight; + yymsp[-9].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-7].minor.yy14,yymsp[-6].minor.yy203,yymsp[-5].minor.yy454,yymsp[-4].minor.yy14,yymsp[-3].minor.yy454,yymsp[-1].minor.yy14,yymsp[-8].minor.yy144,yymsp[0].minor.yy454); + if( yymsp[-9].minor.yy555 ){ + yymsp[-9].minor.yy555->pWinDefn = yymsp[-2].minor.yy211; }else{ - yymsp[-4].minor.yy159 = pLeft; + sqlite3WindowListDelete(pParse->db, yymsp[-2].minor.yy211); } } break; - case 88: /* distinct ::= DISTINCT */ -{yymsp[0].minor.yy392 = SF_Distinct;} - break; - case 89: /* distinct ::= ALL */ -{yymsp[0].minor.yy392 = SF_All;} - break; - case 91: /* sclp ::= */ - case 119: /* orderby_opt ::= */ yytestcase(yyruleno==119); - case 126: /* groupby_opt ::= */ yytestcase(yyruleno==126); - case 199: /* exprlist ::= */ yytestcase(yyruleno==199); - case 205: /* eidlist_opt ::= */ yytestcase(yyruleno==205); -{yymsp[1].minor.yy442 = 0;} - break; - case 92: /* selcollist ::= sclp expr as */ + case 94: /* values ::= VALUES LP nexprlist RP */ { - yymsp[-2].minor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy442, yymsp[-1].minor.yy342.pExpr); - if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-2].minor.yy442, &yymsp[0].minor.yy0, 1); - sqlite3ExprListSetSpan(pParse,yymsp[-2].minor.yy442,&yymsp[-1].minor.yy342); + yymsp[-3].minor.yy555 = sqlite3SelectNew(pParse,yymsp[-1].minor.yy14,0,0,0,0,0,SF_Values,0); } break; - case 93: /* selcollist ::= sclp STAR */ + case 95: /* oneselect ::= mvalues */ +{ + sqlite3MultiValuesEnd(pParse, yymsp[0].minor.yy555); +} + break; + case 96: /* mvalues ::= values COMMA LP nexprlist RP */ + case 97: /* mvalues ::= mvalues COMMA LP nexprlist RP */ yytestcase(yyruleno==97); +{ + yymsp[-4].minor.yy555 = sqlite3MultiValues(pParse, yymsp[-4].minor.yy555, yymsp[-1].minor.yy14); +} + break; + case 98: /* distinct ::= DISTINCT */ +{yymsp[0].minor.yy144 = SF_Distinct;} + break; + case 99: /* distinct ::= ALL */ +{yymsp[0].minor.yy144 = SF_All;} + break; + case 101: /* sclp ::= */ + case 134: /* orderby_opt ::= */ yytestcase(yyruleno==134); + case 144: /* groupby_opt ::= */ yytestcase(yyruleno==144); + case 234: /* exprlist ::= */ yytestcase(yyruleno==234); + case 237: /* paren_exprlist ::= */ yytestcase(yyruleno==237); + case 242: /* eidlist_opt ::= */ yytestcase(yyruleno==242); +{yymsp[1].minor.yy14 = 0;} + break; + case 102: /* selcollist ::= sclp scanpt expr scanpt as */ +{ + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + if( yymsp[0].minor.yy0.n>0 ) sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[0].minor.yy0, 1); + sqlite3ExprListSetSpan(pParse,yymsp[-4].minor.yy14,yymsp[-3].minor.yy168,yymsp[-1].minor.yy168); +} + break; + case 103: /* selcollist ::= sclp scanpt STAR */ { Expr *p = sqlite3Expr(pParse->db, TK_ASTERISK, 0); - yymsp[-1].minor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-1].minor.yy442, p); + sqlite3ExprSetErrorOffset(p, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-2].minor.yy14, p); } break; - case 94: /* selcollist ::= sclp nm DOT STAR */ + case 104: /* selcollist ::= sclp scanpt nm DOT STAR */ { - Expr *pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0, &yymsp[0].minor.yy0); - Expr *pLeft = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight, 0); - yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442, pDot); + Expr *pRight, *pLeft, *pDot; + pRight = sqlite3PExpr(pParse, TK_ASTERISK, 0, 0); + sqlite3ExprSetErrorOffset(pRight, (int)(yymsp[0].minor.yy0.z - pParse->zTail)); + pLeft = tokenExpr(pParse, TK_ID, yymsp[-2].minor.yy0); + pDot = sqlite3PExpr(pParse, TK_DOT, pLeft, pRight); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, pDot); } break; - case 95: /* as ::= AS nm */ - case 106: /* dbnm ::= DOT nm */ yytestcase(yyruleno==106); - case 219: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==219); - case 220: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==220); + case 105: /* as ::= AS nm */ + case 117: /* dbnm ::= DOT nm */ yytestcase(yyruleno==117); + case 258: /* plus_num ::= PLUS INTEGER|FLOAT */ yytestcase(yyruleno==258); + case 259: /* minus_num ::= MINUS INTEGER|FLOAT */ yytestcase(yyruleno==259); {yymsp[-1].minor.yy0 = yymsp[0].minor.yy0;} break; - case 97: /* from ::= */ -{yymsp[1].minor.yy347 = sqlite3DbMallocZero(pParse->db, sizeof(*yymsp[1].minor.yy347));} + case 107: /* from ::= */ + case 110: /* stl_prefix ::= */ yytestcase(yyruleno==110); +{yymsp[1].minor.yy203 = 0;} break; - case 98: /* from ::= FROM seltablist */ + case 108: /* from ::= FROM seltablist */ { - yymsp[-1].minor.yy347 = yymsp[0].minor.yy347; - sqlite3SrcListShiftJoinType(yymsp[-1].minor.yy347); + yymsp[-1].minor.yy203 = yymsp[0].minor.yy203; + sqlite3SrcListShiftJoinType(pParse,yymsp[-1].minor.yy203); } break; - case 99: /* stl_prefix ::= seltablist joinop */ + case 109: /* stl_prefix ::= seltablist joinop */ { - if( ALWAYS(yymsp[-1].minor.yy347 && yymsp[-1].minor.yy347->nSrc>0) ) yymsp[-1].minor.yy347->a[yymsp[-1].minor.yy347->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy392; + if( ALWAYS(yymsp[-1].minor.yy203 && yymsp[-1].minor.yy203->nSrc>0) ) yymsp[-1].minor.yy203->a[yymsp[-1].minor.yy203->nSrc-1].fg.jointype = (u8)yymsp[0].minor.yy144; } break; - case 100: /* stl_prefix ::= */ -{yymsp[1].minor.yy347 = 0;} - break; - case 101: /* seltablist ::= stl_prefix nm dbnm as indexed_opt on_opt using_opt */ + case 111: /* seltablist ::= stl_prefix nm dbnm as on_using */ { - yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,&yymsp[-5].minor.yy0,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); - sqlite3SrcListIndexedBy(pParse, yymsp[-6].minor.yy347, &yymsp[-2].minor.yy0); + yymsp[-4].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-4].minor.yy203,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); } break; - case 102: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_opt using_opt */ + case 112: /* seltablist ::= stl_prefix nm dbnm as indexed_by on_using */ { - yymsp[-8].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-8].minor.yy347,&yymsp[-7].minor.yy0,&yymsp[-6].minor.yy0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); - sqlite3SrcListFuncArgs(pParse, yymsp[-8].minor.yy347, yymsp[-4].minor.yy442); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-1].minor.yy0); } break; - case 103: /* seltablist ::= stl_prefix LP select RP as on_opt using_opt */ + case 113: /* seltablist ::= stl_prefix nm dbnm LP exprlist RP as on_using */ { - yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,yymsp[-4].minor.yy159,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + yymsp[-7].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-7].minor.yy203,&yymsp[-6].minor.yy0,&yymsp[-5].minor.yy0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + sqlite3SrcListFuncArgs(pParse, yymsp[-7].minor.yy203, yymsp[-3].minor.yy14); +} + break; + case 114: /* seltablist ::= stl_prefix LP select RP as on_using */ +{ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,yymsp[-3].minor.yy555,&yymsp[0].minor.yy269); } break; - case 104: /* seltablist ::= stl_prefix LP seltablist RP as on_opt using_opt */ + case 115: /* seltablist ::= stl_prefix LP seltablist RP as on_using */ { - if( yymsp[-6].minor.yy347==0 && yymsp[-2].minor.yy0.n==0 && yymsp[-1].minor.yy122==0 && yymsp[0].minor.yy180==0 ){ - yymsp[-6].minor.yy347 = yymsp[-4].minor.yy347; - }else if( yymsp[-4].minor.yy347->nSrc==1 ){ - yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,0,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); - if( yymsp[-6].minor.yy347 ){ - struct SrcList_item *pNew = &yymsp[-6].minor.yy347->a[yymsp[-6].minor.yy347->nSrc-1]; - struct SrcList_item *pOld = yymsp[-4].minor.yy347->a; + if( yymsp[-5].minor.yy203==0 && yymsp[-1].minor.yy0.n==0 && yymsp[0].minor.yy269.pOn==0 && yymsp[0].minor.yy269.pUsing==0 ){ + yymsp[-5].minor.yy203 = yymsp[-3].minor.yy203; + }else if( ALWAYS(yymsp[-3].minor.yy203!=0) && yymsp[-3].minor.yy203->nSrc==1 ){ + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,0,&yymsp[0].minor.yy269); + if( yymsp[-5].minor.yy203 ){ + SrcItem *pNew = &yymsp[-5].minor.yy203->a[yymsp[-5].minor.yy203->nSrc-1]; + SrcItem *pOld = yymsp[-3].minor.yy203->a; + assert( pOld->fg.fixedSchema==0 ); pNew->zName = pOld->zName; - pNew->zDatabase = pOld->zDatabase; - pNew->pSelect = pOld->pSelect; - pOld->zName = pOld->zDatabase = 0; - pOld->pSelect = 0; + assert( pOld->fg.fixedSchema==0 ); + if( pOld->fg.isSubquery ){ + pNew->fg.isSubquery = 1; + pNew->u4.pSubq = pOld->u4.pSubq; + pOld->u4.pSubq = 0; + pOld->fg.isSubquery = 0; + assert( pNew->u4.pSubq!=0 && pNew->u4.pSubq->pSelect!=0 ); + if( (pNew->u4.pSubq->pSelect->selFlags & SF_NestedFrom)!=0 ){ + pNew->fg.isNestedFrom = 1; + } + }else{ + pNew->u4.zDatabase = pOld->u4.zDatabase; + pOld->u4.zDatabase = 0; + } + if( pOld->fg.isTabFunc ){ + pNew->u1.pFuncArg = pOld->u1.pFuncArg; + pOld->u1.pFuncArg = 0; + pOld->fg.isTabFunc = 0; + pNew->fg.isTabFunc = 1; + } + pOld->zName = 0; } - sqlite3SrcListDelete(pParse->db, yymsp[-4].minor.yy347); + sqlite3SrcListDelete(pParse->db, yymsp[-3].minor.yy203); }else{ Select *pSubquery; - sqlite3SrcListShiftJoinType(yymsp[-4].minor.yy347); - pSubquery = sqlite3SelectNew(pParse,0,yymsp[-4].minor.yy347,0,0,0,0,SF_NestedFrom,0,0); - yymsp[-6].minor.yy347 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-6].minor.yy347,0,0,&yymsp[-2].minor.yy0,pSubquery,yymsp[-1].minor.yy122,yymsp[0].minor.yy180); + sqlite3SrcListShiftJoinType(pParse,yymsp[-3].minor.yy203); + pSubquery = sqlite3SelectNew(pParse,0,yymsp[-3].minor.yy203,0,0,0,0,SF_NestedFrom,0); + yymsp[-5].minor.yy203 = sqlite3SrcListAppendFromTerm(pParse,yymsp[-5].minor.yy203,0,0,&yymsp[-1].minor.yy0,pSubquery,&yymsp[0].minor.yy269); } } break; - case 105: /* dbnm ::= */ - case 114: /* indexed_opt ::= */ yytestcase(yyruleno==114); + case 116: /* dbnm ::= */ + case 131: /* indexed_opt ::= */ yytestcase(yyruleno==131); {yymsp[1].minor.yy0.z=0; yymsp[1].minor.yy0.n=0;} break; - case 107: /* fullname ::= nm dbnm */ -{yymsp[-1].minor.yy347 = sqlite3SrcListAppend(pParse->db,0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} + case 118: /* fullname ::= nm */ +{ + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[0].minor.yy203 = yylhsminor.yy203; break; - case 108: /* joinop ::= COMMA|JOIN */ -{ yymsp[0].minor.yy392 = JT_INNER; } + case 119: /* fullname ::= nm DOT nm */ +{ + yylhsminor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); + if( IN_RENAME_OBJECT && yylhsminor.yy203 ) sqlite3RenameTokenMap(pParse, yylhsminor.yy203->a[0].zName, &yymsp[0].minor.yy0); +} + yymsp[-2].minor.yy203 = yylhsminor.yy203; break; - case 109: /* joinop ::= JOIN_KW JOIN */ -{yymsp[-1].minor.yy392 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + case 120: /* xfullname ::= nm */ +{yymsp[0].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[0].minor.yy0,0); /*A-overwrites-X*/} break; - case 110: /* joinop ::= JOIN_KW nm JOIN */ -{yymsp[-2].minor.yy392 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + case 121: /* xfullname ::= nm DOT nm */ +{yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/} break; - case 111: /* joinop ::= JOIN_KW nm nm JOIN */ -{yymsp[-3].minor.yy392 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + case 122: /* xfullname ::= nm DOT nm AS nm */ +{ + yymsp[-4].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,&yymsp[-2].minor.yy0); /*A-overwrites-X*/ + if( yymsp[-4].minor.yy203 ) yymsp[-4].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} break; - case 112: /* on_opt ::= ON expr */ - case 129: /* having_opt ::= HAVING expr */ yytestcase(yyruleno==129); - case 136: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==136); - case 195: /* case_else ::= ELSE expr */ yytestcase(yyruleno==195); -{yymsp[-1].minor.yy122 = yymsp[0].minor.yy342.pExpr;} + case 123: /* xfullname ::= nm AS nm */ +{ + yymsp[-2].minor.yy203 = sqlite3SrcListAppend(pParse,0,&yymsp[-2].minor.yy0,0); /*A-overwrites-X*/ + if( yymsp[-2].minor.yy203 ) yymsp[-2].minor.yy203->a[0].zAlias = sqlite3NameFromToken(pParse->db, &yymsp[0].minor.yy0); +} break; - case 113: /* on_opt ::= */ - case 128: /* having_opt ::= */ yytestcase(yyruleno==128); - case 135: /* where_opt ::= */ yytestcase(yyruleno==135); - case 196: /* case_else ::= */ yytestcase(yyruleno==196); - case 198: /* case_operand ::= */ yytestcase(yyruleno==198); -{yymsp[1].minor.yy122 = 0;} + case 124: /* joinop ::= COMMA|JOIN */ +{ yymsp[0].minor.yy144 = JT_INNER; } break; - case 115: /* indexed_opt ::= INDEXED BY nm */ + case 125: /* joinop ::= JOIN_KW JOIN */ +{yymsp[-1].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-1].minor.yy0,0,0); /*X-overwrites-A*/} + break; + case 126: /* joinop ::= JOIN_KW nm JOIN */ +{yymsp[-2].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0,0); /*X-overwrites-A*/} + break; + case 127: /* joinop ::= JOIN_KW nm nm JOIN */ +{yymsp[-3].minor.yy144 = sqlite3JoinType(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0);/*X-overwrites-A*/} + break; + case 128: /* on_using ::= ON expr */ +{yymsp[-1].minor.yy269.pOn = yymsp[0].minor.yy454; yymsp[-1].minor.yy269.pUsing = 0;} + break; + case 129: /* on_using ::= USING LP idlist RP */ +{yymsp[-3].minor.yy269.pOn = 0; yymsp[-3].minor.yy269.pUsing = yymsp[-1].minor.yy132;} + break; + case 130: /* on_using ::= */ +{yymsp[1].minor.yy269.pOn = 0; yymsp[1].minor.yy269.pUsing = 0;} + break; + case 132: /* indexed_by ::= INDEXED BY nm */ {yymsp[-2].minor.yy0 = yymsp[0].minor.yy0;} break; - case 116: /* indexed_opt ::= NOT INDEXED */ + case 133: /* indexed_by ::= NOT INDEXED */ {yymsp[-1].minor.yy0.z=0; yymsp[-1].minor.yy0.n=1;} break; - case 117: /* using_opt ::= USING LP idlist RP */ -{yymsp[-3].minor.yy180 = yymsp[-1].minor.yy180;} + case 135: /* orderby_opt ::= ORDER BY sortlist */ + case 145: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==145); +{yymsp[-2].minor.yy14 = yymsp[0].minor.yy14;} break; - case 118: /* using_opt ::= */ - case 144: /* idlist_opt ::= */ yytestcase(yyruleno==144); -{yymsp[1].minor.yy180 = 0;} - break; - case 120: /* orderby_opt ::= ORDER BY sortlist */ - case 127: /* groupby_opt ::= GROUP BY nexprlist */ yytestcase(yyruleno==127); -{yymsp[-2].minor.yy442 = yymsp[0].minor.yy442;} - break; - case 121: /* sortlist ::= sortlist COMMA expr sortorder */ + case 136: /* sortlist ::= sortlist COMMA expr sortorder nulls */ { - yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442,yymsp[-1].minor.yy342.pExpr); - sqlite3ExprListSetSortOrder(yymsp[-3].minor.yy442,yymsp[0].minor.yy392); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14,yymsp[-2].minor.yy454); + sqlite3ExprListSetSortOrder(yymsp[-4].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 122: /* sortlist ::= expr sortorder */ + case 137: /* sortlist ::= expr sortorder nulls */ { - yymsp[-1].minor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[-1].minor.yy342.pExpr); /*A-overwrites-Y*/ - sqlite3ExprListSetSortOrder(yymsp[-1].minor.yy442,yymsp[0].minor.yy392); + yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[-2].minor.yy454); /*A-overwrites-Y*/ + sqlite3ExprListSetSortOrder(yymsp[-2].minor.yy14,yymsp[-1].minor.yy144,yymsp[0].minor.yy144); } break; - case 123: /* sortorder ::= ASC */ -{yymsp[0].minor.yy392 = SQLITE_SO_ASC;} + case 138: /* sortorder ::= ASC */ +{yymsp[0].minor.yy144 = SQLITE_SO_ASC;} break; - case 124: /* sortorder ::= DESC */ -{yymsp[0].minor.yy392 = SQLITE_SO_DESC;} + case 139: /* sortorder ::= DESC */ +{yymsp[0].minor.yy144 = SQLITE_SO_DESC;} break; - case 125: /* sortorder ::= */ -{yymsp[1].minor.yy392 = SQLITE_SO_UNDEFINED;} + case 140: /* sortorder ::= */ + case 143: /* nulls ::= */ yytestcase(yyruleno==143); +{yymsp[1].minor.yy144 = SQLITE_SO_UNDEFINED;} break; - case 130: /* limit_opt ::= */ -{yymsp[1].minor.yy64.pLimit = 0; yymsp[1].minor.yy64.pOffset = 0;} + case 141: /* nulls ::= NULLS FIRST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_ASC;} break; - case 131: /* limit_opt ::= LIMIT expr */ -{yymsp[-1].minor.yy64.pLimit = yymsp[0].minor.yy342.pExpr; yymsp[-1].minor.yy64.pOffset = 0;} + case 142: /* nulls ::= NULLS LAST */ +{yymsp[-1].minor.yy144 = SQLITE_SO_DESC;} break; - case 132: /* limit_opt ::= LIMIT expr OFFSET expr */ -{yymsp[-3].minor.yy64.pLimit = yymsp[-2].minor.yy342.pExpr; yymsp[-3].minor.yy64.pOffset = yymsp[0].minor.yy342.pExpr;} + case 146: /* having_opt ::= */ + case 148: /* limit_opt ::= */ yytestcase(yyruleno==148); + case 153: /* where_opt ::= */ yytestcase(yyruleno==153); + case 155: /* where_opt_ret ::= */ yytestcase(yyruleno==155); + case 232: /* case_else ::= */ yytestcase(yyruleno==232); + case 233: /* case_operand ::= */ yytestcase(yyruleno==233); + case 252: /* vinto ::= */ yytestcase(yyruleno==252); +{yymsp[1].minor.yy454 = 0;} break; - case 133: /* limit_opt ::= LIMIT expr COMMA expr */ -{yymsp[-3].minor.yy64.pOffset = yymsp[-2].minor.yy342.pExpr; yymsp[-3].minor.yy64.pLimit = yymsp[0].minor.yy342.pExpr;} + case 147: /* having_opt ::= HAVING expr */ + case 154: /* where_opt ::= WHERE expr */ yytestcase(yyruleno==154); + case 156: /* where_opt_ret ::= WHERE expr */ yytestcase(yyruleno==156); + case 231: /* case_else ::= ELSE expr */ yytestcase(yyruleno==231); + case 251: /* vinto ::= INTO expr */ yytestcase(yyruleno==251); +{yymsp[-1].minor.yy454 = yymsp[0].minor.yy454;} break; - case 134: /* cmd ::= with DELETE FROM fullname indexed_opt where_opt */ + case 149: /* limit_opt ::= LIMIT expr */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,0);} + break; + case 150: /* limit_opt ::= LIMIT expr OFFSET expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} + break; + case 151: /* limit_opt ::= LIMIT expr COMMA expr */ +{yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_LIMIT,yymsp[0].minor.yy454,yymsp[-2].minor.yy454);} + break; + case 152: /* cmd ::= with DELETE FROM xfullname indexed_opt where_opt_ret */ { - sqlite3WithPush(pParse, yymsp[-5].minor.yy331, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy347, &yymsp[-1].minor.yy0); - sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy347,yymsp[0].minor.yy122); + sqlite3SrcListIndexedBy(pParse, yymsp[-2].minor.yy203, &yymsp[-1].minor.yy0); + sqlite3DeleteFrom(pParse,yymsp[-2].minor.yy203,yymsp[0].minor.yy454,0,0); } break; - case 137: /* cmd ::= with UPDATE orconf fullname indexed_opt SET setlist where_opt */ + case 157: /* where_opt_ret ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-1].minor.yy454 = 0;} + break; + case 158: /* where_opt_ret ::= WHERE expr RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14); yymsp[-3].minor.yy454 = yymsp[-2].minor.yy454;} + break; + case 159: /* cmd ::= with UPDATE orconf xfullname indexed_opt SET setlist from where_opt_ret */ { - sqlite3WithPush(pParse, yymsp[-7].minor.yy331, 1); - sqlite3SrcListIndexedBy(pParse, yymsp[-4].minor.yy347, &yymsp[-3].minor.yy0); - sqlite3ExprListCheckLength(pParse,yymsp[-1].minor.yy442,"set list"); - sqlite3Update(pParse,yymsp[-4].minor.yy347,yymsp[-1].minor.yy442,yymsp[0].minor.yy122,yymsp[-5].minor.yy392); + sqlite3SrcListIndexedBy(pParse, yymsp[-5].minor.yy203, &yymsp[-4].minor.yy0); + sqlite3ExprListCheckLength(pParse,yymsp[-2].minor.yy14,"set list"); + if( yymsp[-1].minor.yy203 ){ + SrcList *pFromClause = yymsp[-1].minor.yy203; + if( pFromClause->nSrc>1 ){ + Select *pSubquery; + Token as; + pSubquery = sqlite3SelectNew(pParse,0,pFromClause,0,0,0,0,SF_NestedFrom,0); + as.n = 0; + as.z = 0; + pFromClause = sqlite3SrcListAppendFromTerm(pParse,0,0,0,&as,pSubquery,0); + } + yymsp[-5].minor.yy203 = sqlite3SrcListAppendList(pParse, yymsp[-5].minor.yy203, pFromClause); + } + sqlite3Update(pParse,yymsp[-5].minor.yy203,yymsp[-2].minor.yy14,yymsp[0].minor.yy454,yymsp[-6].minor.yy144,0,0,0); } break; - case 138: /* setlist ::= setlist COMMA nm EQ expr */ + case 160: /* setlist ::= setlist COMMA nm EQ expr */ { - yymsp[-4].minor.yy442 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy442, yymsp[0].minor.yy342.pExpr); - sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy442, &yymsp[-2].minor.yy0, 1); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse, yymsp[-4].minor.yy14, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, 1); } break; - case 139: /* setlist ::= nm EQ expr */ + case 161: /* setlist ::= setlist COMMA LP idlist RP EQ expr */ { - yylhsminor.yy442 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy342.pExpr); - sqlite3ExprListSetName(pParse, yylhsminor.yy442, &yymsp[-2].minor.yy0, 1); -} - yymsp[-2].minor.yy442 = yylhsminor.yy442; - break; - case 140: /* cmd ::= with insert_cmd INTO fullname idlist_opt select */ -{ - sqlite3WithPush(pParse, yymsp[-5].minor.yy331, 1); - sqlite3Insert(pParse, yymsp[-2].minor.yy347, yymsp[0].minor.yy159, yymsp[-1].minor.yy180, yymsp[-4].minor.yy392); + yymsp[-6].minor.yy14 = sqlite3ExprListAppendVector(pParse, yymsp[-6].minor.yy14, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 141: /* cmd ::= with insert_cmd INTO fullname idlist_opt DEFAULT VALUES */ + case 162: /* setlist ::= nm EQ expr */ { - sqlite3WithPush(pParse, yymsp[-6].minor.yy331, 1); - sqlite3Insert(pParse, yymsp[-3].minor.yy347, 0, yymsp[-2].minor.yy180, yymsp[-5].minor.yy392); + yylhsminor.yy14 = sqlite3ExprListAppend(pParse, 0, yymsp[0].minor.yy454); + sqlite3ExprListSetName(pParse, yylhsminor.yy14, &yymsp[-2].minor.yy0, 1); +} + yymsp[-2].minor.yy14 = yylhsminor.yy14; + break; + case 163: /* setlist ::= LP idlist RP EQ expr */ +{ + yymsp[-4].minor.yy14 = sqlite3ExprListAppendVector(pParse, 0, yymsp[-3].minor.yy132, yymsp[0].minor.yy454); } break; - case 145: /* idlist_opt ::= LP idlist RP */ -{yymsp[-2].minor.yy180 = yymsp[-1].minor.yy180;} - break; - case 146: /* idlist ::= idlist COMMA nm */ -{yymsp[-2].minor.yy180 = sqlite3IdListAppend(pParse->db,yymsp[-2].minor.yy180,&yymsp[0].minor.yy0);} - break; - case 147: /* idlist ::= nm */ -{yymsp[0].minor.yy180 = sqlite3IdListAppend(pParse->db,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} - break; - case 148: /* expr ::= LP expr RP */ -{spanSet(&yymsp[-2].minor.yy342,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ yymsp[-2].minor.yy342.pExpr = yymsp[-1].minor.yy342.pExpr;} - break; - case 149: /* term ::= NULL */ - case 154: /* term ::= INTEGER|FLOAT|BLOB */ yytestcase(yyruleno==154); - case 155: /* term ::= STRING */ yytestcase(yyruleno==155); -{spanExpr(&yymsp[0].minor.yy342,pParse,yymsp[0].major,yymsp[0].minor.yy0);/*A-overwrites-X*/} - break; - case 150: /* expr ::= ID|INDEXED */ - case 151: /* expr ::= JOIN_KW */ yytestcase(yyruleno==151); -{spanExpr(&yymsp[0].minor.yy342,pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} - break; - case 152: /* expr ::= nm DOT nm */ + case 164: /* cmd ::= with insert_cmd INTO xfullname idlist_opt select upsert */ { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - spanSet(&yymsp[-2].minor.yy342,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-2].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp2, 0); + sqlite3Insert(pParse, yymsp[-3].minor.yy203, yymsp[-1].minor.yy555, yymsp[-2].minor.yy132, yymsp[-5].minor.yy144, yymsp[0].minor.yy122); } break; - case 153: /* expr ::= nm DOT nm DOT nm */ + case 165: /* cmd ::= with insert_cmd INTO xfullname idlist_opt DEFAULT VALUES returning */ { - Expr *temp1 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-4].minor.yy0); - Expr *temp2 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[-2].minor.yy0); - Expr *temp3 = sqlite3PExpr(pParse, TK_ID, 0, 0, &yymsp[0].minor.yy0); - Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3, 0); - spanSet(&yymsp[-4].minor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_DOT, temp1, temp4, 0); + sqlite3Insert(pParse, yymsp[-4].minor.yy203, 0, yymsp[-3].minor.yy132, yymsp[-6].minor.yy144, 0); } break; - case 156: /* expr ::= VARIABLE */ + case 166: /* upsert ::= */ +{ yymsp[1].minor.yy122 = 0; } + break; + case 167: /* upsert ::= RETURNING selcollist */ +{ yymsp[-1].minor.yy122 = 0; sqlite3AddReturning(pParse,yymsp[0].minor.yy14); } + break; + case 168: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO UPDATE SET setlist where_opt upsert */ +{ yymsp[-11].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-8].minor.yy14,yymsp[-6].minor.yy454,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,yymsp[0].minor.yy122);} + break; + case 169: /* upsert ::= ON CONFLICT LP sortlist RP where_opt DO NOTHING upsert */ +{ yymsp[-8].minor.yy122 = sqlite3UpsertNew(pParse->db,yymsp[-5].minor.yy14,yymsp[-3].minor.yy454,0,0,yymsp[0].minor.yy122); } + break; + case 170: /* upsert ::= ON CONFLICT DO NOTHING returning */ +{ yymsp[-4].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,0,0,0); } + break; + case 171: /* upsert ::= ON CONFLICT DO UPDATE SET setlist where_opt returning */ +{ yymsp[-7].minor.yy122 = sqlite3UpsertNew(pParse->db,0,0,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454,0);} + break; + case 172: /* returning ::= RETURNING selcollist */ +{sqlite3AddReturning(pParse,yymsp[0].minor.yy14);} + break; + case 175: /* idlist_opt ::= */ +{yymsp[1].minor.yy132 = 0;} + break; + case 176: /* idlist_opt ::= LP idlist RP */ +{yymsp[-2].minor.yy132 = yymsp[-1].minor.yy132;} + break; + case 177: /* idlist ::= idlist COMMA nm */ +{yymsp[-2].minor.yy132 = sqlite3IdListAppend(pParse,yymsp[-2].minor.yy132,&yymsp[0].minor.yy0);} + break; + case 178: /* idlist ::= nm */ +{yymsp[0].minor.yy132 = sqlite3IdListAppend(pParse,0,&yymsp[0].minor.yy0); /*A-overwrites-Y*/} + break; + case 179: /* expr ::= LP expr RP */ +{yymsp[-2].minor.yy454 = yymsp[-1].minor.yy454;} + break; + case 180: /* expr ::= ID|INDEXED|JOIN_KW */ +{yymsp[0].minor.yy454=tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 181: /* expr ::= nm DOT nm */ +{ + Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); + Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp2); +} + yymsp[-2].minor.yy454 = yylhsminor.yy454; + break; + case 182: /* expr ::= nm DOT nm DOT nm */ +{ + Expr *temp1 = tokenExpr(pParse,TK_ID,yymsp[-4].minor.yy0); + Expr *temp2 = tokenExpr(pParse,TK_ID,yymsp[-2].minor.yy0); + Expr *temp3 = tokenExpr(pParse,TK_ID,yymsp[0].minor.yy0); + Expr *temp4 = sqlite3PExpr(pParse, TK_DOT, temp2, temp3); + if( IN_RENAME_OBJECT ){ + sqlite3RenameTokenRemap(pParse, 0, temp1); + } + yylhsminor.yy454 = sqlite3PExpr(pParse, TK_DOT, temp1, temp4); +} + yymsp[-4].minor.yy454 = yylhsminor.yy454; + break; + case 183: /* term ::= NULL|FLOAT|BLOB */ + case 184: /* term ::= STRING */ yytestcase(yyruleno==184); +{yymsp[0].minor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); /*A-overwrites-X*/} + break; + case 185: /* term ::= INTEGER */ +{ + yylhsminor.yy454 = sqlite3ExprAlloc(pParse->db, TK_INTEGER, &yymsp[0].minor.yy0, 1); + if( yylhsminor.yy454 ) yylhsminor.yy454->w.iOfst = (int)(yymsp[0].minor.yy0.z - pParse->zTail); +} + yymsp[0].minor.yy454 = yylhsminor.yy454; + break; + case 186: /* expr ::= VARIABLE */ { if( !(yymsp[0].minor.yy0.z[0]=='#' && sqlite3Isdigit(yymsp[0].minor.yy0.z[1])) ){ - spanExpr(&yymsp[0].minor.yy342, pParse, TK_VARIABLE, yymsp[0].minor.yy0); - sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy342.pExpr); + u32 n = yymsp[0].minor.yy0.n; + yymsp[0].minor.yy454 = tokenExpr(pParse, TK_VARIABLE, yymsp[0].minor.yy0); + sqlite3ExprAssignVarNumber(pParse, yymsp[0].minor.yy454, n); }else{ /* When doing a nested parse, one can include terms in an expression ** that look like this: #1 #2 ... These terms refer to registers ** in the virtual machine. #N is the N-th register. */ Token t = yymsp[0].minor.yy0; /*A-overwrites-X*/ assert( t.n>=2 ); - spanSet(&yymsp[0].minor.yy342, &t, &t); if( pParse->nested==0 ){ sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &t); - yymsp[0].minor.yy342.pExpr = 0; + yymsp[0].minor.yy454 = 0; }else{ - yymsp[0].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_REGISTER, 0, 0, &t); - if( yymsp[0].minor.yy342.pExpr ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy342.pExpr->iTable); + yymsp[0].minor.yy454 = sqlite3PExpr(pParse, TK_REGISTER, 0, 0); + if( yymsp[0].minor.yy454 ) sqlite3GetInt32(&t.z[1], &yymsp[0].minor.yy454->iTable); } } } break; - case 157: /* expr ::= expr COLLATE ID|STRING */ + case 187: /* expr ::= expr COLLATE ID|STRING */ { - yymsp[-2].minor.yy342.pExpr = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy342.pExpr, &yymsp[0].minor.yy0, 1); - yymsp[-2].minor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-2].minor.yy454 = sqlite3ExprAddCollateToken(pParse, yymsp[-2].minor.yy454, &yymsp[0].minor.yy0, 1); } break; - case 158: /* expr ::= CAST LP expr AS typetoken RP */ + case 188: /* expr ::= CAST LP expr AS typetoken RP */ { - spanSet(&yymsp[-5].minor.yy342,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-5].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_CAST, yymsp[-3].minor.yy342.pExpr, 0, &yymsp[-1].minor.yy0); + yymsp[-5].minor.yy454 = sqlite3ExprAlloc(pParse->db, TK_CAST, &yymsp[-1].minor.yy0, 1); + sqlite3ExprAttachSubtrees(pParse->db, yymsp[-5].minor.yy454, yymsp[-3].minor.yy454, 0); } break; - case 159: /* expr ::= ID|INDEXED LP distinct exprlist RP */ + case 189: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP */ { - if( yymsp[-1].minor.yy442 && yymsp[-1].minor.yy442->nExpr>pParse->db->aLimit[SQLITE_LIMIT_FUNCTION_ARG] ){ - sqlite3ErrorMsg(pParse, "too many arguments on function %T", &yymsp[-4].minor.yy0); - } - yylhsminor.yy342.pExpr = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy442, &yymsp[-4].minor.yy0); - spanSet(&yylhsminor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); - if( yymsp[-2].minor.yy392==SF_Distinct && yylhsminor.yy342.pExpr ){ - yylhsminor.yy342.pExpr->flags |= EP_Distinct; - } + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy144); } - yymsp[-4].minor.yy342 = yylhsminor.yy342; + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 160: /* expr ::= ID|INDEXED LP STAR RP */ + case 190: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP */ { - yylhsminor.yy342.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0); - spanSet(&yylhsminor.yy342,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-4].minor.yy14, &yymsp[-7].minor.yy0, yymsp[-5].minor.yy144); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-1].minor.yy14); } - yymsp[-3].minor.yy342 = yylhsminor.yy342; + yymsp[-7].minor.yy454 = yylhsminor.yy454; break; - case 161: /* term ::= CTIME_KW */ + case 191: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP */ { - yylhsminor.yy342.pExpr = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0); - spanSet(&yylhsminor.yy342, &yymsp[0].minor.yy0, &yymsp[0].minor.yy0); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-3].minor.yy0, 0); } - yymsp[0].minor.yy342 = yylhsminor.yy342; + yymsp[-3].minor.yy454 = yylhsminor.yy454; break; - case 162: /* expr ::= expr AND expr */ - case 163: /* expr ::= expr OR expr */ yytestcase(yyruleno==163); - case 164: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==164); - case 165: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==165); - case 166: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==166); - case 167: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==167); - case 168: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==168); - case 169: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==169); -{spanBinaryExpr(pParse,yymsp[-1].major,&yymsp[-2].minor.yy342,&yymsp[0].minor.yy342);} - break; - case 170: /* likeop ::= LIKE_KW|MATCH */ -{yymsp[0].minor.yy318.eOperator = yymsp[0].minor.yy0; yymsp[0].minor.yy318.bNot = 0;/*A-overwrites-X*/} - break; - case 171: /* likeop ::= NOT LIKE_KW|MATCH */ -{yymsp[-1].minor.yy318.eOperator = yymsp[0].minor.yy0; yymsp[-1].minor.yy318.bNot = 1;} - break; - case 172: /* expr ::= expr likeop expr */ + case 192: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist RP filter_over */ { - ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy342.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy342.pExpr); - yymsp[-2].minor.yy342.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy318.eOperator); - exprNot(pParse, yymsp[-1].minor.yy318.bNot, &yymsp[-2].minor.yy342); - yymsp[-2].minor.yy342.zEnd = yymsp[0].minor.yy342.zEnd; - if( yymsp[-2].minor.yy342.pExpr ) yymsp[-2].minor.yy342.pExpr->flags |= EP_InfixFunc; + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-2].minor.yy14, &yymsp[-5].minor.yy0, yymsp[-3].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } + yymsp[-5].minor.yy454 = yylhsminor.yy454; break; - case 173: /* expr ::= expr likeop expr ESCAPE expr */ + case 193: /* expr ::= ID|INDEXED|JOIN_KW LP distinct exprlist ORDER BY sortlist RP filter_over */ { - ExprList *pList; - pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy342.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy342.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy342.pExpr); - yymsp[-4].minor.yy342.pExpr = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy318.eOperator); - exprNot(pParse, yymsp[-3].minor.yy318.bNot, &yymsp[-4].minor.yy342); - yymsp[-4].minor.yy342.zEnd = yymsp[0].minor.yy342.zEnd; - if( yymsp[-4].minor.yy342.pExpr ) yymsp[-4].minor.yy342.pExpr->flags |= EP_InfixFunc; + yylhsminor.yy454 = sqlite3ExprFunction(pParse, yymsp[-5].minor.yy14, &yymsp[-8].minor.yy0, yymsp[-6].minor.yy144); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); + sqlite3ExprAddFunctionOrderBy(pParse, yylhsminor.yy454, yymsp[-2].minor.yy14); } + yymsp[-8].minor.yy454 = yylhsminor.yy454; break; - case 174: /* expr ::= expr ISNULL|NOTNULL */ -{spanUnaryPostfix(pParse,yymsp[0].major,&yymsp[-1].minor.yy342,&yymsp[0].minor.yy0);} - break; - case 175: /* expr ::= expr NOT NULL */ -{spanUnaryPostfix(pParse,TK_NOTNULL,&yymsp[-2].minor.yy342,&yymsp[0].minor.yy0);} - break; - case 176: /* expr ::= expr IS expr */ + case 194: /* expr ::= ID|INDEXED|JOIN_KW LP STAR RP filter_over */ { - spanBinaryExpr(pParse,TK_IS,&yymsp[-2].minor.yy342,&yymsp[0].minor.yy342); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy342.pExpr, yymsp[-2].minor.yy342.pExpr, TK_ISNULL); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[-4].minor.yy0, 0); + sqlite3WindowAttach(pParse, yylhsminor.yy454, yymsp[0].minor.yy211); } + yymsp[-4].minor.yy454 = yylhsminor.yy454; break; - case 177: /* expr ::= expr IS NOT expr */ + case 195: /* term ::= CTIME_KW */ { - spanBinaryExpr(pParse,TK_ISNOT,&yymsp[-3].minor.yy342,&yymsp[0].minor.yy342); - binaryToUnaryIfNull(pParse, yymsp[0].minor.yy342.pExpr, yymsp[-3].minor.yy342.pExpr, TK_NOTNULL); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, 0, &yymsp[0].minor.yy0, 0); } + yymsp[0].minor.yy454 = yylhsminor.yy454; break; - case 178: /* expr ::= NOT expr */ - case 179: /* expr ::= BITNOT expr */ yytestcase(yyruleno==179); -{spanUnaryPrefix(&yymsp[-1].minor.yy342,pParse,yymsp[-1].major,&yymsp[0].minor.yy342,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} - break; - case 180: /* expr ::= MINUS expr */ -{spanUnaryPrefix(&yymsp[-1].minor.yy342,pParse,TK_UMINUS,&yymsp[0].minor.yy342,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} - break; - case 181: /* expr ::= PLUS expr */ -{spanUnaryPrefix(&yymsp[-1].minor.yy342,pParse,TK_UPLUS,&yymsp[0].minor.yy342,&yymsp[-1].minor.yy0);/*A-overwrites-B*/} - break; - case 182: /* between_op ::= BETWEEN */ - case 185: /* in_op ::= IN */ yytestcase(yyruleno==185); -{yymsp[0].minor.yy392 = 0;} - break; - case 184: /* expr ::= expr between_op expr AND expr */ + case 196: /* expr ::= LP nexprlist COMMA expr RP */ { - ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy342.pExpr); - pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy342.pExpr); - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy342.pExpr, 0, 0); - if( yymsp[-4].minor.yy342.pExpr ){ - yymsp[-4].minor.yy342.pExpr->x.pList = pList; + ExprList *pList = sqlite3ExprListAppend(pParse, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_VECTOR, 0, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; + if( ALWAYS(pList->nExpr) ){ + yymsp[-4].minor.yy454->flags |= pList->a[0].pExpr->flags & EP_Propagate; + } }else{ sqlite3ExprListDelete(pParse->db, pList); - } - exprNot(pParse, yymsp[-3].minor.yy392, &yymsp[-4].minor.yy342); - yymsp[-4].minor.yy342.zEnd = yymsp[0].minor.yy342.zEnd; + } } break; - case 187: /* expr ::= expr in_op LP exprlist RP */ + case 197: /* expr ::= expr AND expr */ +{yymsp[-2].minor.yy454=sqlite3ExprAnd(pParse,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} + break; + case 198: /* expr ::= expr OR expr */ + case 199: /* expr ::= expr LT|GT|GE|LE expr */ yytestcase(yyruleno==199); + case 200: /* expr ::= expr EQ|NE expr */ yytestcase(yyruleno==200); + case 201: /* expr ::= expr BITAND|BITOR|LSHIFT|RSHIFT expr */ yytestcase(yyruleno==201); + case 202: /* expr ::= expr PLUS|MINUS expr */ yytestcase(yyruleno==202); + case 203: /* expr ::= expr STAR|SLASH|REM expr */ yytestcase(yyruleno==203); + case 204: /* expr ::= expr CONCAT expr */ yytestcase(yyruleno==204); +{yymsp[-2].minor.yy454=sqlite3PExpr(pParse,yymsp[-1].major,yymsp[-2].minor.yy454,yymsp[0].minor.yy454);} + break; + case 205: /* likeop ::= NOT LIKE_KW|MATCH */ +{yymsp[-1].minor.yy0=yymsp[0].minor.yy0; yymsp[-1].minor.yy0.n|=0x80000000; /*yymsp[-1].minor.yy0-overwrite-yymsp[0].minor.yy0*/} + break; + case 206: /* expr ::= expr likeop expr */ { - if( yymsp[-1].minor.yy442==0 ){ + ExprList *pList; + int bNot = yymsp[-1].minor.yy0.n & 0x80000000; + yymsp[-1].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[0].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-2].minor.yy454); + yymsp[-2].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); + if( bNot ) yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-2].minor.yy454, 0); + if( yymsp[-2].minor.yy454 ) yymsp[-2].minor.yy454->flags |= EP_InfixFunc; +} + break; + case 207: /* expr ::= expr likeop expr ESCAPE expr */ +{ + ExprList *pList; + int bNot = yymsp[-3].minor.yy0.n & 0x80000000; + yymsp[-3].minor.yy0.n &= 0x7fffffff; + pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[-4].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-3].minor.yy0, 0); + if( bNot ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ) yymsp[-4].minor.yy454->flags |= EP_InfixFunc; +} + break; + case 208: /* expr ::= expr ISNULL|NOTNULL */ +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse,yymsp[0].major,yymsp[-1].minor.yy454,0);} + break; + case 209: /* expr ::= expr NOT NULL */ +{yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_NOTNULL,yymsp[-2].minor.yy454,0);} + break; + case 210: /* expr ::= expr IS expr */ +{ + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-2].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-2].minor.yy454, TK_ISNULL); +} + break; + case 211: /* expr ::= expr IS NOT expr */ +{ + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-3].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-3].minor.yy454, TK_NOTNULL); +} + break; + case 212: /* expr ::= expr IS NOT DISTINCT FROM expr */ +{ + yymsp[-5].minor.yy454 = sqlite3PExpr(pParse,TK_IS,yymsp[-5].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-5].minor.yy454, TK_ISNULL); +} + break; + case 213: /* expr ::= expr IS DISTINCT FROM expr */ +{ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse,TK_ISNOT,yymsp[-4].minor.yy454,yymsp[0].minor.yy454); + binaryToUnaryIfNull(pParse, yymsp[0].minor.yy454, yymsp[-4].minor.yy454, TK_NOTNULL); +} + break; + case 214: /* expr ::= NOT expr */ + case 215: /* expr ::= BITNOT expr */ yytestcase(yyruleno==215); +{yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, yymsp[-1].major, yymsp[0].minor.yy454, 0);/*A-overwrites-B*/} + break; + case 216: /* expr ::= PLUS|MINUS expr */ +{ + Expr *p = yymsp[0].minor.yy454; + u8 op = yymsp[-1].major + (TK_UPLUS-TK_PLUS); + assert( TK_UPLUS>TK_PLUS ); + assert( TK_UMINUS == TK_MINUS + (TK_UPLUS - TK_PLUS) ); + if( p && p->op==TK_UPLUS ){ + p->op = op; + yymsp[-1].minor.yy454 = p; + }else{ + yymsp[-1].minor.yy454 = sqlite3PExpr(pParse, op, p, 0); + /*A-overwrites-B*/ + } +} + break; + case 217: /* expr ::= expr PTR expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse, 0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse, pList, yymsp[0].minor.yy454); + yylhsminor.yy454 = sqlite3ExprFunction(pParse, pList, &yymsp[-1].minor.yy0, 0); +} + yymsp[-2].minor.yy454 = yylhsminor.yy454; + break; + case 218: /* between_op ::= BETWEEN */ + case 221: /* in_op ::= IN */ yytestcase(yyruleno==221); +{yymsp[0].minor.yy144 = 0;} + break; + case 220: /* expr ::= expr between_op expr AND expr */ +{ + ExprList *pList = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + pList = sqlite3ExprListAppend(pParse,pList, yymsp[0].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_BETWEEN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = pList; + }else{ + sqlite3ExprListDelete(pParse->db, pList); + } + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); +} + break; + case 223: /* expr ::= expr in_op LP exprlist RP */ +{ + if( yymsp[-1].minor.yy14==0 ){ /* Expressions of the form ** ** expr1 IN () @@ -133686,434 +178429,607 @@ static void yy_reduce( ** simplify to constants 0 (false) and 1 (true), respectively, ** regardless of the value of expr1. */ - sqlite3ExprDelete(pParse->db, yymsp[-4].minor.yy342.pExpr); - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_INTEGER, 0, 0, &sqlite3IntTokens[yymsp[-3].minor.yy392]); - }else if( yymsp[-1].minor.yy442->nExpr==1 ){ - /* Expressions of the form: - ** - ** expr1 IN (?1) - ** expr1 NOT IN (?2) - ** - ** with exactly one value on the RHS can be simplified to something - ** like this: - ** - ** expr1 == ?1 - ** expr1 <> ?2 - ** - ** But, the RHS of the == or <> is marked with the EP_Generic flag - ** so that it may not contribute to the computation of comparison - ** affinity or the collating sequence to use for comparison. Otherwise, - ** the semantics would be subtly different from IN or NOT IN. - */ - Expr *pRHS = yymsp[-1].minor.yy442->a[0].pExpr; - yymsp[-1].minor.yy442->a[0].pExpr = 0; - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy442); - /* pRHS cannot be NULL because a malloc error would have been detected - ** before now and control would have never reached this point */ - if( ALWAYS(pRHS) ){ - pRHS->flags &= ~EP_Collate; - pRHS->flags |= EP_Generic; - } - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, yymsp[-3].minor.yy392 ? TK_NE : TK_EQ, yymsp[-4].minor.yy342.pExpr, pRHS, 0); + sqlite3ExprUnmapAndDelete(pParse, yymsp[-4].minor.yy454); + yymsp[-4].minor.yy454 = sqlite3Expr(pParse->db, TK_STRING, yymsp[-3].minor.yy144 ? "true" : "false"); + if( yymsp[-4].minor.yy454 ) sqlite3ExprIdToTrueFalse(yymsp[-4].minor.yy454); }else{ - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy342.pExpr, 0, 0); - if( yymsp[-4].minor.yy342.pExpr ){ - yymsp[-4].minor.yy342.pExpr->x.pList = yymsp[-1].minor.yy442; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy342.pExpr); + Expr *pRHS = yymsp[-1].minor.yy14->a[0].pExpr; + if( yymsp[-1].minor.yy14->nExpr==1 && sqlite3ExprIsConstant(pParse,pRHS) && yymsp[-4].minor.yy454->op!=TK_VECTOR ){ + yymsp[-1].minor.yy14->a[0].pExpr = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + pRHS = sqlite3PExpr(pParse, TK_UPLUS, pRHS, 0); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_EQ, yymsp[-4].minor.yy454, pRHS); + }else if( yymsp[-1].minor.yy14->nExpr==1 && pRHS->op==TK_SELECT ){ + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pRHS->x.pSelect); + pRHS->x.pSelect = 0; + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy442); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + if( yymsp[-4].minor.yy454==0 ){ + sqlite3ExprListDelete(pParse->db, yymsp[-1].minor.yy14); + }else if( yymsp[-4].minor.yy454->pLeft->op==TK_VECTOR ){ + int nExpr = yymsp[-4].minor.yy454->pLeft->x.pList->nExpr; + Select *pSelectRHS = sqlite3ExprListToValues(pParse, nExpr, yymsp[-1].minor.yy14); + if( pSelectRHS ){ + parserDoubleLinkSelect(pParse, pSelectRHS); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelectRHS); + } + }else{ + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); + } } - exprNot(pParse, yymsp[-3].minor.yy392, &yymsp[-4].minor.yy342); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } - yymsp[-4].minor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; } break; - case 188: /* expr ::= LP select RP */ + case 224: /* expr ::= LP select RP */ { - spanSet(&yymsp[-2].minor.yy342,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ - yymsp[-2].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_SELECT, 0, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy342.pExpr, yymsp[-1].minor.yy159); + yymsp[-2].minor.yy454 = sqlite3PExpr(pParse, TK_SELECT, 0, 0); + sqlite3PExprAddSelect(pParse, yymsp[-2].minor.yy454, yymsp[-1].minor.yy555); } break; - case 189: /* expr ::= expr in_op LP select RP */ + case 225: /* expr ::= expr in_op LP select RP */ { - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy342.pExpr, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy342.pExpr, yymsp[-1].minor.yy159); - exprNot(pParse, yymsp[-3].minor.yy392, &yymsp[-4].minor.yy342); - yymsp[-4].minor.yy342.zEnd = &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n]; + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, yymsp[-1].minor.yy555); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 190: /* expr ::= expr in_op nm dbnm */ + case 226: /* expr ::= expr in_op nm dbnm paren_exprlist */ { - SrcList *pSrc = sqlite3SrcListAppend(pParse->db, 0,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0); - Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0,0); - yymsp[-3].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_IN, yymsp[-3].minor.yy342.pExpr, 0, 0); - sqlite3PExprAddSelect(pParse, yymsp[-3].minor.yy342.pExpr, pSelect); - exprNot(pParse, yymsp[-2].minor.yy392, &yymsp[-3].minor.yy342); - yymsp[-3].minor.yy342.zEnd = yymsp[0].minor.yy0.z ? &yymsp[0].minor.yy0.z[yymsp[0].minor.yy0.n] : &yymsp[-1].minor.yy0.z[yymsp[-1].minor.yy0.n]; + SrcList *pSrc = sqlite3SrcListAppend(pParse, 0,&yymsp[-2].minor.yy0,&yymsp[-1].minor.yy0); + Select *pSelect = sqlite3SelectNew(pParse, 0,pSrc,0,0,0,0,0,0); + if( yymsp[0].minor.yy14 ) sqlite3SrcListFuncArgs(pParse, pSelect ? pSrc : 0, yymsp[0].minor.yy14); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_IN, yymsp[-4].minor.yy454, 0); + sqlite3PExprAddSelect(pParse, yymsp[-4].minor.yy454, pSelect); + if( yymsp[-3].minor.yy144 ) yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_NOT, yymsp[-4].minor.yy454, 0); } break; - case 191: /* expr ::= EXISTS LP select RP */ + case 227: /* expr ::= EXISTS LP select RP */ { Expr *p; - spanSet(&yymsp[-3].minor.yy342,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-B*/ - p = yymsp[-3].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_EXISTS, 0, 0, 0); - sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy159); + p = yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_EXISTS, 0, 0); + sqlite3PExprAddSelect(pParse, p, yymsp[-1].minor.yy555); } break; - case 192: /* expr ::= CASE case_operand case_exprlist case_else END */ + case 228: /* expr ::= CASE case_operand case_exprlist case_else END */ { - spanSet(&yymsp[-4].minor.yy342,&yymsp[-4].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-C*/ - yymsp[-4].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy122, 0, 0); - if( yymsp[-4].minor.yy342.pExpr ){ - yymsp[-4].minor.yy342.pExpr->x.pList = yymsp[-1].minor.yy122 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy442,yymsp[-1].minor.yy122) : yymsp[-2].minor.yy442; - sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy342.pExpr); + yymsp[-4].minor.yy454 = sqlite3PExpr(pParse, TK_CASE, yymsp[-3].minor.yy454, 0); + if( yymsp[-4].minor.yy454 ){ + yymsp[-4].minor.yy454->x.pList = yymsp[-1].minor.yy454 ? sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[-1].minor.yy454) : yymsp[-2].minor.yy14; + sqlite3ExprSetHeightAndFlags(pParse, yymsp[-4].minor.yy454); }else{ - sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy442); - sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy122); + sqlite3ExprListDelete(pParse->db, yymsp[-2].minor.yy14); + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); } } break; - case 193: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ + case 229: /* case_exprlist ::= case_exprlist WHEN expr THEN expr */ { - yymsp[-4].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, yymsp[-2].minor.yy342.pExpr); - yymsp[-4].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy442, yymsp[0].minor.yy342.pExpr); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[-2].minor.yy454); + yymsp[-4].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-4].minor.yy14, yymsp[0].minor.yy454); } break; - case 194: /* case_exprlist ::= WHEN expr THEN expr */ + case 230: /* case_exprlist ::= WHEN expr THEN expr */ { - yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy342.pExpr); - yymsp[-3].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy442, yymsp[0].minor.yy342.pExpr); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,0, yymsp[-2].minor.yy454); + yymsp[-3].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-3].minor.yy14, yymsp[0].minor.yy454); } break; - case 197: /* case_operand ::= expr */ -{yymsp[0].minor.yy122 = yymsp[0].minor.yy342.pExpr; /*A-overwrites-X*/} + case 235: /* nexprlist ::= nexprlist COMMA expr */ +{yymsp[-2].minor.yy14 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy14,yymsp[0].minor.yy454);} break; - case 200: /* nexprlist ::= nexprlist COMMA expr */ -{yymsp[-2].minor.yy442 = sqlite3ExprListAppend(pParse,yymsp[-2].minor.yy442,yymsp[0].minor.yy342.pExpr);} + case 236: /* nexprlist ::= expr */ +{yymsp[0].minor.yy14 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy454); /*A-overwrites-Y*/} break; - case 201: /* nexprlist ::= expr */ -{yymsp[0].minor.yy442 = sqlite3ExprListAppend(pParse,0,yymsp[0].minor.yy342.pExpr); /*A-overwrites-Y*/} + case 238: /* paren_exprlist ::= LP exprlist RP */ + case 243: /* eidlist_opt ::= LP eidlist RP */ yytestcase(yyruleno==243); +{yymsp[-2].minor.yy14 = yymsp[-1].minor.yy14;} break; - case 202: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ + case 239: /* cmd ::= createkw uniqueflag INDEX ifnotexists nm dbnm ON nm LP sortlist RP where_opt */ { - sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, - sqlite3SrcListAppend(pParse->db,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy442, yymsp[-10].minor.yy392, - &yymsp[-11].minor.yy0, yymsp[0].minor.yy122, SQLITE_SO_ASC, yymsp[-8].minor.yy392); + sqlite3CreateIndex(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, + sqlite3SrcListAppend(pParse,0,&yymsp[-4].minor.yy0,0), yymsp[-2].minor.yy14, yymsp[-10].minor.yy144, + &yymsp[-11].minor.yy0, yymsp[0].minor.yy454, SQLITE_SO_ASC, yymsp[-8].minor.yy144, SQLITE_IDXTYPE_APPDEF); + if( IN_RENAME_OBJECT && pParse->pNewIndex ){ + sqlite3RenameTokenMap(pParse, pParse->pNewIndex->zName, &yymsp[-4].minor.yy0); + } } break; - case 203: /* uniqueflag ::= UNIQUE */ - case 244: /* raisetype ::= ABORT */ yytestcase(yyruleno==244); -{yymsp[0].minor.yy392 = OE_Abort;} + case 240: /* uniqueflag ::= UNIQUE */ + case 282: /* raisetype ::= ABORT */ yytestcase(yyruleno==282); +{yymsp[0].minor.yy144 = OE_Abort;} break; - case 204: /* uniqueflag ::= */ -{yymsp[1].minor.yy392 = OE_None;} + case 241: /* uniqueflag ::= */ +{yymsp[1].minor.yy144 = OE_None;} break; - case 206: /* eidlist_opt ::= LP eidlist RP */ -{yymsp[-2].minor.yy442 = yymsp[-1].minor.yy442;} - break; - case 207: /* eidlist ::= eidlist COMMA nm collate sortorder */ + case 244: /* eidlist ::= eidlist COMMA nm collate sortorder */ { - yymsp[-4].minor.yy442 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy442, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); + yymsp[-4].minor.yy14 = parserAddExprIdListTerm(pParse, yymsp[-4].minor.yy14, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); } break; - case 208: /* eidlist ::= nm collate sortorder */ + case 245: /* eidlist ::= nm collate sortorder */ { - yymsp[-2].minor.yy442 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); /*A-overwrites-Y*/ + yymsp[-2].minor.yy14 = parserAddExprIdListTerm(pParse, 0, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy144, yymsp[0].minor.yy144); /*A-overwrites-Y*/ } break; - case 211: /* cmd ::= DROP INDEX ifexists fullname */ -{sqlite3DropIndex(pParse, yymsp[0].minor.yy347, yymsp[-1].minor.yy392);} + case 248: /* cmd ::= DROP INDEX ifexists fullname */ +{sqlite3DropIndex(pParse, yymsp[0].minor.yy203, yymsp[-1].minor.yy144);} break; - case 212: /* cmd ::= VACUUM */ - case 213: /* cmd ::= VACUUM nm */ yytestcase(yyruleno==213); -{sqlite3Vacuum(pParse);} + case 249: /* cmd ::= VACUUM vinto */ +{sqlite3Vacuum(pParse,0,yymsp[0].minor.yy454);} break; - case 214: /* cmd ::= PRAGMA nm dbnm */ + case 250: /* cmd ::= VACUUM nm vinto */ +{sqlite3Vacuum(pParse,&yymsp[-1].minor.yy0,yymsp[0].minor.yy454);} + break; + case 253: /* cmd ::= PRAGMA nm dbnm */ {sqlite3Pragma(pParse,&yymsp[-1].minor.yy0,&yymsp[0].minor.yy0,0,0);} break; - case 215: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ + case 254: /* cmd ::= PRAGMA nm dbnm EQ nmnum */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,0);} break; - case 216: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ + case 255: /* cmd ::= PRAGMA nm dbnm LP nmnum RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,0);} break; - case 217: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ + case 256: /* cmd ::= PRAGMA nm dbnm EQ minus_num */ {sqlite3Pragma(pParse,&yymsp[-3].minor.yy0,&yymsp[-2].minor.yy0,&yymsp[0].minor.yy0,1);} break; - case 218: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ + case 257: /* cmd ::= PRAGMA nm dbnm LP minus_num RP */ {sqlite3Pragma(pParse,&yymsp[-4].minor.yy0,&yymsp[-3].minor.yy0,&yymsp[-1].minor.yy0,1);} break; - case 221: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ + case 260: /* cmd ::= createkw trigger_decl BEGIN trigger_cmd_list END */ { Token all; all.z = yymsp[-3].minor.yy0.z; all.n = (int)(yymsp[0].minor.yy0.z - yymsp[-3].minor.yy0.z) + yymsp[0].minor.yy0.n; - sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy327, &all); + sqlite3FinishTrigger(pParse, yymsp[-1].minor.yy427, &all); } break; - case 222: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ + case 261: /* trigger_decl ::= temp TRIGGER ifnotexists nm dbnm trigger_time trigger_event ON fullname foreach_clause when_clause */ { - sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy392, yymsp[-4].minor.yy410.a, yymsp[-4].minor.yy410.b, yymsp[-2].minor.yy347, yymsp[0].minor.yy122, yymsp[-10].minor.yy392, yymsp[-8].minor.yy392); + sqlite3BeginTrigger(pParse, &yymsp[-7].minor.yy0, &yymsp[-6].minor.yy0, yymsp[-5].minor.yy144, yymsp[-4].minor.yy286.a, yymsp[-4].minor.yy286.b, yymsp[-2].minor.yy203, yymsp[0].minor.yy454, yymsp[-10].minor.yy144, yymsp[-8].minor.yy144); yymsp[-10].minor.yy0 = (yymsp[-6].minor.yy0.n==0?yymsp[-7].minor.yy0:yymsp[-6].minor.yy0); /*A-overwrites-T*/ } break; - case 223: /* trigger_time ::= BEFORE */ -{ yymsp[0].minor.yy392 = TK_BEFORE; } + case 262: /* trigger_time ::= BEFORE|AFTER */ +{ yymsp[0].minor.yy144 = yymsp[0].major; /*A-overwrites-X*/ } break; - case 224: /* trigger_time ::= AFTER */ -{ yymsp[0].minor.yy392 = TK_AFTER; } + case 263: /* trigger_time ::= INSTEAD OF */ +{ yymsp[-1].minor.yy144 = TK_INSTEAD;} break; - case 225: /* trigger_time ::= INSTEAD OF */ -{ yymsp[-1].minor.yy392 = TK_INSTEAD;} + case 264: /* trigger_time ::= */ +{ yymsp[1].minor.yy144 = TK_BEFORE; } break; - case 226: /* trigger_time ::= */ -{ yymsp[1].minor.yy392 = TK_BEFORE; } + case 265: /* trigger_event ::= DELETE|INSERT */ + case 266: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==266); +{yymsp[0].minor.yy286.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy286.b = 0;} break; - case 227: /* trigger_event ::= DELETE|INSERT */ - case 228: /* trigger_event ::= UPDATE */ yytestcase(yyruleno==228); -{yymsp[0].minor.yy410.a = yymsp[0].major; /*A-overwrites-X*/ yymsp[0].minor.yy410.b = 0;} + case 267: /* trigger_event ::= UPDATE OF idlist */ +{yymsp[-2].minor.yy286.a = TK_UPDATE; yymsp[-2].minor.yy286.b = yymsp[0].minor.yy132;} break; - case 229: /* trigger_event ::= UPDATE OF idlist */ -{yymsp[-2].minor.yy410.a = TK_UPDATE; yymsp[-2].minor.yy410.b = yymsp[0].minor.yy180;} + case 268: /* when_clause ::= */ + case 287: /* key_opt ::= */ yytestcase(yyruleno==287); +{ yymsp[1].minor.yy454 = 0; } break; - case 230: /* when_clause ::= */ - case 249: /* key_opt ::= */ yytestcase(yyruleno==249); -{ yymsp[1].minor.yy122 = 0; } + case 269: /* when_clause ::= WHEN expr */ + case 288: /* key_opt ::= KEY expr */ yytestcase(yyruleno==288); +{ yymsp[-1].minor.yy454 = yymsp[0].minor.yy454; } break; - case 231: /* when_clause ::= WHEN expr */ - case 250: /* key_opt ::= KEY expr */ yytestcase(yyruleno==250); -{ yymsp[-1].minor.yy122 = yymsp[0].minor.yy342.pExpr; } - break; - case 232: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ + case 270: /* trigger_cmd_list ::= trigger_cmd_list trigger_cmd SEMI */ { - assert( yymsp[-2].minor.yy327!=0 ); - yymsp[-2].minor.yy327->pLast->pNext = yymsp[-1].minor.yy327; - yymsp[-2].minor.yy327->pLast = yymsp[-1].minor.yy327; + assert( yymsp[-2].minor.yy427!=0 ); + yymsp[-2].minor.yy427->pLast->pNext = yymsp[-1].minor.yy427; + yymsp[-2].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 233: /* trigger_cmd_list ::= trigger_cmd SEMI */ -{ - assert( yymsp[-1].minor.yy327!=0 ); - yymsp[-1].minor.yy327->pLast = yymsp[-1].minor.yy327; + case 271: /* trigger_cmd_list ::= trigger_cmd SEMI */ +{ + assert( yymsp[-1].minor.yy427!=0 ); + yymsp[-1].minor.yy427->pLast = yymsp[-1].minor.yy427; } break; - case 234: /* trnm ::= nm DOT nm */ + case 272: /* trnm ::= nm DOT nm */ { yymsp[-2].minor.yy0 = yymsp[0].minor.yy0; - sqlite3ErrorMsg(pParse, + sqlite3ErrorMsg(pParse, "qualified table names are not allowed on INSERT, UPDATE, and DELETE " "statements within triggers"); } break; - case 235: /* tridxby ::= INDEXED BY nm */ + case 273: /* tridxby ::= INDEXED BY nm */ { sqlite3ErrorMsg(pParse, "the INDEXED BY clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 236: /* tridxby ::= NOT INDEXED */ + case 274: /* tridxby ::= NOT INDEXED */ { sqlite3ErrorMsg(pParse, "the NOT INDEXED clause is not allowed on UPDATE or DELETE statements " "within triggers"); } break; - case 237: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist where_opt */ -{yymsp[-6].minor.yy327 = sqlite3TriggerUpdateStep(pParse->db, &yymsp[-4].minor.yy0, yymsp[-1].minor.yy442, yymsp[0].minor.yy122, yymsp[-5].minor.yy392);} + case 275: /* trigger_cmd ::= UPDATE orconf trnm tridxby SET setlist from where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerUpdateStep(pParse, &yymsp[-6].minor.yy0, yymsp[-2].minor.yy203, yymsp[-3].minor.yy14, yymsp[-1].minor.yy454, yymsp[-7].minor.yy144, yymsp[-8].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-8].minor.yy427 = yylhsminor.yy427; break; - case 238: /* trigger_cmd ::= insert_cmd INTO trnm idlist_opt select */ -{yymsp[-4].minor.yy327 = sqlite3TriggerInsertStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[-1].minor.yy180, yymsp[0].minor.yy159, yymsp[-4].minor.yy392);/*A-overwrites-R*/} - break; - case 239: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt */ -{yymsp[-4].minor.yy327 = sqlite3TriggerDeleteStep(pParse->db, &yymsp[-2].minor.yy0, yymsp[0].minor.yy122);} - break; - case 240: /* trigger_cmd ::= select */ -{yymsp[0].minor.yy327 = sqlite3TriggerSelectStep(pParse->db, yymsp[0].minor.yy159); /*A-overwrites-X*/} - break; - case 241: /* expr ::= RAISE LP IGNORE RP */ + case 276: /* trigger_cmd ::= scanpt insert_cmd INTO trnm idlist_opt select upsert scanpt */ { - spanSet(&yymsp[-3].minor.yy342,&yymsp[-3].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-3].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, 0); - if( yymsp[-3].minor.yy342.pExpr ){ - yymsp[-3].minor.yy342.pExpr->affinity = OE_Ignore; + yylhsminor.yy427 = sqlite3TriggerInsertStep(pParse,&yymsp[-4].minor.yy0,yymsp[-3].minor.yy132,yymsp[-2].minor.yy555,yymsp[-6].minor.yy144,yymsp[-1].minor.yy122,yymsp[-7].minor.yy168,yymsp[0].minor.yy168);/*yylhsminor.yy427-overwrites-yymsp[-6].minor.yy144*/ +} + yymsp[-7].minor.yy427 = yylhsminor.yy427; + break; + case 277: /* trigger_cmd ::= DELETE FROM trnm tridxby where_opt scanpt */ +{yylhsminor.yy427 = sqlite3TriggerDeleteStep(pParse, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy454, yymsp[-5].minor.yy0.z, yymsp[0].minor.yy168);} + yymsp[-5].minor.yy427 = yylhsminor.yy427; + break; + case 278: /* trigger_cmd ::= scanpt select scanpt */ +{yylhsminor.yy427 = sqlite3TriggerSelectStep(pParse->db, yymsp[-1].minor.yy555, yymsp[-2].minor.yy168, yymsp[0].minor.yy168); /*yylhsminor.yy427-overwrites-yymsp[-1].minor.yy555*/} + yymsp[-2].minor.yy427 = yylhsminor.yy427; + break; + case 279: /* expr ::= RAISE LP IGNORE RP */ +{ + yymsp[-3].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, 0, 0); + if( yymsp[-3].minor.yy454 ){ + yymsp[-3].minor.yy454->affExpr = OE_Ignore; } } break; - case 242: /* expr ::= RAISE LP raisetype COMMA nm RP */ + case 280: /* expr ::= RAISE LP raisetype COMMA expr RP */ { - spanSet(&yymsp[-5].minor.yy342,&yymsp[-5].minor.yy0,&yymsp[0].minor.yy0); /*A-overwrites-X*/ - yymsp[-5].minor.yy342.pExpr = sqlite3PExpr(pParse, TK_RAISE, 0, 0, &yymsp[-1].minor.yy0); - if( yymsp[-5].minor.yy342.pExpr ) { - yymsp[-5].minor.yy342.pExpr->affinity = (char)yymsp[-3].minor.yy392; + yymsp[-5].minor.yy454 = sqlite3PExpr(pParse, TK_RAISE, yymsp[-1].minor.yy454, 0); + if( yymsp[-5].minor.yy454 ) { + yymsp[-5].minor.yy454->affExpr = (char)yymsp[-3].minor.yy144; } } break; - case 243: /* raisetype ::= ROLLBACK */ -{yymsp[0].minor.yy392 = OE_Rollback;} + case 281: /* raisetype ::= ROLLBACK */ +{yymsp[0].minor.yy144 = OE_Rollback;} break; - case 245: /* raisetype ::= FAIL */ -{yymsp[0].minor.yy392 = OE_Fail;} + case 283: /* raisetype ::= FAIL */ +{yymsp[0].minor.yy144 = OE_Fail;} break; - case 246: /* cmd ::= DROP TRIGGER ifexists fullname */ + case 284: /* cmd ::= DROP TRIGGER ifexists fullname */ { - sqlite3DropTrigger(pParse,yymsp[0].minor.yy347,yymsp[-1].minor.yy392); + sqlite3DropTrigger(pParse,yymsp[0].minor.yy203,yymsp[-1].minor.yy144); } break; - case 247: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ + case 285: /* cmd ::= ATTACH database_kw_opt expr AS expr key_opt */ { - sqlite3Attach(pParse, yymsp[-3].minor.yy342.pExpr, yymsp[-1].minor.yy342.pExpr, yymsp[0].minor.yy122); + sqlite3Attach(pParse, yymsp[-3].minor.yy454, yymsp[-1].minor.yy454, yymsp[0].minor.yy454); } break; - case 248: /* cmd ::= DETACH database_kw_opt expr */ + case 286: /* cmd ::= DETACH database_kw_opt expr */ { - sqlite3Detach(pParse, yymsp[0].minor.yy342.pExpr); + sqlite3Detach(pParse, yymsp[0].minor.yy454); } break; - case 251: /* cmd ::= REINDEX */ + case 289: /* cmd ::= REINDEX */ {sqlite3Reindex(pParse, 0, 0);} break; - case 252: /* cmd ::= REINDEX nm dbnm */ + case 290: /* cmd ::= REINDEX nm dbnm */ {sqlite3Reindex(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 253: /* cmd ::= ANALYZE */ + case 291: /* cmd ::= ANALYZE */ {sqlite3Analyze(pParse, 0, 0);} break; - case 254: /* cmd ::= ANALYZE nm dbnm */ + case 292: /* cmd ::= ANALYZE nm dbnm */ {sqlite3Analyze(pParse, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0);} break; - case 255: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ + case 293: /* cmd ::= ALTER TABLE fullname RENAME TO nm */ { - sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy347,&yymsp[0].minor.yy0); + sqlite3AlterRenameTable(pParse,yymsp[-3].minor.yy203,&yymsp[0].minor.yy0); } break; - case 256: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ + case 294: /* cmd ::= ALTER TABLE add_column_fullname ADD kwcolumn_opt columnname carglist */ { yymsp[-1].minor.yy0.n = (int)(pParse->sLastToken.z-yymsp[-1].minor.yy0.z) + pParse->sLastToken.n; sqlite3AlterFinishAddColumn(pParse, &yymsp[-1].minor.yy0); } break; - case 257: /* add_column_fullname ::= fullname */ + case 295: /* cmd ::= ALTER TABLE fullname DROP kwcolumn_opt nm */ +{ + sqlite3AlterDropColumn(pParse, yymsp[-3].minor.yy203, &yymsp[0].minor.yy0); +} + break; + case 296: /* add_column_fullname ::= fullname */ { disableLookaside(pParse); - sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy347); + sqlite3AlterBeginAddColumn(pParse, yymsp[0].minor.yy203); } break; - case 258: /* cmd ::= create_vtab */ + case 297: /* cmd ::= ALTER TABLE fullname RENAME kwcolumn_opt nm TO nm */ +{ + sqlite3AlterRenameColumn(pParse, yymsp[-5].minor.yy203, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); +} + break; + case 298: /* cmd ::= create_vtab */ {sqlite3VtabFinishParse(pParse,0);} break; - case 259: /* cmd ::= create_vtab LP vtabarglist RP */ + case 299: /* cmd ::= create_vtab LP vtabarglist RP */ {sqlite3VtabFinishParse(pParse,&yymsp[0].minor.yy0);} break; - case 260: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ + case 300: /* create_vtab ::= createkw VIRTUAL TABLE ifnotexists nm dbnm USING nm */ { - sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy392); + sqlite3VtabBeginParse(pParse, &yymsp[-3].minor.yy0, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-4].minor.yy144); } break; - case 261: /* vtabarg ::= */ + case 301: /* vtabarg ::= */ {sqlite3VtabArgInit(pParse);} break; - case 262: /* vtabargtoken ::= ANY */ - case 263: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==263); - case 264: /* lp ::= LP */ yytestcase(yyruleno==264); + case 302: /* vtabargtoken ::= ANY */ + case 303: /* vtabargtoken ::= lp anylist RP */ yytestcase(yyruleno==303); + case 304: /* lp ::= LP */ yytestcase(yyruleno==304); {sqlite3VtabArgExtend(pParse,&yymsp[0].minor.yy0);} break; - case 265: /* with ::= */ -{yymsp[1].minor.yy331 = 0;} + case 305: /* with ::= WITH wqlist */ + case 306: /* with ::= WITH RECURSIVE wqlist */ yytestcase(yyruleno==306); +{ sqlite3WithPush(pParse, yymsp[0].minor.yy59, 1); } break; - case 266: /* with ::= WITH wqlist */ -{ yymsp[-1].minor.yy331 = yymsp[0].minor.yy331; } + case 307: /* wqas ::= AS */ +{yymsp[0].minor.yy462 = M10d_Any;} break; - case 267: /* with ::= WITH RECURSIVE wqlist */ -{ yymsp[-2].minor.yy331 = yymsp[0].minor.yy331; } + case 308: /* wqas ::= AS MATERIALIZED */ +{yymsp[-1].minor.yy462 = M10d_Yes;} break; - case 268: /* wqlist ::= nm eidlist_opt AS LP select RP */ + case 309: /* wqas ::= AS NOT MATERIALIZED */ +{yymsp[-2].minor.yy462 = M10d_No;} + break; + case 310: /* wqitem ::= withnm eidlist_opt wqas LP select RP */ { - yymsp[-5].minor.yy331 = sqlite3WithAdd(pParse, 0, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy442, yymsp[-1].minor.yy159); /*A-overwrites-X*/ + yymsp[-5].minor.yy67 = sqlite3CteNew(pParse, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy14, yymsp[-1].minor.yy555, yymsp[-3].minor.yy462); /*A-overwrites-X*/ } break; - case 269: /* wqlist ::= wqlist COMMA nm eidlist_opt AS LP select RP */ + case 311: /* withnm ::= nm */ +{pParse->bHasWith = 1;} + break; + case 312: /* wqlist ::= wqitem */ { - yymsp[-7].minor.yy331 = sqlite3WithAdd(pParse, yymsp[-7].minor.yy331, &yymsp[-5].minor.yy0, yymsp[-4].minor.yy442, yymsp[-1].minor.yy159); + yymsp[0].minor.yy59 = sqlite3WithAdd(pParse, 0, yymsp[0].minor.yy67); /*A-overwrites-X*/ } break; + case 313: /* wqlist ::= wqlist COMMA wqitem */ +{ + yymsp[-2].minor.yy59 = sqlite3WithAdd(pParse, yymsp[-2].minor.yy59, yymsp[0].minor.yy67); +} + break; + case 314: /* windowdefn_list ::= windowdefn_list COMMA windowdefn */ +{ + assert( yymsp[0].minor.yy211!=0 ); + sqlite3WindowChain(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy211); + yymsp[0].minor.yy211->pNextWin = yymsp[-2].minor.yy211; + yylhsminor.yy211 = yymsp[0].minor.yy211; +} + yymsp[-2].minor.yy211 = yylhsminor.yy211; + break; + case 315: /* windowdefn ::= nm AS LP window RP */ +{ + if( ALWAYS(yymsp[-1].minor.yy211) ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[-4].minor.yy0.z, yymsp[-4].minor.yy0.n); + } + yylhsminor.yy211 = yymsp[-1].minor.yy211; +} + yymsp[-4].minor.yy211 = yylhsminor.yy211; + break; + case 316: /* window ::= PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yymsp[-4].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, 0); +} + break; + case 317: /* window ::= nm PARTITION BY nexprlist orderby_opt frame_opt */ +{ + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, yymsp[-2].minor.yy14, yymsp[-1].minor.yy14, &yymsp[-5].minor.yy0); +} + yymsp[-5].minor.yy211 = yylhsminor.yy211; + break; + case 318: /* window ::= ORDER BY sortlist frame_opt */ +{ + yymsp[-3].minor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, 0); +} + break; + case 319: /* window ::= nm ORDER BY sortlist frame_opt */ +{ + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, yymsp[-1].minor.yy14, &yymsp[-4].minor.yy0); +} + yymsp[-4].minor.yy211 = yylhsminor.yy211; + break; + case 320: /* window ::= nm frame_opt */ +{ + yylhsminor.yy211 = sqlite3WindowAssemble(pParse, yymsp[0].minor.yy211, 0, 0, &yymsp[-1].minor.yy0); +} + yymsp[-1].minor.yy211 = yylhsminor.yy211; + break; + case 321: /* frame_opt ::= */ +{ + yymsp[1].minor.yy211 = sqlite3WindowAlloc(pParse, 0, TK_UNBOUNDED, 0, TK_CURRENT, 0, 0); +} + break; + case 322: /* frame_opt ::= range_or_rows frame_bound_s frame_exclude_opt */ +{ + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-2].minor.yy144, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, TK_CURRENT, 0, yymsp[0].minor.yy462); +} + yymsp[-2].minor.yy211 = yylhsminor.yy211; + break; + case 323: /* frame_opt ::= range_or_rows BETWEEN frame_bound_s AND frame_bound_e frame_exclude_opt */ +{ + yylhsminor.yy211 = sqlite3WindowAlloc(pParse, yymsp[-5].minor.yy144, yymsp[-3].minor.yy509.eType, yymsp[-3].minor.yy509.pExpr, yymsp[-1].minor.yy509.eType, yymsp[-1].minor.yy509.pExpr, yymsp[0].minor.yy462); +} + yymsp[-5].minor.yy211 = yylhsminor.yy211; + break; + case 325: /* frame_bound_s ::= frame_bound */ + case 327: /* frame_bound_e ::= frame_bound */ yytestcase(yyruleno==327); +{yylhsminor.yy509 = yymsp[0].minor.yy509;} + yymsp[0].minor.yy509 = yylhsminor.yy509; + break; + case 326: /* frame_bound_s ::= UNBOUNDED PRECEDING */ + case 328: /* frame_bound_e ::= UNBOUNDED FOLLOWING */ yytestcase(yyruleno==328); + case 330: /* frame_bound ::= CURRENT ROW */ yytestcase(yyruleno==330); +{yylhsminor.yy509.eType = yymsp[-1].major; yylhsminor.yy509.pExpr = 0;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; + break; + case 329: /* frame_bound ::= expr PRECEDING|FOLLOWING */ +{yylhsminor.yy509.eType = yymsp[0].major; yylhsminor.yy509.pExpr = yymsp[-1].minor.yy454;} + yymsp[-1].minor.yy509 = yylhsminor.yy509; + break; + case 331: /* frame_exclude_opt ::= */ +{yymsp[1].minor.yy462 = 0;} + break; + case 332: /* frame_exclude_opt ::= EXCLUDE frame_exclude */ +{yymsp[-1].minor.yy462 = yymsp[0].minor.yy462;} + break; + case 333: /* frame_exclude ::= NO OTHERS */ + case 334: /* frame_exclude ::= CURRENT ROW */ yytestcase(yyruleno==334); +{yymsp[-1].minor.yy462 = yymsp[-1].major; /*A-overwrites-X*/} + break; + case 335: /* frame_exclude ::= GROUP|TIES */ +{yymsp[0].minor.yy462 = yymsp[0].major; /*A-overwrites-X*/} + break; + case 336: /* window_clause ::= WINDOW windowdefn_list */ +{ yymsp[-1].minor.yy211 = yymsp[0].minor.yy211; } + break; + case 337: /* filter_over ::= filter_clause over_clause */ +{ + if( yymsp[0].minor.yy211 ){ + yymsp[0].minor.yy211->pFilter = yymsp[-1].minor.yy454; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[-1].minor.yy454); + } + yylhsminor.yy211 = yymsp[0].minor.yy211; +} + yymsp[-1].minor.yy211 = yylhsminor.yy211; + break; + case 338: /* filter_over ::= over_clause */ +{ + yylhsminor.yy211 = yymsp[0].minor.yy211; +} + yymsp[0].minor.yy211 = yylhsminor.yy211; + break; + case 339: /* filter_over ::= filter_clause */ +{ + yylhsminor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yylhsminor.yy211 ){ + yylhsminor.yy211->eFrmType = TK_FILTER; + yylhsminor.yy211->pFilter = yymsp[0].minor.yy454; + }else{ + sqlite3ExprDelete(pParse->db, yymsp[0].minor.yy454); + } +} + yymsp[0].minor.yy211 = yylhsminor.yy211; + break; + case 340: /* over_clause ::= OVER LP window RP */ +{ + yymsp[-3].minor.yy211 = yymsp[-1].minor.yy211; + assert( yymsp[-3].minor.yy211!=0 ); +} + break; + case 341: /* over_clause ::= OVER nm */ +{ + yymsp[-1].minor.yy211 = (Window*)sqlite3DbMallocZero(pParse->db, sizeof(Window)); + if( yymsp[-1].minor.yy211 ){ + yymsp[-1].minor.yy211->zName = sqlite3DbStrNDup(pParse->db, yymsp[0].minor.yy0.z, yymsp[0].minor.yy0.n); + } +} + break; + case 342: /* filter_clause ::= FILTER LP WHERE expr RP */ +{ yymsp[-4].minor.yy454 = yymsp[-1].minor.yy454; } + break; + case 343: /* term ::= QNUMBER */ +{ + yylhsminor.yy454=tokenExpr(pParse,yymsp[0].major,yymsp[0].minor.yy0); + sqlite3DequoteNumber(pParse, yylhsminor.yy454); +} + yymsp[0].minor.yy454 = yylhsminor.yy454; + break; default: - /* (270) input ::= cmdlist */ yytestcase(yyruleno==270); - /* (271) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==271); - /* (272) cmdlist ::= ecmd */ yytestcase(yyruleno==272); - /* (273) ecmd ::= SEMI */ yytestcase(yyruleno==273); - /* (274) ecmd ::= explain cmdx SEMI */ yytestcase(yyruleno==274); - /* (275) explain ::= */ yytestcase(yyruleno==275); - /* (276) trans_opt ::= */ yytestcase(yyruleno==276); - /* (277) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==277); - /* (278) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==278); - /* (279) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==279); - /* (280) savepoint_opt ::= */ yytestcase(yyruleno==280); - /* (281) cmd ::= create_table create_table_args */ yytestcase(yyruleno==281); - /* (282) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==282); - /* (283) columnlist ::= columnname carglist */ yytestcase(yyruleno==283); - /* (284) nm ::= ID|INDEXED */ yytestcase(yyruleno==284); - /* (285) nm ::= STRING */ yytestcase(yyruleno==285); - /* (286) nm ::= JOIN_KW */ yytestcase(yyruleno==286); - /* (287) typetoken ::= typename */ yytestcase(yyruleno==287); - /* (288) typename ::= ID|STRING */ yytestcase(yyruleno==288); - /* (289) signed ::= plus_num */ yytestcase(yyruleno==289); - /* (290) signed ::= minus_num */ yytestcase(yyruleno==290); - /* (291) carglist ::= carglist ccons */ yytestcase(yyruleno==291); - /* (292) carglist ::= */ yytestcase(yyruleno==292); - /* (293) ccons ::= NULL onconf */ yytestcase(yyruleno==293); - /* (294) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==294); - /* (295) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==295); - /* (296) conslist ::= tcons */ yytestcase(yyruleno==296); - /* (297) tconscomma ::= */ yytestcase(yyruleno==297); - /* (298) defer_subclause_opt ::= defer_subclause */ yytestcase(yyruleno==298); - /* (299) resolvetype ::= raisetype */ yytestcase(yyruleno==299); - /* (300) selectnowith ::= oneselect */ yytestcase(yyruleno==300); - /* (301) oneselect ::= values */ yytestcase(yyruleno==301); - /* (302) sclp ::= selcollist COMMA */ yytestcase(yyruleno==302); - /* (303) as ::= ID|STRING */ yytestcase(yyruleno==303); - /* (304) expr ::= term */ yytestcase(yyruleno==304); - /* (305) exprlist ::= nexprlist */ yytestcase(yyruleno==305); - /* (306) nmnum ::= plus_num */ yytestcase(yyruleno==306); - /* (307) nmnum ::= nm */ yytestcase(yyruleno==307); - /* (308) nmnum ::= ON */ yytestcase(yyruleno==308); - /* (309) nmnum ::= DELETE */ yytestcase(yyruleno==309); - /* (310) nmnum ::= DEFAULT */ yytestcase(yyruleno==310); - /* (311) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==311); - /* (312) foreach_clause ::= */ yytestcase(yyruleno==312); - /* (313) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==313); - /* (314) trnm ::= nm */ yytestcase(yyruleno==314); - /* (315) tridxby ::= */ yytestcase(yyruleno==315); - /* (316) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==316); - /* (317) database_kw_opt ::= */ yytestcase(yyruleno==317); - /* (318) kwcolumn_opt ::= */ yytestcase(yyruleno==318); - /* (319) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==319); - /* (320) vtabarglist ::= vtabarg */ yytestcase(yyruleno==320); - /* (321) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==321); - /* (322) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==322); - /* (323) anylist ::= */ yytestcase(yyruleno==323); - /* (324) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==324); - /* (325) anylist ::= anylist ANY */ yytestcase(yyruleno==325); + /* (344) input ::= cmdlist */ yytestcase(yyruleno==344); + /* (345) cmdlist ::= cmdlist ecmd */ yytestcase(yyruleno==345); + /* (346) cmdlist ::= ecmd (OPTIMIZED OUT) */ assert(yyruleno!=346); + /* (347) ecmd ::= SEMI */ yytestcase(yyruleno==347); + /* (348) ecmd ::= cmdx SEMI */ yytestcase(yyruleno==348); + /* (349) ecmd ::= explain cmdx SEMI (NEVER REDUCES) */ assert(yyruleno!=349); + /* (350) trans_opt ::= */ yytestcase(yyruleno==350); + /* (351) trans_opt ::= TRANSACTION */ yytestcase(yyruleno==351); + /* (352) trans_opt ::= TRANSACTION nm */ yytestcase(yyruleno==352); + /* (353) savepoint_opt ::= SAVEPOINT */ yytestcase(yyruleno==353); + /* (354) savepoint_opt ::= */ yytestcase(yyruleno==354); + /* (355) cmd ::= create_table create_table_args */ yytestcase(yyruleno==355); + /* (356) table_option_set ::= table_option (OPTIMIZED OUT) */ assert(yyruleno!=356); + /* (357) columnlist ::= columnlist COMMA columnname carglist */ yytestcase(yyruleno==357); + /* (358) columnlist ::= columnname carglist */ yytestcase(yyruleno==358); + /* (359) nm ::= ID|INDEXED|JOIN_KW */ yytestcase(yyruleno==359); + /* (360) nm ::= STRING */ yytestcase(yyruleno==360); + /* (361) typetoken ::= typename */ yytestcase(yyruleno==361); + /* (362) typename ::= ID|STRING */ yytestcase(yyruleno==362); + /* (363) signed ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=363); + /* (364) signed ::= minus_num (OPTIMIZED OUT) */ assert(yyruleno!=364); + /* (365) carglist ::= carglist ccons */ yytestcase(yyruleno==365); + /* (366) carglist ::= */ yytestcase(yyruleno==366); + /* (367) ccons ::= NULL onconf */ yytestcase(yyruleno==367); + /* (368) ccons ::= GENERATED ALWAYS AS generated */ yytestcase(yyruleno==368); + /* (369) ccons ::= AS generated */ yytestcase(yyruleno==369); + /* (370) conslist_opt ::= COMMA conslist */ yytestcase(yyruleno==370); + /* (371) conslist ::= conslist tconscomma tcons */ yytestcase(yyruleno==371); + /* (372) conslist ::= tcons (OPTIMIZED OUT) */ assert(yyruleno!=372); + /* (373) tconscomma ::= */ yytestcase(yyruleno==373); + /* (374) defer_subclause_opt ::= defer_subclause (OPTIMIZED OUT) */ assert(yyruleno!=374); + /* (375) resolvetype ::= raisetype (OPTIMIZED OUT) */ assert(yyruleno!=375); + /* (376) selectnowith ::= oneselect (OPTIMIZED OUT) */ assert(yyruleno!=376); + /* (377) oneselect ::= values */ yytestcase(yyruleno==377); + /* (378) sclp ::= selcollist COMMA */ yytestcase(yyruleno==378); + /* (379) as ::= ID|STRING */ yytestcase(yyruleno==379); + /* (380) indexed_opt ::= indexed_by (OPTIMIZED OUT) */ assert(yyruleno!=380); + /* (381) returning ::= */ yytestcase(yyruleno==381); + /* (382) expr ::= term (OPTIMIZED OUT) */ assert(yyruleno!=382); + /* (383) likeop ::= LIKE_KW|MATCH */ yytestcase(yyruleno==383); + /* (384) case_operand ::= expr */ yytestcase(yyruleno==384); + /* (385) exprlist ::= nexprlist */ yytestcase(yyruleno==385); + /* (386) nmnum ::= plus_num (OPTIMIZED OUT) */ assert(yyruleno!=386); + /* (387) nmnum ::= nm (OPTIMIZED OUT) */ assert(yyruleno!=387); + /* (388) nmnum ::= ON */ yytestcase(yyruleno==388); + /* (389) nmnum ::= DELETE */ yytestcase(yyruleno==389); + /* (390) nmnum ::= DEFAULT */ yytestcase(yyruleno==390); + /* (391) plus_num ::= INTEGER|FLOAT */ yytestcase(yyruleno==391); + /* (392) foreach_clause ::= */ yytestcase(yyruleno==392); + /* (393) foreach_clause ::= FOR EACH ROW */ yytestcase(yyruleno==393); + /* (394) trnm ::= nm */ yytestcase(yyruleno==394); + /* (395) tridxby ::= */ yytestcase(yyruleno==395); + /* (396) database_kw_opt ::= DATABASE */ yytestcase(yyruleno==396); + /* (397) database_kw_opt ::= */ yytestcase(yyruleno==397); + /* (398) kwcolumn_opt ::= */ yytestcase(yyruleno==398); + /* (399) kwcolumn_opt ::= COLUMNKW */ yytestcase(yyruleno==399); + /* (400) vtabarglist ::= vtabarg */ yytestcase(yyruleno==400); + /* (401) vtabarglist ::= vtabarglist COMMA vtabarg */ yytestcase(yyruleno==401); + /* (402) vtabarg ::= vtabarg vtabargtoken */ yytestcase(yyruleno==402); + /* (403) anylist ::= */ yytestcase(yyruleno==403); + /* (404) anylist ::= anylist LP anylist RP */ yytestcase(yyruleno==404); + /* (405) anylist ::= anylist ANY */ yytestcase(yyruleno==405); + /* (406) with ::= */ yytestcase(yyruleno==406); + /* (407) windowdefn_list ::= windowdefn (OPTIMIZED OUT) */ assert(yyruleno!=407); + /* (408) window ::= frame_opt (OPTIMIZED OUT) */ assert(yyruleno!=408); break; /********** End reduce actions ************************************************/ }; - assert( yyrulenoYY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - yypParser->yyidx -= yysize - 1; - yymsp -= yysize-1; - yymsp->stateno = (YYACTIONTYPE)yyact; - yymsp->major = (YYCODETYPE)yygoto; - yyTraceShift(yypParser, yyact); - }else{ - assert( yyact == YY_ACCEPT_ACTION ); - yypParser->yyidx -= yysize; - yy_accept(yypParser); - } + assert( yyrulenoYY_MAX_SHIFT && yyact<=YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( yyact!=YY_ERROR_ACTION ); + + yymsp += yysize+1; + yypParser->yytos = yymsp; + yymsp->stateno = (YYACTIONTYPE)yyact; + yymsp->major = (YYCODETYPE)yygoto; + yyTraceShift(yypParser, yyact, "... then shift"); + return yyact; } /* @@ -134123,18 +179039,20 @@ static void yy_reduce( static void yy_parse_failed( yyParser *yypParser /* The parser */ ){ - sqlite3ParserARG_FETCH; + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sFail!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); + while( yypParser->yytos>yypParser->yystack ) yy_pop_parser_stack(yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE } #endif /* YYNOERRORRECOVERY */ @@ -134146,15 +179064,20 @@ static void yy_syntax_error( int yymajor, /* The major type of the error token */ sqlite3ParserTOKENTYPE yyminor /* The minor type of the error token */ ){ - sqlite3ParserARG_FETCH; + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH #define TOKEN yyminor /************ Begin %syntax_error code ****************************************/ UNUSED_PARAMETER(yymajor); /* Silence some compiler warnings */ - assert( TOKEN.z[0] ); /* The tokenizer always gives us a token */ - sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + if( TOKEN.z[0] ){ + sqlite3ErrorMsg(pParse, "near \"%T\": syntax error", &TOKEN); + }else{ + sqlite3ErrorMsg(pParse, "incomplete input"); + } /************ End %syntax_error code ******************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE } /* @@ -134163,18 +179086,23 @@ static void yy_syntax_error( static void yy_accept( yyParser *yypParser /* The parser */ ){ - sqlite3ParserARG_FETCH; + sqlite3ParserARG_FETCH + sqlite3ParserCTX_FETCH #ifndef NDEBUG if( yyTraceFILE ){ fprintf(yyTraceFILE,"%sAccept!\n",yyTracePrompt); } #endif - while( yypParser->yyidx>=0 ) yy_pop_parser_stack(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif + assert( yypParser->yytos==yypParser->yystack ); /* Here code is inserted which will be executed whenever the ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - sqlite3ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3ParserCTX_STORE } /* The main parser program. @@ -134203,59 +179131,88 @@ SQLITE_PRIVATE void sqlite3Parser( sqlite3ParserARG_PDECL /* Optional %extra_argument parameter */ ){ YYMINORTYPE yyminorunion; - unsigned int yyact; /* The parser action. */ + YYACTIONTYPE yyact; /* The parser action. */ #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) int yyendofinput; /* True if we are at the end of input */ #endif #ifdef YYERRORSYMBOL int yyerrorhit = 0; /* True if yymajor has invoked an error */ #endif - yyParser *yypParser; /* The parser */ + yyParser *yypParser = (yyParser*)yyp; /* The parser */ + sqlite3ParserCTX_FETCH + sqlite3ParserARG_STORE - /* (re)initialize the parser, if necessary */ - yypParser = (yyParser*)yyp; - if( yypParser->yyidx<0 ){ -#if YYSTACKDEPTH<=0 - if( yypParser->yystksz <=0 ){ - yyStackOverflow(yypParser); - return; - } -#endif - yypParser->yyidx = 0; -#ifndef YYNOERRORRECOVERY - yypParser->yyerrcnt = -1; -#endif - yypParser->yystack[0].stateno = 0; - yypParser->yystack[0].major = 0; -#ifndef NDEBUG - if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInitialize. Empty stack. State 0\n", - yyTracePrompt); - } -#endif - } + assert( yypParser->yytos!=0 ); #if !defined(YYERRORSYMBOL) && !defined(YYNOERRORRECOVERY) yyendofinput = (yymajor==0); #endif - sqlite3ParserARG_STORE; + yyact = yypParser->yytos->stateno; #ifndef NDEBUG if( yyTraceFILE ){ - fprintf(yyTraceFILE,"%sInput '%s'\n",yyTracePrompt,yyTokenName[yymajor]); + if( yyact < YY_MIN_REDUCE ){ + fprintf(yyTraceFILE,"%sInput '%s' in state %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact); + }else{ + fprintf(yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + yyTracePrompt,yyTokenName[yymajor],yyact-YY_MIN_REDUCE); + } } #endif - do{ - yyact = yy_find_shift_action(yypParser,(YYCODETYPE)yymajor); - if( yyact <= YY_MAX_SHIFTREDUCE ){ - if( yyact > YY_MAX_SHIFT ) yyact += YY_MIN_REDUCE - YY_MIN_SHIFTREDUCE; - yy_shift(yypParser,yyact,yymajor,yyminor); + while(1){ /* Exit by "break" */ + assert( yypParser->yytos>=yypParser->yystack ); + assert( yyact==yypParser->yytos->stateno ); + yyact = yy_find_shift_action((YYCODETYPE)yymajor,yyact); + if( yyact >= YY_MIN_REDUCE ){ + unsigned int yyruleno = yyact - YY_MIN_REDUCE; /* Reduce by this rule */ +#ifndef NDEBUG + assert( yyruleno<(int)(sizeof(yyRuleName)/sizeof(yyRuleName[0])) ); + if( yyTraceFILE ){ + int yysize = yyRuleInfoNRhs[yyruleno]; + if( yysize ){ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + yyTracePrompt, + yyruleno, yyRuleName[yyruleno], + yyrulenoyytos[yysize].stateno); + }else{ + fprintf(yyTraceFILE, "%sReduce %d [%s]%s.\n", + yyTracePrompt, yyruleno, yyRuleName[yyruleno], + yyrulenoyytos - yypParser->yystack)>yypParser->yyhwm ){ + yypParser->yyhwm++; + assert( yypParser->yyhwm == + (int)(yypParser->yytos - yypParser->yystack)); + } +#endif + if( yypParser->yytos>=yypParser->yystackEnd ){ + if( yyGrowStack(yypParser) ){ + yyStackOverflow(yypParser); + break; + } + } + } + yyact = yy_reduce(yypParser,yyruleno,yymajor,yyminor sqlite3ParserCTX_PARAM); + }else if( yyact <= YY_MAX_SHIFTREDUCE ){ + yy_shift(yypParser,yyact,(YYCODETYPE)yymajor,yyminor); #ifndef YYNOERRORRECOVERY yypParser->yyerrcnt--; #endif - yymajor = YYNOCODE; - }else if( yyact <= YY_MAX_REDUCE ){ - yy_reduce(yypParser,yyact-YY_MIN_REDUCE); + break; + }else if( yyact==YY_ACCEPT_ACTION ){ + yypParser->yytos--; + yy_accept(yypParser); + return; }else{ assert( yyact == YY_ERROR_ACTION ); yyminorunion.yy0 = yyminor; @@ -134270,7 +179227,7 @@ SQLITE_PRIVATE void sqlite3Parser( #ifdef YYERRORSYMBOL /* A syntax error has occurred. ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". + ** grammar defines an error token "ERROR". ** ** This is what we do if the grammar does define ERROR: ** @@ -134290,7 +179247,7 @@ SQLITE_PRIVATE void sqlite3Parser( if( yypParser->yyerrcnt<0 ){ yy_syntax_error(yypParser,yymajor,yyminor); } - yymx = yypParser->yystack[yypParser->yyidx].major; + yymx = yypParser->yytos->major; if( yymx==YYERRORSYMBOL || yyerrorhit ){ #ifndef NDEBUG if( yyTraceFILE ){ @@ -134301,18 +179258,18 @@ SQLITE_PRIVATE void sqlite3Parser( yy_destructor(yypParser, (YYCODETYPE)yymajor, &yyminorunion); yymajor = YYNOCODE; }else{ - while( - yypParser->yyidx >= 0 && - yymx != YYERRORSYMBOL && - (yyact = yy_find_reduce_action( - yypParser->yystack[yypParser->yyidx].stateno, - YYERRORSYMBOL)) >= YY_MIN_REDUCE - ){ + while( yypParser->yytos > yypParser->yystack ){ + yyact = yy_find_reduce_action(yypParser->yytos->stateno, + YYERRORSYMBOL); + if( yyact<=YY_MAX_SHIFTREDUCE ) break; yy_pop_parser_stack(yypParser); } - if( yypParser->yyidx < 0 || yymajor==0 ){ + if( yypParser->yytos <= yypParser->yystack || yymajor==0 ){ yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif yymajor = YYNOCODE; }else if( yymx!=YYERRORSYMBOL ){ yy_shift(yypParser,yyact,YYERRORSYMBOL,yyminor); @@ -134320,6 +179277,8 @@ SQLITE_PRIVATE void sqlite3Parser( } yypParser->yyerrcnt = 3; yyerrorhit = 1; + if( yymajor==YYNOCODE ) break; + yyact = yypParser->yytos->stateno; #elif defined(YYNOERRORRECOVERY) /* If the YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -134330,8 +179289,7 @@ SQLITE_PRIVATE void sqlite3Parser( */ yy_syntax_error(yypParser,yymajor, yyminor); yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); - yymajor = YYNOCODE; - + break; #else /* YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -134349,24 +179307,43 @@ SQLITE_PRIVATE void sqlite3Parser( yy_destructor(yypParser,(YYCODETYPE)yymajor,&yyminorunion); if( yyendofinput ){ yy_parse_failed(yypParser); +#ifndef YYNOERRORRECOVERY + yypParser->yyerrcnt = -1; +#endif } - yymajor = YYNOCODE; + break; #endif } - }while( yymajor!=YYNOCODE && yypParser->yyidx>=0 ); + } #ifndef NDEBUG if( yyTraceFILE ){ - int i; + yyStackEntry *i; + char cDiv = '['; fprintf(yyTraceFILE,"%sReturn. Stack=",yyTracePrompt); - for(i=1; i<=yypParser->yyidx; i++) - fprintf(yyTraceFILE,"%c%s", i==1 ? '[' : ' ', - yyTokenName[yypParser->yystack[i].major]); + for(i=&yypParser->yystack[1]; i<=yypParser->yytos; i++){ + fprintf(yyTraceFILE,"%c%s", cDiv, yyTokenName[i->major]); + cDiv = ' '; + } fprintf(yyTraceFILE,"]\n"); } #endif return; } +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +SQLITE_PRIVATE int sqlite3ParserFallback(int iToken){ +#ifdef YYFALLBACK + assert( iToken<(int)(sizeof(yyFallback)/sizeof(yyFallback[0])) ); + return yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} + /************** End of parse.c ***********************************************/ /************** Begin file tokenize.c ****************************************/ /* @@ -134398,8 +179375,8 @@ SQLITE_PRIVATE void sqlite3Parser( ** all of them need to be used within the switch. */ #define CC_X 0 /* The letter 'x', or start of BLOB literal */ -#define CC_KYWD 1 /* Alphabetics or '_'. Usable in a keyword */ -#define CC_ID 2 /* unicode characters usable in IDs */ +#define CC_KYWD0 1 /* First letter of a keyword */ +#define CC_KYWD 2 /* Alphabetics or '_'. Usable in a keyword */ #define CC_DIGIT 3 /* Digits */ #define CC_DOLLAR 4 /* '$' */ #define CC_VARALPHA 5 /* '@', '#', ':'. Alphabetic SQL variables */ @@ -134424,46 +179401,49 @@ SQLITE_PRIVATE void sqlite3Parser( #define CC_AND 24 /* '&' */ #define CC_TILDA 25 /* '~' */ #define CC_DOT 26 /* '.' */ -#define CC_ILLEGAL 27 /* Illegal character */ +#define CC_ID 27 /* unicode characters usable in IDs */ +#define CC_ILLEGAL 28 /* Illegal character */ +#define CC_NUL 29 /* 0x00 */ +#define CC_BOM 30 /* First byte of UTF8 BOM: 0xEF 0xBB 0xBF */ static const unsigned char aiClass[] = { #ifdef SQLITE_ASCII /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 7, 7, 27, 7, 7, 27, 27, -/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 0x */ 29, 28, 28, 28, 28, 28, 28, 28, 28, 7, 7, 28, 7, 7, 28, 28, +/* 1x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, /* 2x */ 7, 15, 8, 5, 4, 22, 24, 8, 17, 18, 21, 20, 23, 11, 26, 16, /* 3x */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 19, 12, 14, 13, 6, /* 4x */ 5, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 9, 27, 27, 27, 1, +/* 5x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 9, 28, 28, 28, 2, /* 6x */ 8, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, -/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 10, 27, 25, 27, -/* 8x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* 9x */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Ax */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Bx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Cx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Dx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Ex */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, -/* Fx */ 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2 +/* 7x */ 1, 1, 1, 1, 1, 1, 1, 1, 0, 2, 2, 28, 10, 28, 25, 28, +/* 8x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* 9x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Ax */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Cx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Dx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, +/* Ex */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 30, +/* Fx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27 #endif #ifdef SQLITE_EBCDIC /* x0 x1 x2 x3 x4 x5 x6 x7 x8 x9 xa xb xc xd xe xf */ -/* 0x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 7, 7, 27, 27, -/* 1x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 2x */ 27, 27, 27, 27, 27, 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 3x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, -/* 4x */ 7, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 12, 17, 20, 10, -/* 5x */ 24, 27, 27, 27, 27, 27, 27, 27, 27, 27, 15, 4, 21, 18, 19, 27, -/* 6x */ 11, 16, 27, 27, 27, 27, 27, 27, 27, 27, 27, 23, 22, 1, 13, 7, -/* 7x */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 8, 5, 5, 5, 8, 14, 8, -/* 8x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* 9x */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* 9x */ 25, 1, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, -/* Bx */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, 9, 27, 27, 27, 27, 27, -/* Cx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Dx */ 27, 1, 1, 1, 1, 1, 1, 1, 1, 1, 27, 27, 27, 27, 27, 27, -/* Ex */ 27, 27, 1, 1, 1, 1, 1, 0, 1, 1, 27, 27, 27, 27, 27, 27, -/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 27, 27, 27, 27, 27, 27, +/* 0x */ 29, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 7, 7, 28, 28, +/* 1x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 2x */ 28, 28, 28, 28, 28, 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 3x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, +/* 4x */ 7, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 26, 12, 17, 20, 10, +/* 5x */ 24, 28, 28, 28, 28, 28, 28, 28, 28, 28, 15, 4, 21, 18, 19, 28, +/* 6x */ 11, 16, 28, 28, 28, 28, 28, 28, 28, 28, 28, 23, 22, 2, 13, 6, +/* 7x */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 8, 5, 5, 5, 8, 14, 8, +/* 8x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* 9x */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* Ax */ 28, 25, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28, +/* Bx */ 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 9, 28, 28, 28, 28, 28, +/* Cx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* Dx */ 28, 1, 1, 1, 1, 1, 1, 1, 1, 1, 28, 28, 28, 28, 28, 28, +/* Ex */ 28, 28, 1, 1, 1, 1, 1, 0, 2, 2, 28, 28, 28, 28, 28, 28, +/* Fx */ 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 28, 28, 28, 28, 28, 28, #endif }; @@ -134472,7 +179452,7 @@ static const unsigned char aiClass[] = { ** lower-case ASCII equivalent. On ASCII machines, this is just ** an upper-to-lower case map. On EBCDIC machines we also need ** to adjust the encoding. The mapping is only valid for alphabetics -** which are the only characters for which this feature is used. +** which are the only characters for which this feature is used. ** ** Used by keywordhash.h */ @@ -134504,7 +179484,7 @@ const unsigned char ebcdicToAscii[] = { /* ** The sqlite3KeywordCode function looks up an identifier to determine if -** it is a keyword. If it is a keyword, the token code of that keyword is +** it is a keyword. If it is a keyword, the token code of that keyword is ** returned. If the input is not a keyword, TK_ID is returned. ** ** The implementation of this routine was generated by a program, @@ -134528,278 +179508,475 @@ const unsigned char ebcdicToAscii[] = { ** is substantially reduced. This is important for embedded applications ** on platforms with limited memory. */ -/* Hash score: 182 */ +/* Hash score: 231 */ +/* zKWText[] encodes 1007 bytes of keyword text in 667 bytes */ +/* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ +/* ABLEFTHENDEFERRABLELSEXCLUDELETEMPORARYISNULLSAVEPOINTERSECT */ +/* IESNOTNULLIKEXCEPTRANSACTIONATURALTERAISEXCLUSIVEXISTS */ +/* CONSTRAINTOFFSETRIGGERANGENERATEDETACHAVINGLOBEGINNEREFERENCES */ +/* UNIQUERYWITHOUTERELEASEATTACHBETWEENOTHINGROUPSCASCADEFAULT */ +/* CASECOLLATECREATECURRENT_DATEIMMEDIATEJOINSERTMATCHPLANALYZE */ +/* PRAGMATERIALIZEDEFERREDISTINCTUPDATEVALUESVIRTUALWAYSWHENWHERE */ +/* CURSIVEABORTAFTERENAMEANDROPARTITIONAUTOINCREMENTCASTCOLUMN */ +/* COMMITCONFLICTCROSSCURRENT_TIMESTAMPRECEDINGFAILASTFILTER */ +/* EPLACEFIRSTFOLLOWINGFROMFULLIMITIFORDERESTRICTOTHERSOVER */ +/* ETURNINGRIGHTROLLBACKROWSUNBOUNDEDUNIONUSINGVACUUMVIEWINDOWBY */ +/* INITIALLYPRIMARY */ +static const char zKWText[666] = { + 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', + 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', + 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', + 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', + 'E','R','R','A','B','L','E','L','S','E','X','C','L','U','D','E','L','E', + 'T','E','M','P','O','R','A','R','Y','I','S','N','U','L','L','S','A','V', + 'E','P','O','I','N','T','E','R','S','E','C','T','I','E','S','N','O','T', + 'N','U','L','L','I','K','E','X','C','E','P','T','R','A','N','S','A','C', + 'T','I','O','N','A','T','U','R','A','L','T','E','R','A','I','S','E','X', + 'C','L','U','S','I','V','E','X','I','S','T','S','C','O','N','S','T','R', + 'A','I','N','T','O','F','F','S','E','T','R','I','G','G','E','R','A','N', + 'G','E','N','E','R','A','T','E','D','E','T','A','C','H','A','V','I','N', + 'G','L','O','B','E','G','I','N','N','E','R','E','F','E','R','E','N','C', + 'E','S','U','N','I','Q','U','E','R','Y','W','I','T','H','O','U','T','E', + 'R','E','L','E','A','S','E','A','T','T','A','C','H','B','E','T','W','E', + 'E','N','O','T','H','I','N','G','R','O','U','P','S','C','A','S','C','A', + 'D','E','F','A','U','L','T','C','A','S','E','C','O','L','L','A','T','E', + 'C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A','T','E', + 'I','M','M','E','D','I','A','T','E','J','O','I','N','S','E','R','T','M', + 'A','T','C','H','P','L','A','N','A','L','Y','Z','E','P','R','A','G','M', + 'A','T','E','R','I','A','L','I','Z','E','D','E','F','E','R','R','E','D', + 'I','S','T','I','N','C','T','U','P','D','A','T','E','V','A','L','U','E', + 'S','V','I','R','T','U','A','L','W','A','Y','S','W','H','E','N','W','H', + 'E','R','E','C','U','R','S','I','V','E','A','B','O','R','T','A','F','T', + 'E','R','E','N','A','M','E','A','N','D','R','O','P','A','R','T','I','T', + 'I','O','N','A','U','T','O','I','N','C','R','E','M','E','N','T','C','A', + 'S','T','C','O','L','U','M','N','C','O','M','M','I','T','C','O','N','F', + 'L','I','C','T','C','R','O','S','S','C','U','R','R','E','N','T','_','T', + 'I','M','E','S','T','A','M','P','R','E','C','E','D','I','N','G','F','A', + 'I','L','A','S','T','F','I','L','T','E','R','E','P','L','A','C','E','F', + 'I','R','S','T','F','O','L','L','O','W','I','N','G','F','R','O','M','F', + 'U','L','L','I','M','I','T','I','F','O','R','D','E','R','E','S','T','R', + 'I','C','T','O','T','H','E','R','S','O','V','E','R','E','T','U','R','N', + 'I','N','G','R','I','G','H','T','R','O','L','L','B','A','C','K','R','O', + 'W','S','U','N','B','O','U','N','D','E','D','U','N','I','O','N','U','S', + 'I','N','G','V','A','C','U','U','M','V','I','E','W','I','N','D','O','W', + 'B','Y','I','N','I','T','I','A','L','L','Y','P','R','I','M','A','R','Y', +}; +/* aKWHash[i] is the hash value for the i-th keyword */ +static const unsigned char aKWHash[127] = { + 84, 92, 134, 82, 105, 29, 0, 0, 94, 0, 85, 72, 0, + 53, 35, 86, 15, 0, 42, 97, 54, 89, 135, 19, 0, 0, + 140, 0, 40, 129, 0, 22, 107, 0, 9, 0, 0, 123, 80, + 0, 78, 6, 0, 65, 103, 147, 0, 136, 115, 0, 0, 48, + 0, 90, 24, 0, 17, 0, 27, 70, 23, 26, 5, 60, 142, + 110, 122, 0, 73, 91, 71, 145, 61, 120, 74, 0, 49, 0, + 11, 41, 0, 113, 0, 0, 0, 109, 10, 111, 116, 125, 14, + 50, 124, 0, 100, 0, 18, 121, 144, 56, 130, 139, 88, 83, + 37, 30, 126, 0, 0, 108, 51, 131, 128, 0, 34, 0, 0, + 132, 0, 98, 38, 39, 0, 20, 45, 117, 93, +}; +/* aKWNext[] forms the hash collision chain. If aKWHash[i]==0 +** then the i-th keyword has no more hash collisions. Otherwise, +** the next keyword with the same hash is aKWHash[i]-1. */ +static const unsigned char aKWNext[148] = {0, + 0, 0, 0, 0, 4, 0, 43, 0, 0, 106, 114, 0, 0, + 0, 2, 0, 0, 143, 0, 0, 0, 13, 0, 0, 0, 0, + 141, 0, 0, 119, 52, 0, 0, 137, 12, 0, 0, 62, 0, + 138, 0, 133, 0, 0, 36, 0, 0, 28, 77, 0, 0, 0, + 0, 59, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 69, 0, 0, 0, 0, 0, 146, 3, 0, 58, 0, 1, + 75, 0, 0, 0, 31, 0, 0, 0, 0, 0, 127, 0, 104, + 0, 64, 66, 63, 0, 0, 0, 0, 0, 46, 0, 16, 8, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 81, 101, 0, + 112, 21, 7, 67, 0, 79, 96, 118, 0, 0, 68, 0, 0, + 99, 44, 0, 55, 0, 76, 0, 95, 32, 33, 57, 25, 0, + 102, 0, 0, 87, +}; +/* aKWLen[i] is the length (in bytes) of the i-th keyword */ +static const unsigned char aKWLen[148] = {0, + 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, + 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 7, + 6, 9, 4, 2, 6, 5, 9, 9, 4, 7, 3, 2, 4, + 4, 6, 11, 6, 2, 7, 5, 5, 9, 6, 10, 4, 6, + 2, 3, 7, 5, 9, 6, 6, 4, 5, 5, 10, 6, 5, + 7, 4, 5, 7, 6, 7, 7, 6, 5, 7, 3, 7, 4, + 7, 6, 12, 9, 4, 6, 5, 4, 7, 6, 12, 8, 8, + 2, 6, 6, 7, 6, 4, 5, 9, 5, 5, 6, 3, 4, + 9, 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 9, + 4, 4, 6, 7, 5, 9, 4, 4, 5, 2, 5, 8, 6, + 4, 9, 5, 8, 4, 3, 9, 5, 5, 6, 4, 6, 2, + 2, 9, 3, 7, +}; +/* aKWOffset[i] is the index into zKWText[] of the start of +** the text for the i-th keyword. */ +static const unsigned short int aKWOffset[148] = {0, + 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, + 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, + 86, 90, 90, 94, 99, 101, 105, 111, 119, 123, 123, 123, 126, + 129, 132, 137, 142, 146, 147, 152, 156, 160, 168, 174, 181, 184, + 184, 187, 189, 195, 198, 206, 211, 216, 219, 222, 226, 236, 239, + 244, 244, 248, 252, 259, 265, 271, 277, 277, 283, 284, 288, 295, + 299, 306, 312, 324, 333, 335, 341, 346, 348, 355, 359, 370, 377, + 378, 385, 391, 397, 402, 408, 412, 415, 424, 429, 433, 439, 441, + 444, 453, 455, 457, 466, 470, 476, 482, 490, 495, 495, 495, 511, + 520, 523, 527, 532, 539, 544, 553, 557, 560, 565, 567, 571, 579, + 585, 588, 597, 602, 610, 610, 614, 623, 628, 633, 639, 642, 645, + 648, 650, 655, 659, +}; +/* aKWCode[i] is the parser symbol code for the i-th keyword */ +static const unsigned char aKWCode[148] = {0, + TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, + TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, + TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, + TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, + TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, + TK_EXCLUDE, TK_DELETE, TK_TEMP, TK_TEMP, TK_OR, + TK_ISNULL, TK_NULLS, TK_SAVEPOINT, TK_INTERSECT, TK_TIES, + TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, + TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, + TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_CONSTRAINT, + TK_INTO, TK_OFFSET, TK_OF, TK_SET, TK_TRIGGER, + TK_RANGE, TK_GENERATED, TK_DETACH, TK_HAVING, TK_LIKE_KW, + TK_BEGIN, TK_JOIN_KW, TK_REFERENCES, TK_UNIQUE, TK_QUERY, + TK_WITHOUT, TK_WITH, TK_JOIN_KW, TK_RELEASE, TK_ATTACH, + TK_BETWEEN, TK_NOTHING, TK_GROUPS, TK_GROUP, TK_CASCADE, + TK_ASC, TK_DEFAULT, TK_CASE, TK_COLLATE, TK_CREATE, + TK_CTIME_KW, TK_IMMEDIATE, TK_JOIN, TK_INSERT, TK_MATCH, + TK_PLAN, TK_ANALYZE, TK_PRAGMA, TK_MATERIALIZED, TK_DEFERRED, + TK_DISTINCT, TK_IS, TK_UPDATE, TK_VALUES, TK_VIRTUAL, + TK_ALWAYS, TK_WHEN, TK_WHERE, TK_RECURSIVE, TK_ABORT, + TK_AFTER, TK_RENAME, TK_AND, TK_DROP, TK_PARTITION, + TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, TK_COLUMNKW, + TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, TK_CTIME_KW, + TK_CURRENT, TK_PRECEDING, TK_FAIL, TK_LAST, TK_FILTER, + TK_REPLACE, TK_FIRST, TK_FOLLOWING, TK_FROM, TK_JOIN_KW, + TK_LIMIT, TK_IF, TK_ORDER, TK_RESTRICT, TK_OTHERS, + TK_OVER, TK_RETURNING, TK_JOIN_KW, TK_ROLLBACK, TK_ROWS, + TK_ROW, TK_UNBOUNDED, TK_UNION, TK_USING, TK_VACUUM, + TK_VIEW, TK_WINDOW, TK_DO, TK_BY, TK_INITIALLY, + TK_ALL, TK_PRIMARY, +}; +/* Hash table decoded: +** 0: INSERT +** 1: IS +** 2: ROLLBACK TRIGGER +** 3: IMMEDIATE +** 4: PARTITION +** 5: TEMP +** 6: +** 7: +** 8: VALUES WITHOUT +** 9: +** 10: MATCH +** 11: NOTHING +** 12: +** 13: OF +** 14: TIES IGNORE +** 15: PLAN +** 16: INSTEAD INDEXED +** 17: +** 18: TRANSACTION RIGHT +** 19: WHEN +** 20: SET HAVING +** 21: MATERIALIZED IF +** 22: ROWS +** 23: SELECT +** 24: +** 25: +** 26: VACUUM SAVEPOINT +** 27: +** 28: LIKE UNION VIRTUAL REFERENCES +** 29: RESTRICT +** 30: +** 31: THEN REGEXP +** 32: TO +** 33: +** 34: BEFORE +** 35: +** 36: +** 37: FOLLOWING COLLATE CASCADE +** 38: CREATE +** 39: +** 40: CASE REINDEX +** 41: EACH +** 42: +** 43: QUERY +** 44: AND ADD +** 45: PRIMARY ANALYZE +** 46: +** 47: ROW ASC DETACH +** 48: CURRENT_TIME CURRENT_DATE +** 49: +** 50: +** 51: EXCLUSIVE TEMPORARY +** 52: +** 53: DEFERRED +** 54: DEFERRABLE +** 55: +** 56: DATABASE +** 57: +** 58: DELETE VIEW GENERATED +** 59: ATTACH +** 60: END +** 61: EXCLUDE +** 62: ESCAPE DESC +** 63: GLOB +** 64: WINDOW ELSE +** 65: COLUMN +** 66: FIRST +** 67: +** 68: GROUPS ALL +** 69: DISTINCT DROP KEY +** 70: BETWEEN +** 71: INITIALLY +** 72: BEGIN +** 73: FILTER CHECK ACTION +** 74: GROUP INDEX +** 75: +** 76: EXISTS DEFAULT +** 77: +** 78: FOR CURRENT_TIMESTAMP +** 79: EXCEPT +** 80: +** 81: CROSS +** 82: +** 83: +** 84: +** 85: CAST +** 86: FOREIGN AUTOINCREMENT +** 87: COMMIT +** 88: CURRENT AFTER ALTER +** 89: FULL FAIL CONFLICT +** 90: EXPLAIN +** 91: CONSTRAINT +** 92: FROM ALWAYS +** 93: +** 94: ABORT +** 95: +** 96: AS DO +** 97: REPLACE WITH RELEASE +** 98: BY RENAME +** 99: RANGE RAISE +** 100: OTHERS +** 101: USING NULLS +** 102: PRAGMA +** 103: JOIN ISNULL OFFSET +** 104: NOT +** 105: OR LAST LEFT +** 106: LIMIT +** 107: +** 108: +** 109: IN +** 110: INTO +** 111: OVER RECURSIVE +** 112: ORDER OUTER +** 113: +** 114: INTERSECT UNBOUNDED +** 115: +** 116: +** 117: RETURNING ON +** 118: +** 119: WHERE +** 120: NO INNER +** 121: NULL +** 122: +** 123: TABLE +** 124: NATURAL NOTNULL +** 125: PRECEDING +** 126: UPDATE UNIQUE +*/ +/* Check to see if z[0..n-1] is a keyword. If it is, write the +** parser symbol code for that keyword into *pType. Always +** return the integer n (the length of the token). */ static int keywordCode(const char *z, int n, int *pType){ - /* zText[] encodes 834 bytes of keywords in 554 bytes */ - /* REINDEXEDESCAPEACHECKEYBEFOREIGNOREGEXPLAINSTEADDATABASELECT */ - /* ABLEFTHENDEFERRABLELSEXCEPTRANSACTIONATURALTERAISEXCLUSIVE */ - /* XISTSAVEPOINTERSECTRIGGEREFERENCESCONSTRAINTOFFSETEMPORARY */ - /* UNIQUERYWITHOUTERELEASEATTACHAVINGROUPDATEBEGINNERECURSIVE */ - /* BETWEENOTNULLIKECASCADELETECASECOLLATECREATECURRENT_DATEDETACH */ - /* IMMEDIATEJOINSERTMATCHPLANALYZEPRAGMABORTVALUESVIRTUALIMITWHEN */ - /* WHERENAMEAFTEREPLACEANDEFAULTAUTOINCREMENTCASTCOLUMNCOMMIT */ - /* CONFLICTCROSSCURRENT_TIMESTAMPRIMARYDEFERREDISTINCTDROPFAIL */ - /* FROMFULLGLOBYIFISNULLORDERESTRICTRIGHTROLLBACKROWUNIONUSING */ - /* VACUUMVIEWINITIALLY */ - static const char zText[553] = { - 'R','E','I','N','D','E','X','E','D','E','S','C','A','P','E','A','C','H', - 'E','C','K','E','Y','B','E','F','O','R','E','I','G','N','O','R','E','G', - 'E','X','P','L','A','I','N','S','T','E','A','D','D','A','T','A','B','A', - 'S','E','L','E','C','T','A','B','L','E','F','T','H','E','N','D','E','F', - 'E','R','R','A','B','L','E','L','S','E','X','C','E','P','T','R','A','N', - 'S','A','C','T','I','O','N','A','T','U','R','A','L','T','E','R','A','I', - 'S','E','X','C','L','U','S','I','V','E','X','I','S','T','S','A','V','E', - 'P','O','I','N','T','E','R','S','E','C','T','R','I','G','G','E','R','E', - 'F','E','R','E','N','C','E','S','C','O','N','S','T','R','A','I','N','T', - 'O','F','F','S','E','T','E','M','P','O','R','A','R','Y','U','N','I','Q', - 'U','E','R','Y','W','I','T','H','O','U','T','E','R','E','L','E','A','S', - 'E','A','T','T','A','C','H','A','V','I','N','G','R','O','U','P','D','A', - 'T','E','B','E','G','I','N','N','E','R','E','C','U','R','S','I','V','E', - 'B','E','T','W','E','E','N','O','T','N','U','L','L','I','K','E','C','A', - 'S','C','A','D','E','L','E','T','E','C','A','S','E','C','O','L','L','A', - 'T','E','C','R','E','A','T','E','C','U','R','R','E','N','T','_','D','A', - 'T','E','D','E','T','A','C','H','I','M','M','E','D','I','A','T','E','J', - 'O','I','N','S','E','R','T','M','A','T','C','H','P','L','A','N','A','L', - 'Y','Z','E','P','R','A','G','M','A','B','O','R','T','V','A','L','U','E', - 'S','V','I','R','T','U','A','L','I','M','I','T','W','H','E','N','W','H', - 'E','R','E','N','A','M','E','A','F','T','E','R','E','P','L','A','C','E', - 'A','N','D','E','F','A','U','L','T','A','U','T','O','I','N','C','R','E', - 'M','E','N','T','C','A','S','T','C','O','L','U','M','N','C','O','M','M', - 'I','T','C','O','N','F','L','I','C','T','C','R','O','S','S','C','U','R', - 'R','E','N','T','_','T','I','M','E','S','T','A','M','P','R','I','M','A', - 'R','Y','D','E','F','E','R','R','E','D','I','S','T','I','N','C','T','D', - 'R','O','P','F','A','I','L','F','R','O','M','F','U','L','L','G','L','O', - 'B','Y','I','F','I','S','N','U','L','L','O','R','D','E','R','E','S','T', - 'R','I','C','T','R','I','G','H','T','R','O','L','L','B','A','C','K','R', - 'O','W','U','N','I','O','N','U','S','I','N','G','V','A','C','U','U','M', - 'V','I','E','W','I','N','I','T','I','A','L','L','Y', - }; - static const unsigned char aHash[127] = { - 76, 105, 117, 74, 0, 45, 0, 0, 82, 0, 77, 0, 0, - 42, 12, 78, 15, 0, 116, 85, 54, 112, 0, 19, 0, 0, - 121, 0, 119, 115, 0, 22, 93, 0, 9, 0, 0, 70, 71, - 0, 69, 6, 0, 48, 90, 102, 0, 118, 101, 0, 0, 44, - 0, 103, 24, 0, 17, 0, 122, 53, 23, 0, 5, 110, 25, - 96, 0, 0, 124, 106, 60, 123, 57, 28, 55, 0, 91, 0, - 100, 26, 0, 99, 0, 0, 0, 95, 92, 97, 88, 109, 14, - 39, 108, 0, 81, 0, 18, 89, 111, 32, 0, 120, 80, 113, - 62, 46, 84, 0, 0, 94, 40, 59, 114, 0, 36, 0, 0, - 29, 0, 86, 63, 64, 0, 20, 61, 0, 56, - }; - static const unsigned char aNext[124] = { - 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 2, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, - 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 33, 0, 21, 0, 0, 0, 0, 0, 50, - 0, 43, 3, 47, 0, 0, 0, 0, 30, 0, 58, 0, 38, - 0, 0, 0, 1, 66, 0, 0, 67, 0, 41, 0, 0, 0, - 0, 0, 0, 49, 65, 0, 0, 0, 0, 31, 52, 16, 34, - 10, 0, 0, 0, 0, 0, 0, 0, 11, 72, 79, 0, 8, - 0, 104, 98, 0, 107, 0, 87, 0, 75, 51, 0, 27, 37, - 73, 83, 0, 35, 68, 0, 0, - }; - static const unsigned char aLen[124] = { - 7, 7, 5, 4, 6, 4, 5, 3, 6, 7, 3, 6, 6, - 7, 7, 3, 8, 2, 6, 5, 4, 4, 3, 10, 4, 6, - 11, 6, 2, 7, 5, 5, 9, 6, 9, 9, 7, 10, 10, - 4, 6, 2, 3, 9, 4, 2, 6, 5, 7, 4, 5, 7, - 6, 6, 5, 6, 5, 5, 9, 7, 7, 3, 2, 4, 4, - 7, 3, 6, 4, 7, 6, 12, 6, 9, 4, 6, 5, 4, - 7, 6, 5, 6, 7, 5, 4, 5, 6, 5, 7, 3, 7, - 13, 2, 2, 4, 6, 6, 8, 5, 17, 12, 7, 8, 8, - 2, 4, 4, 4, 4, 4, 2, 2, 6, 5, 8, 5, 8, - 3, 5, 5, 6, 4, 9, 3, - }; - static const unsigned short int aOffset[124] = { - 0, 2, 2, 8, 9, 14, 16, 20, 23, 25, 25, 29, 33, - 36, 41, 46, 48, 53, 54, 59, 62, 65, 67, 69, 78, 81, - 86, 91, 95, 96, 101, 105, 109, 117, 122, 128, 136, 142, 152, - 159, 162, 162, 165, 167, 167, 171, 176, 179, 184, 184, 188, 192, - 199, 204, 209, 212, 218, 221, 225, 234, 240, 240, 240, 243, 246, - 250, 251, 255, 261, 265, 272, 278, 290, 296, 305, 307, 313, 318, - 320, 327, 332, 337, 343, 349, 354, 358, 361, 367, 371, 378, 380, - 387, 389, 391, 400, 404, 410, 416, 424, 429, 429, 445, 452, 459, - 460, 467, 471, 475, 479, 483, 486, 488, 490, 496, 500, 508, 513, - 521, 524, 529, 534, 540, 544, 549, - }; - static const unsigned char aCode[124] = { - TK_REINDEX, TK_INDEXED, TK_INDEX, TK_DESC, TK_ESCAPE, - TK_EACH, TK_CHECK, TK_KEY, TK_BEFORE, TK_FOREIGN, - TK_FOR, TK_IGNORE, TK_LIKE_KW, TK_EXPLAIN, TK_INSTEAD, - TK_ADD, TK_DATABASE, TK_AS, TK_SELECT, TK_TABLE, - TK_JOIN_KW, TK_THEN, TK_END, TK_DEFERRABLE, TK_ELSE, - TK_EXCEPT, TK_TRANSACTION,TK_ACTION, TK_ON, TK_JOIN_KW, - TK_ALTER, TK_RAISE, TK_EXCLUSIVE, TK_EXISTS, TK_SAVEPOINT, - TK_INTERSECT, TK_TRIGGER, TK_REFERENCES, TK_CONSTRAINT, TK_INTO, - TK_OFFSET, TK_OF, TK_SET, TK_TEMP, TK_TEMP, - TK_OR, TK_UNIQUE, TK_QUERY, TK_WITHOUT, TK_WITH, - TK_JOIN_KW, TK_RELEASE, TK_ATTACH, TK_HAVING, TK_GROUP, - TK_UPDATE, TK_BEGIN, TK_JOIN_KW, TK_RECURSIVE, TK_BETWEEN, - TK_NOTNULL, TK_NOT, TK_NO, TK_NULL, TK_LIKE_KW, - TK_CASCADE, TK_ASC, TK_DELETE, TK_CASE, TK_COLLATE, - TK_CREATE, TK_CTIME_KW, TK_DETACH, TK_IMMEDIATE, TK_JOIN, - TK_INSERT, TK_MATCH, TK_PLAN, TK_ANALYZE, TK_PRAGMA, - TK_ABORT, TK_VALUES, TK_VIRTUAL, TK_LIMIT, TK_WHEN, - TK_WHERE, TK_RENAME, TK_AFTER, TK_REPLACE, TK_AND, - TK_DEFAULT, TK_AUTOINCR, TK_TO, TK_IN, TK_CAST, - TK_COLUMNKW, TK_COMMIT, TK_CONFLICT, TK_JOIN_KW, TK_CTIME_KW, - TK_CTIME_KW, TK_PRIMARY, TK_DEFERRED, TK_DISTINCT, TK_IS, - TK_DROP, TK_FAIL, TK_FROM, TK_JOIN_KW, TK_LIKE_KW, - TK_BY, TK_IF, TK_ISNULL, TK_ORDER, TK_RESTRICT, - TK_JOIN_KW, TK_ROLLBACK, TK_ROW, TK_UNION, TK_USING, - TK_VACUUM, TK_VIEW, TK_INITIALLY, TK_ALL, - }; int i, j; const char *zKW; - if( n>=2 ){ - i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n) % 127; - for(i=((int)aHash[i])-1; i>=0; i=((int)aNext[i])-1){ - if( aLen[i]!=n ) continue; - j = 0; - zKW = &zText[aOffset[i]]; + assert( n>=2 ); + i = ((charMap(z[0])*4) ^ (charMap(z[n-1])*3) ^ n*1) % 127; + for(i=(int)aKWHash[i]; i>0; i=aKWNext[i]){ + if( aKWLen[i]!=n ) continue; + zKW = &zKWText[aKWOffset[i]]; #ifdef SQLITE_ASCII - while( j=2 ) keywordCode((char*)z, n, &id); return id; } -#define SQLITE_N_KEYWORD 124 +#define SQLITE_N_KEYWORD 147 +SQLITE_API int sqlite3_keyword_name(int i,const char **pzName,int *pnName){ + if( i<0 || i>=SQLITE_N_KEYWORD ) return SQLITE_ERROR; + i++; + *pzName = zKWText + aKWOffset[i]; + *pnName = aKWLen[i]; + return SQLITE_OK; +} +SQLITE_API int sqlite3_keyword_count(void){ return SQLITE_N_KEYWORD; } +SQLITE_API int sqlite3_keyword_check(const char *zName, int nName){ + return TK_ID!=sqlite3KeywordCode((const u8*)zName, nName); +} /************** End of keywordhash.h *****************************************/ /************** Continuing where we left off in tokenize.c *******************/ @@ -134810,14 +179987,14 @@ SQLITE_PRIVATE int sqlite3KeywordCode(const unsigned char *z, int n){ ** IdChar(X) will be true. Otherwise it is false. ** ** For ASCII, any character with the high-order bit set is -** allowed in an identifier. For 7-bit characters, +** allowed in an identifier. For 7-bit characters, ** sqlite3IsIdChar[X] must be 1. ** ** For EBCDIC, the rules are more complex but have the same ** end result. ** ** Ticket #1066. the SQL standard does not allow '$' in the -** middle of identifiers. But many SQL implementations do. +** middle of identifiers. But many SQL implementations do. ** SQLite will allow '$' in identifiers for compatibility. ** But the feature is undocumented. */ @@ -134843,14 +180020,88 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[] = { #define IdChar(C) (((c=C)>=0x42 && sqlite3IsEbcdicIdChar[c-0x40])) #endif -/* Make the IdChar function accessible from ctime.c */ -#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* Make the IdChar function accessible from ctime.c and alter.c */ SQLITE_PRIVATE int sqlite3IsIdChar(u8 c){ return IdChar(c); } -#endif +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** Return the id of the next token in string (*pz). Before returning, set +** (*pz) to point to the byte following the parsed token. +*/ +static int getToken(const unsigned char **pz){ + const unsigned char *z = *pz; + int t; /* Token type to return */ + do { + z += sqlite3GetToken(z, &t); + }while( t==TK_SPACE ); + if( t==TK_ID + || t==TK_STRING + || t==TK_JOIN_KW + || t==TK_WINDOW + || t==TK_OVER + || sqlite3ParserFallback(t)==TK_ID + ){ + t = TK_ID; + } + *pz = z; + return t; +} /* -** Return the length (in bytes) of the token that begins at z[0]. +** The following three functions are called immediately after the tokenizer +** reads the keywords WINDOW, OVER and FILTER, respectively, to determine +** whether the token should be treated as a keyword or an SQL identifier. +** This cannot be handled by the usual lemon %fallback method, due to +** the ambiguity in some constructions. e.g. +** +** SELECT sum(x) OVER ... +** +** In the above, "OVER" might be a keyword, or it might be an alias for the +** sum(x) expression. If a "%fallback ID OVER" directive were added to +** grammar, then SQLite would always treat "OVER" as an alias, making it +** impossible to call a window-function without a FILTER clause. +** +** WINDOW is treated as a keyword if: +** +** * the following token is an identifier, or a keyword that can fallback +** to being an identifier, and +** * the token after than one is TK_AS. +** +** OVER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is either TK_LP or an identifier. +** +** FILTER is a keyword if: +** +** * the previous token was TK_RP, and +** * the next token is TK_LP. +*/ +static int analyzeWindowKeyword(const unsigned char *z){ + int t; + t = getToken(&z); + if( t!=TK_ID ) return TK_ID; + t = getToken(&z); + if( t!=TK_AS ) return TK_ID; + return TK_WINDOW; +} +static int analyzeOverKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP ){ + int t = getToken(&z); + if( t==TK_LP || t==TK_ID ) return TK_OVER; + } + return TK_ID; +} +static int analyzeFilterKeyword(const unsigned char *z, int lastToken){ + if( lastToken==TK_RP && getToken(&z)==TK_LP ){ + return TK_FILTER; + } + return TK_ID; +} +#endif /* SQLITE_OMIT_WINDOWFUNC */ + +/* +** Return the length (in bytes) of the token that begins at z[0]. ** Store the token type in *tokenType before returning. */ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ @@ -134873,6 +180124,9 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ for(i=2; (c=z[i])!=0 && c!='\n'; i++){} *tokenType = TK_SPACE; /* IMP: R-22934-25134 */ return i; + }else if( z[1]=='>' ){ + *tokenType = TK_PTR; + return 2 + (z[2]=='>'); } *tokenType = TK_MINUS; return 1; @@ -135007,36 +180261,68 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } /* If the next character is a digit, this is a floating point ** number that begins with ".". Fall thru into the next case */ + /* no break */ deliberate_fall_through } case CC_DIGIT: { testcase( z[0]=='0' ); testcase( z[0]=='1' ); testcase( z[0]=='2' ); testcase( z[0]=='3' ); testcase( z[0]=='4' ); testcase( z[0]=='5' ); testcase( z[0]=='6' ); testcase( z[0]=='7' ); testcase( z[0]=='8' ); - testcase( z[0]=='9' ); + testcase( z[0]=='9' ); testcase( z[0]=='.' ); *tokenType = TK_INTEGER; #ifndef SQLITE_OMIT_HEX_INTEGER if( z[0]=='0' && (z[1]=='x' || z[1]=='X') && sqlite3Isxdigit(z[2]) ){ - for(i=3; sqlite3Isxdigit(z[i]); i++){} - return i; - } + for(i=3; 1; i++){ + if( sqlite3Isxdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + }else #endif - for(i=0; sqlite3Isdigit(z[i]); i++){} + { + for(i=0; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } #ifndef SQLITE_OMIT_FLOATING_POINT - if( z[i]=='.' ){ - i++; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } - if( (z[i]=='e' || z[i]=='E') && - ( sqlite3Isdigit(z[i+1]) - || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) - ) - ){ - i += 2; - while( sqlite3Isdigit(z[i]) ){ i++; } - *tokenType = TK_FLOAT; - } + if( z[i]=='.' ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i++; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } + if( (z[i]=='e' || z[i]=='E') && + ( sqlite3Isdigit(z[i+1]) + || ((z[i+1]=='+' || z[i+1]=='-') && sqlite3Isdigit(z[i+2])) + ) + ){ + if( *tokenType==TK_INTEGER ) *tokenType = TK_FLOAT; + for(i+=2; 1; i++){ + if( sqlite3Isdigit(z[i])==0 ){ + if( z[i]==SQLITE_DIGIT_SEPARATOR ){ + *tokenType = TK_QNUMBER; + }else{ + break; + } + } + } + } #endif + } while( IdChar(z[i]) ){ *tokenType = TK_ILLEGAL; i++; @@ -135083,8 +180369,9 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ if( n==0 ) *tokenType = TK_ILLEGAL; return i; } - case CC_KYWD: { - for(i=1; aiClass[z[i]]<=CC_KYWD; i++){} + case CC_KYWD0: { + if( aiClass[z[1]]>CC_KYWD ){ i = 1; break; } + for(i=2; aiClass[z[i]]<=CC_KYWD; i++){} if( IdChar(z[i]) ){ /* This token started out using characters that can appear in keywords, ** but z[i] is a character not allowed within keywords, so this must @@ -135111,11 +180398,25 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ #endif /* If it is not a BLOB literal, then it must be an ID, since no ** SQL keywords start with the letter 'x'. Fall through */ + /* no break */ deliberate_fall_through } + case CC_KYWD: case CC_ID: { i = 1; break; } + case CC_BOM: { + if( z[1]==0xbb && z[2]==0xbf ){ + *tokenType = TK_SPACE; + return 3; + } + i = 1; + break; + } + case CC_NUL: { + *tokenType = TK_ILLEGAL; + return 0; + } default: { *tokenType = TK_ILLEGAL; return 1; @@ -135127,78 +180428,121 @@ SQLITE_PRIVATE int sqlite3GetToken(const unsigned char *z, int *tokenType){ } /* -** Run the parser on the given SQL string. The parser structure is -** passed in. An SQLITE_ status code is returned. If an error occurs -** then an and attempt is made to write an error message into -** memory obtained from sqlite3_malloc() and to make *pzErrMsg point to that -** error message. +** Run the parser on the given SQL string. */ -SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzErrMsg){ +SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql){ int nErr = 0; /* Number of errors encountered */ - int i; /* Loop counter */ void *pEngine; /* The LEMON-generated LALR(1) parser */ + int n = 0; /* Length of the next token token */ int tokenType; /* type of the next token */ int lastTokenParsed = -1; /* type of the previous token */ sqlite3 *db = pParse->db; /* The database connection */ int mxSqlLen; /* Max length of an SQL string */ + Parse *pParentParse = 0; /* Outer parse context, if any */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + yyParser sEngine; /* Space to hold the Lemon-generated Parser object */ +#endif + VVA_ONLY( u8 startedWithOom = db->mallocFailed ); assert( zSql!=0 ); mxSqlLen = db->aLimit[SQLITE_LIMIT_SQL_LENGTH]; if( db->nVdbeActive==0 ){ - db->u1.isInterrupted = 0; + AtomicStore(&db->u1.isInterrupted, 0); } pParse->rc = SQLITE_OK; pParse->zTail = zSql; - i = 0; - assert( pzErrMsg!=0 ); - /* sqlite3ParserTrace(stdout, "parser: "); */ - pEngine = sqlite3ParserAlloc(sqlite3Malloc); +#ifdef SQLITE_DEBUG + if( db->flags & SQLITE_ParserTrace ){ + printf("parser: [[[%s]]]\n", zSql); + sqlite3ParserTrace(stdout, "parser: "); + }else{ + sqlite3ParserTrace(0, 0); + } +#endif +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + pEngine = &sEngine; + sqlite3ParserInit(pEngine, pParse); +#else + pEngine = sqlite3ParserAlloc(sqlite3Malloc, pParse); if( pEngine==0 ){ sqlite3OomFault(db); return SQLITE_NOMEM_BKPT; } +#endif assert( pParse->pNewTable==0 ); assert( pParse->pNewTrigger==0 ); assert( pParse->nVar==0 ); - assert( pParse->nzVar==0 ); - assert( pParse->azVar==0 ); - while( zSql[i]!=0 ){ - assert( i>=0 ); - pParse->sLastToken.z = &zSql[i]; - pParse->sLastToken.n = sqlite3GetToken((unsigned char*)&zSql[i],&tokenType); - i += pParse->sLastToken.n; - if( i>mxSqlLen ){ + assert( pParse->pVList==0 ); + pParentParse = db->pParse; + db->pParse = pParse; + while( 1 ){ + n = sqlite3GetToken((u8*)zSql, &tokenType); + mxSqlLen -= n; + if( mxSqlLen<0 ){ pParse->rc = SQLITE_TOOBIG; + pParse->nErr++; break; } +#ifndef SQLITE_OMIT_WINDOWFUNC + if( tokenType>=TK_WINDOW ){ + assert( tokenType==TK_SPACE || tokenType==TK_OVER || tokenType==TK_FILTER + || tokenType==TK_ILLEGAL || tokenType==TK_WINDOW + || tokenType==TK_QNUMBER + ); +#else if( tokenType>=TK_SPACE ){ - assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL ); - if( db->u1.isInterrupted ){ + assert( tokenType==TK_SPACE || tokenType==TK_ILLEGAL + || tokenType==TK_QNUMBER + ); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + if( AtomicLoad(&db->u1.isInterrupted) ){ pParse->rc = SQLITE_INTERRUPT; + pParse->nErr++; break; } - if( tokenType==TK_ILLEGAL ){ - sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", - &pParse->sLastToken); + if( tokenType==TK_SPACE ){ + zSql += n; + continue; + } + if( zSql[0]==0 ){ + /* Upon reaching the end of input, call the parser two more times + ** with tokens TK_SEMI and 0, in that order. */ + if( lastTokenParsed==TK_SEMI ){ + tokenType = 0; + }else if( lastTokenParsed==0 ){ + break; + }else{ + tokenType = TK_SEMI; + } + n = 0; +#ifndef SQLITE_OMIT_WINDOWFUNC + }else if( tokenType==TK_WINDOW ){ + assert( n==6 ); + tokenType = analyzeWindowKeyword((const u8*)&zSql[6]); + }else if( tokenType==TK_OVER ){ + assert( n==4 ); + tokenType = analyzeOverKeyword((const u8*)&zSql[4], lastTokenParsed); + }else if( tokenType==TK_FILTER ){ + assert( n==6 ); + tokenType = analyzeFilterKeyword((const u8*)&zSql[6], lastTokenParsed); +#endif /* SQLITE_OMIT_WINDOWFUNC */ + }else if( tokenType!=TK_QNUMBER ){ + Token x; + x.z = zSql; + x.n = n; + sqlite3ErrorMsg(pParse, "unrecognized token: \"%T\"", &x); break; } - }else{ - sqlite3Parser(pEngine, tokenType, pParse->sLastToken, pParse); - lastTokenParsed = tokenType; - if( pParse->rc!=SQLITE_OK || db->mallocFailed ) break; } + pParse->sLastToken.z = zSql; + pParse->sLastToken.n = n; + sqlite3Parser(pEngine, tokenType, pParse->sLastToken); + lastTokenParsed = tokenType; + zSql += n; + assert( db->mallocFailed==0 || pParse->rc!=SQLITE_OK || startedWithOom ); + if( pParse->rc!=SQLITE_OK ) break; } assert( nErr==0 ); - pParse->zTail = &zSql[i]; - if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ - assert( zSql[i]==0 ); - if( lastTokenParsed!=TK_SEMI ){ - sqlite3Parser(pEngine, TK_SEMI, pParse->sLastToken, pParse); - } - if( pParse->rc==SQLITE_OK && db->mallocFailed==0 ){ - sqlite3Parser(pEngine, 0, pParse->sLastToken, pParse); - } - } #ifdef YYTRACKMAXSTACKDEPTH sqlite3_mutex_enter(sqlite3MallocMutex()); sqlite3StatusHighwater(SQLITE_STATUS_PARSER_STACK, @@ -135206,61 +180550,177 @@ SQLITE_PRIVATE int sqlite3RunParser(Parse *pParse, const char *zSql, char **pzEr ); sqlite3_mutex_leave(sqlite3MallocMutex()); #endif /* YYDEBUG */ +#ifdef sqlite3Parser_ENGINEALWAYSONSTACK + sqlite3ParserFinalize(pEngine); +#else sqlite3ParserFree(pEngine, sqlite3_free); +#endif if( db->mallocFailed ){ pParse->rc = SQLITE_NOMEM_BKPT; } - if( pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE && pParse->zErrMsg==0 ){ - pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); - } - assert( pzErrMsg!=0 ); - if( pParse->zErrMsg ){ - *pzErrMsg = pParse->zErrMsg; - sqlite3_log(pParse->rc, "%s", *pzErrMsg); - pParse->zErrMsg = 0; + if( pParse->zErrMsg || (pParse->rc!=SQLITE_OK && pParse->rc!=SQLITE_DONE) ){ + if( pParse->zErrMsg==0 ){ + pParse->zErrMsg = sqlite3MPrintf(db, "%s", sqlite3ErrStr(pParse->rc)); + } + sqlite3_log(pParse->rc, "%s in \"%s\"", pParse->zErrMsg, pParse->zTail); nErr++; } - if( pParse->pVdbe && pParse->nErr>0 && pParse->nested==0 ){ - sqlite3VdbeDelete(pParse->pVdbe); - pParse->pVdbe = 0; - } -#ifndef SQLITE_OMIT_SHARED_CACHE - if( pParse->nested==0 ){ - sqlite3DbFree(db, pParse->aTableLock); - pParse->aTableLock = 0; - pParse->nTableLock = 0; - } -#endif + pParse->zTail = zSql; #ifndef SQLITE_OMIT_VIRTUALTABLE sqlite3_free(pParse->apVtabLock); #endif - if( !IN_DECLARE_VTAB ){ - /* If the pParse->declareVtab flag is set, do not delete any table + if( pParse->pNewTable && !IN_SPECIAL_PARSE ){ + /* If the pParse->declareVtab flag is set, do not delete any table ** structure built up in pParse->pNewTable. The calling code (see vtab.c) ** will take responsibility for freeing the Table structure. */ sqlite3DeleteTable(db, pParse->pNewTable); } - - if( pParse->pWithToFree ) sqlite3WithDelete(db, pParse->pWithToFree); - sqlite3DeleteTrigger(db, pParse->pNewTrigger); - for(i=pParse->nzVar-1; i>=0; i--) sqlite3DbFree(db, pParse->azVar[i]); - sqlite3DbFree(db, pParse->azVar); - while( pParse->pAinc ){ - AutoincInfo *p = pParse->pAinc; - pParse->pAinc = p->pNext; - sqlite3DbFree(db, p); - } - while( pParse->pZombieTab ){ - Table *p = pParse->pZombieTab; - pParse->pZombieTab = p->pNextZombie; - sqlite3DeleteTable(db, p); + if( pParse->pNewTrigger && !IN_RENAME_OBJECT ){ + sqlite3DeleteTrigger(db, pParse->pNewTrigger); } + if( pParse->pVList ) sqlite3DbNNFreeNN(db, pParse->pVList); + db->pParse = pParentParse; assert( nErr==0 || pParse->rc!=SQLITE_OK ); return nErr; } + +#ifdef SQLITE_ENABLE_NORMALIZE +/* +** Insert a single space character into pStr if the current string +** ends with an identifier +*/ +static void addSpaceSeparator(sqlite3_str *pStr){ + if( pStr->nChar && sqlite3IsIdChar(pStr->zText[pStr->nChar-1]) ){ + sqlite3_str_append(pStr, " ", 1); + } +} + +/* +** Compute a normalization of the SQL given by zSql[0..nSql-1]. Return +** the normalization in space obtained from sqlite3DbMalloc(). Or return +** NULL if anything goes wrong or if zSql is NULL. +*/ +SQLITE_PRIVATE char *sqlite3Normalize( + Vdbe *pVdbe, /* VM being reprepared */ + const char *zSql /* The original SQL string */ +){ + sqlite3 *db; /* The database connection */ + int i; /* Next unread byte of zSql[] */ + int n; /* length of current token */ + int tokenType; /* type of current token */ + int prevType = 0; /* Previous non-whitespace token */ + int nParen; /* Number of nested levels of parentheses */ + int iStartIN; /* Start of RHS of IN operator in z[] */ + int nParenAtIN; /* Value of nParent at start of RHS of IN operator */ + u32 j; /* Bytes of normalized SQL generated so far */ + sqlite3_str *pStr; /* The normalized SQL string under construction */ + + db = sqlite3VdbeDb(pVdbe); + tokenType = -1; + nParen = iStartIN = nParenAtIN = 0; + pStr = sqlite3_str_new(db); + assert( pStr!=0 ); /* sqlite3_str_new() never returns NULL */ + for(i=0; zSql[i] && pStr->accError==0; i+=n){ + if( tokenType!=TK_SPACE ){ + prevType = tokenType; + } + n = sqlite3GetToken((unsigned char*)zSql+i, &tokenType); + if( NEVER(n<=0) ) break; + switch( tokenType ){ + case TK_SPACE: { + break; + } + case TK_NULL: { + if( prevType==TK_IS || prevType==TK_NOT ){ + sqlite3_str_append(pStr, " NULL", 5); + break; + } + /* Fall through */ + } + case TK_STRING: + case TK_INTEGER: + case TK_FLOAT: + case TK_VARIABLE: + case TK_BLOB: { + sqlite3_str_append(pStr, "?", 1); + break; + } + case TK_LP: { + nParen++; + if( prevType==TK_IN ){ + iStartIN = pStr->nChar; + nParenAtIN = nParen; + } + sqlite3_str_append(pStr, "(", 1); + break; + } + case TK_RP: { + if( iStartIN>0 && nParen==nParenAtIN ){ + assert( pStr->nChar>=(u32)iStartIN ); + pStr->nChar = iStartIN+1; + sqlite3_str_append(pStr, "?,?,?", 5); + iStartIN = 0; + } + nParen--; + sqlite3_str_append(pStr, ")", 1); + break; + } + case TK_ID: { + iStartIN = 0; + j = pStr->nChar; + if( sqlite3Isquote(zSql[i]) ){ + char *zId = sqlite3DbStrNDup(db, zSql+i, n); + int nId; + int eType = 0; + if( zId==0 ) break; + sqlite3Dequote(zId); + if( zSql[i]=='"' && sqlite3VdbeUsesDoubleQuotedString(pVdbe, zId) ){ + sqlite3_str_append(pStr, "?", 1); + sqlite3DbFree(db, zId); + break; + } + nId = sqlite3Strlen30(zId); + if( sqlite3GetToken((u8*)zId, &eType)==nId && eType==TK_ID ){ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zId, nId); + }else{ + sqlite3_str_appendf(pStr, "\"%w\"", zId); + } + sqlite3DbFree(db, zId); + }else{ + addSpaceSeparator(pStr); + sqlite3_str_append(pStr, zSql+i, n); + } + while( jnChar ){ + pStr->zText[j] = sqlite3Tolower(pStr->zText[j]); + j++; + } + break; + } + case TK_SELECT: { + iStartIN = 0; + /* fall through */ + } + default: { + if( sqlite3IsIdChar(zSql[i]) ) addSpaceSeparator(pStr); + j = pStr->nChar; + sqlite3_str_append(pStr, zSql+i, n); + while( jnChar ){ + pStr->zText[j] = sqlite3Toupper(pStr->zText[j]); + j++; + } + break; + } + } + } + if( tokenType!=TK_SEMI ) sqlite3_str_append(pStr, ";", 1); + return sqlite3_str_finish(pStr); +} +#endif /* SQLITE_ENABLE_NORMALIZE */ + /************** End of tokenize.c ********************************************/ /************** Begin file complete.c ****************************************/ /* @@ -135331,7 +180791,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** (2) NORMAL We are in the middle of statement which ends with a single ** semicolon. ** -** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of +** (3) EXPLAIN The keyword EXPLAIN has been seen at the beginning of ** a statement. ** ** (4) CREATE The keyword CREATE has been seen at the beginning of a @@ -135366,7 +180826,7 @@ SQLITE_PRIVATE const char sqlite3IsEbcdicIdChar[]; ** to recognize the end of a trigger can be omitted. All we have to do ** is look for a semicolon that is not part of an string or comment. */ -SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *zSql){ +SQLITE_API int sqlite3_complete(const char *zSql){ u8 state = 0; /* Current state, using numbers defined in header comment */ u8 token; /* Value of the next token */ @@ -135531,7 +180991,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_complete(const char *zSql){ ** above, except that the parameter is required to be UTF-16 encoded, not ** UTF-8. */ -SQLITE_API int SQLITE_STDCALL sqlite3_complete16(const void *zSql){ +SQLITE_API int sqlite3_complete16(const void *zSql){ sqlite3_value *pVal; char const *zSql8; int rc; @@ -135627,6 +181087,10 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db); */ /* #include "sqlite3.h" */ +#ifdef SQLITE_OMIT_VIRTUALTABLE +# undef SQLITE_ENABLE_RTREE +#endif + #if 0 extern "C" { #endif /* __cplusplus */ @@ -135640,7 +181104,7 @@ SQLITE_PRIVATE int sqlite3RtreeInit(sqlite3 *db); /************** End of rtree.h ***********************************************/ /************** Continuing where we left off in main.c ***********************/ #endif -#ifdef SQLITE_ENABLE_ICU +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) /************** Include sqliteicu.h in the middle of main.c ******************/ /************** Begin file sqliteicu.h ***************************************/ /* @@ -135670,45 +181134,102 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db); } /* extern "C" */ #endif /* __cplusplus */ - /************** End of sqliteicu.h *******************************************/ /************** Continuing where we left off in main.c ***********************/ #endif -#ifdef SQLITE_ENABLE_JSON1 -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3*); -#endif + +/* +** This is an extension initializer that is a no-op and always +** succeeds, except that it fails if the fault-simulation is set +** to 500. +*/ +static int sqlite3TestExtInit(sqlite3 *db){ + (void)db; + return sqlite3FaultSim(500); +} + + +/* +** Forward declarations of external module initializer functions +** for modules that need them. +*/ #ifdef SQLITE_ENABLE_FTS5 SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3*); #endif +#ifdef SQLITE_ENABLE_STMTVTAB +SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3*); +#endif +#ifdef SQLITE_EXTRA_AUTOEXT +int SQLITE_EXTRA_AUTOEXT(sqlite3*); +#endif +/* +** An array of pointers to extension initializer functions for +** built-in extensions. +*/ +static int (*const sqlite3BuiltinExtensions[])(sqlite3*) = { +#ifdef SQLITE_ENABLE_FTS3 + sqlite3Fts3Init, +#endif +#ifdef SQLITE_ENABLE_FTS5 + sqlite3Fts5Init, +#endif +#if defined(SQLITE_ENABLE_ICU) || defined(SQLITE_ENABLE_ICU_COLLATIONS) + sqlite3IcuInit, +#endif +#ifdef SQLITE_ENABLE_RTREE + sqlite3RtreeInit, +#endif +#ifdef SQLITE_ENABLE_DBPAGE_VTAB + sqlite3DbpageRegister, +#endif +#ifdef SQLITE_ENABLE_DBSTAT_VTAB + sqlite3DbstatRegister, +#endif + sqlite3TestExtInit, +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) + sqlite3JsonTableFunctions, +#endif +#ifdef SQLITE_ENABLE_STMTVTAB + sqlite3StmtVtabInit, +#endif +#ifdef SQLITE_ENABLE_BYTECODE_VTAB + sqlite3VdbeBytecodeVtabInit, +#endif +#ifdef SQLITE_EXTRA_AUTOEXT + SQLITE_EXTRA_AUTOEXT, +#endif +}; #ifndef SQLITE_AMALGAMATION /* IMPLEMENTATION-OF: R-46656-45156 The sqlite3_version[] string constant -** contains the text of SQLITE_VERSION macro. +** contains the text of SQLITE_VERSION macro. */ SQLITE_API const char sqlite3_version[] = SQLITE_VERSION; #endif /* IMPLEMENTATION-OF: R-53536-42575 The sqlite3_libversion() function returns -** a pointer to the to the sqlite3_version[] string constant. +** a pointer to the to the sqlite3_version[] string constant. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_libversion(void){ return sqlite3_version; } +SQLITE_API const char *sqlite3_libversion(void){ return sqlite3_version; } -/* IMPLEMENTATION-OF: R-63124-39300 The sqlite3_sourceid() function returns a +/* IMPLEMENTATION-OF: R-25063-23286 The sqlite3_sourceid() function returns a ** pointer to a string constant whose value is the same as the -** SQLITE_SOURCE_ID C preprocessor macro. +** SQLITE_SOURCE_ID C preprocessor macro. Except if SQLite is built using +** an edited copy of the amalgamation, then the last four characters of +** the hash might be different from SQLITE_SOURCE_ID. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } +/* SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } */ /* IMPLEMENTATION-OF: R-35210-63508 The sqlite3_libversion_number() function ** returns an integer equal to SQLITE_VERSION_NUMBER. */ -SQLITE_API int SQLITE_STDCALL sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } +SQLITE_API int sqlite3_libversion_number(void){ return SQLITE_VERSION_NUMBER; } /* IMPLEMENTATION-OF: R-20790-14025 The sqlite3_threadsafe() function returns ** zero if and only if SQLite was compiled with mutexing code omitted due to ** the SQLITE_THREADSAFE compile-time option being set to 0. */ -SQLITE_API int SQLITE_STDCALL sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } +SQLITE_API int sqlite3_threadsafe(void){ return SQLITE_THREADSAFE; } /* ** When compiling the test fixture or with debugging enabled (on Win32), @@ -135751,13 +181272,13 @@ SQLITE_API char *sqlite3_temp_directory = 0; SQLITE_API char *sqlite3_data_directory = 0; /* -** Initialize SQLite. +** Initialize SQLite. ** ** This routine must be called to initialize the memory allocation, ** VFS, and mutex subsystems prior to doing any serious work with ** SQLite. But as long as you do not compile with SQLITE_OMIT_AUTOINIT ** this routine will be called automatically by key routines such as -** sqlite3_open(). +** sqlite3_open(). ** ** This routine is a no-op except on its very first call for the process, ** or for the first call after a call to sqlite3_shutdown. @@ -135781,8 +181302,8 @@ SQLITE_API char *sqlite3_data_directory = 0; ** * Recursive calls to this routine from thread X return immediately ** without blocking. */ -SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ - MUTEX_LOGIC( sqlite3_mutex *pMaster; ) /* The main static mutex */ +SQLITE_API int sqlite3_initialize(void){ + MUTEX_LOGIC( sqlite3_mutex *pMainMtx; ) /* The main static mutex */ int rc; /* Result code */ #ifdef SQLITE_EXTRA_INIT int bRunExtraInit = 0; /* Extra initialization needed */ @@ -135805,9 +181326,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ ** must be complete. So isInit must not be set until the very end ** of this routine. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_OK; + if( sqlite3GlobalConfig.isInit ){ + sqlite3MemoryBarrier(); + return SQLITE_OK; + } - /* Make sure the mutex subsystem is initialized. If unable to + /* Make sure the mutex subsystem is initialized. If unable to ** initialize the mutex subsystem, return early with the error. ** If the system is so sick that we are unable to allocate a mutex, ** there is not much SQLite is going to be able to do. @@ -135819,13 +181343,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ if( rc ) return rc; /* Initialize the malloc() system and the recursive pInitMutex mutex. - ** This operation is protected by the STATIC_MASTER mutex. Note that + ** This operation is protected by the STATIC_MAIN mutex. Note that ** MutexAlloc() is called for a static mutex prior to initializing the ** malloc subsystem - this implies that the allocation of a static ** mutex must not require support from the malloc subsystem. */ - MUTEX_LOGIC( pMaster = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER); ) - sqlite3_mutex_enter(pMaster); + MUTEX_LOGIC( pMainMtx = sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN); ) + sqlite3_mutex_enter(pMainMtx); sqlite3GlobalConfig.isMutexInit = 1; if( !sqlite3GlobalConfig.isMallocInit ){ rc = sqlite3MallocInit(); @@ -135843,7 +181367,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ if( rc==SQLITE_OK ){ sqlite3GlobalConfig.nRefInitMutex++; } - sqlite3_mutex_leave(pMaster); + sqlite3_mutex_leave(pMainMtx); /* If rc is not SQLITE_OK at this point, then either the malloc ** subsystem could not be initialized or the system failed to allocate @@ -135883,9 +181407,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ sqlite3GlobalConfig.isPCacheInit = 1; rc = sqlite3OsInit(); } +#ifndef SQLITE_OMIT_DESERIALIZE if( rc==SQLITE_OK ){ - sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, + rc = sqlite3MemdbInit(); + } +#endif + if( rc==SQLITE_OK ){ + sqlite3PCacheBufferSetup( sqlite3GlobalConfig.pPage, sqlite3GlobalConfig.szPage, sqlite3GlobalConfig.nPage); + sqlite3MemoryBarrier(); sqlite3GlobalConfig.isInit = 1; #ifdef SQLITE_EXTRA_INIT bRunExtraInit = 1; @@ -135898,14 +181428,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ /* Go back under the static mutex and clean up the recursive ** mutex to prevent a resource leak. */ - sqlite3_mutex_enter(pMaster); + sqlite3_mutex_enter(pMainMtx); sqlite3GlobalConfig.nRefInitMutex--; if( sqlite3GlobalConfig.nRefInitMutex<=0 ){ assert( sqlite3GlobalConfig.nRefInitMutex==0 ); sqlite3_mutex_free(sqlite3GlobalConfig.pInitMutex); sqlite3GlobalConfig.pInitMutex = 0; } - sqlite3_mutex_leave(pMaster); + sqlite3_mutex_leave(pMainMtx); /* The following is just a sanity check to make sure SQLite has ** been compiled correctly. It is important to run this code, but @@ -135915,7 +181445,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ #ifndef NDEBUG #ifndef SQLITE_OMIT_FLOATING_POINT /* This section of code's only "output" is via assert() statements. */ - if ( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ u64 x = (((u64)1)<<63)-1; double y; assert(sizeof(x)==8); @@ -135935,7 +181465,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ rc = SQLITE_EXTRA_INIT(0); } #endif - return rc; } @@ -135947,7 +181476,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_initialize(void){ ** on when SQLite is already shut down. If SQLite is already shut down ** when this routine is invoked, then this routine is a harmless no-op. */ -SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){ +SQLITE_API int sqlite3_shutdown(void){ #ifdef SQLITE_OMIT_WSD int rc = sqlite3_wsd_init(4096, 24); if( rc!=SQLITE_OK ){ @@ -136001,13 +181530,25 @@ SQLITE_API int SQLITE_STDCALL sqlite3_shutdown(void){ ** threadsafe. Failure to heed these warnings can lead to unpredictable ** behavior. */ -SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ +SQLITE_API int sqlite3_config(int op, ...){ va_list ap; int rc = SQLITE_OK; - /* sqlite3_config() shall return SQLITE_MISUSE if it is invoked while - ** the SQLite library is in use. */ - if( sqlite3GlobalConfig.isInit ) return SQLITE_MISUSE_BKPT; + /* sqlite3_config() normally returns SQLITE_MISUSE if it is invoked while + ** the SQLite library is in use. Except, a few selected opcodes + ** are allowed. + */ + if( sqlite3GlobalConfig.isInit ){ + static const u64 mAnytimeConfigOption = 0 + | MASKBIT64( SQLITE_CONFIG_LOG ) + | MASKBIT64( SQLITE_CONFIG_PCACHE_HDRSZ ) + ; + if( op<0 || op>63 || (MASKBIT64(op) & mAnytimeConfigOption)==0 ){ + return SQLITE_MISUSE_BKPT; + } + testcase( op==SQLITE_CONFIG_LOG ); + testcase( op==SQLITE_CONFIG_PCACHE_HDRSZ ); + } va_start(ap, op); switch( op ){ @@ -136076,20 +181617,15 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ break; } case SQLITE_CONFIG_MEMSTATUS: { + assert( !sqlite3GlobalConfig.isInit ); /* Cannot change at runtime */ /* EVIDENCE-OF: R-61275-35157 The SQLITE_CONFIG_MEMSTATUS option takes ** single argument of type int, interpreted as a boolean, which enables ** or disables the collection of memory allocation statistics. */ sqlite3GlobalConfig.bMemstat = va_arg(ap, int); break; } - case SQLITE_CONFIG_SCRATCH: { - /* EVIDENCE-OF: R-08404-60887 There are three arguments to - ** SQLITE_CONFIG_SCRATCH: A pointer an 8-byte aligned memory buffer from - ** which the scratch allocations will be drawn, the size of each scratch - ** allocation (sz), and the maximum number of scratch allocations (N). */ - sqlite3GlobalConfig.pScratch = va_arg(ap, void*); - sqlite3GlobalConfig.szScratch = va_arg(ap, int); - sqlite3GlobalConfig.nScratch = va_arg(ap, int); + case SQLITE_CONFIG_SMALL_MALLOC: { + sqlite3GlobalConfig.bSmallMalloc = va_arg(ap, int); break; } case SQLITE_CONFIG_PAGECACHE: { @@ -136107,7 +181643,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ ** a single parameter which is a pointer to an integer and writes into ** that integer the number of extra bytes per page required for each page ** in SQLITE_CONFIG_PAGECACHE. */ - *va_arg(ap, int*) = + *va_arg(ap, int*) = sqlite3HeaderSizeBtree() + sqlite3HeaderSizePcache() + sqlite3HeaderSizePcache1(); @@ -136194,7 +181730,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ sqlite3GlobalConfig.nLookaside = va_arg(ap, int); break; } - + /* Record a pointer to the logger function and its first argument. ** The default is NULL. Logging is disabled if the function pointer is ** NULL. @@ -136205,8 +181741,10 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ ** sqlite3GlobalConfig.xLog = va_arg(ap, void(*)(void*,int,const char*)); */ typedef void(*LOGFUNC_t)(void*,int,const char*); - sqlite3GlobalConfig.xLog = va_arg(ap, LOGFUNC_t); - sqlite3GlobalConfig.pLogArg = va_arg(ap, void*); + LOGFUNC_t xLog = va_arg(ap, LOGFUNC_t); + void *pLogArg = va_arg(ap, void*); + AtomicStore(&sqlite3GlobalConfig.xLog, xLog); + AtomicStore(&sqlite3GlobalConfig.pLogArg, pLogArg); break; } @@ -136220,7 +181758,8 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ ** argument of type int. If non-zero, then URI handling is globally ** enabled. If the parameter is zero, then URI handling is globally ** disabled. */ - sqlite3GlobalConfig.bOpenUri = va_arg(ap, int); + int bOpenUri = va_arg(ap, int); + AtomicStore(&sqlite3GlobalConfig.bOpenUri, bOpenUri); break; } @@ -136287,6 +181826,36 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ break; } +#ifdef SQLITE_ENABLE_SORTER_REFERENCES + case SQLITE_CONFIG_SORTERREF_SIZE: { + int iVal = va_arg(ap, int); + if( iVal<0 ){ + iVal = SQLITE_DEFAULT_SORTERREF_SIZE; + } + sqlite3GlobalConfig.szSorterRef = (u32)iVal; + break; + } +#endif /* SQLITE_ENABLE_SORTER_REFERENCES */ + +#ifndef SQLITE_OMIT_DESERIALIZE + case SQLITE_CONFIG_MEMDB_MAXSIZE: { + sqlite3GlobalConfig.mxMemdbSize = va_arg(ap, sqlite3_int64); + break; + } +#endif /* SQLITE_OMIT_DESERIALIZE */ + + case SQLITE_CONFIG_ROWID_IN_VIEW: { + int *pVal = va_arg(ap,int*); +#ifdef SQLITE_ALLOW_ROWID_IN_VIEW + if( 0==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = TF_NoVisibleRowid; + if( 1==*pVal ) sqlite3GlobalConfig.mNoVisibleRowid = 0; + *pVal = (sqlite3GlobalConfig.mNoVisibleRowid==0); +#else + *pVal = 0; +#endif + break; + } + default: { rc = SQLITE_ERROR; break; @@ -136298,7 +181867,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ /* ** Set up the lookaside buffers for a database connection. -** Return SQLITE_OK on success. +** Return SQLITE_OK on success. ** If lookaside is already active, return SQLITE_BUSY. ** ** The sz parameter is the number of bytes in each lookaside slot. @@ -136310,11 +181879,15 @@ SQLITE_API int SQLITE_CDECL sqlite3_config(int op, ...){ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ #ifndef SQLITE_OMIT_LOOKASIDE void *pStart; - if( db->lookaside.nOut ){ + sqlite3_int64 szAlloc = sz*(sqlite3_int64)cnt; + int nBig; /* Number of full-size slots */ + int nSm; /* Number smaller LOOKASIDE_SMALL-byte slots */ + + if( sqlite3LookasideUsed(db,0)>0 ){ return SQLITE_BUSY; } /* Free any existing lookaside buffer for this handle before - ** allocating a new one so we don't have to have space for + ** allocating a new one so we don't have to have space for ** both at the same time. */ if( db->lookaside.bMalloced ){ @@ -136331,34 +181904,72 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ pStart = 0; }else if( pBuf==0 ){ sqlite3BeginBenignMalloc(); - pStart = sqlite3Malloc( sz*cnt ); /* IMP: R-61949-35727 */ + pStart = sqlite3Malloc( szAlloc ); /* IMP: R-61949-35727 */ sqlite3EndBenignMalloc(); - if( pStart ) cnt = sqlite3MallocSize(pStart)/sz; + if( pStart ) szAlloc = sqlite3MallocSize(pStart); }else{ pStart = pBuf; } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + if( sz>=LOOKASIDE_SMALL*3 ){ + nBig = szAlloc/(3*LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else if( sz>=LOOKASIDE_SMALL*2 ){ + nBig = szAlloc/(LOOKASIDE_SMALL+sz); + nSm = (szAlloc - sz*nBig)/LOOKASIDE_SMALL; + }else +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + if( sz>0 ){ + nBig = szAlloc/sz; + nSm = 0; + }else{ + nBig = nSm = 0; + } db->lookaside.pStart = pStart; + db->lookaside.pInit = 0; db->lookaside.pFree = 0; db->lookaside.sz = (u16)sz; + db->lookaside.szTrue = (u16)sz; if( pStart ){ int i; LookasideSlot *p; assert( sz > (int)sizeof(LookasideSlot*) ); p = (LookasideSlot*)pStart; - for(i=cnt-1; i>=0; i--){ - p->pNext = db->lookaside.pFree; - db->lookaside.pFree = p; + for(i=0; ipNext = db->lookaside.pInit; + db->lookaside.pInit = p; p = (LookasideSlot*)&((u8*)p)[sz]; } +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = p; + for(i=0; ipNext = db->lookaside.pSmallInit; + db->lookaside.pSmallInit = p; + p = (LookasideSlot*)&((u8*)p)[LOOKASIDE_SMALL]; + } +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + assert( ((uptr)p)<=szAlloc + (uptr)pStart ); db->lookaside.pEnd = p; db->lookaside.bDisable = 0; db->lookaside.bMalloced = pBuf==0 ?1:0; + db->lookaside.nSlot = nBig+nSm; }else{ - db->lookaside.pStart = db; - db->lookaside.pEnd = db; + db->lookaside.pStart = 0; +#ifndef SQLITE_OMIT_TWOSIZE_LOOKASIDE + db->lookaside.pSmallInit = 0; + db->lookaside.pSmallFree = 0; + db->lookaside.pMiddle = 0; +#endif /* SQLITE_OMIT_TWOSIZE_LOOKASIDE */ + db->lookaside.pEnd = 0; db->lookaside.bDisable = 1; + db->lookaside.sz = 0; db->lookaside.bMalloced = 0; + db->lookaside.nSlot = 0; } + db->lookaside.pTrueEnd = db->lookaside.pEnd; + assert( sqlite3LookasideUsed(db,0)==0 ); #endif /* SQLITE_OMIT_LOOKASIDE */ return SQLITE_OK; } @@ -136366,7 +181977,7 @@ static int setupLookaside(sqlite3 *db, void *pBuf, int sz, int cnt){ /* ** Return the mutex associated with a database connection. */ -SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; @@ -136380,7 +181991,7 @@ SQLITE_API sqlite3_mutex *SQLITE_STDCALL sqlite3_db_mutex(sqlite3 *db){ ** Free up as much memory as we can from the given database ** connection. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ +SQLITE_API int sqlite3_db_release_memory(sqlite3 *db){ int i; #ifdef SQLITE_ENABLE_API_ARMOR @@ -136404,7 +182015,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_release_memory(sqlite3 *db){ ** Flush any dirty pages in the pager-cache for any attached database ** to disk. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3 *db){ +SQLITE_API int sqlite3_db_cacheflush(sqlite3 *db){ int i; int rc = SQLITE_OK; int bSeenBusy = 0; @@ -136416,7 +182027,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3 *db){ sqlite3BtreeEnterAll(db); for(i=0; rc==SQLITE_OK && inDb; i++){ Btree *pBt = db->aDb[i].pBt; - if( pBt && sqlite3BtreeIsInTrans(pBt) ){ + if( pBt && sqlite3BtreeTxnState(pBt)==SQLITE_TXN_WRITE ){ Pager *pPager = sqlite3BtreePager(pBt); rc = sqlite3PagerFlush(pPager); if( rc==SQLITE_BUSY ){ @@ -136433,11 +182044,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_cacheflush(sqlite3 *db){ /* ** Configuration settings for an individual database connection */ -SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ +SQLITE_API int sqlite3_db_config(sqlite3 *db, int op, ...){ va_list ap; int rc; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif + sqlite3_mutex_enter(db->mutex); va_start(ap, op); switch( op ){ + case SQLITE_DBCONFIG_MAINDBNAME: { + /* IMP: R-06824-28531 */ + /* IMP: R-36257-52125 */ + db->aDb[0].zDbSName = va_arg(ap,char*); + rc = SQLITE_OK; + break; + } case SQLITE_DBCONFIG_LOOKASIDE: { void *pBuf = va_arg(ap, void*); /* IMP: R-26835-10964 */ int sz = va_arg(ap, int); /* IMP: R-47871-25994 */ @@ -136452,8 +182075,23 @@ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ } aFlagOp[] = { { SQLITE_DBCONFIG_ENABLE_FKEY, SQLITE_ForeignKeys }, { SQLITE_DBCONFIG_ENABLE_TRIGGER, SQLITE_EnableTrigger }, + { SQLITE_DBCONFIG_ENABLE_VIEW, SQLITE_EnableView }, { SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER, SQLITE_Fts3Tokenizer }, { SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION, SQLITE_LoadExtension }, + { SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE, SQLITE_NoCkptOnClose }, + { SQLITE_DBCONFIG_ENABLE_QPSG, SQLITE_EnableQPSG }, + { SQLITE_DBCONFIG_TRIGGER_EQP, SQLITE_TriggerEQP }, + { SQLITE_DBCONFIG_RESET_DATABASE, SQLITE_ResetDatabase }, + { SQLITE_DBCONFIG_DEFENSIVE, SQLITE_Defensive }, + { SQLITE_DBCONFIG_WRITABLE_SCHEMA, SQLITE_WriteSchema| + SQLITE_NoSchemaError }, + { SQLITE_DBCONFIG_LEGACY_ALTER_TABLE, SQLITE_LegacyAlter }, + { SQLITE_DBCONFIG_DQS_DDL, SQLITE_DqsDDL }, + { SQLITE_DBCONFIG_DQS_DML, SQLITE_DqsDML }, + { SQLITE_DBCONFIG_LEGACY_FILE_FORMAT, SQLITE_LegacyFileFmt }, + { SQLITE_DBCONFIG_TRUSTED_SCHEMA, SQLITE_TrustedSchema }, + { SQLITE_DBCONFIG_STMT_SCANSTATUS, SQLITE_StmtScanStatus }, + { SQLITE_DBCONFIG_REVERSE_SCANORDER, SQLITE_ReverseOrder }, }; unsigned int i; rc = SQLITE_ERROR; /* IMP: R-42790-23372 */ @@ -136461,14 +182099,14 @@ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ if( aFlagOp[i].op==op ){ int onoff = va_arg(ap, int); int *pRes = va_arg(ap, int*); - int oldFlags = db->flags; + u64 oldFlags = db->flags; if( onoff>0 ){ db->flags |= aFlagOp[i].mask; }else if( onoff==0 ){ - db->flags &= ~aFlagOp[i].mask; + db->flags &= ~(u64)aFlagOp[i].mask; } if( oldFlags!=db->flags ){ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); } if( pRes ){ *pRes = (db->flags & aFlagOp[i].mask)!=0; @@ -136481,55 +182119,59 @@ SQLITE_API int SQLITE_CDECL sqlite3_db_config(sqlite3 *db, int op, ...){ } } va_end(ap); + sqlite3_mutex_leave(db->mutex); return rc; } - -/* -** Return true if the buffer z[0..n-1] contains all spaces. -*/ -static int allSpaces(const char *z, int n){ - while( n>0 && z[n-1]==' ' ){ n--; } - return n==0; -} - /* ** This is the default collating function named "BINARY" which is always ** available. -** -** If the padFlag argument is not NULL then space padding at the end -** of strings is ignored. This implements the RTRIM collation. */ static int binCollFunc( - void *padFlag, + void *NotUsed, int nKey1, const void *pKey1, int nKey2, const void *pKey2 ){ int rc, n; + UNUSED_PARAMETER(NotUsed); n = nKey1xCmp!=binCollFunc || strcmp(p->zName,"BINARY")==0 ); + return p==0 || p->xCmp==binCollFunc; +} + +/* +** Another built-in collating sequence: NOCASE. ** ** This collating sequence is intended to be used for "case independent ** comparison". SQLite's knowledge of upper and lower case equivalents @@ -136554,7 +182196,7 @@ static int nocaseCollatingFunc( /* ** Return the ROWID of the most recent insert */ -SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3 *db){ +SQLITE_API sqlite_int64 sqlite3_last_insert_rowid(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; @@ -136564,10 +182206,25 @@ SQLITE_API sqlite_int64 SQLITE_STDCALL sqlite3_last_insert_rowid(sqlite3 *db){ return db->lastRowid; } +/* +** Set the value returned by the sqlite3_last_insert_rowid() API function. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3 *db, sqlite3_int64 iRowid){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return; + } +#endif + sqlite3_mutex_enter(db->mutex); + db->lastRowid = iRowid; + sqlite3_mutex_leave(db->mutex); +} + /* ** Return the number of changes in the most recent call to sqlite3_exec(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){ +SQLITE_API sqlite3_int64 sqlite3_changes64(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; @@ -136576,11 +182233,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_changes(sqlite3 *db){ #endif return db->nChange; } +SQLITE_API int sqlite3_changes(sqlite3 *db){ + return (int)sqlite3_changes64(db); +} /* ** Return the number of changes since the database handle was opened. */ -SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){ +SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; @@ -136589,6 +182249,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_total_changes(sqlite3 *db){ #endif return db->nTotalChange; } +SQLITE_API int sqlite3_total_changes(sqlite3 *db){ + return (int)sqlite3_total_changes64(db); +} /* ** Close all open savepoints. This function only manipulates fields of the @@ -136613,7 +182276,9 @@ SQLITE_PRIVATE void sqlite3CloseSavepoints(sqlite3 *db){ ** with SQLITE_ANY as the encoding. */ static void functionDestroy(sqlite3 *db, FuncDef *p){ - FuncDestructor *pDestructor = p->u.pDestructor; + FuncDestructor *pDestructor; + assert( (p->funcFlags & SQLITE_FUNC_BUILTIN)==0 ); + pDestructor = p->u.pDestructor; if( pDestructor ){ pDestructor->nRef--; if( pDestructor->nRef==0 ){ @@ -136634,7 +182299,7 @@ static void disconnectAllVtab(sqlite3 *db){ sqlite3BtreeEnterAll(db); for(i=0; inDb; i++){ Schema *pSchema = db->aDb[i].pSchema; - if( db->aDb[i].pSchema ){ + if( pSchema ){ for(p=sqliteHashFirst(&pSchema->tblHash); p; p=sqliteHashNext(p)){ Table *pTab = (Table *)sqliteHashData(p); if( IsVirtual(pTab) ) sqlite3VtabDisconnect(db, pTab); @@ -136656,7 +182321,7 @@ static void disconnectAllVtab(sqlite3 *db){ /* ** Return TRUE if database connection db has unfinalized prepared -** statements or unfinished sqlite3_backup objects. +** statements or unfinished sqlite3_backup objects. */ static int connectionIsBusy(sqlite3 *db){ int j; @@ -136682,6 +182347,9 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); + if( db->mTrace & SQLITE_TRACE_CLOSE ){ + db->trace.xV2(SQLITE_TRACE_CLOSE, db->pTraceArg, db, 0); + } /* Force xDisconnect calls on all virtual tables */ disconnectAllVtab(db); @@ -136712,24 +182380,62 @@ static int sqlite3Close(sqlite3 *db, int forceZombie){ } #endif + while( db->pDbData ){ + DbClientData *p = db->pDbData; + db->pDbData = p->pNext; + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + sqlite3_free(p); + } + /* Convert the connection into a zombie and then close it. */ - db->magic = SQLITE_MAGIC_ZOMBIE; + db->eOpenState = SQLITE_STATE_ZOMBIE; sqlite3LeaveMutexAndCloseZombie(db); return SQLITE_OK; } +/* +** Return the transaction state for a single databse, or the maximum +** transaction state over all attached databases if zSchema is null. +*/ +SQLITE_API int sqlite3_txn_state(sqlite3 *db, const char *zSchema){ + int iDb, nDb; + int iTxn = -1; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return -1; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( zSchema ){ + nDb = iDb = sqlite3FindDbName(db, zSchema); + if( iDb<0 ) nDb--; + }else{ + iDb = 0; + nDb = db->nDb-1; + } + for(; iDb<=nDb; iDb++){ + Btree *pBt = db->aDb[iDb].pBt; + int x = pBt!=0 ? sqlite3BtreeTxnState(pBt) : SQLITE_TXN_NONE; + if( x>iTxn ) iTxn = x; + } + sqlite3_mutex_leave(db->mutex); + return iTxn; +} + /* ** Two variations on the public interface for closing a database ** connection. The sqlite3_close() version returns SQLITE_BUSY and -** leaves the connection option if there are unfinalized prepared +** leaves the connection open if there are unfinalized prepared ** statements or unfinished sqlite3_backups. The sqlite3_close_v2() ** version forces the connection to become a zombie if there are ** unclosed resources, and arranges for deallocation when the last ** prepare statement or sqlite3_backup closes. */ -SQLITE_API int SQLITE_STDCALL sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } -SQLITE_API int SQLITE_STDCALL sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } +SQLITE_API int sqlite3_close(sqlite3 *db){ return sqlite3Close(db,0); } +SQLITE_API int sqlite3_close_v2(sqlite3 *db){ return sqlite3Close(db,1); } /* @@ -136748,7 +182454,7 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ ** or if the connection has not yet been closed by sqlite3_close_v2(), ** then just leave the mutex and return. */ - if( db->magic!=SQLITE_MAGIC_ZOMBIE || connectionIsBusy(db) ){ + if( db->eOpenState!=SQLITE_STATE_ZOMBIE || connectionIsBusy(db) ){ sqlite3_mutex_leave(db->mutex); return; } @@ -136820,11 +182526,8 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ #ifndef SQLITE_OMIT_VIRTUALTABLE for(i=sqliteHashFirst(&db->aModule); i; i=sqliteHashNext(i)){ Module *pMod = (Module *)sqliteHashData(i); - if( pMod->xDestroy ){ - pMod->xDestroy(pMod->pAux); - } sqlite3VtabEponymousTableClear(db, pMod); - sqlite3DbFree(db, pMod); + sqlite3VtabModuleUnref(db, pMod); } sqlite3HashClear(&db->aModule); #endif @@ -136837,19 +182540,22 @@ SQLITE_PRIVATE void sqlite3LeaveMutexAndCloseZombie(sqlite3 *db){ sqlite3_free(db->auth.zAuthPW); #endif - db->magic = SQLITE_MAGIC_ERROR; + db->eOpenState = SQLITE_STATE_ERROR; /* The temp-database schema is allocated differently from the other schema ** objects (using sqliteMalloc() directly, instead of sqlite3BtreeSchema()). ** So it needs to be freed here. Todo: Why not roll the temp schema into - ** the same sqliteMalloc() as the one that allocates the database + ** the same sqliteMalloc() as the one that allocates the database ** structure? */ sqlite3DbFree(db, db->aDb[1].pSchema); + if( db->xAutovacDestr ){ + db->xAutovacDestr(db->pAutovacPagesArg); + } sqlite3_mutex_leave(db->mutex); - db->magic = SQLITE_MAGIC_CLOSED; + db->eOpenState = SQLITE_STATE_CLOSED; sqlite3_mutex_free(db->mutex); - assert( db->lookaside.nOut==0 ); /* Fails on a lookaside memory leak */ + assert( sqlite3LookasideUsed(db,0)==0 ); if( db->lookaside.bMalloced ){ sqlite3_free(db->lookaside.pStart); } @@ -136870,19 +182576,19 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ assert( sqlite3_mutex_held(db->mutex) ); sqlite3BeginBenignMalloc(); - /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). + /* Obtain all b-tree mutexes before making any calls to BtreeRollback(). ** This is important in case the transaction being rolled back has ** modified the database schema. If the b-tree mutexes are not taken ** here, then another shared-cache connection might sneak in between ** the database rollback and schema reset, which can cause false ** corruption reports in some cases. */ sqlite3BtreeEnterAll(db); - schemaChange = (db->flags & SQLITE_InternChanges)!=0 && db->init.busy==0; + schemaChange = (db->mDbFlags & DBFLAG_SchemaChange)!=0 && db->init.busy==0; for(i=0; inDb; i++){ Btree *p = db->aDb[i].pBt; if( p ){ - if( sqlite3BtreeIsInTrans(p) ){ + if( sqlite3BtreeTxnState(p)==SQLITE_TXN_WRITE ){ inTrans = 1; } sqlite3BtreeRollback(p, tripCode, !schemaChange); @@ -136891,8 +182597,8 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ sqlite3VtabRollback(db); sqlite3EndBenignMalloc(); - if( (db->flags&SQLITE_InternChanges)!=0 && db->init.busy==0 ){ - sqlite3ExpirePreparedStatements(db); + if( schemaChange ){ + sqlite3ExpirePreparedStatements(db, 0); sqlite3ResetAllSchemasOfConnection(db); } sqlite3BtreeLeaveAll(db); @@ -136900,7 +182606,7 @@ SQLITE_PRIVATE void sqlite3RollbackAll(sqlite3 *db, int tripCode){ /* Any deferred constraint violations have now been resolved. */ db->nDeferredCons = 0; db->nDeferredImmCons = 0; - db->flags &= ~SQLITE_DeferFKs; + db->flags &= ~(u64)(SQLITE_DeferFKs|SQLITE_CorruptRdOnly); /* If one has been configured, invoke the rollback-hook callback */ if( db->xRollbackCallback && (inTrans || !db->autoCommit) ){ @@ -136920,6 +182626,7 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ switch( rc ){ case SQLITE_OK: zName = "SQLITE_OK"; break; case SQLITE_ERROR: zName = "SQLITE_ERROR"; break; + case SQLITE_ERROR_SNAPSHOT: zName = "SQLITE_ERROR_SNAPSHOT"; break; case SQLITE_INTERNAL: zName = "SQLITE_INTERNAL"; break; case SQLITE_PERM: zName = "SQLITE_PERM"; break; case SQLITE_ABORT: zName = "SQLITE_ABORT"; break; @@ -136932,9 +182639,10 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_NOMEM: zName = "SQLITE_NOMEM"; break; case SQLITE_READONLY: zName = "SQLITE_READONLY"; break; case SQLITE_READONLY_RECOVERY: zName = "SQLITE_READONLY_RECOVERY"; break; - case SQLITE_READONLY_CANTLOCK: zName = "SQLITE_READONLY_CANTLOCK"; break; + case SQLITE_READONLY_CANTINIT: zName = "SQLITE_READONLY_CANTINIT"; break; case SQLITE_READONLY_ROLLBACK: zName = "SQLITE_READONLY_ROLLBACK"; break; case SQLITE_READONLY_DBMOVED: zName = "SQLITE_READONLY_DBMOVED"; break; + case SQLITE_READONLY_DIRECTORY: zName = "SQLITE_READONLY_DIRECTORY";break; case SQLITE_INTERRUPT: zName = "SQLITE_INTERRUPT"; break; case SQLITE_IOERR: zName = "SQLITE_IOERR"; break; case SQLITE_IOERR_READ: zName = "SQLITE_IOERR_READ"; break; @@ -136972,6 +182680,7 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_CANTOPEN_ISDIR: zName = "SQLITE_CANTOPEN_ISDIR"; break; case SQLITE_CANTOPEN_FULLPATH: zName = "SQLITE_CANTOPEN_FULLPATH"; break; case SQLITE_CANTOPEN_CONVPATH: zName = "SQLITE_CANTOPEN_CONVPATH"; break; + case SQLITE_CANTOPEN_SYMLINK: zName = "SQLITE_CANTOPEN_SYMLINK"; break; case SQLITE_PROTOCOL: zName = "SQLITE_PROTOCOL"; break; case SQLITE_EMPTY: zName = "SQLITE_EMPTY"; break; case SQLITE_SCHEMA: zName = "SQLITE_SCHEMA"; break; @@ -137003,6 +182712,7 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ case SQLITE_NOTICE_RECOVER_WAL: zName = "SQLITE_NOTICE_RECOVER_WAL";break; case SQLITE_NOTICE_RECOVER_ROLLBACK: zName = "SQLITE_NOTICE_RECOVER_ROLLBACK"; break; + case SQLITE_NOTICE_RBU: zName = "SQLITE_NOTICE_RBU"; break; case SQLITE_WARNING: zName = "SQLITE_WARNING"; break; case SQLITE_WARNING_AUTOINDEX: zName = "SQLITE_WARNING_AUTOINDEX"; break; case SQLITE_DONE: zName = "SQLITE_DONE"; break; @@ -137024,10 +182734,10 @@ SQLITE_PRIVATE const char *sqlite3ErrName(int rc){ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ static const char* const aMsg[] = { /* SQLITE_OK */ "not an error", - /* SQLITE_ERROR */ "SQL logic error or missing database", + /* SQLITE_ERROR */ "SQL logic error", /* SQLITE_INTERNAL */ 0, /* SQLITE_PERM */ "access permission denied", - /* SQLITE_ABORT */ "callback requested query abort", + /* SQLITE_ABORT */ "query aborted", /* SQLITE_BUSY */ "database is locked", /* SQLITE_LOCKED */ "database table is locked", /* SQLITE_NOMEM */ "out of memory", @@ -137039,17 +182749,23 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ /* SQLITE_FULL */ "database or disk is full", /* SQLITE_CANTOPEN */ "unable to open database file", /* SQLITE_PROTOCOL */ "locking protocol", - /* SQLITE_EMPTY */ "table contains no data", + /* SQLITE_EMPTY */ 0, /* SQLITE_SCHEMA */ "database schema has changed", /* SQLITE_TOOBIG */ "string or blob too big", /* SQLITE_CONSTRAINT */ "constraint failed", /* SQLITE_MISMATCH */ "datatype mismatch", - /* SQLITE_MISUSE */ "library routine called out of sequence", + /* SQLITE_MISUSE */ "bad parameter or other API misuse", +#ifdef SQLITE_DISABLE_LFS /* SQLITE_NOLFS */ "large file support is disabled", +#else + /* SQLITE_NOLFS */ 0, +#endif /* SQLITE_AUTH */ "authorization denied", - /* SQLITE_FORMAT */ "auxiliary database format error", - /* SQLITE_RANGE */ "bind or column index out of range", - /* SQLITE_NOTADB */ "file is encrypted or is not a database", + /* SQLITE_FORMAT */ 0, + /* SQLITE_RANGE */ "column index out of range", + /* SQLITE_NOTADB */ "file is not a database", + /* SQLITE_NOTICE */ "notification message", + /* SQLITE_WARNING */ "warning message", }; const char *zErr = "unknown error"; switch( rc ){ @@ -137057,6 +182773,14 @@ SQLITE_PRIVATE const char *sqlite3ErrStr(int rc){ zErr = "abort due to ROLLBACK"; break; } + case SQLITE_ROW: { + zErr = "another row available"; + break; + } + case SQLITE_DONE: { + zErr = "no more rows available"; + break; + } default: { rc &= 0xff; if( ALWAYS(rc>=0) && rcbusyTimeout; + int tmout = db->busyTimeout; int delay, prior; assert( count>=0 ); @@ -137096,16 +182825,18 @@ static int sqliteDefaultBusyCallback( delay = delays[NDELAY-1]; prior = totals[NDELAY-1] + delay*(count-(NDELAY-1)); } - if( prior + delay > timeout ){ - delay = timeout - prior; + if( prior + delay > tmout ){ + delay = tmout - prior; if( delay<=0 ) return 0; } sqlite3OsSleep(db->pVfs, delay*1000); return 1; #else + /* This case for unix systems that lack usleep() support. Sleeping + ** must be done in increments of whole seconds */ sqlite3 *db = (sqlite3 *)ptr; - int timeout = ((sqlite3 *)ptr)->busyTimeout; - if( (count+1)*1000 > timeout ){ + int tmout = ((sqlite3 *)ptr)->busyTimeout; + if( (count+1)*1000 > tmout ){ return 0; } sqlite3OsSleep(db->pVfs, 1000000); @@ -137116,27 +182847,29 @@ static int sqliteDefaultBusyCallback( /* ** Invoke the given busy handler. ** -** This routine is called when an operation failed with a lock. +** This routine is called when an operation failed to acquire a +** lock on VFS file pFile. +** ** If this routine returns non-zero, the lock is retried. If it ** returns 0, the operation aborts with an SQLITE_BUSY error. */ SQLITE_PRIVATE int sqlite3InvokeBusyHandler(BusyHandler *p){ int rc; - if( NEVER(p==0) || p->xFunc==0 || p->nBusy<0 ) return 0; - rc = p->xFunc(p->pArg, p->nBusy); + if( p->xBusyHandler==0 || p->nBusy<0 ) return 0; + rc = p->xBusyHandler(p->pBusyArg, p->nBusy); if( rc==0 ){ p->nBusy = -1; }else{ p->nBusy++; } - return rc; + return rc; } /* ** This routine sets the busy callback for an Sqlite database to the ** given callback function with the given argument. */ -SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( +SQLITE_API int sqlite3_busy_handler( sqlite3 *db, int (*xBusy)(void*,int), void *pArg @@ -137145,8 +182878,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif sqlite3_mutex_enter(db->mutex); - db->busyHandler.xFunc = xBusy; - db->busyHandler.pArg = pArg; + db->busyHandler.xBusyHandler = xBusy; + db->busyHandler.pBusyArg = pArg; db->busyHandler.nBusy = 0; db->busyTimeout = 0; sqlite3_mutex_leave(db->mutex); @@ -137159,10 +182892,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_busy_handler( ** given callback function with the given argument. The progress callback will ** be invoked every nOps opcodes. */ -SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler( - sqlite3 *db, +SQLITE_API void sqlite3_progress_handler( + sqlite3 *db, int nOps, - int (*xProgress)(void*), + int (*xProgress)(void*), void *pArg ){ #ifdef SQLITE_ENABLE_API_ARMOR @@ -137190,12 +182923,13 @@ SQLITE_API void SQLITE_STDCALL sqlite3_progress_handler( ** This routine installs a default busy handler that waits for the ** specified number of milliseconds before returning 0. */ -SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){ +SQLITE_API int sqlite3_busy_timeout(sqlite3 *db, int ms){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif if( ms>0 ){ - sqlite3_busy_handler(db, sqliteDefaultBusyCallback, (void*)db); + sqlite3_busy_handler(db, (int(*)(void*,int))sqliteDefaultBusyCallback, + (void*)db); db->busyTimeout = ms; }else{ sqlite3_busy_handler(db, 0, 0); @@ -137206,22 +182940,39 @@ SQLITE_API int SQLITE_STDCALL sqlite3_busy_timeout(sqlite3 *db, int ms){ /* ** Cause any pending operation to stop at its earliest opportunity. */ -SQLITE_API void SQLITE_STDCALL sqlite3_interrupt(sqlite3 *db){ +SQLITE_API void sqlite3_interrupt(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR - if( !sqlite3SafetyCheckOk(db) ){ + if( !sqlite3SafetyCheckOk(db) + && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) + ){ (void)SQLITE_MISUSE_BKPT; return; } #endif - db->u1.isInterrupted = 1; + AtomicStore(&db->u1.isInterrupted, 1); } +/* +** Return true or false depending on whether or not an interrupt is +** pending on connection db. +*/ +SQLITE_API int sqlite3_is_interrupted(sqlite3 *db){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) + && (db==0 || db->eOpenState!=SQLITE_STATE_ZOMBIE) + ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + return AtomicLoad(&db->u1.isInterrupted)!=0; +} /* ** This function is exactly the same as sqlite3_create_function(), except ** that it is designed to be called by internal code. The difference is ** that if a malloc() fails in sqlite3_create_function(), an error code -** is returned and the mallocFailed flag cleared. +** is returned and the mallocFailed flag cleared. */ SQLITE_PRIVATE int sqlite3CreateFunc( sqlite3 *db, @@ -137232,26 +182983,38 @@ SQLITE_PRIVATE int sqlite3CreateFunc( void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), void (*xStep)(sqlite3_context*,int,sqlite3_value **), void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), FuncDestructor *pDestructor ){ FuncDef *p; - int nName; int extraFlags; assert( sqlite3_mutex_held(db->mutex) ); - if( zFunctionName==0 || - (xSFunc && (xFinal || xStep)) || - (!xSFunc && (xFinal && !xStep)) || - (!xSFunc && (!xFinal && xStep)) || - (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) || - (255<(nName = sqlite3Strlen30( zFunctionName))) ){ + assert( xValue==0 || xSFunc==0 ); + if( zFunctionName==0 /* Must have a valid name */ + || (xSFunc!=0 && xFinal!=0) /* Not both xSFunc and xFinal */ + || ((xFinal==0)!=(xStep==0)) /* Both or neither of xFinal and xStep */ + || ((xValue==0)!=(xInverse==0)) /* Both or neither of xValue, xInverse */ + || (nArg<-1 || nArg>SQLITE_MAX_FUNCTION_ARG) + || (255funcFlags & SQLITE_FUNC_ENCMASK)==enc && p->nArg==nArg ){ + if( p && (p->funcFlags & SQLITE_FUNC_ENCMASK)==(u32)enc && p->nArg==nArg ){ if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify user-function due to active statements"); assert( !db->mallocFailed ); return SQLITE_BUSY; }else{ - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); } + }else if( xSFunc==0 && xFinal==0 ){ + /* Trying to delete a function that does not exist. This is a no-op. + ** https://sqlite.org/forum/forumpost/726219164b */ + return SQLITE_OK; } p = sqlite3FindFunction(db, zFunctionName, nArg, (u8)enc, 1); @@ -137312,40 +183092,35 @@ SQLITE_PRIVATE int sqlite3CreateFunc( p->u.pDestructor = pDestructor; p->funcFlags = (p->funcFlags & SQLITE_FUNC_ENCMASK) | extraFlags; testcase( p->funcFlags & SQLITE_DETERMINISTIC ); + testcase( p->funcFlags & SQLITE_DIRECTONLY ); p->xSFunc = xSFunc ? xSFunc : xStep; p->xFinalize = xFinal; + p->xValue = xValue; + p->xInverse = xInverse; p->pUserData = pUserData; p->nArg = (u16)nArg; return SQLITE_OK; } /* -** Create new user functions. +** Worker function used by utf-8 APIs that create new functions: +** +** sqlite3_create_function() +** sqlite3_create_function_v2() +** sqlite3_create_window_function() */ -SQLITE_API int SQLITE_STDCALL sqlite3_create_function( +static int createFunctionApi( sqlite3 *db, const char *zFunc, int nArg, int enc, void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), - void (*xFinal)(sqlite3_context*) -){ - return sqlite3_create_function_v2(db, zFunc, nArg, enc, p, xSFunc, xStep, - xFinal, 0); -} - -SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( - sqlite3 *db, - const char *zFunc, - int nArg, - int enc, - void *p, - void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), - void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xSFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*), - void (*xDestroy)(void *) + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) ){ int rc = SQLITE_ERROR; FuncDestructor *pArg = 0; @@ -137357,19 +183132,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( #endif sqlite3_mutex_enter(db->mutex); if( xDestroy ){ - pArg = (FuncDestructor *)sqlite3DbMallocZero(db, sizeof(FuncDestructor)); + pArg = (FuncDestructor *)sqlite3Malloc(sizeof(FuncDestructor)); if( !pArg ){ + sqlite3OomFault(db); xDestroy(p); goto out; } + pArg->nRef = 0; pArg->xDestroy = xDestroy; pArg->pUserData = p; } - rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, xSFunc, xStep, xFinal, pArg); + rc = sqlite3CreateFunc(db, zFunc, nArg, enc, p, + xSFunc, xStep, xFinal, xValue, xInverse, pArg + ); if( pArg && pArg->nRef==0 ){ - assert( rc!=SQLITE_OK ); + assert( rc!=SQLITE_OK || (xStep==0 && xFinal==0) ); xDestroy(p); - sqlite3DbFree(db, pArg); + sqlite3_free(pArg); } out: @@ -137378,8 +183157,54 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function_v2( return rc; } +/* +** Create new user functions. +*/ +SQLITE_API int sqlite3_create_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, 0); +} +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xSFunc)(sqlite3_context*,int,sqlite3_value **), + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, xSFunc, xStep, + xFinal, 0, 0, xDestroy); +} +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunc, + int nArg, + int enc, + void *p, + void (*xStep)(sqlite3_context*,int,sqlite3_value **), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value **), + void (*xDestroy)(void *) +){ + return createFunctionApi(db, zFunc, nArg, enc, p, 0, xStep, + xFinal, xValue, xInverse, xDestroy); +} + #ifndef SQLITE_OMIT_UTF16 -SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( +SQLITE_API int sqlite3_create_function16( sqlite3 *db, const void *zFunctionName, int nArg, @@ -137398,7 +183223,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( sqlite3_mutex_enter(db->mutex); assert( !db->mallocFailed ); zFunc8 = sqlite3Utf16to8(db, zFunctionName, -1, SQLITE_UTF16NATIVE); - rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0); + rc = sqlite3CreateFunc(db, zFunc8, nArg, eTextRep, p, xSFunc,xStep,xFinal,0,0,0); sqlite3DbFree(db, zFunc8); rc = sqlite3ApiExit(db, rc); sqlite3_mutex_leave(db->mutex); @@ -137407,24 +183232,47 @@ SQLITE_API int SQLITE_STDCALL sqlite3_create_function16( #endif +/* +** The following is the implementation of an SQL function that always +** fails with an error message stating that the function is used in the +** wrong context. The sqlite3_overload_function() API might construct +** SQL function that use this routine so that the functions will exist +** for name resolution but are actually overloaded by the xFindFunction +** method of virtual tables. +*/ +static void sqlite3InvalidFunction( + sqlite3_context *context, /* The function calling context */ + int NotUsed, /* Number of arguments to the function */ + sqlite3_value **NotUsed2 /* Value of each argument */ +){ + const char *zName = (const char*)sqlite3_user_data(context); + char *zErr; + UNUSED_PARAMETER2(NotUsed, NotUsed2); + zErr = sqlite3_mprintf( + "unable to use function %s in the requested context", zName); + sqlite3_result_error(context, zErr, -1); + sqlite3_free(zErr); +} + /* ** Declare that a function has been overloaded by a virtual table. ** ** If the function already exists as a regular global function, then ** this routine is a no-op. If the function does not exist, then create -** a new one that always throws a run-time error. +** a new one that always throws a run-time error. ** ** When virtual tables intend to provide an overloaded function, they ** should call this routine to make sure the global function exists. ** A global function must exist in order for name resolution to work ** properly. */ -SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( +SQLITE_API int sqlite3_overload_function( sqlite3 *db, const char *zName, int nArg ){ - int rc = SQLITE_OK; + int rc; + char *zCopy; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) || zName==0 || nArg<-2 ){ @@ -137432,25 +183280,26 @@ SQLITE_API int SQLITE_STDCALL sqlite3_overload_function( } #endif sqlite3_mutex_enter(db->mutex); - if( sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)==0 ){ - rc = sqlite3CreateFunc(db, zName, nArg, SQLITE_UTF8, - 0, sqlite3InvalidFunction, 0, 0, 0); - } - rc = sqlite3ApiExit(db, rc); + rc = sqlite3FindFunction(db, zName, nArg, SQLITE_UTF8, 0)!=0; sqlite3_mutex_leave(db->mutex); - return rc; + if( rc ) return SQLITE_OK; + zCopy = sqlite3_mprintf("%s", zName); + if( zCopy==0 ) return SQLITE_NOMEM; + return sqlite3_create_function_v2(db, zName, nArg, SQLITE_UTF8, + zCopy, sqlite3InvalidFunction, 0, 0, sqlite3_free); } #ifndef SQLITE_OMIT_TRACE /* ** Register a trace function. The pArg from the previously registered trace -** is returned. +** is returned. ** ** A NULL trace function means that no tracing is executes. A non-NULL ** trace is a pointer to a function that is invoked at the start of each ** SQL statement. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*,const char*), void *pArg){ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API void *sqlite3_trace(sqlite3 *db, void(*xTrace)(void*,const char*), void *pArg){ void *pOld; #ifdef SQLITE_ENABLE_API_ARMOR @@ -137461,20 +183310,47 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_trace(sqlite3 *db, void (*xTrace)(void*, #endif sqlite3_mutex_enter(db->mutex); pOld = db->pTraceArg; - db->xTrace = xTrace; + db->mTrace = xTrace ? SQLITE_TRACE_LEGACY : 0; + db->trace.xLegacy = xTrace; db->pTraceArg = pArg; sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ + +/* Register a trace callback using the version-2 interface. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3 *db, /* Trace this connection */ + unsigned mTrace, /* Mask of events to be traced */ + int(*xTrace)(unsigned,void*,void*,void*), /* Callback to invoke */ + void *pArg /* Context */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( mTrace==0 ) xTrace = 0; + if( xTrace==0 ) mTrace = 0; + db->mTrace = mTrace; + db->trace.xV2 = xTrace; + db->pTraceArg = pArg; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + +#ifndef SQLITE_OMIT_DEPRECATED /* -** Register a profile function. The pArg from the previously registered -** profile function is returned. +** Register a profile function. The pArg from the previously registered +** profile function is returned. ** ** A NULL profile function means that no profiling is executes. A non-NULL ** profile is a pointer to a function that is invoked at the conclusion of ** each SQL statement that is run. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_profile( +SQLITE_API void *sqlite3_profile( sqlite3 *db, void (*xProfile)(void*,const char*,sqlite_uint64), void *pArg @@ -137491,9 +183367,12 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_profile( pOld = db->pProfileArg; db->xProfile = xProfile; db->pProfileArg = pArg; + db->mTrace &= SQLITE_TRACE_NONLEGACY_MASK; + if( db->xProfile ) db->mTrace |= SQLITE_TRACE_XPROFILE; sqlite3_mutex_leave(db->mutex); return pOld; } +#endif /* SQLITE_OMIT_DEPRECATED */ #endif /* SQLITE_OMIT_TRACE */ /* @@ -137501,7 +183380,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_profile( ** If the invoked function returns non-zero, then the commit becomes a ** rollback. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook( +SQLITE_API void *sqlite3_commit_hook( sqlite3 *db, /* Attach the hook to this database */ int (*xCallback)(void*), /* Function to invoke on each commit */ void *pArg /* Argument to the function */ @@ -137526,7 +183405,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_commit_hook( ** Register a callback to be invoked each time a row is updated, ** inserted or deleted using this database connection. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( +SQLITE_API void *sqlite3_update_hook( sqlite3 *db, /* Attach the hook to this database */ void (*xCallback)(void*,int,char const *,char const *,sqlite_int64), void *pArg /* Argument to the function */ @@ -137551,7 +183430,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_update_hook( ** Register a callback to be invoked each time a transaction is rolled ** back by this database connection. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook( +SQLITE_API void *sqlite3_rollback_hook( sqlite3 *db, /* Attach the hook to this database */ void (*xCallback)(void*), /* Callback function */ void *pArg /* Argument to the function */ @@ -137577,13 +183456,19 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_rollback_hook( ** Register a callback to be invoked each time a row is updated, ** inserted or deleted using this database connection. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_preupdate_hook( +SQLITE_API void *sqlite3_preupdate_hook( sqlite3 *db, /* Attach the hook to this database */ void(*xCallback)( /* Callback function */ void*,sqlite3*,int,char const*,char const*,sqlite3_int64,sqlite3_int64), void *pArg /* First callback argument */ ){ void *pRet; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( db==0 ){ + return 0; + } +#endif sqlite3_mutex_enter(db->mutex); pRet = db->pPreUpdateArg; db->xPreUpdateCallback = xCallback; @@ -137593,13 +183478,41 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_preupdate_hook( } #endif /* SQLITE_ENABLE_PREUPDATE_HOOK */ +/* +** Register a function to be invoked prior to each autovacuum that +** determines the number of pages to vacuum. +*/ +SQLITE_API int sqlite3_autovacuum_pages( + sqlite3 *db, /* Attach the hook to this database */ + unsigned int (*xCallback)(void*,const char*,u32,u32,u32), + void *pArg, /* Argument to the function */ + void (*xDestructor)(void*) /* Destructor for pArg */ +){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + if( xDestructor ) xDestructor(pArg); + return SQLITE_MISUSE_BKPT; + } +#endif + sqlite3_mutex_enter(db->mutex); + if( db->xAutovacDestr ){ + db->xAutovacDestr(db->pAutovacPagesArg); + } + db->xAutovacPages = xCallback; + db->pAutovacPagesArg = pArg; + db->xAutovacDestr = xDestructor; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + + #ifndef SQLITE_OMIT_WAL /* ** The sqlite3_wal_hook() callback registered by sqlite3_wal_autocheckpoint(). ** Invoke sqlite3_wal_checkpoint if the number of frames in the log file ** is greater than sqlite3.pWalArg cast to an integer (the value configured by ** wal_autocheckpoint()). -*/ +*/ SQLITE_PRIVATE int sqlite3WalDefaultHook( void *pClientData, /* Argument */ sqlite3 *db, /* Connection */ @@ -137626,7 +183539,7 @@ SQLITE_PRIVATE int sqlite3WalDefaultHook( ** using sqlite3_wal_hook() disables the automatic checkpoint mechanism ** configured by this function. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame){ #ifdef SQLITE_OMIT_WAL UNUSED_PARAMETER(db); UNUSED_PARAMETER(nFrame); @@ -137647,7 +183560,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_autocheckpoint(sqlite3 *db, int nFrame ** Register a callback to be invoked each time a transaction is written ** into the write-ahead-log by this database connection. */ -SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( +SQLITE_API void *sqlite3_wal_hook( sqlite3 *db, /* Attach the hook to this db handle */ int(*xCallback)(void *, sqlite3*, const char*, int), void *pArg /* First argument passed to xCallback() */ @@ -137674,7 +183587,7 @@ SQLITE_API void *SQLITE_STDCALL sqlite3_wal_hook( /* ** Checkpoint database zDb. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( +SQLITE_API int sqlite3_wal_checkpoint_v2( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of attached database (or NULL) */ int eMode, /* SQLITE_CHECKPOINT_* value */ @@ -137685,7 +183598,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( return SQLITE_OK; #else int rc; /* Return code */ - int iDb = SQLITE_MAX_ATTACHED; /* sqlite3.aDb[] index of db to checkpoint */ + int iDb; /* Schema to checkpoint */ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; @@ -137702,12 +183615,14 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( if( eModeSQLITE_CHECKPOINT_TRUNCATE ){ /* EVIDENCE-OF: R-03996-12088 The M parameter must be a valid checkpoint ** mode: */ - return SQLITE_MISUSE; + return SQLITE_MISUSE_BKPT; } sqlite3_mutex_enter(db->mutex); if( zDb && zDb[0] ){ iDb = sqlite3FindDbName(db, zDb); + }else{ + iDb = SQLITE_MAX_DB; /* This means process all schemas */ } if( iDb<0 ){ rc = SQLITE_ERROR; @@ -137718,6 +183633,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( sqlite3Error(db, rc); } rc = sqlite3ApiExit(db, rc); + + /* If there are no active statements, clear the interrupt flag at this + ** point. */ + if( db->nVdbeActive==0 ){ + AtomicStore(&db->u1.isInterrupted, 0); + } + sqlite3_mutex_leave(db->mutex); return rc; #endif @@ -137726,10 +183648,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint_v2( /* ** Checkpoint database zDb. If zDb is NULL, or if the buffer zDb points -** to contains a zero-length string, all attached databases are +** to contains a zero-length string, all attached databases are ** checkpointed. */ -SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb){ /* EVIDENCE-OF: R-41613-20553 The sqlite3_wal_checkpoint(D,X) is equivalent to ** sqlite3_wal_checkpoint_v2(D,X,SQLITE_CHECKPOINT_PASSIVE,0,0). */ return sqlite3_wal_checkpoint_v2(db,zDb,SQLITE_CHECKPOINT_PASSIVE,0,0); @@ -137740,20 +183662,21 @@ SQLITE_API int SQLITE_STDCALL sqlite3_wal_checkpoint(sqlite3 *db, const char *zD ** Run a checkpoint on database iDb. This is a no-op if database iDb is ** not currently open in WAL mode. ** -** If a transaction is open on the database being checkpointed, this -** function returns SQLITE_LOCKED and a checkpoint is not attempted. If -** an error occurs while running the checkpoint, an SQLite error code is +** If a transaction is open on the database being checkpointed, this +** function returns SQLITE_LOCKED and a checkpoint is not attempted. If +** an error occurs while running the checkpoint, an SQLite error code is ** returned (i.e. SQLITE_IOERR). Otherwise, SQLITE_OK. ** ** The mutex on database handle db should be held by the caller. The mutex ** associated with the specific b-tree being checkpointed is taken by ** this function while the checkpoint is running. ** -** If iDb is passed SQLITE_MAX_ATTACHED, then all attached databases are +** If iDb is passed SQLITE_MAX_DB then all attached databases are ** checkpointed. If an error is encountered it is returned immediately - ** no attempt is made to checkpoint any remaining databases. ** -** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL or RESTART. +** Parameter eMode is one of SQLITE_CHECKPOINT_PASSIVE, FULL, RESTART +** or TRUNCATE. */ SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog, int *pnCkpt){ int rc = SQLITE_OK; /* Return code */ @@ -137763,9 +183686,11 @@ SQLITE_PRIVATE int sqlite3Checkpoint(sqlite3 *db, int iDb, int eMode, int *pnLog assert( sqlite3_mutex_held(db->mutex) ); assert( !pnLog || *pnLog==-1 ); assert( !pnCkpt || *pnCkpt==-1 ); + testcase( iDb==SQLITE_MAX_ATTACHED ); /* See forum post a006d86f72 */ + testcase( iDb==SQLITE_MAX_DB ); for(i=0; inDb && rc==SQLITE_OK; i++){ - if( i==iDb || iDb==SQLITE_MAX_ATTACHED ){ + if( i==iDb || iDb==SQLITE_MAX_DB ){ rc = sqlite3BtreeCheckpoint(db->aDb[i].pBt, eMode, pnLog, pnCkpt); pnLog = 0; pnCkpt = 0; @@ -137820,7 +183745,7 @@ SQLITE_PRIVATE int sqlite3TempInMemory(const sqlite3 *db){ ** Return UTF-8 encoded English language explanation of the most recent ** error. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ +SQLITE_API const char *sqlite3_errmsg(sqlite3 *db){ const char *z; if( !db ){ return sqlite3ErrStr(SQLITE_NOMEM_BKPT); @@ -137833,7 +183758,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ z = sqlite3ErrStr(SQLITE_NOMEM_BKPT); }else{ testcase( db->pErr==0 ); - z = (char*)sqlite3_value_text(db->pErr); + z = db->errCode ? (char*)sqlite3_value_text(db->pErr) : 0; assert( !db->mallocFailed ); if( z==0 ){ z = sqlite3ErrStr(db->errCode); @@ -137843,22 +183768,32 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_errmsg(sqlite3 *db){ return z; } +/* +** Return the byte offset of the most recent error +*/ +SQLITE_API int sqlite3_error_offset(sqlite3 *db){ + int iOffset = -1; + if( db && sqlite3SafetyCheckSickOrOk(db) && db->errCode ){ + sqlite3_mutex_enter(db->mutex); + iOffset = db->errByteOffset; + sqlite3_mutex_leave(db->mutex); + } + return iOffset; +} + #ifndef SQLITE_OMIT_UTF16 /* ** Return UTF-16 encoded English language explanation of the most recent ** error. */ -SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ +SQLITE_API const void *sqlite3_errmsg16(sqlite3 *db){ static const u16 outOfMem[] = { 'o', 'u', 't', ' ', 'o', 'f', ' ', 'm', 'e', 'm', 'o', 'r', 'y', 0 }; static const u16 misuse[] = { - 'l', 'i', 'b', 'r', 'a', 'r', 'y', ' ', - 'r', 'o', 'u', 't', 'i', 'n', 'e', ' ', - 'c', 'a', 'l', 'l', 'e', 'd', ' ', - 'o', 'u', 't', ' ', - 'o', 'f', ' ', - 's', 'e', 'q', 'u', 'e', 'n', 'c', 'e', 0 + 'b', 'a', 'd', ' ', 'p', 'a', 'r', 'a', 'm', 'e', 't', 'e', 'r', ' ', + 'o', 'r', ' ', 'o', 't', 'h', 'e', 'r', ' ', 'A', 'P', 'I', ' ', + 'm', 'i', 's', 'u', 's', 'e', 0 }; const void *z; @@ -137893,7 +183828,7 @@ SQLITE_API const void *SQLITE_STDCALL sqlite3_errmsg16(sqlite3 *db){ ** Return the most recent error code generated by an SQLite routine. If NULL is ** passed to this function, we assume a malloc() failed during sqlite3_open(). */ -SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ +SQLITE_API int sqlite3_errcode(sqlite3 *db){ if( db && !sqlite3SafetyCheckSickOrOk(db) ){ return SQLITE_MISUSE_BKPT; } @@ -137902,7 +183837,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_errcode(sqlite3 *db){ } return db->errCode & db->errMask; } -SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db){ if( db && !sqlite3SafetyCheckSickOrOk(db) ){ return SQLITE_MISUSE_BKPT; } @@ -137911,16 +183846,16 @@ SQLITE_API int SQLITE_STDCALL sqlite3_extended_errcode(sqlite3 *db){ } return db->errCode; } -SQLITE_API int SQLITE_STDCALL sqlite3_system_errno(sqlite3 *db){ +SQLITE_API int sqlite3_system_errno(sqlite3 *db){ return db ? db->iSysErrno : 0; -} +} /* ** Return a string that describes the kind of error specified in the ** argument. For now, this simply calls the internal sqlite3ErrStr() ** function. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){ +SQLITE_API const char *sqlite3_errstr(int rc){ return sqlite3ErrStr(rc); } @@ -137930,7 +183865,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_errstr(int rc){ */ static int createCollation( sqlite3* db, - const char *zName, + const char *zName, u8 enc, void* pCtx, int(*xCompare)(void*,int,const void*,int,const void*), @@ -137938,7 +183873,7 @@ static int createCollation( ){ CollSeq *pColl; int enc2; - + assert( sqlite3_mutex_held(db->mutex) ); /* If SQLITE_UTF16 is specified as the encoding type, transform this @@ -137955,25 +183890,25 @@ static int createCollation( return SQLITE_MISUSE_BKPT; } - /* Check if this call is removing or replacing an existing collation + /* Check if this call is removing or replacing an existing collation ** sequence. If so, and there are active VMs, return busy. If there ** are no active VMs, invalidate any pre-compiled statements. */ pColl = sqlite3FindCollSeq(db, (u8)enc2, zName, 0); if( pColl && pColl->xCmp ){ if( db->nVdbeActive ){ - sqlite3ErrorWithMsg(db, SQLITE_BUSY, + sqlite3ErrorWithMsg(db, SQLITE_BUSY, "unable to delete/modify collation sequence due to active statements"); return SQLITE_BUSY; } - sqlite3ExpirePreparedStatements(db); + sqlite3ExpirePreparedStatements(db, 0); /* If collation sequence pColl was created directly by a call to ** sqlite3_create_collation, and not generated by synthCollSeq(), ** then any copies made by synthCollSeq() need to be invalidated. ** Also, collation destructor - CollSeq.xDel() - function may need ** to be called. - */ + */ if( (pColl->enc & ~SQLITE_UTF16_ALIGNED)==enc2 ){ CollSeq *aColl = sqlite3HashFind(&db->aCollSeq, zName); int j; @@ -138068,7 +184003,7 @@ static const int aHardLimit[] = { ** It merely prevents new constructs that exceed the limit ** from forming. */ -SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ +SQLITE_API int sqlite3_limit(sqlite3 *db, int limitId, int newLimit){ int oldLimit; #ifdef SQLITE_ENABLE_API_ARMOR @@ -138106,6 +184041,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLim if( newLimit>=0 ){ /* IMP: R-52476-28732 */ if( newLimit>aHardLimit[limitId] ){ newLimit = aHardLimit[limitId]; /* IMP: R-51463-25634 */ + }else if( newLimit<1 && limitId==SQLITE_LIMIT_LENGTH ){ + newLimit = 1; } db->aLimit[limitId] = newLimit; } @@ -138122,17 +184059,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_limit(sqlite3 *db, int limitId, int newLim ** query parameter. The second argument contains the URI (or non-URI filename) ** itself. When this function is called the *pFlags variable should contain ** the default flags to open the database handle with. The value stored in -** *pFlags may be updated before returning if the URI filename contains +** *pFlags may be updated before returning if the URI filename contains ** "cache=xxx" or "mode=xxx" query parameters. ** ** If successful, SQLITE_OK is returned. In this case *ppVfs is set to point to ** the VFS that should be used to open the database file. *pzFile is set to -** point to a buffer containing the name of the file to open. It is the -** responsibility of the caller to eventually call sqlite3_free() to release -** this buffer. +** point to a buffer containing the name of the file to open. The value +** stored in *pzFile is a database name acceptable to sqlite3_uri_parameter() +** and is in the same format as names created using sqlite3_create_filename(). +** The caller must invoke sqlite3_free_filename() (not sqlite3_free()!) on +** the value returned in *pzFile to avoid a memory leak. ** ** If an error occurs, then an SQLite error code is returned and *pzErrMsg -** may be set to point to a buffer containing an English language error +** may be set to point to a buffer containing an English language error ** message. It is the responsibility of the caller to eventually release ** this buffer by calling sqlite3_free(). */ @@ -138140,7 +184079,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( const char *zDefaultVfs, /* VFS to use if no "vfs=xxx" query option */ const char *zUri, /* Nul-terminated URI to parse */ unsigned int *pFlags, /* IN/OUT: SQLITE_OPEN_XXX flags */ - sqlite3_vfs **ppVfs, /* OUT: VFS to use */ + sqlite3_vfs **ppVfs, /* OUT: VFS to use */ char **pzFile, /* OUT: Filename component of URI */ char **pzErrMsg /* OUT: Error message (if rc!=SQLITE_OK) */ ){ @@ -138153,17 +184092,17 @@ SQLITE_PRIVATE int sqlite3ParseUri( assert( *pzErrMsg==0 ); - if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ - || sqlite3GlobalConfig.bOpenUri) /* IMP: R-51689-46548 */ - && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ + if( ((flags & SQLITE_OPEN_URI) /* IMP: R-48725-32206 */ + || AtomicLoad(&sqlite3GlobalConfig.bOpenUri)) /* IMP: R-51689-46548 */ + && nUri>=5 && memcmp(zUri, "file:", 5)==0 /* IMP: R-57884-37496 */ ){ char *zOpt; int eState; /* Parser state when parsing URI */ int iIn; /* Input character index */ int iOut = 0; /* Output character index */ - u64 nByte = nUri+2; /* Bytes of space to allocate */ + u64 nByte = nUri+8; /* Bytes of space to allocate */ - /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen + /* Make sure the SQLITE_OPEN_URI flag is set to indicate to the VFS xOpen ** method that there may be extra parameters following the file-name. */ flags |= SQLITE_OPEN_URI; @@ -138171,6 +184110,9 @@ SQLITE_PRIVATE int sqlite3ParseUri( zFile = sqlite3_malloc64(nByte); if( !zFile ) return SQLITE_NOMEM_BKPT; + memset(zFile, 0, 4); /* 4-byte of 0x00 is the start of DB name marker */ + zFile += 4; + iIn = 5; #ifdef SQLITE_ALLOW_URI_AUTHORITY if( strncmp(zUri+5, "///", 3)==0 ){ @@ -138178,7 +184120,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( /* The following condition causes URIs with five leading / characters ** like file://///host/path to be converted into UNCs like //host/path. ** The correct URI for that UNC has only two or four leading / characters - ** file://host/path or file:////host/path. But 5 leading slashes is a + ** file://host/path or file:////host/path. But 5 leading slashes is a ** common error, we are told, so we handle it as a special case. */ if( strncmp(zUri+7, "///", 3)==0 ){ iIn++; } }else if( strncmp(zUri+5, "//localhost/", 12)==0 ){ @@ -138190,7 +184132,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( iIn = 7; while( zUri[iIn] && zUri[iIn]!='/' ) iIn++; if( iIn!=7 && (iIn!=16 || memcmp("localhost", &zUri[7], 9)) ){ - *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", + *pzErrMsg = sqlite3_mprintf("invalid uri authority: %.*s", iIn-7, &zUri[7]); rc = SQLITE_ERROR; goto parse_uri_out; @@ -138198,8 +184140,8 @@ SQLITE_PRIVATE int sqlite3ParseUri( } #endif - /* Copy the filename and any query parameters into the zFile buffer. - ** Decode %HH escape codes along the way. + /* Copy the filename and any query parameters into the zFile buffer. + ** Decode %HH escape codes along the way. ** ** Within this loop, variable eState may be set to 0, 1 or 2, depending ** on the parsing context. As follows: @@ -138211,20 +184153,21 @@ SQLITE_PRIVATE int sqlite3ParseUri( eState = 0; while( (c = zUri[iIn])!=0 && c!='#' ){ iIn++; - if( c=='%' - && sqlite3Isxdigit(zUri[iIn]) - && sqlite3Isxdigit(zUri[iIn+1]) + if( c=='%' + && sqlite3Isxdigit(zUri[iIn]) + && sqlite3Isxdigit(zUri[iIn+1]) ){ int octet = (sqlite3HexToInt(zUri[iIn++]) << 4); octet += sqlite3HexToInt(zUri[iIn++]); assert( octet>=0 && octet<256 ); if( octet==0 ){ +#ifndef SQLITE_ENABLE_URI_00_ERROR /* This branch is taken when "%00" appears within the URI. In this ** case we ignore all text in the remainder of the path, name or ** value currently being parsed. So ignore the current character ** and skip to the next "?", "=" or "&", as appropriate. */ - while( (c = zUri[iIn])!=0 && c!='#' + while( (c = zUri[iIn])!=0 && c!='#' && (eState!=0 || c!='?') && (eState!=1 || (c!='=' && c!='&')) && (eState!=2 || c!='&') @@ -138232,6 +184175,12 @@ SQLITE_PRIVATE int sqlite3ParseUri( iIn++; } continue; +#else + /* If ENABLE_URI_00_ERROR is defined, "%00" in a URI is an error. */ + *pzErrMsg = sqlite3_mprintf("unexpected %%00 in uri"); + rc = SQLITE_ERROR; + goto parse_uri_out; +#endif } c = octet; }else if( eState==1 && (c=='&' || c=='=') ){ @@ -138253,10 +184202,9 @@ SQLITE_PRIVATE int sqlite3ParseUri( zFile[iOut++] = c; } if( eState==1 ) zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; - zFile[iOut++] = '\0'; + memset(zFile+iOut, 0, 4); /* end-of-options + empty journal filenames */ - /* Check if there were any options specified that should be interpreted + /* Check if there were any options specified that should be interpreted ** here. Options that are interpreted here include "vfs" and those that ** correspond to flags that may be passed to the sqlite3_open_v2() ** method. */ @@ -138292,7 +184240,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( if( nOpt==4 && memcmp("mode", zOpt, 4)==0 ){ static struct OpenMode aOpenMode[] = { { "ro", SQLITE_OPEN_READONLY }, - { "rw", SQLITE_OPEN_READWRITE }, + { "rw", SQLITE_OPEN_READWRITE }, { "rwc", SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE }, { "memory", SQLITE_OPEN_MEMORY }, { 0, 0 } @@ -138334,11 +184282,14 @@ SQLITE_PRIVATE int sqlite3ParseUri( } }else{ - zFile = sqlite3_malloc64(nUri+2); + zFile = sqlite3_malloc64(nUri+8); if( !zFile ) return SQLITE_NOMEM_BKPT; - memcpy(zFile, zUri, nUri); - zFile[nUri] = '\0'; - zFile[nUri+1] = '\0'; + memset(zFile, 0, 4); + zFile += 4; + if( nUri ){ + memcpy(zFile, zUri, nUri); + } + memset(zFile+nUri, 0, 4); flags &= ~SQLITE_OPEN_URI; } @@ -138349,7 +184300,7 @@ SQLITE_PRIVATE int sqlite3ParseUri( } parse_uri_out: if( rc!=SQLITE_OK ){ - sqlite3_free(zFile); + sqlite3_free_filename(zFile); zFile = 0; } *pFlags = flags; @@ -138357,10 +184308,26 @@ SQLITE_PRIVATE int sqlite3ParseUri( return rc; } +/* +** This routine does the core work of extracting URI parameters from a +** database filename for the sqlite3_uri_parameter() interface. +*/ +static const char *uriParameter(const char *zFilename, const char *zParam){ + zFilename += sqlite3Strlen30(zFilename) + 1; + while( ALWAYS(zFilename!=0) && zFilename[0] ){ + int x = strcmp(zFilename, zParam); + zFilename += sqlite3Strlen30(zFilename) + 1; + if( x==0 ) return zFilename; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return 0; +} + + /* ** This routine does the work of opening a database on behalf of -** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" +** sqlite3_open() and sqlite3_open16(). The database filename "zFilename" ** is UTF-8 encoded. */ static int openDatabase( @@ -138374,6 +184341,7 @@ static int openDatabase( int isThreadsafe; /* True for threadsafe connections */ char *zOpen = 0; /* Filename argument to pass to BtreeOpen() */ char *zErrMsg = 0; /* Error message from sqlite3ParseUri() */ + int i; /* Loop counter */ #ifdef SQLITE_ENABLE_API_ARMOR if( ppDb==0 ) return SQLITE_MISUSE_BKPT; @@ -138384,26 +184352,6 @@ static int openDatabase( if( rc ) return rc; #endif - /* Only allow sensible combinations of bits in the flags argument. - ** Throw an error if any non-sense combination is used. If we - ** do not block illegal combinations here, it could trigger - ** assert() statements in deeper layers. Sensible combinations - ** are: - ** - ** 1: SQLITE_OPEN_READONLY - ** 2: SQLITE_OPEN_READWRITE - ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE - */ - assert( SQLITE_OPEN_READONLY == 0x01 ); - assert( SQLITE_OPEN_READWRITE == 0x02 ); - assert( SQLITE_OPEN_CREATE == 0x04 ); - testcase( (1<<(flags&7))==0x02 ); /* READONLY */ - testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ - testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ - if( ((1<<(flags&7)) & 0x46)==0 ){ - return SQLITE_MISUSE_BKPT; /* IMP: R-65497-44594 */ - } - if( sqlite3GlobalConfig.bCoreMutex==0 ){ isThreadsafe = 0; }else if( flags & SQLITE_OPEN_NOMUTEX ){ @@ -138413,6 +184361,7 @@ static int openDatabase( }else{ isThreadsafe = sqlite3GlobalConfig.bFullMutex; } + if( flags & SQLITE_OPEN_PRIVATECACHE ){ flags &= ~SQLITE_OPEN_SHAREDCACHE; }else if( sqlite3GlobalConfig.sharedCacheEnabled ){ @@ -138425,18 +184374,18 @@ static int openDatabase( ** dealt with in the previous code block. Besides these, the only ** valid input flags for sqlite3_open_v2() are SQLITE_OPEN_READONLY, ** SQLITE_OPEN_READWRITE, SQLITE_OPEN_CREATE, SQLITE_OPEN_SHAREDCACHE, - ** SQLITE_OPEN_PRIVATECACHE, and some reserved bits. Silently mask - ** off all other flags. + ** SQLITE_OPEN_PRIVATECACHE, SQLITE_OPEN_EXRESCODE, and some reserved + ** bits. Silently mask off all other flags. */ flags &= ~( SQLITE_OPEN_DELETEONCLOSE | SQLITE_OPEN_EXCLUSIVE | SQLITE_OPEN_MAIN_DB | - SQLITE_OPEN_TEMP_DB | - SQLITE_OPEN_TRANSIENT_DB | - SQLITE_OPEN_MAIN_JOURNAL | - SQLITE_OPEN_TEMP_JOURNAL | - SQLITE_OPEN_SUBJOURNAL | - SQLITE_OPEN_MASTER_JOURNAL | + SQLITE_OPEN_TEMP_DB | + SQLITE_OPEN_TRANSIENT_DB | + SQLITE_OPEN_MAIN_JOURNAL | + SQLITE_OPEN_TEMP_JOURNAL | + SQLITE_OPEN_SUBJOURNAL | + SQLITE_OPEN_SUPER_JOURNAL | SQLITE_OPEN_NOMUTEX | SQLITE_OPEN_FULLMUTEX | SQLITE_OPEN_WAL @@ -138445,19 +184394,28 @@ static int openDatabase( /* Allocate the sqlite data structure */ db = sqlite3MallocZero( sizeof(sqlite3) ); if( db==0 ) goto opendb_out; - if( isThreadsafe ){ + if( isThreadsafe +#ifdef SQLITE_ENABLE_MULTITHREADED_CHECKS + || sqlite3GlobalConfig.bCoreMutex +#endif + ){ db->mutex = sqlite3MutexAlloc(SQLITE_MUTEX_RECURSIVE); if( db->mutex==0 ){ sqlite3_free(db); db = 0; goto opendb_out; } + if( isThreadsafe==0 ){ + sqlite3MutexWarnOnContention(db->mutex); + } } sqlite3_mutex_enter(db->mutex); - db->errMask = 0xff; + db->errMask = (flags & SQLITE_OPEN_EXRESCODE)!=0 ? 0xffffffff : 0xff; db->nDb = 2; - db->magic = SQLITE_MAGIC_BUSY; + db->eOpenState = SQLITE_STATE_BUSY; db->aDb = db->aDbStatic; + db->lookaside.bDisable = 1; + db->lookaside.sz = 0; assert( sizeof(db->aLimit)==sizeof(aHardLimit) ); memcpy(db->aLimit, aHardLimit, sizeof(db->aLimit)); @@ -138466,8 +184424,47 @@ static int openDatabase( db->nextAutovac = -1; db->szMmap = sqlite3GlobalConfig.szMmap; db->nextPagesize = 0; + db->init.azInit = sqlite3StdType; /* Any array of string ptrs will do */ +#ifdef SQLITE_ENABLE_SORTER_MMAP + /* Beginning with version 3.37.0, using the VFS xFetch() API to memory-map + ** the temporary files used to do external sorts (see code in vdbesort.c) + ** is disabled. It can still be used either by defining + ** SQLITE_ENABLE_SORTER_MMAP at compile time or by using the + ** SQLITE_TESTCTRL_SORTER_MMAP test-control at runtime. */ db->nMaxSorterMmap = 0x7FFFFFFF; - db->flags |= SQLITE_ShortColNames | SQLITE_EnableTrigger | SQLITE_CacheSpill +#endif + db->flags |= SQLITE_ShortColNames + | SQLITE_EnableTrigger + | SQLITE_EnableView + | SQLITE_CacheSpill +#if !defined(SQLITE_TRUSTED_SCHEMA) || SQLITE_TRUSTED_SCHEMA+0!=0 + | SQLITE_TrustedSchema +#endif +/* The SQLITE_DQS compile-time option determines the default settings +** for SQLITE_DBCONFIG_DQS_DDL and SQLITE_DBCONFIG_DQS_DML. +** +** SQLITE_DQS SQLITE_DBCONFIG_DQS_DDL SQLITE_DBCONFIG_DQS_DML +** ---------- ----------------------- ----------------------- +** undefined on on +** 3 on on +** 2 on off +** 1 off on +** 0 off off +** +** Legacy behavior is 3 (double-quoted string literals are allowed anywhere) +** and so that is the default. But developers are encouraged to use +** -DSQLITE_DQS=0 (best) or -DSQLITE_DQS=1 (second choice) if possible. +*/ +#if !defined(SQLITE_DQS) +# define SQLITE_DQS 3 +#endif +#if (SQLITE_DQS&1)==1 + | SQLITE_DqsDML +#endif +#if (SQLITE_DQS&2)==2 + | SQLITE_DqsDDL +#endif + #if !defined(SQLITE_DEFAULT_AUTOMATIC_INDEX) || SQLITE_DEFAULT_AUTOMATIC_INDEX | SQLITE_AutoIndex #endif @@ -138494,6 +184491,18 @@ static int openDatabase( #endif #if defined(SQLITE_ENABLE_FTS3_TOKENIZER) | SQLITE_Fts3Tokenizer +#endif +#if defined(SQLITE_ENABLE_QPSG) + | SQLITE_EnableQPSG +#endif +#if defined(SQLITE_DEFAULT_DEFENSIVE) + | SQLITE_Defensive +#endif +#if defined(SQLITE_DEFAULT_LEGACY_ALTER_TABLE) + | SQLITE_LegacyAlter +#endif +#if defined(SQLITE_ENABLE_STMT_SCANSTATUS) + | SQLITE_StmtScanStatus #endif ; sqlite3HashInit(&db->aCollSeq); @@ -138512,25 +184521,61 @@ static int openDatabase( createCollation(db, sqlite3StrBINARY, SQLITE_UTF16BE, 0, binCollFunc, 0); createCollation(db, sqlite3StrBINARY, SQLITE_UTF16LE, 0, binCollFunc, 0); createCollation(db, "NOCASE", SQLITE_UTF8, 0, nocaseCollatingFunc, 0); - createCollation(db, "RTRIM", SQLITE_UTF8, (void*)1, binCollFunc, 0); + createCollation(db, "RTRIM", SQLITE_UTF8, 0, rtrimCollFunc, 0); if( db->mallocFailed ){ goto opendb_out; } - /* EVIDENCE-OF: R-08308-17224 The default collating function for all - ** strings is BINARY. - */ - db->pDfltColl = sqlite3FindCollSeq(db, SQLITE_UTF8, sqlite3StrBINARY, 0); - assert( db->pDfltColl!=0 ); - /* Parse the filename/URI argument. */ +#if SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) + /* Process magic filenames ":localStorage:" and ":sessionStorage:" */ + if( zFilename && zFilename[0]==':' ){ + if( strcmp(zFilename, ":localStorage:")==0 ){ + zFilename = "file:local?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + }else if( strcmp(zFilename, ":sessionStorage:")==0 ){ + zFilename = "file:session?vfs=kvvfs"; + flags |= SQLITE_OPEN_URI; + } + } +#endif /* SQLITE_OS_UNIX && defined(SQLITE_OS_KV_OPTIONAL) */ + + /* Parse the filename/URI argument + ** + ** Only allow sensible combinations of bits in the flags argument. + ** Throw an error if any non-sense combination is used. If we + ** do not block illegal combinations here, it could trigger + ** assert() statements in deeper layers. Sensible combinations + ** are: + ** + ** 1: SQLITE_OPEN_READONLY + ** 2: SQLITE_OPEN_READWRITE + ** 6: SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE + */ db->openFlags = flags; - rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + assert( SQLITE_OPEN_READONLY == 0x01 ); + assert( SQLITE_OPEN_READWRITE == 0x02 ); + assert( SQLITE_OPEN_CREATE == 0x04 ); + testcase( (1<<(flags&7))==0x02 ); /* READONLY */ + testcase( (1<<(flags&7))==0x04 ); /* READWRITE */ + testcase( (1<<(flags&7))==0x40 ); /* READWRITE | CREATE */ + if( ((1<<(flags&7)) & 0x46)==0 ){ + rc = SQLITE_MISUSE_BKPT; /* IMP: R-18321-05872 */ + }else{ + if( zFilename==0 ) zFilename = ":memory:"; + rc = sqlite3ParseUri(zVfs, zFilename, &flags, &db->pVfs, &zOpen, &zErrMsg); + } if( rc!=SQLITE_OK ){ if( rc==SQLITE_NOMEM ) sqlite3OomFault(db); sqlite3ErrorWithMsg(db, rc, zErrMsg ? "%s" : 0, zErrMsg); sqlite3_free(zErrMsg); goto opendb_out; } + assert( db->pVfs!=0 ); +#if SQLITE_OS_KV || defined(SQLITE_OS_KV_OPTIONAL) + if( sqlite3_stricmp(db->pVfs->zName, "kvvfs")==0 ){ + db->temp_store = 2; + } +#endif /* Open the backend database driver */ rc = sqlite3BtreeOpen(db->pVfs, zOpen, db, &db->aDb[0].pBt, 0, @@ -138544,19 +184589,21 @@ static int openDatabase( } sqlite3BtreeEnter(db->aDb[0].pBt); db->aDb[0].pSchema = sqlite3SchemaGet(db, db->aDb[0].pBt); - if( !db->mallocFailed ) ENC(db) = SCHEMA_ENC(db); + if( !db->mallocFailed ){ + sqlite3SetTextEncoding(db, SCHEMA_ENC(db)); + } sqlite3BtreeLeave(db->aDb[0].pBt); db->aDb[1].pSchema = sqlite3SchemaGet(db, 0); /* The default safety_level for the main database is FULL; for the temp - ** database it is OFF. This matches the pager layer defaults. + ** database it is OFF. This matches the pager layer defaults. */ - db->aDb[0].zName = "main"; + db->aDb[0].zDbSName = "main"; db->aDb[0].safety_level = SQLITE_DEFAULT_SYNCHRONOUS+1; - db->aDb[1].zName = "temp"; + db->aDb[1].zDbSName = "temp"; db->aDb[1].safety_level = PAGER_SYNCHRONOUS_OFF; - db->magic = SQLITE_MAGIC_OPEN; + db->eOpenState = SQLITE_STATE_OPEN; if( db->mallocFailed ){ goto opendb_out; } @@ -138567,11 +184614,17 @@ static int openDatabase( */ sqlite3Error(db, SQLITE_OK); sqlite3RegisterPerConnectionBuiltinFunctions(db); + rc = sqlite3_errcode(db); + + + /* Load compiled-in extensions */ + for(i=0; rc==SQLITE_OK && imallocFailed ){ - extern int sqlite3Fts1Init(sqlite3*); - rc = sqlite3Fts1Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_FTS2 - if( !db->mallocFailed && rc==SQLITE_OK ){ - extern int sqlite3Fts2Init(sqlite3*); - rc = sqlite3Fts2Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_FTS3 /* automatically defined by SQLITE_ENABLE_FTS4 */ - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts3Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_FTS5 - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3Fts5Init(db); - } -#endif - -#ifdef SQLITE_ENABLE_ICU - if( !db->mallocFailed && rc==SQLITE_OK ){ - rc = sqlite3IcuInit(db); - } -#endif - -#ifdef SQLITE_ENABLE_RTREE - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3RtreeInit(db); - } -#endif - -#ifdef SQLITE_ENABLE_DBSTAT_VTAB - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3DbstatRegister(db); - } -#endif - -#ifdef SQLITE_ENABLE_JSON1 - if( !db->mallocFailed && rc==SQLITE_OK){ - rc = sqlite3Json1Init(db); - } +#ifdef SQLITE_ENABLE_INTERNAL_FUNCTIONS + /* Testing use only!!! The -DSQLITE_ENABLE_INTERNAL_FUNCTIONS=1 compile-time + ** option gives access to internal functions by default. + ** Testing use only!!! */ + db->mDbFlags |= DBFLAG_InternalFunc; #endif /* -DSQLITE_DEFAULT_LOCKING_MODE=1 makes EXCLUSIVE the default locking @@ -138655,12 +184665,12 @@ opendb_out: sqlite3_mutex_leave(db->mutex); } rc = sqlite3_errcode(db); - assert( db!=0 || rc==SQLITE_NOMEM ); - if( rc==SQLITE_NOMEM ){ + assert( db!=0 || (rc&0xff)==SQLITE_NOMEM ); + if( (rc&0xff)==SQLITE_NOMEM ){ sqlite3_close(db); db = 0; }else if( rc!=SQLITE_OK ){ - db->magic = SQLITE_MAGIC_SICK; + db->eOpenState = SQLITE_STATE_SICK; } *ppDb = db; #ifdef SQLITE_ENABLE_SQLLOG @@ -138670,36 +184680,22 @@ opendb_out: sqlite3GlobalConfig.xSqllog(pArg, db, zFilename, 0); } #endif -#if defined(SQLITE_HAS_CODEC) - if( rc==SQLITE_OK ){ - const char *zHexKey = sqlite3_uri_parameter(zOpen, "hexkey"); - if( zHexKey && zHexKey[0] ){ - u8 iByte; - int i; - char zKey[40]; - for(i=0, iByte=0; imutex); + for(p=db->pDbData; p; p=p->pNext){ + if( strcmp(p->zName, zName)==0 ){ + void *pResult = p->pData; + sqlite3_mutex_leave(db->mutex); + return pResult; + } + } + sqlite3_mutex_leave(db->mutex); + return 0; +} + +/* +** Add new client data to a database connection. +*/ +SQLITE_API int sqlite3_set_clientdata( + sqlite3 *db, /* Attach client data to this connection */ + const char *zName, /* Name of the client data */ + void *pData, /* The client data itself */ + void (*xDestructor)(void*) /* Destructor */ +){ + DbClientData *p, **pp; + sqlite3_mutex_enter(db->mutex); + pp = &db->pDbData; + for(p=db->pDbData; p && strcmp(p->zName,zName); p=p->pNext){ + pp = &p->pNext; + } + if( p ){ + assert( p->pData!=0 ); + if( p->xDestructor ) p->xDestructor(p->pData); + if( pData==0 ){ + *pp = p->pNext; + sqlite3_free(p); + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + } + }else if( pData==0 ){ + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; + }else{ + size_t n = strlen(zName); + p = sqlite3_malloc64( sizeof(DbClientData)+n+1 ); + if( p==0 ){ + if( xDestructor ) xDestructor(pData); + sqlite3_mutex_leave(db->mutex); + return SQLITE_NOMEM; + } + memcpy(p->zName, zName, n+1); + p->pNext = db->pDbData; + db->pDbData = p; + } + p->pData = pData; + p->xDestructor = xDestructor; + sqlite3_mutex_leave(db->mutex); + return SQLITE_OK; +} + + #ifndef SQLITE_OMIT_DEPRECATED /* ** This function is now an anachronism. It used to be used to recover from a ** malloc() failure, but SQLite now does this automatically. */ -SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){ +SQLITE_API int sqlite3_global_recover(void){ return SQLITE_OK; } #endif @@ -138873,7 +184932,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_global_recover(void){ ** by default. Autocommit is disabled by a BEGIN statement and reenabled ** by the next COMMIT or ROLLBACK. */ -SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ +SQLITE_API int sqlite3_get_autocommit(sqlite3 *db){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ (void)SQLITE_MISUSE_BKPT; @@ -138894,31 +184953,39 @@ SQLITE_API int SQLITE_STDCALL sqlite3_get_autocommit(sqlite3 *db){ ** 2. Invoke sqlite3_log() to provide the source code location where ** a low-level error is first detected. */ -static int reportError(int iErr, int lineno, const char *zType){ +SQLITE_PRIVATE int sqlite3ReportError(int iErr, int lineno, const char *zType){ sqlite3_log(iErr, "%s at line %d of [%.10s]", zType, lineno, 20+sqlite3_sourceid()); return iErr; } SQLITE_PRIVATE int sqlite3CorruptError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CORRUPT, lineno, "database corruption"); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, "database corruption"); } SQLITE_PRIVATE int sqlite3MisuseError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_MISUSE, lineno, "misuse"); + return sqlite3ReportError(SQLITE_MISUSE, lineno, "misuse"); } SQLITE_PRIVATE int sqlite3CantopenError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_CANTOPEN, lineno, "cannot open file"); + return sqlite3ReportError(SQLITE_CANTOPEN, lineno, "cannot open file"); } +#if defined(SQLITE_DEBUG) || defined(SQLITE_ENABLE_CORRUPT_PGNO) +SQLITE_PRIVATE int sqlite3CorruptPgnoError(int lineno, Pgno pgno){ + char zMsg[100]; + sqlite3_snprintf(sizeof(zMsg), zMsg, "database corruption page %d", pgno); + testcase( sqlite3GlobalConfig.xLog!=0 ); + return sqlite3ReportError(SQLITE_CORRUPT, lineno, zMsg); +} +#endif #ifdef SQLITE_DEBUG SQLITE_PRIVATE int sqlite3NomemError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_NOMEM, lineno, "OOM"); + return sqlite3ReportError(SQLITE_NOMEM, lineno, "OOM"); } SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ testcase( sqlite3GlobalConfig.xLog!=0 ); - return reportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); + return sqlite3ReportError(SQLITE_IOERR_NOMEM, lineno, "I/O OOM error"); } #endif @@ -138930,7 +184997,7 @@ SQLITE_PRIVATE int sqlite3IoerrnomemError(int lineno){ ** SQLite no longer uses thread-specific data so this routine is now a ** no-op. It is retained for historical compatibility. */ -SQLITE_API void SQLITE_STDCALL sqlite3_thread_cleanup(void){ +SQLITE_API void sqlite3_thread_cleanup(void){ } #endif @@ -138938,7 +185005,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3_thread_cleanup(void){ ** Return meta information about a specific column of a database table. ** See comment in sqlite3.h (sqlite.h.in) for details. */ -SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( +SQLITE_API int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ const char *zTableName, /* Table name */ @@ -138977,18 +185044,18 @@ SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( /* Locate the table in question */ pTab = sqlite3FindTable(db, zTableName, zDbName); - if( !pTab || pTab->pSelect ){ + if( !pTab || IsView(pTab) ){ pTab = 0; goto error_out; } /* Find the column for which info is requested */ if( zColumnName==0 ){ - /* Query for existance of table only */ + /* Query for existence of table only */ }else{ for(iCol=0; iColnCol; iCol++){ pCol = &pTab->aCol[iCol]; - if( 0==sqlite3StrICmp(pCol->zName, zColumnName) ){ + if( 0==sqlite3StrICmp(pCol->zCnName, zColumnName) ){ break; } } @@ -139006,16 +185073,16 @@ SQLITE_API int SQLITE_STDCALL sqlite3_table_column_metadata( /* The following block stores the meta information that will be returned ** to the caller in local variables zDataType, zCollSeq, notnull, primarykey ** and autoinc. At this point there are two possibilities: - ** - ** 1. The specified column name was rowid", "oid" or "_rowid_" - ** and there is no explicitly declared IPK column. ** - ** 2. The table is not a view and the column name identified an + ** 1. The specified column name was rowid", "oid" or "_rowid_" + ** and there is no explicitly declared IPK column. + ** + ** 2. The table is not a view and the column name identified an ** explicitly declared column. Copy meta information from *pCol. - */ + */ if( pCol ){ zDataType = sqlite3ColumnType(pCol,0); - zCollSeq = pCol->zColl; + zCollSeq = sqlite3ColumnColl(pCol); notnull = pCol->notNull!=0; primarykey = (pCol->colFlags & COLFLAG_PRIMKEY)!=0; autoinc = pTab->iPKey==iCol && (pTab->tabFlags & TF_Autoincrement)!=0; @@ -139056,23 +185123,23 @@ error_out: /* ** Sleep for a little while. Return the amount of time slept. */ -SQLITE_API int SQLITE_STDCALL sqlite3_sleep(int ms){ +SQLITE_API int sqlite3_sleep(int ms){ sqlite3_vfs *pVfs; int rc; pVfs = sqlite3_vfs_find(0); if( pVfs==0 ) return 0; - /* This function works in milliseconds, but the underlying OsSleep() + /* This function works in milliseconds, but the underlying OsSleep() ** API uses microseconds. Hence the 1000's. */ - rc = (sqlite3OsSleep(pVfs, 1000*ms)/1000); + rc = (sqlite3OsSleep(pVfs, ms<0 ? 0 : 1000*ms)/1000); return rc; } /* ** Enable or disable the extended result codes. */ -SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int onoff){ +SQLITE_API int sqlite3_extended_result_codes(sqlite3 *db, int onoff){ #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; #endif @@ -139085,7 +185152,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_extended_result_codes(sqlite3 *db, int ono /* ** Invoke the xFileControl method on a particular database. */ -SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ +SQLITE_API int sqlite3_file_control(sqlite3 *db, const char *zDbName, int op, void *pArg){ int rc = SQLITE_ERROR; Btree *pBtree; @@ -139111,10 +185178,23 @@ SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbN }else if( op==SQLITE_FCNTL_JOURNAL_POINTER ){ *(sqlite3_file**)pArg = sqlite3PagerJrnlFile(pPager); rc = SQLITE_OK; - }else if( fd->pMethods ){ - rc = sqlite3OsFileControl(fd, op, pArg); + }else if( op==SQLITE_FCNTL_DATA_VERSION ){ + *(unsigned int*)pArg = sqlite3PagerDataVersion(pPager); + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESERVE_BYTES ){ + int iNew = *(int*)pArg; + *(int*)pArg = sqlite3BtreeGetRequestedReserve(pBtree); + if( iNew>=0 && iNew<=255 ){ + sqlite3BtreeSetPageSize(pBtree, 0, iNew, 0); + } + rc = SQLITE_OK; + }else if( op==SQLITE_FCNTL_RESET_CACHE ){ + sqlite3BtreeClearCache(pBtree); + rc = SQLITE_OK; }else{ - rc = SQLITE_NOTFOUND; + int nSave = db->busyHandler.nBusy; + rc = sqlite3OsFileControl(fd, op, pArg); + db->busyHandler.nBusy = nSave; } sqlite3BtreeLeave(pBtree); } @@ -139125,9 +185205,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_file_control(sqlite3 *db, const char *zDbN /* ** Interface to the testing logic. */ -SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ +SQLITE_API int sqlite3_test_control(int op, ...){ int rc = 0; -#ifdef SQLITE_OMIT_BUILTIN_TEST +#ifdef SQLITE_UNTESTABLE UNUSED_PARAMETER(op); #else va_list ap; @@ -139152,15 +185232,60 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ break; } - /* - ** Reset the PRNG back to its uninitialized state. The next call - ** to sqlite3_randomness() will reseed the PRNG using a single call - ** to the xRandomness method of the default VFS. + /* sqlite3_test_control(SQLITE_TESTCTRL_PRNG_SEED, int x, sqlite3 *db); + ** + ** Control the seed for the pseudo-random number generator (PRNG) that + ** is built into SQLite. Cases: + ** + ** x!=0 && db!=0 Seed the PRNG to the current value of the + ** schema cookie in the main database for db, or + ** x if the schema cookie is zero. This case + ** is convenient to use with database fuzzers + ** as it allows the fuzzer some control over the + ** the PRNG seed. + ** + ** x!=0 && db==0 Seed the PRNG to the value of x. + ** + ** x==0 && db==0 Revert to default behavior of using the + ** xRandomness method on the primary VFS. + ** + ** This test-control also resets the PRNG so that the new seed will + ** be used for the next call to sqlite3_randomness(). */ - case SQLITE_TESTCTRL_PRNG_RESET: { +#ifndef SQLITE_OMIT_WSD + case SQLITE_TESTCTRL_PRNG_SEED: { + int x = va_arg(ap, int); + int y; + sqlite3 *db = va_arg(ap, sqlite3*); + assert( db==0 || db->aDb[0].pSchema!=0 ); + if( db && (y = db->aDb[0].pSchema->schema_cookie)!=0 ){ x = y; } + sqlite3Config.iPrngSeed = x; sqlite3_randomness(0,0); break; } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_FK_NO_ACTION, sqlite3 *db, int b); + ** + ** If b is true, then activate the SQLITE_FkNoAction setting. If b is + ** false then clearn that setting. If the SQLITE_FkNoAction setting is + ** abled, all foreign key ON DELETE and ON UPDATE actions behave as if + ** they were NO ACTION, regardless of how they are defined. + ** + ** NB: One must usually run "PRAGMA writable_schema=RESET" after + ** using this test-control, before it will take full effect. failing + ** to reset the schema can result in some unexpected behavior. + */ + case SQLITE_TESTCTRL_FK_NO_ACTION: { + sqlite3 *db = va_arg(ap, sqlite3*); + int b = va_arg(ap, int); + if( b ){ + db->flags |= SQLITE_FkNoAction; + }else{ + db->flags &= ~SQLITE_FkNoAction; + } + break; + } /* ** sqlite3_test_control(BITVEC_TEST, size, program) @@ -139189,12 +185314,16 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ ** sqlite3_test_control(). */ case SQLITE_TESTCTRL_FAULT_INSTALL: { - /* MSVC is picky about pulling func ptrs from va lists. - ** http://support.microsoft.com/kb/47961 + /* A bug in MSVC prevents it from understanding pointers to functions + ** types in the second argument to va_arg(). Work around the problem + ** using a typedef. + ** http://support.microsoft.com/kb/47961 <-- dead hyperlink + ** Search at http://web.archive.org/ to find the 2015-03-16 archive + ** of the link above to see the original text. ** sqlite3GlobalConfig.xTestCallback = va_arg(ap, int(*)(int)); */ - typedef int(*TESTCALLBACKFUNC_t)(int); - sqlite3GlobalConfig.xTestCallback = va_arg(ap, TESTCALLBACKFUNC_t); + typedef int(*sqlite3FaultFuncType)(int); + sqlite3GlobalConfig.xTestCallback = va_arg(ap, sqlite3FaultFuncType); rc = sqlite3FaultSim(0); break; } @@ -139202,7 +185331,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ /* ** sqlite3_test_control(BENIGN_MALLOC_HOOKS, xBegin, xEnd) ** - ** Register hooks to call to indicate which malloc() failures + ** Register hooks to call to indicate which malloc() failures ** are benign. */ case SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS: { @@ -139253,6 +185382,30 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ volatile int x = 0; assert( /*side-effects-ok*/ (x = va_arg(ap,int))!=0 ); rc = x; +#if defined(SQLITE_DEBUG) + /* Invoke these debugging routines so that the compiler does not + ** issue "defined but not used" warnings. */ + if( x==9999 ){ + sqlite3ShowExpr(0); + sqlite3ShowExpr(0); + sqlite3ShowExprList(0); + sqlite3ShowIdList(0); + sqlite3ShowSrcList(0); + sqlite3ShowWith(0); + sqlite3ShowUpsert(0); +#ifndef SQLITE_OMIT_TRIGGER + sqlite3ShowTriggerStep(0); + sqlite3ShowTriggerStepList(0); + sqlite3ShowTrigger(0); + sqlite3ShowTriggerList(0); +#endif +#ifndef SQLITE_OMIT_WINDOWFUNC + sqlite3ShowWindow(0); + sqlite3ShowWinFunc(0); +#endif + sqlite3ShowSelect(0); + } +#endif break; } @@ -139263,7 +185416,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ ** This action provides a run-time test to see how the ALWAYS and ** NEVER macros were defined at compile-time. ** - ** The return value is ALWAYS(X). + ** The return value is ALWAYS(X) if X is true, or 0 if X is false. ** ** The recommended test is X==2. If the return value is 2, that means ** ALWAYS() and NEVER() are both no-op pass-through macros, which is the @@ -139286,7 +185439,7 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_ALWAYS: { int x = va_arg(ap,int); - rc = ALWAYS(x); + rc = x ? ALWAYS(x) : 0; break; } @@ -139300,29 +185453,15 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ ** 10 little-endian, determined at run-time ** 432101 big-endian, determined at compile-time ** 123410 little-endian, determined at compile-time - */ + */ case SQLITE_TESTCTRL_BYTEORDER: { rc = SQLITE_BYTEORDER*100 + SQLITE_LITTLEENDIAN*10 + SQLITE_BIGENDIAN; break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_RESERVE, sqlite3 *db, int N) - ** - ** Set the nReserve size to N for the main database on the database - ** connection db. - */ - case SQLITE_TESTCTRL_RESERVE: { - sqlite3 *db = va_arg(ap, sqlite3*); - int x = va_arg(ap,int); - sqlite3_mutex_enter(db->mutex); - sqlite3BtreeSetPageSize(db->aDb[0].pBt, 0, x, 0); - sqlite3_mutex_leave(db->mutex); - break; - } - /* sqlite3_test_control(SQLITE_TESTCTRL_OPTIMIZATIONS, sqlite3 *db, int N) ** - ** Enable or disable various optimizations for testing purposes. The + ** Enable or disable various optimizations for testing purposes. The ** argument N is a bitmask of optimizations to be disabled. For normal ** operation N should be 0. The idea is that a test program (like the ** SQL Logic Test or SLT test module) can run the same SQL multiple times @@ -139331,52 +185470,54 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_OPTIMIZATIONS: { sqlite3 *db = va_arg(ap, sqlite3*); - db->dbOptFlags = (u16)(va_arg(ap, int) & 0xffff); + db->dbOptFlags = va_arg(ap, u32); break; } -#ifdef SQLITE_N_KEYWORD - /* sqlite3_test_control(SQLITE_TESTCTRL_ISKEYWORD, const char *zWord) + /* sqlite3_test_control(SQLITE_TESTCTRL_GETOPT, sqlite3 *db, int *N) ** - ** If zWord is a keyword recognized by the parser, then return the - ** number of keywords. Or if zWord is not a keyword, return 0. - ** - ** This test feature is only available in the amalgamation since - ** the SQLITE_N_KEYWORD macro is not defined in this file if SQLite - ** is built using separate source files. + ** Write the current optimization settings into *N. A zero bit means that + ** the optimization is on, and a 1 bit means that the optimization is off. */ - case SQLITE_TESTCTRL_ISKEYWORD: { - const char *zWord = va_arg(ap, const char*); - int n = sqlite3Strlen30(zWord); - rc = (sqlite3KeywordCode((u8*)zWord, n)!=TK_ID) ? SQLITE_N_KEYWORD : 0; - break; - } -#endif - - /* sqlite3_test_control(SQLITE_TESTCTRL_SCRATCHMALLOC, sz, &pNew, pFree); - ** - ** Pass pFree into sqlite3ScratchFree(). - ** If sz>0 then allocate a scratch buffer into pNew. - */ - case SQLITE_TESTCTRL_SCRATCHMALLOC: { - void *pFree, **ppNew; - int sz; - sz = va_arg(ap, int); - ppNew = va_arg(ap, void**); - pFree = va_arg(ap, void*); - if( sz ) *ppNew = sqlite3ScratchMalloc(sz); - sqlite3ScratchFree(pFree); + case SQLITE_TESTCTRL_GETOPT: { + sqlite3 *db = va_arg(ap, sqlite3*); + int *pN = va_arg(ap, int*); + *pN = db->dbOptFlags; break; } - /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, int onoff); + /* sqlite3_test_control(SQLITE_TESTCTRL_LOCALTIME_FAULT, onoff, xAlt); ** - ** If parameter onoff is non-zero, configure the wrappers so that all - ** subsequent calls to localtime() and variants fail. If onoff is zero, - ** undo this setting. + ** If parameter onoff is 1, subsequent calls to localtime() fail. + ** If 2, then invoke xAlt() instead of localtime(). If 0, normal + ** processing. + ** + ** xAlt arguments are void pointers, but they really want to be: + ** + ** int xAlt(const time_t*, struct tm*); + ** + ** xAlt should write results in to struct tm object of its 2nd argument + ** and return zero on success, or return non-zero on failure. */ case SQLITE_TESTCTRL_LOCALTIME_FAULT: { sqlite3GlobalConfig.bLocaltimeFault = va_arg(ap, int); + if( sqlite3GlobalConfig.bLocaltimeFault==2 ){ + typedef int(*sqlite3LocaltimeType)(const void*,void*); + sqlite3GlobalConfig.xAltLocaltime = va_arg(ap, sqlite3LocaltimeType); + }else{ + sqlite3GlobalConfig.xAltLocaltime = 0; + } + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_INTERNAL_FUNCTIONS, sqlite3*); + ** + ** Toggle the ability to use internal functions on or off for + ** the database connection given in the argument. + */ + case SQLITE_TESTCTRL_INTERNAL_FUNCTIONS: { + sqlite3 *db = va_arg(ap, sqlite3*); + db->mDbFlags ^= DBFLAG_InternalFunc; break; } @@ -139386,22 +185527,49 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ ** formed and never corrupt. This flag is clear by default, indicating that ** database files might have arbitrary corruption. Setting the flag during ** testing causes certain assert() statements in the code to be activated - ** that demonstrat invariants on well-formed database files. + ** that demonstrate invariants on well-formed database files. */ case SQLITE_TESTCTRL_NEVER_CORRUPT: { sqlite3GlobalConfig.neverCorrupt = va_arg(ap, int); break; } + /* sqlite3_test_control(SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS, int); + ** + ** Set or clear a flag that causes SQLite to verify that type, name, + ** and tbl_name fields of the sqlite_schema table. This is normally + ** on, but it is sometimes useful to turn it off for testing. + ** + ** 2020-07-22: Disabling EXTRA_SCHEMA_CHECKS also disables the + ** verification of rootpage numbers when parsing the schema. This + ** is useful to make it easier to reach strange internal error states + ** during testing. The EXTRA_SCHEMA_CHECKS setting is always enabled + ** in production. + */ + case SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS: { + sqlite3GlobalConfig.bExtraSchemaChecks = va_arg(ap, int); + break; + } + + /* Set the threshold at which OP_Once counters reset back to zero. + ** By default this is 0x7ffffffe (over 2 billion), but that value is + ** too big to test in a reasonable amount of time, so this control is + ** provided to set a small and easily reachable reset value. + */ + case SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD: { + sqlite3GlobalConfig.iOnceResetThreshold = va_arg(ap, int); + break; + } /* sqlite3_test_control(SQLITE_TESTCTRL_VDBE_COVERAGE, xCallback, ptr); ** - ** Set the VDBE coverage callback function to xCallback with context + ** Set the VDBE coverage callback function to xCallback with context ** pointer ptr. */ case SQLITE_TESTCTRL_VDBE_COVERAGE: { #ifdef SQLITE_VDBE_COVERAGE - typedef void (*branch_callback)(void*,int,u8,u8); + typedef void (*branch_callback)(void*,unsigned int, + unsigned char,unsigned char); sqlite3GlobalConfig.xVdbeBranch = va_arg(ap,branch_callback); sqlite3GlobalConfig.pVdbeBranchArg = va_arg(ap,void*); #endif @@ -139443,25 +185611,249 @@ SQLITE_API int SQLITE_CDECL sqlite3_test_control(int op, ...){ */ case SQLITE_TESTCTRL_IMPOSTER: { sqlite3 *db = va_arg(ap, sqlite3*); + int iDb; sqlite3_mutex_enter(db->mutex); - db->init.iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); - db->init.busy = db->init.imposterTable = va_arg(ap,int); - db->init.newTnum = va_arg(ap,int); - if( db->init.busy==0 && db->init.newTnum>0 ){ - sqlite3ResetAllSchemasOfConnection(db); + iDb = sqlite3FindDbName(db, va_arg(ap,const char*)); + if( iDb>=0 ){ + db->init.iDb = iDb; + db->init.busy = db->init.imposterTable = va_arg(ap,int); + db->init.newTnum = va_arg(ap,int); + if( db->init.busy==0 && db->init.newTnum>0 ){ + sqlite3ResetAllSchemasOfConnection(db); + } } sqlite3_mutex_leave(db->mutex); break; } + +#if defined(YYCOVERAGE) + /* sqlite3_test_control(SQLITE_TESTCTRL_PARSER_COVERAGE, FILE *out) + ** + ** This test control (only available when SQLite is compiled with + ** -DYYCOVERAGE) writes a report onto "out" that shows all + ** state/lookahead combinations in the parser state machine + ** which are never exercised. If any state is missed, make the + ** return code SQLITE_ERROR. + */ + case SQLITE_TESTCTRL_PARSER_COVERAGE: { + FILE *out = va_arg(ap, FILE*); + if( sqlite3ParserCoverage(out) ) rc = SQLITE_ERROR; + break; + } +#endif /* defined(YYCOVERAGE) */ + + /* sqlite3_test_control(SQLITE_TESTCTRL_RESULT_INTREAL, sqlite3_context*); + ** + ** This test-control causes the most recent sqlite3_result_int64() value + ** to be interpreted as a MEM_IntReal instead of as an MEM_Int. Normally, + ** MEM_IntReal values only arise during an INSERT operation of integer + ** values into a REAL column, so they can be challenging to test. This + ** test-control enables us to write an intreal() SQL function that can + ** inject an intreal() value at arbitrary places in an SQL statement, + ** for testing purposes. + */ + case SQLITE_TESTCTRL_RESULT_INTREAL: { + sqlite3_context *pCtx = va_arg(ap, sqlite3_context*); + sqlite3ResultIntReal(pCtx); + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_SEEK_COUNT, + ** sqlite3 *db, // Database connection + ** u64 *pnSeek // Write seek count here + ** ); + ** + ** This test-control queries the seek-counter on the "main" database + ** file. The seek-counter is written into *pnSeek and is then reset. + ** The seek-count is only available if compiled with SQLITE_DEBUG. + */ + case SQLITE_TESTCTRL_SEEK_COUNT: { + sqlite3 *db = va_arg(ap, sqlite3*); + u64 *pn = va_arg(ap, sqlite3_uint64*); + *pn = sqlite3BtreeSeekCount(db->aDb->pBt); + (void)db; /* Silence harmless unused variable warning */ + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_TRACEFLAGS, op, ptr) + ** + ** "ptr" is a pointer to a u32. + ** + ** op==0 Store the current sqlite3TreeTrace in *ptr + ** op==1 Set sqlite3TreeTrace to the value *ptr + ** op==2 Store the current sqlite3WhereTrace in *ptr + ** op==3 Set sqlite3WhereTrace to the value *ptr + */ + case SQLITE_TESTCTRL_TRACEFLAGS: { + int opTrace = va_arg(ap, int); + u32 *ptr = va_arg(ap, u32*); + switch( opTrace ){ + case 0: *ptr = sqlite3TreeTrace; break; + case 1: sqlite3TreeTrace = *ptr; break; + case 2: *ptr = sqlite3WhereTrace; break; + case 3: sqlite3WhereTrace = *ptr; break; + } + break; + } + + /* sqlite3_test_control(SQLITE_TESTCTRL_LOGEST, + ** double fIn, // Input value + ** int *pLogEst, // sqlite3LogEstFromDouble(fIn) + ** u64 *pInt, // sqlite3LogEstToInt(*pLogEst) + ** int *pLogEst2 // sqlite3LogEst(*pInt) + ** ); + ** + ** Test access for the LogEst conversion routines. + */ + case SQLITE_TESTCTRL_LOGEST: { + double rIn = va_arg(ap, double); + LogEst rLogEst = sqlite3LogEstFromDouble(rIn); + int *pI1 = va_arg(ap,int*); + u64 *pU64 = va_arg(ap,u64*); + int *pI2 = va_arg(ap,int*); + *pI1 = rLogEst; + *pU64 = sqlite3LogEstToInt(rLogEst); + *pI2 = sqlite3LogEst(*pU64); + break; + } + +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) + /* sqlite3_test_control(SQLITE_TESTCTRL_TUNE, id, *piValue) + ** + ** If "id" is an integer between 1 and SQLITE_NTUNE then set the value + ** of the id-th tuning parameter to *piValue. If "id" is between -1 + ** and -SQLITE_NTUNE, then write the current value of the (-id)-th + ** tuning parameter into *piValue. + ** + ** Tuning parameters are for use during transient development builds, + ** to help find the best values for constants in the query planner. + ** Access tuning parameters using the Tuning(ID) macro. Set the + ** parameters in the CLI using ".testctrl tune ID VALUE". + ** + ** Transient use only. Tuning parameters should not be used in + ** checked-in code. + */ + case SQLITE_TESTCTRL_TUNE: { + int id = va_arg(ap, int); + int *piValue = va_arg(ap, int*); + if( id>0 && id<=SQLITE_NTUNE ){ + Tuning(id) = *piValue; + }else if( id<0 && id>=-SQLITE_NTUNE ){ + *piValue = Tuning(-id); + }else{ + rc = SQLITE_NOTFOUND; + } + break; + } +#endif + + /* sqlite3_test_control(SQLITE_TESTCTRL_JSON_SELFCHECK, &onOff); + ** + ** Activate or deactivate validation of JSONB that is generated from + ** text. Off by default, as the validation is slow. Validation is + ** only available if compiled using SQLITE_DEBUG. + ** + ** If onOff is initially 1, then turn it on. If onOff is initially + ** off, turn it off. If onOff is initially -1, then change onOff + ** to be the current setting. + */ + case SQLITE_TESTCTRL_JSON_SELFCHECK: { +#if defined(SQLITE_DEBUG) && !defined(SQLITE_OMIT_WSD) + int *pOnOff = va_arg(ap, int*); + if( *pOnOff<0 ){ + *pOnOff = sqlite3Config.bJsonSelfcheck; + }else{ + sqlite3Config.bJsonSelfcheck = (u8)((*pOnOff)&0xff); + } +#endif + break; + } } va_end(ap); -#endif /* SQLITE_OMIT_BUILTIN_TEST */ +#endif /* SQLITE_UNTESTABLE */ return rc; } +/* +** The Pager stores the Database filename, Journal filename, and WAL filename +** consecutively in memory, in that order. The database filename is prefixed +** by four zero bytes. Locate the start of the database filename by searching +** backwards for the first byte following four consecutive zero bytes. +** +** This only works if the filename passed in was obtained from the Pager. +*/ +static const char *databaseName(const char *zName){ + while( zName[-1]!=0 || zName[-2]!=0 || zName[-3]!=0 || zName[-4]!=0 ){ + zName--; + } + return zName; +} + +/* +** Append text z[] to the end of p[]. Return a pointer to the first +** character after then zero terminator on the new text in p[]. +*/ +static char *appendText(char *p, const char *z){ + size_t n = strlen(z); + memcpy(p, z, n+1); + return p+n+1; +} + +/* +** Allocate memory to hold names for a database, journal file, WAL file, +** and query parameters. The pointer returned is valid for use by +** sqlite3_filename_database() and sqlite3_uri_parameter() and related +** functions. +** +** Memory layout must be compatible with that generated by the pager +** and expected by sqlite3_uri_parameter() and databaseName(). +*/ +SQLITE_API const char *sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +){ + sqlite3_int64 nByte; + int i; + char *pResult, *p; + nByte = strlen(zDatabase) + strlen(zJournal) + strlen(zWal) + 10; + for(i=0; i0 ){ zFilename += sqlite3Strlen30(zFilename) + 1; - if( x==0 ) return zFilename; zFilename += sqlite3Strlen30(zFilename) + 1; } - return 0; + return zFilename[0] ? zFilename : 0; } /* ** Return a boolean value for a query parameter. */ -SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ +SQLITE_API int sqlite3_uri_boolean(const char *zFilename, const char *zParam, int bDflt){ const char *z = sqlite3_uri_parameter(zFilename, zParam); bDflt = bDflt!=0; return z ? sqlite3GetBoolean(z, bDflt) : bDflt; @@ -139494,39 +185894,84 @@ SQLITE_API int SQLITE_STDCALL sqlite3_uri_boolean(const char *zFilename, const c /* ** Return a 64-bit integer value for a query parameter. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3_uri_int64( +SQLITE_API sqlite3_int64 sqlite3_uri_int64( const char *zFilename, /* Filename as passed to xOpen */ const char *zParam, /* URI parameter sought */ sqlite3_int64 bDflt /* return if parameter is missing */ ){ const char *z = sqlite3_uri_parameter(zFilename, zParam); sqlite3_int64 v; - if( z && sqlite3DecOrHexToI64(z, &v)==SQLITE_OK ){ + if( z && sqlite3DecOrHexToI64(z, &v)==0 ){ bDflt = v; } return bDflt; } +/* +** Translate a filename that was handed to a VFS routine into the corresponding +** database, journal, or WAL file. +** +** It is an error to pass this routine a filename string that was not +** passed into the VFS from the SQLite core. Doing so is similar to +** passing free() a pointer that was not obtained from malloc() - it is +** an error that we cannot easily detect but that will likely cause memory +** corruption. +*/ +SQLITE_API const char *sqlite3_filename_database(const char *zFilename){ + if( zFilename==0 ) return 0; + return databaseName(zFilename); +} +SQLITE_API const char *sqlite3_filename_journal(const char *zFilename){ + if( zFilename==0 ) return 0; + zFilename = databaseName(zFilename); + zFilename += sqlite3Strlen30(zFilename) + 1; + while( ALWAYS(zFilename) && zFilename[0] ){ + zFilename += sqlite3Strlen30(zFilename) + 1; + zFilename += sqlite3Strlen30(zFilename) + 1; + } + return zFilename + 1; +} +SQLITE_API const char *sqlite3_filename_wal(const char *zFilename){ +#ifdef SQLITE_OMIT_WAL + return 0; +#else + zFilename = sqlite3_filename_journal(zFilename); + if( zFilename ) zFilename += sqlite3Strlen30(zFilename) + 1; + return zFilename; +#endif +} + /* ** Return the Btree pointer identified by zDbName. Return NULL if not found. */ SQLITE_PRIVATE Btree *sqlite3DbNameToBtree(sqlite3 *db, const char *zDbName){ - int i; - for(i=0; inDb; i++){ - if( db->aDb[i].pBt - && (zDbName==0 || sqlite3StrICmp(zDbName, db->aDb[i].zName)==0) - ){ - return db->aDb[i].pBt; - } + int iDb = zDbName ? sqlite3FindDbName(db, zDbName) : 0; + return iDb<0 ? 0 : db->aDb[iDb].pBt; +} + +/* +** Return the name of the N-th database schema. Return NULL if N is out +** of range. +*/ +SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N){ +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + if( N<0 || N>=db->nDb ){ + return 0; + }else{ + return db->aDb[N].zDbSName; } - return 0; } /* ** Return the filename of the database associated with a database ** connection. */ -SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const char *zDbName){ +SQLITE_API const char *sqlite3_db_filename(sqlite3 *db, const char *zDbName){ Btree *pBt; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -139542,7 +185987,7 @@ SQLITE_API const char *SQLITE_STDCALL sqlite3_db_filename(sqlite3 *db, const cha ** Return 1 if database is read-only or 0 if read/write. Return -1 if ** no such database exists. */ -SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName){ Btree *pBt; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -139556,17 +186001,16 @@ SQLITE_API int SQLITE_STDCALL sqlite3_db_readonly(sqlite3 *db, const char *zDbNa #ifdef SQLITE_ENABLE_SNAPSHOT /* -** Obtain a snapshot handle for the snapshot of database zDb currently +** Obtain a snapshot handle for the snapshot of database zDb currently ** being read by handle db. */ -SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_get( - sqlite3 *db, +SQLITE_API int sqlite3_snapshot_get( + sqlite3 *db, const char *zDb, sqlite3_snapshot **ppSnapshot ){ int rc = SQLITE_ERROR; #ifndef SQLITE_OMIT_WAL - int iDb; #ifdef SQLITE_ENABLE_API_ARMOR if( !sqlite3SafetyCheckOk(db) ){ @@ -139575,13 +186019,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_get( #endif sqlite3_mutex_enter(db->mutex); - iDb = sqlite3FindDbName(db, zDb); - if( iDb==0 || iDb>1 ){ - Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInTrans(pBt) ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + if( db->autoCommit==0 ){ + int iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( SQLITE_TXN_WRITE!=sqlite3BtreeTxnState(pBt) ){ + Pager *pPager = sqlite3BtreePager(pBt); + i64 dummy = 0; + sqlite3PagerSnapshotOpen(pPager, (sqlite3_snapshot*)&dummy); + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotGet(sqlite3BtreePager(pBt), ppSnapshot); + } } } } @@ -139592,11 +186042,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_get( } /* -** Open a read-transaction on the snapshot idendified by pSnapshot. +** Open a read-transaction on the snapshot identified by pSnapshot. */ -SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_open( - sqlite3 *db, - const char *zDb, +SQLITE_API int sqlite3_snapshot_open( + sqlite3 *db, + const char *zDb, sqlite3_snapshot *pSnapshot ){ int rc = SQLITE_ERROR; @@ -139613,11 +186063,29 @@ SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_open( iDb = sqlite3FindDbName(db, zDb); if( iDb==0 || iDb>1 ){ Btree *pBt = db->aDb[iDb].pBt; - if( 0==sqlite3BtreeIsInReadTrans(pBt) ){ - rc = sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), pSnapshot); + if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_WRITE ){ + Pager *pPager = sqlite3BtreePager(pBt); + int bUnlock = 0; + if( sqlite3BtreeTxnState(pBt)!=SQLITE_TXN_NONE ){ + if( db->nVdbeActive==0 ){ + rc = sqlite3PagerSnapshotCheck(pPager, pSnapshot); + if( rc==SQLITE_OK ){ + bUnlock = 1; + rc = sqlite3BtreeCommit(pBt); + } + } + }else{ + rc = SQLITE_OK; + } if( rc==SQLITE_OK ){ - rc = sqlite3BtreeBeginTrans(pBt, 0); - sqlite3PagerSnapshotOpen(sqlite3BtreePager(pBt), 0); + rc = sqlite3PagerSnapshotOpen(pPager, pSnapshot); + } + if( rc==SQLITE_OK ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + sqlite3PagerSnapshotOpen(pPager, 0); + } + if( bUnlock ){ + sqlite3PagerSnapshotUnlock(pPager); } } } @@ -139628,14 +186096,98 @@ SQLITE_API int SQLITE_STDCALL sqlite3_snapshot_open( return rc; } +/* +** Recover as many snapshots as possible from the wal file associated with +** schema zDb of database db. +*/ +SQLITE_API int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb){ + int rc = SQLITE_ERROR; +#ifndef SQLITE_OMIT_WAL + int iDb; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ){ + return SQLITE_MISUSE_BKPT; + } +#endif + + sqlite3_mutex_enter(db->mutex); + iDb = sqlite3FindDbName(db, zDb); + if( iDb==0 || iDb>1 ){ + Btree *pBt = db->aDb[iDb].pBt; + if( SQLITE_TXN_NONE==sqlite3BtreeTxnState(pBt) ){ + rc = sqlite3BtreeBeginTrans(pBt, 0, 0); + if( rc==SQLITE_OK ){ + rc = sqlite3PagerSnapshotRecover(sqlite3BtreePager(pBt)); + sqlite3BtreeCommit(pBt); + } + } + } + sqlite3_mutex_leave(db->mutex); +#endif /* SQLITE_OMIT_WAL */ + return rc; +} + /* ** Free a snapshot handle obtained from sqlite3_snapshot_get(). */ -SQLITE_API void SQLITE_STDCALL sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ +SQLITE_API void sqlite3_snapshot_free(sqlite3_snapshot *pSnapshot){ sqlite3_free(pSnapshot); } #endif /* SQLITE_ENABLE_SNAPSHOT */ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +/* +** Given the name of a compile-time option, return true if that option +** was used and false if not. +** +** The name can optionally begin with "SQLITE_" but the "SQLITE_" prefix +** is not required for a match. +*/ +SQLITE_API int sqlite3_compileoption_used(const char *zOptName){ + int i, n; + int nOpt; + const char **azCompileOpt; + +#ifdef SQLITE_ENABLE_API_ARMOR + if( zOptName==0 ){ + (void)SQLITE_MISUSE_BKPT; + return 0; + } +#endif + + azCompileOpt = sqlite3CompileOptions(&nOpt); + + if( sqlite3StrNICmp(zOptName, "SQLITE_", 7)==0 ) zOptName += 7; + n = sqlite3Strlen30(zOptName); + + /* Since nOpt is normally in single digits, a linear search is + ** adequate. No need for a binary search. */ + for(i=0; i=0 && NxUnlockNotify!=db->xUnlockNotify; + pp=&sqlite3BlockedList; + *pp && (*pp)->xUnlockNotify!=db->xUnlockNotify; pp=&(*pp)->pNextBlocked ); db->pNextBlocked = *pp; @@ -139748,20 +186300,20 @@ static void addToBlockedList(sqlite3 *db){ } /* -** Obtain the STATIC_MASTER mutex. +** Obtain the STATIC_MAIN mutex. */ static void enterMutex(void){ - sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_enter(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); checkListProperties(0); } /* -** Release the STATIC_MASTER mutex. +** Release the STATIC_MAIN mutex. */ static void leaveMutex(void){ assertMutexHeld(); checkListProperties(0); - sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MASTER)); + sqlite3_mutex_leave(sqlite3MutexAlloc(SQLITE_MUTEX_STATIC_MAIN)); } /* @@ -139785,13 +186337,16 @@ static void leaveMutex(void){ ** on the same "db". If xNotify==0 then any prior callbacks are immediately ** cancelled. */ -SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( +SQLITE_API int sqlite3_unlock_notify( sqlite3 *db, void (*xNotify)(void **, int), void *pArg ){ int rc = SQLITE_OK; +#ifdef SQLITE_ENABLE_API_ARMOR + if( !sqlite3SafetyCheckOk(db) ) return SQLITE_MISUSE_BKPT; +#endif sqlite3_mutex_enter(db->mutex); enterMutex(); @@ -139802,9 +186357,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( db->xUnlockNotify = 0; db->pUnlockArg = 0; }else if( 0==db->pBlockingConnection ){ - /* The blocking transaction has been concluded. Or there never was a + /* The blocking transaction has been concluded. Or there never was a ** blocking transaction. In either case, invoke the notify callback - ** immediately. + ** immediately. */ xNotify(&pArg, 1); }else{ @@ -139830,7 +186385,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_unlock_notify( } /* -** This function is called while stepping or preparing a statement +** This function is called while stepping or preparing a statement ** associated with connection db. The operation will return SQLITE_LOCKED ** to the user because it requires a lock that will not be available ** until connection pBlocker concludes its current transaction. @@ -139846,7 +186401,7 @@ SQLITE_PRIVATE void sqlite3ConnectionBlocked(sqlite3 *db, sqlite3 *pBlocker){ /* ** This function is called when -** the transaction opened by database db has just finished. Locks held +** the transaction opened by database db has just finished. Locks held ** by database connection db have been released. ** ** This function loops through each entry in the blocked connections @@ -139872,7 +186427,7 @@ SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ void *aStatic[16]; /* Starter space for aArg[]. No malloc required */ aArg = aStatic; - enterMutex(); /* Enter STATIC_MASTER mutex */ + enterMutex(); /* Enter STATIC_MAIN mutex */ /* This loop runs once for each entry in the blocked-connections list. */ for(pp=&sqlite3BlockedList; *pp; /* no-op */ ){ @@ -139906,7 +186461,7 @@ SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ }else{ /* This occurs when the array of context pointers that need to ** be passed to the unlock-notify callback is larger than the - ** aStatic[] array allocated on the stack and the attempt to + ** aStatic[] array allocated on the stack and the attempt to ** allocate a larger array from the heap has failed. ** ** This is a difficult situation to handle. Returning an error @@ -139914,17 +186469,17 @@ SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ ** is returned the transaction on connection db will still be ** closed and the unlock-notify callbacks on blocked connections ** will go unissued. This might cause the application to wait - ** indefinitely for an unlock-notify callback that will never + ** indefinitely for an unlock-notify callback that will never ** arrive. ** ** Instead, invoke the unlock-notify callback with the context ** array already accumulated. We can then clear the array and - ** begin accumulating any further context pointers without + ** begin accumulating any further context pointers without ** requiring any dynamic allocation. This is sub-optimal because ** it means that instead of one callback with a large array of ** context pointers the application will receive two or more ** callbacks with smaller arrays of context pointers, which will - ** reduce the applications ability to prioritize multiple + ** reduce the applications ability to prioritize multiple ** connections. But it is the best that can be done under the ** circumstances. */ @@ -139955,11 +186510,11 @@ SQLITE_PRIVATE void sqlite3ConnectionUnlocked(sqlite3 *db){ xUnlockNotify(aArg, nArg); } sqlite3_free(aDyn); - leaveMutex(); /* Leave STATIC_MASTER mutex */ + leaveMutex(); /* Leave STATIC_MAIN mutex */ } /* -** This is called when the database connection passed as an argument is +** This is called when the database connection passed as an argument is ** being closed. The connection is removed from the blocked list. */ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ @@ -140036,7 +186591,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** A doclist (document list) holds a docid-sorted list of hits for a ** given term. Doclists hold docids and associated token positions. ** A docid is the unique integer identifier for a single document. -** A position is the index of a word within the document. The first +** A position is the index of a word within the document. The first ** word of the document has a position of 0. ** ** FTS3 used to optionally store character offsets using a compile-time @@ -140061,7 +186616,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** ** Here, array { X } means zero or more occurrences of X, adjacent in ** memory. A "position" is an index of a token in the token stream -** generated by the tokenizer. Note that POS_END and POS_COLUMN occur +** generated by the tokenizer. Note that POS_END and POS_COLUMN occur ** in the same logical place as the position element, and act as sentinals ** ending a position list array. POS_END is 0. POS_COLUMN is 1. ** The positions numbers are not stored literally but rather as two more @@ -140085,7 +186640,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ ** a document record consists of a docid followed by a position-list and ** a doclist consists of one or more document records. ** -** A bare doclist omits the position information, becoming an +** A bare doclist omits the position information, becoming an ** array of varint-encoded docids. ** **** Segment leaf nodes **** @@ -140281,7 +186836,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ #ifndef _FTSINT_H #define _FTSINT_H -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif @@ -140304,7 +186859,7 @@ SQLITE_PRIVATE void sqlite3ConnectionClosed(sqlite3 *db){ /* If not building as part of the core, include sqlite3ext.h. */ #ifndef SQLITE_CORE -/* # include "sqlite3ext.h" */ +/* # include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT3 #endif @@ -140348,7 +186903,7 @@ SQLITE_EXTENSION_INIT3 ** When an fts3 table is created, it passes any arguments passed to ** the tokenizer clause of the CREATE VIRTUAL TABLE statement to the ** sqlite3_tokenizer_module.xCreate() function of the requested tokenizer -** implementation. The xCreate() function in turn returns an +** implementation. The xCreate() function in turn returns an ** sqlite3_tokenizer structure representing the specific tokenizer to ** be used for the fts3 table (customized by the tokenizer clause arguments). ** @@ -140380,7 +186935,7 @@ struct sqlite3_tokenizer_module { ** then argc is set to 2, and the argv[] array contains pointers ** to the strings "arg1" and "arg2". ** - ** This method should return either SQLITE_OK (0), or an SQLite error + ** This method should return either SQLITE_OK (0), or an SQLite error ** code. If SQLITE_OK is returned, then *ppTokenizer should be set ** to point at the newly created tokenizer structure. The generic ** sqlite3_tokenizer.pModule variable should not be initialized by @@ -140401,7 +186956,7 @@ struct sqlite3_tokenizer_module { /* ** Create a tokenizer cursor to tokenize an input buffer. The caller ** is responsible for ensuring that the input buffer remains valid - ** until the cursor is closed (using the xClose() method). + ** until the cursor is closed (using the xClose() method). */ int (*xOpen)( sqlite3_tokenizer *pTokenizer, /* Tokenizer object */ @@ -140410,7 +186965,7 @@ struct sqlite3_tokenizer_module { ); /* - ** Destroy an existing tokenizer cursor. The fts3 module calls this + ** Destroy an existing tokenizer cursor. The fts3 module calls this ** method exactly once for each successful call to xOpen(). */ int (*xClose)(sqlite3_tokenizer_cursor *pCursor); @@ -140421,7 +186976,7 @@ struct sqlite3_tokenizer_module { ** "OUT" variables identified below, or SQLITE_DONE to indicate that ** the end of the buffer has been reached, or an SQLite error code. ** - ** *ppToken should be set to point at a buffer containing the + ** *ppToken should be set to point at a buffer containing the ** normalized version of the token (i.e. after any case-folding and/or ** stemming has been performed). *pnBytes should be set to the length ** of this buffer in bytes. The input text that generated the token is @@ -140433,7 +186988,7 @@ struct sqlite3_tokenizer_module { ** ** The buffer *ppToken is set to point at is managed by the tokenizer ** implementation. It is only required to be valid until the next call - ** to xNext() or xClose(). + ** to xNext() or xClose(). */ /* TODO(shess) current implementation requires pInput to be ** nul-terminated. This should either be fixed, or pInput/nBytes @@ -140451,7 +187006,7 @@ struct sqlite3_tokenizer_module { ** Methods below this point are only available if iVersion>=1. */ - /* + /* ** Configure the language id of a tokenizer cursor. */ int (*xLanguageid)(sqlite3_tokenizer_cursor *pCsr, int iLangid); @@ -140520,7 +187075,7 @@ struct Fts3Hash { } *ht; }; -/* Each element in the hash table is an instance of the following +/* Each element in the hash table is an instance of the following ** structure. All elements are stored on a single doubly-linked list. ** ** Again, this structure is intended to be opaque, but it can't really @@ -140539,10 +187094,10 @@ struct Fts3HashElem { ** (including the null-terminator, if any). Case ** is respected in comparisons. ** -** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. +** FTS3_HASH_BINARY pKey points to binary data nKey bytes long. ** memcmp() is used to compare keys. ** -** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. +** A copy of the key is made if the copyKey parameter to fts3HashInit is 1. */ #define FTS3_HASH_STRING 1 #define FTS3_HASH_BINARY 2 @@ -140595,7 +187150,7 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi /* ** This constant determines the maximum depth of an FTS expression tree -** that the library will create and use. FTS uses recursion to perform +** that the library will create and use. FTS uses recursion to perform ** various operations on the query tree, so the disadvantage of a large ** limit is that it may allow very large queries to use large amounts ** of stack space (perhaps causing a stack overflow). @@ -140613,11 +187168,11 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi #define FTS3_MERGE_COUNT 16 /* -** This is the maximum amount of data (in bytes) to store in the +** This is the maximum amount of data (in bytes) to store in the ** Fts3Table.pendingTerms hash table. Normally, the hash table is ** populated as documents are inserted/updated/deleted in a transaction ** and used to create a new segment when the transaction is committed. -** However if this limit is reached midway through a transaction, a new +** However if this limit is reached midway through a transaction, a new ** segment is created and the hash table cleared immediately. */ #define FTS3_MAX_PENDING_DATA (1*1024*1024) @@ -140643,10 +187198,12 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi */ #define FTS3_VARINT_MAX 10 +#define FTS3_BUFFER_PADDING 8 + /* ** FTS4 virtual tables may maintain multiple indexes - one index of all terms ** in the document set and zero or more prefix indexes. All indexes are stored -** as one or more b+-trees in the %_segments and %_segdir tables. +** as one or more b+-trees in the %_segments and %_segdir tables. ** ** It is possible to determine which index a b+-tree belongs to based on the ** value stored in the "%_segdir.level" column. Given this value L, the index @@ -140654,8 +187211,8 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi ** level values between 0 and 1023 (inclusive) belong to index 0, all levels ** between 1024 and 2047 to index 1, and so on. ** -** It is considered impossible for an index to use more than 1024 levels. In -** theory though this may happen, but only after at least +** It is considered impossible for an index to use more than 1024 levels. In +** theory though this may happen, but only after at least ** (FTS3_MERGE_COUNT^1024) separate flushes of the pending-terms tables. */ #define FTS3_SEGDIR_MAXLEVEL 1024 @@ -140673,11 +187230,23 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi ** Terminator values for position-lists and column-lists. */ #define POS_COLUMN (1) /* Column-list terminator */ -#define POS_END (0) /* Position-list terminator */ +#define POS_END (0) /* Position-list terminator */ + +/* +** The assert_fts3_nc() macro is similar to the assert() macro, except that it +** is used for assert() conditions that are true only if it can be +** guranteed that the database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +SQLITE_API extern int sqlite3_fts3_may_be_corrupt; +# define assert_fts3_nc(x) assert(sqlite3_fts3_may_be_corrupt || (x)) +#else +# define assert_fts3_nc(x) assert(x) +#endif /* ** This section provides definitions to allow the -** FTS3 extension to be compiled outside of the +** FTS3 extension to be compiled outside of the ** amalgamation. */ #ifndef SQLITE_AMALGAMATION @@ -140685,17 +187254,18 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem(const Fts3Hash *, const voi ** Macros indicating that conditional expressions are always true or ** false. */ -#ifdef SQLITE_COVERAGE_TEST -# define ALWAYS(x) (1) -# define NEVER(X) (0) -#elif defined(SQLITE_DEBUG) -# define ALWAYS(x) sqlite3Fts3Always((x)!=0) -# define NEVER(x) sqlite3Fts3Never((x)!=0) -SQLITE_PRIVATE int sqlite3Fts3Always(int b); -SQLITE_PRIVATE int sqlite3Fts3Never(int b); +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) #else -# define ALWAYS(x) (x) -# define NEVER(x) (x) +# define ALWAYS(X) (X) +# define NEVER(X) (X) #endif /* @@ -140715,7 +187285,7 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ /* ** Activate assert() only if SQLITE_TEST is enabled. */ -#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) # define NDEBUG 1 #endif @@ -140730,6 +187300,11 @@ typedef sqlite3_int64 i64; /* 8-byte signed integer */ # define TESTONLY(X) #endif +#define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) +#define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) + +#define deliberate_fall_through + #endif /* SQLITE_AMALGAMATION */ #ifdef SQLITE_DEBUG @@ -140773,11 +187348,13 @@ struct Fts3Table { char *zLanguageid; /* languageid=xxx option, or NULL */ int nAutoincrmerge; /* Value configured by 'automerge' */ u32 nLeafAdd; /* Number of leaf blocks added this trans */ + int bLock; /* Used to prevent recursive content= tbls */ - /* Precompiled statements used by the implementation. Each of these - ** statements is run and reset within a single virtual table API call. + /* Precompiled statements used by the implementation. Each of these + ** statements is run and reset within a single virtual table API call. */ sqlite3_stmt *aStmt[40]; + sqlite3_stmt *pSeekStmt; /* Cache for fts3CursorSeekStmt() */ char *zReadExprlist; char *zWriteExprlist; @@ -140791,9 +187368,10 @@ struct Fts3Table { int nPgsz; /* Page size for host database */ char *zSegmentsTbl; /* Name of %_segments table */ sqlite3_blob *pSegments; /* Blob handle open on %_segments table */ + int iSavepoint; - /* - ** The following array of hash tables is used to buffer pending index + /* + ** The following array of hash tables is used to buffer pending index ** updates during transactions. All pending updates buffered at any one ** time must share a common language-id (see the FTS4 langid= feature). ** The current language id is stored in variable iPrevLangid. @@ -140803,10 +187381,10 @@ struct Fts3Table { ** terms that appear in the document set. Each subsequent index in aIndex[] ** is an index of prefixes of a specific length. ** - ** Variable nPendingData contains an estimate the memory consumed by the + ** Variable nPendingData contains an estimate the memory consumed by the ** pending data structures, including hash table overhead, but not including ** malloc overhead. When nPendingData exceeds nMaxPendingData, all hash - ** tables are flushed to disk. Variable iPrevDocid is the docid of the most + ** tables are flushed to disk. Variable iPrevDocid is the docid of the most ** recently inserted record. */ int nIndex; /* Size of aIndex[] */ @@ -140830,13 +187408,23 @@ struct Fts3Table { int mxSavepoint; /* Largest valid xSavepoint integer */ #endif -#ifdef SQLITE_TEST +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) /* True to disable the incremental doclist optimization. This is controled ** by special insert command 'test-no-incr-doclist'. */ int bNoIncrDoclist; + + /* Number of segments in a level */ + int nMergeCount; #endif }; +/* Macro to find the number of segments to merge */ +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) +# define MergeCount(P) ((P)->nMergeCount) +#else +# define MergeCount(P) FTS3_MERGE_COUNT +#endif + /* ** When the core wants to read from the virtual table, it creates a ** virtual table cursor (an instance of the following structure) using @@ -140847,6 +187435,7 @@ struct Fts3Cursor { i16 eSearch; /* Search strategy (see below) */ u8 isEof; /* True if at End Of Results */ u8 isRequireSeek; /* True if must seek pStmt to %_content row */ + u8 bSeekStmt; /* True if pStmt is a seek */ sqlite3_stmt *pStmt; /* Prepared statement in use by the cursor */ Fts3Expr *pExpr; /* Parsed MATCH query string */ int iLangid; /* Language being queried for */ @@ -140878,10 +187467,10 @@ struct Fts3Cursor { ** ** CREATE VIRTUAL TABLE ex1 USING fts3(a,b,c,d); ** SELECT docid FROM ex1 WHERE b MATCH 'one two three'; -** +** ** Because the LHS of the MATCH operator is 2nd column "b", ** Fts3Cursor.eSearch will be set to FTS3_FULLTEXT_SEARCH+1. (+0 for a, -** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" +** +1 for b, +2 for c, +3 for d.) If the LHS of MATCH were "ex1" ** indicating that all columns should be searched, ** then eSearch would be set to FTS3_FULLTEXT_SEARCH+4. */ @@ -140940,8 +187529,8 @@ struct Fts3Phrase { char *pOrPoslist; i64 iOrDocid; - /* Variables below this point are populated by fts3_expr.c when parsing - ** a MATCH expression. Everything above is part of the evaluation phase. + /* Variables below this point are populated by fts3_expr.c when parsing + ** a MATCH expression. Everything above is part of the evaluation phase. */ int nToken; /* Number of tokens in the phrase */ int iColumn; /* Index of column this phrase must match */ @@ -140951,10 +187540,10 @@ struct Fts3Phrase { /* ** A tree of these objects forms the RHS of a MATCH operator. ** -** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist -** points to a malloced buffer, size nDoclist bytes, containing the results -** of this phrase query in FTS3 doclist format. As usual, the initial -** "Length" field found in doclists stored on disk is omitted from this +** If Fts3Expr.eType is FTSQUERY_PHRASE and isLoaded is true, then aDoclist +** points to a malloced buffer, size nDoclist bytes, containing the results +** of this phrase query in FTS3 doclist format. As usual, the initial +** "Length" field found in doclists stored on disk is omitted from this ** buffer. ** ** Variable aMI is used only for FTSQUERY_NEAR nodes to store the global @@ -140966,7 +187555,7 @@ struct Fts3Phrase { ** aMI[iCol*3 + 1] = Number of occurrences ** aMI[iCol*3 + 2] = Number of rows containing at least one instance ** -** The aMI array is allocated using sqlite3_malloc(). It should be freed +** The aMI array is allocated using sqlite3_malloc(). It should be freed ** when the expression node is. */ struct Fts3Expr { @@ -140990,7 +187579,7 @@ struct Fts3Expr { /* ** Candidate values for Fts3Query.eType. Note that the order of the first -** four values is in order of precedence when parsing expressions. For +** four values is in order of precedence when parsing expressions. For ** example, the following: ** ** "a OR b AND c NOT d NEAR e" @@ -141047,7 +187636,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStart(Fts3Table*, Fts3MultiSegReader*, Ft SQLITE_PRIVATE int sqlite3Fts3SegReaderStep(Fts3Table *, Fts3MultiSegReader *); SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish(Fts3MultiSegReader *); -SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, +SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor(Fts3Table *, int, int, int, const char *, int, int, int, Fts3MultiSegReader *); /* Flags allowed as part of the 4th argument to SegmentReaderIterate() */ @@ -141073,7 +187662,7 @@ struct Fts3MultiSegReader { int nAdvance; /* How many seg-readers to advance */ Fts3SegFilter *pFilter; /* Pointer to filter object */ char *aBuffer; /* Buffer to merge doclists in */ - int nBuffer; /* Allocated size of aBuffer[] in bytes */ + i64 nBuffer; /* Allocated size of aBuffer[] in bytes */ int iColFilter; /* If >=0, filter for this column */ int bRestart; @@ -141099,6 +187688,8 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table*,int,int); SQLITE_PRIVATE void sqlite3Fts3ErrMsg(char**,const char*,...); SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *, sqlite3_int64); SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *, sqlite_int64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarintU(const char *, sqlite_uint64 *); +SQLITE_PRIVATE int sqlite3Fts3GetVarintBounded(const char*,const char*,sqlite3_int64*); SQLITE_PRIVATE int sqlite3Fts3GetVarint32(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3VarintLen(sqlite3_uint64); SQLITE_PRIVATE void sqlite3Fts3Dequote(char *); @@ -141107,11 +187698,12 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats(Fts3Cursor *, Fts3Expr *, u32 *); SQLITE_PRIVATE int sqlite3Fts3FirstFilter(sqlite3_int64, char *, int, char *); SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int*, Fts3Table*); SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc); +SQLITE_PRIVATE int sqlite3Fts3ReadInt(const char *z, int *pnOut); /* fts3_tokenizer.c */ SQLITE_PRIVATE const char *sqlite3Fts3NextToken(const char *, int *); SQLITE_PRIVATE int sqlite3Fts3InitHashTable(sqlite3 *, Fts3Hash *, const char *); -SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, +SQLITE_PRIVATE int sqlite3Fts3InitTokenizer(Fts3Hash *pHash, const char *, sqlite3_tokenizer **, char ** ); SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char); @@ -141130,9 +187722,10 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse(sqlite3_tokenizer *, int, ); SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *); #ifdef SQLITE_TEST -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db); +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash*); SQLITE_PRIVATE int sqlite3Fts3InitTerm(sqlite3 *db); #endif +SQLITE_PRIVATE void *sqlite3Fts3MallocZero(i64 nByte); SQLITE_PRIVATE int sqlite3Fts3OpenTokenizer(sqlite3_tokenizer *, int, const char *, int, sqlite3_tokenizer_cursor ** @@ -141147,12 +187740,12 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( Fts3Table*, Fts3MultiSegReader*, int, const char*, int); SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( Fts3Table *, Fts3MultiSegReader *, sqlite3_int64 *, char **, int *); -SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); +SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist(Fts3Cursor *, Fts3Expr *, int iCol, char **); SQLITE_PRIVATE int sqlite3Fts3MsrOvfl(Fts3Cursor *, Fts3MultiSegReader *, int *); SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr); /* fts3_tokenize_vtab.c */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *); +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3*, Fts3Hash *, void(*xDestroy)(void*)); /* fts3_unicode2.c (functions generated by parsing unicode text files) */ #ifndef SQLITE_DISABLE_FTS3_UNICODE @@ -141161,6 +187754,10 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int); SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); #endif +SQLITE_PRIVATE int sqlite3Fts3ExprIterate(Fts3Expr*, int (*x)(Fts3Expr*,int,void*), void*); + +SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk); + #endif /* !SQLITE_CORE || SQLITE_ENABLE_FTS3 */ #endif /* _FTSINT_H */ @@ -141180,24 +187777,33 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int); /* #include */ /* #include "fts3.h" */ -#ifndef SQLITE_CORE +#ifndef SQLITE_CORE /* # include "sqlite3ext.h" */ SQLITE_EXTENSION_INIT1 #endif +typedef struct Fts3HashWrapper Fts3HashWrapper; +struct Fts3HashWrapper { + Fts3Hash hash; /* Hash table */ + int nRef; /* Number of pointers to this object */ +}; + static int fts3EvalNext(Fts3Cursor *pCsr); static int fts3EvalStart(Fts3Cursor *pCsr); static int fts3TermSegReaderCursor( Fts3Cursor *, const char *, int, int, Fts3MultiSegReader **); -#ifndef SQLITE_AMALGAMATION -# if defined(SQLITE_DEBUG) -SQLITE_PRIVATE int sqlite3Fts3Always(int b) { assert( b ); return b; } -SQLITE_PRIVATE int sqlite3Fts3Never(int b) { assert( !b ); return b; } -# endif +/* +** This variable is set to false when running tests for which the on disk +** structures should not be corrupt. Otherwise, true. If it is false, extra +** assert() conditions in the fts3 code are activated - conditions that are +** only true if it is guaranteed that the fts3 database is not corrupt. +*/ +#ifdef SQLITE_DEBUG +SQLITE_API int sqlite3_fts3_may_be_corrupt = 1; #endif -/* +/* ** Write a 64-bit variable-length integer to memory starting at p[0]. ** The length of data written will be between 1 and FTS3_VARINT_MAX bytes. ** The number of bytes written is returned. @@ -141215,19 +187821,15 @@ SQLITE_PRIVATE int sqlite3Fts3PutVarint(char *p, sqlite_int64 v){ } #define GETVARINT_STEP(v, ptr, shift, mask1, mask2, var, ret) \ - v = (v & mask1) | ( (*ptr++) << shift ); \ + v = (v & mask1) | ( (*(const unsigned char*)(ptr++)) << shift ); \ if( (v & mask2)==0 ){ var = v; return ret; } #define GETVARINT_INIT(v, ptr, shift, mask1, mask2, var, ret) \ v = (*ptr++); \ if( (v & mask2)==0 ){ var = v; return ret; } -/* -** Read a 64-bit variable-length integer from memory starting at p[0]. -** Return the number of bytes read, or 0 on error. -** The value is stored in *v. -*/ -SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ - const char *pStart = p; +SQLITE_PRIVATE int sqlite3Fts3GetVarintU(const char *pBuf, sqlite_uint64 *v){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; u32 a; u64 b; int shift; @@ -141248,24 +187850,62 @@ SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *p, sqlite_int64 *v){ } /* -** Similar to sqlite3Fts3GetVarint(), except that the output is truncated to a -** 32-bit integer before it is returned. +** Read a 64-bit variable-length integer from memory starting at p[0]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarint(const char *pBuf, sqlite_int64 *v){ + return sqlite3Fts3GetVarintU(pBuf, (sqlite3_uint64*)v); +} + +/* +** Read a 64-bit variable-length integer from memory starting at p[0] and +** not extending past pEnd[-1]. +** Return the number of bytes read, or 0 on error. +** The value is stored in *v. +*/ +SQLITE_PRIVATE int sqlite3Fts3GetVarintBounded( + const char *pBuf, + const char *pEnd, + sqlite_int64 *v +){ + const unsigned char *p = (const unsigned char*)pBuf; + const unsigned char *pStart = p; + const unsigned char *pX = (const unsigned char*)pEnd; + u64 b = 0; + int shift; + for(shift=0; shift<=63; shift+=7){ + u64 c = p=0 ); return 5; } @@ -141304,7 +187944,7 @@ SQLITE_PRIVATE void sqlite3Fts3Dequote(char *z){ int iOut = 0; /* Index of next byte to write to output */ /* If the first byte was a '[', then the close-quote character is a ']' */ - if( quote=='[' ) quote = ']'; + if( quote=='[' ) quote = ']'; while( z[iIn] ){ if( z[iIn]==quote ){ @@ -141340,14 +187980,14 @@ static void fts3GetDeltaVarint(char **pp, sqlite3_int64 *pVal){ ** varint is part of. */ static void fts3GetReverseVarint( - char **pp, - char *pStart, + char **pp, + char *pStart, sqlite3_int64 *pVal ){ sqlite3_int64 iVal; char *p; - /* Pointer p now points at the first byte past the varint we are + /* Pointer p now points at the first byte past the varint we are ** interested in. So, unless the doclist is corrupt, the 0x80 bit is ** clear on character p[-1]. */ for(p = (*pp)-2; p>=pStart && *p&0x80; p--); @@ -141369,6 +188009,7 @@ static int fts3DisconnectMethod(sqlite3_vtab *pVtab){ assert( p->pSegments==0 ); /* Free any prepared statements held */ + sqlite3_finalize(p->pSeekStmt); for(i=0; iaStmt); i++){ sqlite3_finalize(p->aStmt[i]); } @@ -141433,13 +188074,18 @@ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ sqlite3 *db = p->db; /* Database handle */ /* Drop the shadow tables */ - if( p->zContentTbl==0 ){ - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_content'", zDb, p->zName); - } - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segments'", zDb,p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_segdir'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_docsize'", zDb, p->zName); - fts3DbExec(&rc, db, "DROP TABLE IF EXISTS %Q.'%q_stat'", zDb, p->zName); + fts3DbExec(&rc, db, + "DROP TABLE IF EXISTS %Q.'%q_segments';" + "DROP TABLE IF EXISTS %Q.'%q_segdir';" + "DROP TABLE IF EXISTS %Q.'%q_docsize';" + "DROP TABLE IF EXISTS %Q.'%q_stat';" + "%s DROP TABLE IF EXISTS %Q.'%q_content';", + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + zDb, p->zName, + (p->zContentTbl ? "--" : ""), zDb,p->zName + ); /* If everything has worked, invoke fts3DisconnectMethod() to free the ** memory associated with the Fts3Table structure and return SQLITE_OK. @@ -141454,7 +188100,7 @@ static int fts3DestroyMethod(sqlite3_vtab *pVtab){ ** passed as the first argument. This is done as part of the xConnect() ** and xCreate() methods. ** -** If *pRc is non-zero when this function is called, it is a no-op. +** If *pRc is non-zero when this function is called, it is a no-op. ** Otherwise, if an error occurs, an SQLite error code is stored in *pRc ** before returning. */ @@ -141468,6 +188114,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ zLanguageid = (p->zLanguageid ? p->zLanguageid : "__langid"); sqlite3_vtab_config(p->db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(p->db, SQLITE_VTAB_INNOCUOUS); /* Create a list of user columns for the virtual table */ zCols = sqlite3_mprintf("%Q, ", p->azColumn[0]); @@ -141477,7 +188124,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ /* Create the whole "CREATE TABLE" statement to pass to SQLite */ zSql = sqlite3_mprintf( - "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", + "CREATE TABLE x(%s %Q HIDDEN, docid HIDDEN, %Q HIDDEN)", zCols, p->zName, zLanguageid ); if( !zCols || !zSql ){ @@ -141496,7 +188143,7 @@ static void fts3DeclareVtab(int *pRc, Fts3Table *p){ ** Create the %_stat table if it does not already exist. */ SQLITE_PRIVATE void sqlite3Fts3CreateStatTable(int *pRc, Fts3Table *p){ - fts3DbExec(pRc, p->db, + fts3DbExec(pRc, p->db, "CREATE TABLE IF NOT EXISTS %Q.'%q_stat'" "(id INTEGER PRIMARY KEY, value BLOB);", p->zDb, p->zName @@ -141532,9 +188179,9 @@ static int fts3CreateTables(Fts3Table *p){ zContentCols = sqlite3_mprintf("%z, langid", zContentCols, zLanguageid); } if( zContentCols==0 ) rc = SQLITE_NOMEM; - + /* Create the content table */ - fts3DbExec(&rc, db, + fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_content'(%s)", p->zDb, p->zName, zContentCols ); @@ -141542,11 +188189,11 @@ static int fts3CreateTables(Fts3Table *p){ } /* Create other tables */ - fts3DbExec(&rc, db, + fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segments'(blockid INTEGER PRIMARY KEY, block BLOB);", p->zDb, p->zName ); - fts3DbExec(&rc, db, + fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_segdir'(" "level INTEGER," "idx INTEGER," @@ -141559,7 +188206,7 @@ static int fts3CreateTables(Fts3Table *p){ p->zDb, p->zName ); if( p->bHasDocsize ){ - fts3DbExec(&rc, db, + fts3DbExec(&rc, db, "CREATE TABLE %Q.'%q_docsize'(docid INTEGER PRIMARY KEY, size BLOB);", p->zDb, p->zName ); @@ -141574,7 +188221,7 @@ static int fts3CreateTables(Fts3Table *p){ /* ** Store the current database page-size in bytes in p->nPgsz. ** -** If *pRc is non-zero when this function is called, it is a no-op. +** If *pRc is non-zero when this function is called, it is a no-op. ** Otherwise, if an error occurs, an SQLite error code is stored in *pRc ** before returning. */ @@ -141583,7 +188230,7 @@ static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ int rc; /* Return code */ char *zSql; /* SQL text "PRAGMA %Q.page_size" */ sqlite3_stmt *pStmt; /* Compiled "PRAGMA %Q.page_size" statement */ - + zSql = sqlite3_mprintf("PRAGMA %Q.page_size", p->zDb); if( !zSql ){ rc = SQLITE_NOMEM; @@ -141609,11 +188256,11 @@ static void fts3DatabasePageSize(int *pRc, Fts3Table *p){ ** ** = ** -** There may not be whitespace surrounding the "=" character. The +** There may not be whitespace surrounding the "=" character. The ** term may be quoted, but the may not. */ static int fts3IsSpecialColumn( - const char *z, + const char *z, int *pnKey, char **pzValue ){ @@ -141671,10 +188318,10 @@ static void fts3Appendf( ** memory. */ static char *fts3QuoteId(char const *zInput){ - int nRet; + sqlite3_int64 nRet; char *zRet; nRet = 2 + (int)strlen(zInput)*2 + 1; - zRet = sqlite3_malloc(nRet); + zRet = sqlite3_malloc64(nRet); if( zRet ){ int i; char *z = zRet; @@ -141690,7 +188337,7 @@ static char *fts3QuoteId(char const *zInput){ } /* -** Return a list of comma separated SQL expressions and a FROM clause that +** Return a list of comma separated SQL expressions and a FROM clause that ** could be used in a SELECT statement such as the following: ** ** SELECT FROM %_content AS x ... @@ -141741,7 +188388,7 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ fts3Appendf(pRc, &zRet, ", x.%Q", p->zLanguageid); } } - fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", + fts3Appendf(pRc, &zRet, " FROM '%q'.'%q%s' AS x", p->zDb, (p->zContentTbl ? p->zContentTbl : p->zName), (p->zContentTbl ? "" : "_content") @@ -141756,7 +188403,7 @@ static char *fts3ReadExprList(Fts3Table *p, const char *zFunc, int *pRc){ ** ** If argument zFunc is not NULL, then all but the first question mark ** is preceded by zFunc and an open bracket, and followed by a closed -** bracket. For example, if zFunc is "zip" and the FTS3 table has three +** bracket. For example, if zFunc is "zip" and the FTS3 table has three ** user-defined text columns, the following string is returned: ** ** "?, zip(?), zip(?), zip(?)" @@ -141791,13 +188438,29 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ return zRet; } +/* +** Buffer z contains a positive integer value encoded as utf-8 text. +** Decode this value and store it in *pnOut, returning the number of bytes +** consumed. If an overflow error occurs return a negative value. +*/ +SQLITE_PRIVATE int sqlite3Fts3ReadInt(const char *z, int *pnOut){ + u64 iVal = 0; + int i; + for(i=0; z[i]>='0' && z[i]<='9'; i++){ + iVal = iVal*10 + (z[i] - '0'); + if( iVal>0x7FFFFFFF ) return -1; + } + *pnOut = (int)iVal; + return i; +} + /* ** This function interprets the string at (*pp) as a non-negative integer -** value. It reads the integer and sets *pnOut to the value read, then +** value. It reads the integer and sets *pnOut to the value read, then ** sets *pp to point to the byte immediately following the last byte of ** the integer value. ** -** Only decimal digits ('0'..'9') may be part of an integer value. +** Only decimal digits ('0'..'9') may be part of an integer value. ** ** If *pp does not being with a decimal digit SQLITE_ERROR is returned and ** the output value undefined. Otherwise SQLITE_OK is returned. @@ -141806,19 +188469,17 @@ static char *fts3WriteExprList(Fts3Table *p, const char *zFunc, int *pRc){ */ static int fts3GobbleInt(const char **pp, int *pnOut){ const int MAX_NPREFIX = 10000000; - const char *p; /* Iterator pointer */ int nInt = 0; /* Output value */ - - for(p=*pp; p[0]>='0' && p[0]<='9'; p++){ - nInt = nInt * 10 + (p[0] - '0'); - if( nInt>MAX_NPREFIX ){ - nInt = 0; - break; - } + int nByte; + nByte = sqlite3Fts3ReadInt(*pp, &nInt); + if( nInt>MAX_NPREFIX ){ + nInt = 0; + } + if( nByte==0 ){ + return SQLITE_ERROR; } - if( p==*pp ) return SQLITE_ERROR; *pnOut = nInt; - *pp = p; + *pp += nByte; return SQLITE_OK; } @@ -141855,7 +188516,7 @@ static int fts3PrefixParameter( } } - aIndex = sqlite3_malloc(sizeof(struct Fts3Index) * nIndex); + aIndex = sqlite3_malloc64(sizeof(struct Fts3Index) * nIndex); *apIndex = aIndex; if( !aIndex ){ return SQLITE_NOMEM; @@ -141918,7 +188579,7 @@ static int fts3ContentColumns( char **pzErr /* OUT: error message */ ){ int rc = SQLITE_OK; /* Return code */ - char *zSql; /* "SELECT *" statement on zTbl */ + char *zSql; /* "SELECT *" statement on zTbl */ sqlite3_stmt *pStmt = 0; /* Compiled version of zSql */ zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", zDb, zTbl); @@ -141934,7 +188595,7 @@ static int fts3ContentColumns( if( rc==SQLITE_OK ){ const char **azCol; /* Output array */ - int nStr = 0; /* Size of all column names (incl. 0x00) */ + sqlite3_int64 nStr = 0; /* Size of all column names (incl. 0x00) */ int nCol; /* Number of table columns */ int i; /* Used to iterate through columns */ @@ -141944,11 +188605,11 @@ static int fts3ContentColumns( nCol = sqlite3_column_count(pStmt); for(i=0; ihash; Fts3Table *p = 0; /* Pointer to allocated vtab */ int rc = SQLITE_OK; /* Return code */ int i; /* Iterator variable */ - int nByte; /* Size of allocation used for *p */ + sqlite3_int64 nByte; /* Size of allocation used for *p */ int iCol; /* Column index */ int nString = 0; /* Bytes required to hold all column names */ int nCol = 0; /* Number of columns in the FTS table */ @@ -142030,10 +188691,10 @@ static int fts3InitVtab( nName = (int)strlen(argv[2]) + 1; nByte = sizeof(const char *) * (argc-2); - aCol = (const char **)sqlite3_malloc(nByte); + aCol = (const char **)sqlite3_malloc64(nByte); if( aCol ){ memset((void*)aCol, 0, nByte); - azNotindexed = (char **)sqlite3_malloc(nByte); + azNotindexed = (char **)sqlite3_malloc64(nByte); } if( azNotindexed ){ memset(azNotindexed, 0, nByte); @@ -142060,9 +188721,9 @@ static int fts3InitVtab( char *zVal; /* Check if this is a tokenizer specification */ - if( !pTokenizer + if( !pTokenizer && strlen(z)>8 - && 0==sqlite3_strnicmp(z, "tokenize", 8) + && 0==sqlite3_strnicmp(z, "tokenize", 8) && 0==sqlite3Fts3IsIdChar(z[8]) ){ rc = sqlite3Fts3InitTokenizer(pHash, &z[9], &pTokenizer, pzErr); @@ -142094,65 +188755,66 @@ static int fts3InitVtab( break; } } - if( iOpt==SizeofArray(aFts4Opt) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); - rc = SQLITE_ERROR; - }else{ - switch( iOpt ){ - case 0: /* MATCHINFO */ - if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); - rc = SQLITE_ERROR; - } - bNoDocsize = 1; - break; + switch( iOpt ){ + case 0: /* MATCHINFO */ + if( strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "fts3", 4) ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized matchinfo: %s", zVal); + rc = SQLITE_ERROR; + } + bNoDocsize = 1; + break; - case 1: /* PREFIX */ - sqlite3_free(zPrefix); - zPrefix = zVal; - zVal = 0; - break; + case 1: /* PREFIX */ + sqlite3_free(zPrefix); + zPrefix = zVal; + zVal = 0; + break; - case 2: /* COMPRESS */ - sqlite3_free(zCompress); - zCompress = zVal; - zVal = 0; - break; + case 2: /* COMPRESS */ + sqlite3_free(zCompress); + zCompress = zVal; + zVal = 0; + break; - case 3: /* UNCOMPRESS */ - sqlite3_free(zUncompress); - zUncompress = zVal; - zVal = 0; - break; + case 3: /* UNCOMPRESS */ + sqlite3_free(zUncompress); + zUncompress = zVal; + zVal = 0; + break; - case 4: /* ORDER */ - if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) - && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) - ){ - sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); - rc = SQLITE_ERROR; - } - bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); - break; + case 4: /* ORDER */ + if( (strlen(zVal)!=3 || sqlite3_strnicmp(zVal, "asc", 3)) + && (strlen(zVal)!=4 || sqlite3_strnicmp(zVal, "desc", 4)) + ){ + sqlite3Fts3ErrMsg(pzErr, "unrecognized order: %s", zVal); + rc = SQLITE_ERROR; + } + bDescIdx = (zVal[0]=='d' || zVal[0]=='D'); + break; - case 5: /* CONTENT */ - sqlite3_free(zContent); - zContent = zVal; - zVal = 0; - break; + case 5: /* CONTENT */ + sqlite3_free(zContent); + zContent = zVal; + zVal = 0; + break; - case 6: /* LANGUAGEID */ - assert( iOpt==6 ); - sqlite3_free(zLanguageid); - zLanguageid = zVal; - zVal = 0; - break; + case 6: /* LANGUAGEID */ + assert( iOpt==6 ); + sqlite3_free(zLanguageid); + zLanguageid = zVal; + zVal = 0; + break; - case 7: /* NOTINDEXED */ - azNotindexed[nNotindexed++] = zVal; - zVal = 0; - break; - } + case 7: /* NOTINDEXED */ + azNotindexed[nNotindexed++] = zVal; + zVal = 0; + break; + + default: + assert( iOpt==SizeofArray(aFts4Opt) ); + sqlite3Fts3ErrMsg(pzErr, "unrecognized parameter: %s", z); + rc = SQLITE_ERROR; + break; } sqlite3_free(zVal); } @@ -142173,17 +188835,17 @@ static int fts3InitVtab( ** TABLE statement, use all columns from the content table. */ if( rc==SQLITE_OK && zContent ){ - sqlite3_free(zCompress); - sqlite3_free(zUncompress); + sqlite3_free(zCompress); + sqlite3_free(zUncompress); zCompress = 0; zUncompress = 0; if( nCol==0 ){ - sqlite3_free((void*)aCol); + sqlite3_free((void*)aCol); aCol = 0; rc = fts3ContentColumns(db, argv[1], zContent,&aCol,&nCol,&nString,pzErr); /* If a languageid= option was specified, remove the language id - ** column from the aCol[] array. */ + ** column from the aCol[] array. */ if( rc==SQLITE_OK && zLanguageid ){ int j; for(j=0; jpTokenizer = pTokenizer; p->nMaxPendingData = FTS3_MAX_PENDING_DATA; p->bHasDocsize = (isFts4 && bNoDocsize==0); - p->bHasStat = isFts4; - p->bFts4 = isFts4; - p->bDescIdx = bDescIdx; + p->bHasStat = (u8)isFts4; + p->bFts4 = (u8)isFts4; + p->bDescIdx = (u8)bDescIdx; p->nAutoincrmerge = 0xff; /* 0xff means setting unknown */ p->zContentTbl = zContent; p->zLanguageid = zLanguageid; @@ -142270,10 +188932,12 @@ static int fts3InitVtab( /* Fill in the azColumn array */ for(iCol=0; iCol0 ){ + memcpy(zCsr, z, n); + } zCsr[n] = '\0'; sqlite3Fts3Dequote(zCsr); p->azColumn[iCol] = zCsr; @@ -142287,7 +188951,7 @@ static int fts3InitVtab( for(i=0; iazColumn[iCol], zNot, n) + && 0==sqlite3_strnicmp(p->azColumn[iCol], zNot, n) ){ p->abNotindexed[iCol] = 1; sqlite3_free(zNot); @@ -142311,7 +188975,7 @@ static int fts3InitVtab( p->zWriteExprlist = fts3WriteExprList(p, zCompress, &rc); if( rc!=SQLITE_OK ) goto fts3_init_out; - /* If this is an xCreate call, create the underlying tables in the + /* If this is an xCreate call, create the underlying tables in the ** database. TODO: For xConnect(), it could verify that said tables exist. */ if( isCreate ){ @@ -142330,6 +188994,10 @@ static int fts3InitVtab( fts3DatabasePageSize(&rc, p); p->nNodeSize = p->nPgsz-35; +#if defined(SQLITE_DEBUG)||defined(SQLITE_TEST) + p->nMergeCount = FTS3_MERGE_COUNT; +#endif + /* Declare the table schema to SQLite. */ fts3DeclareVtab(&rc, p); @@ -142407,11 +189075,11 @@ static void fts3SetUniqueFlag(sqlite3_index_info *pIdxInfo){ #endif } -/* +/* ** Implementation of the xBestIndex method for FTS3 tables. There ** are three possible strategies, in order of preference: ** -** 1. Direct lookup by rowid or docid. +** 1. Direct lookup by rowid or docid. ** 2. Full-text search using a MATCH operator on a non-docid column. ** 3. Linear scan of %_content table. */ @@ -142425,8 +189093,12 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ int iDocidLe = -1; /* Index of docid<=x constraint, if present */ int iIdx; + if( p->bLock ){ + return SQLITE_ERROR; + } + /* By default use a full table scan. This is an expensive option, - ** so search through the constraints to see if a more efficient + ** so search through the constraints to see if a more efficient ** strategy is possible. */ pInfo->idxNum = FTS3_FULLSCAN_SEARCH; @@ -142462,12 +189134,12 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ ** ** If there is more than one MATCH constraint available, use the first ** one encountered. If there is both a MATCH constraint and a direct - ** rowid/docid lookup, prefer the MATCH strategy. This is done even + ** rowid/docid lookup, prefer the MATCH strategy. This is done even ** though the rowid/docid lookup is faster than a MATCH query, selecting - ** it would lead to an "unable to use function MATCH in the requested + ** it would lead to an "unable to use function MATCH in the requested ** context" error. */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH + if( pCons->op==SQLITE_INDEX_CONSTRAINT_MATCH && pCons->iColumn>=0 && pCons->iColumn<=p->nColumn ){ pInfo->idxNum = FTS3_FULLTEXT_SEARCH + pCons->iColumn; @@ -142476,7 +189148,7 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } /* Equality constraint on the langid column */ - if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ + if( pCons->op==SQLITE_INDEX_CONSTRAINT_EQ && pCons->iColumn==p->nColumn + 2 ){ iLangidCons = i; @@ -142504,22 +189176,22 @@ static int fts3BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ if( iCons>=0 ){ pInfo->aConstraintUsage[iCons].argvIndex = iIdx++; pInfo->aConstraintUsage[iCons].omit = 1; - } + } if( iLangidCons>=0 ){ pInfo->idxNum |= FTS3_HAVE_LANGID; pInfo->aConstraintUsage[iLangidCons].argvIndex = iIdx++; - } + } if( iDocidGe>=0 ){ pInfo->idxNum |= FTS3_HAVE_DOCID_GE; pInfo->aConstraintUsage[iDocidGe].argvIndex = iIdx++; - } + } if( iDocidLe>=0 ){ pInfo->idxNum |= FTS3_HAVE_DOCID_LE; pInfo->aConstraintUsage[iDocidLe].argvIndex = iIdx++; - } + } /* Regardless of the strategy selected, FTS can deliver rows in rowid (or - ** docid) order. Both ascending and descending are possible. + ** docid) order. Both ascending and descending are possible. */ if( pInfo->nOrderBy==1 ){ struct sqlite3_index_orderby *pOrder = &pInfo->aOrderBy[0]; @@ -142546,7 +189218,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ UNUSED_PARAMETER(pVTab); /* Allocate a buffer large enough for an Fts3Cursor structure. If the - ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, + ** allocation succeeds, zero it and return SQLITE_OK. Otherwise, ** if the allocation fails, return SQLITE_NOMEM. */ *ppCsr = pCsr = (sqlite3_vtab_cursor *)sqlite3_malloc(sizeof(Fts3Cursor)); @@ -142557,6 +189229,39 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ return SQLITE_OK; } +/* +** Finalize the statement handle at pCsr->pStmt. +** +** Or, if that statement handle is one created by fts3CursorSeekStmt(), +** and the Fts3Table.pSeekStmt slot is currently NULL, save the statement +** pointer there instead of finalizing it. +*/ +static void fts3CursorFinalizeStmt(Fts3Cursor *pCsr){ + if( pCsr->bSeekStmt ){ + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; + if( p->pSeekStmt==0 ){ + p->pSeekStmt = pCsr->pStmt; + sqlite3_reset(pCsr->pStmt); + pCsr->pStmt = 0; + } + pCsr->bSeekStmt = 0; + } + sqlite3_finalize(pCsr->pStmt); +} + +/* +** Free all resources currently held by the cursor passed as the only +** argument. +*/ +static void fts3ClearCursor(Fts3Cursor *pCsr){ + fts3CursorFinalizeStmt(pCsr); + sqlite3Fts3FreeDeferredTokens(pCsr); + sqlite3_free(pCsr->aDoclist); + sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + sqlite3Fts3ExprFree(pCsr->pExpr); + memset(&(&pCsr->base)[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); +} + /* ** Close the cursor. For additional information see the documentation ** on the xClose method of the virtual table interface. @@ -142564,11 +189269,7 @@ static int fts3OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); - sqlite3_finalize(pCsr->pStmt); - sqlite3Fts3ExprFree(pCsr->pExpr); - sqlite3Fts3FreeDeferredTokens(pCsr); - sqlite3_free(pCsr->aDoclist); - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); + fts3ClearCursor(pCsr); assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); sqlite3_free(pCsr); return SQLITE_OK; @@ -142582,40 +189283,49 @@ static int fts3CloseMethod(sqlite3_vtab_cursor *pCursor){ ** ** (or the equivalent for a content=xxx table) and set pCsr->pStmt to ** it. If an error occurs, return an SQLite error code. -** -** Otherwise, set *ppStmt to point to pCsr->pStmt and return SQLITE_OK. */ -static int fts3CursorSeekStmt(Fts3Cursor *pCsr, sqlite3_stmt **ppStmt){ +static int fts3CursorSeekStmt(Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->pStmt==0 ){ Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; char *zSql; - zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); - if( !zSql ) return SQLITE_NOMEM; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); - sqlite3_free(zSql); + if( p->pSeekStmt ){ + pCsr->pStmt = p->pSeekStmt; + p->pSeekStmt = 0; + }else{ + zSql = sqlite3_mprintf("SELECT %s WHERE rowid = ?", p->zReadExprlist); + if( !zSql ) return SQLITE_NOMEM; + p->bLock++; + rc = sqlite3_prepare_v3( + p->db, zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; + sqlite3_free(zSql); + } + if( rc==SQLITE_OK ) pCsr->bSeekStmt = 1; } - *ppStmt = pCsr->pStmt; return rc; } /* ** Position the pCsr->pStmt statement so that it is on the row ** of the %_content table that contains the last match. Return -** SQLITE_OK on success. +** SQLITE_OK on success. */ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ int rc = SQLITE_OK; if( pCsr->isRequireSeek ){ - sqlite3_stmt *pStmt = 0; - - rc = fts3CursorSeekStmt(pCsr, &pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ + Fts3Table *pTab = (Fts3Table*)pCsr->base.pVtab; + pTab->bLock++; sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iPrevId); pCsr->isRequireSeek = 0; if( SQLITE_ROW==sqlite3_step(pCsr->pStmt) ){ + pTab->bLock--; return SQLITE_OK; }else{ + pTab->bLock--; rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK && ((Fts3Table *)pCsr->base.pVtab)->zContentTbl==0 ){ /* If no row was found and no error has occurred, then the %_content @@ -142636,7 +189346,7 @@ static int fts3CursorSeek(sqlite3_context *pContext, Fts3Cursor *pCsr){ /* ** This function is used to process a single interior node when searching -** a b-tree for a term or term prefix. The node data is passed to this +** a b-tree for a term or term prefix. The node data is passed to this ** function via the zNode/nNode parameters. The term to search for is ** passed in zTerm/nTerm. ** @@ -142661,13 +189371,14 @@ static int fts3ScanInteriorNode( const char *zCsr = zNode; /* Cursor to iterate through node */ const char *zEnd = &zCsr[nNode];/* End of interior node buffer */ char *zBuffer = 0; /* Buffer to load terms into */ - int nAlloc = 0; /* Size of allocated buffer */ + i64 nAlloc = 0; /* Size of allocated buffer */ int isFirstTerm = 1; /* True when processing first term on page */ - sqlite3_int64 iChild; /* Block id of child node to descend to */ + u64 iChild; /* Block id of child node to descend to */ + int nBuffer = 0; /* Total term size */ - /* Skip over the 'height' varint that occurs at the start of every + /* Skip over the 'height' varint that occurs at the start of every ** interior node. Then load the blockid of the left-child of the b-tree - ** node into variable iChild. + ** node into variable iChild. ** ** Even if the data structure on disk is corrupted, this (reading two ** varints from the buffer) does not risk an overread. If zNode is a @@ -142678,34 +189389,38 @@ static int fts3ScanInteriorNode( ** table, then there are always 20 bytes of zeroed padding following the ** nNode bytes of content (see sqlite3Fts3ReadBlock() for details). */ - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); - zCsr += sqlite3Fts3GetVarint(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild); + zCsr += sqlite3Fts3GetVarintU(zCsr, &iChild); if( zCsr>zEnd ){ return FTS_CORRUPT_VTAB; } - + while( zCsrnBuffer ){ + rc = FTS_CORRUPT_VTAB; + goto finish_scan; + } } isFirstTerm = 0; zCsr += fts3GetVarint32(zCsr, &nSuffix); - - if( nPrefix<0 || nSuffix<0 || &zCsr[nSuffix]>zEnd ){ + + assert( nPrefix>=0 && nSuffix>=0 ); + if( nPrefix>zCsr-zNode || nSuffix>zEnd-zCsr || nSuffix==0 ){ rc = FTS_CORRUPT_VTAB; goto finish_scan; } - if( nPrefix+nSuffix>nAlloc ){ + if( (i64)nPrefix+nSuffix>nAlloc ){ char *zNew; - nAlloc = (nPrefix+nSuffix) * 2; - zNew = (char *)sqlite3_realloc(zBuffer, nAlloc); + nAlloc = ((i64)nPrefix+nSuffix) * 2; + zNew = (char *)sqlite3_realloc64(zBuffer, nAlloc); if( !zNew ){ rc = SQLITE_NOMEM; goto finish_scan; @@ -142719,8 +189434,8 @@ static int fts3ScanInteriorNode( /* Compare the term we are searching for with the term just loaded from ** the interior node. If the specified term is greater than or equal - ** to the term from the interior node, then all terms on the sub-tree - ** headed by node iChild are smaller than zTerm. No need to search + ** to the term from the interior node, then all terms on the sub-tree + ** headed by node iChild are smaller than zTerm. No need to search ** iChild. ** ** If the interior node term is larger than the specified term, then @@ -142728,20 +189443,20 @@ static int fts3ScanInteriorNode( */ cmp = memcmp(zTerm, zBuffer, (nBuffer>nTerm ? nTerm : nBuffer)); if( piFirst && (cmp<0 || (cmp==0 && nBuffer>nTerm)) ){ - *piFirst = iChild; + *piFirst = (i64)iChild; piFirst = 0; } if( piLast && cmp<0 ){ - *piLast = iChild; + *piLast = (i64)iChild; piLast = 0; } iChild++; }; - if( piFirst ) *piFirst = iChild; - if( piLast ) *piLast = iChild; + if( piFirst ) *piFirst = (i64)iChild; + if( piLast ) *piLast = (i64)iChild; finish_scan: sqlite3_free(zBuffer); @@ -142756,20 +189471,20 @@ static int fts3ScanInteriorNode( ** node for the range of leaf nodes that may contain the specified term ** or terms for which the specified term is a prefix. ** -** If piLeaf is not NULL, then *piLeaf is set to the blockid of the +** If piLeaf is not NULL, then *piLeaf is set to the blockid of the ** left-most leaf node in the tree that may contain the specified term. ** If piLeaf2 is not NULL, then *piLeaf2 is set to the blockid of the ** right-most leaf node that may contain a term for which the specified ** term is a prefix. ** -** It is possible that the range of returned leaf nodes does not contain -** the specified term or any terms for which it is a prefix. However, if the +** It is possible that the range of returned leaf nodes does not contain +** the specified term or any terms for which it is a prefix. However, if the ** segment does contain any such terms, they are stored within the identified ** range. Because this function only inspects interior segment nodes (and ** never loads leaf nodes into memory), it is not possible to be sure. ** ** If an error occurs, an error code other than SQLITE_OK is returned. -*/ +*/ static int fts3SelectLeaf( Fts3Table *p, /* Virtual table handle */ const char *zTerm, /* Term to select leaves for */ @@ -142786,7 +189501,7 @@ static int fts3SelectLeaf( fts3GetVarint32(zNode, &iHeight); rc = fts3ScanInteriorNode(zTerm, nTerm, zNode, nNode, piLeaf, piLeaf2); - assert( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); + assert_fts3_nc( !piLeaf2 || !piLeaf || rc!=SQLITE_OK || (*piLeaf<=*piLeaf2) ); if( rc==SQLITE_OK && iHeight>1 ){ char *zBlob = 0; /* Blob read from %_segments table */ @@ -142806,7 +189521,13 @@ static int fts3SelectLeaf( rc = sqlite3Fts3ReadBlock(p, piLeaf?*piLeaf:*piLeaf2, &zBlob, &nBlob, 0); } if( rc==SQLITE_OK ){ - rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + int iNewHeight = 0; + fts3GetVarint32(zBlob, &iNewHeight); + if( iNewHeight>=iHeight ){ + rc = FTS_CORRUPT_VTAB; + }else{ + rc = fts3SelectLeaf(p, zTerm, nTerm, zBlob, nBlob, piLeaf, piLeaf2); + } } sqlite3_free(zBlob); } @@ -142815,7 +189536,7 @@ static int fts3SelectLeaf( } /* -** This function is used to create delta-encoded serialized lists of FTS3 +** This function is used to create delta-encoded serialized lists of FTS3 ** varints. Each call to this function appends a single varint to a list. */ static void fts3PutDeltaVarint( @@ -142823,17 +189544,17 @@ static void fts3PutDeltaVarint( sqlite3_int64 *piPrev, /* IN/OUT: Previous value written to list */ sqlite3_int64 iVal /* Write this value to the list */ ){ - assert( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); + assert_fts3_nc( iVal-*piPrev > 0 || (*piPrev==0 && iVal==0) ); *pp += sqlite3Fts3PutVarint(*pp, iVal-*piPrev); *piPrev = iVal; } /* -** When this function is called, *ppPoslist is assumed to point to the +** When this function is called, *ppPoslist is assumed to point to the ** start of a position-list. After it returns, *ppPoslist points to the ** first byte after the position-list. ** -** A position list is list of positions (delta encoded) and columns for +** A position list is list of positions (delta encoded) and columns for ** a single document record of a doclist. So, in other words, this ** routine advances *ppPoslist so that it points to the next docid in ** the doclist, or to the first byte past the end of the doclist. @@ -142846,12 +189567,12 @@ static void fts3PoslistCopy(char **pp, char **ppPoslist){ char *pEnd = *ppPoslist; char c = 0; - /* The end of a position list is marked by a zero encoded as an FTS3 + /* The end of a position list is marked by a zero encoded as an FTS3 ** varint. A single POS_END (0) byte. Except, if the 0 byte is preceded by ** a byte with the 0x80 bit set, then it is not a varint 0, but the tail ** of some other, multi-byte, value. ** - ** The following while-loop moves pEnd to point to the first byte that is not + ** The following while-loop moves pEnd to point to the first byte that is not ** immediately preceded by a byte with the 0x80 bit set. Then increments ** pEnd once more so that it points to the byte immediately following the ** last byte in the position-list. @@ -142873,7 +189594,7 @@ static void fts3PoslistCopy(char **pp, char **ppPoslist){ } /* -** When this function is called, *ppPoslist is assumed to point to the +** When this function is called, *ppPoslist is assumed to point to the ** start of a column-list. After it returns, *ppPoslist points to the ** to the terminator (POS_COLUMN or POS_END) byte of the column-list. ** @@ -142911,10 +189632,11 @@ static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ } /* -** Value used to signify the end of an position-list. This is safe because -** it is not possible to have a document with 2^31 terms. +** Value used to signify the end of an position-list. This must be +** as large or larger than any value that might appear on the +** position-list, even a position list that has been corrupted. */ -#define POSITION_LIST_END 0x7fffffff +#define POSITION_LIST_END LARGEST_INT64 /* ** This function is used to help parse position-lists. When this function is @@ -142923,7 +189645,7 @@ static void fts3ColumnlistCopy(char **pp, char **ppPoslist){ ** (in which case **pp will be a terminator bytes POS_END (0) or ** (1)). ** -** If *pp points past the end of the current position-list, set *pi to +** If *pp points past the end of the current position-list, set *pi to ** POSITION_LIST_END and return. Otherwise, read the next varint from *pp, ** increment the current value of *pi by the value read, and set *pp to ** point to the next value before returning. @@ -142939,7 +189661,9 @@ static void fts3ReadNextPos( sqlite3_int64 *pi /* IN/OUT: Value read from position-list */ ){ if( (**pp)&0xFE ){ - fts3GetDeltaVarint(pp, pi); + int iVal; + *pp += fts3GetVarint32((*pp), &iVal); + *pi += iVal; *pi -= 2; }else{ *pi = POSITION_LIST_END; @@ -142951,7 +189675,7 @@ static void fts3ReadNextPos( ** the value of iCol encoded as a varint to *pp. This will start a new ** column list. ** -** Set *pp to point to the byte just after the last byte written before +** Set *pp to point to the byte just after the last byte written before ** returning (do not modify it if iCol==0). Return the total number of bytes ** written (0 if iCol==0). */ @@ -142973,7 +189697,7 @@ static int fts3PutColNumber(char **pp, int iCol){ ** updated appropriately. The caller is responsible for insuring ** that there is enough space in *pp to hold the complete output. */ -static void fts3PoslistMerge( +static int fts3PoslistMerge( char **pp, /* Output buffer */ char **pp1, /* Left input list */ char **pp2 /* Right input list */ @@ -142986,12 +189710,18 @@ static void fts3PoslistMerge( int iCol1; /* The current column index in pp1 */ int iCol2; /* The current column index in pp2 */ - if( *p1==POS_COLUMN ) fts3GetVarint32(&p1[1], &iCol1); - else if( *p1==POS_END ) iCol1 = POSITION_LIST_END; + if( *p1==POS_COLUMN ){ + fts3GetVarint32(&p1[1], &iCol1); + if( iCol1==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p1==POS_END ) iCol1 = 0x7fffffff; else iCol1 = 0; - if( *p2==POS_COLUMN ) fts3GetVarint32(&p2[1], &iCol2); - else if( *p2==POS_END ) iCol2 = POSITION_LIST_END; + if( *p2==POS_COLUMN ){ + fts3GetVarint32(&p2[1], &iCol2); + if( iCol2==0 ) return FTS_CORRUPT_VTAB; + } + else if( *p2==POS_END ) iCol2 = 0x7fffffff; else iCol2 = 0; if( iCol1==iCol2 ){ @@ -143004,7 +189734,7 @@ static void fts3PoslistMerge( /* At this point, both p1 and p2 point to the start of column-lists ** for the same column (the column with index iCol1 and iCol2). - ** A column-list is a list of non-negative delta-encoded varints, each + ** A column-list is a list of non-negative delta-encoded varints, each ** incremented by 2 before being stored. Each list is terminated by a ** POS_END (0) or POS_COLUMN (1). The following block merges the two lists ** and writes the results to buffer p. p is left pointing to the byte @@ -143013,8 +189743,11 @@ static void fts3PoslistMerge( */ fts3GetDeltaVarint(&p1, &i1); fts3GetDeltaVarint(&p2, &i2); + if( i1<2 || i2<2 ){ + break; + } do { - fts3PutDeltaVarint(&p, &iPrev, (i1pos(*pp1) && pos(*pp2)-pos(*pp1)<=nToken). i.e. ** when the *pp1 token appears before the *pp2 token, but not more than nToken @@ -143080,14 +189814,19 @@ static int fts3PoslistPhraseMerge( /* Never set both isSaveLeft and isExact for the same invocation. */ assert( isSaveLeft==0 || isExact==0 ); - assert( p!=0 && *p1!=0 && *p2!=0 ); - if( *p1==POS_COLUMN ){ + assert_fts3_nc( p!=0 && *p1!=0 && *p2!=0 ); + if( *p1==POS_COLUMN ){ p1++; p1 += fts3GetVarint32(p1, &iCol1); + /* iCol1==0 indicates corruption. Column 0 does not have a POS_COLUMN + ** entry, so this is actually end-of-doclist. */ + if( iCol1==0 ) return 0; } - if( *p2==POS_COLUMN ){ + if( *p2==POS_COLUMN ){ p2++; p2 += fts3GetVarint32(p2, &iCol2); + /* As above, iCol2==0 indicates corruption. */ + if( iCol2==0 ) return 0; } while( 1 ){ @@ -143102,14 +189841,13 @@ static int fts3PoslistPhraseMerge( p += sqlite3Fts3PutVarint(p, iCol1); } - assert( *p1!=POS_END && *p1!=POS_COLUMN ); - assert( *p2!=POS_END && *p2!=POS_COLUMN ); fts3GetDeltaVarint(&p1, &iPos1); iPos1 -= 2; fts3GetDeltaVarint(&p2, &iPos2); iPos2 -= 2; + if( iPos1<0 || iPos2<0 ) break; while( 1 ){ - if( iPos2==iPos1+nToken - || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) + if( iPos2==iPos1+nToken + || (isExact==0 && iPos2>iPos1 && iPos2<=iPos1+nToken) ){ sqlite3_int64 iSave; iSave = isSaveLeft ? iPos1 : iPos2; @@ -143144,8 +189882,8 @@ static int fts3PoslistPhraseMerge( /* Advance pointer p1 or p2 (whichever corresponds to the smaller of ** iCol1 and iCol2) so that it points to either the 0x00 that marks the - ** end of the position list, or the 0x01 that precedes the next - ** column-number in the position list. + ** end of the position list, or the 0x01 that precedes the next + ** column-number in the position list. */ else if( iCol1=pEnd ){ *pp = 0; }else{ - sqlite3_int64 iVal; - *pp += sqlite3Fts3GetVarint(*pp, &iVal); + u64 iVal; + *pp += sqlite3Fts3GetVarintU(*pp, &iVal); if( bDescIdx ){ - *pVal -= iVal; + *pVal = (i64)((u64)*pVal - iVal); }else{ - *pVal += iVal; + *pVal = (i64)((u64)*pVal + iVal); } } } @@ -143270,9 +190008,9 @@ static void fts3GetDeltaVarint3( ** end of the value written. ** ** If *pbFirst is zero when this function is called, the value written to -** the buffer is that of parameter iVal. +** the buffer is that of parameter iVal. ** -** If *pbFirst is non-zero when this function is called, then the value +** If *pbFirst is non-zero when this function is called, then the value ** written is either (iVal-*piPrev) (if bDescIdx is zero) or (*piPrev-iVal) ** (if bDescIdx is non-zero). ** @@ -143286,14 +190024,16 @@ static void fts3PutDeltaVarint3( int *pbFirst, /* IN/OUT: True after first int written */ sqlite3_int64 iVal /* Write this value to the list */ ){ - sqlite3_int64 iWrite; + sqlite3_uint64 iWrite; if( bDescIdx==0 || *pbFirst==0 ){ - iWrite = iVal - *piPrev; + assert_fts3_nc( *pbFirst==0 || iVal>=*piPrev ); + iWrite = (u64)iVal - (u64)*piPrev; }else{ - iWrite = *piPrev - iVal; + assert_fts3_nc( *piPrev>=iVal ); + iWrite = (u64)*piPrev - (u64)iVal; } assert( *pbFirst || *piPrev==0 ); - assert( *pbFirst==0 || iWrite>0 ); + assert_fts3_nc( *pbFirst==0 || iWrite>0 ); *pp += sqlite3Fts3PutVarint(*pp, iWrite); *piPrev = iVal; *pbFirst = 1; @@ -143303,17 +190043,18 @@ static void fts3PutDeltaVarint3( /* ** This macro is used by various functions that merge doclists. The two ** arguments are 64-bit docid values. If the value of the stack variable -** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). +** bDescDoclist is 0 when this macro is invoked, then it returns (i1-i2). ** Otherwise, (i2-i1). ** ** Using this makes it easier to write code that can merge doclists that are ** sorted in either ascending or descending order. */ -#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1-i2)) +/* #define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i64)((u64)i1-i2)) */ +#define DOCID_CMP(i1, i2) ((bDescDoclist?-1:1) * (i1>i2?1:((i1==i2)?0:-1))) /* ** This function does an "OR" merge of two doclists (output contains all -** positions contained in either argument doclist). If the docids in the +** positions contained in either argument doclist). If the docids in the ** input doclists are sorted in ascending order, parameter bDescDoclist ** should be false. If they are sorted in ascending order, it should be ** passed a non-zero value. @@ -143331,6 +190072,7 @@ static int fts3DoclistOrMerge( char *a2, int n2, /* Second doclist */ char **paOut, int *pnOut /* OUT: Malloc'd doclist */ ){ + int rc = SQLITE_OK; sqlite3_int64 i1 = 0; sqlite3_int64 i2 = 0; sqlite3_int64 iPrev = 0; @@ -143352,12 +190094,12 @@ static int fts3DoclistOrMerge( ** current and previous docid (a positive number - since the list is in ** ascending order). ** - ** The first docid written to the output is therefore encoded using the + ** The first docid written to the output is therefore encoded using the ** same number of bytes as it is in whichever of the input lists it is - ** read from. And each subsequent docid read from the same input list + ** read from. And each subsequent docid read from the same input list ** consumes either the same or less bytes as it did in the input (since ** the difference between it and the previous value in the output must - ** be a positive value less than or equal to the delta value read from + ** be a positive value less than or equal to the delta value read from ** the input list). The same argument applies to all but the first docid ** read from the 'other' list. And to the contents of all position lists ** that will be copied and merged from the input to the output. @@ -143369,12 +190111,12 @@ static int fts3DoclistOrMerge( ** ** The space required to store the output is therefore the sum of the ** sizes of the two inputs, plus enough space for exactly one of the input - ** docids to grow. + ** docids to grow. ** - ** A symetric argument may be made if the doclists are in descending + ** A symetric argument may be made if the doclists are in descending ** order. */ - aOut = sqlite3_malloc(n1+n2+FTS3_VARINT_MAX-1); + aOut = sqlite3_malloc64((i64)n1+n2+FTS3_VARINT_MAX-1+FTS3_BUFFER_PADDING); if( !aOut ) return SQLITE_NOMEM; p = aOut; @@ -143385,7 +190127,8 @@ static int fts3DoclistOrMerge( if( p2 && p1 && iDiff==0 ){ fts3PutDeltaVarint3(&p, bDescDoclist, &iPrev, &bFirstOut, i1); - fts3PoslistMerge(&p, &p1, &p2); + rc = fts3PoslistMerge(&p, &p1, &p2); + if( rc ) break; fts3GetDeltaVarint3(&p1, pEnd1, bDescDoclist, &i1); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); }else if( !p2 || (p1 && iDiff<0) ){ @@ -143397,12 +190140,20 @@ static int fts3DoclistOrMerge( fts3PoslistCopy(&p, &p2); fts3GetDeltaVarint3(&p2, pEnd2, bDescDoclist, &i2); } + + assert( (p-aOut)<=((p1?(p1-a1):n1)+(p2?(p2-a2):n2)+FTS3_VARINT_MAX-1) ); } + if( rc!=SQLITE_OK ){ + sqlite3_free(aOut); + p = aOut = 0; + }else{ + assert( (p-aOut)<=n1+n2+FTS3_VARINT_MAX-1 ); + memset(&aOut[(p-aOut)], 0, FTS3_BUFFER_PADDING); + } *paOut = aOut; *pnOut = (int)(p-aOut); - assert( *pnOut<=n1+n2+FTS3_VARINT_MAX-1 ); - return SQLITE_OK; + return rc; } /* @@ -143412,7 +190163,7 @@ static int fts3DoclistOrMerge( ** exactly nDist tokens before it. ** ** If the docids in the input doclists are sorted in ascending order, -** parameter bDescDoclist should be false. If they are sorted in ascending +** parameter bDescDoclist should be false. If they are sorted in ascending ** order, it should be passed a non-zero value. ** ** The right-hand input doclist is overwritten by this function. @@ -143437,7 +190188,7 @@ static int fts3DoclistPhraseMerge( assert( nDist>0 ); if( bDescDoclist ){ - aOut = sqlite3_malloc(*pnRight + FTS3_VARINT_MAX); + aOut = sqlite3_malloc64((sqlite3_int64)*pnRight + FTS3_VARINT_MAX); if( aOut==0 ) return SQLITE_NOMEM; }else{ aOut = aRight; @@ -143508,7 +190259,7 @@ SQLITE_PRIVATE int sqlite3Fts3FirstFilter( fts3ColumnlistCopy(0, &p); } - while( pbDescIdx, + int rc = fts3DoclistOrMerge(p->bDescIdx, pTS->aaOutput[i], pTS->anOutput[i], aOut, nOut, &aNew, &nNew ); if( rc!=SQLITE_OK ){ @@ -143602,25 +190353,26 @@ static int fts3TermSelectMerge( ){ if( pTS->aaOutput[0]==0 ){ /* If this is the first term selected, copy the doclist to the output - ** buffer using memcpy(). + ** buffer using memcpy(). ** - ** Add FTS3_VARINT_MAX bytes of unused space to the end of the + ** Add FTS3_VARINT_MAX bytes of unused space to the end of the ** allocation. This is so as to ensure that the buffer is big enough ** to hold the current doclist AND'd with any other doclist. If the ** doclists are stored in order=ASC order, this padding would not be ** required (since the size of [doclistA AND doclistB] is always less ** than or equal to the size of [doclistA] in that case). But this is - ** not true for order=DESC. For example, a doclist containing (1, -1) + ** not true for order=DESC. For example, a doclist containing (1, -1) ** may be smaller than (-1), as in the first example the -1 may be stored ** as a single-byte delta, whereas in the second it must be stored as a ** FTS3_VARINT_MAX byte varint. ** ** Similar padding is added in the fts3DoclistOrMerge() function. */ - pTS->aaOutput[0] = sqlite3_malloc(nDoclist + FTS3_VARINT_MAX + 1); + pTS->aaOutput[0] = sqlite3_malloc64((i64)nDoclist + FTS3_VARINT_MAX + 1); pTS->anOutput[0] = nDoclist; if( pTS->aaOutput[0] ){ memcpy(pTS->aaOutput[0], aDoclist, nDoclist); + memset(&pTS->aaOutput[0][nDoclist], 0, FTS3_VARINT_MAX); }else{ return SQLITE_NOMEM; } @@ -143639,7 +190391,7 @@ static int fts3TermSelectMerge( char *aNew; int nNew; - int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, + int rc = fts3DoclistOrMerge(p->bDescIdx, aMerge, nMerge, pTS->aaOutput[iOut], pTS->anOutput[iOut], &aNew, &nNew ); if( rc!=SQLITE_OK ){ @@ -143650,7 +190402,7 @@ static int fts3TermSelectMerge( if( aMerge!=aDoclist ) sqlite3_free(aMerge); sqlite3_free(pTS->aaOutput[iOut]); pTS->aaOutput[iOut] = 0; - + aMerge = aNew; nMerge = nNew; if( (iOut+1)==SizeofArray(pTS->aaOutput) ){ @@ -143667,13 +190419,13 @@ static int fts3TermSelectMerge( ** Append SegReader object pNew to the end of the pCsr->apSegment[] array. */ static int fts3SegReaderCursorAppend( - Fts3MultiSegReader *pCsr, + Fts3MultiSegReader *pCsr, Fts3SegReader *pNew ){ if( (pCsr->nSegment%16)==0 ){ Fts3SegReader **apNew; - int nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); - apNew = (Fts3SegReader **)sqlite3_realloc(pCsr->apSegment, nByte); + sqlite3_int64 nByte = (pCsr->nSegment + 16)*sizeof(Fts3SegReader*); + apNew = (Fts3SegReader **)sqlite3_realloc64(pCsr->apSegment, nByte); if( !apNew ){ sqlite3Fts3SegReaderFree(pNew); return SQLITE_NOMEM; @@ -143706,13 +190458,13 @@ static int fts3SegReaderCursor( sqlite3_stmt *pStmt = 0; /* Statement to iterate through segments */ int rc2; /* Result of sqlite3_reset() */ - /* If iLevel is less than 0 and this is not a scan, include a seg-reader + /* If iLevel is less than 0 and this is not a scan, include a seg-reader ** for the pending-terms. If this is a scan, then this call must be being ** made by an fts4aux module, not an FTS table. In this case calling - ** Fts3SegReaderPending might segfault, as the data structures used by + ** Fts3SegReaderPending might segfault, as the data structures used by ** fts4aux are not completely populated. So it's easiest to filter these ** calls out here. */ - if( iLevel<0 && p->aIndex ){ + if( iLevel<0 && p->aIndex && p->iPrevLangid==iLangid ){ Fts3SegReader *pSeg = 0; rc = sqlite3Fts3SegReaderPending(p, iIndex, zTerm, nTerm, isPrefix||isScan, &pSeg); if( rc==SQLITE_OK && pSeg ){ @@ -143737,16 +190489,16 @@ static int fts3SegReaderCursor( /* If zTerm is not NULL, and this segment is not stored entirely on its ** root node, the range of leaves scanned can be reduced. Do this. */ - if( iStartBlock && zTerm ){ + if( iStartBlock && zTerm && zRoot ){ sqlite3_int64 *pi = (isPrefix ? &iLeavesEndBlock : 0); rc = fts3SelectLeaf(p, zTerm, nTerm, zRoot, nRoot, &iStartBlock, pi); if( rc!=SQLITE_OK ) goto finished; if( isPrefix==0 && isScan==0 ) iLeavesEndBlock = iStartBlock; } - - rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, + + rc = sqlite3Fts3SegReaderNew(pCsr->nSegment+1, (isPrefix==0 && isScan==0), - iStartBlock, iLeavesEndBlock, + iStartBlock, iLeavesEndBlock, iEndBlock, zRoot, nRoot, &pSeg ); if( rc!=SQLITE_OK ) goto finished; @@ -143762,7 +190514,7 @@ static int fts3SegReaderCursor( } /* -** Set up a cursor object for iterating through a full-text index or a +** Set up a cursor object for iterating through a full-text index or a ** single level therein. */ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( @@ -143778,7 +190530,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderCursor( ){ assert( iIndex>=0 && iIndexnIndex ); assert( iLevel==FTS3_SEGCURSOR_ALL - || iLevel==FTS3_SEGCURSOR_PENDING + || iLevel==FTS3_SEGCURSOR_PENDING || iLevel>=0 ); assert( iLevelnIndex; i++){ if( p->aIndex[i].nPrefix==nTerm ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 0, 0, pSegcsr ); pSegcsr->bLookup = 1; @@ -143852,7 +190604,7 @@ static int fts3TermSegReaderCursor( for(i=1; bFound==0 && inIndex; i++){ if( p->aIndex[i].nPrefix==nTerm+1 ){ bFound = 1; - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, i, FTS3_SEGCURSOR_ALL, zTerm, nTerm, 1, 0, pSegcsr ); if( rc==SQLITE_OK ){ @@ -143865,7 +190617,7 @@ static int fts3TermSegReaderCursor( } if( bFound==0 ){ - rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, + rc = sqlite3Fts3SegReaderCursor(p, pCsr->iLangid, 0, FTS3_SEGCURSOR_ALL, zTerm, nTerm, isPrefix, 0, pSegcsr ); pSegcsr->bLookup = !isPrefix; @@ -143913,7 +190665,7 @@ static int fts3TermSelect( rc = sqlite3Fts3SegReaderStart(p, pSegcsr, &filter); while( SQLITE_OK==rc - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) + && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pSegcsr)) ){ rc = fts3TermSelectMerge(p, &tsc, pSegcsr->aDoclist, pSegcsr->nDoclist); } @@ -143942,7 +190694,7 @@ static int fts3TermSelect( ** ** If the isPoslist argument is true, then it is assumed that the doclist ** contains a position-list following each docid. Otherwise, it is assumed -** that the doclist is simply a list of docids stored as delta encoded +** that the doclist is simply a list of docids stored as delta encoded ** varints. */ static int fts3DoclistCountDocids(char *aList, int nList){ @@ -143975,6 +190727,8 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ int rc; Fts3Cursor *pCsr = (Fts3Cursor *)pCursor; if( pCsr->eSearch==FTS3_DOCID_SEARCH || pCsr->eSearch==FTS3_FULLSCAN_SEARCH ){ + Fts3Table *pTab = (Fts3Table*)pCursor->pVtab; + pTab->bLock++; if( SQLITE_ROW!=sqlite3_step(pCsr->pStmt) ){ pCsr->isEof = 1; rc = sqlite3_reset(pCsr->pStmt); @@ -143982,6 +190736,7 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->iPrevId = sqlite3_column_int64(pCsr->pStmt, 0); rc = SQLITE_OK; } + pTab->bLock--; }else{ rc = fts3EvalNext((Fts3Cursor *)pCursor); } @@ -143989,18 +190744,6 @@ static int fts3NextMethod(sqlite3_vtab_cursor *pCursor){ return rc; } -/* -** The following are copied from sqliteInt.h. -** -** Constants for the largest and smallest possible 64-bit signed integers. -** These macros are designed to work correctly on both 32-bit and 64-bit -** compilers. -*/ -#ifndef SQLITE_AMALGAMATION -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - /* ** If the numeric type of argument pVal is "integer", then return it ** converted to a 64-bit signed integer. Otherwise, return a copy of @@ -144054,6 +190797,10 @@ static int fts3FilterMethod( UNUSED_PARAMETER(idxStr); UNUSED_PARAMETER(nVal); + if( p->bLock ){ + return SQLITE_ERROR; + } + eSearch = (idxNum & 0x0000FFFF); assert( eSearch>=0 && eSearch<=(FTS3_FULLTEXT_SEARCH+p->nColumn) ); assert( p->pSegments==0 ); @@ -144067,11 +190814,7 @@ static int fts3FilterMethod( assert( iIdx==nVal ); /* In case the cursor has been used before, clear it now. */ - sqlite3_finalize(pCsr->pStmt); - sqlite3_free(pCsr->aDoclist); - sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); - sqlite3Fts3ExprFree(pCsr->pExpr); - memset(&pCursor[1], 0, sizeof(Fts3Cursor)-sizeof(sqlite3_vtab_cursor)); + fts3ClearCursor(pCsr); /* Set the lower and upper bounds on docids to return */ pCsr->iMinDocid = fts3DocidRange(pDocidGe, SMALLEST_INT64); @@ -144097,7 +190840,7 @@ static int fts3FilterMethod( assert( p->base.zErrMsg==0 ); rc = sqlite3Fts3ExprParse(p->pTokenizer, pCsr->iLangid, - p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, + p->azColumn, p->bFts4, p->nColumn, iCol, zQuery, -1, &pCsr->pExpr, &p->base.zErrMsg ); if( rc!=SQLITE_OK ){ @@ -144124,18 +190867,22 @@ static int fts3FilterMethod( (pCsr->bDesc ? "DESC" : "ASC") ); }else{ - zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", + zSql = sqlite3_mprintf("SELECT %s ORDER BY rowid %s", p->zReadExprlist, (pCsr->bDesc ? "DESC" : "ASC") ); } if( zSql ){ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pCsr->pStmt, 0); + p->bLock++; + rc = sqlite3_prepare_v3( + p->db,zSql,-1,SQLITE_PREPARE_PERSISTENT,&pCsr->pStmt,0 + ); + p->bLock--; sqlite3_free(zSql); }else{ rc = SQLITE_NOMEM; } }else if( eSearch==FTS3_DOCID_SEARCH ){ - rc = fts3CursorSeekStmt(pCsr, &pCsr->pStmt); + rc = fts3CursorSeekStmt(pCsr); if( rc==SQLITE_OK ){ rc = sqlite3_bind_value(pCsr->pStmt, 1, pCons); } @@ -144145,15 +190892,20 @@ static int fts3FilterMethod( return fts3NextMethod(pCursor); } -/* -** This is the xEof method of the virtual table. SQLite calls this +/* +** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts3EofMethod(sqlite3_vtab_cursor *pCursor){ - return ((Fts3Cursor *)pCursor)->isEof; + Fts3Cursor *pCsr = (Fts3Cursor*)pCursor; + if( pCsr->isEof ){ + fts3ClearCursor(pCsr); + pCsr->isEof = 1; + } + return pCsr->isEof; } -/* +/* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts3 ** exposes %_content.docid as the rowid for the virtual table. The @@ -144165,7 +190917,7 @@ static int fts3RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ return SQLITE_OK; } -/* +/* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. ** @@ -144188,41 +190940,45 @@ static int fts3ColumnMethod( /* The column value supplied by SQLite must be in range. */ assert( iCol>=0 && iCol<=p->nColumn+2 ); - if( iCol==p->nColumn+1 ){ - /* This call is a request for the "docid" column. Since "docid" is an - ** alias for "rowid", use the xRowid() method to obtain the value. - */ - sqlite3_result_int64(pCtx, pCsr->iPrevId); - }else if( iCol==p->nColumn ){ - /* The extra column whose name is the same as the table. - ** Return a blob which is a pointer to the cursor. */ - sqlite3_result_blob(pCtx, &pCsr, sizeof(pCsr), SQLITE_TRANSIENT); - }else if( iCol==p->nColumn+2 && pCsr->pExpr ){ - sqlite3_result_int64(pCtx, pCsr->iLangid); - }else{ - /* The requested column is either a user column (one that contains - ** indexed data), or the language-id column. */ - rc = fts3CursorSeek(0, pCsr); + switch( iCol-p->nColumn ){ + case 0: + /* The special 'table-name' column */ + sqlite3_result_pointer(pCtx, pCsr, "fts3cursor", 0); + break; - if( rc==SQLITE_OK ){ - if( iCol==p->nColumn+2 ){ - int iLangid = 0; - if( p->zLanguageid ){ - iLangid = sqlite3_column_int(pCsr->pStmt, p->nColumn+1); - } - sqlite3_result_int(pCtx, iLangid); - }else if( sqlite3_data_count(pCsr->pStmt)>(iCol+1) ){ + case 1: + /* The docid column */ + sqlite3_result_int64(pCtx, pCsr->iPrevId); + break; + + case 2: + if( pCsr->pExpr ){ + sqlite3_result_int64(pCtx, pCsr->iLangid); + break; + }else if( p->zLanguageid==0 ){ + sqlite3_result_int(pCtx, 0); + break; + }else{ + iCol = p->nColumn; + /* no break */ deliberate_fall_through + } + + default: + /* A user column. Or, if this is a full-table scan, possibly the + ** language-id column. Seek the cursor. */ + rc = fts3CursorSeek(0, pCsr); + if( rc==SQLITE_OK && sqlite3_data_count(pCsr->pStmt)-1>iCol ){ sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); } - } + break; } assert( ((Fts3Table *)pCsr->base.pVtab)->pSegments==0 ); return rc; } -/* -** This function is the implementation of the xUpdate callback used by +/* +** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be ** inserted, updated or deleted. */ @@ -144257,16 +191013,18 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ ** ** Of course, updating the input segments also involves deleting a bunch ** of blocks from the segments table. But this is not considered overhead - ** as it would also be required by a crisis-merge that used the same input + ** as it would also be required by a crisis-merge that used the same input ** segments. */ const u32 nMinMerge = 64; /* Minimum amount of incr-merge work to do */ Fts3Table *p = (Fts3Table*)pVtab; - int rc = sqlite3Fts3PendingTermsFlush(p); + int rc; + i64 iLastRowid = sqlite3_last_insert_rowid(p->db); - if( rc==SQLITE_OK - && p->nLeafAdd>(nMinMerge/16) + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK + && p->nLeafAdd>(nMinMerge/16) && p->nAutoincrmerge && p->nAutoincrmerge!=0xff ){ int mxLevel = 0; /* Maximum relative level value in db */ @@ -144279,6 +191037,7 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ if( A>(int)nMinMerge ) rc = sqlite3Fts3Incrmerge(p, A, p->nAutoincrmerge); } sqlite3Fts3SegmentsClose(p); + sqlite3_set_last_insert_rowid(p->db, iLastRowid); return rc; } @@ -144291,17 +191050,11 @@ static int fts3SyncMethod(sqlite3_vtab *pVtab){ static int fts3SetHasStat(Fts3Table *p){ int rc = SQLITE_OK; if( p->bHasStat==2 ){ - const char *zFmt ="SELECT 1 FROM %Q.sqlite_master WHERE tbl_name='%q_stat'"; - char *zSql = sqlite3_mprintf(zFmt, p->zDb, p->zName); - if( zSql ){ - sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, 0); - if( rc==SQLITE_OK ){ - int bHasStat = (sqlite3_step(pStmt)==SQLITE_ROW); - rc = sqlite3_finalize(pStmt); - if( rc==SQLITE_OK ) p->bHasStat = bHasStat; - } - sqlite3_free(zSql); + char *zTbl = sqlite3_mprintf("%s_stat", p->zName); + if( zTbl ){ + int res = sqlite3_table_column_metadata(p->db, p->zDb, zTbl, 0,0,0,0,0,0); + sqlite3_free(zTbl); + p->bHasStat = (res==SQLITE_OK); }else{ rc = SQLITE_NOMEM; } @@ -144310,18 +191063,24 @@ static int fts3SetHasStat(Fts3Table *p){ } /* -** Implementation of xBegin() method. +** Implementation of xBegin() method. */ static int fts3BeginMethod(sqlite3_vtab *pVtab){ Fts3Table *p = (Fts3Table*)pVtab; + int rc; UNUSED_PARAMETER(pVtab); assert( p->pSegments==0 ); assert( p->nPendingData==0 ); assert( p->inTransaction!=1 ); - TESTONLY( p->inTransaction = 1 ); - TESTONLY( p->mxSavepoint = -1; ); p->nLeafAdd = 0; - return fts3SetHasStat(p); + rc = fts3SetHasStat(p); +#ifdef SQLITE_DEBUG + if( rc==SQLITE_OK ){ + p->inTransaction = 1; + p->mxSavepoint = -1; + } +#endif + return rc; } /* @@ -144366,17 +191125,17 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ /* Skip backwards passed any trailing 0x00 bytes added by NearTrim() */ while( p>pStart && (c=*p--)==0 ); - /* Search backwards for a varint with value zero (the end of the previous + /* Search backwards for a varint with value zero (the end of the previous ** poslist). This is an 0x00 byte preceded by some byte that does not ** have the 0x80 bit set. */ - while( p>pStart && (*p & 0x80) | c ){ - c = *p--; + while( p>pStart && (*p & 0x80) | c ){ + c = *p--; } assert( p==pStart || c==0 ); /* At this point p points to that preceding byte without the 0x80 bit ** set. So to find the start of the poslist, skip forward 2 bytes then - ** over a varint. + ** over a varint. ** ** Normally. The other case is that p==pStart and the poslist to return ** is the first in the doclist. In this case do not skip forward 2 bytes. @@ -144397,7 +191156,7 @@ static void fts3ReversePoslist(char *pStart, char **ppPoslist){ ** offsets() and optimize() SQL functions. ** ** If the value passed as the third argument is a blob of size -** sizeof(Fts3Cursor*), then the blob contents are copied to the +** sizeof(Fts3Cursor*), then the blob contents are copied to the ** output variable *ppCsr and SQLITE_OK is returned. Otherwise, an error ** message is written to context pContext and SQLITE_ERROR returned. The ** string passed via zFunc is used as part of the error message. @@ -144408,18 +191167,17 @@ static int fts3FunctionArg( sqlite3_value *pVal, /* argv[0] passed to function */ Fts3Cursor **ppCsr /* OUT: Store cursor handle here */ ){ - Fts3Cursor *pRet; - if( sqlite3_value_type(pVal)!=SQLITE_BLOB - || sqlite3_value_bytes(pVal)!=sizeof(Fts3Cursor *) - ){ + int rc; + *ppCsr = (Fts3Cursor*)sqlite3_value_pointer(pVal, "fts3cursor"); + if( (*ppCsr)!=0 ){ + rc = SQLITE_OK; + }else{ char *zErr = sqlite3_mprintf("illegal first argument to %s", zFunc); sqlite3_result_error(pContext, zErr, -1); sqlite3_free(zErr); - return SQLITE_ERROR; + rc = SQLITE_ERROR; } - memcpy(&pRet, sqlite3_value_blob(pVal), sizeof(Fts3Cursor *)); - *ppCsr = pRet; - return SQLITE_OK; + return rc; } /* @@ -144443,7 +191201,7 @@ static void fts3SnippetFunc( assert( nVal>=1 ); if( nVal>6 ){ - sqlite3_result_error(pContext, + sqlite3_result_error(pContext, "wrong number of arguments to function snippet()", -1); return; } @@ -144451,9 +191209,13 @@ static void fts3SnippetFunc( switch( nVal ){ case 6: nToken = sqlite3_value_int(apVal[5]); + /* no break */ deliberate_fall_through case 5: iCol = sqlite3_value_int(apVal[4]); + /* no break */ deliberate_fall_through case 4: zEllipsis = (const char*)sqlite3_value_text(apVal[3]); + /* no break */ deliberate_fall_through case 3: zEnd = (const char*)sqlite3_value_text(apVal[2]); + /* no break */ deliberate_fall_through case 2: zStart = (const char*)sqlite3_value_text(apVal[1]); } if( !zEllipsis || !zEnd || !zStart ){ @@ -144485,8 +191247,8 @@ static void fts3OffsetsFunc( } } -/* -** Implementation of the special optimize() function for FTS3. This +/* +** Implementation of the special optimize() function for FTS3. This ** function merges all segments in the database to a single segment. ** Example usage is: ** @@ -144595,10 +191357,10 @@ static int fts3RenameMethod( /* At this point it must be known if the %_stat table exists or not. ** So bHasStat may not be 2. */ rc = fts3SetHasStat(p); - + /* As it happens, the pending terms table is always empty here. This is - ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction - ** always opens a savepoint transaction. And the xSavepoint() method + ** because an "ALTER TABLE RENAME TABLE" statement inside a transaction + ** always opens a savepoint transaction. And the xSavepoint() method ** flushes the pending terms table. But leave the (no-op) call to ** PendingTermsFlush() in in case that changes. */ @@ -144607,6 +191369,8 @@ static int fts3RenameMethod( rc = sqlite3Fts3PendingTermsFlush(p); } + p->bIgnoreSavepoint = 1; + if( p->zContentTbl==0 ){ fts3DbExec(&rc, db, "ALTER TABLE %Q.'%q_content' RENAME TO '%q_content';", @@ -144634,6 +191398,8 @@ static int fts3RenameMethod( "ALTER TABLE %Q.'%q_segdir' RENAME TO '%q_segdir';", p->zDb, p->zName, zName ); + + p->bIgnoreSavepoint = 0; return rc; } @@ -144644,12 +191410,28 @@ static int fts3RenameMethod( */ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ int rc = SQLITE_OK; - UNUSED_PARAMETER(iSavepoint); - assert( ((Fts3Table *)pVtab)->inTransaction ); - assert( ((Fts3Table *)pVtab)->mxSavepoint < iSavepoint ); - TESTONLY( ((Fts3Table *)pVtab)->mxSavepoint = iSavepoint ); - if( ((Fts3Table *)pVtab)->bIgnoreSavepoint==0 ){ - rc = fts3SyncMethod(pVtab); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint<=iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + + if( pTab->bIgnoreSavepoint==0 ){ + if( fts3HashCount(&pTab->aIndex[0].hPending)>0 ){ + char *zSql = sqlite3_mprintf("INSERT INTO %Q.%Q(%Q) VALUES('flush')", + pTab->zDb, pTab->zName, pTab->zName + ); + if( zSql ){ + pTab->bIgnoreSavepoint = 1; + rc = sqlite3_exec(pTab->db, zSql, 0, 0, 0); + pTab->bIgnoreSavepoint = 0; + sqlite3_free(zSql); + }else{ + rc = SQLITE_NOMEM; + } + } + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } } return rc; } @@ -144660,12 +191442,11 @@ static int fts3SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - TESTONLY( Fts3Table *p = (Fts3Table*)pVtab ); - UNUSED_PARAMETER(iSavepoint); - UNUSED_PARAMETER(pVtab); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint-1 ); + Fts3Table *pTab = (Fts3Table*)pVtab; + assert( pTab->inTransaction ); + assert( pTab->mxSavepoint >= iSavepoint ); + TESTONLY( pTab->mxSavepoint = iSavepoint-1 ); + pTab->iSavepoint = iSavepoint; return SQLITE_OK; } @@ -144675,17 +191456,67 @@ static int fts3ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** Discard the contents of the pending terms table. */ static int fts3RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts3Table *p = (Fts3Table*)pVtab; + Fts3Table *pTab = (Fts3Table*)pVtab; UNUSED_PARAMETER(iSavepoint); - assert( p->inTransaction ); - assert( p->mxSavepoint >= iSavepoint ); - TESTONLY( p->mxSavepoint = iSavepoint ); - sqlite3Fts3PendingTermsClear(p); + assert( pTab->inTransaction ); + TESTONLY( pTab->mxSavepoint = iSavepoint ); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + sqlite3Fts3PendingTermsClear(pTab); + } return SQLITE_OK; } +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts3ShadowName(const char *zName){ + static const char *azName[] = { + "content", "docsize", "segdir", "segments", "stat", + }; + unsigned int i; + for(i=0; ibFts4 ? 4 : 3, zSchema, zTabname, sqlite3_errstr(rc)); + if( *pzErr ) rc = SQLITE_OK; + }else if( rc==SQLITE_OK && bOk==0 ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS%d table %s.%s", + p->bFts4 ? 4 : 3, zSchema, zTabname); + if( *pzErr==0 ) rc = SQLITE_NOMEM; + } + sqlite3Fts3SegmentsClose(p); + return rc; +} + + + static const sqlite3_module fts3Module = { - /* iVersion */ 2, + /* iVersion */ 4, /* xCreate */ fts3CreateMethod, /* xConnect */ fts3ConnectMethod, /* xBestIndex */ fts3BestIndexMethod, @@ -144708,6 +191539,8 @@ static const sqlite3_module fts3Module = { /* xSavepoint */ fts3SavepointMethod, /* xRelease */ fts3ReleaseMethod, /* xRollbackTo */ fts3RollbackToMethod, + /* xShadowName */ fts3ShadowName, + /* xIntegrity */ fts3IntegrityMethod, }; /* @@ -144716,13 +191549,16 @@ static const sqlite3_module fts3Module = { ** allocated for the tokenizer hash table. */ static void hashDestroy(void *p){ - Fts3Hash *pHash = (Fts3Hash *)p; - sqlite3Fts3HashClear(pHash); - sqlite3_free(pHash); + Fts3HashWrapper *pHash = (Fts3HashWrapper *)p; + pHash->nRef--; + if( pHash->nRef<=0 ){ + sqlite3Fts3HashClear(&pHash->hash); + sqlite3_free(pHash); + } } /* -** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are +** The fts3 built-in tokenizers - "simple", "porter" and "icu"- are ** implemented in files fts3_tokenizer1.c, fts3_porter.c and fts3_icu.c ** respectively. The following three forward declarations are for functions ** declared in these files used to retrieve the respective implementations. @@ -144748,7 +191584,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule(sqlite3_tokenizer_module const */ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ int rc = SQLITE_OK; - Fts3Hash *pHash = 0; + Fts3HashWrapper *pHash = 0; const sqlite3_tokenizer_module *pSimple = 0; const sqlite3_tokenizer_module *pPorter = 0; #ifndef SQLITE_DISABLE_FTS3_UNICODE @@ -144776,23 +191612,24 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ sqlite3Fts3PorterTokenizerModule(&pPorter); /* Allocate and initialize the hash-table used to store tokenizers. */ - pHash = sqlite3_malloc(sizeof(Fts3Hash)); + pHash = sqlite3_malloc(sizeof(Fts3HashWrapper)); if( !pHash ){ rc = SQLITE_NOMEM; }else{ - sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); + sqlite3Fts3HashInit(&pHash->hash, FTS3_HASH_STRING, 1); + pHash->nRef = 0; } /* Load the built-in tokenizers into the hash table */ if( rc==SQLITE_OK ){ - if( sqlite3Fts3HashInsert(pHash, "simple", 7, (void *)pSimple) - || sqlite3Fts3HashInsert(pHash, "porter", 7, (void *)pPorter) + if( sqlite3Fts3HashInsert(&pHash->hash, "simple", 7, (void *)pSimple) + || sqlite3Fts3HashInsert(&pHash->hash, "porter", 7, (void *)pPorter) #ifndef SQLITE_DISABLE_FTS3_UNICODE - || sqlite3Fts3HashInsert(pHash, "unicode61", 10, (void *)pUnicode) + || sqlite3Fts3HashInsert(&pHash->hash, "unicode61", 10, (void *)pUnicode) #endif #ifdef SQLITE_ENABLE_ICU - || (pIcu && sqlite3Fts3HashInsert(pHash, "icu", 4, (void *)pIcu)) + || (pIcu && sqlite3Fts3HashInsert(&pHash->hash, "icu", 4, (void *)pIcu)) #endif ){ rc = SQLITE_NOMEM; @@ -144801,32 +191638,35 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ #ifdef SQLITE_TEST if( rc==SQLITE_OK ){ - rc = sqlite3Fts3ExprInitTestInterface(db); + rc = sqlite3Fts3ExprInitTestInterface(db, &pHash->hash); } #endif - /* Create the virtual table wrapper around the hash-table and overload - ** the two scalar functions. If this is successful, register the + /* Create the virtual table wrapper around the hash-table and overload + ** the four scalar functions. If this is successful, register the ** module with sqlite. */ - if( SQLITE_OK==rc - && SQLITE_OK==(rc = sqlite3Fts3InitHashTable(db, pHash, "fts3_tokenizer")) + if( SQLITE_OK==rc + && SQLITE_OK==(rc=sqlite3Fts3InitHashTable(db,&pHash->hash,"fts3_tokenizer")) && SQLITE_OK==(rc = sqlite3_overload_function(db, "snippet", -1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "offsets", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 1)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "matchinfo", 2)) && SQLITE_OK==(rc = sqlite3_overload_function(db, "optimize", 1)) ){ + pHash->nRef++; rc = sqlite3_create_module_v2( db, "fts3", &fts3Module, (void *)pHash, hashDestroy ); if( rc==SQLITE_OK ){ + pHash->nRef++; rc = sqlite3_create_module_v2( - db, "fts4", &fts3Module, (void *)pHash, 0 + db, "fts4", &fts3Module, (void *)pHash, hashDestroy ); } if( rc==SQLITE_OK ){ - rc = sqlite3Fts3InitTok(db, (void *)pHash); + pHash->nRef++; + rc = sqlite3Fts3InitTok(db, (void *)pHash, hashDestroy); } return rc; } @@ -144835,7 +191675,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ /* An error has occurred. Delete the hash table and return the error code. */ assert( rc!=SQLITE_OK ); if( pHash ){ - sqlite3Fts3HashClear(pHash); + sqlite3Fts3HashClear(&pHash->hash); sqlite3_free(pHash); } return rc; @@ -144843,7 +191683,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ /* ** Allocate an Fts3MultiSegReader for each token in the expression headed -** by pExpr. +** by pExpr. ** ** An Fts3SegReader object is a cursor that can seek or scan a range of ** entries within a single segment b-tree. An Fts3MultiSegReader uses multiple @@ -144853,7 +191693,7 @@ SQLITE_PRIVATE int sqlite3Fts3Init(sqlite3 *db){ ** If the allocated Fts3MultiSegReader just seeks to a single entry in a ** segment b-tree (if the term is not a prefix or it is a prefix for which ** there exists prefix b-tree of the right length) then it may be traversed -** and merged incrementally. Otherwise, it has to be merged into an in-memory +** and merged incrementally. Otherwise, it has to be merged into an in-memory ** doclist and then traversed. */ static void fts3EvalAllocateReaders( @@ -144870,7 +191710,7 @@ static void fts3EvalAllocateReaders( *pnToken += nToken; for(i=0; ipPhrase->aToken[i]; - int rc = fts3TermSegReaderCursor(pCsr, + int rc = fts3TermSegReaderCursor(pCsr, pToken->z, pToken->n, pToken->isPrefix, &pToken->pSegcsr ); if( rc!=SQLITE_OK ){ @@ -144988,6 +191828,7 @@ static int fts3EvalPhraseLoad( return rc; } +#ifndef SQLITE_DISABLE_FTS4_DEFERRED /* ** This function is called on each phrase after the position lists for ** any deferred tokens have been loaded into memory. It updates the phrases @@ -145003,8 +191844,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ char *aPoslist = 0; /* Position list for deferred tokens */ int nPoslist = 0; /* Number of bytes in aPoslist */ int iPrev = -1; /* Token number of previous deferred token */ - - assert( pPhrase->doclist.bFreeList==0 ); + char *aFree = (pPhrase->doclist.bFreeList ? pPhrase->doclist.pList : 0); for(iToken=0; iTokennToken; iToken++){ Fts3PhraseToken *pToken = &pPhrase->aToken[iToken]; @@ -145018,6 +191858,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ if( pList==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -145038,6 +191879,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nPoslist = (int)(aOut - aPoslist); if( nPoslist==0 ){ sqlite3_free(aPoslist); + sqlite3_free(aFree); pPhrase->doclist.pList = 0; pPhrase->doclist.nList = 0; return SQLITE_OK; @@ -145070,13 +191912,14 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ nDistance = iPrev - nMaxUndeferred; } - aOut = (char *)sqlite3_malloc(nPoslist+8); + aOut = (char *)sqlite3Fts3MallocZero(nPoslist+FTS3_BUFFER_PADDING); if( !aOut ){ sqlite3_free(aPoslist); return SQLITE_NOMEM; } - + pPhrase->doclist.pList = aOut; + assert( p1 && p2 ); if( fts3PoslistPhraseMerge(&aOut, nDistance, 0, 1, &p1, &p2) ){ pPhrase->doclist.bFreeList = 1; pPhrase->doclist.nList = (int)(aOut - pPhrase->doclist.pList); @@ -145089,8 +191932,10 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ } } + if( pPhrase->doclist.pList!=aFree ) sqlite3_free(aFree); return SQLITE_OK; } +#endif /* SQLITE_DISABLE_FTS4_DEFERRED */ /* ** Maximum number of tokens a phrase may have to be considered for the @@ -145099,7 +191944,7 @@ static int fts3EvalDeferredPhrase(Fts3Cursor *pCsr, Fts3Phrase *pPhrase){ #define MAX_INCR_PHRASE_TOKENS 4 /* -** This function is called for each Fts3Phrase in a full-text query +** This function is called for each Fts3Phrase in a full-text query ** expression to initialize the mechanism for returning rows. Once this ** function has been called successfully on an Fts3Phrase, it may be ** used with fts3EvalPhraseNext() to iterate through the matching docids. @@ -145117,14 +191962,14 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ /* Determine if doclists may be loaded from disk incrementally. This is ** possible if the bOptOk argument is true, the FTS doclists will be - ** scanned in forward order, and the phrase consists of + ** scanned in forward order, and the phrase consists of ** MAX_INCR_PHRASE_TOKENS or fewer tokens, none of which are are "^first" ** tokens or prefix tokens that cannot use a prefix-index. */ int bHaveIncr = 0; - int bIncrOk = (bOptOk - && pCsr->bDesc==pTab->bDescIdx + int bIncrOk = (bOptOk + && pCsr->bDesc==pTab->bDescIdx && p->nToken<=MAX_INCR_PHRASE_TOKENS && p->nToken>0 -#ifdef SQLITE_TEST +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) && pTab->bNoIncrDoclist==0 #endif ); @@ -145158,12 +192003,12 @@ static int fts3EvalPhraseStart(Fts3Cursor *pCsr, int bOptOk, Fts3Phrase *p){ } /* -** This function is used to iterate backwards (from the end to start) +** This function is used to iterate backwards (from the end to start) ** through doclists. It is used by this module to iterate through phrase ** doclists in reverse and by the fts3_write.c module to iterate through ** pending-terms lists when writing to databases with "order=desc". ** -** The doclist may be sorted in ascending (parameter bDescIdx==0) or +** The doclist may be sorted in ascending (parameter bDescIdx==0) or ** descending (parameter bDescIdx==1) order of docid. Regardless, this ** function iterates from the end of the doclist to the beginning. */ @@ -145180,7 +192025,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistPrev( assert( nDoclist>0 ); assert( *pbEof==0 ); - assert( p || *piDocid==0 ); + assert_fts3_nc( p || *piDocid==0 ); assert( !p || (p>aDoclist && p<&aDoclist[nDoclist]) ); if( p==0 ){ @@ -145235,7 +192080,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistNext( assert( nDoclist>0 ); assert( *pbEof==0 ); - assert( p || *piDocid==0 ); + assert_fts3_nc( p || *piDocid==0 ); assert( !p || (p>=aDoclist && p<=&aDoclist[nDoclist]) ); if( p==0 ){ @@ -145243,7 +192088,7 @@ SQLITE_PRIVATE void sqlite3Fts3DoclistNext( p += sqlite3Fts3GetVarint(p, piDocid); }else{ fts3PoslistCopy(0, &p); - while( p<&aDoclist[nDoclist] && *p==0 ) p++; + while( p<&aDoclist[nDoclist] && *p==0 ) p++; if( p>=&aDoclist[nDoclist] ){ *pbEof = 1; }else{ @@ -145266,15 +192111,16 @@ static void fts3EvalDlPhraseNext( u8 *pbEof ){ char *pIter; /* Used to iterate through aAll */ - char *pEnd = &pDL->aAll[pDL->nAll]; /* 1 byte past end of aAll */ - + char *pEnd; /* 1 byte past end of aAll */ + if( pDL->pNextDocid ){ pIter = pDL->pNextDocid; + assert( pDL->aAll!=0 || pIter==0 ); }else{ pIter = pDL->aAll; } - if( pIter>=pEnd ){ + if( pIter==0 || pIter>=(pEnd = pDL->aAll + pDL->nAll) ){ /* We have already reached the end of this doclist. EOF. */ *pbEof = 1; }else{ @@ -145315,12 +192161,12 @@ struct TokenDoclist { }; /* -** Token pToken is an incrementally loaded token that is part of a +** Token pToken is an incrementally loaded token that is part of a ** multi-token phrase. Advance it to the next matching document in the ** database and populate output variable *p with the details of the new ** entry. Or, if the iterator has reached EOF, set *pbEof to true. ** -** If an error occurs, return an SQLite error code. Otherwise, return +** If an error occurs, return an SQLite error code. Otherwise, return ** SQLITE_OK. */ static int incrPhraseTokenNext( @@ -145361,18 +192207,18 @@ static int incrPhraseTokenNext( /* ** The phrase iterator passed as the second argument: ** -** * features at least one token that uses an incremental doclist, and +** * features at least one token that uses an incremental doclist, and ** ** * does not contain any deferred tokens. ** ** Advance it to the next matching documnent in the database and populate -** the Fts3Doclist.pList and nList fields. +** the Fts3Doclist.pList and nList fields. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to ** 1 before returning. Otherwise, if no error occurs and the iterator is ** successfully advanced, *pbEof is set to 0. ** -** If an error occurs, return an SQLite error code. Otherwise, return +** If an error occurs, return an SQLite error code. Otherwise, return ** SQLITE_OK. */ static int fts3EvalIncrPhraseNext( @@ -145389,8 +192235,8 @@ static int fts3EvalIncrPhraseNext( ** one incremental token. In which case the bIncr flag is set. */ assert( p->bIncr==1 ); - if( p->nToken==1 && p->bIncr ){ - rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, + if( p->nToken==1 ){ + rc = sqlite3Fts3MsrIncrNext(pTab, p->aToken[0].pSegcsr, &pDL->iDocid, &pDL->pList, &pDL->nList ); if( pDL->pList==0 ) bEof = 1; @@ -145420,8 +192266,8 @@ static int fts3EvalIncrPhraseNext( /* Keep advancing iterators until they all point to the same document */ for(i=0; inToken; i++){ - while( rc==SQLITE_OK && bEof==0 - && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 + while( rc==SQLITE_OK && bEof==0 + && a[i].bIgnore==0 && DOCID_CMP(a[i].iDocid, iMax)<0 ){ rc = incrPhraseTokenNext(pTab, p, i, &a[i], &bEof); if( DOCID_CMP(a[i].iDocid, iMax)>0 ){ @@ -145435,9 +192281,10 @@ static int fts3EvalIncrPhraseNext( if( bEof==0 ){ int nList = 0; int nByte = a[p->nToken-1].nList; - char *aDoclist = sqlite3_malloc(nByte+1); + char *aDoclist = sqlite3_malloc64((i64)nByte+FTS3_BUFFER_PADDING); if( !aDoclist ) return SQLITE_NOMEM; memcpy(aDoclist, a[p->nToken-1].pList, nByte+1); + memset(&aDoclist[nByte], 0, FTS3_BUFFER_PADDING); for(i=0; i<(p->nToken-1); i++){ if( a[i].bIgnore==0 ){ @@ -145467,8 +192314,8 @@ static int fts3EvalIncrPhraseNext( } /* -** Attempt to move the phrase iterator to point to the next matching docid. -** If an error occurs, return an SQLite error code. Otherwise, return +** Attempt to move the phrase iterator to point to the next matching docid. +** If an error occurs, return an SQLite error code. Otherwise, return ** SQLITE_OK. ** ** If there is no "next" entry and no error occurs, then *pbEof is set to @@ -145487,7 +192334,7 @@ static int fts3EvalPhraseNext( if( p->bIncr ){ rc = fts3EvalIncrPhraseNext(pCsr, p, pbEof); }else if( pCsr->bDesc!=pTab->bDescIdx && pDL->nAll ){ - sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, + sqlite3Fts3DoclistPrev(pTab->bDescIdx, pDL->aAll, pDL->nAll, &pDL->pNextDocid, &pDL->iDocid, &pDL->nList, pbEof ); pDL->pList = pDL->pNextDocid; @@ -145547,7 +192394,7 @@ static void fts3EvalStartReaders( ** Tokens are divided into AND/NEAR clusters. All tokens in a cluster belong ** to phrases that are connected only by AND and NEAR operators (not OR or ** NOT). When determining tokens to defer, each AND/NEAR cluster is considered -** separately. The root of a tokens AND/NEAR cluster is stored in +** separately. The root of a tokens AND/NEAR cluster is stored in ** Fts3TokenAndCost.pRoot. */ typedef struct Fts3TokenAndCost Fts3TokenAndCost; @@ -145615,26 +192462,26 @@ static void fts3EvalTokenCosts( ** write this value to *pnPage and return SQLITE_OK. Otherwise, return ** an SQLite error code. ** -** The average document size in pages is calculated by first calculating +** The average document size in pages is calculated by first calculating ** determining the average size in bytes, B. If B is less than the amount ** of data that will fit on a single leaf page of an intkey table in ** this database, then the average docsize is 1. Otherwise, it is 1 plus ** the number of overflow pages consumed by a record B bytes in size. */ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ + int rc = SQLITE_OK; if( pCsr->nRowAvg==0 ){ /* The average document size, which is required to calculate the cost - ** of each doclist, has not yet been determined. Read the required + ** of each doclist, has not yet been determined. Read the required ** data from the %_stat table to calculate it. ** - ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 + ** Entry 0 of the %_stat table is a blob containing (nCol+1) FTS3 ** varints, where nCol is the number of columns in the FTS3 table. ** The first varint is the number of documents currently stored in ** the table. The following nCol varints contain the total amount of ** data stored in all rows of each column of the table, from left ** to right. */ - int rc; Fts3Table *p = (Fts3Table*)pCsr->base.pVtab; sqlite3_stmt *pStmt; sqlite3_int64 nDoc = 0; @@ -145645,12 +192492,13 @@ static int fts3EvalAverageDocsize(Fts3Cursor *pCsr, int *pnPage){ rc = sqlite3Fts3SelectDoctotal(p, &pStmt); if( rc!=SQLITE_OK ) return rc; a = sqlite3_column_blob(pStmt, 0); - assert( a ); - - pEnd = &a[sqlite3_column_bytes(pStmt, 0)]; - a += sqlite3Fts3GetVarint(a, &nDoc); - while( anDoc = nDoc; pCsr->nRowAvg = (int)(((nByte / nDoc) + p->nPgsz) / p->nPgsz); - assert( pCsr->nRowAvg>0 ); + assert( pCsr->nRowAvg>0 ); rc = sqlite3_reset(pStmt); - if( rc!=SQLITE_OK ) return rc; } *pnPage = pCsr->nRowAvg; - return SQLITE_OK; + return rc; } /* -** This function is called to select the tokens (if any) that will be +** This function is called to select the tokens (if any) that will be ** deferred. The array aTC[] has already been populated when this is ** called. ** -** This function is called once for each AND/NEAR cluster in the +** This function is called once for each AND/NEAR cluster in the ** expression. Each invocation determines which tokens to defer within ** the cluster with root node pRoot. See comments above the definition ** of struct Fts3TokenAndCost for more details. @@ -145723,8 +192570,8 @@ static int fts3EvalSelectDeferred( assert( rc!=SQLITE_OK || nDocSize>0 ); - /* Iterate through all tokens in this AND/NEAR cluster, in ascending order - ** of the number of overflow pages that will be loaded by the pager layer + /* Iterate through all tokens in this AND/NEAR cluster, in ascending order + ** of the number of overflow pages that will be loaded by the pager layer ** to retrieve the entire doclist for the token from the full-text index. ** Load the doclists for tokens that are either: ** @@ -145735,7 +192582,7 @@ static int fts3EvalSelectDeferred( ** ** After each token doclist is loaded, merge it with the others from the ** same phrase and count the number of documents that the merged doclist - ** contains. Set variable "nMinEst" to the smallest number of documents in + ** contains. Set variable "nMinEst" to the smallest number of documents in ** any phrase doclist for which 1 or more token doclists have been loaded. ** Let nOther be the number of other phrases for which it is certain that ** one or more tokens will not be deferred. @@ -145751,8 +192598,8 @@ static int fts3EvalSelectDeferred( /* Set pTC to point to the cheapest remaining token. */ for(iTC=0; iTCnOvfl) + if( aTC[iTC].pToken && aTC[iTC].pRoot==pRoot + && (!pTC || aTC[iTC].nOvflnOvfl) ){ pTC = &aTC[iTC]; } @@ -145761,7 +192608,7 @@ static int fts3EvalSelectDeferred( if( ii && pTC->nOvfl>=((nMinEst+(nLoad4/4)-1)/(nLoad4/4))*nDocSize ){ /* The number of overflow pages to load for this (and therefore all - ** subsequent) tokens is greater than the estimated number of pages + ** subsequent) tokens is greater than the estimated number of pages ** that will be loaded if all subsequent tokens are deferred. */ Fts3PhraseToken *pToken = pTC->pToken; @@ -145770,7 +192617,7 @@ static int fts3EvalSelectDeferred( pToken->pSegcsr = 0; }else{ /* Set nLoad4 to the value of (4^nOther) for the next iteration of the - ** for-loop. Except, limit the value to 2^24 to prevent it from + ** for-loop. Except, limit the value to 2^24 to prevent it from ** overflowing the 32-bit integer it is stored in. */ if( ii<12 ) nLoad4 = nLoad4*4; @@ -145828,16 +192675,15 @@ static int fts3EvalStart(Fts3Cursor *pCsr){ #ifndef SQLITE_DISABLE_FTS4_DEFERRED if( rc==SQLITE_OK && nToken>1 && pTab->bFts4 ){ Fts3TokenAndCost *aTC; - Fts3Expr **apOr; - aTC = (Fts3TokenAndCost *)sqlite3_malloc( + aTC = (Fts3TokenAndCost *)sqlite3_malloc64( sizeof(Fts3TokenAndCost) * nToken + sizeof(Fts3Expr *) * nOr * 2 ); - apOr = (Fts3Expr **)&aTC[nToken]; if( !aTC ){ rc = SQLITE_NOMEM; }else{ + Fts3Expr **apOr = (Fts3Expr **)&aTC[nToken]; int ii; Fts3TokenAndCost *pTC = aTC; Fts3Expr **ppOr = apOr; @@ -145883,7 +192729,7 @@ static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ ** ** Parameter nNear is passed the NEAR distance of the expression (5 in ** the example above). When this function is called, *paPoslist points to -** the position list, and *pnToken is the number of phrase tokens in, the +** the position list, and *pnToken is the number of phrase tokens in the ** phrase on the other side of the NEAR operator to pPhrase. For example, ** if pPhrase refers to the "def ghi" phrase, then *paPoslist points to ** the position list associated with phrase "abc". @@ -145892,7 +192738,7 @@ static void fts3EvalInvalidatePoslist(Fts3Phrase *pPhrase){ ** close to a position in the *paPoslist position list are removed. If this ** leaves 0 positions, zero is returned. Otherwise, non-zero. ** -** Before returning, *paPoslist is set to point to the position lsit +** Before returning, *paPoslist is set to point to the position lsit ** associated with pPhrase. And *pnToken is set to the number of tokens in ** pPhrase. */ @@ -145906,8 +192752,8 @@ static int fts3EvalNearTrim( int nParam1 = nNear + pPhrase->nToken; int nParam2 = nNear + *pnToken; int nNew; - char *p2; - char *pOut; + char *p2; + char *pOut; int res; assert( pPhrase->doclist.pList ); @@ -145918,10 +192764,12 @@ static int fts3EvalNearTrim( ); if( res ){ nNew = (int)(pOut - pPhrase->doclist.pList) - 1; - assert( pPhrase->doclist.pList[nNew]=='\0' ); - assert( nNew<=pPhrase->doclist.nList && nNew>0 ); - memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); - pPhrase->doclist.nList = nNew; + assert_fts3_nc( nNew<=pPhrase->doclist.nList && nNew>0 ); + if( nNew>=0 && nNew<=pPhrase->doclist.nList ){ + assert( pPhrase->doclist.pList[nNew]=='\0' ); + memset(&pPhrase->doclist.pList[nNew], 0, pPhrase->doclist.nList - nNew); + pPhrase->doclist.nList = nNew; + } *paPoslist = pPhrase->doclist.pList; *pnToken = pPhrase->nToken; } @@ -145954,19 +192802,19 @@ static int fts3EvalNearTrim( ** ** 1. Deferred tokens are not taken into account. If a phrase consists ** entirely of deferred tokens, it is assumed to match every row in -** the db. In this case the position-list is not populated at all. +** the db. In this case the position-list is not populated at all. ** ** Or, if a phrase contains one or more deferred tokens and one or -** more non-deferred tokens, then the expression is advanced to the +** more non-deferred tokens, then the expression is advanced to the ** next possible match, considering only non-deferred tokens. In other ** words, if the phrase is "A B C", and "B" is deferred, the expression -** is advanced to the next row that contains an instance of "A * C", +** is advanced to the next row that contains an instance of "A * C", ** where "*" may match any single token. The position list in this case ** is populated as for "A * C" before returning. ** -** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is +** 2. NEAR is treated as AND. If the expression is "x NEAR y", it is ** advanced to point to the next row that matches "x AND y". -** +** ** See sqlite3Fts3EvalTestDeferred() for details on testing if a row is ** really a match, taking into account deferred tokens and NEAR operators. */ @@ -145975,9 +192823,8 @@ static void fts3EvalNextRow( Fts3Expr *pExpr, /* Expr. to advance to next matching row */ int *pRc /* IN/OUT: Error code */ ){ - if( *pRc==SQLITE_OK ){ + if( *pRc==SQLITE_OK && pExpr->bEof==0 ){ int bDescDoclist = pCsr->bDesc; /* Used by DOCID_CMP() macro */ - assert( pExpr->bEof==0 ); pExpr->bStart = 1; switch( pExpr->eType ){ @@ -146015,7 +192862,8 @@ static void fts3EvalNextRow( pExpr->iDocid = pLeft->iDocid; pExpr->bEof = (pLeft->bEof || pRight->bEof); if( pExpr->eType==FTSQUERY_NEAR && pExpr->bEof ){ - if( pRight->pPhrase && pRight->pPhrase->doclist.aAll ){ + assert( pRight->eType==FTSQUERY_PHRASE ); + if( pRight->pPhrase->doclist.aAll ){ Fts3Doclist *pDl = &pRight->pPhrase->doclist; while( *pRc==SQLITE_OK && pRight->bEof==0 ){ memset(pDl->pList, 0, pDl->nList); @@ -146029,22 +192877,23 @@ static void fts3EvalNextRow( fts3EvalNextRow(pCsr, pLeft, pRc); } } + pRight->bEof = pLeft->bEof = 1; } } break; } - + case FTSQUERY_OR: { Fts3Expr *pLeft = pExpr->pLeft; Fts3Expr *pRight = pExpr->pRight; sqlite3_int64 iCmp = DOCID_CMP(pLeft->iDocid, pRight->iDocid); - assert( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); - assert( pRight->bStart || pLeft->iDocid==pRight->iDocid ); + assert_fts3_nc( pLeft->bStart || pLeft->iDocid==pRight->iDocid ); + assert_fts3_nc( pRight->bStart || pLeft->iDocid==pRight->iDocid ); if( pRight->bEof || (pLeft->bEof==0 && iCmp<0) ){ fts3EvalNextRow(pCsr, pLeft, pRc); - }else if( pLeft->bEof || (pRight->bEof==0 && iCmp>0) ){ + }else if( pLeft->bEof || iCmp>0 ){ fts3EvalNextRow(pCsr, pRight, pRc); }else{ fts3EvalNextRow(pCsr, pLeft, pRc); @@ -146073,9 +192922,9 @@ static void fts3EvalNextRow( fts3EvalNextRow(pCsr, pLeft, pRc); if( pLeft->bEof==0 ){ - while( !*pRc - && !pRight->bEof - && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 + while( !*pRc + && !pRight->bEof + && DOCID_CMP(pLeft->iDocid, pRight->iDocid)>0 ){ fts3EvalNextRow(pCsr, pRight, pRc); } @@ -146100,14 +192949,14 @@ static void fts3EvalNextRow( ** If *pRc is not SQLITE_OK, or if pExpr is not the root node of a NEAR ** cluster, then this function returns 1 immediately. ** -** Otherwise, it checks if the current row really does match the NEAR -** expression, using the data currently stored in the position lists -** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. +** Otherwise, it checks if the current row really does match the NEAR +** expression, using the data currently stored in the position lists +** (Fts3Expr->pPhrase.doclist.pList/nList) for each phrase in the expression. ** ** If the current row is a match, the position list associated with each ** phrase in the NEAR expression is edited in place to contain only those ** phrase instances sufficiently close to their peers to satisfy all NEAR -** constraints. In this case it returns 1. If the NEAR expression does not +** constraints. In this case it returns 1. If the NEAR expression does not ** match the current row, 0 is returned. The position lists may or may not ** be edited if 0 is returned. */ @@ -146130,57 +192979,53 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ ** | | ** "w" "x" ** - ** The right-hand child of a NEAR node is always a phrase. The + ** The right-hand child of a NEAR node is always a phrase. The ** left-hand child may be either a phrase or a NEAR node. There are ** no exceptions to this - it's the way the parser in fts3_expr.c works. */ - if( *pRc==SQLITE_OK - && pExpr->eType==FTSQUERY_NEAR - && pExpr->bEof==0 + if( *pRc==SQLITE_OK + && pExpr->eType==FTSQUERY_NEAR && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ - Fts3Expr *p; - int nTmp = 0; /* Bytes of temp space */ + Fts3Expr *p; + sqlite3_int64 nTmp = 0; /* Bytes of temp space */ char *aTmp; /* Temp space for PoslistNearMerge() */ /* Allocate temporary working space. */ for(p=pExpr; p->pLeft; p=p->pLeft){ + assert( p->pRight->pPhrase->doclist.nList>0 ); nTmp += p->pRight->pPhrase->doclist.nList; } nTmp += p->pPhrase->doclist.nList; - if( nTmp==0 ){ + aTmp = sqlite3_malloc64(nTmp*2 + FTS3_VARINT_MAX); + if( !aTmp ){ + *pRc = SQLITE_NOMEM; res = 0; }else{ - aTmp = sqlite3_malloc(nTmp*2); - if( !aTmp ){ - *pRc = SQLITE_NOMEM; - res = 0; - }else{ - char *aPoslist = p->pPhrase->doclist.pList; - int nToken = p->pPhrase->nToken; + char *aPoslist = p->pPhrase->doclist.pList; + int nToken = p->pPhrase->nToken; - for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ - Fts3Phrase *pPhrase = p->pRight->pPhrase; - int nNear = p->nNear; - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } - - aPoslist = pExpr->pRight->pPhrase->doclist.pList; - nToken = pExpr->pRight->pPhrase->nToken; - for(p=pExpr->pLeft; p && res; p=p->pLeft){ - int nNear; - Fts3Phrase *pPhrase; - assert( p->pParent && p->pParent->pLeft==p ); - nNear = p->pParent->nNear; - pPhrase = ( - p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase - ); - res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); - } + for(p=p->pParent;res && p && p->eType==FTSQUERY_NEAR; p=p->pParent){ + Fts3Phrase *pPhrase = p->pRight->pPhrase; + int nNear = p->nNear; + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); } - sqlite3_free(aTmp); + aPoslist = pExpr->pRight->pPhrase->doclist.pList; + nToken = pExpr->pRight->pPhrase->nToken; + for(p=pExpr->pLeft; p && res; p=p->pLeft){ + int nNear; + Fts3Phrase *pPhrase; + assert( p->pParent && p->pParent->pLeft==p ); + nNear = p->pParent->nNear; + pPhrase = ( + p->eType==FTSQUERY_NEAR ? p->pRight->pPhrase : p->pPhrase + ); + res = fts3EvalNearTrim(nNear, aTmp, &aPoslist, &nToken, pPhrase); + } } + + sqlite3_free(aTmp); } return res; @@ -146189,12 +193034,12 @@ static int fts3EvalNearTest(Fts3Expr *pExpr, int *pRc){ /* ** This function is a helper function for sqlite3Fts3EvalTestDeferred(). ** Assuming no error occurs or has occurred, It returns non-zero if the -** expression passed as the second argument matches the row that pCsr +** expression passed as the second argument matches the row that pCsr ** currently points to, or zero if it does not. ** ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. -** If an error occurs during execution of this function, *pRc is set to -** the appropriate SQLite error code. In this case the returned value is +** If an error occurs during execution of this function, *pRc is set to +** the appropriate SQLite error code. In this case the returned value is ** undefined. */ static int fts3EvalTestExpr( @@ -146213,10 +193058,10 @@ static int fts3EvalTestExpr( && fts3EvalNearTest(pExpr, pRc) ); - /* If the NEAR expression does not match any rows, zero the doclist for + /* If the NEAR expression does not match any rows, zero the doclist for ** all phrases involved in the NEAR. This is because the snippet(), - ** offsets() and matchinfo() functions are not supposed to recognize - ** any instances of phrases that are part of unmatched NEAR queries. + ** offsets() and matchinfo() functions are not supposed to recognize + ** any instances of phrases that are part of unmatched NEAR queries. ** For example if this expression: ** ** ... MATCH 'a OR (b NEAR c)' @@ -146228,8 +193073,8 @@ static int fts3EvalTestExpr( ** then any snippet() should ony highlight the "a" term, not the "b" ** (as "b" is part of a non-matching NEAR clause). */ - if( bHit==0 - && pExpr->eType==FTSQUERY_NEAR + if( bHit==0 + && pExpr->eType==FTSQUERY_NEAR && (pExpr->pParent==0 || pExpr->pParent->eType!=FTSQUERY_NEAR) ){ Fts3Expr *p; @@ -146261,11 +193106,10 @@ static int fts3EvalTestExpr( default: { #ifndef SQLITE_DISABLE_FTS4_DEFERRED - if( pCsr->pDeferred - && (pExpr->iDocid==pCsr->iPrevId || pExpr->bDeferred) - ){ + if( pCsr->pDeferred && (pExpr->bDeferred || ( + pExpr->iDocid==pCsr->iPrevId && pExpr->pPhrase->doclist.pList + ))){ Fts3Phrase *pPhrase = pExpr->pPhrase; - assert( pExpr->bDeferred || pPhrase->doclist.bFreeList==0 ); if( pExpr->bDeferred ){ fts3EvalInvalidatePoslist(pPhrase); } @@ -146275,7 +193119,10 @@ static int fts3EvalTestExpr( }else #endif { - bHit = (pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId); + bHit = ( + pExpr->bEof==0 && pExpr->iDocid==pCsr->iPrevId + && pExpr->pPhrase->doclist.nList>0 + ); } break; } @@ -146317,7 +193164,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalTestDeferred(Fts3Cursor *pCsr, int *pRc){ ** memory and scan it to determine the position list for each deferred ** token. Then, see if this row is really a match, considering deferred ** tokens and NEAR operators (neither of which were taken into account - ** earlier, by fts3EvalNextRow()). + ** earlier, by fts3EvalNextRow()). */ if( pCsr->pDeferred ){ rc = fts3CursorSeek(0, pCsr); @@ -146372,7 +193219,7 @@ static int fts3EvalNext(Fts3Cursor *pCsr){ /* ** Restart interation for expression pExpr so that the next call to -** fts3EvalNext() visits the first row. Do not allow incremental +** fts3EvalNext() visits the first row. Do not allow incremental ** loading or merging of phrase doclists for this iteration. ** ** If *pRc is other than SQLITE_OK when this function is called, it is @@ -146415,22 +193262,21 @@ static void fts3EvalRestart( } /* -** After allocating the Fts3Expr.aMI[] array for each phrase in the +** After allocating the Fts3Expr.aMI[] array for each phrase in the ** expression rooted at pExpr, the cursor iterates through all rows matched ** by pExpr, calling this function for each row. This function increments ** the values in Fts3Expr.aMI[] according to the position-list currently -** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase +** found in Fts3Expr.pPhrase->doclist.pList for each of the phrase ** expression nodes. */ -static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ +static void fts3EvalUpdateCounts(Fts3Expr *pExpr, int nCol){ if( pExpr ){ Fts3Phrase *pPhrase = pExpr->pPhrase; if( pPhrase && pPhrase->doclist.pList ){ int iCol = 0; char *p = pPhrase->doclist.pList; - assert( *p ); - while( 1 ){ + do{ u8 c = 0; int iCnt = 0; while( 0xFE & (*p | c) ){ @@ -146446,14 +193292,30 @@ static void fts3EvalUpdateCounts(Fts3Expr *pExpr){ if( *p==0x00 ) break; p++; p += fts3GetVarint32(p, &iCol); - } + }while( iColpLeft); - fts3EvalUpdateCounts(pExpr->pRight); + fts3EvalUpdateCounts(pExpr->pLeft, nCol); + fts3EvalUpdateCounts(pExpr->pRight, nCol); } } +/* +** This is an sqlite3Fts3ExprIterate() callback. If the Fts3Expr.aMI[] array +** has not yet been allocated, allocate and zero it. Otherwise, just zero +** it. +*/ +static int fts3AllocateMSI(Fts3Expr *pExpr, int iPhrase, void *pCtx){ + Fts3Table *pTab = (Fts3Table*)pCtx; + UNUSED_PARAMETER(iPhrase); + if( pExpr->aMI==0 ){ + pExpr->aMI = (u32 *)sqlite3_malloc64(pTab->nColumn * 3 * sizeof(u32)); + if( pExpr->aMI==0 ) return SQLITE_NOMEM; + } + memset(pExpr->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); + return SQLITE_OK; +} + /* ** Expression pExpr must be of type FTSQUERY_PHRASE. ** @@ -146475,7 +193337,6 @@ static int fts3EvalGatherStats( if( pExpr->aMI==0 ){ Fts3Table *pTab = (Fts3Table *)pCsr->base.pVtab; Fts3Expr *pRoot; /* Root of NEAR expression */ - Fts3Expr *p; /* Iterator used for several purposes */ sqlite3_int64 iPrevId = pCsr->iPrevId; sqlite3_int64 iDocid; @@ -146483,7 +193344,9 @@ static int fts3EvalGatherStats( /* Find the root of the NEAR expression */ pRoot = pExpr; - while( pRoot->pParent && pRoot->pParent->eType==FTSQUERY_NEAR ){ + while( pRoot->pParent + && (pRoot->pParent->eType==FTSQUERY_NEAR || pRoot->bDeferred) + ){ pRoot = pRoot->pParent; } iDocid = pRoot->iDocid; @@ -146491,14 +193354,8 @@ static int fts3EvalGatherStats( assert( pRoot->bStart ); /* Allocate space for the aMSI[] array of each FTSQUERY_PHRASE node */ - for(p=pRoot; p; p=p->pLeft){ - Fts3Expr *pE = (p->eType==FTSQUERY_PHRASE?p:p->pRight); - assert( pE->aMI==0 ); - pE->aMI = (u32 *)sqlite3_malloc(pTab->nColumn * 3 * sizeof(u32)); - if( !pE->aMI ) return SQLITE_NOMEM; - memset(pE->aMI, 0, pTab->nColumn * 3 * sizeof(u32)); - } - + rc = sqlite3Fts3ExprIterate(pRoot, fts3AllocateMSI, (void*)pTab); + if( rc!=SQLITE_OK ) return rc; fts3EvalRestart(pCsr, pRoot, &rc); while( pCsr->isEof==0 && rc==SQLITE_OK ){ @@ -146514,13 +193371,13 @@ static int fts3EvalGatherStats( pCsr->isRequireSeek = 1; pCsr->isMatchinfoNeeded = 1; pCsr->iPrevId = pRoot->iDocid; - }while( pCsr->isEof==0 - && pRoot->eType==FTSQUERY_NEAR - && sqlite3Fts3EvalTestDeferred(pCsr, &rc) + }while( pCsr->isEof==0 + && pRoot->eType==FTSQUERY_NEAR + && sqlite3Fts3EvalTestDeferred(pCsr, &rc) ); if( rc==SQLITE_OK && pCsr->isEof==0 ){ - fts3EvalUpdateCounts(pRoot); + fts3EvalUpdateCounts(pRoot, pTab->nColumn); } } @@ -146531,7 +193388,7 @@ static int fts3EvalGatherStats( pRoot->bEof = bEof; }else{ /* Caution: pRoot may iterate through docids in ascending or descending - ** order. For this reason, even though it seems more defensive, the + ** order. For this reason, even though it seems more defensive, the ** do loop can not be written: ** ** do {...} while( pRoot->iDocidbEof==0 ); + assert_fts3_nc( pRoot->bEof==0 ); + if( pRoot->bEof ) rc = FTS_CORRUPT_VTAB; }while( pRoot->iDocid!=iDocid && rc==SQLITE_OK ); } } @@ -146547,10 +193405,10 @@ static int fts3EvalGatherStats( } /* -** This function is used by the matchinfo() module to query a phrase +** This function is used by the matchinfo() module to query a phrase ** expression node for the following information: ** -** 1. The total number of occurrences of the phrase in each column of +** 1. The total number of occurrences of the phrase in each column of ** the FTS table (considering all rows), and ** ** 2. For each column, the number of rows in the table for which the @@ -146564,12 +193422,12 @@ static int fts3EvalGatherStats( ** ** Caveats: ** -** * If a phrase consists entirely of deferred tokens, then all output +** * If a phrase consists entirely of deferred tokens, then all output ** values are set to the number of documents in the table. In other -** words we assume that very common tokens occur exactly once in each +** words we assume that very common tokens occur exactly once in each ** column of each row of the table. ** -** * If a phrase contains some deferred tokens (and some non-deferred +** * If a phrase contains some deferred tokens (and some non-deferred ** tokens), count the potential occurrence identified by considering ** the non-deferred tokens instead of actual phrase occurrences. ** @@ -146607,14 +193465,14 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhraseStats( /* ** The expression pExpr passed as the second argument to this function -** must be of type FTSQUERY_PHRASE. +** must be of type FTSQUERY_PHRASE. ** ** The returned value is either NULL or a pointer to a buffer containing ** a position-list indicating the occurrences of the phrase in column iCol -** of the current row. +** of the current row. ** -** More specifically, the returned buffer contains 1 varint for each -** occurrence of the phrase in the column, stored using the normal (delta+2) +** More specifically, the returned buffer contains 1 varint for each +** occurrence of the phrase in the column, stored using the normal (delta+2) ** compression and is terminated by either an 0x01 or 0x00 byte. For example, ** if the requested column contains "a b X c d X X" and the position-list ** for 'X' is requested, the buffer returned may contain: @@ -146636,7 +193494,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( int iThis; sqlite3_int64 iDocid; - /* If this phrase is applies specifically to some column other than + /* If this phrase is applies specifically to some column other than ** column iCol, return a NULL pointer. */ *ppOut = 0; assert( iCol>=0 && iColnColumn ); @@ -146653,10 +193511,11 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( u8 bTreeEof = 0; Fts3Expr *p; /* Used to iterate from pExpr to root */ Fts3Expr *pNear; /* Most senior NEAR ancestor (or pExpr) */ + Fts3Expr *pRun; /* Closest non-deferred ancestor of pNear */ int bMatch; - /* Check if this phrase descends from an OR expression node. If not, - ** return NULL. Otherwise, the entry that corresponds to docid + /* Check if this phrase descends from an OR expression node. If not, + ** return NULL. Otherwise, the entry that corresponds to docid ** pCsr->iPrevId may lie earlier in the doclist buffer. Or, if the ** tree that the node is part of has been marked as EOF, but the node ** itself is not EOF, then it may point to an earlier entry. */ @@ -146667,22 +193526,30 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( if( p->bEof ) bTreeEof = 1; } if( bOr==0 ) return SQLITE_OK; + pRun = pNear; + while( pRun->bDeferred ){ + assert( pRun->pParent ); + pRun = pRun->pParent; + } /* This is the descendent of an OR node. In this case we cannot use ** an incremental phrase. Load the entire doclist for the phrase ** into memory in this case. */ if( pPhrase->bIncr ){ - int bEofSave = pNear->bEof; - fts3EvalRestart(pCsr, pNear, &rc); - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); - if( bEofSave==0 && pNear->iDocid==iDocid ) break; + int bEofSave = pRun->bEof; + fts3EvalRestart(pCsr, pRun, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); + if( bEofSave==0 && pRun->iDocid==iDocid ) break; } assert( rc!=SQLITE_OK || pPhrase->bIncr==0 ); + if( rc==SQLITE_OK && pRun->bEof!=bEofSave ){ + rc = FTS_CORRUPT_VTAB; + } } if( bTreeEof ){ - while( rc==SQLITE_OK && !pNear->bEof ){ - fts3EvalNextRow(pCsr, pNear, &rc); + while( rc==SQLITE_OK && !pRun->bEof ){ + fts3EvalNextRow(pCsr, pRun, &rc); } } if( rc!=SQLITE_OK ) return rc; @@ -146704,7 +193571,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( (pIter >= (pPh->doclist.aAll + pPh->doclist.nAll)); while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)<0 ) && bEof==0 ){ sqlite3Fts3DoclistNext( - bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, &pIter, &iDocid, &bEof ); } @@ -146713,7 +193580,7 @@ SQLITE_PRIVATE int sqlite3Fts3EvalPhrasePoslist( while( (pIter==0 || DOCID_CMP(iDocid, pCsr->iPrevId)>0 ) && bEof==0 ){ int dummy; sqlite3Fts3DoclistPrev( - bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, + bDescDoclist, pPh->doclist.aAll, pPh->doclist.nAll, &pIter, &iDocid, &dummy, &bEof ); } @@ -146788,8 +193655,8 @@ SQLITE_PRIVATE int sqlite3Fts3Corrupt(){ #ifdef _WIN32 __declspec(dllexport) #endif -SQLITE_API int SQLITE_STDCALL sqlite3_fts3_init( - sqlite3 *db, +SQLITE_API int sqlite3_fts3_init( + sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ @@ -146870,7 +193737,7 @@ static int fts3auxConnectMethod( char const *zFts3; /* Name of fts3 table */ int nDb; /* Result of strlen(zDb) */ int nFts3; /* Result of strlen(zFts3) */ - int nByte; /* Bytes of space to allocate here */ + sqlite3_int64 nByte; /* Bytes of space to allocate here */ int rc; /* value returned by declare_vtab() */ Fts3auxTable *p; /* Virtual table object to return */ @@ -146883,11 +193750,11 @@ static int fts3auxConnectMethod( */ if( argc!=4 && argc!=5 ) goto bad_args; - zDb = argv[1]; + zDb = argv[1]; nDb = (int)strlen(zDb); if( argc==5 ){ if( nDb==4 && 0==sqlite3_strnicmp("temp", zDb, 4) ){ - zDb = argv[3]; + zDb = argv[3]; nDb = (int)strlen(zDb); zFts3 = argv[4]; }else{ @@ -146902,7 +193769,7 @@ static int fts3auxConnectMethod( if( rc!=SQLITE_OK ) return rc; nByte = sizeof(Fts3auxTable) + sizeof(Fts3Table) + nDb + nFts3 + 2; - p = (Fts3auxTable *)sqlite3_malloc(nByte); + p = (Fts3auxTable *)sqlite3_malloc64(nByte); if( !p ) return SQLITE_NOMEM; memset(p, 0, nByte); @@ -146951,7 +193818,7 @@ static int fts3auxDisconnectMethod(sqlite3_vtab *pVtab){ ** xBestIndex - Analyze a WHERE and ORDER BY clause. */ static int fts3auxBestIndexMethod( - sqlite3_vtab *pVTab, + sqlite3_vtab *pVTab, sqlite3_index_info *pInfo ){ int i; @@ -146964,14 +193831,14 @@ static int fts3auxBestIndexMethod( UNUSED_PARAMETER(pVTab); /* This vtab delivers always results in "ORDER BY term ASC" order. */ - if( pInfo->nOrderBy==1 - && pInfo->aOrderBy[0].iColumn==0 + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 && pInfo->aOrderBy[0].desc==0 ){ pInfo->orderByConsumed = 1; } - /* Search for equality and range constraints on the "term" column. + /* Search for equality and range constraints on the "term" column. ** And equality constraints on the hidden "languageid" column. */ for(i=0; inConstraint; i++){ if( pInfo->aConstraint[i].usable ){ @@ -147052,11 +193919,11 @@ static int fts3auxCloseMethod(sqlite3_vtab_cursor *pCursor){ static int fts3auxGrowStatArray(Fts3auxCursor *pCsr, int nSize){ if( nSize>pCsr->nStat ){ struct Fts3auxColstats *aNew; - aNew = (struct Fts3auxColstats *)sqlite3_realloc(pCsr->aStat, + aNew = (struct Fts3auxColstats *)sqlite3_realloc64(pCsr->aStat, sizeof(struct Fts3auxColstats) * nSize ); if( aNew==0 ) return SQLITE_NOMEM; - memset(&aNew[pCsr->nStat], 0, + memset(&aNew[pCsr->nStat], 0, sizeof(struct Fts3auxColstats) * (nSize - pCsr->nStat) ); pCsr->aStat = aNew; @@ -147101,6 +193968,7 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ if( fts3auxGrowStatArray(pCsr, 2) ) return SQLITE_NOMEM; memset(pCsr->aStat, 0, sizeof(struct Fts3auxColstats) * pCsr->nStat); iCol = 0; + rc = SQLITE_OK; while( iaStat[iCol+1].nDoc++; eState = 2; @@ -147152,7 +194024,6 @@ static int fts3auxNextMethod(sqlite3_vtab_cursor *pCursor){ } pCsr->iCol = 0; - rc = SQLITE_OK; }else{ pCsr->isEof = 1; } @@ -147210,6 +194081,7 @@ static int fts3auxFilterMethod( sqlite3Fts3SegReaderFinish(&pCsr->csr); sqlite3_free((void *)pCsr->filter.zTerm); sqlite3_free(pCsr->aStat); + sqlite3_free(pCsr->zStop); memset(&pCsr->csr, 0, ((u8*)&pCsr[1]) - (u8*)&pCsr->csr); pCsr->filter.flags = FTS3_SEGMENT_REQUIRE_POS|FTS3_SEGMENT_IGNORE_EMPTY; @@ -147220,17 +194092,17 @@ static int fts3auxFilterMethod( assert( (iEq==0 && iGe==-1) || (iEq==-1 && iGe==0) ); if( zStr ){ pCsr->filter.zTerm = sqlite3_mprintf("%s", zStr); - pCsr->filter.nTerm = sqlite3_value_bytes(apVal[0]); if( pCsr->filter.zTerm==0 ) return SQLITE_NOMEM; + pCsr->filter.nTerm = (int)strlen(pCsr->filter.zTerm); } } if( iLe>=0 ){ pCsr->zStop = sqlite3_mprintf("%s", sqlite3_value_text(apVal[iLe])); - pCsr->nStop = sqlite3_value_bytes(apVal[iLe]); if( pCsr->zStop==0 ) return SQLITE_NOMEM; + pCsr->nStop = (int)strlen(pCsr->zStop); } - + if( iLangid>=0 ){ iLangVal = sqlite3_value_int(apVal[iLangid]); @@ -147343,7 +194215,9 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ @@ -147368,15 +194242,15 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ ****************************************************************************** ** ** This module contains code that implements a parser for fts3 query strings -** (the right-hand argument to the MATCH operator). Because the supported +** (the right-hand argument to the MATCH operator). Because the supported ** syntax is relatively simple, the whole tokenizer/parser system is -** hand-coded. +** hand-coded. */ /* #include "fts3Int.h" */ #if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS3) /* -** By default, this module parses the legacy syntax that has been +** By default, this module parses the legacy syntax that has been ** traditionally used by fts3. Or, if SQLITE_ENABLE_FTS3_PARENTHESIS ** is defined, then it uses the new syntax. The differences between ** the new and the old syntaxes are: @@ -147385,7 +194259,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ ** ** b) The new syntax supports the AND and NOT operators. The old does not. ** -** c) The old syntax supports the "-" token qualifier. This is not +** c) The old syntax supports the "-" token qualifier. This is not ** supported by the new syntax (it is replaced by the NOT operator). ** ** d) When using the old syntax, the OR operator has a greater precedence @@ -147394,7 +194268,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ ** ** If compiled with SQLITE_TEST defined, then this module exports the ** symbol "int sqlite3_fts3_enable_parentheses". Setting this variable -** to zero causes the module to use the old syntax. If it is set to +** to zero causes the module to use the old syntax. If it is set to ** non-zero the new syntax is activated. This is so both syntaxes can ** be tested using a single build of testfixture. ** @@ -147423,7 +194297,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitAux(sqlite3 *db){ #ifdef SQLITE_TEST SQLITE_API int sqlite3_fts3_enable_parentheses = 0; #else -# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS +# ifdef SQLITE_ENABLE_FTS3_PARENTHESIS # define sqlite3_fts3_enable_parentheses 1 # else # define sqlite3_fts3_enable_parentheses 0 @@ -147441,7 +194315,7 @@ SQLITE_API int sqlite3_fts3_enable_parentheses = 0; /* ** isNot: ** This variable is used by function getNextNode(). When getNextNode() is -** called, it sets ParseContext.isNot to true if the 'next node' is a +** called, it sets ParseContext.isNot to true if the 'next node' is a ** FTSQUERY_PHRASE with a unary "-" attached to it. i.e. "mysql" in the ** FTS3 query "sqlite -mysql". Otherwise, ParseContext.isNot is set to ** zero. @@ -147460,7 +194334,7 @@ struct ParseContext { }; /* -** This function is equivalent to the standard isspace() function. +** This function is equivalent to the standard isspace() function. ** ** The standard isspace() can be awkward to use safely, because although it ** is defined to accept an argument of type int, its behavior when passed @@ -147476,11 +194350,11 @@ static int fts3isspace(char c){ /* ** Allocate nByte bytes of memory using sqlite3_malloc(). If successful, -** zero the memory before returning a pointer to it. If unsuccessful, +** zero the memory before returning a pointer to it. If unsuccessful, ** return NULL. */ -static void *fts3MallocZero(int nByte){ - void *pRet = sqlite3_malloc(nByte); +SQLITE_PRIVATE void *sqlite3Fts3MallocZero(sqlite3_int64 nByte){ + void *pRet = sqlite3_malloc64(nByte); if( pRet ) memset(pRet, 0, nByte); return pRet; } @@ -147524,7 +194398,7 @@ static int fts3ExprParse(ParseContext *, const char *, int, Fts3Expr **, int *); ** structure of type FTSQUERY_PHRASE containing a phrase consisting of this ** single token and set *ppExpr to point to it. If the end of the buffer is ** reached before a token is found, set *ppExpr to zero. It is the -** responsibility of the caller to eventually deallocate the allocated +** responsibility of the caller to eventually deallocate the allocated ** Fts3Expr structure (if any) by passing it to sqlite3_free(). ** ** Return SQLITE_OK if successful, or SQLITE_NOMEM if a memory allocation @@ -147555,12 +194429,12 @@ static int getNextToken( if( rc==SQLITE_OK ){ const char *zToken; int nToken = 0, iStart = 0, iEnd = 0, iPosition = 0; - int nByte; /* total space to allocate */ + sqlite3_int64 nByte; /* total space to allocate */ rc = pModule->xNext(pCursor, &zToken, &nToken, &iStart, &iEnd, &iPosition); if( rc==SQLITE_OK ){ nByte = sizeof(Fts3Expr) + sizeof(Fts3Phrase) + nToken; - pRet = (Fts3Expr *)fts3MallocZero(nByte); + pRet = (Fts3Expr *)sqlite3Fts3MallocZero(nByte); if( !pRet ){ rc = SQLITE_NOMEM; }else{ @@ -147578,8 +194452,8 @@ static int getNextToken( } while( 1 ){ - if( !sqlite3_fts3_enable_parentheses - && iStart>0 && z[iStart-1]=='-' + if( !sqlite3_fts3_enable_parentheses + && iStart>0 && z[iStart-1]=='-' ){ pParse->isNot = 1; iStart--; @@ -147599,7 +194473,7 @@ static int getNextToken( pModule->xClose(pCursor); } - + *ppExpr = pRet; return rc; } @@ -147609,8 +194483,8 @@ static int getNextToken( ** Enlarge a memory allocation. If an out-of-memory allocation occurs, ** then free the old allocation. */ -static void *fts3ReallocOrFree(void *pOrig, int nNew){ - void *pRet = sqlite3_realloc(pOrig, nNew); +static void *fts3ReallocOrFree(void *pOrig, sqlite3_int64 nNew){ + void *pRet = sqlite3_realloc64(pOrig, nNew); if( !pRet ){ sqlite3_free(pOrig); } @@ -147621,7 +194495,7 @@ static void *fts3ReallocOrFree(void *pOrig, int nNew){ ** Buffer zInput, length nInput, contains the contents of a quoted string ** that appeared as part of an fts3 query expression. Neither quote character ** is included in the buffer. This function attempts to tokenize the entire -** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE +** input buffer and create an Fts3Expr structure of type FTSQUERY_PHRASE ** containing the results. ** ** If successful, SQLITE_OK is returned and *ppExpr set to point at the @@ -147646,7 +194520,7 @@ static int getNextString( int nToken = 0; /* The final Fts3Expr data structure, including the Fts3Phrase, - ** Fts3PhraseToken structures token buffers are all stored as a single + ** Fts3PhraseToken structures token buffers are all stored as a single ** allocation so that the expression can be freed with a single call to ** sqlite3_free(). Setting this up requires a two pass approach. ** @@ -147655,7 +194529,7 @@ static int getNextString( ** to assemble data in two dynamic buffers: ** ** Buffer p: Points to the Fts3Expr structure, followed by the Fts3Phrase - ** structure, followed by the array of Fts3PhraseToken + ** structure, followed by the array of Fts3PhraseToken ** structures. This pass only populates the Fts3PhraseToken array. ** ** Buffer zTemp: Contains copies of all tokens. @@ -147676,10 +194550,11 @@ static int getNextString( Fts3PhraseToken *pToken; p = fts3ReallocOrFree(p, nSpace + ii*sizeof(Fts3PhraseToken)); - if( !p ) goto no_mem; - zTemp = fts3ReallocOrFree(zTemp, nTemp + nByte); - if( !zTemp ) goto no_mem; + if( !zTemp || !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } assert( nToken==ii ); pToken = &((Fts3Phrase *)(&p[1]))->aToken[ii]; @@ -147694,9 +194569,6 @@ static int getNextString( nToken = ii+1; } } - - pModule->xClose(pCursor); - pCursor = 0; } if( rc==SQLITE_DONE ){ @@ -147704,7 +194576,10 @@ static int getNextString( char *zBuf = 0; p = fts3ReallocOrFree(p, nSpace + nToken*sizeof(Fts3PhraseToken) + nTemp); - if( !p ) goto no_mem; + if( !p ){ + rc = SQLITE_NOMEM; + goto getnextstring_out; + } memset(p, 0, (char *)&(((Fts3Phrase *)&p[1])->aToken[0])-(char *)p); p->eType = FTSQUERY_PHRASE; p->pPhrase = (Fts3Phrase *)&p[1]; @@ -147712,11 +194587,9 @@ static int getNextString( p->pPhrase->nToken = nToken; zBuf = (char *)&p->pPhrase->aToken[nToken]; + assert( nTemp==0 || zTemp ); if( zTemp ){ memcpy(zBuf, zTemp, nTemp); - sqlite3_free(zTemp); - }else{ - assert( nTemp==0 ); } for(jj=0; jjpPhrase->nToken; jj++){ @@ -147726,21 +194599,21 @@ static int getNextString( rc = SQLITE_OK; } - *ppExpr = p; - return rc; -no_mem: - + getnextstring_out: if( pCursor ){ pModule->xClose(pCursor); } sqlite3_free(zTemp); - sqlite3_free(p); - *ppExpr = 0; - return SQLITE_NOMEM; + if( rc!=SQLITE_OK ){ + sqlite3_free(p); + p = 0; + } + *ppExpr = p; + return rc; } /* -** The output variable *ppExpr is populated with an allocated Fts3Expr +** The output variable *ppExpr is populated with an allocated Fts3Expr ** structure, or set to 0 if the end of the input buffer is reached. ** ** Returns an SQLite error code. SQLITE_OK if everything works, SQLITE_NOMEM @@ -147776,7 +194649,7 @@ static int getNextNode( pParse->isNot = 0; /* Skip over any whitespace before checking for a keyword, an open or - ** close bracket, or a quoted string. + ** close bracket, or a quoted string. */ while( nInput>0 && fts3isspace(*zInput) ){ nInput--; @@ -147803,22 +194676,19 @@ static int getNextNode( if( pKey->eType==FTSQUERY_NEAR ){ assert( nKey==4 ); if( zInput[4]=='/' && zInput[5]>='0' && zInput[5]<='9' ){ - nNear = 0; - for(nKey=5; zInput[nKey]>='0' && zInput[nKey]<='9'; nKey++){ - nNear = nNear * 10 + (zInput[nKey] - '0'); - } + nKey += 1+sqlite3Fts3ReadInt(&zInput[nKey+1], &nNear); } } /* At this point this is probably a keyword. But for that to be true, ** the next byte must contain either whitespace, an open or close - ** parenthesis, a quote character, or EOF. + ** parenthesis, a quote character, or EOF. */ cNext = zInput[nKey]; - if( fts3isspace(cNext) + if( fts3isspace(cNext) || cNext=='"' || cNext=='(' || cNext==')' || cNext==0 ){ - pRet = (Fts3Expr *)fts3MallocZero(sizeof(Fts3Expr)); + pRet = (Fts3Expr *)sqlite3Fts3MallocZero(sizeof(Fts3Expr)); if( !pRet ){ return SQLITE_NOMEM; } @@ -147853,8 +194723,12 @@ static int getNextNode( if( *zInput=='(' ){ int nConsumed = 0; pParse->nNest++; +#if !defined(SQLITE_MAX_EXPR_DEPTH) + if( pParse->nNest>1000 ) return SQLITE_ERROR; +#elif SQLITE_MAX_EXPR_DEPTH>0 + if( pParse->nNest>SQLITE_MAX_EXPR_DEPTH ) return SQLITE_ERROR; +#endif rc = fts3ExprParse(pParse, zInput+1, nInput-1, ppExpr, &nConsumed); - if( rc==SQLITE_OK && !*ppExpr ){ rc = SQLITE_DONE; } *pnConsumed = (int)(zInput - z) + 1 + nConsumed; return rc; }else if( *zInput==')' ){ @@ -147865,15 +194739,15 @@ static int getNextNode( } } - /* If control flows to this point, this must be a regular token, or + /* If control flows to this point, this must be a regular token, or ** the end of the input. Read a regular token using the sqlite3_tokenizer ** interface. Before doing so, figure out if there is an explicit - ** column specifier for the token. + ** column specifier for the token. ** ** TODO: Strangely, it is not possible to associate a column specifier ** with a quoted phrase, only with a single token. Not sure if this was ** an implementation artifact or an intentional decision when fts3 was - ** first implemented. Whichever it was, this module duplicates the + ** first implemented. Whichever it was, this module duplicates the ** limitation. */ iCol = pParse->iDefaultCol; @@ -147881,8 +194755,8 @@ static int getNextNode( for(ii=0; iinCol; ii++){ const char *zStr = pParse->azCol[ii]; int nStr = (int)strlen(zStr); - if( nInput>nStr && zInput[nStr]==':' - && sqlite3_strnicmp(zStr, zInput, nStr)==0 + if( nInput>nStr && zInput[nStr]==':' + && sqlite3_strnicmp(zStr, zInput, nStr)==0 ){ iCol = ii; iColLen = (int)((zInput - z) + nStr + 1); @@ -147927,7 +194801,7 @@ static int opPrecedence(Fts3Expr *p){ } /* -** Argument ppHead contains a pointer to the current head of a query +** Argument ppHead contains a pointer to the current head of a query ** expression tree being parsed. pPrev is the expression node most recently ** inserted into the tree. This function adds pNew, which is always a binary ** operator node, into the expression tree based on the relative precedence @@ -147957,7 +194831,7 @@ static void insertBinaryOperator( /* ** Parse the fts3 query expression found in buffer z, length n. This function -** returns either when the end of the buffer is reached or an unmatched +** returns either when the end of the buffer is reached or an unmatched ** closing bracket - ')' - is encountered. ** ** If successful, SQLITE_OK is returned, *ppExpr is set to point to the @@ -147989,11 +194863,11 @@ static int fts3ExprParse( if( p ){ int isPhrase; - if( !sqlite3_fts3_enable_parentheses - && p->eType==FTSQUERY_PHRASE && pParse->isNot + if( !sqlite3_fts3_enable_parentheses + && p->eType==FTSQUERY_PHRASE && pParse->isNot ){ /* Create an implicit NOT operator. */ - Fts3Expr *pNot = fts3MallocZero(sizeof(Fts3Expr)); + Fts3Expr *pNot = sqlite3Fts3MallocZero(sizeof(Fts3Expr)); if( !pNot ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; @@ -148027,7 +194901,7 @@ static int fts3ExprParse( /* Insert an implicit AND operator. */ Fts3Expr *pAnd; assert( pRet && pPrev ); - pAnd = fts3MallocZero(sizeof(Fts3Expr)); + pAnd = sqlite3Fts3MallocZero(sizeof(Fts3Expr)); if( !pAnd ){ sqlite3Fts3ExprFree(p); rc = SQLITE_NOMEM; @@ -148111,13 +194985,13 @@ exprparse_out: } /* -** Return SQLITE_ERROR if the maximum depth of the expression tree passed +** Return SQLITE_ERROR if the maximum depth of the expression tree passed ** as the only argument is more than nMaxDepth. */ static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ int rc = SQLITE_OK; if( p ){ - if( nMaxDepth<0 ){ + if( nMaxDepth<0 ){ rc = SQLITE_TOOBIG; }else{ rc = fts3ExprCheckDepth(p->pLeft, nMaxDepth-1); @@ -148132,12 +195006,12 @@ static int fts3ExprCheckDepth(Fts3Expr *p, int nMaxDepth){ /* ** This function attempts to transform the expression tree at (*pp) to ** an equivalent but more balanced form. The tree is modified in place. -** If successful, SQLITE_OK is returned and (*pp) set to point to the -** new root expression node. +** If successful, SQLITE_OK is returned and (*pp) set to point to the +** new root expression node. ** ** nMaxDepth is the maximum allowable depth of the balanced sub-tree. ** -** Otherwise, if an error occurs, an SQLite error code is returned and +** Otherwise, if an error occurs, an SQLite error code is returned and ** expression (*pp) freed. */ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ @@ -148153,7 +195027,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ if( rc==SQLITE_OK ){ if( (eType==FTSQUERY_AND || eType==FTSQUERY_OR) ){ Fts3Expr **apLeaf; - apLeaf = (Fts3Expr **)sqlite3_malloc(sizeof(Fts3Expr *) * nMaxDepth); + apLeaf = (Fts3Expr **)sqlite3_malloc64(sizeof(Fts3Expr *) * nMaxDepth); if( 0==apLeaf ){ rc = SQLITE_NOMEM; }else{ @@ -148252,7 +195126,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ } pRoot = p; }else{ - /* An error occurred. Delete the contents of the apLeaf[] array + /* An error occurred. Delete the contents of the apLeaf[] array ** and pFree list. Everything else is cleaned up by the call to ** sqlite3Fts3ExprFree(pRoot) below. */ Fts3Expr *pDel; @@ -148294,7 +195168,7 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ } } } - + if( rc!=SQLITE_OK ){ sqlite3Fts3ExprFree(pRoot); pRoot = 0; @@ -148308,9 +195182,9 @@ static int fts3ExprBalance(Fts3Expr **pp, int nMaxDepth){ ** differences: ** ** 1. It does not do expression rebalancing. -** 2. It does not check that the expression does not exceed the +** 2. It does not check that the expression does not exceed the ** maximum allowable depth. -** 3. Even if it fails, *ppExpr may still be set to point to an +** 3. Even if it fails, *ppExpr may still be set to point to an ** expression tree. It should be deleted using sqlite3Fts3ExprFree() ** in this case. */ @@ -148349,7 +195223,7 @@ static int fts3ExprParseUnbalanced( if( rc==SQLITE_OK && sParse.nNest ){ rc = SQLITE_ERROR; } - + return rc; } @@ -148368,7 +195242,7 @@ static int fts3ExprParseUnbalanced( ** The first parameter, pTokenizer, is passed the fts3 tokenizer module to ** use to normalize query tokens while parsing the expression. The azCol[] ** array, which is assumed to contain nCol entries, should contain the names -** of each column in the target fts3 table, in order from left to right. +** of each column in the target fts3 table, in order from left to right. ** Column names must be nul-terminated strings. ** ** The iDefaultCol parameter should be passed the index of the table column @@ -148391,7 +195265,7 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( int rc = fts3ExprParseUnbalanced( pTokenizer, iLangid, azCol, bFts4, nCol, iDefaultCol, z, n, ppExpr ); - + /* Rebalance the expression. And check that its depth does not exceed ** SQLITE_FTS3_MAX_EXPR_DEPTH. */ if( rc==SQLITE_OK && *ppExpr ){ @@ -148406,7 +195280,7 @@ SQLITE_PRIVATE int sqlite3Fts3ExprParse( *ppExpr = 0; if( rc==SQLITE_TOOBIG ){ sqlite3Fts3ErrMsg(pzErr, - "FTS expression tree is too large (maximum depth %d)", + "FTS expression tree is too large (maximum depth %d)", SQLITE_FTS3_MAX_EXPR_DEPTH ); rc = SQLITE_ERROR; @@ -148465,42 +195339,14 @@ SQLITE_PRIVATE void sqlite3Fts3ExprFree(Fts3Expr *pDel){ /* #include */ -/* -** Function to query the hash-table of tokenizers (see README.tokenizers). -*/ -static int queryTestTokenizer( - sqlite3 *db, - const char *zName, - const sqlite3_tokenizer_module **pp -){ - int rc; - sqlite3_stmt *pStmt; - const char zSql[] = "SELECT fts3_tokenizer(?)"; - - *pp = 0; - rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0); - if( rc!=SQLITE_OK ){ - return rc; - } - - sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); - if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ - memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); - } - } - - return sqlite3_finalize(pStmt); -} - /* ** Return a pointer to a buffer containing a text representation of the ** expression passed as the first argument. The buffer is obtained from -** sqlite3_malloc(). It is the responsibility of the caller to use +** sqlite3_malloc(). It is the responsibility of the caller to use ** sqlite3_free() to release the memory. If an OOM condition is encountered, ** NULL is returned. ** -** If the second argument is not NULL, then its contents are prepended to +** If the second argument is not NULL, then its contents are prepended to ** the returned expression text and then freed using sqlite3_free(). */ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ @@ -148514,7 +195360,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ zBuf = sqlite3_mprintf( "%zPHRASE %d 0", zBuf, pPhrase->iColumn); for(i=0; zBuf && inToken; i++){ - zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, + zBuf = sqlite3_mprintf("%z %.*s%s", zBuf, pPhrase->aToken[i].n, pPhrase->aToken[i].z, (pPhrase->aToken[i].isPrefix?"+":"") ); @@ -148547,7 +195393,7 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ } /* -** This is the implementation of a scalar SQL function used to test the +** This is the implementation of a scalar SQL function used to test the ** expression parser. It should be called as follows: ** ** fts3_exprtest(, , , ...); @@ -148560,12 +195406,12 @@ static char *exprToString(Fts3Expr *pExpr, char *zBuf){ ** ** SELECT fts3_exprtest('simple', 'Bill col2:Bloggs', 'col1', 'col2'); */ -static void fts3ExprTest( +static void fts3ExprTestCommon( + int bRebalance, sqlite3_context *context, int argc, sqlite3_value **argv ){ - sqlite3_tokenizer_module const *pModule = 0; sqlite3_tokenizer *pTokenizer = 0; int rc; char **azCol = 0; @@ -148575,37 +195421,33 @@ static void fts3ExprTest( int ii; Fts3Expr *pExpr; char *zBuf = 0; - sqlite3 *db = sqlite3_context_db_handle(context); + Fts3Hash *pHash = (Fts3Hash*)sqlite3_user_data(context); + const char *zTokenizer = 0; + char *zErr = 0; if( argc<3 ){ - sqlite3_result_error(context, + sqlite3_result_error(context, "Usage: fts3_exprtest(tokenizer, expr, col1, ...", -1 ); return; } - rc = queryTestTokenizer(db, - (const char *)sqlite3_value_text(argv[0]), &pModule); - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - }else if( !pModule ){ - sqlite3_result_error(context, "No such tokenizer module", -1); - goto exprtest_out; + zTokenizer = (const char*)sqlite3_value_text(argv[0]); + rc = sqlite3Fts3InitTokenizer(pHash, zTokenizer, &pTokenizer, &zErr); + if( rc!=SQLITE_OK ){ + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_error(context, zErr, -1); + } + sqlite3_free(zErr); + return; } - rc = pModule->xCreate(0, 0, &pTokenizer); - assert( rc==SQLITE_NOMEM || rc==SQLITE_OK ); - if( rc==SQLITE_NOMEM ){ - sqlite3_result_error_nomem(context); - goto exprtest_out; - } - pTokenizer->pModule = pModule; - zExpr = (const char *)sqlite3_value_text(argv[1]); nExpr = sqlite3_value_bytes(argv[1]); nCol = argc-2; - azCol = (char **)sqlite3_malloc(nCol*sizeof(char *)); + azCol = (char **)sqlite3_malloc64(nCol*sizeof(char *)); if( !azCol ){ sqlite3_result_error_nomem(context); goto exprtest_out; @@ -148614,7 +195456,7 @@ static void fts3ExprTest( azCol[ii] = (char *)sqlite3_value_text(argv[ii+2]); } - if( sqlite3_user_data(context) ){ + if( bRebalance ){ char *zDummy = 0; rc = sqlite3Fts3ExprParse( pTokenizer, 0, azCol, 0, nCol, nCol, zExpr, nExpr, &pExpr, &zDummy @@ -148640,23 +195482,38 @@ static void fts3ExprTest( sqlite3Fts3ExprFree(pExpr); exprtest_out: - if( pModule && pTokenizer ){ - rc = pModule->xDestroy(pTokenizer); + if( pTokenizer ){ + rc = pTokenizer->pModule->xDestroy(pTokenizer); } sqlite3_free(azCol); } +static void fts3ExprTest( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(0, context, argc, argv); +} +static void fts3ExprTestRebalance( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + fts3ExprTestCommon(1, context, argc, argv); +} + /* -** Register the query expression parser test function fts3_exprtest() -** with database connection db. +** Register the query expression parser test function fts3_exprtest() +** with database connection db. */ -SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ +SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3 *db, Fts3Hash *pHash){ int rc = sqlite3_create_function( - db, "fts3_exprtest", -1, SQLITE_UTF8, 0, fts3ExprTest, 0, 0 + db, "fts3_exprtest", -1, SQLITE_UTF8, (void*)pHash, fts3ExprTest, 0, 0 ); if( rc==SQLITE_OK ){ - rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", - -1, SQLITE_UTF8, (void *)1, fts3ExprTest, 0, 0 + rc = sqlite3_create_function(db, "fts3_exprtest_rebalance", + -1, SQLITE_UTF8, (void*)pHash, fts3ExprTestRebalance, 0, 0 ); } return rc; @@ -148704,8 +195561,8 @@ SQLITE_PRIVATE int sqlite3Fts3ExprInitTestInterface(sqlite3* db){ /* ** Malloc and Free functions */ -static void *fts3HashMalloc(int n){ - void *p = sqlite3_malloc(n); +static void *fts3HashMalloc(sqlite3_int64 n){ + void *p = sqlite3_malloc64(n); if( p ){ memset(p, 0, n); } @@ -148719,8 +195576,8 @@ static void fts3HashFree(void *p){ ** fields of the Hash structure. ** ** "pNew" is a pointer to the hash table that is to be initialized. -** keyClass is one of the constants -** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass +** keyClass is one of the constants +** FTS3_HASH_BINARY or FTS3_HASH_STRING. The value of keyClass ** determines what kind of key the hash table will use. "copyKey" is ** true if the hash table should make its own private copy of keys and ** false if it should just use the supplied pointer. @@ -148797,7 +195654,7 @@ static int fts3BinCompare(const void *pKey1, int n1, const void *pKey2, int n2){ /* ** Return a pointer to the appropriate hash function given the key class. ** -** The C syntax in this function definition may be unfamilar to some +** The C syntax in this function definition may be unfamilar to some ** programmers, so we provide the following additional explanation: ** ** The name of the function is "ftsHashFunction". The function takes a @@ -148857,7 +195714,7 @@ static void fts3HashInsertElement( /* Resize the hash table so that it cantains "new_size" buckets. -** "new_size" must be a power of 2. The hash table might fail +** "new_size" must be a power of 2. The hash table might fail ** to resize if sqliteMalloc() fails. ** ** Return non-zero if a memory allocation error occurs. @@ -148902,7 +195759,7 @@ static Fts3HashElem *fts3FindElementByHash( count = pEntry->count; xCompare = ftsCompareFunction(pH->keyClass); while( count-- && elem ){ - if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ + if( (*xCompare)(elem->pKey,elem->nKey,pKey,nKey)==0 ){ return elem; } elem = elem->next; @@ -148921,7 +195778,7 @@ static void fts3RemoveElementByHash( ){ struct _fts3ht *pEntry; if( elem->prev ){ - elem->prev->next = elem->next; + elem->prev->next = elem->next; }else{ pH->first = elem->next; } @@ -148949,8 +195806,8 @@ static void fts3RemoveElementByHash( } SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( - const Fts3Hash *pH, - const void *pKey, + const Fts3Hash *pH, + const void *pKey, int nKey ){ int h; /* A hash on key */ @@ -148964,7 +195821,7 @@ SQLITE_PRIVATE Fts3HashElem *sqlite3Fts3HashFindElem( return fts3FindElementByHash(pH,pKey,nKey, h & (pH->htsize-1)); } -/* +/* ** Attempt to locate an element of the hash table pH with a key ** that matches pKey,nKey. Return the data for this element if it is ** found, or NULL if there is no match. @@ -149138,7 +195995,7 @@ static int porterDestroy(sqlite3_tokenizer *pTokenizer){ /* ** Prepare to begin tokenizing a particular string. The input ** string to be tokenized is zInput[0..nInput-1]. A cursor -** used to incrementally tokenize this string is returned in +** used to incrementally tokenize this string is returned in ** *ppCursor. */ static int porterOpen( @@ -149191,7 +196048,7 @@ static const char cType[] = { /* ** isConsonant() and isVowel() determine if their first character in ** the string they point to is a consonant or a vowel, according -** to Porter ruls. +** to Porter ruls. ** ** A consonate is any letter other than 'a', 'e', 'i', 'o', or 'u'. ** 'Y' is a consonant unless it follows another consonant, @@ -149311,11 +196168,11 @@ static int star_oh(const char *z){ /* ** If the word ends with zFrom and xCond() is true for the stem -** of the word that preceeds the zFrom ending, then change the +** of the word that preceeds the zFrom ending, then change the ** ending to zTo. ** ** The input word *pz and zFrom are both in reverse order. zTo -** is in normal order. +** is in normal order. ** ** Return TRUE if zFrom matches. Return FALSE if zFrom does not ** match. Not that TRUE is returned even if xCond() fails and @@ -149384,9 +196241,9 @@ static void copy_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ ** word contains digits, 3 bytes are taken from the beginning and ** 3 bytes from the end. For long words without digits, 10 bytes ** are taken from each end. US-ASCII case folding still applies. -** -** If the input word contains not digits but does characters not -** in [a-zA-Z] then no stemming is attempted and this routine just +** +** If the input word contains not digits but does characters not +** in [a-zA-Z] then no stemming is attempted and this routine just ** copies the input into the input into the output with US-ASCII ** case folding. ** @@ -149431,11 +196288,11 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ } } - /* Step 1b */ + /* Step 1b */ z2 = z; if( stem(&z, "dee", "ee", m_gt_0) ){ /* Do nothing. The work was all in the test */ - }else if( + }else if( (stem(&z, "gni", "", hasVowel) || stem(&z, "de", "", hasVowel)) && z!=z2 ){ @@ -149474,7 +196331,7 @@ static void porter_stemmer(const char *zIn, int nIn, char *zOut, int *pnOut){ stem(&z, "igol", "log", m_gt_0); break; case 'l': - if( !stem(&z, "ilb", "ble", m_gt_0) + if( !stem(&z, "ilb", "ble", m_gt_0) && !stem(&z, "illa", "al", m_gt_0) && !stem(&z, "iltne", "ent", m_gt_0) && !stem(&z, "ile", "e", m_gt_0) @@ -149676,7 +196533,7 @@ static int porterNext( if( n>c->nAllocated ){ char *pNew; c->nAllocated = n+20; - pNew = sqlite3_realloc(c->zToken, c->nAllocated); + pNew = sqlite3_realloc64(c->zToken, c->nAllocated); if( !pNew ) return SQLITE_NOMEM; c->zToken = pNew; } @@ -149762,7 +196619,7 @@ static int fts3TokenizerEnabled(sqlite3_context *context){ } /* -** Implementation of the SQL scalar function for accessing the underlying +** Implementation of the SQL scalar function for accessing the underlying ** hash table. This function may be called as follows: ** ** SELECT (); @@ -149799,7 +196656,7 @@ static void fts3TokenizerFunc( nName = sqlite3_value_bytes(argv[0])+1; if( argc==2 ){ - if( fts3TokenizerEnabled(context) ){ + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[1]) ){ void *pOld; int n = sqlite3_value_bytes(argv[1]); if( zName==0 || n!=sizeof(pPtr) ){ @@ -149826,7 +196683,9 @@ static void fts3TokenizerFunc( return; } } - sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + if( fts3TokenizerEnabled(context) || sqlite3_value_frombind(argv[0]) ){ + sqlite3_result_blob(context, (void *)&pPtr, sizeof(pPtr), SQLITE_TRANSIENT); + } } SQLITE_PRIVATE int sqlite3Fts3IsIdChar(char c){ @@ -149914,8 +196773,8 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( int iArg = 0; z = &z[n+1]; while( zpModule = m; + (*ppTok)->pModule = m; } sqlite3_free((void *)aArg); } @@ -149944,11 +196803,11 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( #ifdef SQLITE_TEST -#include +#include "tclsqlite.h" /* #include */ /* -** Implementation of a special SQL scalar function for testing tokenizers +** Implementation of a special SQL scalar function for testing tokenizers ** designed to be used in concert with the Tcl testing framework. This ** function must be called with two or more arguments: ** @@ -149960,9 +196819,9 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( ** ** The return value is a string that may be interpreted as a Tcl ** list. For each token in the , three elements are -** added to the returned list. The first is the token position, the +** added to the returned list. The first is the token position, the ** second is the token text (folded, stemmed, etc.) and the third is the -** substring of associated with the token. For example, +** substring of associated with the token. For example, ** using the built-in "simple" tokenizer: ** ** SELECT fts_tokenizer_test('simple', 'I don't see how'); @@ -149970,7 +196829,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTokenizer( ** will return the string: ** ** "{0 i I 1 dont don't 2 see see 3 how how}" -** +** */ static void testFunc( sqlite3_context *context, @@ -150065,8 +196924,8 @@ finish: static int registerTokenizer( - sqlite3 *db, - char *zName, + sqlite3 *db, + char *zName, const sqlite3_tokenizer_module *p ){ int rc; @@ -150088,8 +196947,8 @@ int registerTokenizer( static int queryTokenizer( - sqlite3 *db, - char *zName, + sqlite3 *db, + char *zName, const sqlite3_tokenizer_module **pp ){ int rc; @@ -150104,7 +196963,9 @@ int queryTokenizer( sqlite3_bind_text(pStmt, 1, zName, -1, SQLITE_STATIC); if( SQLITE_ROW==sqlite3_step(pStmt) ){ - if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB ){ + if( sqlite3_column_type(pStmt, 0)==SQLITE_BLOB + && sqlite3_column_bytes(pStmt, 0)==sizeof(*pp) + ){ memcpy((void *)pp, sqlite3_column_blob(pStmt, 0), sizeof(*pp)); } } @@ -150172,28 +197033,28 @@ static void intTestFunc( /* ** Set up SQL objects in database db used to access the contents of ** the hash table pointed to by argument pHash. The hash table must -** been initialized to use string keys, and to take a private copy +** been initialized to use string keys, and to take a private copy ** of the key when a value is inserted. i.e. by a call similar to: ** ** sqlite3Fts3HashInit(pHash, FTS3_HASH_STRING, 1); ** ** This function adds a scalar function (see header comment above ** fts3TokenizerFunc() in this file for details) and, if ENABLE_TABLE is -** defined at compilation time, a temporary virtual table (see header -** comment above struct HashTableVtab) to the database schema. Both +** defined at compilation time, a temporary virtual table (see header +** comment above struct HashTableVtab) to the database schema. Both ** provide read/write access to the contents of *pHash. ** ** The third argument to this function, zName, is used as the name ** of both the scalar and, if created, the virtual table. */ SQLITE_PRIVATE int sqlite3Fts3InitHashTable( - sqlite3 *db, - Fts3Hash *pHash, + sqlite3 *db, + Fts3Hash *pHash, const char *zName ){ int rc = SQLITE_OK; void *p = (void *)pHash; - const int any = SQLITE_ANY; + const int any = SQLITE_UTF8|SQLITE_DIRECTONLY; #ifdef SQLITE_TEST char *zTest = 0; @@ -150342,7 +197203,7 @@ static int simpleDestroy(sqlite3_tokenizer *pTokenizer){ /* ** Prepare to begin tokenizing a particular string. The input ** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in +** used to incrementally tokenize this string is returned in ** *ppCursor. */ static int simpleOpen( @@ -150420,7 +197281,7 @@ static int simpleNext( if( n>c->nTokenAllocated ){ char *pNew; c->nTokenAllocated = n+20; - pNew = sqlite3_realloc(c->pToken, c->nTokenAllocated); + pNew = sqlite3_realloc64(c->pToken, c->nTokenAllocated); if( !pNew ) return SQLITE_NOMEM; c->pToken = pNew; } @@ -150497,8 +197358,8 @@ SQLITE_PRIVATE void sqlite3Fts3SimpleTokenizerModule( ** ** input = ** -** The virtual table module tokenizes this , using the FTS3 -** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE +** The virtual table module tokenizes this , using the FTS3 +** tokenizer specified by the arguments to the CREATE VIRTUAL TABLE ** statement and returns one row for each token in the result. With ** fields set as follows: ** @@ -150567,7 +197428,7 @@ static int fts3tokQueryTokenizer( /* ** The second argument, argv[], is an array of pointers to nul-terminated -** strings. This function makes a copy of the array and strings into a +** strings. This function makes a copy of the array and strings into a ** single block of memory. It then dequotes any of the strings that appear ** to be quoted. ** @@ -150594,7 +197455,7 @@ static int fts3tokDequoteArray( nByte += (int)(strlen(argv[i]) + 1); } - *pazDequote = azDequote = sqlite3_malloc(sizeof(char *)*argc + nByte); + *pazDequote = azDequote = sqlite3_malloc64(sizeof(char *)*argc + nByte); if( azDequote==0 ){ rc = SQLITE_NOMEM; }else{ @@ -150623,7 +197484,7 @@ static int fts3tokDequoteArray( ** and xCreate are identical operations. ** ** argv[0]: module name -** argv[1]: database name +** argv[1]: database name ** argv[2]: table name ** argv[3]: first argument (tokenizer name) */ @@ -150660,7 +197521,8 @@ static int fts3tokConnectMethod( assert( (rc==SQLITE_OK)==(pMod!=0) ); if( rc==SQLITE_OK ){ - const char * const *azArg = (const char * const *)&azDequote[1]; + const char * const *azArg = 0; + if( nDequote>1 ) azArg = (const char * const *)&azDequote[1]; rc = pMod->xCreate((nDequote>1 ? nDequote-1 : 0), azArg, &pTok); } @@ -150703,16 +197565,16 @@ static int fts3tokDisconnectMethod(sqlite3_vtab *pVtab){ ** xBestIndex - Analyze a WHERE and ORDER BY clause. */ static int fts3tokBestIndexMethod( - sqlite3_vtab *pVTab, + sqlite3_vtab *pVTab, sqlite3_index_info *pInfo ){ int i; UNUSED_PARAMETER(pVTab); for(i=0; inConstraint; i++){ - if( pInfo->aConstraint[i].usable - && pInfo->aConstraint[i].iColumn==0 - && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ + if( pInfo->aConstraint[i].usable + && pInfo->aConstraint[i].iColumn==0 + && pInfo->aConstraint[i].op==SQLITE_INDEX_CONSTRAINT_EQ ){ pInfo->idxNum = 1; pInfo->aConstraintUsage[i].argvIndex = 1; @@ -150818,11 +197680,11 @@ static int fts3tokFilterMethod( if( idxNum==1 ){ const char *zByte = (const char *)sqlite3_value_text(apVal[0]); int nByte = sqlite3_value_bytes(apVal[0]); - pCsr->zInput = sqlite3_malloc(nByte+1); + pCsr->zInput = sqlite3_malloc64(nByte+1); if( pCsr->zInput==0 ){ rc = SQLITE_NOMEM; }else{ - memcpy(pCsr->zInput, zByte, nByte); + if( nByte>0 ) memcpy(pCsr->zInput, zByte, nByte); pCsr->zInput[nByte] = 0; rc = pTab->pMod->xOpen(pTab->pTok, pCsr->zInput, nByte, &pCsr->pCsr); if( rc==SQLITE_OK ){ @@ -150891,7 +197753,7 @@ static int fts3tokRowidMethod( ** Register the fts3tok module with database connection db. Return SQLITE_OK ** if successful or an error code if sqlite3_create_module() fails. */ -SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ +SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash, void(*xDestroy)(void*)){ static const sqlite3_module fts3tok_module = { 0, /* iVersion */ fts3tokConnectMethod, /* xCreate */ @@ -150915,11 +197777,15 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ 0, /* xRename */ 0, /* xSavepoint */ 0, /* xRelease */ - 0 /* xRollbackTo */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; int rc; /* Return code */ - rc = sqlite3_create_module(db, "fts3tokenize", &fts3tok_module, (void*)pHash); + rc = sqlite3_create_module_v2( + db, "fts3tokenize", &fts3tok_module, (void*)pHash, xDestroy + ); return rc; } @@ -150942,7 +197808,7 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ ** This file is part of the SQLite FTS3 extension module. Specifically, ** this file contains code to insert, update and delete rows from FTS3 ** tables. It also contains code to merge FTS3 b-tree segments. Some -** of the sub-routines used to merge segments are also used by the query +** of the sub-routines used to merge segments are also used by the query ** code in fts3.c. */ @@ -150952,13 +197818,13 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ /* #include */ /* #include */ /* #include */ - +/* #include */ #define FTS_MAX_APPENDABLE_HEIGHT 16 /* ** When full-text index nodes are loaded from disk, the buffer that they -** are loaded into has the following number of bytes of padding at the end +** are loaded into has the following number of bytes of padding at the end ** of it. i.e. if a full-text index node is 900 bytes in size, then a buffer ** of 920 bytes is allocated for it. ** @@ -150975,10 +197841,10 @@ SQLITE_PRIVATE int sqlite3Fts3InitTok(sqlite3 *db, Fts3Hash *pHash){ ** method before retrieving all query results (as may happen, for example, ** if a query has a LIMIT clause). ** -** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD +** Incremental loading is used for b-tree nodes FTS3_NODE_CHUNK_THRESHOLD ** bytes and larger. Nodes are loaded in chunks of FTS3_NODE_CHUNKSIZE bytes. -** The code is written so that the hard lower-limit for each of these values -** is 1. Clearly such small values would be inefficient, but can be useful +** The code is written so that the hard lower-limit for each of these values +** is 1. Clearly such small values would be inefficient, but can be useful ** for testing purposes. ** ** If this module is built with SQLITE_TEST defined, these constants may @@ -150991,12 +197857,12 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; # define FTS3_NODE_CHUNKSIZE test_fts3_node_chunksize # define FTS3_NODE_CHUNK_THRESHOLD test_fts3_node_chunk_threshold #else -# define FTS3_NODE_CHUNKSIZE (4*1024) +# define FTS3_NODE_CHUNKSIZE (4*1024) # define FTS3_NODE_CHUNK_THRESHOLD (FTS3_NODE_CHUNKSIZE*4) #endif /* -** The two values that may be meaningfully bound to the :1 parameter in +** The values that may be meaningfully bound to the :1 parameter in ** statements SQL_REPLACE_STAT and SQL_SELECT_STAT. */ #define FTS_STAT_DOCTOTAL 0 @@ -151005,7 +197871,7 @@ int test_fts3_node_chunk_threshold = (4*1024)*4; /* ** If FTS_LOG_MERGES is defined, call sqlite3_log() to report each automatic -** and incremental merge operation that takes place. This is used for +** and incremental merge operation that takes place. This is used for ** debugging FTS only, it should not usually be turned on in production ** systems. */ @@ -151091,7 +197957,7 @@ struct Fts3SegReader { char *aDoclist; /* Pointer to doclist of current entry */ int nDoclist; /* Size of doclist in current entry */ - /* The following variables are used by fts3SegReaderNextDocid() to iterate + /* The following variables are used by fts3SegReaderNextDocid() to iterate ** through the current doclist (aDoclist/nDoclist). */ char *pOffsetList; @@ -151136,11 +198002,11 @@ struct SegmentWriter { ** fts3NodeFree() ** ** When a b+tree is written to the database (either as a result of a merge -** or the pending-terms table being flushed), leaves are written into the +** or the pending-terms table being flushed), leaves are written into the ** database file as soon as they are completely populated. The interior of ** the tree is assembled in memory and written out only once all leaves have ** been populated and stored. This is Ok, as the b+-tree fanout is usually -** very large, meaning that the interior of the tree consumes relatively +** very large, meaning that the interior of the tree consumes relatively ** little memory. */ struct SegmentNode { @@ -151161,7 +198027,7 @@ struct SegmentNode { */ #define SQL_DELETE_CONTENT 0 #define SQL_IS_EMPTY 1 -#define SQL_DELETE_ALL_CONTENT 2 +#define SQL_DELETE_ALL_CONTENT 2 #define SQL_DELETE_ALL_SEGMENTS 3 #define SQL_DELETE_ALL_SEGDIR 4 #define SQL_DELETE_ALL_DOCSIZE 5 @@ -151209,7 +198075,7 @@ struct SegmentNode { ** Otherwise, an SQLite error code is returned and *pp is set to 0. ** ** If argument apVal is not NULL, then it must point to an array with -** at least as many entries as the requested statement has bound +** at least as many entries as the requested statement has bound ** parameters. The values are bound to the statements parameters before ** returning. */ @@ -151233,7 +198099,7 @@ static int fts3SqlStmt( /* 10 */ "SELECT coalesce((SELECT max(blockid) FROM %Q.'%q_segments') + 1, 1)", /* 11 */ "REPLACE INTO %Q.'%q_segdir' VALUES(?,?,?,?,?,?)", - /* Return segments in order from oldest to newest.*/ + /* Return segments in order from oldest to newest.*/ /* 12 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? ORDER BY idx ASC", /* 13 */ "SELECT idx, start_block, leaves_end_block, end_block, root " @@ -151264,13 +198130,15 @@ static int fts3SqlStmt( ** returns zero rows. */ /* 28 */ "SELECT level, count(*) AS cnt FROM %Q.'%q_segdir' " " GROUP BY level HAVING cnt>=?" - " ORDER BY (level %% 1024) ASC LIMIT 1", + " ORDER BY (level %% 1024) ASC, 2 DESC LIMIT 1", /* Estimate the upper limit on the number of leaf nodes in a new segment -** created by merging the oldest :2 segments from absolute level :1. See +** created by merging the oldest :2 segments from absolute level :1. See ** function sqlite3Fts3Incrmerge() for details. */ /* 29 */ "SELECT 2 * total(1 + leaves_end_block - start_block) " - " FROM %Q.'%q_segdir' WHERE level = ? AND idx < ?", + " FROM (SELECT * FROM %Q.'%q_segdir' " + " WHERE level = ? ORDER BY idx ASC LIMIT ?" + " )", /* SQL_DELETE_SEGDIR_ENTRY ** Delete the %_segdir entry on absolute level :1 with index :2. */ @@ -151282,7 +198150,7 @@ static int fts3SqlStmt( /* 31 */ "UPDATE %Q.'%q_segdir' SET idx = ? WHERE level=? AND idx=?", /* SQL_SELECT_SEGDIR -** Read a single entry from the %_segdir table. The entry from absolute +** Read a single entry from the %_segdir table. The entry from absolute ** level :1 with index value :2. */ /* 32 */ "SELECT idx, start_block, leaves_end_block, end_block, root " "FROM %Q.'%q_segdir' WHERE level = ? AND idx = ?", @@ -151306,7 +198174,7 @@ static int fts3SqlStmt( ** Return the largest relative level in the FTS index or indexes. */ /* 36 */ "SELECT max( level %% 1024 ) FROM %Q.'%q_segdir'", - /* Return segments in order from oldest to newest.*/ + /* Return segments in order from oldest to newest.*/ /* 37 */ "SELECT level, idx, end_block " "FROM %Q.'%q_segdir' WHERE level BETWEEN ? AND ? " "ORDER BY level DESC, idx ASC", @@ -151322,13 +198190,15 @@ static int fts3SqlStmt( assert( SizeofArray(azSql)==SizeofArray(p->aStmt) ); assert( eStmt=0 ); - + pStmt = p->aStmt[eStmt]; if( !pStmt ){ + int f = SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB; char *zSql; if( eStmt==SQL_CONTENT_INSERT ){ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName, p->zWriteExprlist); }else if( eStmt==SQL_SELECT_CONTENT_BY_ROWID ){ + f &= ~SQLITE_PREPARE_NO_VTAB; zSql = sqlite3_mprintf(azSql[eStmt], p->zReadExprlist); }else{ zSql = sqlite3_mprintf(azSql[eStmt], p->zDb, p->zName); @@ -151336,7 +198206,7 @@ static int fts3SqlStmt( if( !zSql ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(p->db, zSql, -1, &pStmt, NULL); + rc = sqlite3_prepare_v3(p->db, zSql, -1, f, &pStmt, NULL); sqlite3_free(zSql); assert( rc==SQLITE_OK || pStmt==0 ); p->aStmt[eStmt] = pStmt; @@ -151425,7 +198295,7 @@ static void fts3SqlExec( sqlite3_stmt *pStmt; int rc; if( *pRC ) return; - rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); + rc = fts3SqlStmt(p, eStmt, &pStmt, apVal); if( rc==SQLITE_OK ){ sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); @@ -151435,22 +198305,22 @@ static void fts3SqlExec( /* -** This function ensures that the caller has obtained an exclusive -** shared-cache table-lock on the %_segdir table. This is required before +** This function ensures that the caller has obtained an exclusive +** shared-cache table-lock on the %_segdir table. This is required before ** writing data to the fts3 table. If this lock is not acquired first, then ** the caller may end up attempting to take this lock as part of committing -** a transaction, causing SQLite to return SQLITE_LOCKED or +** a transaction, causing SQLite to return SQLITE_LOCKED or ** LOCKED_SHAREDCACHEto a COMMIT command. ** -** It is best to avoid this because if FTS3 returns any error when -** committing a transaction, the whole transaction will be rolled back. -** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. -** It can still happen if the user locks the underlying tables directly +** It is best to avoid this because if FTS3 returns any error when +** committing a transaction, the whole transaction will be rolled back. +** And this is not what users expect when they get SQLITE_LOCKED_SHAREDCACHE. +** It can still happen if the user locks the underlying tables directly ** instead of accessing them via FTS. */ static int fts3Writelock(Fts3Table *p){ int rc = SQLITE_OK; - + if( p->nPendingData==0 ){ sqlite3_stmt *pStmt; rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_LEVEL, &pStmt, 0); @@ -151467,7 +198337,7 @@ static int fts3Writelock(Fts3Table *p){ /* ** FTS maintains a separate indexes for each language-id (a 32-bit integer). ** Within each language id, a separate index is maintained to store the -** document terms, and each configured prefix size (configured the FTS +** document terms, and each configured prefix size (configured the FTS ** "prefix=" option). And each index consists of multiple levels ("relative ** levels"). ** @@ -151477,14 +198347,14 @@ static int fts3Writelock(Fts3Table *p){ ** separate component values into the single 64-bit integer value that ** can be used to query the %_segdir table. ** -** Specifically, each language-id/index combination is allocated 1024 +** Specifically, each language-id/index combination is allocated 1024 ** 64-bit integer level values ("absolute levels"). The main terms index ** for language-id 0 is allocate values 0-1023. The first prefix index ** (if any) for language-id 0 is allocated values 1024-2047. And so on. ** Language 1 indexes are allocated immediately following language 0. ** ** So, for a system with nPrefix prefix indexes configured, the block of -** absolute levels that corresponds to language-id iLangid and index +** absolute levels that corresponds to language-id iLangid and index ** iIndex starts at absolute level ((iLangid * (nPrefix+1) + iIndex) * 1024). */ static sqlite3_int64 getAbsoluteLevel( @@ -151494,7 +198364,7 @@ static sqlite3_int64 getAbsoluteLevel( int iLevel /* Level of segments */ ){ sqlite3_int64 iBase; /* First absolute level for iLangid/iIndex */ - assert( iLangid>=0 ); + assert_fts3_nc( iLangid>=0 ); assert( p->nIndex>0 ); assert( iIndex>=0 && iIndexnIndex ); @@ -151505,7 +198375,7 @@ static sqlite3_int64 getAbsoluteLevel( /* ** Set *ppStmt to a statement handle that may be used to iterate through ** all rows in the %_segdir table, from oldest to newest. If successful, -** return SQLITE_OK. If an error occurs while preparing the statement, +** return SQLITE_OK. If an error occurs while preparing the statement, ** return an SQLite error code. ** ** There is only ever one instance of this SQL statement compiled for @@ -151536,16 +198406,16 @@ SQLITE_PRIVATE int sqlite3Fts3AllSegdirs( if( iLevel<0 ){ /* "SELECT * FROM %_segdir WHERE level BETWEEN ? AND ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL_RANGE, &pStmt, 0); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pStmt, 2, + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } }else{ /* "SELECT * FROM %_segdir WHERE level = ? ORDER BY ..." */ rc = fts3SqlStmt(p, SQL_SELECT_LEVEL, &pStmt, 0); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex,iLevel)); } } @@ -151574,7 +198444,7 @@ static int fts3PendingListAppendVarint( /* Allocate or grow the PendingList as required. */ if( !p ){ - p = sqlite3_malloc(sizeof(*p) + 100); + p = sqlite3_malloc64(sizeof(*p) + 100); if( !p ){ return SQLITE_NOMEM; } @@ -151583,14 +198453,14 @@ static int fts3PendingListAppendVarint( p->nData = 0; } else if( p->nData+FTS3_VARINT_MAX+1>p->nSpace ){ - int nNew = p->nSpace * 2; - p = sqlite3_realloc(p, sizeof(*p) + nNew); + i64 nNew = p->nSpace * 2; + p = sqlite3_realloc64(p, sizeof(*p) + nNew); if( !p ){ sqlite3_free(*pp); *pp = 0; return SQLITE_NOMEM; } - p->nSpace = nNew; + p->nSpace = (int)nNew; p->aData = (char *)&p[1]; } @@ -151623,7 +198493,7 @@ static int fts3PendingListAppend( assert( !p || p->iLastDocid<=iDocid ); if( !p || p->iLastDocid!=iDocid ){ - sqlite3_int64 iDelta = iDocid - (p ? p->iLastDocid : 0); + u64 iDelta = (u64)iDocid - (u64)(p ? p->iLastDocid : 0); if( p ){ assert( p->nDatanSpace ); assert( p->aData[p->nData]==0 ); @@ -151689,7 +198559,7 @@ static int fts3PendingTermsAddOne( } if( fts3PendingListAppend(&pList, p->iPrevDocid, iCol, iPos, &rc) ){ if( pList==fts3HashInsert(pHash, zToken, nToken, pList) ){ - /* Malloc failed while inserting the new entry. This can only + /* Malloc failed while inserting the new entry. This can only ** happen if there was no previous entry for this token. */ assert( 0==fts3HashFind(pHash, zToken, nToken) ); @@ -151735,7 +198605,7 @@ static int fts3PendingTermsAdd( assert( pTokenizer && pModule ); /* If the user has inserted a NULL value, this function may be called with - ** zText==0. In this case, add zero token entries to the hash table and + ** zText==0. In this case, add zero token entries to the hash table and ** return early. */ if( zText==0 ){ *pnWord = 0; @@ -151766,8 +198636,8 @@ static int fts3PendingTermsAdd( rc = fts3PendingTermsAddOne( p, iCol, iPos, &p->aIndex[0].hPending, zToken, nToken ); - - /* Add the term to each of the prefix indexes that it is not too + + /* Add the term to each of the prefix indexes that it is not too ** short for. */ for(i=1; rc==SQLITE_OK && inIndex; i++){ struct Fts3Index *pIndex = &p->aIndex[i]; @@ -151783,8 +198653,8 @@ static int fts3PendingTermsAdd( return (rc==SQLITE_DONE ? SQLITE_OK : rc); } -/* -** Calling this function indicates that subsequent calls to +/* +** Calling this function indicates that subsequent calls to ** fts3PendingTermsAdd() are to add term/position-list pairs for the ** contents of the document with docid iDocid. */ @@ -151803,10 +198673,10 @@ static int fts3PendingTermsDocid( ** buffer was half empty, that would let the less frequent terms ** generate longer doclists. */ - if( iDocidiPrevDocid + if( iDocidiPrevDocid || (iDocid==p->iPrevDocid && p->bPrevDelete==0) || p->iPrevLangid!=iLangid - || p->nPendingData>p->nMaxPendingData + || p->nPendingData>p->nMaxPendingData ){ int rc = sqlite3Fts3PendingTermsFlush(p); if( rc!=SQLITE_OK ) return rc; @@ -151818,7 +198688,7 @@ static int fts3PendingTermsDocid( } /* -** Discard the contents of the pending-terms hash tables. +** Discard the contents of the pending-terms hash tables. */ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ int i; @@ -151843,9 +198713,9 @@ SQLITE_PRIVATE void sqlite3Fts3PendingTermsClear(Fts3Table *p){ ** fts3InsertData(). Parameter iDocid is the docid of the new row. */ static int fts3InsertTerms( - Fts3Table *p, - int iLangid, - sqlite3_value **apVal, + Fts3Table *p, + int iLangid, + sqlite3_value **apVal, u32 *aSz ){ int i; /* Iterator variable */ @@ -151908,7 +198778,7 @@ static int fts3InsertData( rc = fts3SqlStmt(p, SQL_CONTENT_INSERT, &pContentInsert, &apVal[1]); if( rc==SQLITE_OK && p->zLanguageid ){ rc = sqlite3_bind_int( - pContentInsert, p->nColumn+2, + pContentInsert, p->nColumn+2, sqlite3_value_int(apVal[p->nColumn+4]) ); } @@ -151935,8 +198805,8 @@ static int fts3InsertData( if( rc!=SQLITE_OK ) return rc; } - /* Execute the statement to insert the record. Set *piDocid to the - ** new docid value. + /* Execute the statement to insert the record. Set *piDocid to the + ** new docid value. */ sqlite3_step(pContentInsert); rc = sqlite3_reset(pContentInsert); @@ -151986,7 +198856,7 @@ static int langidFromSelect(Fts3Table *p, sqlite3_stmt *pSelect){ ** (an integer) of a row about to be deleted. Remove all terms from the ** full-text index. */ -static void fts3DeleteTerms( +static void fts3DeleteTerms( int *pRC, /* Result code */ Fts3Table *p, /* The FTS table to delete from */ sqlite3_value *pRowid, /* The docid to be deleted */ @@ -152033,7 +198903,7 @@ static void fts3DeleteTerms( */ static int fts3SegmentMerge(Fts3Table *, int, int, int); -/* +/* ** This function allocates a new level iLevel index in the segdir table. ** Usually, indexes are allocated within a level sequentially starting ** with 0, so the allocated index is one greater than the value returned @@ -152042,17 +198912,17 @@ static int fts3SegmentMerge(Fts3Table *, int, int, int); ** SELECT max(idx) FROM %_segdir WHERE level = :iLevel ** ** However, if there are already FTS3_MERGE_COUNT indexes at the requested -** level, they are merged into a single level (iLevel+1) segment and the +** level, they are merged into a single level (iLevel+1) segment and the ** allocated index is 0. ** ** If successful, *piIdx is set to the allocated index slot and SQLITE_OK ** returned. Otherwise, an SQLite error code is returned. */ static int fts3AllocateSegdirIdx( - Fts3Table *p, + Fts3Table *p, int iLangid, /* Language id */ int iIndex, /* Index for p->aIndex */ - int iLevel, + int iLevel, int *piIdx ){ int rc; /* Return Code */ @@ -152080,7 +198950,7 @@ static int fts3AllocateSegdirIdx( ** segment and allocate (newly freed) index 0 at level iLevel. Otherwise, ** if iNext is less than FTS3_MERGE_COUNT, allocate index iNext. */ - if( iNext>=FTS3_MERGE_COUNT ){ + if( iNext>=MergeCount(p) ){ fts3LogMerge(16, getAbsoluteLevel(p, iLangid, iIndex, iLevel)); rc = fts3SegmentMerge(p, iLangid, iIndex, iLevel); *piIdx = 0; @@ -152100,7 +198970,7 @@ static int fts3AllocateSegdirIdx( ** This function reads data from a single row of the %_segments table. The ** specific row is identified by the iBlockid parameter. If paBlob is not ** NULL, then a buffer is allocated using sqlite3_malloc() and populated -** with the contents of the blob stored in the "block" column of the +** with the contents of the blob stored in the "block" column of the ** identified table row is. Whether or not paBlob is NULL, *pnBlob is set ** to the size of the blob in bytes before returning. ** @@ -152147,7 +199017,7 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( int nByte = sqlite3_blob_bytes(p->pSegments); *pnBlob = nByte; if( paBlob ){ - char *aByte = sqlite3_malloc(nByte + FTS3_NODE_PADDING); + char *aByte = sqlite3_malloc64((i64)nByte + FTS3_NODE_PADDING); if( !aByte ){ rc = SQLITE_NOMEM; }else{ @@ -152164,6 +199034,8 @@ SQLITE_PRIVATE int sqlite3Fts3ReadBlock( } *paBlob = aByte; } + }else if( rc==SQLITE_ERROR ){ + rc = FTS_CORRUPT_VTAB; } return rc; @@ -152177,14 +199049,14 @@ SQLITE_PRIVATE void sqlite3Fts3SegmentsClose(Fts3Table *p){ sqlite3_blob_close(p->pSegments); p->pSegments = 0; } - + static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ int nRead; /* Number of bytes to read */ int rc; /* Return code */ nRead = MIN(pReader->nNode - pReader->nPopulate, FTS3_NODE_CHUNKSIZE); rc = sqlite3_blob_read( - pReader->pBlob, + pReader->pBlob, &pReader->aNode[pReader->nPopulate], nRead, pReader->nPopulate @@ -152204,10 +199076,10 @@ static int fts3SegReaderIncrRead(Fts3SegReader *pReader){ static int fts3SegReaderRequire(Fts3SegReader *pReader, char *pFrom, int nByte){ int rc = SQLITE_OK; - assert( !pReader->pBlob + assert( !pReader->pBlob || (pFrom>=pReader->aNode && pFrom<&pReader->aNode[pReader->nNode]) ); - while( pReader->pBlob && rc==SQLITE_OK + while( pReader->pBlob && rc==SQLITE_OK && (pFrom - pReader->aNode + nByte)>pReader->nPopulate ){ rc = fts3SegReaderIncrRead(pReader); @@ -152233,7 +199105,7 @@ static void fts3SegReaderSetEof(Fts3SegReader *pSeg){ ** SQLITE_DONE. Otherwise, an SQLite error code. */ static int fts3SegReaderNext( - Fts3Table *p, + Fts3Table *p, Fts3SegReader *pReader, int bIncr ){ @@ -152258,9 +199130,19 @@ static int fts3SegReaderNext( char *aCopy; PendingList *pList = (PendingList *)fts3HashData(pElem); int nCopy = pList->nData+1; - pReader->zTerm = (char *)fts3HashKey(pElem); - pReader->nTerm = fts3HashKeysize(pElem); - aCopy = (char*)sqlite3_malloc(nCopy); + + int nTerm = fts3HashKeysize(pElem); + if( (nTerm+1)>pReader->nTermAlloc ){ + sqlite3_free(pReader->zTerm); + pReader->zTerm = (char*)sqlite3_malloc64(((i64)nTerm+1)*2); + if( !pReader->zTerm ) return SQLITE_NOMEM; + pReader->nTermAlloc = (nTerm+1)*2; + } + memcpy(pReader->zTerm, fts3HashKey(pElem), nTerm); + pReader->zTerm[nTerm] = '\0'; + pReader->nTerm = nTerm; + + aCopy = (char*)sqlite3_malloc64(nCopy); if( !aCopy ) return SQLITE_NOMEM; memcpy(aCopy, pList->aData, nCopy); pReader->nNode = pReader->nDoclist = nCopy; @@ -152273,15 +199155,17 @@ static int fts3SegReaderNext( fts3SegReaderSetEof(pReader); - /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf + /* If iCurrentBlock>=iLeafEndBlock, this is an EOF condition. All leaf ** blocks have already been traversed. */ - assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock ); +#ifdef CORRUPT_DB + assert( pReader->iCurrentBlock<=pReader->iLeafEndBlock || CORRUPT_DB ); +#endif if( pReader->iCurrentBlock>=pReader->iLeafEndBlock ){ return SQLITE_OK; } rc = sqlite3Fts3ReadBlock( - p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, + p, ++pReader->iCurrentBlock, &pReader->aNode, &pReader->nNode, (bIncr ? &pReader->nPopulate : 0) ); if( rc!=SQLITE_OK ) return rc; @@ -152297,20 +199181,24 @@ static int fts3SegReaderNext( rc = fts3SegReaderRequire(pReader, pNext, FTS3_VARINT_MAX*2); if( rc!=SQLITE_OK ) return rc; - - /* Because of the FTS3_NODE_PADDING bytes of padding, the following is + + /* Because of the FTS3_NODE_PADDING bytes of padding, the following is ** safe (no risk of overread) even if the node data is corrupted. */ pNext += fts3GetVarint32(pNext, &nPrefix); pNext += fts3GetVarint32(pNext, &nSuffix); - if( nPrefix<0 || nSuffix<=0 - || &pNext[nSuffix]>&pReader->aNode[pReader->nNode] + if( nSuffix<=0 + || (&pReader->aNode[pReader->nNode] - pNext)pReader->nTerm ){ return FTS_CORRUPT_VTAB; } - if( nPrefix+nSuffix>pReader->nTermAlloc ){ - int nNew = (nPrefix+nSuffix)*2; - char *zNew = sqlite3_realloc(pReader->zTerm, nNew); + /* Both nPrefix and nSuffix were read by fts3GetVarint32() and so are + ** between 0 and 0x7FFFFFFF. But the sum of the two may cause integer + ** overflow - hence the (i64) casts. */ + if( (i64)nPrefix+nSuffix>(i64)pReader->nTermAlloc ){ + i64 nNew = ((i64)nPrefix+nSuffix)*2; + char *zNew = sqlite3_realloc64(pReader->zTerm, nNew); if( !zNew ){ return SQLITE_NOMEM; } @@ -152329,11 +199217,12 @@ static int fts3SegReaderNext( pReader->pOffsetList = 0; /* Check that the doclist does not appear to extend past the end of the - ** b-tree node. And that the final byte of the doclist is 0x00. If either + ** b-tree node. And that the final byte of the doclist is 0x00. If either ** of these statements is untrue, then the data structure is corrupt. */ - if( &pReader->aDoclist[pReader->nDoclist]>&pReader->aNode[pReader->nNode] + if( pReader->nDoclist > pReader->nNode-(pReader->aDoclist-pReader->aNode) || (pReader->nPopulate==0 && pReader->aDoclist[pReader->nDoclist-1]) + || pReader->nDoclist==0 ){ return FTS_CORRUPT_VTAB; } @@ -152353,7 +199242,7 @@ static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ pReader->iDocid = 0; pReader->nOffsetList = 0; sqlite3Fts3DoclistPrev(0, - pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, + pReader->aDoclist, pReader->nDoclist, &pReader->pOffsetList, &pReader->iDocid, &pReader->nOffsetList, &bEof ); }else{ @@ -152369,8 +199258,8 @@ static int fts3SegReaderFirstDocid(Fts3Table *pTab, Fts3SegReader *pReader){ /* ** Advance the SegReader to point to the next docid in the doclist ** associated with the current term. -** -** If arguments ppOffsetList and pnOffsetList are not NULL, then +** +** If arguments ppOffsetList and pnOffsetList are not NULL, then ** *ppOffsetList is set to point to the first column-offset list ** in the doclist entry (i.e. immediately past the docid varint). ** *pnOffsetList is set to the length of the set of column-offset @@ -152413,22 +199302,22 @@ static int fts3SegReaderNextDocid( ** following block advances it to point one byte past the end of ** the same offset list. */ while( 1 ){ - + /* The following line of code (and the "p++" below the while() loop) is - ** normally all that is required to move pointer p to the desired + ** normally all that is required to move pointer p to the desired ** position. The exception is if this node is being loaded from disk ** incrementally and pointer "p" now points to the first byte past ** the populated part of pReader->aNode[]. */ while( *p | c ) c = *p++ & 0x80; assert( *p==0 ); - + if( pReader->pBlob==0 || p<&pReader->aNode[pReader->nPopulate] ) break; rc = fts3SegReaderIncrRead(pReader); if( rc!=SQLITE_OK ) return rc; } p++; - + /* If required, populate the output variables with a pointer to and the ** size of the previous offset-list. */ @@ -152439,7 +199328,7 @@ static int fts3SegReaderNextDocid( /* List may have been edited in place by fts3EvalNearTrim() */ while( ppOffsetList = p + sqlite3Fts3GetVarint(p, &iDelta); + u64 iDelta; + pReader->pOffsetList = p + sqlite3Fts3GetVarintU(p, &iDelta); if( pTab->bDescIdx ){ - pReader->iDocid -= iDelta; + pReader->iDocid = (i64)((u64)pReader->iDocid - iDelta); }else{ - pReader->iDocid += iDelta; + pReader->iDocid = (i64)((u64)pReader->iDocid + iDelta); } } } } - return SQLITE_OK; + return rc; } SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( - Fts3Cursor *pCsr, + Fts3Cursor *pCsr, Fts3MultiSegReader *pMsr, int *pnOvfl ){ @@ -152481,8 +199370,8 @@ SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( for(ii=0; rc==SQLITE_OK && iinSegment; ii++){ Fts3SegReader *pReader = pMsr->apSegment[ii]; - if( !fts3SegReaderIsPending(pReader) - && !fts3SegReaderIsRootOnly(pReader) + if( !fts3SegReaderIsPending(pReader) + && !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_int64 jj; for(jj=pReader->iStartBlock; jj<=pReader->iLeafEndBlock; jj++){ @@ -152500,14 +199389,12 @@ SQLITE_PRIVATE int sqlite3Fts3MsrOvfl( } /* -** Free all allocations associated with the iterator passed as the +** Free all allocations associated with the iterator passed as the ** second argument. */ SQLITE_PRIVATE void sqlite3Fts3SegReaderFree(Fts3SegReader *pReader){ if( pReader ){ - if( !fts3SegReaderIsPending(pReader) ){ - sqlite3_free(pReader->zTerm); - } + sqlite3_free(pReader->zTerm); if( !fts3SegReaderIsRootOnly(pReader) ){ sqlite3_free(pReader->aNode); } @@ -152532,12 +199419,17 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( Fts3SegReader *pReader; /* Newly allocated SegReader object */ int nExtra = 0; /* Bytes to allocate segment root node */ - assert( iStartLeaf<=iEndLeaf ); + assert( zRoot!=0 || nRoot==0 ); +#ifdef CORRUPT_DB + assert( zRoot!=0 || CORRUPT_DB ); +#endif + if( iStartLeaf==0 ){ + if( iEndLeaf!=0 ) return FTS_CORRUPT_VTAB; nExtra = nRoot + FTS3_NODE_PADDING; } - pReader = (Fts3SegReader *)sqlite3_malloc(sizeof(Fts3SegReader) + nExtra); + pReader = (Fts3SegReader *)sqlite3_malloc64(sizeof(Fts3SegReader) + nExtra); if( !pReader ){ return SQLITE_NOMEM; } @@ -152553,7 +199445,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderNew( pReader->aNode = (char *)&pReader[1]; pReader->rootOnly = 1; pReader->nNode = nRoot; - memcpy(pReader->aNode, zRoot, nRoot); + if( nRoot ) memcpy(pReader->aNode, zRoot, nRoot); memset(&pReader->aNode[nRoot], 0, FTS3_NODE_PADDING); }else{ pReader->iCurrentBlock = iStartLeaf-1; @@ -152629,7 +199521,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( if( nElem==nAlloc ){ Fts3HashElem **aElem2; nAlloc += 16; - aElem2 = (Fts3HashElem **)sqlite3_realloc( + aElem2 = (Fts3HashElem **)sqlite3_realloc64( aElem, nAlloc*sizeof(Fts3HashElem *) ); if( !aElem2 ){ @@ -152654,7 +199546,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( }else{ /* The query is a simple term lookup that matches at most one term in - ** the index. All that is required is a straight hash-lookup. + ** the index. All that is required is a straight hash-lookup. ** ** Because the stack address of pE may be accessed via the aElem pointer ** below, the "Fts3HashElem *pE" must be declared so that it is valid @@ -152668,8 +199560,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } if( nElem>0 ){ - int nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); - pReader = (Fts3SegReader *)sqlite3_malloc(nByte); + sqlite3_int64 nByte; + nByte = sizeof(Fts3SegReader) + (nElem+1)*sizeof(Fts3HashElem *); + pReader = (Fts3SegReader *)sqlite3_malloc64(nByte); if( !pReader ){ rc = SQLITE_NOMEM; }else{ @@ -152688,7 +199581,7 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderPending( } /* -** Compare the entries pointed to by two Fts3SegReader structures. +** Compare the entries pointed to by two Fts3SegReader structures. ** Comparison is as follows: ** ** 1) EOF is greater than not EOF. @@ -152717,7 +199610,7 @@ static int fts3SegReaderCmp(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ if( rc==0 ){ rc = pRhs->iIdx - pLhs->iIdx; } - assert( rc!=0 ); + assert_fts3_nc( rc!=0 ); return rc; } @@ -152759,7 +199652,7 @@ static int fts3SegReaderDoclistCmpRev(Fts3SegReader *pLhs, Fts3SegReader *pRhs){ /* ** Compare the term that the Fts3SegReader object passed as the first argument -** points to with the term specified by arguments zTerm and nTerm. +** points to with the term specified by arguments zTerm and nTerm. ** ** If the pSeg iterator is already at EOF, return 0. Otherwise, return ** -ve if the pSeg term is less than zTerm/nTerm, 0 if the two terms are @@ -152820,7 +199713,7 @@ static void fts3SegReaderSort( #endif } -/* +/* ** Insert a record into the %_segments table. */ static int fts3WriteSegment( @@ -152836,6 +199729,7 @@ static int fts3WriteSegment( sqlite3_bind_blob(pStmt, 2, z, n, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); } return rc; } @@ -152861,7 +199755,7 @@ SQLITE_PRIVATE int sqlite3Fts3MaxLevel(Fts3Table *p, int *pnMax){ return rc; } -/* +/* ** Insert a record into the %_segdir table. */ static int fts3WriteSegdir( @@ -152892,13 +199786,14 @@ static int fts3WriteSegdir( sqlite3_bind_blob(pStmt, 6, zRoot, nRoot, SQLITE_STATIC); sqlite3_step(pStmt); rc = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 6); } return rc; } /* ** Return the size of the common prefix (if any) shared by zPrev and -** zNext, in bytes. For example, +** zNext, in bytes. For example, ** ** fts3PrefixCompress("abc", 3, "abcdef", 6) // returns 3 ** fts3PrefixCompress("abX", 3, "abcdef", 6) // returns 2 @@ -152911,8 +199806,8 @@ static int fts3PrefixCompress( int nNext /* Size of buffer zNext in bytes */ ){ int n; - UNUSED_PARAMETER(nNext); - for(n=0; nzTerm, pTree->nTerm, zTerm, nTerm); nSuffix = nTerm-nPrefix; + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + nReq += sqlite3Fts3VarintLen(nPrefix)+sqlite3Fts3VarintLen(nSuffix)+nSuffix; if( nReq<=p->nNodeSize || !pTree->zTerm ){ @@ -152951,11 +199851,11 @@ static int fts3NodeAddTerm( ** and the static node buffer (p->nNodeSize bytes) is not large ** enough. Use a separately malloced buffer instead This wastes ** p->nNodeSize bytes, but since this scenario only comes about when - ** the database contain two terms that share a prefix of almost 2KB, - ** this is not expected to be a serious problem. + ** the database contain two terms that share a prefix of almost 2KB, + ** this is not expected to be a serious problem. */ assert( pTree->aData==(char *)&pTree[1] ); - pTree->aData = (char *)sqlite3_malloc(nReq); + pTree->aData = (char *)sqlite3_malloc64(nReq); if( !pTree->aData ){ return SQLITE_NOMEM; } @@ -152973,7 +199873,7 @@ static int fts3NodeAddTerm( if( isCopyTerm ){ if( pTree->nMalloczMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pTree->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -152996,10 +199896,10 @@ static int fts3NodeAddTerm( ** If this is the first node in the tree, the term is added to it. ** ** Otherwise, the term is not added to the new node, it is left empty for - ** now. Instead, the term is inserted into the parent of pTree. If pTree + ** now. Instead, the term is inserted into the parent of pTree. If pTree ** has no parent, one is created here. */ - pNew = (SegmentNode *)sqlite3_malloc(sizeof(SegmentNode) + p->nNodeSize); + pNew = (SegmentNode *)sqlite3_malloc64(sizeof(SegmentNode) + p->nNodeSize); if( !pNew ){ return SQLITE_NOMEM; } @@ -153021,7 +199921,7 @@ static int fts3NodeAddTerm( pTree->zMalloc = 0; }else{ pNew->pLeftmost = pNew; - rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); + rc = fts3NodeAddTerm(p, &pNew, isCopyTerm, zTerm, nTerm); } *ppTree = pNew; @@ -153032,8 +199932,8 @@ static int fts3NodeAddTerm( ** Helper function for fts3NodeWrite(). */ static int fts3TreeFinishNode( - SegmentNode *pTree, - int iHeight, + SegmentNode *pTree, + int iHeight, sqlite3_int64 iLeftChild ){ int nStart; @@ -153046,15 +199946,15 @@ static int fts3TreeFinishNode( /* ** Write the buffer for the segment node pTree and all of its peers to the -** database. Then call this function recursively to write the parent of -** pTree and its peers to the database. +** database. Then call this function recursively to write the parent of +** pTree and its peers to the database. ** ** Except, if pTree is a root node, do not write it to the database. Instead, ** set output variables *paRoot and *pnRoot to contain the root node. ** ** If successful, SQLITE_OK is returned and output variable *piLast is ** set to the largest blockid written to the database (or zero if no -** blocks were written to the db). Otherwise, an SQLite error code is +** blocks were written to the db). Otherwise, an SQLite error code is ** returned. */ static int fts3NodeWrite( @@ -153082,7 +199982,7 @@ static int fts3NodeWrite( for(pIter=pTree->pLeftmost; pIter && rc==SQLITE_OK; pIter=pIter->pRight){ int nStart = fts3TreeFinishNode(pIter, iHeight, iNextLeaf); int nWrite = pIter->nData - nStart; - + rc = fts3WriteSegment(p, iNextFree, &pIter->aData[nStart], nWrite); iNextFree++; iNextLeaf += (pIter->nEntry+1); @@ -153128,7 +200028,7 @@ static void fts3NodeFree(SegmentNode *pTree){ */ static int fts3SegWriterAdd( Fts3Table *p, /* Virtual table handle */ - SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ + SegmentWriter **ppWriter, /* IN/OUT: SegmentWriter handle */ int isCopyTerm, /* True if buffer zTerm must be copied */ const char *zTerm, /* Pointer to buffer containing term */ int nTerm, /* Size of term in bytes */ @@ -153137,7 +200037,7 @@ static int fts3SegWriterAdd( ){ int nPrefix; /* Size of term prefix in bytes */ int nSuffix; /* Size of term suffix in bytes */ - int nReq; /* Number of bytes required on leaf page */ + i64 nReq; /* Number of bytes required on leaf page */ int nData; SegmentWriter *pWriter = *ppWriter; @@ -153146,13 +200046,13 @@ static int fts3SegWriterAdd( sqlite3_stmt *pStmt; /* Allocate the SegmentWriter structure */ - pWriter = (SegmentWriter *)sqlite3_malloc(sizeof(SegmentWriter)); + pWriter = (SegmentWriter *)sqlite3_malloc64(sizeof(SegmentWriter)); if( !pWriter ) return SQLITE_NOMEM; memset(pWriter, 0, sizeof(SegmentWriter)); *ppWriter = pWriter; /* Allocate a buffer in which to accumulate data */ - pWriter->aData = (char *)sqlite3_malloc(p->nNodeSize); + pWriter->aData = (char *)sqlite3_malloc64(p->nNodeSize); if( !pWriter->aData ) return SQLITE_NOMEM; pWriter->nSize = p->nNodeSize; @@ -153171,6 +200071,11 @@ static int fts3SegWriterAdd( nPrefix = fts3PrefixCompress(pWriter->zTerm, pWriter->nTerm, zTerm, nTerm); nSuffix = nTerm-nPrefix; + /* If nSuffix is zero or less, then zTerm/nTerm must be a prefix of + ** pWriter->zTerm/pWriter->nTerm. i.e. must be equal to or less than when + ** compared with BINARY collation. This indicates corruption. */ + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; + /* Figure out how many bytes are required by this new entry */ nReq = sqlite3Fts3VarintLen(nPrefix) + /* varint containing prefix size */ sqlite3Fts3VarintLen(nSuffix) + /* varint containing suffix size */ @@ -153182,6 +200087,7 @@ static int fts3SegWriterAdd( int rc; /* The current leaf node is full. Write it out to the database. */ + if( pWriter->iFree==LARGEST_INT64 ) return FTS_CORRUPT_VTAB; rc = fts3WriteSegment(p, pWriter->iFree++, pWriter->aData, nData); if( rc!=SQLITE_OK ) return rc; p->nLeafAdd++; @@ -153221,7 +200127,7 @@ static int fts3SegWriterAdd( ** the buffer to make it large enough. */ if( nReq>pWriter->nSize ){ - char *aNew = sqlite3_realloc(pWriter->aData, nReq); + char *aNew = sqlite3_realloc64(pWriter->aData, nReq); if( !aNew ) return SQLITE_NOMEM; pWriter->aData = aNew; pWriter->nSize = nReq; @@ -153231,9 +200137,11 @@ static int fts3SegWriterAdd( /* Append the prefix-compressed term and doclist to the buffer. */ nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nPrefix); nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nSuffix); + assert( nSuffix>0 ); memcpy(&pWriter->aData[nData], &zTerm[nPrefix], nSuffix); nData += nSuffix; nData += sqlite3Fts3PutVarint(&pWriter->aData[nData], nDoclist); + assert( nDoclist>0 ); memcpy(&pWriter->aData[nData], aDoclist, nDoclist); pWriter->nData = nData + nDoclist; @@ -153244,7 +200152,7 @@ static int fts3SegWriterAdd( */ if( isCopyTerm ){ if( nTerm>pWriter->nMalloc ){ - char *zNew = sqlite3_realloc(pWriter->zMalloc, nTerm*2); + char *zNew = sqlite3_realloc64(pWriter->zMalloc, (i64)nTerm*2); if( !zNew ){ return SQLITE_NOMEM; } @@ -153253,6 +200161,7 @@ static int fts3SegWriterAdd( pWriter->zTerm = zNew; } assert( pWriter->zTerm==pWriter->zMalloc ); + assert( nTerm>0 ); memcpy(pWriter->zTerm, zTerm, nTerm); }else{ pWriter->zTerm = (char *)zTerm; @@ -153288,12 +200197,12 @@ static int fts3SegWriterFlush( pWriter->iFirst, pWriter->iFree, &iLast, &zRoot, &nRoot); } if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir(p, iLevel, iIdx, + rc = fts3WriteSegdir(p, iLevel, iIdx, pWriter->iFirst, iLastLeaf, iLast, pWriter->nLeafData, zRoot, nRoot); } }else{ /* The entire tree fits on the root node. Write it to the segdir table. */ - rc = fts3WriteSegdir(p, iLevel, iIdx, + rc = fts3WriteSegdir(p, iLevel, iIdx, 0, 0, 0, pWriter->nLeafData, pWriter->aData, pWriter->nData); } p->nLeafAdd++; @@ -153301,7 +200210,7 @@ static int fts3SegWriterFlush( } /* -** Release all memory held by the SegmentWriter object passed as the +** Release all memory held by the SegmentWriter object passed as the ** first argument. */ static void fts3SegWriterFree(SegmentWriter *pWriter){ @@ -153351,9 +200260,9 @@ static int fts3IsEmpty(Fts3Table *p, sqlite3_value *pRowid, int *pisEmpty){ ** Return SQLITE_OK if successful, or an SQLite error code if not. */ static int fts3SegmentMaxLevel( - Fts3Table *p, + Fts3Table *p, int iLangid, - int iIndex, + int iIndex, sqlite3_int64 *pnMax ){ sqlite3_stmt *pStmt; @@ -153369,7 +200278,7 @@ static int fts3SegmentMaxLevel( rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int64(pStmt, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pStmt, 2, + sqlite3_bind_int64(pStmt, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); if( SQLITE_ROW==sqlite3_step(pStmt) ){ @@ -153398,8 +200307,8 @@ static int fts3SegmentIsMaxLevel(Fts3Table *p, i64 iAbsLevel, int *pbMax){ int rc = fts3SqlStmt(p, SQL_SELECT_SEGDIR_MAX_LEVEL, &pStmt, 0); if( rc!=SQLITE_OK ) return rc; sqlite3_bind_int64(pStmt, 1, iAbsLevel+1); - sqlite3_bind_int64(pStmt, 2, - ((iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL + sqlite3_bind_int64(pStmt, 2, + (((u64)iAbsLevel/FTS3_SEGDIR_MAXLEVEL)+1) * FTS3_SEGDIR_MAXLEVEL ); *pbMax = 0; @@ -153436,9 +200345,9 @@ static int fts3DeleteSegment( ** This function is used after merging multiple segments into a single large ** segment to delete the old, now redundant, segment b-trees. Specifically, ** it: -** -** 1) Deletes all %_segments entries for the segments associated with -** each of the SegReader objects in the array passed as the third +** +** 1) Deletes all %_segments entries for the segments associated with +** each of the SegReader objects in the array passed as the third ** argument, and ** ** 2) deletes all %_segdir entries with level iLevel, or all %_segdir @@ -153470,7 +200379,7 @@ static int fts3DeleteSegdir( rc = fts3SqlStmt(p, SQL_DELETE_SEGDIR_RANGE, &pDelete, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pDelete, 1, getAbsoluteLevel(p, iLangid, iIndex, 0)); - sqlite3_bind_int64(pDelete, 2, + sqlite3_bind_int64(pDelete, 2, getAbsoluteLevel(p, iLangid, iIndex, FTS3_SEGDIR_MAXLEVEL-1) ); } @@ -153492,7 +200401,7 @@ static int fts3DeleteSegdir( } /* -** When this function is called, buffer *ppList (size *pnList bytes) contains +** When this function is called, buffer *ppList (size *pnList bytes) contains ** a position list that may (or may not) feature multiple columns. This ** function adjusts the pointer *ppList and the length *pnList so that they ** identify the subset of the position list that corresponds to column iCol. @@ -153519,7 +200428,7 @@ static void fts3ColumnFilter( while( 1 ){ char c = 0; while( p0){ memset(&pList[nList], 0, pEnd - &pList[nList]); } *ppList = pList; @@ -153551,17 +200460,20 @@ static void fts3ColumnFilter( static int fts3MsrBufferData( Fts3MultiSegReader *pMsr, /* Multi-segment-reader handle */ char *pList, - int nList + i64 nList ){ - if( nList>pMsr->nBuffer ){ + if( (nList+FTS3_NODE_PADDING)>pMsr->nBuffer ){ char *pNew; - pMsr->nBuffer = nList*2; - pNew = (char *)sqlite3_realloc(pMsr->aBuffer, pMsr->nBuffer); + int nNew = nList*2 + FTS3_NODE_PADDING; + pNew = (char *)sqlite3_realloc64(pMsr->aBuffer, nNew); if( !pNew ) return SQLITE_NOMEM; pMsr->aBuffer = pNew; + pMsr->nBuffer = nNew; } + assert( nList>0 ); memcpy(pMsr->aBuffer, pList, nList); + memset(&pMsr->aBuffer[nList], 0, FTS3_NODE_PADDING); return SQLITE_OK; } @@ -153599,7 +200511,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( rc = fts3SegReaderNextDocid(p, apSegment[0], &pList, &nList); j = 1; - while( rc==SQLITE_OK + while( rc==SQLITE_OK && jpOffsetList && apSegment[j]->iDocid==iDocid @@ -153611,7 +200523,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrNext( fts3SegReaderSort(pMsr->apSegment, nMerge, j, xCmp); if( nList>0 && fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pMsr, pList, nList+1); + rc = fts3MsrBufferData(pMsr, pList, (i64)nList+1); if( rc!=SQLITE_OK ) return rc; assert( (pMsr->aBuffer[nList] & 0xFE)==0x00 ); pList = pMsr->aBuffer; @@ -153642,7 +200554,7 @@ static int fts3SegReaderStart( int i; int nSeg = pCsr->nSegment; - /* If the Fts3SegFilter defines a specific term (or term prefix) to search + /* If the Fts3SegFilter defines a specific term (or term prefix) to search ** for, then advance each segment iterator until it points to a term of ** equal or greater value than the specified term. This prevents many ** unnecessary merge/sort operations for the case where single segment @@ -153726,7 +200638,7 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrStart( ** sqlite3Fts3SegReaderStart() ** sqlite3Fts3SegReaderStep() ** -** then the entire doclist for the term is available in +** then the entire doclist for the term is available in ** MultiSegReader.aDoclist/nDoclist. */ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ @@ -153748,6 +200660,19 @@ SQLITE_PRIVATE int sqlite3Fts3MsrIncrRestart(Fts3MultiSegReader *pCsr){ return SQLITE_OK; } +static int fts3GrowSegReaderBuffer(Fts3MultiSegReader *pCsr, i64 nReq){ + if( nReq>pCsr->nBuffer ){ + char *aNew; + pCsr->nBuffer = nReq*2; + aNew = sqlite3_realloc64(pCsr->aBuffer, pCsr->nBuffer); + if( !aNew ){ + return SQLITE_NOMEM; + } + pCsr->aBuffer = aNew; + } + return SQLITE_OK; +} + SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( Fts3Table *p, /* Virtual table handle */ @@ -153774,9 +200699,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( do { int nMerge; int i; - + /* Advance the first pCsr->nAdvance entries in the apSegment[] array - ** forward. Then sort the list in order of current term again. + ** forward. Then sort the list in order of current term again. */ for(i=0; inAdvance; i++){ Fts3SegReader *pSeg = apSegment[i]; @@ -153798,39 +200723,40 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( pCsr->zTerm = apSegment[0]->zTerm; /* If this is a prefix-search, and if the term that apSegment[0] points - ** to does not share a suffix with pFilter->zTerm/nTerm, then all + ** to does not share a suffix with pFilter->zTerm/nTerm, then all ** required callbacks have been made. In this case exit early. ** ** Similarly, if this is a search for an exact match, and the first term ** of segment apSegment[0] is not a match, exit early. */ if( pFilter->zTerm && !isScan ){ - if( pCsr->nTermnTerm + if( pCsr->nTermnTerm || (!isPrefix && pCsr->nTerm>pFilter->nTerm) - || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) + || memcmp(pCsr->zTerm, pFilter->zTerm, pFilter->nTerm) ){ break; } } nMerge = 1; - while( nMergeaNode - && apSegment[nMerge]->nTerm==pCsr->nTerm + && apSegment[nMerge]->nTerm==pCsr->nTerm && 0==memcmp(pCsr->zTerm, apSegment[nMerge]->zTerm, pCsr->nTerm) ){ nMerge++; } assert( isIgnoreEmpty || (isRequirePos && !isColFilter) ); - if( nMerge==1 - && !isIgnoreEmpty - && !isFirst + if( nMerge==1 + && !isIgnoreEmpty + && !isFirst && (p->bDescIdx==0 || fts3SegReaderIsPending(apSegment[0])==0) ){ pCsr->nDoclist = apSegment[0]->nDoclist; if( fts3SegReaderIsPending(apSegment[0]) ){ - rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, pCsr->nDoclist); + rc = fts3MsrBufferData(pCsr, apSegment[0]->aDoclist, + (i64)pCsr->nDoclist); pCsr->aDoclist = pCsr->aBuffer; }else{ pCsr->aDoclist = apSegment[0]->aDoclist; @@ -153870,32 +200796,27 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( if( !isIgnoreEmpty || nList>0 ){ - /* Calculate the 'docid' delta value to write into the merged + /* Calculate the 'docid' delta value to write into the merged ** doclist. */ sqlite3_int64 iDelta; if( p->bDescIdx && nDoclist>0 ){ - iDelta = iPrev - iDocid; + if( iPrev<=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iPrev - (u64)iDocid); }else{ - iDelta = iDocid - iPrev; + if( nDoclist>0 && iPrev>=iDocid ) return FTS_CORRUPT_VTAB; + iDelta = (i64)((u64)iDocid - (u64)iPrev); } - assert( iDelta>0 || (nDoclist==0 && iDelta==iDocid) ); - assert( nDoclist>0 || iDelta==iDocid ); nByte = sqlite3Fts3VarintLen(iDelta) + (isRequirePos?nList+1:0); - if( nDoclist+nByte>pCsr->nBuffer ){ - char *aNew; - pCsr->nBuffer = (nDoclist+nByte)*2; - aNew = sqlite3_realloc(pCsr->aBuffer, pCsr->nBuffer); - if( !aNew ){ - return SQLITE_NOMEM; - } - pCsr->aBuffer = aNew; - } + + rc = fts3GrowSegReaderBuffer(pCsr, + (i64)nByte+nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; if( isFirst ){ char *a = &pCsr->aBuffer[nDoclist]; int nWrite; - + nWrite = sqlite3Fts3FirstFilter(iDelta, pList, nList, a); if( nWrite ){ iPrev = iDocid; @@ -153915,6 +200836,9 @@ SQLITE_PRIVATE int sqlite3Fts3SegReaderStep( fts3SegReaderSort(apSegment, nMerge, j, xCmp); } if( nDoclist>0 ){ + rc = fts3GrowSegReaderBuffer(pCsr, (i64)nDoclist+FTS3_NODE_PADDING); + if( rc ) return rc; + memset(&pCsr->aBuffer[nDoclist], 0, FTS3_NODE_PADDING); pCsr->aDoclist = pCsr->aBuffer; pCsr->nDoclist = nDoclist; rc = SQLITE_ROW; @@ -153945,18 +200869,18 @@ SQLITE_PRIVATE void sqlite3Fts3SegReaderFinish( } /* -** Decode the "end_block" field, selected by column iCol of the SELECT -** statement passed as the first argument. +** Decode the "end_block" field, selected by column iCol of the SELECT +** statement passed as the first argument. ** ** The "end_block" field may contain either an integer, or a text field -** containing the text representation of two non-negative integers separated -** by one or more space (0x20) characters. In the first case, set *piEndBlock -** to the integer value and *pnByte to zero before returning. In the second, +** containing the text representation of two non-negative integers separated +** by one or more space (0x20) characters. In the first case, set *piEndBlock +** to the integer value and *pnByte to zero before returning. In the second, ** set *piEndBlock to the first value and *pnByte to the second. */ static void fts3ReadEndBlockField( - sqlite3_stmt *pStmt, - int iCol, + sqlite3_stmt *pStmt, + int iCol, i64 *piEndBlock, i64 *pnByte ){ @@ -153964,11 +200888,11 @@ static void fts3ReadEndBlockField( if( zText ){ int i; int iMul = 1; - i64 iVal = 0; + u64 iVal = 0; for(i=0; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *piEndBlock = iVal; + *piEndBlock = (i64)iVal; while( zText[i]==' ' ) i++; iVal = 0; if( zText[i]=='-' ){ @@ -153978,7 +200902,7 @@ static void fts3ReadEndBlockField( for(/* no-op */; zText[i]>='0' && zText[i]<='9'; i++){ iVal = iVal*10 + (zText[i] - '0'); } - *pnByte = (iVal * (i64)iMul); + *pnByte = ((i64)iVal * (i64)iMul); } } @@ -154002,10 +200926,10 @@ static int fts3PromoteSegments( i64 iLast = (iAbsLevel/FTS3_SEGDIR_MAXLEVEL + 1) * FTS3_SEGDIR_MAXLEVEL - 1; i64 nLimit = (nByte*3)/2; - /* Loop through all entries in the %_segdir table corresponding to + /* Loop through all entries in the %_segdir table corresponding to ** segments in this index on levels greater than iAbsLevel. If there is - ** at least one such segment, and it is possible to determine that all - ** such segments are smaller than nLimit bytes in size, they will be + ** at least one such segment, and it is possible to determine that all + ** such segments are smaller than nLimit bytes in size, they will be ** promoted to level iAbsLevel. */ sqlite3_bind_int64(pRange, 1, iAbsLevel+1); sqlite3_bind_int64(pRange, 2, iLast); @@ -154013,7 +200937,7 @@ static int fts3PromoteSegments( i64 nSize = 0, dummy; fts3ReadEndBlockField(pRange, 2, &dummy, &nSize); if( nSize<=0 || nSize>nLimit ){ - /* If nSize==0, then the %_segdir.end_block field does not not + /* If nSize==0, then the %_segdir.end_block field does not not ** contain a size value. This happens if it was written by an ** old version of FTS. In this case it is not possible to determine ** the size of the segment, and so segment promotion does not @@ -154079,18 +201003,18 @@ static int fts3PromoteSegments( } /* -** Merge all level iLevel segments in the database into a single +** Merge all level iLevel segments in the database into a single ** iLevel+1 segment. Or, if iLevel<0, merge all segments into a -** single segment with a level equal to the numerically largest level +** single segment with a level equal to the numerically largest level ** currently present in the database. ** ** If this function is called with iLevel<0, but there is only one -** segment in the database, SQLITE_DONE is returned immediately. -** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, +** segment in the database, SQLITE_DONE is returned immediately. +** Otherwise, if successful, SQLITE_OK is returned. If an error occurs, ** an SQLite error code is returned. */ static int fts3SegmentMerge( - Fts3Table *p, + Fts3Table *p, int iLangid, /* Language id to merge */ int iIndex, /* Index in p->aIndex[] to merge */ int iLevel /* Level to merge */ @@ -154134,7 +201058,7 @@ static int fts3SegmentMerge( }else{ /* This call is to merge all segments at level iLevel. find the next ** available segment index at level iLevel+1. The call to - ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to + ** fts3AllocateSegdirIdx() will merge the segments at level iLevel+1 to ** a single iLevel+2 segment if necessary. */ assert( FTS3_SEGCURSOR_PENDING==-1 ); iNewLevel = getAbsoluteLevel(p, iLangid, iIndex, iLevel+1); @@ -154144,8 +201068,10 @@ static int fts3SegmentMerge( if( rc!=SQLITE_OK ) goto finished; assert( csr.nSegment>0 ); - assert( iNewLevel>=getAbsoluteLevel(p, iLangid, iIndex, 0) ); - assert( iNewLevel=getAbsoluteLevel(p, iLangid, iIndex, 0) ); + assert_fts3_nc( + iNewLevelnIndex; i++){ rc = fts3SegmentMerge(p, p->iPrevLangid, i, FTS3_SEGCURSOR_PENDING); if( rc==SQLITE_DONE ) rc = SQLITE_OK; } - sqlite3Fts3PendingTermsClear(p); /* Determine the auto-incr-merge setting if unknown. If enabled, ** estimate the number of leaf blocks of content to be written @@ -154216,6 +201141,10 @@ SQLITE_PRIVATE int sqlite3Fts3PendingTermsFlush(Fts3Table *p){ rc = sqlite3_reset(pStmt); } } + + if( rc==SQLITE_OK ){ + sqlite3Fts3PendingTermsClear(p); + } return rc; } @@ -154244,14 +201173,16 @@ static void fts3DecodeIntArray( const char *zBuf, /* The BLOB containing the varints */ int nBuf /* size of the BLOB */ ){ - int i, j; - UNUSED_PARAMETER(nBuf); - for(i=j=0; inColumn ); + pBlob = sqlite3_malloc64( 10*(sqlite3_int64)p->nColumn ); if( pBlob==0 ){ *pRC = SQLITE_NOMEM; return; @@ -154291,7 +201222,7 @@ static void fts3InsertDocsize( /* ** Record 0 of the %_stat table contains a blob consisting of N varints, ** where N is the number of user defined columns in the fts3 table plus -** two. If nCol is the number of user defined columns, then values of the +** two. If nCol is the number of user defined columns, then values of the ** varints are set as follows: ** ** Varint 0: Total number of rows in the table. @@ -154320,7 +201251,7 @@ static void fts3UpdateDocTotals( const int nStat = p->nColumn+2; if( *pRC ) return; - a = sqlite3_malloc( (sizeof(u32)+10)*nStat ); + a = sqlite3_malloc64( (sizeof(u32)+10)*(sqlite3_int64)nStat ); if( a==0 ){ *pRC = SQLITE_NOMEM; return; @@ -154371,11 +201302,12 @@ static void fts3UpdateDocTotals( sqlite3_bind_blob(pStmt, 2, pBlob, nBlob, SQLITE_STATIC); sqlite3_step(pStmt); *pRC = sqlite3_reset(pStmt); + sqlite3_bind_null(pStmt, 2); sqlite3_free(a); } /* -** Merge the entire database so that there is one segment for each +** Merge the entire database so that there is one segment for each ** iIndex/iLangid combination. */ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ @@ -154383,7 +201315,10 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ int rc; sqlite3_stmt *pAllLangid = 0; - rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + rc = sqlite3Fts3PendingTermsFlush(p); + if( rc==SQLITE_OK ){ + rc = fts3SqlStmt(p, SQL_SELECT_ALL_LANGID, &pAllLangid, 0); + } if( rc==SQLITE_OK ){ int rc2; sqlite3_bind_int(pAllLangid, 1, p->iPrevLangid); @@ -154404,7 +201339,6 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ } sqlite3Fts3SegmentsClose(p); - sqlite3Fts3PendingTermsClear(p); return (rc==SQLITE_OK && bReturnDone && bSeenDone) ? SQLITE_DONE : rc; } @@ -154414,7 +201348,7 @@ static int fts3DoOptimize(Fts3Table *p, int bReturnDone){ ** ** INSERT INTO () VALUES('rebuild'); ** -** The entire FTS index is discarded and rebuilt. If the table is one +** The entire FTS index is discarded and rebuilt. If the table is one ** created using the content=xxx option, then the new index is based on ** the current contents of the xxx table. Otherwise, it is rebuilt based ** on the contents of the %_content table. @@ -154440,8 +201374,8 @@ static int fts3DoRebuild(Fts3Table *p){ } if( rc==SQLITE_OK ){ - int nByte = sizeof(u32) * (p->nColumn+1)*3; - aSz = (u32 *)sqlite3_malloc(nByte); + sqlite3_int64 nByte = sizeof(u32) * ((sqlite3_int64)p->nColumn+1)*3; + aSz = (u32 *)sqlite3_malloc64(nByte); if( aSz==0 ){ rc = SQLITE_NOMEM; }else{ @@ -154494,9 +201428,9 @@ static int fts3DoRebuild(Fts3Table *p){ /* -** This function opens a cursor used to read the input data for an +** This function opens a cursor used to read the input data for an ** incremental merge operation. Specifically, it opens a cursor to scan -** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute +** the oldest nSeg segments (idx=0 through idx=(nSeg-1)) in absolute ** level iAbsLevel. */ static int fts3IncrmergeCsr( @@ -154506,13 +201440,13 @@ static int fts3IncrmergeCsr( Fts3MultiSegReader *pCsr /* Cursor object to populate */ ){ int rc; /* Return Code */ - sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ - int nByte; /* Bytes allocated at pCsr->apSegment[] */ + sqlite3_stmt *pStmt = 0; /* Statement used to read %_segdir entry */ + sqlite3_int64 nByte; /* Bytes allocated at pCsr->apSegment[] */ /* Allocate space for the Fts3MultiSegReader.aCsr[] array */ memset(pCsr, 0, sizeof(*pCsr)); nByte = sizeof(Fts3SegReader *) * nSeg; - pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc(nByte); + pCsr->apSegment = (Fts3SegReader **)sqlite3_malloc64(nByte); if( pCsr->apSegment==0 ){ rc = SQLITE_NOMEM; @@ -154561,7 +201495,7 @@ struct Blob { }; /* -** This structure is used to build up buffers containing segment b-tree +** This structure is used to build up buffers containing segment b-tree ** nodes (blocks). */ struct NodeWriter { @@ -154618,7 +201552,7 @@ struct NodeReader { static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ if( *pRc==SQLITE_OK && nMin>pBlob->nAlloc ){ int nAlloc = nMin; - char *a = (char *)sqlite3_realloc(pBlob->a, nAlloc); + char *a = (char *)sqlite3_realloc64(pBlob->a, nAlloc); if( a ){ pBlob->nAlloc = nAlloc; pBlob->a = a; @@ -154630,12 +201564,12 @@ static void blobGrowBuffer(Blob *pBlob, int nMin, int *pRc){ /* ** Attempt to advance the node-reader object passed as the first argument to -** the next entry on the node. +** the next entry on the node. ** -** Return an error code if an error occurs (SQLITE_NOMEM is possible). +** Return an error code if an error occurs (SQLITE_NOMEM is possible). ** Otherwise return SQLITE_OK. If there is no next entry on the node ** (e.g. because the current entry is the last) set NodeReader->aNode to -** NULL to indicate EOF. Otherwise, populate the NodeReader structure output +** NULL to indicate EOF. Otherwise, populate the NodeReader structure output ** variables for the new entry. */ static int nodeReaderNext(NodeReader *p){ @@ -154655,21 +201589,26 @@ static int nodeReaderNext(NodeReader *p){ } p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &nSuffix); + if( nPrefix>p->term.n || nSuffix>p->nNode-p->iOff || nSuffix==0 ){ + return FTS_CORRUPT_VTAB; + } blobGrowBuffer(&p->term, nPrefix+nSuffix, &rc); - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(p->term.a!=0) ){ memcpy(&p->term.a[nPrefix], &p->aNode[p->iOff], nSuffix); p->term.n = nPrefix+nSuffix; p->iOff += nSuffix; if( p->iChild==0 ){ p->iOff += fts3GetVarint32(&p->aNode[p->iOff], &p->nDoclist); + if( (p->nNode-p->iOff)nDoclist ){ + return FTS_CORRUPT_VTAB; + } p->aDoclist = &p->aNode[p->iOff]; p->iOff += p->nDoclist; } } } - assert( p->iOff<=p->nNode ); - + assert_fts3_nc( p->iOff<=p->nNode ); return rc; } @@ -154683,7 +201622,7 @@ static void nodeReaderRelease(NodeReader *p){ /* ** Initialize a node-reader object to read the node in buffer aNode/nNode. ** -** If successful, SQLITE_OK is returned and the NodeReader object set to +** If successful, SQLITE_OK is returned and the NodeReader object set to ** point to the first entry on the node (if any). Otherwise, an SQLite ** error code is returned. */ @@ -154693,14 +201632,14 @@ static int nodeReaderInit(NodeReader *p, const char *aNode, int nNode){ p->nNode = nNode; /* Figure out if this is a leaf or an internal node. */ - if( p->aNode[0] ){ + if( aNode && aNode[0] ){ /* An internal node. */ p->iOff = 1 + sqlite3Fts3GetVarint(&p->aNode[1], &p->iChild); }else{ p->iOff = 1; } - return nodeReaderNext(p); + return aNode ? nodeReaderNext(p) : SQLITE_OK; } /* @@ -154732,17 +201671,18 @@ static int fts3IncrmergePush( int nSpace; /* Figure out how much space the key will consume if it is written to - ** the current node of layer iLayer. Due to the prefix compression, + ** the current node of layer iLayer. Due to the prefix compression, ** the space required changes depending on which node the key is to ** be added to. */ nPrefix = fts3PrefixCompress(pNode->key.a, pNode->key.n, zTerm, nTerm); nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; nSpace = sqlite3Fts3VarintLen(nPrefix); nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; - if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ + if( pNode->key.n==0 || (pNode->block.n + nSpace)<=p->nNodeSize ){ /* If the current node of layer iLayer contains zero keys, or if adding - ** the key to it will not cause it to grow to larger than nNodeSize + ** the key to it will not cause it to grow to larger than nNodeSize ** bytes in size, write the key here. */ Blob *pBlk = &pNode->block; @@ -154761,6 +201701,8 @@ static int fts3IncrmergePush( pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nPrefix); } pBlk->n += sqlite3Fts3PutVarint(&pBlk->a[pBlk->n], nSuffix); + assert( nPrefix+nSuffix<=nTerm ); + assert( nPrefix>=0 ); memcpy(&pBlk->a[pBlk->n], &zTerm[nPrefix], nSuffix); pBlk->n += nSuffix; @@ -154798,12 +201740,12 @@ static int fts3IncrmergePush( ** A node header is a single 0x00 byte for a leaf node, or a height varint ** followed by the left-hand-child varint for an internal node. ** -** The term to be appended is passed via arguments zTerm/nTerm. For a +** The term to be appended is passed via arguments zTerm/nTerm. For a ** leaf node, the doclist is passed as aDoclist/nDoclist. For an internal ** node, both aDoclist and nDoclist must be passed 0. ** ** If the size of the value in blob pPrev is zero, then this is the first -** term written to the node. Otherwise, pPrev contains a copy of the +** term written to the node. Otherwise, pPrev contains a copy of the ** previous term. Before this function returns, it is updated to contain a ** copy of zTerm/nTerm. ** @@ -154820,7 +201762,7 @@ static int fts3AppendToNode( const char *zTerm, /* New term to write */ int nTerm, /* Size of zTerm in bytes */ const char *aDoclist, /* Doclist (or NULL) to write */ - int nDoclist /* Size of aDoclist in bytes */ + int nDoclist /* Size of aDoclist in bytes */ ){ int rc = SQLITE_OK; /* Return code */ int bFirst = (pPrev->n==0); /* True if this is the first term written */ @@ -154830,13 +201772,16 @@ static int fts3AppendToNode( /* Node must have already been started. There must be a doclist for a ** leaf node, and there must not be a doclist for an internal node. */ assert( pNode->n>0 ); - assert( (pNode->a[0]=='\0')==(aDoclist!=0) ); + assert_fts3_nc( (pNode->a[0]=='\0')==(aDoclist!=0) ); blobGrowBuffer(pPrev, nTerm, &rc); if( rc!=SQLITE_OK ) return rc; + assert( pPrev!=0 ); + assert( pPrev->a!=0 ); nPrefix = fts3PrefixCompress(pPrev->a, pPrev->n, zTerm, nTerm); nSuffix = nTerm - nPrefix; + if( nSuffix<=0 ) return FTS_CORRUPT_VTAB; memcpy(pPrev->a, zTerm, nTerm); pPrev->n = nTerm; @@ -154882,19 +201827,24 @@ static int fts3IncrmergeAppend( pLeaf = &pWriter->aNodeWriter[0]; nPrefix = fts3PrefixCompress(pLeaf->key.a, pLeaf->key.n, zTerm, nTerm); nSuffix = nTerm - nPrefix; + if(nSuffix<=0 ) return FTS_CORRUPT_VTAB; nSpace = sqlite3Fts3VarintLen(nPrefix); nSpace += sqlite3Fts3VarintLen(nSuffix) + nSuffix; nSpace += sqlite3Fts3VarintLen(nDoclist) + nDoclist; /* If the current block is not empty, and if adding this term/doclist - ** to the current block would make it larger than Fts3Table.nNodeSize - ** bytes, write this block out to the database. */ - if( pLeaf->block.n>0 && (pLeaf->block.n + nSpace)>p->nNodeSize ){ + ** to the current block would make it larger than Fts3Table.nNodeSize bytes, + ** and if there is still room for another leaf page, write this block out to + ** the database. */ + if( pLeaf->block.n>0 + && (pLeaf->block.n + nSpace)>p->nNodeSize + && pLeaf->iBlock < (pWriter->iStart + pWriter->nLeafEst) + ){ rc = fts3WriteSegment(p, pLeaf->iBlock, pLeaf->block.a, pLeaf->block.n); pWriter->nWork++; - /* Add the current term to the parent node. The term added to the + /* Add the current term to the parent node. The term added to the ** parent must: ** ** a) be greater than the largest term on the leaf node just written @@ -154959,7 +201909,7 @@ static void fts3IncrmergeRelease( NodeWriter *pRoot; /* NodeWriter for root node */ int rc = *pRc; /* Error code */ - /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment + /* Set iRoot to the index in pWriter->aNodeWriter[] of the output segment ** root node. If the segment fits entirely on a single leaf node, iRoot ** will be set to 0. If the root node is the parent of the leaves, iRoot ** will be 1. And so on. */ @@ -154977,17 +201927,17 @@ static void fts3IncrmergeRelease( /* The entire output segment fits on a single node. Normally, this means ** the node would be stored as a blob in the "root" column of the %_segdir - ** table. However, this is not permitted in this case. The problem is that - ** space has already been reserved in the %_segments table, and so the - ** start_block and end_block fields of the %_segdir table must be populated. - ** And, by design or by accident, released versions of FTS cannot handle + ** table. However, this is not permitted in this case. The problem is that + ** space has already been reserved in the %_segments table, and so the + ** start_block and end_block fields of the %_segdir table must be populated. + ** And, by design or by accident, released versions of FTS cannot handle ** segments that fit entirely on the root node with start_block!=0. ** - ** Instead, create a synthetic root node that contains nothing but a + ** Instead, create a synthetic root node that contains nothing but a ** pointer to the single content node. So that the segment consists of a ** single leaf and a single interior (root) node. ** - ** Todo: Better might be to defer allocating space in the %_segments + ** Todo: Better might be to defer allocating space in the %_segments ** table until we are sure it is needed. */ if( iRoot==0 ){ @@ -155015,7 +201965,7 @@ static void fts3IncrmergeRelease( /* Write the %_segdir record. */ if( rc==SQLITE_OK ){ - rc = fts3WriteSegdir(p, + rc = fts3WriteSegdir(p, pWriter->iAbsLevel+1, /* level */ pWriter->iIdx, /* idx */ pWriter->iStart, /* start_block */ @@ -155046,7 +201996,11 @@ static int fts3TermCmp( int nCmp = MIN(nLhs, nRhs); int res; - res = memcmp(zLhs, zRhs, nCmp); + if( nCmp && ALWAYS(zLhs) && ALWAYS(zRhs) ){ + res = memcmp(zLhs, zRhs, nCmp); + }else{ + res = 0; + } if( res==0 ) res = nLhs - nRhs; return res; @@ -155054,11 +202008,11 @@ static int fts3TermCmp( /* -** Query to see if the entry in the %_segments table with blockid iEnd is +** Query to see if the entry in the %_segments table with blockid iEnd is ** NULL. If no error occurs and the entry is NULL, set *pbRes 1 before -** returning. Otherwise, set *pbRes to 0. +** returning. Otherwise, set *pbRes to 0. ** -** Or, if an error occurs while querying the database, return an SQLite +** Or, if an error occurs while querying the database, return an SQLite ** error code. The final value of *pbRes is undefined in this case. ** ** This is used to test if a segment is an "appendable" segment. If it @@ -155076,14 +202030,14 @@ static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ if( SQLITE_ROW==sqlite3_step(pCheck) ) bRes = 1; rc = sqlite3_reset(pCheck); } - + *pbRes = bRes; return rc; } /* ** This function is called when initializing an incremental-merge operation. -** It checks if the existing segment with index value iIdx at absolute level +** It checks if the existing segment with index value iIdx at absolute level ** (iAbsLevel+1) can be appended to by the incremental merge. If it can, the ** merge-writer object *pWriter is initialized to write to it. ** @@ -155092,7 +202046,7 @@ static int fts3IsAppendable(Fts3Table *p, sqlite3_int64 iEnd, int *pbRes){ ** * It was initially created as an appendable segment (with all required ** space pre-allocated), and ** -** * The first key read from the input (arguments zKey and nKey) is +** * The first key read from the input (arguments zKey and nKey) is ** greater than the largest key currently stored in the potential ** output segment. */ @@ -155130,6 +202084,10 @@ static int fts3IncrmergeLoad( pWriter->bNoLeafData = (pWriter->nLeafData==0); nRoot = sqlite3_column_bytes(pSelect, 4); aRoot = sqlite3_column_blob(pSelect, 4); + if( aRoot==0 ){ + sqlite3_reset(pSelect); + return nRoot ? SQLITE_NOMEM : FTS_CORRUPT_VTAB; + } }else{ return sqlite3_reset(pSelect); } @@ -155165,6 +202123,10 @@ static int fts3IncrmergeLoad( int i; int nHeight = (int)aRoot[0]; NodeWriter *pNode; + if( nHeight<1 || nHeight>=FTS_MAX_APPENDABLE_HEIGHT ){ + sqlite3_reset(pSelect); + return FTS_CORRUPT_VTAB; + } pWriter->nLeafEst = (int)((iEnd - iStart) + 1)/FTS_MAX_APPENDABLE_HEIGHT; pWriter->iStart = iStart; @@ -155178,34 +202140,46 @@ static int fts3IncrmergeLoad( pNode = &pWriter->aNodeWriter[nHeight]; pNode->iBlock = pWriter->iStart + pWriter->nLeafEst*nHeight; - blobGrowBuffer(&pNode->block, MAX(nRoot, p->nNodeSize), &rc); + blobGrowBuffer(&pNode->block, + MAX(nRoot, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); if( rc==SQLITE_OK ){ memcpy(pNode->block.a, aRoot, nRoot); pNode->block.n = nRoot; + memset(&pNode->block.a[nRoot], 0, FTS3_NODE_PADDING); } for(i=nHeight; i>=0 && rc==SQLITE_OK; i--){ NodeReader reader; + memset(&reader, 0, sizeof(reader)); pNode = &pWriter->aNodeWriter[i]; - rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); - while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); - blobGrowBuffer(&pNode->key, reader.term.n, &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->key.a, reader.term.a, reader.term.n); - pNode->key.n = reader.term.n; - if( i>0 ){ - char *aBlock = 0; - int nBlock = 0; - pNode = &pWriter->aNodeWriter[i-1]; - pNode->iBlock = reader.iChild; - rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock, 0); - blobGrowBuffer(&pNode->block, MAX(nBlock, p->nNodeSize), &rc); - if( rc==SQLITE_OK ){ - memcpy(pNode->block.a, aBlock, nBlock); - pNode->block.n = nBlock; + if( pNode->block.a){ + rc = nodeReaderInit(&reader, pNode->block.a, pNode->block.n); + while( reader.aNode && rc==SQLITE_OK ) rc = nodeReaderNext(&reader); + blobGrowBuffer(&pNode->key, reader.term.n, &rc); + if( rc==SQLITE_OK ){ + assert_fts3_nc( reader.term.n>0 || reader.aNode==0 ); + if( reader.term.n>0 ){ + memcpy(pNode->key.a, reader.term.a, reader.term.n); + } + pNode->key.n = reader.term.n; + if( i>0 ){ + char *aBlock = 0; + int nBlock = 0; + pNode = &pWriter->aNodeWriter[i-1]; + pNode->iBlock = reader.iChild; + rc = sqlite3Fts3ReadBlock(p, reader.iChild, &aBlock, &nBlock,0); + blobGrowBuffer(&pNode->block, + MAX(nBlock, p->nNodeSize)+FTS3_NODE_PADDING, &rc + ); + if( rc==SQLITE_OK ){ + memcpy(pNode->block.a, aBlock, nBlock); + pNode->block.n = nBlock; + memset(&pNode->block.a[nBlock], 0, FTS3_NODE_PADDING); + } + sqlite3_free(aBlock); } - sqlite3_free(aBlock); } } nodeReaderRelease(&reader); @@ -155222,13 +202196,13 @@ static int fts3IncrmergeLoad( /* ** Determine the largest segment index value that exists within absolute ** level iAbsLevel+1. If no error occurs, set *piIdx to this value plus -** one before returning SQLITE_OK. Or, if there are no segments at all +** one before returning SQLITE_OK. Or, if there are no segments at all ** within level iAbsLevel, set *piIdx to zero. ** ** If an error occurs, return an SQLite error code. The final value of ** *piIdx is undefined in this case. */ -static int fts3IncrmergeOutputIdx( +static int fts3IncrmergeOutputIdx( Fts3Table *p, /* FTS Table handle */ sqlite3_int64 iAbsLevel, /* Absolute index of input segments */ int *piIdx /* OUT: Next free index at iAbsLevel+1 */ @@ -155247,7 +202221,7 @@ static int fts3IncrmergeOutputIdx( return rc; } -/* +/* ** Allocate an appendable output segment on absolute level iAbsLevel+1 ** with idx value iIdx. ** @@ -155261,7 +202235,7 @@ static int fts3IncrmergeOutputIdx( ** When an appendable segment is allocated, it is estimated that the ** maximum number of leaf blocks that may be required is the sum of the ** number of leaf blocks consumed by the input segments, plus the number -** of input segments, multiplied by two. This value is stored in stack +** of input segments, multiplied by two. This value is stored in stack ** variable nLeafEst. ** ** A total of 16*nLeafEst blocks are allocated when an appendable segment @@ -155270,10 +202244,10 @@ static int fts3IncrmergeOutputIdx( ** of interior nodes that are parents of the leaf nodes start at block ** (start_block + (1 + end_block - start_block) / 16). And so on. ** -** In the actual code below, the value "16" is replaced with the +** In the actual code below, the value "16" is replaced with the ** pre-processor macro FTS_MAX_APPENDABLE_HEIGHT. */ -static int fts3IncrmergeWriter( +static int fts3IncrmergeWriter( Fts3Table *p, /* Fts3 table handle */ sqlite3_int64 iAbsLevel, /* Absolute level of input segments */ int iIdx, /* Index of new output segment */ @@ -155311,7 +202285,7 @@ static int fts3IncrmergeWriter( if( rc!=SQLITE_OK ) return rc; /* Insert the marker in the %_segments table to make sure nobody tries - ** to steal the space just allocated. This is also used to identify + ** to steal the space just allocated. This is also used to identify ** appendable segments. */ rc = fts3WriteSegment(p, pWriter->iEnd, 0, 0); if( rc!=SQLITE_OK ) return rc; @@ -155328,13 +202302,13 @@ static int fts3IncrmergeWriter( } /* -** Remove an entry from the %_segdir table. This involves running the +** Remove an entry from the %_segdir table. This involves running the ** following two statements: ** ** DELETE FROM %_segdir WHERE level = :iAbsLevel AND idx = :iIdx ** UPDATE %_segdir SET idx = idx - 1 WHERE level = :iAbsLevel AND idx > :iIdx ** -** The DELETE statement removes the specific %_segdir level. The UPDATE +** The DELETE statement removes the specific %_segdir level. The UPDATE ** statement ensures that the remaining segments have contiguously allocated ** idx values. */ @@ -155382,7 +202356,7 @@ static int fts3RepackSegdirLevel( if( nIdx>=nAlloc ){ int *aNew; nAlloc += 16; - aNew = sqlite3_realloc(aIdx, nAlloc*sizeof(int)); + aNew = sqlite3_realloc64(aIdx, nAlloc*sizeof(int)); if( !aNew ){ rc = SQLITE_NOMEM; break; @@ -155448,7 +202422,10 @@ static int fts3TruncateNode( NodeReader reader; /* Reader object */ Blob prev = {0, 0, 0}; /* Previous term written to new node */ int rc = SQLITE_OK; /* Return code */ - int bLeaf = aNode[0]=='\0'; /* True for a leaf node */ + int bLeaf; /* True for a leaf node */ + + if( nNode<1 ) return FTS_CORRUPT_VTAB; + bLeaf = aNode[0]=='\0'; /* Allocate required output space */ blobGrowBuffer(pNew, nNode, &rc); @@ -155456,8 +202433,8 @@ static int fts3TruncateNode( pNew->n = 0; /* Populate new node buffer */ - for(rc = nodeReaderInit(&reader, aNode, nNode); - rc==SQLITE_OK && reader.aNode; + for(rc = nodeReaderInit(&reader, aNode, nNode); + rc==SQLITE_OK && reader.aNode; rc = nodeReaderNext(&reader) ){ if( pNew->n==0 ){ @@ -155484,7 +202461,7 @@ static int fts3TruncateNode( } /* -** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute +** Remove all terms smaller than zTerm/nTerm from segment iIdx in absolute ** level iAbsLevel. This may involve deleting entries from the %_segments ** table, and modifying existing entries in both the %_segments and %_segdir ** tables. @@ -155559,6 +202536,7 @@ static int fts3TruncateSegment( sqlite3_bind_int(pChomp, 4, iIdx); sqlite3_step(pChomp); rc = sqlite3_reset(pChomp); + sqlite3_bind_null(pChomp, 2); } } @@ -155607,9 +202585,9 @@ static int fts3IncrmergeChomp( } *pnRem = 0; }else{ - /* The incremental merge did not copy all the data from this + /* The incremental merge did not copy all the data from this ** segment to the upper level. The segment is modified in place - ** so that it contains no keys smaller than zTerm/nTerm. */ + ** so that it contains no keys smaller than zTerm/nTerm. */ const char *zTerm = pSeg->zTerm; int nTerm = pSeg->nTerm; rc = fts3TruncateSegment(p, iAbsLevel, pSeg->iIdx, zTerm, nTerm); @@ -155638,13 +202616,14 @@ static int fts3IncrmergeHintStore(Fts3Table *p, Blob *pHint){ sqlite3_bind_blob(pReplace, 2, pHint->a, pHint->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } return rc; } /* -** Load an incr-merge hint from the database. The incr-merge hint, if one +** Load an incr-merge hint from the database. The incr-merge hint, if one ** exists, is stored in the rowid==1 row of the %_stat table. ** ** If successful, populate blob *pHint with the value read from the %_stat @@ -155666,7 +202645,7 @@ static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ if( aHint ){ blobGrowBuffer(pHint, nHint, &rc); if( rc==SQLITE_OK ){ - memcpy(pHint->a, aHint, nHint); + if( ALWAYS(pHint->a!=0) ) memcpy(pHint->a, aHint, nHint); pHint->n = nHint; } } @@ -155681,7 +202660,7 @@ static int fts3IncrmergeHintLoad(Fts3Table *p, Blob *pHint){ /* ** If *pRc is not SQLITE_OK when this function is called, it is a no-op. ** Otherwise, append an entry to the hint stored in blob *pHint. Each entry -** consists of two varints, the absolute level number of the input segments +** consists of two varints, the absolute level number of the input segments ** and the number of input segments. ** ** If successful, leave *pRc set to SQLITE_OK and return. If an error occurs, @@ -155702,7 +202681,7 @@ static void fts3IncrmergeHintPush( /* ** Read the last entry (most recently pushed) from the hint blob *pHint -** and then remove the entry. Write the two values read to *piAbsLevel and +** and then remove the entry. Write the two values read to *piAbsLevel and ** *pnInput before returning. ** ** If no error occurs, return SQLITE_OK. If the hint blob in *pHint does @@ -155712,13 +202691,17 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ const int nHint = pHint->n; int i; - i = pHint->n-2; + i = pHint->n-1; + if( (pHint->a[i] & 0x80) ) return FTS_CORRUPT_VTAB; while( i>0 && (pHint->a[i-1] & 0x80) ) i--; + if( i==0 ) return FTS_CORRUPT_VTAB; + i--; while( i>0 && (pHint->a[i-1] & 0x80) ) i--; pHint->n = i; i += sqlite3Fts3GetVarint(&pHint->a[i], piAbsLevel); i += fts3GetVarint32(&pHint->a[i], pnInput); + assert( i<=nHint ); if( i!=nHint ) return FTS_CORRUPT_VTAB; return SQLITE_OK; @@ -155728,10 +202711,10 @@ static int fts3IncrmergeHintPop(Blob *pHint, i64 *piAbsLevel, int *pnInput){ /* ** Attempt an incremental merge that writes nMerge leaf blocks. ** -** Incremental merges happen nMin segments at a time. The segments -** to be merged are the nMin oldest segments (the ones with the smallest -** values for the _segdir.idx field) in the highest level that contains -** at least nMin segments. Multiple merges might occur in an attempt to +** Incremental merges happen nMin segments at a time. The segments +** to be merged are the nMin oldest segments (the ones with the smallest +** values for the _segdir.idx field) in the highest level that contains +** at least nMin segments. Multiple merges might occur in an attempt to ** write the quota of nMerge leaf blocks. */ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ @@ -155747,7 +202730,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Allocate space for the cursor, filter and writer objects */ const int nAlloc = sizeof(*pCsr) + sizeof(*pFilter) + sizeof(*pWriter); - pWriter = (IncrmergeWriter *)sqlite3_malloc(nAlloc); + pWriter = (IncrmergeWriter *)sqlite3_malloc64(nAlloc); if( !pWriter ) return SQLITE_NOMEM; pFilter = (Fts3SegFilter *)&pWriter[1]; pCsr = (Fts3MultiSegReader *)&pFilter[1]; @@ -155762,7 +202745,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* Search the %_segdir table for the absolute level with the smallest ** relative level number that contains at least nMin segments, if any. ** If one is found, set iAbsLevel to the absolute level number and - ** nSeg to nMin. If no level with at least nMin segments can be found, + ** nSeg to nMin. If no level with at least nMin segments can be found, ** set nSeg to -1. */ rc = fts3SqlStmt(p, SQL_FIND_MERGE_LEVEL, &pFindLevel, 0); @@ -155778,7 +202761,7 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* If the hint read from the %_stat table is not empty, check if the ** last entry in it specifies a relative level smaller than or equal - ** to the level identified by the block above (if any). If so, this + ** to the level identified by the block above (if any). If so, this ** iteration of the loop will work on merging at the hinted level. */ if( rc==SQLITE_OK && hint.n ){ @@ -155788,8 +202771,14 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ rc = fts3IncrmergeHintPop(&hint, &iHintAbsLevel, &nHintSeg); if( nSeg<0 || (iAbsLevel % nMod) >= (iHintAbsLevel % nMod) ){ + /* Based on the scan in the block above, it is known that there + ** are no levels with a relative level smaller than that of + ** iAbsLevel with more than nSeg segments, or if nSeg is -1, + ** no levels with more than nMin segments. Use this to limit the + ** value of nHintSeg to avoid a large memory allocation in case the + ** merge-hint is corrupt*/ iAbsLevel = iHintAbsLevel; - nSeg = nHintSeg; + nSeg = MIN(MAX(nMin,nSeg), nHintSeg); bUseHint = 1; bDirtyHint = 1; }else{ @@ -155802,13 +202791,19 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ /* If nSeg is less that zero, then there is no level with at least ** nMin segments and no hint in the %_stat table. No work to do. ** Exit early in this case. */ - if( nSeg<0 ) break; + if( nSeg<=0 ) break; - /* Open a cursor to iterate through the contents of the oldest nSeg - ** indexes of absolute level iAbsLevel. If this cursor is opened using + assert( nMod<=0x7FFFFFFF ); + if( iAbsLevel<0 || iAbsLevel>(nMod<<32) ){ + rc = FTS_CORRUPT_VTAB; + break; + } + + /* Open a cursor to iterate through the contents of the oldest nSeg + ** indexes of absolute level iAbsLevel. If this cursor is opened using ** the 'hint' parameters, it is possible that there are less than nSeg ** segments available in level iAbsLevel. In this case, no work is - ** done on iAbsLevel - fall through to the next iteration of the loop + ** done on iAbsLevel - fall through to the next iteration of the loop ** to start work on some other level. */ memset(pWriter, 0, nAlloc); pFilter->flags = FTS3_SEGMENT_REQUIRE_POS; @@ -155830,8 +202825,15 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ } if( SQLITE_OK==rc && pCsr->nSegment==nSeg && SQLITE_OK==(rc = sqlite3Fts3SegReaderStart(p, pCsr, pFilter)) - && SQLITE_ROW==(rc = sqlite3Fts3SegReaderStep(p, pCsr)) ){ + int bEmpty = 0; + rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( rc==SQLITE_OK ){ + bEmpty = 1; + }else if( rc!=SQLITE_ROW ){ + sqlite3Fts3SegReaderFinish(pCsr); + break; + } if( bUseHint && iIdx>0 ){ const char *zKey = pCsr->zTerm; int nKey = pCsr->nTerm; @@ -155842,11 +202844,13 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ if( rc==SQLITE_OK && pWriter->nLeafEst ){ fts3LogMerge(nSeg, iAbsLevel); - do { - rc = fts3IncrmergeAppend(p, pWriter, pCsr); - if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); - if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; - }while( rc==SQLITE_ROW ); + if( bEmpty==0 ){ + do { + rc = fts3IncrmergeAppend(p, pWriter, pCsr); + if( rc==SQLITE_OK ) rc = sqlite3Fts3SegReaderStep(p, pCsr); + if( pWriter->nWork>=nRem && rc==SQLITE_ROW ) rc = SQLITE_OK; + }while( rc==SQLITE_ROW ); + } /* Update or delete the input segments */ if( rc==SQLITE_OK ){ @@ -155885,11 +202889,14 @@ SQLITE_PRIVATE int sqlite3Fts3Incrmerge(Fts3Table *p, int nMerge, int nMin){ ** Convert the text beginning at *pz into an integer and return ** its value. Advance *pz to point to the first character past ** the integer. +** +** This function used for parameters to merge= and incrmerge= +** commands. */ static int fts3Getint(const char **pz){ const char *z = *pz; int i = 0; - while( (*z)>='0' && (*z)<='9' ) i = 10*i + *(z++) - '0'; + while( (*z)>='0' && (*z)<='9' && i<214748363 ) i = 10*i + *(z++) - '0'; *pz = z; return i; } @@ -155908,7 +202915,7 @@ static int fts3DoIncrmerge( const char *zParam /* Nul-terminated string containing "A,B" */ ){ int rc; - int nMin = (FTS3_MERGE_COUNT / 2); + int nMin = (MergeCount(p) / 2); int nMerge = 0; const char *z = zParam; @@ -155953,7 +202960,7 @@ static int fts3DoAutoincrmerge( int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; p->nAutoincrmerge = fts3Getint(&zParam); - if( p->nAutoincrmerge==1 || p->nAutoincrmerge>FTS3_MERGE_COUNT ){ + if( p->nAutoincrmerge==1 || p->nAutoincrmerge>MergeCount(p) ){ p->nAutoincrmerge = 8; } if( !p->bHasStat ){ @@ -156015,7 +203022,7 @@ static u64 fts3ChecksumIndex( int rc; u64 cksum = 0; - assert( *pRc==SQLITE_OK ); + if( *pRc ) return 0; memset(&filter, 0, sizeof(filter)); memset(&csr, 0, sizeof(csr)); @@ -156036,12 +203043,12 @@ static u64 fts3ChecksumIndex( i64 iDocid = 0; i64 iCol = 0; - i64 iPos = 0; + u64 iPos = 0; pCsr += sqlite3Fts3GetVarint(pCsr, &iDocid); while( pCsrbDescIdx ){ + iDocid = (i64)((u64)iDocid - iVal); + }else{ + iDocid = (i64)((u64)iDocid + iVal); + } } }else{ iPos += (iVal - 2); @@ -156075,10 +203086,10 @@ static u64 fts3ChecksumIndex( ** to true and return SQLITE_OK. Or if the contents do not match, set *pbOk ** to false before returning. ** -** If an error occurs (e.g. an OOM or IO error), return an SQLite error +** If an error occurs (e.g. an OOM or IO error), return an SQLite error ** code. The final value of *pbOk is undefined in this case. */ -static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ +SQLITE_PRIVATE int sqlite3Fts3IntegrityCheck(Fts3Table *p, int *pbOk){ int rc = SQLITE_OK; /* Return code */ u64 cksum1 = 0; /* Checksum based on FTS index contents */ u64 cksum2 = 0; /* Checksum based on %_content contents */ @@ -156106,7 +203117,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_tokenizer_module const *pModule = p->pTokenizer->pModule; sqlite3_stmt *pStmt = 0; char *zSql; - + zSql = sqlite3_mprintf("SELECT %s" , p->zReadExprlist); if( !zSql ){ rc = SQLITE_NOMEM; @@ -156123,10 +203134,9 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ for(iCol=0; rc==SQLITE_OK && iColnColumn; iCol++){ if( p->abNotindexed[iCol]==0 ){ const char *zText = (const char *)sqlite3_column_text(pStmt, iCol+1); - int nText = sqlite3_column_bytes(pStmt, iCol+1); sqlite3_tokenizer_cursor *pT = 0; - rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, nText,&pT); + rc = sqlite3Fts3OpenTokenizer(p->pTokenizer, iLang, zText, -1, &pT); while( rc==SQLITE_OK ){ char const *zToken; /* Buffer containing token */ int nToken = 0; /* Number of bytes in token */ @@ -156157,7 +203167,12 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ sqlite3_finalize(pStmt); } - *pbOk = (cksum1==cksum2); + if( rc==SQLITE_CORRUPT_VTAB ){ + rc = SQLITE_OK; + *pbOk = 0; + }else{ + *pbOk = (rc==SQLITE_OK && cksum1==cksum2); + } return rc; } @@ -156166,7 +203181,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ ** the FTS index are correct, return SQLITE_OK. Or, if the contents of the ** FTS index are incorrect, return SQLITE_CORRUPT_VTAB. ** -** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite +** Or, if an error (e.g. an OOM or IO error) occurs, return an SQLite ** error code. ** ** The integrity-check works as follows. For each token and indexed token @@ -156175,7 +203190,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ ** ** + The index number (0 for the main index, 1 for the first prefix ** index etc.), -** + The token (or token prefix) text itself, +** + The token (or token prefix) text itself, ** + The language-id of the row it appears in, ** + The docid of the row it appears in, ** + The column it appears in, and @@ -156186,7 +203201,7 @@ static int fts3IntegrityCheck(Fts3Table *p, int *pbOk){ ** ** The integrity-check code calculates the same checksum in two ways: ** -** 1. By scanning the contents of the FTS index, and +** 1. By scanning the contents of the FTS index, and ** 2. By scanning and tokenizing the content table. ** ** If the two checksums are identical, the integrity-check is deemed to have @@ -156197,7 +203212,7 @@ static int fts3DoIntegrityCheck( ){ int rc; int bOk = 0; - rc = fts3IntegrityCheck(p, &bOk); + rc = sqlite3Fts3IntegrityCheck(p, &bOk); if( rc==SQLITE_OK && bOk==0 ) rc = FTS_CORRUPT_VTAB; return rc; } @@ -156207,11 +203222,11 @@ static int fts3DoIntegrityCheck( ** ** "INSERT INTO tbl(tbl) VALUES()" ** -** Argument pVal contains the result of . Currently the only +** Argument pVal contains the result of . Currently the only ** meaningful value to insert is the text 'optimize'. */ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ - int rc; /* Return Code */ + int rc = SQLITE_ERROR; /* Return Code */ const char *zVal = (const char *)sqlite3_value_text(pVal); int nVal = sqlite3_value_bytes(pVal); @@ -156227,21 +203242,30 @@ static int fts3SpecialInsert(Fts3Table *p, sqlite3_value *pVal){ rc = fts3DoIncrmerge(p, &zVal[6]); }else if( nVal>10 && 0==sqlite3_strnicmp(zVal, "automerge=", 10) ){ rc = fts3DoAutoincrmerge(p, &zVal[10]); -#ifdef SQLITE_TEST - }else if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ - p->nNodeSize = atoi(&zVal[9]); - rc = SQLITE_OK; - }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ - p->nMaxPendingData = atoi(&zVal[11]); - rc = SQLITE_OK; - }else if( nVal>21 && 0==sqlite3_strnicmp(zVal, "test-no-incr-doclist=", 21) ){ - p->bNoIncrDoclist = atoi(&zVal[21]); - rc = SQLITE_OK; -#endif - }else{ - rc = SQLITE_ERROR; + }else if( nVal==5 && 0==sqlite3_strnicmp(zVal, "flush", 5) ){ + rc = sqlite3Fts3PendingTermsFlush(p); } - +#if defined(SQLITE_DEBUG) || defined(SQLITE_TEST) + else{ + int v; + if( nVal>9 && 0==sqlite3_strnicmp(zVal, "nodesize=", 9) ){ + v = atoi(&zVal[9]); + if( v>=24 && v<=p->nPgsz-35 ) p->nNodeSize = v; + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal, "maxpending=", 9) ){ + v = atoi(&zVal[11]); + if( v>=64 && v<=FTS3_MAX_PENDING_DATA ) p->nMaxPendingData = v; + rc = SQLITE_OK; + }else if( nVal>21 && 0==sqlite3_strnicmp(zVal,"test-no-incr-doclist=",21) ){ + p->bNoIncrDoclist = atoi(&zVal[21]); + rc = SQLITE_OK; + }else if( nVal>11 && 0==sqlite3_strnicmp(zVal,"mergecount=",11) ){ + v = atoi(&zVal[11]); + if( v>=4 && v<=FTS3_MERGE_COUNT && (v&1)==0 ) p->nMergeCount = v; + rc = SQLITE_OK; + } + } +#endif return rc; } @@ -156259,7 +203283,7 @@ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredDoclists(Fts3Cursor *pCsr){ } /* -** Free all entries in the pCsr->pDeffered list. Entries are added to +** Free all entries in the pCsr->pDeffered list. Entries are added to ** this list using sqlite3Fts3DeferToken(). */ SQLITE_PRIVATE void sqlite3Fts3FreeDeferredTokens(Fts3Cursor *pCsr){ @@ -156287,14 +203311,14 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ int i; /* Used to iterate through table columns */ sqlite3_int64 iDocid; /* Docid of the row pCsr points to */ Fts3DeferredToken *pDef; /* Used to iterate through deferred tokens */ - + Fts3Table *p = (Fts3Table *)pCsr->base.pVtab; sqlite3_tokenizer *pT = p->pTokenizer; sqlite3_tokenizer_module const *pModule = pT->pModule; - + assert( pCsr->isRequireSeek==0 ); iDocid = sqlite3_column_int64(pCsr->pStmt, 0); - + for(i=0; inColumn && rc==SQLITE_OK; i++){ if( p->abNotindexed[i]==0 ){ const char *zText = (const char *)sqlite3_column_text(pCsr->pStmt, i+1); @@ -156335,8 +203359,8 @@ SQLITE_PRIVATE int sqlite3Fts3CacheDeferredDoclists(Fts3Cursor *pCsr){ } SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( - Fts3DeferredToken *p, - char **ppData, + Fts3DeferredToken *p, + char **ppData, int *pnData ){ char *pRet; @@ -156350,13 +203374,13 @@ SQLITE_PRIVATE int sqlite3Fts3DeferredTokenList( return SQLITE_OK; } - pRet = (char *)sqlite3_malloc(p->pList->nData); + pRet = (char *)sqlite3_malloc64(p->pList->nData); if( !pRet ) return SQLITE_NOMEM; nSkip = sqlite3Fts3GetVarint(p->pList->aData, &dummy); *pnData = p->pList->nData - nSkip; *ppData = pRet; - + memcpy(pRet, &p->pList->aData[nSkip], *pnData); return SQLITE_OK; } @@ -156370,13 +203394,13 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( int iCol /* Column that token must appear in (or -1) */ ){ Fts3DeferredToken *pDeferred; - pDeferred = sqlite3_malloc(sizeof(*pDeferred)); + pDeferred = sqlite3_malloc64(sizeof(*pDeferred)); if( !pDeferred ){ return SQLITE_NOMEM; } memset(pDeferred, 0, sizeof(*pDeferred)); pDeferred->pToken = pToken; - pDeferred->pNext = pCsr->pDeferred; + pDeferred->pNext = pCsr->pDeferred; pDeferred->iCol = iCol; pCsr->pDeferred = pDeferred; @@ -156393,8 +203417,8 @@ SQLITE_PRIVATE int sqlite3Fts3DeferToken( ** of subsiduary data structures accordingly. */ static int fts3DeleteByRowid( - Fts3Table *p, - sqlite3_value *pRowid, + Fts3Table *p, + sqlite3_value *pRowid, int *pnChng, /* IN/OUT: Decrement if row is deleted */ u32 *aSzDel ){ @@ -156432,14 +203456,14 @@ static int fts3DeleteByRowid( ** This function does the work for the xUpdate method of FTS3 virtual ** tables. The schema of the virtual table being: ** -** CREATE TABLE
            ( +** CREATE TABLE
            ( ** , -**
            HIDDEN, -** docid HIDDEN, +**
            HIDDEN, +** docid HIDDEN, ** HIDDEN ** ); ** -** +** */ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( sqlite3_vtab *pVtab, /* FTS3 vtab object */ @@ -156449,7 +203473,6 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( ){ Fts3Table *p = (Fts3Table *)pVtab; int rc = SQLITE_OK; /* Return Code */ - int isRemove = 0; /* True for an UPDATE or DELETE */ u32 *aSzIns = 0; /* Sizes of inserted documents */ u32 *aSzDel = 0; /* Sizes of deleted documents */ int nChng = 0; /* Net change in number of documents */ @@ -156460,7 +203483,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( assert( p->bHasStat==0 || p->bHasStat==1 ); assert( p->pSegments==0 ); - assert( + assert( nArg==1 /* DELETE operations */ || nArg==(2 + p->nColumn + 3) /* INSERT or UPDATE operations */ ); @@ -156469,9 +203492,9 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( ** ** INSERT INTO xyz(xyz) VALUES('command'); */ - if( nArg>1 - && sqlite3_value_type(apVal[0])==SQLITE_NULL - && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL + if( nArg>1 + && sqlite3_value_type(apVal[0])==SQLITE_NULL + && sqlite3_value_type(apVal[p->nColumn+2])!=SQLITE_NULL ){ rc = fts3SpecialInsert(p, apVal[p->nColumn+2]); goto update_out; @@ -156483,7 +203506,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( } /* Allocate space to hold the change in document sizes */ - aSzDel = sqlite3_malloc( sizeof(aSzDel[0])*(p->nColumn+1)*2 ); + aSzDel = sqlite3_malloc64(sizeof(aSzDel[0])*((sqlite3_int64)p->nColumn+1)*2); if( aSzDel==0 ){ rc = SQLITE_NOMEM; goto update_out; @@ -156510,24 +203533,24 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( pNewRowid = apVal[1]; } - if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( + if( sqlite3_value_type(pNewRowid)!=SQLITE_NULL && ( sqlite3_value_type(apVal[0])==SQLITE_NULL || sqlite3_value_int64(apVal[0])!=sqlite3_value_int64(pNewRowid) )){ /* The new rowid is not NULL (in this case the rowid will be - ** automatically assigned and there is no chance of a conflict), and + ** automatically assigned and there is no chance of a conflict), and ** the statement is either an INSERT or an UPDATE that modifies the ** rowid column. So if the conflict mode is REPLACE, then delete any - ** existing row with rowid=pNewRowid. + ** existing row with rowid=pNewRowid. ** - ** Or, if the conflict mode is not REPLACE, insert the new record into + ** Or, if the conflict mode is not REPLACE, insert the new record into ** the %_content table. If we hit the duplicate rowid constraint (or any ** other error) while doing so, return immediately. ** ** This branch may also run if pNewRowid contains a value that cannot - ** be losslessly converted to an integer. In this case, the eventual + ** be losslessly converted to an integer. In this case, the eventual ** call to fts3InsertData() (either just below or further on in this - ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is + ** function) will return SQLITE_MISMATCH. If fts3DeleteByRowid is ** invoked, it will delete zero rows (since no row will have ** docid=$pNewRowid if $pNewRowid is not an integer value). */ @@ -156547,9 +203570,8 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( if( sqlite3_value_type(apVal[0])!=SQLITE_NULL ){ assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER ); rc = fts3DeleteByRowid(p, apVal[0], &nChng, aSzDel); - isRemove = 1; } - + /* If this is an INSERT or UPDATE operation, insert the new record. */ if( nArg>1 && rc==SQLITE_OK ){ int iLangid = sqlite3_value_int(apVal[2 + p->nColumn + 2]); @@ -156559,7 +203581,7 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( rc = FTS_CORRUPT_VTAB; } } - if( rc==SQLITE_OK && (!isRemove || *pRowid!=p->iPrevDocid ) ){ + if( rc==SQLITE_OK ){ rc = fts3PendingTermsDocid(p, 0, iLangid, *pRowid); } if( rc==SQLITE_OK ){ @@ -156582,10 +203604,10 @@ SQLITE_PRIVATE int sqlite3Fts3UpdateMethod( return rc; } -/* +/* ** Flush any data in the pending-terms hash table to disk. If successful, -** merge all segments in the database (including the new segment, if -** there was any data to flush) into a single segment. +** merge all segments in the database (including the new segment, if +** there was any data to flush) into a single segment. */ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ int rc; @@ -156627,6 +203649,10 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ /* #include */ /* #include */ +#ifndef SQLITE_AMALGAMATION +typedef sqlite3_int64 i64; +#endif + /* ** Characters that may appear in the second argument to matchinfo(). */ @@ -156641,13 +203667,13 @@ SQLITE_PRIVATE int sqlite3Fts3Optimize(Fts3Table *p){ #define FTS3_MATCHINFO_LHITS_BM 'b' /* nCol*nPhrase values */ /* -** The default value for the second argument to matchinfo(). +** The default value for the second argument to matchinfo(). */ #define FTS3_MATCHINFO_DEFAULT "pcx" /* -** Used as an fts3ExprIterate() context when loading phrase doclists to +** Used as an sqlite3Fts3ExprIterate() context when loading phrase doclists to ** Fts3Expr.aDoclist[]/nDoclist. */ typedef struct LoadDoclistCtx LoadDoclistCtx; @@ -156658,7 +203684,7 @@ struct LoadDoclistCtx { }; /* -** The following types are used as part of the implementation of the +** The following types are used as part of the implementation of the ** fts3BestSnippet() routine. */ typedef struct SnippetIter SnippetIter; @@ -156677,9 +203703,9 @@ struct SnippetIter { struct SnippetPhrase { int nToken; /* Number of tokens in phrase */ char *pList; /* Pointer to start of phrase position list */ - int iHead; /* Next value in position list */ + i64 iHead; /* Next value in position list */ char *pHead; /* Position list data following iHead */ - int iTail; /* Next value in trailing position list */ + i64 iTail; /* Next value in trailing position list */ char *pTail; /* Position list data following iTail */ }; @@ -156691,7 +203717,7 @@ struct SnippetFragment { }; /* -** This type is used as an fts3ExprIterate() context object while +** This type is used as an sqlite3Fts3ExprIterate() context object while ** accumulating the data returned by the matchinfo() function. */ typedef struct MatchInfo MatchInfo; @@ -156738,17 +203764,18 @@ struct StrBuffer { /* ** Allocate a two-slot MatchinfoBuffer object. */ -static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ +static MatchinfoBuffer *fts3MIBufferNew(size_t nElem, const char *zMatchinfo){ MatchinfoBuffer *pRet; - int nByte = sizeof(u32) * (2*nElem + 1) + sizeof(MatchinfoBuffer); - int nStr = (int)strlen(zMatchinfo); + sqlite3_int64 nByte = sizeof(u32) * (2*(sqlite3_int64)nElem + 1) + + sizeof(MatchinfoBuffer); + sqlite3_int64 nStr = strlen(zMatchinfo); - pRet = sqlite3_malloc(nByte + nStr+1); + pRet = sqlite3Fts3MallocZero(nByte + nStr+1); if( pRet ){ - memset(pRet, 0, nByte); pRet->aMatchinfo[0] = (u8*)(&pRet->aMatchinfo[1]) - (u8*)pRet; - pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + sizeof(u32)*(nElem+1); - pRet->nElem = nElem; + pRet->aMatchinfo[1+nElem] = pRet->aMatchinfo[0] + + sizeof(u32)*((int)nElem+1); + pRet->nElem = (int)nElem; pRet->zMatchinfo = ((char*)pRet) + nByte; memcpy(pRet->zMatchinfo, zMatchinfo, nStr+1); pRet->aRef[0] = 1; @@ -156760,8 +203787,8 @@ static MatchinfoBuffer *fts3MIBufferNew(int nElem, const char *zMatchinfo){ static void fts3MIBufferFree(void *p){ MatchinfoBuffer *pBuf = (MatchinfoBuffer*)((u8*)p - ((u32*)p)[-1]); - assert( (u32*)p==&pBuf->aMatchinfo[1] - || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] + assert( (u32*)p==&pBuf->aMatchinfo[1] + || (u32*)p==&pBuf->aMatchinfo[pBuf->nElem+2] ); if( (u32*)p==&pBuf->aMatchinfo[1] ){ pBuf->aRef[1] = 0; @@ -156788,7 +203815,7 @@ static void (*fts3MIBufferAlloc(MatchinfoBuffer *p, u32 **paOut))(void*){ aOut = &p->aMatchinfo[p->nElem+2]; xRet = fts3MIBufferFree; }else{ - aOut = (u32*)sqlite3_malloc(p->nElem * sizeof(u32)); + aOut = (u32*)sqlite3_malloc64(p->nElem * sizeof(u32)); if( aOut ){ xRet = sqlite3_free; if( p->bGlobal ) memcpy(aOut, &p->aMatchinfo[1], p->nElem*sizeof(u32)); @@ -156817,7 +203844,7 @@ SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ } } -/* +/* ** End of MatchinfoBuffer code. *************************************************************************/ @@ -156842,14 +203869,14 @@ SQLITE_PRIVATE void sqlite3Fts3MIBufferFree(MatchinfoBuffer *p){ ** After it returns, *piPos contains the value of the next element of the ** list and *pp is advanced to the following varint. */ -static void fts3GetDeltaPosition(char **pp, int *piPos){ +static void fts3GetDeltaPosition(char **pp, i64 *piPos){ int iVal; *pp += fts3GetVarint32(*pp, &iVal); *piPos += (iVal-2); } /* -** Helper function for fts3ExprIterate() (see below). +** Helper function for sqlite3Fts3ExprIterate() (see below). */ static int fts3ExprIterate2( Fts3Expr *pExpr, /* Expression to iterate phrases of */ @@ -156878,12 +203905,12 @@ static int fts3ExprIterate2( ** are part of a sub-tree that is the right-hand-side of a NOT operator. ** For each phrase node found, the supplied callback function is invoked. ** -** If the callback function returns anything other than SQLITE_OK, +** If the callback function returns anything other than SQLITE_OK, ** the iteration is abandoned and the error code returned immediately. ** Otherwise, SQLITE_OK is returned after a callback has been made for ** all eligible phrase nodes. */ -static int fts3ExprIterate( +SQLITE_PRIVATE int sqlite3Fts3ExprIterate( Fts3Expr *pExpr, /* Expression to iterate phrases of */ int (*x)(Fts3Expr*,int,void*), /* Callback function to invoke for phrases */ void *pCtx /* Second argument to pass to callback */ @@ -156892,10 +203919,9 @@ static int fts3ExprIterate( return fts3ExprIterate2(pExpr, &iPhrase, x, pCtx); } - /* -** This is an fts3ExprIterate() callback used while loading the doclists -** for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also +** This is an sqlite3Fts3ExprIterate() callback used while loading the +** doclists for each phrase into Fts3Expr.aDoclist[]/nDoclist. See also ** fts3ExprLoadDoclists(). */ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ @@ -156913,11 +203939,11 @@ static int fts3ExprLoadDoclistsCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ /* ** Load the doclists for each phrase in the query associated with FTS3 cursor -** pCsr. +** pCsr. ** -** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable -** phrases in the expression (all phrases except those directly or -** indirectly descended from the right-hand-side of a NOT operator). If +** If pnPhrase is not NULL, then *pnPhrase is set to the number of matchable +** phrases in the expression (all phrases except those directly or +** indirectly descended from the right-hand-side of a NOT operator). If ** pnToken is not NULL, then it is set to the number of tokens in all ** matchable phrases of the expression. */ @@ -156927,9 +203953,9 @@ static int fts3ExprLoadDoclists( int *pnToken /* OUT: Number of tokens in query */ ){ int rc; /* Return Code */ - LoadDoclistCtx sCtx = {0,0,0}; /* Context for fts3ExprIterate() */ + LoadDoclistCtx sCtx = {0,0,0}; /* Context for sqlite3Fts3ExprIterate() */ sCtx.pCsr = pCsr; - rc = fts3ExprIterate(pCsr->pExpr, fts3ExprLoadDoclistsCb, (void *)&sCtx); + rc = sqlite3Fts3ExprIterate(pCsr->pExpr,fts3ExprLoadDoclistsCb,(void*)&sCtx); if( pnPhrase ) *pnPhrase = sCtx.nPhrase; if( pnToken ) *pnToken = sCtx.nToken; return rc; @@ -156942,19 +203968,19 @@ static int fts3ExprPhraseCountCb(Fts3Expr *pExpr, int iPhrase, void *ctx){ } static int fts3ExprPhraseCount(Fts3Expr *pExpr){ int nPhrase = 0; - (void)fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprPhraseCountCb, (void *)&nPhrase); return nPhrase; } /* -** Advance the position list iterator specified by the first two +** Advance the position list iterator specified by the first two ** arguments so that it points to the first element with a value greater ** than or equal to parameter iNext. */ -static void fts3SnippetAdvance(char **ppIter, int *piIter, int iNext){ +static void fts3SnippetAdvance(char **ppIter, i64 *piIter, int iNext){ char *pIter = *ppIter; if( pIter ){ - int iIter = *piIter; + i64 iIter = *piIter; while( iIternSnippet>=0 ); pIter->iCurrent = iStart = iEnd - pIter->nSnippet + 1; for(i=0; inPhrase; i++){ SnippetPhrase *pPhrase = &pIter->aPhrase[i]; @@ -157016,7 +204043,7 @@ static int fts3SnippetNextCandidate(SnippetIter *pIter){ } /* -** Retrieve information about the current candidate snippet of snippet +** Retrieve information about the current candidate snippet of snippet ** iterator pIter. */ static void fts3SnippetDetails( @@ -157037,13 +204064,14 @@ static void fts3SnippetDetails( SnippetPhrase *pPhrase = &pIter->aPhrase[i]; if( pPhrase->pTail ){ char *pCsr = pPhrase->pTail; - int iCsr = pPhrase->iTail; + i64 iCsr = pPhrase->iTail; - while( iCsr<(iStart+pIter->nSnippet) ){ + while( iCsr<(iStart+pIter->nSnippet) && iCsr>=iStart ){ int j; - u64 mPhrase = (u64)1 << i; + u64 mPhrase = (u64)1 << (i%64); u64 mPos = (u64)1 << (iCsr - iStart); - assert( iCsr>=iStart ); + assert( iCsr>=iStart && (iCsr - iStart)<=64 ); + assert( i>=0 ); if( (mCover|mCovered)&mPhrase ){ iScore++; }else{ @@ -157051,7 +204079,7 @@ static void fts3SnippetDetails( } mCover |= mPhrase; - for(j=0; jnToken; j++){ + for(j=0; jnToken && jnSnippet; j++){ mHighlight |= (mPos>>j); } @@ -157069,8 +204097,9 @@ static void fts3SnippetDetails( } /* -** This function is an fts3ExprIterate() callback used by fts3BestSnippet(). -** Each invocation populates an element of the SnippetIter.aPhrase[] array. +** This function is an sqlite3Fts3ExprIterate() callback used by +** fts3BestSnippet(). Each invocation populates an element of the +** SnippetIter.aPhrase[] array. */ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ SnippetIter *p = (SnippetIter *)ctx; @@ -157082,17 +204111,20 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ rc = sqlite3Fts3EvalPhrasePoslist(p->pCsr, pExpr, p->iCol, &pCsr); assert( rc==SQLITE_OK || pCsr==0 ); if( pCsr ){ - int iFirst = 0; + i64 iFirst = 0; pPhrase->pList = pCsr; fts3GetDeltaPosition(&pCsr, &iFirst); - assert( iFirst>=0 ); - pPhrase->pHead = pCsr; - pPhrase->pTail = pCsr; - pPhrase->iHead = iFirst; - pPhrase->iTail = iFirst; + if( iFirst<0 ){ + rc = FTS_CORRUPT_VTAB; + }else{ + pPhrase->pHead = pCsr; + pPhrase->pTail = pCsr; + pPhrase->iHead = iFirst; + pPhrase->iTail = iFirst; + } }else{ assert( rc!=SQLITE_OK || ( - pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 + pPhrase->pList==0 && pPhrase->pHead==0 && pPhrase->pTail==0 )); } @@ -157100,14 +204132,14 @@ static int fts3SnippetFindPositions(Fts3Expr *pExpr, int iPhrase, void *ctx){ } /* -** Select the fragment of text consisting of nFragment contiguous tokens +** Select the fragment of text consisting of nFragment contiguous tokens ** from column iCol that represent the "best" snippet. The best snippet ** is the snippet with the highest score, where scores are calculated ** by adding: ** ** (a) +1 point for each occurrence of a matchable phrase in the snippet. ** -** (b) +1000 points for the first occurrence of each matchable phrase in +** (b) +1000 points for the first occurrence of each matchable phrase in ** the snippet for which the corresponding mCovered bit is not set. ** ** The selected snippet parameters are stored in structure *pFragment before @@ -157126,7 +204158,7 @@ static int fts3BestSnippet( int rc; /* Return Code */ int nList; /* Number of phrases in expression */ SnippetIter sIter; /* Iterates through snippet candidates */ - int nByte; /* Number of bytes of space to allocate */ + sqlite3_int64 nByte; /* Number of bytes of space to allocate */ int iBestScore = -1; /* Best snippet score found so far */ int i; /* Loop counter */ @@ -157144,11 +204176,10 @@ static int fts3BestSnippet( ** the required space using malloc(). */ nByte = sizeof(SnippetPhrase) * nList; - sIter.aPhrase = (SnippetPhrase *)sqlite3_malloc(nByte); + sIter.aPhrase = (SnippetPhrase *)sqlite3Fts3MallocZero(nByte); if( !sIter.aPhrase ){ return SQLITE_NOMEM; } - memset(sIter.aPhrase, 0, nByte); /* Initialize the contents of the SnippetIter object. Then iterate through ** the set of phrases in the expression to populate the aPhrase[] array. @@ -157158,17 +204189,19 @@ static int fts3BestSnippet( sIter.nSnippet = nSnippet; sIter.nPhrase = nList; sIter.iCurrent = -1; - rc = fts3ExprIterate(pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter); + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3SnippetFindPositions, (void*)&sIter + ); if( rc==SQLITE_OK ){ /* Set the *pmSeen output variable. */ for(i=0; iiCol = iCol; @@ -157214,8 +204247,8 @@ static int fts3StringAppend( ** appended data. */ if( pStr->n+nAppend+1>=pStr->nAlloc ){ - int nAlloc = pStr->nAlloc+nAppend+100; - char *zNew = sqlite3_realloc(pStr->z, nAlloc); + sqlite3_int64 nAlloc = pStr->nAlloc+(sqlite3_int64)nAppend+100; + char *zNew = sqlite3_realloc64(pStr->z, nAlloc); if( !zNew ){ return SQLITE_NOMEM; } @@ -157240,8 +204273,8 @@ static int fts3StringAppend( ** ** ........X.....X ** -** This function "shifts" the beginning of the snippet forward in the -** document so that there are approximately the same number of +** This function "shifts" the beginning of the snippet forward in the +** document so that there are approximately the same number of ** non-highlighted terms to the right of the final highlighted term as there ** are to the left of the first highlighted term. For example, to this: ** @@ -157249,8 +204282,8 @@ static int fts3StringAppend( ** ** This is done as part of extracting the snippet text, not when selecting ** the snippet. Snippet selection is done based on doclists only, so there -** is no way for fts3BestSnippet() to know whether or not the document -** actually contains terms that follow the final highlighted term. +** is no way for fts3BestSnippet() to know whether or not the document +** actually contains terms that follow the final highlighted term. */ static int fts3SnippetShift( Fts3Table *pTab, /* FTS3 table snippet comes from */ @@ -157270,6 +204303,7 @@ static int fts3SnippetShift( for(nLeft=0; !(hlmask & ((u64)1 << nLeft)); nLeft++); for(nRight=0; !(hlmask & ((u64)1 << (nSnippet-1-nRight))); nRight++); + assert( (nSnippet-1-nRight)<=63 && (nSnippet-1-nRight)>=0 ); nDesired = (nLeft-nRight)/2; /* Ideally, the start of the snippet should be pushed forward in the @@ -157339,7 +204373,7 @@ static int fts3SnippetText( int iCol = pFragment->iCol+1; /* Query column to extract text from */ sqlite3_tokenizer_module *pMod; /* Tokenizer module methods object */ sqlite3_tokenizer_cursor *pC; /* Tokenizer cursor open on zDoc/nDoc */ - + zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol); if( zDoc==0 ){ if( sqlite3_column_type(pCsr->pStmt, iCol)!=SQLITE_NULL ){ @@ -157379,7 +204413,7 @@ static int fts3SnippetText( if( rc==SQLITE_DONE ){ /* Special case - the last token of the snippet is also the last token ** of the column. Append any punctuation that occurred between the end - ** of the previous token and the end of the document to the output. + ** of the previous token and the end of the document to the output. ** Then break out of the loop. */ rc = fts3StringAppend(pOut, &zDoc[iEnd], -1); } @@ -157396,7 +204430,7 @@ static int fts3SnippetText( /* Now that the shift has been done, check if the initial "..." are ** required. They are required if (a) this is not the first fragment, - ** or (b) this fragment does not begin at position 0 of its column. + ** or (b) this fragment does not begin at position 0 of its column. */ if( rc==SQLITE_OK ){ if( iPos>0 || iFragment>0 ){ @@ -157432,8 +204466,8 @@ static int fts3SnippetText( /* -** This function is used to count the entries in a column-list (a -** delta-encoded list of term offsets within a single column of a single +** This function is used to count the entries in a column-list (a +** delta-encoded list of term offsets within a single column of a single ** row). When this function is called, *ppCollist should point to the ** beginning of the first varint in the column-list (the varint that ** contains the position of the first matching term in the column data). @@ -157462,7 +204496,7 @@ static int fts3ColumnlistCount(char **ppCollist){ /* ** This function gathers 'y' or 'b' data for a single phrase. */ -static void fts3ExprLHits( +static int fts3ExprLHits( Fts3Expr *pExpr, /* Phrase expression node */ MatchInfo *p /* Matchinfo context */ ){ @@ -157479,7 +204513,7 @@ static void fts3ExprLHits( iStart = pExpr->iPhrase * ((p->nCol + 31) / 32); } - while( 1 ){ + if( pIter ) while( 1 ){ int nHit = fts3ColumnlistCount(&pIter); if( (pPhrase->iColumn>=pTab->nColumn || pPhrase->iColumn==iCol) ){ if( p->flag==FTS3_MATCHINFO_LHITS ){ @@ -157492,34 +204526,38 @@ static void fts3ExprLHits( if( *pIter!=0x01 ) break; pIter++; pIter += fts3GetVarint32(pIter, &iCol); + if( iCol>=p->nCol ) return FTS_CORRUPT_VTAB; } + return SQLITE_OK; } /* ** Gather the results for matchinfo directives 'y' and 'b'. */ -static void fts3ExprLHitGather( +static int fts3ExprLHitGather( Fts3Expr *pExpr, MatchInfo *p ){ + int rc = SQLITE_OK; assert( (pExpr->pLeft==0)==(pExpr->pRight==0) ); if( pExpr->bEof==0 && pExpr->iDocid==p->pCursor->iPrevId ){ if( pExpr->pLeft ){ - fts3ExprLHitGather(pExpr->pLeft, p); - fts3ExprLHitGather(pExpr->pRight, p); + rc = fts3ExprLHitGather(pExpr->pLeft, p); + if( rc==SQLITE_OK ) rc = fts3ExprLHitGather(pExpr->pRight, p); }else{ - fts3ExprLHits(pExpr, p); + rc = fts3ExprLHits(pExpr, p); } } + return rc; } /* -** fts3ExprIterate() callback used to collect the "global" matchinfo stats -** for a single query. +** sqlite3Fts3ExprIterate() callback used to collect the "global" matchinfo +** stats for a single query. ** -** fts3ExprIterate() callback to load the 'global' elements of a -** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements -** of the matchinfo array that are constant for all rows returned by the +** sqlite3Fts3ExprIterate() callback to load the 'global' elements of a +** FTS3_MATCHINFO_HITS matchinfo array. The global stats are those elements +** of the matchinfo array that are constant for all rows returned by the ** current query. ** ** Argument pCtx is actually a pointer to a struct of type MatchInfo. This @@ -157535,7 +204573,7 @@ static void fts3ExprLHitGather( ** at least one instance of phrase iPhrase. ** ** If the phrase pExpr consists entirely of deferred tokens, then all X and -** Y values are set to nDoc, where nDoc is the number of documents in the +** Y values are set to nDoc, where nDoc is the number of documents in the ** file system. This is done because the full-text index doclist is required ** to calculate these values properly, and the full-text index doclist is ** not available for deferred tokens. @@ -157552,8 +204590,8 @@ static int fts3ExprGlobalHitsCb( } /* -** fts3ExprIterate() callback used to collect the "local" part of the -** FTS3_MATCHINFO_HITS array. The local stats are those elements of the +** sqlite3Fts3ExprIterate() callback used to collect the "local" part of the +** FTS3_MATCHINFO_HITS array. The local stats are those elements of the ** array that are different for each row returned by the query. */ static int fts3ExprLocalHitsCb( @@ -157580,7 +204618,7 @@ static int fts3ExprLocalHitsCb( } static int fts3MatchinfoCheck( - Fts3Table *pTab, + Fts3Table *pTab, char cArg, char **pzErr ){ @@ -157600,13 +204638,13 @@ static int fts3MatchinfoCheck( return SQLITE_ERROR; } -static int fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ - int nVal; /* Number of integers output by cArg */ +static size_t fts3MatchinfoSize(MatchInfo *pInfo, char cArg){ + size_t nVal; /* Number of integers output by cArg */ switch( cArg ){ case FTS3_MATCHINFO_NDOC: - case FTS3_MATCHINFO_NPHRASE: - case FTS3_MATCHINFO_NCOL: + case FTS3_MATCHINFO_NPHRASE: + case FTS3_MATCHINFO_NCOL: nVal = 1; break; @@ -157637,11 +204675,15 @@ static int fts3MatchinfoSelectDoctotal( Fts3Table *pTab, sqlite3_stmt **ppStmt, sqlite3_int64 *pnDoc, - const char **paLen + const char **paLen, + const char **ppEnd ){ sqlite3_stmt *pStmt; const char *a; + const char *pEnd; sqlite3_int64 nDoc; + int n; + if( !*ppStmt ){ int rc = sqlite3Fts3SelectDoctotal(pTab, ppStmt); @@ -157650,17 +204692,25 @@ static int fts3MatchinfoSelectDoctotal( pStmt = *ppStmt; assert( sqlite3_data_count(pStmt)==1 ); + n = sqlite3_column_bytes(pStmt, 0); a = sqlite3_column_blob(pStmt, 0); - a += sqlite3Fts3GetVarint(a, &nDoc); - if( nDoc==0 ) return FTS_CORRUPT_VTAB; - *pnDoc = (u32)nDoc; + if( a==0 ){ + return FTS_CORRUPT_VTAB; + } + pEnd = a + n; + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nDoc); + if( nDoc<=0 || a>pEnd ){ + return FTS_CORRUPT_VTAB; + } + *pnDoc = nDoc; if( paLen ) *paLen = a; + if( ppEnd ) *ppEnd = pEnd; return SQLITE_OK; } /* -** An instance of the following structure is used to store state while +** An instance of the following structure is used to store state while ** iterating through a multi-column position-list corresponding to the ** hits for a single phrase on a single row in order to calculate the ** values for a matchinfo() FTS3_MATCHINFO_LCS request. @@ -157673,7 +204723,7 @@ struct LcsIterator { int iPos; /* Current position */ }; -/* +/* ** If LcsIterator.iCol is set to the following value, the iterator has ** finished iterating through all offsets for all columns. */ @@ -157695,10 +204745,12 @@ static int fts3MatchinfoLcsCb( ** position list for the next column. */ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ - char *pRead = pIter->pRead; + char *pRead; sqlite3_int64 iRead; int rc = 0; + if( NEVER(pIter==0) ) return 1; + pRead = pIter->pRead; pRead += sqlite3Fts3GetVarint(pRead, &iRead); if( iRead==0 || iRead==1 ){ pRead = 0; @@ -157710,16 +204762,16 @@ static int fts3LcsIteratorAdvance(LcsIterator *pIter){ pIter->pRead = pRead; return rc; } - + /* -** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. +** This function implements the FTS3_MATCHINFO_LCS matchinfo() flag. ** ** If the call is successful, the longest-common-substring lengths for each -** column are written into the first nCol elements of the pInfo->aMatchinfo[] +** column are written into the first nCol elements of the pInfo->aMatchinfo[] ** array before returning. SQLITE_OK is returned in this case. ** ** Otherwise, if an error occurs, an SQLite error code is returned and the -** data written to the first nCol elements of pInfo->aMatchinfo[] is +** data written to the first nCol elements of pInfo->aMatchinfo[] is ** undefined. */ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ @@ -157727,14 +204779,14 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int i; int iCol; int nToken = 0; + int rc = SQLITE_OK; /* Allocate and populate the array of LcsIterator objects. The array ** contains one element for each matchable phrase in the query. **/ - aIter = sqlite3_malloc(sizeof(LcsIterator) * pCsr->nPhrase); + aIter = sqlite3Fts3MallocZero(sizeof(LcsIterator) * pCsr->nPhrase); if( !aIter ) return SQLITE_NOMEM; - memset(aIter, 0, sizeof(LcsIterator) * pCsr->nPhrase); - (void)fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); + (void)sqlite3Fts3ExprIterate(pCsr->pExpr, fts3MatchinfoLcsCb, (void*)aIter); for(i=0; inPhrase; i++){ LcsIterator *pIter = &aIter[i]; @@ -157747,13 +204799,16 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ int nLive = 0; /* Number of iterators in aIter not at EOF */ for(i=0; inPhrase; i++){ - int rc; LcsIterator *pIt = &aIter[i]; rc = sqlite3Fts3EvalPhrasePoslist(pCsr, pIt->pExpr, iCol, &pIt->pRead); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ) goto matchinfo_lcs_out; if( pIt->pRead ){ pIt->iPos = pIt->iPosOffset; - fts3LcsIteratorAdvance(&aIter[i]); + fts3LcsIteratorAdvance(pIt); + if( pIt->pRead==0 ){ + rc = FTS_CORRUPT_VTAB; + goto matchinfo_lcs_out; + } nLive++; } } @@ -157785,13 +204840,14 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ pInfo->aMatchinfo[iCol] = nLcs; } + matchinfo_lcs_out: sqlite3_free(aIter); - return SQLITE_OK; + return rc; } /* ** Populate the buffer pInfo->aMatchinfo[] with an array of integers to -** be returned by the matchinfo() function. Argument zArg contains the +** be returned by the matchinfo() function. Argument zArg contains the ** format string passed as the second argument to matchinfo (or the ** default value "pcx" if no second argument was specified). The format ** string has already been validated and the pInfo->aMatchinfo[] array @@ -157802,7 +204858,7 @@ static int fts3MatchinfoLcs(Fts3Cursor *pCsr, MatchInfo *pInfo){ ** rows (i.e. FTS3_MATCHINFO_NPHRASE, NCOL, NDOC, AVGLENGTH and part of HITS) ** have already been populated. ** -** Return SQLITE_OK if successful, or an SQLite error code if an error +** Return SQLITE_OK if successful, or an SQLite error code if an error ** occurs. If a value other than SQLITE_OK is returned, the state the ** pInfo->aMatchinfo[] buffer is left in is undefined. */ @@ -157827,27 +204883,32 @@ static int fts3MatchinfoValues( case FTS3_MATCHINFO_NCOL: if( bGlobal ) pInfo->aMatchinfo[0] = pInfo->nCol; break; - + case FTS3_MATCHINFO_NDOC: if( bGlobal ){ sqlite3_int64 nDoc = 0; - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0); + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, 0, 0); pInfo->aMatchinfo[0] = (u32)nDoc; } break; - case FTS3_MATCHINFO_AVGLENGTH: + case FTS3_MATCHINFO_AVGLENGTH: if( bGlobal ){ sqlite3_int64 nDoc; /* Number of rows in table */ const char *a; /* Aggregate column length array */ + const char *pEnd; /* First byte past end of length array */ - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a); + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &nDoc, &a, &pEnd); if( rc==SQLITE_OK ){ int iCol; for(iCol=0; iColnCol; iCol++){ u32 iVal; sqlite3_int64 nToken; a += sqlite3Fts3GetVarint(a, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } iVal = (u32)(((u32)(nToken&0xffffffff)+nDoc/2)/nDoc); pInfo->aMatchinfo[iCol] = iVal; } @@ -157861,9 +204922,14 @@ static int fts3MatchinfoValues( if( rc==SQLITE_OK ){ int iCol; const char *a = sqlite3_column_blob(pSelectDocsize, 0); + const char *pEnd = a + sqlite3_column_bytes(pSelectDocsize, 0); for(iCol=0; iColnCol; iCol++){ sqlite3_int64 nToken; - a += sqlite3Fts3GetVarint(a, &nToken); + a += sqlite3Fts3GetVarintBounded(a, pEnd, &nToken); + if( a>pEnd ){ + rc = SQLITE_CORRUPT_VTAB; + break; + } pInfo->aMatchinfo[iCol] = (u32)nToken; } } @@ -157880,9 +204946,9 @@ static int fts3MatchinfoValues( case FTS3_MATCHINFO_LHITS_BM: case FTS3_MATCHINFO_LHITS: { - int nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); + size_t nZero = fts3MatchinfoSize(pInfo, zArg[i]) * sizeof(u32); memset(pInfo->aMatchinfo, 0, nZero); - fts3ExprLHitGather(pCsr->pExpr, pInfo); + rc = fts3ExprLHitGather(pCsr->pExpr, pInfo); break; } @@ -157894,14 +204960,14 @@ static int fts3MatchinfoValues( if( rc!=SQLITE_OK ) break; if( bGlobal ){ if( pCsr->pDeferred ){ - rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc, 0); + rc = fts3MatchinfoSelectDoctotal(pTab, &pSelect, &pInfo->nDoc,0,0); if( rc!=SQLITE_OK ) break; } - rc = fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); + rc = sqlite3Fts3ExprIterate(pExpr, fts3ExprGlobalHitsCb,(void*)pInfo); sqlite3Fts3EvalTestDeferred(pCsr, &rc); if( rc!=SQLITE_OK ) break; } - (void)fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); + (void)sqlite3Fts3ExprIterate(pExpr, fts3ExprLocalHitsCb,(void*)pInfo); break; } } @@ -157915,7 +204981,7 @@ static int fts3MatchinfoValues( /* -** Populate pCsr->aMatchinfo[] with data for the current row. The +** Populate pCsr->aMatchinfo[] with data for the current row. The ** 'matchinfo' data is an array of 32-bit unsigned integers (C type u32). */ static void fts3GetMatchinfo( @@ -157935,8 +205001,8 @@ static void fts3GetMatchinfo( sInfo.pCursor = pCsr; sInfo.nCol = pTab->nColumn; - /* If there is cached matchinfo() data, but the format string for the - ** cache does not match the format string for this request, discard + /* If there is cached matchinfo() data, but the format string for the + ** cache does not match the format string for this request, discard ** the cached data. */ if( pCsr->pMIBuffer && strcmp(pCsr->pMIBuffer->zMatchinfo, zArg) ){ sqlite3Fts3MIBufferFree(pCsr->pMIBuffer); @@ -157944,12 +205010,12 @@ static void fts3GetMatchinfo( } /* If Fts3Cursor.pMIBuffer is NULL, then this is the first time the - ** matchinfo function has been called for this query. In this case + ** matchinfo function has been called for this query. In this case ** allocate the array used to accumulate the matchinfo data and ** initialize those elements that are constant for every row. */ if( pCsr->pMIBuffer==0 ){ - int nMatchinfo = 0; /* Number of u32 elements in match-info */ + size_t nMatchinfo = 0; /* Number of u32 elements in match-info */ int i; /* Used to iterate through zArg */ /* Determine the number of phrases in the query */ @@ -158019,7 +205085,7 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( /* The returned text includes up to four fragments of text extracted from ** the data in the current row. The first iteration of the for(...) loop - ** below attempts to locate a single fragment of text nToken tokens in + ** below attempts to locate a single fragment of text nToken tokens in ** size that contains at least one instance of all phrases in the query ** expression that appear in the current row. If such a fragment of text ** cannot be found, the second iteration of the loop attempts to locate @@ -158034,6 +205100,10 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( return; } + /* Limit the snippet length to 64 tokens. */ + if( nToken<-64 ) nToken = -64; + if( nToken>+64 ) nToken = +64; + for(nSnippet=1; 1; nSnippet++){ int iSnip; /* Loop counter 0..nSnippet-1 */ @@ -158086,7 +205156,7 @@ SQLITE_PRIVATE void sqlite3Fts3Snippet( assert( nFToken>0 ); for(i=0; ipPhrase->nToken; if( pList ){ fts3GetDeltaPosition(&pList, &iPos); - assert( iPos>=0 ); + assert_fts3_nc( iPos>=0 ); } for(iTerm=0; iTermiPrevId; sCtx.pCsr = pCsr; - /* Loop through the table columns, appending offset information to + /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ for(iCol=0; iColnColumn; iCol++){ @@ -158197,19 +205267,21 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( const char *zDoc; int nDoc; - /* Initialize the contents of sCtx.aTerm[] for column iCol. There is - ** no way that this operation can fail, so the return code from - ** fts3ExprIterate() can be discarded. + /* Initialize the contents of sCtx.aTerm[] for column iCol. This + ** operation may fail if the database contains corrupt records. */ sCtx.iCol = iCol; sCtx.iTerm = 0; - (void)fts3ExprIterate(pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx); + rc = sqlite3Fts3ExprIterate( + pCsr->pExpr, fts3ExprTermOffsetInit, (void*)&sCtx + ); + if( rc!=SQLITE_OK ) goto offsets_out; - /* Retreive the text stored in column iCol. If an SQL NULL is stored + /* Retreive the text stored in column iCol. If an SQL NULL is stored ** in column iCol, jump immediately to the next iteration of the loop. ** If an OOM occurs while retrieving the data (this can happen if SQLite - ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM - ** to the caller. + ** needs to transform the data from utf-16 to utf-8), return SQLITE_NOMEM + ** to the caller. */ zDoc = (const char *)sqlite3_column_text(pCsr->pStmt, iCol+1); nDoc = sqlite3_column_bytes(pCsr->pStmt, iCol+1); @@ -158245,7 +205317,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( /* All offsets for this column have been gathered. */ rc = SQLITE_DONE; }else{ - assert( iCurrent<=iMinPos ); + assert_fts3_nc( iCurrent<=iMinPos ); if( 0==(0xFE&*pTerm->pList) ){ pTerm->pList = 0; }else{ @@ -158256,7 +205328,7 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( } if( rc==SQLITE_OK ){ char aBuffer[64]; - sqlite3_snprintf(sizeof(aBuffer), aBuffer, + sqlite3_snprintf(sizeof(aBuffer), aBuffer, "%d %d %d %d ", iCol, pTerm-sCtx.aTerm, iStart, iEnd-iStart ); rc = fts3StringAppend(&res, aBuffer, -1); @@ -158401,7 +205473,7 @@ typedef struct unicode_cursor unicode_cursor; struct unicode_tokenizer { sqlite3_tokenizer base; - int bRemoveDiacritic; + int eRemoveDiacritic; int nException; int *aiException; }; @@ -158437,7 +205509,7 @@ static int unicodeDestroy(sqlite3_tokenizer *pTokenizer){ ** ** For each codepoint in the zIn/nIn string, this function checks if the ** sqlite3FtsUnicodeIsalnum() function already returns the desired result. -** If so, no action is taken. Otherwise, the codepoint is added to the +** If so, no action is taken. Otherwise, the codepoint is added to the ** unicode_tokenizer.aiException[] array. For the purposes of tokenization, ** the return value of sqlite3FtsUnicodeIsalnum() is inverted for all ** codepoints in the aiException[] array. @@ -158455,16 +205527,16 @@ static int unicodeAddExceptions( ){ const unsigned char *z = (const unsigned char *)zIn; const unsigned char *zTerm = &z[nIn]; - int iCode; + unsigned int iCode; int nEntry = 0; assert( bAlnum==0 || bAlnum==1 ); while( zaiException, (p->nException+nEntry)*sizeof(int)); + aNew = sqlite3_realloc64(p->aiException,(p->nException+nEntry)*sizeof(int)); if( aNew==0 ) return SQLITE_NOMEM; nNew = p->nException; z = (const unsigned char *)zIn; while( zi; j--) aNew[j] = aNew[j-1]; - aNew[i] = iCode; + aNew[i] = (int)iCode; nNew++; } } @@ -158546,17 +205618,20 @@ static int unicodeCreate( pNew = (unicode_tokenizer *) sqlite3_malloc(sizeof(unicode_tokenizer)); if( pNew==NULL ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(unicode_tokenizer)); - pNew->bRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; for(i=0; rc==SQLITE_OK && ibRemoveDiacritic = 1; + pNew->eRemoveDiacritic = 1; } else if( n==19 && memcmp("remove_diacritics=0", z, 19)==0 ){ - pNew->bRemoveDiacritic = 0; + pNew->eRemoveDiacritic = 0; + } + else if( n==19 && memcmp("remove_diacritics=2", z, 19)==0 ){ + pNew->eRemoveDiacritic = 2; } else if( n>=11 && memcmp("tokenchars=", z, 11)==0 ){ rc = unicodeAddExceptions(pNew, 1, &z[11], n-11); @@ -158581,7 +205656,7 @@ static int unicodeCreate( /* ** Prepare to begin tokenizing a particular string. The input ** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in +** used to incrementally tokenize this string is returned in ** *ppCursor. */ static int unicodeOpen( @@ -158601,6 +205676,7 @@ static int unicodeOpen( pCsr->aInput = (const unsigned char *)aInput; if( aInput==0 ){ pCsr->nInput = 0; + pCsr->aInput = (const unsigned char*)""; }else if( nInput<0 ){ pCsr->nInput = (int)strlen(aInput); }else{ @@ -158637,7 +205713,7 @@ static int unicodeNext( ){ unicode_cursor *pCsr = (unicode_cursor *)pC; unicode_tokenizer *p = ((unicode_tokenizer *)pCsr->base.pTokenizer); - int iCode = 0; + unsigned int iCode = 0; char *zOut; const unsigned char *z = &pCsr->aInput[pCsr->iOff]; const unsigned char *zStart = z; @@ -158645,11 +205721,11 @@ static int unicodeNext( const unsigned char *zTerm = &pCsr->aInput[pCsr->nInput]; /* Scan past any delimiter characters before the start of the next token. - ** Return SQLITE_DONE early if this takes us all the way to the end of + ** Return SQLITE_DONE early if this takes us all the way to the end of ** the input. */ while( z=zTerm ) return SQLITE_DONE; @@ -158660,7 +205736,7 @@ static int unicodeNext( /* Grow the output buffer if required. */ if( (zOut-pCsr->zToken)>=(pCsr->nAlloc-4) ){ - char *zNew = sqlite3_realloc(pCsr->zToken, pCsr->nAlloc+64); + char *zNew = sqlite3_realloc64(pCsr->zToken, pCsr->nAlloc+64); if( !zNew ) return SQLITE_NOMEM; zOut = &zNew[zOut - pCsr->zToken]; pCsr->zToken = zNew; @@ -158669,7 +205745,7 @@ static int unicodeNext( /* Write the folded case of the last character read to the output */ zEnd = z; - iOut = sqlite3FtsUnicodeFold(iCode, p->bRemoveDiacritic); + iOut = sqlite3FtsUnicodeFold((int)iCode, p->eRemoveDiacritic); if( iOut ){ WRITE_UTF8(zOut, iOut); } @@ -158677,8 +205753,8 @@ static int unicodeNext( /* If the cursor is not at EOF, read the next character */ if( z>=zTerm ) break; READ_UTF8(z, zTerm, iCode); - }while( unicodeIsAlnum(p, iCode) - || sqlite3FtsUnicodeIsdiacritic(iCode) + }while( unicodeIsAlnum(p, (int)iCode) + || sqlite3FtsUnicodeIsdiacritic((int)iCode) ); /* Set the output variables and return. */ @@ -158692,7 +205768,7 @@ static int unicodeNext( } /* -** Set *ppModule to a pointer to the sqlite3_tokenizer_module +** Set *ppModule to a pointer to the sqlite3_tokenizer_module ** structure for the unicode tokenizer. */ SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const **ppModule){ @@ -158714,7 +205790,7 @@ SQLITE_PRIVATE void sqlite3Fts3UnicodeTokenizer(sqlite3_tokenizer_module const * /************** End of fts3_unicode.c ****************************************/ /************** Begin file fts3_unicode2.c ***********************************/ /* -** 2012 May 25 +** 2012-05-25 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -158747,11 +205823,11 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ ** range of unicode codepoints that are not either letters or numbers (i.e. ** codepoints for which this function should return 0). ** - ** The most significant 22 bits in each 32-bit value contain the first + ** The most significant 22 bits in each 32-bit value contain the first ** codepoint in the range. The least significant 10 bits are used to store - ** the size of the range (always at least 1). In other words, the value - ** ((C<<22) + N) represents a range of N codepoints starting with codepoint - ** C. It is not possible to represent a range larger than 1023 codepoints + ** the size of the range (always at least 1). In other words, the value + ** ((C<<22) + N) represents a range of N codepoints starting with codepoint + ** C. It is not possible to represent a range larger than 1023 codepoints ** using this format. */ static const unsigned int aEntry[] = { @@ -158842,9 +205918,9 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, }; - if( c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( c<(1<<22) ){ + if( (unsigned int)c<128 ){ + return ( (aAscii[c >> 5] & ((unsigned int)1 << (c & 0x001F)))==0 ); + }else if( (unsigned int)c<(1<<22) ){ unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; int iRes = 0; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; @@ -158874,32 +205950,48 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsalnum(int c){ ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int remove_diacritic(int c){ +static int remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { - 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, - 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, - 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, - 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; - char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -158916,7 +206008,8 @@ static int remove_diacritic(int c){ } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } @@ -158929,8 +206022,8 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ unsigned int mask1 = 0x000361F8; if( c<768 || c>817 ) return 0; return (c < 768+32) ? - (mask0 & (1 << (c-768))) : - (mask1 & (1 << (c-768-32))); + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); } @@ -158943,7 +206036,7 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeIsdiacritic(int c){ ** The results are undefined if the value passed to this function ** is less than zero. */ -SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ +SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. @@ -159020,33 +206113,34 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, - {65313, 14, 26}, + {65313, 14, 26}, }; static const unsigned short aiOff[] = { - 1, 2, 8, 15, 16, 26, 28, 32, - 37, 38, 40, 48, 63, 64, 69, 71, - 79, 80, 116, 202, 203, 205, 206, 207, - 209, 210, 211, 213, 214, 217, 218, 219, - 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, - 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, - 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, - 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, - 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, - 65514, 65521, 65527, 65528, 65529, + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, }; int ret = c; - assert( c>=0 ); assert( sizeof(unsigned short)==2 && sizeof(unsigned char)==1 ); if( c<128 ){ if( c>='A' && c<='Z' ) ret = c + ('a' - 'A'); }else if( c<65536 ){ + const struct TableEntry *p; int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; int iLo = 0; int iRes = -1; + assert( c>aEntry[0].iCode ); while( iHi>=iLo ){ int iTest = (iHi + iLo) / 2; int cmp = (c - aEntry[iTest].iCode); @@ -159057,19 +206151,19 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ iHi = iTest-1; } } - assert( iRes<0 || c>=aEntry[iRes].iCode ); - if( iRes>=0 ){ - const struct TableEntry *p = &aEntry[iRes]; - if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ - ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; - assert( ret>0 ); - } + assert( iRes>=0 && c>=aEntry[iRes].iCode ); + p = &aEntry[iRes]; + if( c<(p->iCode + p->nRange) && 0==(0x01 & p->flags & (p->iCode ^ c)) ){ + ret = (c + (aiOff[p->flags>>1])) & 0x0000FFFF; + assert( ret>0 ); } - if( bRemoveDiacritic ) ret = remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = remove_diacritic(ret, eRemoveDiacritic==2); + } } - + else if( c>=66560 && c<66600 ){ ret = c + 40; } @@ -159080,6 +206174,5475 @@ SQLITE_PRIVATE int sqlite3FtsUnicodeFold(int c, int bRemoveDiacritic){ #endif /* !defined(SQLITE_DISABLE_FTS3_UNICODE) */ /************** End of fts3_unicode2.c ***************************************/ +/************** Begin file json.c ********************************************/ +/* +** 2015-08-12 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** SQLite JSON functions. +** +** This file began as an extension in ext/misc/json1.c in 2015. That +** extension proved so useful that it has now been moved into the core. +** +** The original design stored all JSON as pure text, canonical RFC-8259. +** Support for JSON-5 extensions was added with version 3.42.0 (2023-05-16). +** All generated JSON text still conforms strictly to RFC-8259, but text +** with JSON-5 extensions is accepted as input. +** +** Beginning with version 3.45.0 (circa 2024-01-01), these routines also +** accept BLOB values that have JSON encoded using a binary representation +** called "JSONB". The name JSONB comes from PostgreSQL, however the on-disk +** format SQLite JSONB is completely different and incompatible with +** PostgreSQL JSONB. +** +** Decoding and interpreting JSONB is still O(N) where N is the size of +** the input, the same as text JSON. However, the constant of proportionality +** for JSONB is much smaller due to faster parsing. The size of each +** element in JSONB is encoded in its header, so there is no need to search +** for delimiters using persnickety syntax rules. JSONB seems to be about +** 3x faster than text JSON as a result. JSONB is also tends to be slightly +** smaller than text JSON, by 5% or 10%, but there are corner cases where +** JSONB can be slightly larger. So you are not far mistaken to say that +** a JSONB blob is the same size as the equivalent RFC-8259 text. +** +** +** THE JSONB ENCODING: +** +** Every JSON element is encoded in JSONB as a header and a payload. +** The header is between 1 and 9 bytes in size. The payload is zero +** or more bytes. +** +** The lower 4 bits of the first byte of the header determines the +** element type: +** +** 0: NULL +** 1: TRUE +** 2: FALSE +** 3: INT -- RFC-8259 integer literal +** 4: INT5 -- JSON5 integer literal +** 5: FLOAT -- RFC-8259 floating point literal +** 6: FLOAT5 -- JSON5 floating point literal +** 7: TEXT -- Text literal acceptable to both SQL and JSON +** 8: TEXTJ -- Text containing RFC-8259 escapes +** 9: TEXT5 -- Text containing JSON5 and/or RFC-8259 escapes +** 10: TEXTRAW -- Text containing unescaped syntax characters +** 11: ARRAY +** 12: OBJECT +** +** The other three possible values (13-15) are reserved for future +** enhancements. +** +** The upper 4 bits of the first byte determine the size of the header +** and sometimes also the size of the payload. If X is the first byte +** of the element and if X>>4 is between 0 and 11, then the payload +** will be that many bytes in size and the header is exactly one byte +** in size. Other four values for X>>4 (12-15) indicate that the header +** is more than one byte in size and that the payload size is determined +** by the remainder of the header, interpreted as a unsigned big-endian +** integer. +** +** Value of X>>4 Size integer Total header size +** ------------- -------------------- ----------------- +** 12 1 byte (0-255) 2 +** 13 2 byte (0-65535) 3 +** 14 4 byte (0-4294967295) 5 +** 15 8 byte (0-1.8e19) 9 +** +** The payload size need not be expressed in its minimal form. For example, +** if the payload size is 10, the size can be expressed in any of 5 different +** ways: (1) (X>>4)==10, (2) (X>>4)==12 following by on 0x0a byte, +** (3) (X>>4)==13 followed by 0x00 and 0x0a, (4) (X>>4)==14 followed by +** 0x00 0x00 0x00 0x0a, or (5) (X>>4)==15 followed by 7 bytes of 0x00 and +** a single byte of 0x0a. The shorter forms are preferred, of course, but +** sometimes when generating JSONB, the payload size is not known in advance +** and it is convenient to reserve sufficient header space to cover the +** largest possible payload size and then come back later and patch up +** the size when it becomes known, resulting in a non-minimal encoding. +** +** The value (X>>4)==15 is not actually used in the current implementation +** (as SQLite is currently unable handle BLOBs larger than about 2GB) +** but is included in the design to allow for future enhancements. +** +** The payload follows the header. NULL, TRUE, and FALSE have no payload and +** their payload size must always be zero. The payload for INT, INT5, +** FLOAT, FLOAT5, TEXT, TEXTJ, TEXT5, and TEXTROW is text. Note that the +** "..." or '...' delimiters are omitted from the various text encodings. +** The payload for ARRAY and OBJECT is a list of additional elements that +** are the content for the array or object. The payload for an OBJECT +** must be an even number of elements. The first element of each pair is +** the label and must be of type TEXT, TEXTJ, TEXT5, or TEXTRAW. +** +** A valid JSONB blob consists of a single element, as described above. +** Usually this will be an ARRAY or OBJECT element which has many more +** elements as its content. But the overall blob is just a single element. +** +** Input validation for JSONB blobs simply checks that the element type +** code is between 0 and 12 and that the total size of the element +** (header plus payload) is the same as the size of the BLOB. If those +** checks are true, the BLOB is assumed to be JSONB and processing continues. +** Errors are only raised if some other miscoding is discovered during +** processing. +** +** Additional information can be found in the doc/jsonb.md file of the +** canonical SQLite source tree. +*/ +#ifndef SQLITE_OMIT_JSON +/* #include "sqliteInt.h" */ + +/* JSONB element types +*/ +#define JSONB_NULL 0 /* "null" */ +#define JSONB_TRUE 1 /* "true" */ +#define JSONB_FALSE 2 /* "false" */ +#define JSONB_INT 3 /* integer acceptable to JSON and SQL */ +#define JSONB_INT5 4 /* integer in 0x000 notation */ +#define JSONB_FLOAT 5 /* float acceptable to JSON and SQL */ +#define JSONB_FLOAT5 6 /* float with JSON5 extensions */ +#define JSONB_TEXT 7 /* Text compatible with both JSON and SQL */ +#define JSONB_TEXTJ 8 /* Text with JSON escapes */ +#define JSONB_TEXT5 9 /* Text with JSON-5 escape */ +#define JSONB_TEXTRAW 10 /* SQL text that needs escaping for JSON */ +#define JSONB_ARRAY 11 /* An array */ +#define JSONB_OBJECT 12 /* An object */ + +/* Human-readable names for the JSONB values. The index for each +** string must correspond to the JSONB_* integer above. +*/ +static const char * const jsonbType[] = { + "null", "true", "false", "integer", "integer", + "real", "real", "text", "text", "text", + "text", "array", "object", "", "", "", "" +}; + +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function, resulting in a 7% overall performance +** increase for the text-JSON parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). +*/ +static const char jsonIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define jsonIsspace(x) (jsonIsSpace[(unsigned char)x]) + +/* +** The set of all space characters recognized by jsonIsspace(). +** Useful as the second argument to strspn(). +*/ +static const char jsonSpaces[] = "\011\012\015\040"; + +/* +** Characters that are special to JSON. Control characters, +** '"' and '\\' and '\''. Actually, '\'' is not special to +** canonical JSON, but it is special in JSON-5, so we include +** it in the set of special characters. +*/ +static const char jsonIsOk[256] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 +}; + +/* Objects */ +typedef struct JsonCache JsonCache; +typedef struct JsonString JsonString; +typedef struct JsonParse JsonParse; + +/* +** Magic number used for the JSON parse cache in sqlite3_get_auxdata() +*/ +#define JSON_CACHE_ID (-429938) /* Cache entry */ +#define JSON_CACHE_SIZE 4 /* Max number of cache entries */ + +/* +** jsonUnescapeOneChar() returns this invalid code point if it encounters +** a syntax error. +*/ +#define JSON_INVALID_CHAR 0x99999 + +/* A cache mapping JSON text into JSONB blobs. +** +** Each cache entry is a JsonParse object with the following restrictions: +** +** * The bReadOnly flag must be set +** +** * The aBlob[] array must be owned by the JsonParse object. In other +** words, nBlobAlloc must be non-zero. +** +** * eEdit and delta must be zero. +** +** * zJson must be an RCStr. In other words bJsonIsRCStr must be true. +*/ +struct JsonCache { + sqlite3 *db; /* Database connection */ + int nUsed; /* Number of active entries in the cache */ + JsonParse *a[JSON_CACHE_SIZE]; /* One line for each cache entry */ +}; + +/* An instance of this object represents a JSON string +** under construction. Really, this is a generic string accumulator +** that can be and is used to create strings other than JSON. +** +** If the generated string is longer than will fit into the zSpace[] buffer, +** then it will be an RCStr string. This aids with caching of large +** JSON strings. +*/ +struct JsonString { + sqlite3_context *pCtx; /* Function context - put error messages here */ + char *zBuf; /* Append JSON content here */ + u64 nAlloc; /* Bytes of storage available in zBuf[] */ + u64 nUsed; /* Bytes of zBuf[] currently used */ + u8 bStatic; /* True if zBuf is static space */ + u8 eErr; /* True if an error has been encountered */ + char zSpace[100]; /* Initial static space */ +}; + +/* Allowed values for JsonString.eErr */ +#define JSTRING_OOM 0x01 /* Out of memory */ +#define JSTRING_MALFORMED 0x02 /* Malformed JSONB */ +#define JSTRING_ERR 0x04 /* Error already sent to sqlite3_result */ + +/* The "subtype" set for text JSON values passed through using +** sqlite3_result_subtype() and sqlite3_value_subtype(). +*/ +#define JSON_SUBTYPE 74 /* Ascii for "J" */ + +/* +** Bit values for the flags passed into various SQL function implementations +** via the sqlite3_user_data() value. +*/ +#define JSON_JSON 0x01 /* Result is always JSON */ +#define JSON_SQL 0x02 /* Result is always SQL */ +#define JSON_ABPATH 0x03 /* Allow abbreviated JSON path specs */ +#define JSON_ISSET 0x04 /* json_set(), not json_insert() */ +#define JSON_BLOB 0x08 /* Use the BLOB output format */ + + +/* A parsed JSON value. Lifecycle: +** +** 1. JSON comes in and is parsed into a JSONB value in aBlob. The +** original text is stored in zJson. This step is skipped if the +** input is JSONB instead of text JSON. +** +** 2. The aBlob[] array is searched using the JSON path notation, if needed. +** +** 3. Zero or more changes are made to aBlob[] (via json_remove() or +** json_replace() or json_patch() or similar). +** +** 4. New JSON text is generated from the aBlob[] for output. This step +** is skipped if the function is one of the jsonb_* functions that +** returns JSONB instead of text JSON. +*/ +struct JsonParse { + u8 *aBlob; /* JSONB representation of JSON value */ + u32 nBlob; /* Bytes of aBlob[] actually used */ + u32 nBlobAlloc; /* Bytes allocated to aBlob[]. 0 if aBlob is external */ + char *zJson; /* Json text used for parsing */ + sqlite3 *db; /* The database connection to which this object belongs */ + int nJson; /* Length of the zJson string in bytes */ + u32 nJPRef; /* Number of references to this object */ + u32 iErr; /* Error location in zJson[] */ + u16 iDepth; /* Nesting depth */ + u8 nErr; /* Number of errors seen */ + u8 oom; /* Set to true if out of memory */ + u8 bJsonIsRCStr; /* True if zJson is an RCStr */ + u8 hasNonstd; /* True if input uses non-standard features like JSON5 */ + u8 bReadOnly; /* Do not modify. */ + /* Search and edit information. See jsonLookupStep() */ + u8 eEdit; /* Edit operation to apply */ + int delta; /* Size change due to the edit */ + u32 nIns; /* Number of bytes to insert */ + u32 iLabel; /* Location of label if search landed on an object value */ + u8 *aIns; /* Content to be inserted */ +}; + +/* Allowed values for JsonParse.eEdit */ +#define JEDIT_DEL 1 /* Delete if exists */ +#define JEDIT_REPL 2 /* Overwrite if exists */ +#define JEDIT_INS 3 /* Insert if not exists */ +#define JEDIT_SET 4 /* Insert or overwrite */ + +/* +** Maximum nesting depth of JSON for this implementation. +** +** This limit is needed to avoid a stack overflow in the recursive +** descent parser. A depth of 1000 is far deeper than any sane JSON +** should go. Historical note: This limit was 2000 prior to version 3.42.0 +*/ +#ifndef SQLITE_JSON_MAX_DEPTH +# define JSON_MAX_DEPTH 1000 +#else +# define JSON_MAX_DEPTH SQLITE_JSON_MAX_DEPTH +#endif + +/* +** Allowed values for the flgs argument to jsonParseFuncArg(); +*/ +#define JSON_EDITABLE 0x01 /* Generate a writable JsonParse object */ +#define JSON_KEEPERROR 0x02 /* Return non-NULL even if there is an error */ + +/************************************************************************** +** Forward references +**************************************************************************/ +static void jsonReturnStringAsBlob(JsonString*); +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson); +static u32 jsonTranslateBlobToText(const JsonParse*,u32,JsonString*); +static void jsonReturnParse(sqlite3_context*,JsonParse*); +static JsonParse *jsonParseFuncArg(sqlite3_context*,sqlite3_value*,u32); +static void jsonParseFree(JsonParse*); +static u32 jsonbPayloadSize(const JsonParse*, u32, u32*); +static u32 jsonUnescapeOneChar(const char*, u32, u32*); + +/************************************************************************** +** Utility routines for dealing with JsonCache objects +**************************************************************************/ + +/* +** Free a JsonCache object. +*/ +static void jsonCacheDelete(JsonCache *p){ + int i; + for(i=0; inUsed; i++){ + jsonParseFree(p->a[i]); + } + sqlite3DbFree(p->db, p); +} +static void jsonCacheDeleteGeneric(void *p){ + jsonCacheDelete((JsonCache*)p); +} + +/* +** Insert a new entry into the cache. If the cache is full, expel +** the least recently used entry. Return SQLITE_OK on success or a +** result code otherwise. +** +** Cache entries are stored in age order, oldest first. +*/ +static int jsonCacheInsert( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + JsonParse *pParse /* The parse object to be added to the cache */ +){ + JsonCache *p; + + assert( pParse->zJson!=0 ); + assert( pParse->bJsonIsRCStr ); + assert( pParse->delta==0 ); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + sqlite3 *db = sqlite3_context_db_handle(ctx); + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) return SQLITE_NOMEM; + p->db = db; + sqlite3_set_auxdata(ctx, JSON_CACHE_ID, p, jsonCacheDeleteGeneric); + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ) return SQLITE_NOMEM; + } + if( p->nUsed >= JSON_CACHE_SIZE ){ + jsonParseFree(p->a[0]); + memmove(p->a, &p->a[1], (JSON_CACHE_SIZE-1)*sizeof(p->a[0])); + p->nUsed = JSON_CACHE_SIZE-1; + } + assert( pParse->nBlobAlloc>0 ); + pParse->eEdit = 0; + pParse->nJPRef++; + pParse->bReadOnly = 1; + p->a[p->nUsed] = pParse; + p->nUsed++; + return SQLITE_OK; +} + +/* +** Search for a cached translation the json text supplied by pArg. Return +** the JsonParse object if found. Return NULL if not found. +** +** When a match if found, the matching entry is moved to become the +** most-recently used entry if it isn't so already. +** +** The JsonParse object returned still belongs to the Cache and might +** be deleted at any moment. If the caller whants the JsonParse to +** linger, it needs to increment the nPJRef reference counter. +*/ +static JsonParse *jsonCacheSearch( + sqlite3_context *ctx, /* The SQL statement context holding the cache */ + sqlite3_value *pArg /* Function argument containing SQL text */ +){ + JsonCache *p; + int i; + const char *zJson; + int nJson; + + if( sqlite3_value_type(pArg)!=SQLITE_TEXT ){ + return 0; + } + zJson = (const char*)sqlite3_value_text(pArg); + if( zJson==0 ) return 0; + nJson = sqlite3_value_bytes(pArg); + + p = sqlite3_get_auxdata(ctx, JSON_CACHE_ID); + if( p==0 ){ + return 0; + } + for(i=0; inUsed; i++){ + if( p->a[i]->zJson==zJson ) break; + } + if( i>=p->nUsed ){ + for(i=0; inUsed; i++){ + if( p->a[i]->nJson!=nJson ) continue; + if( memcmp(p->a[i]->zJson, zJson, nJson)==0 ) break; + } + } + if( inUsed ){ + if( inUsed-1 ){ + /* Make the matching entry the most recently used entry */ + JsonParse *tmp = p->a[i]; + memmove(&p->a[i], &p->a[i+1], (p->nUsed-i-1)*sizeof(tmp)); + p->a[p->nUsed-1] = tmp; + i = p->nUsed - 1; + } + assert( p->a[i]->delta==0 ); + return p->a[i]; + }else{ + return 0; + } +} + +/************************************************************************** +** Utility routines for dealing with JsonString objects +**************************************************************************/ + +/* Turn uninitialized bulk memory into a valid JsonString object +** holding a zero-length string. +*/ +static void jsonStringZero(JsonString *p){ + p->zBuf = p->zSpace; + p->nAlloc = sizeof(p->zSpace); + p->nUsed = 0; + p->bStatic = 1; +} + +/* Initialize the JsonString object +*/ +static void jsonStringInit(JsonString *p, sqlite3_context *pCtx){ + p->pCtx = pCtx; + p->eErr = 0; + jsonStringZero(p); +} + +/* Free all allocated memory and reset the JsonString object back to its +** initial state. +*/ +static void jsonStringReset(JsonString *p){ + if( !p->bStatic ) sqlite3RCStrUnref(p->zBuf); + jsonStringZero(p); +} + +/* Report an out-of-memory (OOM) condition +*/ +static void jsonStringOom(JsonString *p){ + p->eErr |= JSTRING_OOM; + if( p->pCtx ) sqlite3_result_error_nomem(p->pCtx); + jsonStringReset(p); +} + +/* Enlarge pJson->zBuf so that it can hold at least N more bytes. +** Return zero on success. Return non-zero on an OOM error +*/ +static int jsonStringGrow(JsonString *p, u32 N){ + u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; + char *zNew; + if( p->bStatic ){ + if( p->eErr ) return 1; + zNew = sqlite3RCStrNew(nTotal); + if( zNew==0 ){ + jsonStringOom(p); + return SQLITE_NOMEM; + } + memcpy(zNew, p->zBuf, (size_t)p->nUsed); + p->zBuf = zNew; + p->bStatic = 0; + }else{ + p->zBuf = sqlite3RCStrResize(p->zBuf, nTotal); + if( p->zBuf==0 ){ + p->eErr |= JSTRING_OOM; + jsonStringZero(p); + return SQLITE_NOMEM; + } + } + p->nAlloc = nTotal; + return SQLITE_OK; +} + +/* Append N bytes from zIn onto the end of the JsonString string. +*/ +static SQLITE_NOINLINE void jsonStringExpandAndAppend( + JsonString *p, + const char *zIn, + u32 N +){ + assert( N>0 ); + if( jsonStringGrow(p,N) ) return; + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; +} +static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ + if( N==0 ) return; + if( N+p->nUsed >= p->nAlloc ){ + jsonStringExpandAndAppend(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} +static void jsonAppendRawNZ(JsonString *p, const char *zIn, u32 N){ + assert( N>0 ); + if( N+p->nUsed >= p->nAlloc ){ + jsonStringExpandAndAppend(p,zIn,N); + }else{ + memcpy(p->zBuf+p->nUsed, zIn, N); + p->nUsed += N; + } +} + +/* Append formatted text (not to exceed N bytes) to the JsonString. +*/ +static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ + va_list ap; + if( (p->nUsed + N >= p->nAlloc) && jsonStringGrow(p, N) ) return; + va_start(ap, zFormat); + sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); + va_end(ap); + p->nUsed += (int)strlen(p->zBuf+p->nUsed); +} + +/* Append a single character +*/ +static SQLITE_NOINLINE void jsonAppendCharExpand(JsonString *p, char c){ + if( jsonStringGrow(p,1) ) return; + p->zBuf[p->nUsed++] = c; +} +static void jsonAppendChar(JsonString *p, char c){ + if( p->nUsed>=p->nAlloc ){ + jsonAppendCharExpand(p,c); + }else{ + p->zBuf[p->nUsed++] = c; + } +} + +/* Remove a single character from the end of the string +*/ +static void jsonStringTrimOneChar(JsonString *p){ + if( p->eErr==0 ){ + assert( p->nUsed>0 ); + p->nUsed--; + } +} + + +/* Make sure there is a zero terminator on p->zBuf[] +** +** Return true on success. Return false if an OOM prevents this +** from happening. +*/ +static int jsonStringTerminate(JsonString *p){ + jsonAppendChar(p, 0); + jsonStringTrimOneChar(p); + return p->eErr==0; +} + +/* Append a comma separator to the output buffer, if the previous +** character is not '[' or '{'. +*/ +static void jsonAppendSeparator(JsonString *p){ + char c; + if( p->nUsed==0 ) return; + c = p->zBuf[p->nUsed-1]; + if( c=='[' || c=='{' ) return; + jsonAppendChar(p, ','); +} + +/* c is a control character. Append the canonical JSON representation +** of that control character to p. +** +** This routine assumes that the output buffer has already been enlarged +** sufficiently to hold the worst-case encoding plus a nul terminator. +*/ +static void jsonAppendControlChar(JsonString *p, u8 c){ + static const char aSpecial[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 + }; + assert( sizeof(aSpecial)==32 ); + assert( aSpecial['\b']=='b' ); + assert( aSpecial['\f']=='f' ); + assert( aSpecial['\n']=='n' ); + assert( aSpecial['\r']=='r' ); + assert( aSpecial['\t']=='t' ); + assert( c>=0 && cnUsed+7 <= p->nAlloc ); + if( aSpecial[c] ){ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = aSpecial[c]; + p->nUsed += 2; + }else{ + p->zBuf[p->nUsed] = '\\'; + p->zBuf[p->nUsed+1] = 'u'; + p->zBuf[p->nUsed+2] = '0'; + p->zBuf[p->nUsed+3] = '0'; + p->zBuf[p->nUsed+4] = "0123456789abcdef"[c>>4]; + p->zBuf[p->nUsed+5] = "0123456789abcdef"[c&0xf]; + p->nUsed += 6; + } +} + +/* Append the N-byte string in zIn to the end of the JsonString string +** under construction. Enclose the string in double-quotes ("...") and +** escape any double-quotes or backslash characters contained within the +** string. +** +** This routine is a high-runner. There is a measurable performance +** increase associated with unwinding the jsonIsOk[] loop. +*/ +static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ + u32 k; + u8 c; + const u8 *z = (const u8*)zIn; + if( z==0 ) return; + if( (N+p->nUsed+2 >= p->nAlloc) && jsonStringGrow(p,N+2)!=0 ) return; + p->zBuf[p->nUsed++] = '"'; + while( 1 /*exit-by-break*/ ){ + k = 0; + /* The following while() is the 4-way unwound equivalent of + ** + ** while( k=N ){ + while( k=N ){ + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + } + break; + } + if( k>0 ){ + memcpy(&p->zBuf[p->nUsed], z, k); + p->nUsed += k; + z += k; + N -= k; + } + c = z[0]; + if( c=='"' || c=='\\' ){ + if( (p->nUsed+N+3 > p->nAlloc) && jsonStringGrow(p,N+3)!=0 ) return; + p->zBuf[p->nUsed++] = '\\'; + p->zBuf[p->nUsed++] = c; + }else if( c=='\'' ){ + p->zBuf[p->nUsed++] = c; + }else{ + if( (p->nUsed+N+7 > p->nAlloc) && jsonStringGrow(p,N+7)!=0 ) return; + jsonAppendControlChar(p, c); + } + z++; + N--; + } + p->zBuf[p->nUsed++] = '"'; + assert( p->nUsednAlloc ); +} + +/* +** Append an sqlite3_value (such as a function parameter) to the JSON +** string under construction in p. +*/ +static void jsonAppendSqlValue( + JsonString *p, /* Append to this JSON string */ + sqlite3_value *pValue /* Value to append */ +){ + switch( sqlite3_value_type(pValue) ){ + case SQLITE_NULL: { + jsonAppendRawNZ(p, "null", 4); + break; + } + case SQLITE_FLOAT: { + jsonPrintf(100, p, "%!0.15g", sqlite3_value_double(pValue)); + break; + } + case SQLITE_INTEGER: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + jsonAppendRaw(p, z, n); + break; + } + case SQLITE_TEXT: { + const char *z = (const char*)sqlite3_value_text(pValue); + u32 n = (u32)sqlite3_value_bytes(pValue); + if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ + jsonAppendRaw(p, z, n); + }else{ + jsonAppendString(p, z, n); + } + break; + } + default: { + if( jsonFuncArgMightBeBinary(pValue) ){ + JsonParse px; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(pValue); + px.nBlob = sqlite3_value_bytes(pValue); + jsonTranslateBlobToText(&px, 0, p); + }else if( p->eErr==0 ){ + sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); + p->eErr = JSTRING_ERR; + jsonStringReset(p); + } + break; + } + } +} + +/* Make the text in p (which is probably a generated JSON text string) +** the result of the SQL function. +** +** The JsonString is reset. +** +** If pParse and ctx are both non-NULL, then the SQL string in p is +** loaded into the zJson field of the pParse object as a RCStr and the +** pParse is added to the cache. +*/ +static void jsonReturnString( + JsonString *p, /* String to return */ + JsonParse *pParse, /* JSONB source or NULL */ + sqlite3_context *ctx /* Where to cache */ +){ + assert( (pParse!=0)==(ctx!=0) ); + assert( ctx==0 || ctx==p->pCtx ); + if( p->eErr==0 ){ + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(p->pCtx)); + if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(p); + }else if( p->bStatic ){ + sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + }else if( jsonStringTerminate(p) ){ + if( pParse && pParse->bJsonIsRCStr==0 && pParse->nBlobAlloc>0 ){ + int rc; + pParse->zJson = sqlite3RCStrRef(p->zBuf); + pParse->nJson = p->nUsed; + pParse->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, pParse); + if( rc==SQLITE_NOMEM ){ + sqlite3_result_error_nomem(ctx); + jsonStringReset(p); + return; + } + } + sqlite3_result_text64(p->pCtx, sqlite3RCStrRef(p->zBuf), p->nUsed, + sqlite3RCStrUnref, + SQLITE_UTF8); + }else{ + sqlite3_result_error_nomem(p->pCtx); + } + }else if( p->eErr & JSTRING_OOM ){ + sqlite3_result_error_nomem(p->pCtx); + }else if( p->eErr & JSTRING_MALFORMED ){ + sqlite3_result_error(p->pCtx, "malformed JSON", -1); + } + jsonStringReset(p); +} + +/************************************************************************** +** Utility routines for dealing with JsonParse objects +**************************************************************************/ + +/* +** Reclaim all memory allocated by a JsonParse object. But do not +** delete the JsonParse object itself. +*/ +static void jsonParseReset(JsonParse *pParse){ + assert( pParse->nJPRef<=1 ); + if( pParse->bJsonIsRCStr ){ + sqlite3RCStrUnref(pParse->zJson); + pParse->zJson = 0; + pParse->nJson = 0; + pParse->bJsonIsRCStr = 0; + } + if( pParse->nBlobAlloc ){ + sqlite3DbFree(pParse->db, pParse->aBlob); + pParse->aBlob = 0; + pParse->nBlob = 0; + pParse->nBlobAlloc = 0; + } +} + +/* +** Decrement the reference count on the JsonParse object. When the +** count reaches zero, free the object. +*/ +static void jsonParseFree(JsonParse *pParse){ + if( pParse ){ + if( pParse->nJPRef>1 ){ + pParse->nJPRef--; + }else{ + jsonParseReset(pParse); + sqlite3DbFree(pParse->db, pParse); + } + } +} + +/************************************************************************** +** Utility routines for the JSON text parser +**************************************************************************/ + +/* +** Translate a single byte of Hex into an integer. +** This routine only gives a correct answer if h really is a valid hexadecimal +** character: 0..9a..fA..F. But unlike sqlite3HexToInt(), it does not +** assert() if the digit is not hex. +*/ +static u8 jsonHexToInt(int h){ +#ifdef SQLITE_ASCII + h += 9*(1&(h>>6)); +#endif +#ifdef SQLITE_EBCDIC + h += 9*(1&~(h>>4)); +#endif + return (u8)(h & 0xf); +} + +/* +** Convert a 4-byte hex string into an integer +*/ +static u32 jsonHexToInt4(const char *z){ + u32 v; + v = (jsonHexToInt(z[0])<<12) + + (jsonHexToInt(z[1])<<8) + + (jsonHexToInt(z[2])<<4) + + jsonHexToInt(z[3]); + return v; +} + +/* +** Return true if z[] begins with 2 (or more) hexadecimal digits +*/ +static int jsonIs2Hex(const char *z){ + return sqlite3Isxdigit(z[0]) && sqlite3Isxdigit(z[1]); +} + +/* +** Return true if z[] begins with 4 (or more) hexadecimal digits +*/ +static int jsonIs4Hex(const char *z){ + return jsonIs2Hex(z) && jsonIs2Hex(&z[2]); +} + +/* +** Return the number of bytes of JSON5 whitespace at the beginning of +** the input string z[]. +** +** JSON5 whitespace consists of any of the following characters: +** +** Unicode UTF-8 Name +** U+0009 09 horizontal tab +** U+000a 0a line feed +** U+000b 0b vertical tab +** U+000c 0c form feed +** U+000d 0d carriage return +** U+0020 20 space +** U+00a0 c2 a0 non-breaking space +** U+1680 e1 9a 80 ogham space mark +** U+2000 e2 80 80 en quad +** U+2001 e2 80 81 em quad +** U+2002 e2 80 82 en space +** U+2003 e2 80 83 em space +** U+2004 e2 80 84 three-per-em space +** U+2005 e2 80 85 four-per-em space +** U+2006 e2 80 86 six-per-em space +** U+2007 e2 80 87 figure space +** U+2008 e2 80 88 punctuation space +** U+2009 e2 80 89 thin space +** U+200a e2 80 8a hair space +** U+2028 e2 80 a8 line separator +** U+2029 e2 80 a9 paragraph separator +** U+202f e2 80 af narrow no-break space (NNBSP) +** U+205f e2 81 9f medium mathematical space (MMSP) +** U+3000 e3 80 80 ideographical space +** U+FEFF ef bb bf byte order mark +** +** In addition, comments between '/', '*' and '*', '/' and +** from '/', '/' to end-of-line are also considered to be whitespace. +*/ +static int json5Whitespace(const char *zIn){ + int n = 0; + const u8 *z = (u8*)zIn; + while( 1 /*exit by "goto whitespace_done"*/ ){ + switch( z[n] ){ + case 0x09: + case 0x0a: + case 0x0b: + case 0x0c: + case 0x0d: + case 0x20: { + n++; + break; + } + case '/': { + if( z[n+1]=='*' && z[n+2]!=0 ){ + int j; + for(j=n+3; z[j]!='/' || z[j-1]!='*'; j++){ + if( z[j]==0 ) goto whitespace_done; + } + n = j+1; + break; + }else if( z[n+1]=='/' ){ + int j; + char c; + for(j=n+2; (c = z[j])!=0; j++){ + if( c=='\n' || c=='\r' ) break; + if( 0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2]) + ){ + j += 2; + break; + } + } + n = j; + if( z[n] ) n++; + break; + } + goto whitespace_done; + } + case 0xc2: { + if( z[n+1]==0xa0 ){ + n += 2; + break; + } + goto whitespace_done; + } + case 0xe1: { + if( z[n+1]==0x9a && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe2: { + if( z[n+1]==0x80 ){ + u8 c = z[n+2]; + if( c<0x80 ) goto whitespace_done; + if( c<=0x8a || c==0xa8 || c==0xa9 || c==0xaf ){ + n += 3; + break; + } + }else if( z[n+1]==0x81 && z[n+2]==0x9f ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xe3: { + if( z[n+1]==0x80 && z[n+2]==0x80 ){ + n += 3; + break; + } + goto whitespace_done; + } + case 0xef: { + if( z[n+1]==0xbb && z[n+2]==0xbf ){ + n += 3; + break; + } + goto whitespace_done; + } + default: { + goto whitespace_done; + } + } + } + whitespace_done: + return n; +} + +/* +** Extra floating-point literals to allow in JSON. +*/ +static const struct NanInfName { + char c1; + char c2; + char n; + char eType; + char nRepl; + char *zMatch; + char *zRepl; +} aNanInfName[] = { + { 'i', 'I', 3, JSONB_FLOAT, 7, "inf", "9.0e999" }, + { 'i', 'I', 8, JSONB_FLOAT, 7, "infinity", "9.0e999" }, + { 'n', 'N', 3, JSONB_NULL, 4, "NaN", "null" }, + { 'q', 'Q', 4, JSONB_NULL, 4, "QNaN", "null" }, + { 's', 'S', 4, JSONB_NULL, 4, "SNaN", "null" }, +}; + + +/* +** Report the wrong number of arguments for json_insert(), json_replace() +** or json_set(). +*/ +static void jsonWrongNumArgs( + sqlite3_context *pCtx, + const char *zFuncName +){ + char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", + zFuncName); + sqlite3_result_error(pCtx, zMsg, -1); + sqlite3_free(zMsg); +} + +/**************************************************************************** +** Utility routines for dealing with the binary BLOB representation of JSON +****************************************************************************/ + +/* +** Expand pParse->aBlob so that it holds at least N bytes. +** +** Return the number of errors. +*/ +static int jsonBlobExpand(JsonParse *pParse, u32 N){ + u8 *aNew; + u32 t; + assert( N>pParse->nBlobAlloc ); + if( pParse->nBlobAlloc==0 ){ + t = 100; + }else{ + t = pParse->nBlobAlloc*2; + } + if( tdb, pParse->aBlob, t); + if( aNew==0 ){ pParse->oom = 1; return 1; } + pParse->aBlob = aNew; + pParse->nBlobAlloc = t; + return 0; +} + +/* +** If pParse->aBlob is not previously editable (because it is taken +** from sqlite3_value_blob(), as indicated by the fact that +** pParse->nBlobAlloc==0 and pParse->nBlob>0) then make it editable +** by making a copy into space obtained from malloc. +** +** Return true on success. Return false on OOM. +*/ +static int jsonBlobMakeEditable(JsonParse *pParse, u32 nExtra){ + u8 *aOld; + u32 nSize; + assert( !pParse->bReadOnly ); + if( pParse->oom ) return 0; + if( pParse->nBlobAlloc>0 ) return 1; + aOld = pParse->aBlob; + nSize = pParse->nBlob + nExtra; + pParse->aBlob = 0; + if( jsonBlobExpand(pParse, nSize) ){ + return 0; + } + assert( pParse->nBlobAlloc >= pParse->nBlob + nExtra ); + memcpy(pParse->aBlob, aOld, pParse->nBlob); + return 1; +} + +/* Expand pParse->aBlob and append one bytes. +*/ +static SQLITE_NOINLINE void jsonBlobExpandAndAppendOneByte( + JsonParse *pParse, + u8 c +){ + jsonBlobExpand(pParse, pParse->nBlob+1); + if( pParse->oom==0 ){ + assert( pParse->nBlob+1<=pParse->nBlobAlloc ); + pParse->aBlob[pParse->nBlob++] = c; + } +} + +/* Append a single character. +*/ +static void jsonBlobAppendOneByte(JsonParse *pParse, u8 c){ + if( pParse->nBlob >= pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendOneByte(pParse, c); + }else{ + pParse->aBlob[pParse->nBlob++] = c; + } +} + +/* Slow version of jsonBlobAppendNode() that first resizes the +** pParse->aBlob structure. +*/ +static void jsonBlobAppendNode(JsonParse*,u8,u32,const void*); +static SQLITE_NOINLINE void jsonBlobExpandAndAppendNode( + JsonParse *pParse, + u8 eType, + u32 szPayload, + const void *aPayload +){ + if( jsonBlobExpand(pParse, pParse->nBlob+szPayload+9) ) return; + jsonBlobAppendNode(pParse, eType, szPayload, aPayload); +} + + +/* Append an node type byte together with the payload size and +** possibly also the payload. +** +** If aPayload is not NULL, then it is a pointer to the payload which +** is also appended. If aPayload is NULL, the pParse->aBlob[] array +** is resized (if necessary) so that it is big enough to hold the +** payload, but the payload is not appended and pParse->nBlob is left +** pointing to where the first byte of payload will eventually be. +*/ +static void jsonBlobAppendNode( + JsonParse *pParse, /* The JsonParse object under construction */ + u8 eType, /* Node type. One of JSONB_* */ + u32 szPayload, /* Number of bytes of payload */ + const void *aPayload /* The payload. Might be NULL */ +){ + u8 *a; + if( pParse->nBlob+szPayload+9 > pParse->nBlobAlloc ){ + jsonBlobExpandAndAppendNode(pParse,eType,szPayload,aPayload); + return; + } + assert( pParse->aBlob!=0 ); + a = &pParse->aBlob[pParse->nBlob]; + if( szPayload<=11 ){ + a[0] = eType | (szPayload<<4); + pParse->nBlob += 1; + }else if( szPayload<=0xff ){ + a[0] = eType | 0xc0; + a[1] = szPayload & 0xff; + pParse->nBlob += 2; + }else if( szPayload<=0xffff ){ + a[0] = eType | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + pParse->nBlob += 3; + }else{ + a[0] = eType | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + pParse->nBlob += 5; + } + if( aPayload ){ + pParse->nBlob += szPayload; + memcpy(&pParse->aBlob[pParse->nBlob-szPayload], aPayload, szPayload); + } +} + +/* Change the payload size for the node at index i to be szPayload. +*/ +static int jsonBlobChangePayloadSize( + JsonParse *pParse, + u32 i, + u32 szPayload +){ + u8 *a; + u8 szType; + u8 nExtra; + u8 nNeeded; + int delta; + if( pParse->oom ) return 0; + a = &pParse->aBlob[i]; + szType = a[0]>>4; + if( szType<=11 ){ + nExtra = 0; + }else if( szType==12 ){ + nExtra = 1; + }else if( szType==13 ){ + nExtra = 2; + }else{ + nExtra = 4; + } + if( szPayload<=11 ){ + nNeeded = 0; + }else if( szPayload<=0xff ){ + nNeeded = 1; + }else if( szPayload<=0xffff ){ + nNeeded = 2; + }else{ + nNeeded = 4; + } + delta = nNeeded - nExtra; + if( delta ){ + u32 newSize = pParse->nBlob + delta; + if( delta>0 ){ + if( newSize>pParse->nBlobAlloc && jsonBlobExpand(pParse, newSize) ){ + return 0; /* OOM error. Error state recorded in pParse->oom. */ + } + a = &pParse->aBlob[i]; + memmove(&a[1+delta], &a[1], pParse->nBlob - (i+1)); + }else{ + memmove(&a[1], &a[1-delta], pParse->nBlob - (i+1-delta)); + } + pParse->nBlob = newSize; + } + if( nNeeded==0 ){ + a[0] = (a[0] & 0x0f) | (szPayload<<4); + }else if( nNeeded==1 ){ + a[0] = (a[0] & 0x0f) | 0xc0; + a[1] = szPayload & 0xff; + }else if( nNeeded==2 ){ + a[0] = (a[0] & 0x0f) | 0xd0; + a[1] = (szPayload >> 8) & 0xff; + a[2] = szPayload & 0xff; + }else{ + a[0] = (a[0] & 0x0f) | 0xe0; + a[1] = (szPayload >> 24) & 0xff; + a[2] = (szPayload >> 16) & 0xff; + a[3] = (szPayload >> 8) & 0xff; + a[4] = szPayload & 0xff; + } + return delta; +} + +/* +** If z[0] is 'u' and is followed by exactly 4 hexadecimal character, +** then set *pOp to JSONB_TEXTJ and return true. If not, do not make +** any changes to *pOp and return false. +*/ +static int jsonIs4HexB(const char *z, int *pOp){ + if( z[0]!='u' ) return 0; + if( !jsonIs4Hex(&z[1]) ) return 0; + *pOp = JSONB_TEXTJ; + return 1; +} + +/* +** Check a single element of the JSONB in pParse for validity. +** +** The element to be checked starts at offset i and must end at on the +** last byte before iEnd. +** +** Return 0 if everything is correct. Return the 1-based byte offset of the +** error if a problem is detected. (In other words, if the error is at offset +** 0, return 1). +*/ +static u32 jsonbValidityCheck( + const JsonParse *pParse, /* Input JSONB. Only aBlob and nBlob are used */ + u32 i, /* Start of element as pParse->aBlob[i] */ + u32 iEnd, /* One more than the last byte of the element */ + u32 iDepth /* Current nesting depth */ +){ + u32 n, sz, j, k; + const u8 *z; + u8 x; + if( iDepth>JSON_MAX_DEPTH ) return i+1; + sz = 0; + n = jsonbPayloadSize(pParse, i, &sz); + if( NEVER(n==0) ) return i+1; /* Checked by caller */ + if( NEVER(i+n+sz!=iEnd) ) return i+1; /* Checked by caller */ + z = pParse->aBlob; + x = z[i] & 0x0f; + switch( x ){ + case JSONB_NULL: + case JSONB_TRUE: + case JSONB_FALSE: { + return n+sz==1 ? 0 : i+1; + } + case JSONB_INT: { + if( sz<1 ) return i+1; + j = i+n; + if( z[j]=='-' ){ + j++; + if( sz<2 ) return i+1; + } + k = i+n+sz; + while( jk ) return j+1; + if( z[j+1]!='.' && z[j+1]!='e' && z[j+1]!='E' ) return j+1; + j++; + } + for(; j0 ) return j+1; + if( x==JSONB_FLOAT && (j==k-1 || !sqlite3Isdigit(z[j+1])) ){ + return j+1; + } + seen = 1; + continue; + } + if( z[j]=='e' || z[j]=='E' ){ + if( seen==2 ) return j+1; + if( j==k-1 ) return j+1; + if( z[j+1]=='+' || z[j+1]=='-' ){ + j++; + if( j==k-1 ) return j+1; + } + seen = 2; + continue; + } + return j+1; + } + if( seen==0 ) return i+1; + return 0; + } + case JSONB_TEXT: { + j = i+n; + k = j+sz; + while( j=k ){ + return j+1; + }else if( strchr("\"\\/bfnrt",z[j+1])!=0 ){ + j++; + }else if( z[j+1]=='u' ){ + if( j+5>=k ) return j+1; + if( !jsonIs4Hex((const char*)&z[j+2]) ) return j+1; + j++; + }else if( x!=JSONB_TEXT5 ){ + return j+1; + }else{ + u32 c = 0; + u32 szC = jsonUnescapeOneChar((const char*)&z[j], k-j, &c); + if( c==JSON_INVALID_CHAR ) return j+1; + j += szC - 1; + } + } + j++; + } + return 0; + } + case JSONB_TEXTRAW: { + return 0; + } + case JSONB_ARRAY: { + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + j += n + sz; + } + assert( j==k ); + return 0; + } + case JSONB_OBJECT: { + u32 cnt = 0; + u32 sub; + j = i+n; + k = j+sz; + while( jk ) return j+1; + if( (cnt & 1)==0 ){ + x = z[j] & 0x0f; + if( xJSONB_TEXTRAW ) return j+1; + } + sub = jsonbValidityCheck(pParse, j, j+n+sz, iDepth+1); + if( sub ) return sub; + cnt++; + j += n + sz; + } + assert( j==k ); + if( (cnt & 1)!=0 ) return j+1; + return 0; + } + default: { + return i+1; + } + } +} + +/* +** Translate a single element of JSON text at pParse->zJson[i] into +** its equivalent binary JSONB representation. Append the translation into +** pParse->aBlob[] beginning at pParse->nBlob. The size of +** pParse->aBlob[] is increased as necessary. +** +** Return the index of the first character past the end of the element parsed, +** or one of the following special result codes: +** +** 0 End of input +** -1 Syntax error or OOM +** -2 '}' seen \ +** -3 ']' seen \___ For these returns, pParse->iErr is set to +** -4 ',' seen / the index in zJson[] of the seen character +** -5 ':' seen / +*/ +static int jsonTranslateTextToBlob(JsonParse *pParse, u32 i){ + char c; + u32 j; + u32 iThis, iStart; + int x; + u8 t; + const char *z = pParse->zJson; +json_parse_restart: + switch( (u8)z[i] ){ + case '{': { + /* Parse object */ + iThis = pParse->nBlob; + jsonBlobAppendNode(pParse, JSONB_OBJECT, pParse->nJson-i, 0); + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + iStart = pParse->nBlob; + for(j=i+1;;j++){ + u32 iBlob = pParse->nBlob; + x = jsonTranslateTextToBlob(pParse, j); + if( x<=0 ){ + int op; + if( x==(-2) ){ + j = pParse->iErr; + if( pParse->nBlob!=(u32)iStart ) pParse->hasNonstd = 1; + break; + } + j += json5Whitespace(&z[j]); + op = JSONB_TEXT; + if( sqlite3JsonId1(z[j]) + || (z[j]=='\\' && jsonIs4HexB(&z[j+1], &op)) + ){ + int k = j+1; + while( (sqlite3JsonId2(z[k]) && json5Whitespace(&z[k])==0) + || (z[k]=='\\' && jsonIs4HexB(&z[k+1], &op)) + ){ + k++; + } + assert( iBlob==pParse->nBlob ); + jsonBlobAppendNode(pParse, op, k-j, &z[j]); + pParse->hasNonstd = 1; + x = k; + }else{ + if( x!=-1 ) pParse->iErr = j; + return -1; + } + } + if( pParse->oom ) return -1; + t = pParse->aBlob[iBlob] & 0x0f; + if( tJSONB_TEXTRAW ){ + pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==':' ){ + j++; + }else{ + if( jsonIsspace(z[j]) ){ + /* strspn() is not helpful here */ + do{ j++; }while( jsonIsspace(z[j]) ); + if( z[j]==':' ){ + j++; + goto parse_object_value; + } + } + x = jsonTranslateTextToBlob(pParse, j); + if( x!=(-5) ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = pParse->iErr+1; + } + parse_object_value: + x = jsonTranslateTextToBlob(pParse, j); + if( x<=0 ){ + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + }else{ + if( jsonIsspace(z[j]) ){ + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); + if( z[j]==',' ){ + continue; + }else if( z[j]=='}' ){ + break; + } + } + x = jsonTranslateTextToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-2) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + pParse->iDepth--; + return j+1; + } + case '[': { + /* Parse array */ + iThis = pParse->nBlob; + assert( i<=(u32)pParse->nJson ); + jsonBlobAppendNode(pParse, JSONB_ARRAY, pParse->nJson - i, 0); + iStart = pParse->nBlob; + if( pParse->oom ) return -1; + if( ++pParse->iDepth > JSON_MAX_DEPTH ){ + pParse->iErr = i; + return -1; + } + for(j=i+1;;j++){ + x = jsonTranslateTextToBlob(pParse, j); + if( x<=0 ){ + if( x==(-3) ){ + j = pParse->iErr; + if( pParse->nBlob!=iStart ) pParse->hasNonstd = 1; + break; + } + if( x!=(-1) ) pParse->iErr = j; + return -1; + } + j = x; + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + }else{ + if( jsonIsspace(z[j]) ){ + j += 1 + (u32)strspn(&z[j+1], jsonSpaces); + if( z[j]==',' ){ + continue; + }else if( z[j]==']' ){ + break; + } + } + x = jsonTranslateTextToBlob(pParse, j); + if( x==(-4) ){ + j = pParse->iErr; + continue; + } + if( x==(-3) ){ + j = pParse->iErr; + break; + } + } + pParse->iErr = j; + return -1; + } + jsonBlobChangePayloadSize(pParse, iThis, pParse->nBlob - iStart); + pParse->iDepth--; + return j+1; + } + case '\'': { + u8 opcode; + char cDelim; + pParse->hasNonstd = 1; + opcode = JSONB_TEXT; + goto parse_string; + case '"': + /* Parse string */ + opcode = JSONB_TEXT; + parse_string: + cDelim = z[i]; + j = i+1; + while( 1 /*exit-by-break*/ ){ + if( jsonIsOk[(u8)z[j]] ){ + if( !jsonIsOk[(u8)z[j+1]] ){ + j += 1; + }else if( !jsonIsOk[(u8)z[j+2]] ){ + j += 2; + }else{ + j += 3; + continue; + } + } + c = z[j]; + if( c==cDelim ){ + break; + }else if( c=='\\' ){ + c = z[++j]; + if( c=='"' || c=='\\' || c=='/' || c=='b' || c=='f' + || c=='n' || c=='r' || c=='t' + || (c=='u' && jsonIs4Hex(&z[j+1])) ){ + if( opcode==JSONB_TEXT ) opcode = JSONB_TEXTJ; + }else if( c=='\'' || c=='0' || c=='v' || c=='\n' + || (0xe2==(u8)c && 0x80==(u8)z[j+1] + && (0xa8==(u8)z[j+2] || 0xa9==(u8)z[j+2])) + || (c=='x' && jsonIs2Hex(&z[j+1])) ){ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='\r' ){ + if( z[j+1]=='\n' ) j++; + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else{ + pParse->iErr = j; + return -1; + } + }else if( c<=0x1f ){ + if( c==0 ){ + pParse->iErr = j; + return -1; + } + /* Control characters are not allowed in canonical JSON string + ** literals, but are allowed in JSON5 string literals. */ + opcode = JSONB_TEXT5; + pParse->hasNonstd = 1; + }else if( c=='"' ){ + opcode = JSONB_TEXT5; + } + j++; + } + jsonBlobAppendNode(pParse, opcode, j-1-i, &z[i+1]); + return j+1; + } + case 't': { + if( strncmp(z+i,"true",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_TRUE); + return i+4; + } + pParse->iErr = i; + return -1; + } + case 'f': { + if( strncmp(z+i,"false",5)==0 && !sqlite3Isalnum(z[i+5]) ){ + jsonBlobAppendOneByte(pParse, JSONB_FALSE); + return i+5; + } + pParse->iErr = i; + return -1; + } + case '+': { + u8 seenE; + pParse->hasNonstd = 1; + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + goto parse_number; + case '.': + if( sqlite3Isdigit(z[i+1]) ){ + pParse->hasNonstd = 1; + t = 0x03; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + seenE = 0; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + case '-': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + /* Parse number */ + t = 0x00; /* Bit 0x01: JSON5. Bit 0x02: FLOAT */ + parse_number: + seenE = 0; + assert( '-' < '0' ); + assert( '+' < '0' ); + assert( '.' < '0' ); + c = z[i]; + + if( c<='0' ){ + if( c=='0' ){ + if( (z[i+1]=='x' || z[i+1]=='X') && sqlite3Isxdigit(z[i+2]) ){ + assert( t==0x00 ); + pParse->hasNonstd = 1; + t = 0x01; + for(j=i+3; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + }else if( sqlite3Isdigit(z[i+1]) ){ + pParse->iErr = i+1; + return -1; + } + }else{ + if( !sqlite3Isdigit(z[i+1]) ){ + /* JSON5 allows for "+Infinity" and "-Infinity" using exactly + ** that case. SQLite also allows these in any case and it allows + ** "+inf" and "-inf". */ + if( (z[i+1]=='I' || z[i+1]=='i') + && sqlite3StrNICmp(&z[i+1], "inf",3)==0 + ){ + pParse->hasNonstd = 1; + if( z[i]=='-' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + } + return i + (sqlite3StrNICmp(&z[i+4],"inity",5)==0 ? 9 : 4); + } + if( z[i+1]=='.' ){ + pParse->hasNonstd = 1; + t |= 0x01; + goto parse_number_2; + } + pParse->iErr = i; + return -1; + } + if( z[i+1]=='0' ){ + if( sqlite3Isdigit(z[i+2]) ){ + pParse->iErr = i+1; + return -1; + }else if( (z[i+2]=='x' || z[i+2]=='X') && sqlite3Isxdigit(z[i+3]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + for(j=i+4; sqlite3Isxdigit(z[j]); j++){} + goto parse_number_finish; + } + } + } + } + + parse_number_2: + for(j=i+1;; j++){ + c = z[j]; + if( sqlite3Isdigit(c) ) continue; + if( c=='.' ){ + if( (t & 0x02)!=0 ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + if( seenE ){ + pParse->iErr = j; + return -1; + } + t |= 0x02; + seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ){ + pParse->iErr = j; + return -1; + } + continue; + } + break; + } + if( z[j-1]<'0' ){ + if( ALWAYS(z[j-1]=='.') && ALWAYS(j-2>=i) && sqlite3Isdigit(z[j-2]) ){ + pParse->hasNonstd = 1; + t |= 0x01; + }else{ + pParse->iErr = j; + return -1; + } + } + parse_number_finish: + assert( JSONB_INT+0x01==JSONB_INT5 ); + assert( JSONB_FLOAT+0x01==JSONB_FLOAT5 ); + assert( JSONB_INT+0x02==JSONB_FLOAT ); + if( z[i]=='+' ) i++; + jsonBlobAppendNode(pParse, JSONB_INT+t, j-i, &z[i]); + return j; + } + case '}': { + pParse->iErr = i; + return -2; /* End of {...} */ + } + case ']': { + pParse->iErr = i; + return -3; /* End of [...] */ + } + case ',': { + pParse->iErr = i; + return -4; /* List separator */ + } + case ':': { + pParse->iErr = i; + return -5; /* Object label/value separator */ + } + case 0: { + return 0; /* End of file */ + } + case 0x09: + case 0x0a: + case 0x0d: + case 0x20: { + i += 1 + (u32)strspn(&z[i+1], jsonSpaces); + goto json_parse_restart; + } + case 0x0b: + case 0x0c: + case '/': + case 0xc2: + case 0xe1: + case 0xe2: + case 0xe3: + case 0xef: { + j = json5Whitespace(&z[i]); + if( j>0 ){ + i += j; + pParse->hasNonstd = 1; + goto json_parse_restart; + } + pParse->iErr = i; + return -1; + } + case 'n': { + if( strncmp(z+i,"null",4)==0 && !sqlite3Isalnum(z[i+4]) ){ + jsonBlobAppendOneByte(pParse, JSONB_NULL); + return i+4; + } + /* fall-through into the default case that checks for NaN */ + /* no break */ deliberate_fall_through + } + default: { + u32 k; + int nn; + c = z[i]; + for(k=0; khasNonstd = 1; + return i + nn; + } + pParse->iErr = i; + return -1; /* Syntax error */ + } + } /* End switch(z[i]) */ +} + + +/* +** Parse a complete JSON string. Return 0 on success or non-zero if there +** are any errors. If an error occurs, free all memory held by pParse, +** but not pParse itself. +** +** pParse must be initialized to an empty parse object prior to calling +** this routine. +*/ +static int jsonConvertTextToBlob( + JsonParse *pParse, /* Initialize and fill this JsonParse object */ + sqlite3_context *pCtx /* Report errors here */ +){ + int i; + const char *zJson = pParse->zJson; + i = jsonTranslateTextToBlob(pParse, 0); + if( pParse->oom ) i = -1; + if( i>0 ){ +#ifdef SQLITE_DEBUG + assert( pParse->iDepth==0 ); + if( sqlite3Config.bJsonSelfcheck ){ + assert( jsonbValidityCheck(pParse, 0, pParse->nBlob, 0)==0 ); + } +#endif + while( jsonIsspace(zJson[i]) ) i++; + if( zJson[i] ){ + i += json5Whitespace(&zJson[i]); + if( zJson[i] ){ + if( pCtx ) sqlite3_result_error(pCtx, "malformed JSON", -1); + jsonParseReset(pParse); + return 1; + } + pParse->hasNonstd = 1; + } + } + if( i<=0 ){ + if( pCtx!=0 ){ + if( pParse->oom ){ + sqlite3_result_error_nomem(pCtx); + }else{ + sqlite3_result_error(pCtx, "malformed JSON", -1); + } + } + jsonParseReset(pParse); + return 1; + } + return 0; +} + +/* +** The input string pStr is a well-formed JSON text string. Convert +** this into the JSONB format and make it the return value of the +** SQL function. +*/ +static void jsonReturnStringAsBlob(JsonString *pStr){ + JsonParse px; + memset(&px, 0, sizeof(px)); + jsonStringTerminate(pStr); + if( pStr->eErr ){ + sqlite3_result_error_nomem(pStr->pCtx); + return; + } + px.zJson = pStr->zBuf; + px.nJson = pStr->nUsed; + px.db = sqlite3_context_db_handle(pStr->pCtx); + (void)jsonTranslateTextToBlob(&px, 0); + if( px.oom ){ + sqlite3DbFree(px.db, px.aBlob); + sqlite3_result_error_nomem(pStr->pCtx); + }else{ + assert( px.nBlobAlloc>0 ); + assert( !px.bReadOnly ); + sqlite3_result_blob(pStr->pCtx, px.aBlob, px.nBlob, SQLITE_DYNAMIC); + } +} + +/* The byte at index i is a node type-code. This routine +** determines the payload size for that node and writes that +** payload size in to *pSz. It returns the offset from i to the +** beginning of the payload. Return 0 on error. +*/ +static u32 jsonbPayloadSize(const JsonParse *pParse, u32 i, u32 *pSz){ + u8 x; + u32 sz; + u32 n; + if( NEVER(i>pParse->nBlob) ){ + *pSz = 0; + return 0; + } + x = pParse->aBlob[i]>>4; + if( x<=11 ){ + sz = x; + n = 1; + }else if( x==12 ){ + if( i+1>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = pParse->aBlob[i+1]; + n = 2; + }else if( x==13 ){ + if( i+2>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+1]<<8) + pParse->aBlob[i+2]; + n = 3; + }else if( x==14 ){ + if( i+4>=pParse->nBlob ){ + *pSz = 0; + return 0; + } + sz = ((u32)pParse->aBlob[i+1]<<24) + (pParse->aBlob[i+2]<<16) + + (pParse->aBlob[i+3]<<8) + pParse->aBlob[i+4]; + n = 5; + }else{ + if( i+8>=pParse->nBlob + || pParse->aBlob[i+1]!=0 + || pParse->aBlob[i+2]!=0 + || pParse->aBlob[i+3]!=0 + || pParse->aBlob[i+4]!=0 + ){ + *pSz = 0; + return 0; + } + sz = (pParse->aBlob[i+5]<<24) + (pParse->aBlob[i+6]<<16) + + (pParse->aBlob[i+7]<<8) + pParse->aBlob[i+8]; + n = 9; + } + if( (i64)i+sz+n > pParse->nBlob + && (i64)i+sz+n > pParse->nBlob-pParse->delta + ){ + sz = 0; + n = 0; + } + *pSz = sz; + return n; +} + + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToText( + const JsonParse *pParse, /* the complete parse of the JSON */ + u32 i, /* Start rendering at this index */ + JsonString *pOut /* Write JSON here */ +){ + u32 sz, n, j, iEnd; + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + jsonAppendRawNZ(pOut, "null", 4); + return i+1; + } + case JSONB_TRUE: { + jsonAppendRawNZ(pOut, "true", 4); + return i+1; + } + case JSONB_FALSE: { + jsonAppendRawNZ(pOut, "false", 5); + return i+1; + } + case JSONB_INT: + case JSONB_FLOAT: { + if( sz==0 ) goto malformed_jsonb; + jsonAppendRaw(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_INT5: { /* Integer literal in hexadecimal notation */ + u32 k = 2; + sqlite3_uint64 u = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + int bOverflow = 0; + if( sz==0 ) goto malformed_jsonb; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + }else if( zIn[0]=='+' ){ + k++; + } + for(; keErr |= JSTRING_MALFORMED; + break; + }else if( (u>>60)!=0 ){ + bOverflow = 1; + }else{ + u = u*16 + sqlite3HexToInt(zIn[k]); + } + } + jsonPrintf(100,pOut,bOverflow?"9.0e999":"%llu", u); + break; + } + case JSONB_FLOAT5: { /* Float literal missing digits beside "." */ + u32 k = 0; + const char *zIn = (const char*)&pParse->aBlob[i+n]; + if( sz==0 ) goto malformed_jsonb; + if( zIn[0]=='-' ){ + jsonAppendChar(pOut, '-'); + k++; + } + if( zIn[k]=='.' ){ + jsonAppendChar(pOut, '0'); + } + for(; kaBlob[i+n], sz); + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXT5: { + const char *zIn; + u32 k; + u32 sz2 = sz; + zIn = (const char*)&pParse->aBlob[i+n]; + jsonAppendChar(pOut, '"'); + while( sz2>0 ){ + for(k=0; k0 ){ + jsonAppendRawNZ(pOut, zIn, k); + if( k>=sz2 ){ + break; + } + zIn += k; + sz2 -= k; + } + if( zIn[0]=='"' ){ + jsonAppendRawNZ(pOut, "\\\"", 2); + zIn++; + sz2--; + continue; + } + if( zIn[0]<=0x1f ){ + if( pOut->nUsed+7>pOut->nAlloc && jsonStringGrow(pOut,7) ) break; + jsonAppendControlChar(pOut, zIn[0]); + zIn++; + sz2--; + continue; + } + assert( zIn[0]=='\\' ); + assert( sz2>=1 ); + if( sz2<2 ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + switch( (u8)zIn[1] ){ + case '\'': + jsonAppendChar(pOut, '\''); + break; + case 'v': + jsonAppendRawNZ(pOut, "\\u0009", 6); + break; + case 'x': + if( sz2<4 ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 2; + break; + } + jsonAppendRawNZ(pOut, "\\u00", 4); + jsonAppendRawNZ(pOut, &zIn[2], 2); + zIn += 2; + sz2 -= 2; + break; + case '0': + jsonAppendRawNZ(pOut, "\\u0000", 6); + break; + case '\r': + if( sz2>2 && zIn[2]=='\n' ){ + zIn++; + sz2--; + } + break; + case '\n': + break; + case 0xe2: + /* '\' followed by either U+2028 or U+2029 is ignored as + ** whitespace. Not that in UTF8, U+2028 is 0xe2 0x80 0x29. + ** U+2029 is the same except for the last byte */ + if( sz2<4 + || 0x80!=(u8)zIn[2] + || (0xa8!=(u8)zIn[3] && 0xa9!=(u8)zIn[3]) + ){ + pOut->eErr |= JSTRING_MALFORMED; + sz2 = 2; + break; + } + zIn += 2; + sz2 -= 2; + break; + default: + jsonAppendRawNZ(pOut, zIn, 2); + break; + } + assert( sz2>=2 ); + zIn += 2; + sz2 -= 2; + } + jsonAppendChar(pOut, '"'); + break; + } + case JSONB_TEXTRAW: { + jsonAppendString(pOut, (const char*)&pParse->aBlob[i+n], sz); + break; + } + case JSONB_ARRAY: { + jsonAppendChar(pOut, '['); + j = i+n; + iEnd = j+sz; + while( jeErr==0 ){ + j = jsonTranslateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, ','); + } + if( j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); + jsonAppendChar(pOut, ']'); + break; + } + case JSONB_OBJECT: { + int x = 0; + jsonAppendChar(pOut, '{'); + j = i+n; + iEnd = j+sz; + while( jeErr==0 ){ + j = jsonTranslateBlobToText(pParse, j, pOut); + jsonAppendChar(pOut, (x++ & 1) ? ',' : ':'); + } + if( (x & 1)!=0 || j>iEnd ) pOut->eErr |= JSTRING_MALFORMED; + if( sz>0 ) jsonStringTrimOneChar(pOut); + jsonAppendChar(pOut, '}'); + break; + } + + default: { + malformed_jsonb: + pOut->eErr |= JSTRING_MALFORMED; + break; + } + } + return i+n+sz; +} + +/* Context for recursion of json_pretty() +*/ +typedef struct JsonPretty JsonPretty; +struct JsonPretty { + JsonParse *pParse; /* The BLOB being rendered */ + JsonString *pOut; /* Generate pretty output into this string */ + const char *zIndent; /* Use this text for indentation */ + u32 szIndent; /* Bytes in zIndent[] */ + u32 nIndent; /* Current level of indentation */ +}; + +/* Append indentation to the pretty JSON under construction */ +static void jsonPrettyIndent(JsonPretty *pPretty){ + u32 jj; + for(jj=0; jjnIndent; jj++){ + jsonAppendRaw(pPretty->pOut, pPretty->zIndent, pPretty->szIndent); + } +} + +/* +** Translate the binary JSONB representation of JSON beginning at +** pParse->aBlob[i] into a JSON text string. Append the JSON +** text onto the end of pOut. Return the index in pParse->aBlob[] +** of the first byte past the end of the element that is translated. +** +** This is a variant of jsonTranslateBlobToText() that "pretty-prints" +** the output. Extra whitespace is inserted to make the JSON easier +** for humans to read. +** +** If an error is detected in the BLOB input, the pOut->eErr flag +** might get set to JSTRING_MALFORMED. But not all BLOB input errors +** are detected. So a malformed JSONB input might either result +** in an error, or in incorrect JSON. +** +** The pOut->eErr JSTRING_OOM flag is set on a OOM. +*/ +static u32 jsonTranslateBlobToPrettyText( + JsonPretty *pPretty, /* Pretty-printing context */ + u32 i /* Start rendering at this index */ +){ + u32 sz, n, j, iEnd; + const JsonParse *pParse = pPretty->pParse; + JsonString *pOut = pPretty->pOut; + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + pOut->eErr |= JSTRING_MALFORMED; + return pParse->nBlob+1; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_ARRAY: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '['); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, ']'); + i = iEnd; + break; + } + case JSONB_OBJECT: { + j = i+n; + iEnd = j+sz; + jsonAppendChar(pOut, '{'); + if( jnIndent++; + while( pOut->eErr==0 ){ + jsonPrettyIndent(pPretty); + j = jsonTranslateBlobToText(pParse, j, pOut); + if( j>iEnd ){ + pOut->eErr |= JSTRING_MALFORMED; + break; + } + jsonAppendRawNZ(pOut, ": ", 2); + j = jsonTranslateBlobToPrettyText(pPretty, j); + if( j>=iEnd ) break; + jsonAppendRawNZ(pOut, ",\n", 2); + } + jsonAppendChar(pOut, '\n'); + pPretty->nIndent--; + jsonPrettyIndent(pPretty); + } + jsonAppendChar(pOut, '}'); + i = iEnd; + break; + } + default: { + i = jsonTranslateBlobToText(pParse, i, pOut); + break; + } + } + return i; +} + + +/* Return true if the input pJson +** +** For performance reasons, this routine does not do a detailed check of the +** input BLOB to ensure that it is well-formed. Hence, false positives are +** possible. False negatives should never occur, however. +*/ +static int jsonFuncArgMightBeBinary(sqlite3_value *pJson){ + u32 sz, n; + const u8 *aBlob; + int nBlob; + JsonParse s; + if( sqlite3_value_type(pJson)!=SQLITE_BLOB ) return 0; + aBlob = sqlite3_value_blob(pJson); + nBlob = sqlite3_value_bytes(pJson); + if( nBlob<1 ) return 0; + if( NEVER(aBlob==0) || (aBlob[0] & 0x0f)>JSONB_OBJECT ) return 0; + memset(&s, 0, sizeof(s)); + s.aBlob = (u8*)aBlob; + s.nBlob = nBlob; + n = jsonbPayloadSize(&s, 0, &sz); + if( n==0 ) return 0; + if( sz+n!=(u32)nBlob ) return 0; + if( (aBlob[0] & 0x0f)<=JSONB_FALSE && sz>0 ) return 0; + return sz+n==(u32)nBlob; +} + +/* +** Given that a JSONB_ARRAY object starts at offset i, return +** the number of entries in that array. +*/ +static u32 jsonbArrayCount(JsonParse *pParse, u32 iRoot){ + u32 n, sz, i, iEnd; + u32 k = 0; + n = jsonbPayloadSize(pParse, iRoot, &sz); + iEnd = iRoot+n+sz; + for(i=iRoot+n; n>0 && idelta. +*/ +static void jsonAfterEditSizeAdjust(JsonParse *pParse, u32 iRoot){ + u32 sz = 0; + u32 nBlob; + assert( pParse->delta!=0 ); + assert( pParse->nBlobAlloc >= pParse->nBlob ); + nBlob = pParse->nBlob; + pParse->nBlob = pParse->nBlobAlloc; + (void)jsonbPayloadSize(pParse, iRoot, &sz); + pParse->nBlob = nBlob; + sz += pParse->delta; + pParse->delta += jsonBlobChangePayloadSize(pParse, iRoot, sz); +} + +/* +** Modify the JSONB blob at pParse->aBlob by removing nDel bytes of +** content beginning at iDel, and replacing them with nIns bytes of +** content given by aIns. +** +** nDel may be zero, in which case no bytes are removed. But iDel is +** still important as new bytes will be insert beginning at iDel. +** +** aIns may be zero, in which case space is created to hold nIns bytes +** beginning at iDel, but that space is uninitialized. +** +** Set pParse->oom if an OOM occurs. +*/ +static void jsonBlobEdit( + JsonParse *pParse, /* The JSONB to be modified is in pParse->aBlob */ + u32 iDel, /* First byte to be removed */ + u32 nDel, /* Number of bytes to remove */ + const u8 *aIns, /* Content to insert */ + u32 nIns /* Bytes of content to insert */ +){ + i64 d = (i64)nIns - (i64)nDel; + if( d!=0 ){ + if( pParse->nBlob + d > pParse->nBlobAlloc ){ + jsonBlobExpand(pParse, pParse->nBlob+d); + if( pParse->oom ) return; + } + memmove(&pParse->aBlob[iDel+nIns], + &pParse->aBlob[iDel+nDel], + pParse->nBlob - (iDel+nDel)); + pParse->nBlob += d; + pParse->delta += d; + } + if( nIns && aIns ) memcpy(&pParse->aBlob[iDel], aIns, nIns); +} + +/* +** Return the number of escaped newlines to be ignored. +** An escaped newline is a one of the following byte sequences: +** +** 0x5c 0x0a +** 0x5c 0x0d +** 0x5c 0x0d 0x0a +** 0x5c 0xe2 0x80 0xa8 +** 0x5c 0xe2 0x80 0xa9 +*/ +static u32 jsonBytesToBypass(const char *z, u32 n){ + u32 i = 0; + while( i+10 ); + assert( z[0]=='\\' ); + if( n<2 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + switch( (u8)z[1] ){ + case 'u': { + u32 v, vlo; + if( n<6 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + v = jsonHexToInt4(&z[2]); + if( (v & 0xfc00)==0xd800 + && n>=12 + && z[6]=='\\' + && z[7]=='u' + && ((vlo = jsonHexToInt4(&z[8]))&0xfc00)==0xdc00 + ){ + *piOut = ((v&0x3ff)<<10) + (vlo&0x3ff) + 0x10000; + return 12; + }else{ + *piOut = v; + return 6; + } + } + case 'b': { *piOut = '\b'; return 2; } + case 'f': { *piOut = '\f'; return 2; } + case 'n': { *piOut = '\n'; return 2; } + case 'r': { *piOut = '\r'; return 2; } + case 't': { *piOut = '\t'; return 2; } + case 'v': { *piOut = '\v'; return 2; } + case '0': { *piOut = 0; return 2; } + case '\'': + case '"': + case '/': + case '\\':{ *piOut = z[1]; return 2; } + case 'x': { + if( n<4 ){ + *piOut = JSON_INVALID_CHAR; + return n; + } + *piOut = (jsonHexToInt(z[2])<<4) | jsonHexToInt(z[3]); + return 4; + } + case 0xe2: + case '\r': + case '\n': { + u32 nSkip = jsonBytesToBypass(z, n); + if( nSkip==0 ){ + *piOut = JSON_INVALID_CHAR; + return n; + }else if( nSkip==n ){ + *piOut = 0; + return n; + }else if( z[nSkip]=='\\' ){ + return nSkip + jsonUnescapeOneChar(&z[nSkip], n-nSkip, piOut); + }else{ + int sz = sqlite3Utf8ReadLimited((u8*)&z[nSkip], n-nSkip, piOut); + return nSkip + sz; + } + } + default: { + *piOut = JSON_INVALID_CHAR; + return 2; + } + } +} + + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. +** +** In this version, we know that one or the other or both of the +** two comparands contains an escape sequence. +*/ +static SQLITE_NOINLINE int jsonLabelCompareEscaped( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + u32 cLeft, cRight; + assert( rawLeft==0 || rawRight==0 ); + while( 1 /*exit-by-return*/ ){ + if( nLeft==0 ){ + cLeft = 0; + }else if( rawLeft || zLeft[0]!='\\' ){ + cLeft = ((u8*)zLeft)[0]; + if( cLeft>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zLeft, nLeft, &cLeft); + zLeft += sz; + nLeft -= sz; + }else{ + zLeft++; + nLeft--; + } + }else{ + u32 n = jsonUnescapeOneChar(zLeft, nLeft, &cLeft); + zLeft += n; + assert( n<=nLeft ); + nLeft -= n; + } + if( nRight==0 ){ + cRight = 0; + }else if( rawRight || zRight[0]!='\\' ){ + cRight = ((u8*)zRight)[0]; + if( cRight>=0xc0 ){ + int sz = sqlite3Utf8ReadLimited((u8*)zRight, nRight, &cRight); + zRight += sz; + nRight -= sz; + }else{ + zRight++; + nRight--; + } + }else{ + u32 n = jsonUnescapeOneChar(zRight, nRight, &cRight); + zRight += n; + assert( n<=nRight ); + nRight -= n; + } + if( cLeft!=cRight ) return 0; + if( cLeft==0 ) return 1; + } +} + +/* +** Compare two object labels. Return 1 if they are equal and +** 0 if they differ. Return -1 if an OOM occurs. +*/ +static int jsonLabelCompare( + const char *zLeft, /* The left label */ + u32 nLeft, /* Size of the left label in bytes */ + int rawLeft, /* True if zLeft contains no escapes */ + const char *zRight, /* The right label */ + u32 nRight, /* Size of the right label in bytes */ + int rawRight /* True if zRight is escape-free */ +){ + if( rawLeft && rawRight ){ + /* Simpliest case: Neither label contains escapes. A simple + ** memcmp() is sufficient. */ + if( nLeft!=nRight ) return 0; + return memcmp(zLeft, zRight, nLeft)==0; + }else{ + return jsonLabelCompareEscaped(zLeft, nLeft, rawLeft, + zRight, nRight, rawRight); + } +} + +/* +** Error returns from jsonLookupStep() +*/ +#define JSON_LOOKUP_ERROR 0xffffffff +#define JSON_LOOKUP_NOTFOUND 0xfffffffe +#define JSON_LOOKUP_PATHERROR 0xfffffffd +#define JSON_LOOKUP_ISERROR(x) ((x)>=JSON_LOOKUP_PATHERROR) + +/* Forward declaration */ +static u32 jsonLookupStep(JsonParse*,u32,const char*,u32); + + +/* This helper routine for jsonLookupStep() populates pIns with +** binary data that is to be inserted into pParse. +** +** In the common case, pIns just points to pParse->aIns and pParse->nIns. +** But if the zPath of the original edit operation includes path elements +** that go deeper, additional substructure must be created. +** +** For example: +** +** json_insert('{}', '$.a.b.c', 123); +** +** The search stops at '$.a' But additional substructure must be +** created for the ".b.c" part of the patch so that the final result +** is: {"a":{"b":{"c"::123}}}. This routine populates pIns with +** the binary equivalent of {"b":{"c":123}} so that it can be inserted. +** +** The caller is responsible for resetting pIns when it has finished +** using the substructure. +*/ +static u32 jsonCreateEditSubstructure( + JsonParse *pParse, /* The original JSONB that is being edited */ + JsonParse *pIns, /* Populate this with the blob data to insert */ + const char *zTail /* Tail of the path that determins substructure */ +){ + static const u8 emptyObject[] = { JSONB_ARRAY, JSONB_OBJECT }; + int rc; + memset(pIns, 0, sizeof(*pIns)); + pIns->db = pParse->db; + if( zTail[0]==0 ){ + /* No substructure. Just insert what is given in pParse. */ + pIns->aBlob = pParse->aIns; + pIns->nBlob = pParse->nIns; + rc = 0; + }else{ + /* Construct the binary substructure */ + pIns->nBlob = 1; + pIns->aBlob = (u8*)&emptyObject[zTail[0]=='.']; + pIns->eEdit = pParse->eEdit; + pIns->nIns = pParse->nIns; + pIns->aIns = pParse->aIns; + rc = jsonLookupStep(pIns, 0, zTail, 0); + pParse->oom |= pIns->oom; + } + return rc; /* Error code only */ +} + +/* +** Search along zPath to find the Json element specified. Return an +** index into pParse->aBlob[] for the start of that element's value. +** +** If the value found by this routine is the value half of label/value pair +** within an object, then set pPath->iLabel to the start of the corresponding +** label, before returning. +** +** Return one of the JSON_LOOKUP error codes if problems are seen. +** +** This routine will also modify the blob. If pParse->eEdit is one of +** JEDIT_DEL, JEDIT_REPL, JEDIT_INS, or JEDIT_SET, then changes might be +** made to the selected value. If an edit is performed, then the return +** value does not necessarily point to the select element. If an edit +** is performed, the return value is only useful for detecting error +** conditions. +*/ +static u32 jsonLookupStep( + JsonParse *pParse, /* The JSON to search */ + u32 iRoot, /* Begin the search at this element of aBlob[] */ + const char *zPath, /* The path to search */ + u32 iLabel /* Label if iRoot is a value of in an object */ +){ + u32 i, j, k, nKey, sz, n, iEnd, rc; + const char *zKey; + u8 x; + + if( zPath[0]==0 ){ + if( pParse->eEdit && jsonBlobMakeEditable(pParse, pParse->nIns) ){ + n = jsonbPayloadSize(pParse, iRoot, &sz); + sz += n; + if( pParse->eEdit==JEDIT_DEL ){ + if( iLabel>0 ){ + sz += iRoot - iLabel; + iRoot = iLabel; + } + jsonBlobEdit(pParse, iRoot, sz, 0, 0); + }else if( pParse->eEdit==JEDIT_INS ){ + /* Already exists, so json_insert() is a no-op */ + }else{ + /* json_set() or json_replace() */ + jsonBlobEdit(pParse, iRoot, sz, pParse->aIns, pParse->nIns); + } + } + pParse->iLabel = iLabel; + return iRoot; + } + if( zPath[0]=='.' ){ + int rawKey = 1; + x = pParse->aBlob[iRoot]; + zPath++; + if( zPath[0]=='"' ){ + zKey = zPath + 1; + for(i=1; zPath[i] && zPath[i]!='"'; i++){ + if( zPath[i]=='\\' && zPath[i+1]!=0 ) i++; + } + nKey = i-1; + if( zPath[i] ){ + i++; + }else{ + return JSON_LOOKUP_PATHERROR; + } + testcase( nKey==0 ); + rawKey = memchr(zKey, '\\', nKey)==0; + }else{ + zKey = zPath; + for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} + nKey = i; + if( nKey==0 ){ + return JSON_LOOKUP_PATHERROR; + } + } + if( (x & 0x0f)!=JSONB_OBJECT ) return JSON_LOOKUP_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + j = iRoot + n; /* j is the index of a label */ + iEnd = j+sz; + while( jaBlob[j] & 0x0f; + if( xJSONB_TEXTRAW ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + k = j+n; /* k is the index of the label text */ + if( k+sz>=iEnd ) return JSON_LOOKUP_ERROR; + zLabel = (const char*)&pParse->aBlob[k]; + rawLabel = x==JSONB_TEXT || x==JSONB_TEXTRAW; + if( jsonLabelCompare(zKey, nKey, rawKey, zLabel, sz, rawLabel) ){ + u32 v = k+sz; /* v is the index of the value */ + if( ((pParse->aBlob[v])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, v, &sz); + if( n==0 || v+n+sz>iEnd ) return JSON_LOOKUP_ERROR; + assert( j>0 ); + rc = jsonLookupStep(pParse, v, &zPath[i], j); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + j = k+sz; + if( ((pParse->aBlob[j])&0x0f)>JSONB_OBJECT ) return JSON_LOOKUP_ERROR; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( pParse->eEdit>=JEDIT_INS ){ + u32 nIns; /* Total bytes to insert (label+value) */ + JsonParse v; /* BLOB encoding of the value to be inserted */ + JsonParse ix; /* Header of the label to be inserted */ + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + memset(&ix, 0, sizeof(ix)); + ix.db = pParse->db; + jsonBlobAppendNode(&ix, rawKey?JSONB_TEXTRAW:JSONB_TEXT5, nKey, 0); + pParse->oom |= ix.oom; + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, ix.nBlob+nKey+v.nBlob) + ){ + assert( !pParse->oom ); + nIns = ix.nBlob + nKey + v.nBlob; + jsonBlobEdit(pParse, j, 0, 0, nIns); + if( !pParse->oom ){ + assert( pParse->aBlob!=0 ); /* Because pParse->oom!=0 */ + assert( ix.aBlob!=0 ); /* Because pPasre->oom!=0 */ + memcpy(&pParse->aBlob[j], ix.aBlob, ix.nBlob); + k = j + ix.nBlob; + memcpy(&pParse->aBlob[k], zKey, nKey); + k += nKey; + memcpy(&pParse->aBlob[k], v.aBlob, v.nBlob); + if( ALWAYS(pParse->delta) ) jsonAfterEditSizeAdjust(pParse, iRoot); + } + } + jsonParseReset(&v); + jsonParseReset(&ix); + return rc; + } + }else if( zPath[0]=='[' ){ + x = pParse->aBlob[iRoot] & 0x0f; + if( x!=JSONB_ARRAY ) return JSON_LOOKUP_NOTFOUND; + n = jsonbPayloadSize(pParse, iRoot, &sz); + k = 0; + i = 1; + while( sqlite3Isdigit(zPath[i]) ){ + k = k*10 + zPath[i] - '0'; + i++; + } + if( i<2 || zPath[i]!=']' ){ + if( zPath[1]=='#' ){ + k = jsonbArrayCount(pParse, iRoot); + i = 2; + if( zPath[2]=='-' && sqlite3Isdigit(zPath[3]) ){ + unsigned int nn = 0; + i = 3; + do{ + nn = nn*10 + zPath[i] - '0'; + i++; + }while( sqlite3Isdigit(zPath[i]) ); + if( nn>k ) return JSON_LOOKUP_NOTFOUND; + k -= nn; + } + if( zPath[i]!=']' ){ + return JSON_LOOKUP_PATHERROR; + } + }else{ + return JSON_LOOKUP_PATHERROR; + } + } + j = iRoot+n; + iEnd = j+sz; + while( jdelta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + k--; + n = jsonbPayloadSize(pParse, j, &sz); + if( n==0 ) return JSON_LOOKUP_ERROR; + j += n+sz; + } + if( j>iEnd ) return JSON_LOOKUP_ERROR; + if( k>0 ) return JSON_LOOKUP_NOTFOUND; + if( pParse->eEdit>=JEDIT_INS ){ + JsonParse v; + testcase( pParse->eEdit==JEDIT_INS ); + testcase( pParse->eEdit==JEDIT_SET ); + rc = jsonCreateEditSubstructure(pParse, &v, &zPath[i+1]); + if( !JSON_LOOKUP_ISERROR(rc) + && jsonBlobMakeEditable(pParse, v.nBlob) + ){ + assert( !pParse->oom ); + jsonBlobEdit(pParse, j, 0, v.aBlob, v.nBlob); + } + jsonParseReset(&v); + if( pParse->delta ) jsonAfterEditSizeAdjust(pParse, iRoot); + return rc; + } + }else{ + return JSON_LOOKUP_PATHERROR; + } + return JSON_LOOKUP_NOTFOUND; +} + +/* +** Convert a JSON BLOB into text and make that text the return value +** of an SQL function. +*/ +static void jsonReturnTextJsonFromBlob( + sqlite3_context *ctx, + const u8 *aBlob, + u32 nBlob +){ + JsonParse x; + JsonString s; + + if( NEVER(aBlob==0) ) return; + memset(&x, 0, sizeof(x)); + x.aBlob = (u8*)aBlob; + x.nBlob = nBlob; + jsonStringInit(&s, ctx); + jsonTranslateBlobToText(&x, 0, &s); + jsonReturnString(&s, 0, 0); +} + + +/* +** Return the value of the BLOB node at index i. +** +** If the value is a primitive, return it as an SQL value. +** If the value is an array or object, return it as either +** JSON text or the BLOB encoding, depending on the JSON_B flag +** on the userdata. +*/ +static void jsonReturnFromBlob( + JsonParse *pParse, /* Complete JSON parse tree */ + u32 i, /* Index of the node */ + sqlite3_context *pCtx, /* Return value for this function */ + int textOnly /* return text JSON. Disregard user-data */ +){ + u32 n, sz; + int rc; + sqlite3 *db = sqlite3_context_db_handle(pCtx); + + n = jsonbPayloadSize(pParse, i, &sz); + if( n==0 ){ + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; + } + switch( pParse->aBlob[i] & 0x0f ){ + case JSONB_NULL: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_null(pCtx); + break; + } + case JSONB_TRUE: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_int(pCtx, 1); + break; + } + case JSONB_FALSE: { + if( sz ) goto returnfromblob_malformed; + sqlite3_result_int(pCtx, 0); + break; + } + case JSONB_INT5: + case JSONB_INT: { + sqlite3_int64 iRes = 0; + char *z; + int bNeg = 0; + char x; + if( sz==0 ) goto returnfromblob_malformed; + x = (char)pParse->aBlob[i+n]; + if( x=='-' ){ + if( sz<2 ) goto returnfromblob_malformed; + n++; + sz--; + bNeg = 1; + } + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3DecOrHexToI64(z, &iRes); + sqlite3DbFree(db, z); + if( rc==0 ){ + sqlite3_result_int64(pCtx, bNeg ? -iRes : iRes); + }else if( rc==3 && bNeg ){ + sqlite3_result_int64(pCtx, SMALLEST_INT64); + }else if( rc==1 ){ + goto returnfromblob_malformed; + }else{ + if( bNeg ){ n--; sz++; } + goto to_double; + } + break; + } + case JSONB_FLOAT5: + case JSONB_FLOAT: { + double r; + char *z; + if( sz==0 ) goto returnfromblob_malformed; + to_double: + z = sqlite3DbStrNDup(db, (const char*)&pParse->aBlob[i+n], (int)sz); + if( z==0 ) goto returnfromblob_oom; + rc = sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); + sqlite3DbFree(db, z); + if( rc<=0 ) goto returnfromblob_malformed; + sqlite3_result_double(pCtx, r); + break; + } + case JSONB_TEXTRAW: + case JSONB_TEXT: { + sqlite3_result_text(pCtx, (char*)&pParse->aBlob[i+n], sz, + SQLITE_TRANSIENT); + break; + } + case JSONB_TEXT5: + case JSONB_TEXTJ: { + /* Translate JSON formatted string into raw text */ + u32 iIn, iOut; + const char *z; + char *zOut; + u32 nOut = sz; + z = (const char*)&pParse->aBlob[i+n]; + zOut = sqlite3DbMallocRaw(db, nOut+1); + if( zOut==0 ) goto returnfromblob_oom; + for(iIn=iOut=0; iIn=2 ); + zOut[iOut++] = (char)(0xc0 | (v>>6)); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v<0x10000 ){ + assert( szEscape>=3 ); + zOut[iOut++] = 0xe0 | (v>>12); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + }else if( v==JSON_INVALID_CHAR ){ + /* Silently ignore illegal unicode */ + }else{ + assert( szEscape>=4 ); + zOut[iOut++] = 0xf0 | (v>>18); + zOut[iOut++] = 0x80 | ((v>>12)&0x3f); + zOut[iOut++] = 0x80 | ((v>>6)&0x3f); + zOut[iOut++] = 0x80 | (v&0x3f); + } + iIn += szEscape - 1; + }else{ + zOut[iOut++] = c; + } + } /* end for() */ + assert( iOut<=nOut ); + zOut[iOut] = 0; + sqlite3_result_text(pCtx, zOut, iOut, SQLITE_DYNAMIC); + break; + } + case JSONB_ARRAY: + case JSONB_OBJECT: { + int flags = textOnly ? 0 : SQLITE_PTR_TO_INT(sqlite3_user_data(pCtx)); + if( flags & JSON_BLOB ){ + sqlite3_result_blob(pCtx, &pParse->aBlob[i], sz+n, SQLITE_TRANSIENT); + }else{ + jsonReturnTextJsonFromBlob(pCtx, &pParse->aBlob[i], sz+n); + } + break; + } + default: { + goto returnfromblob_malformed; + } + } + return; + +returnfromblob_oom: + sqlite3_result_error_nomem(pCtx); + return; + +returnfromblob_malformed: + sqlite3_result_error(pCtx, "malformed JSON", -1); + return; +} + +/* +** pArg is a function argument that might be an SQL value or a JSON +** value. Figure out what it is and encode it as a JSONB blob. +** Return the results in pParse. +** +** pParse is uninitialized upon entry. This routine will handle the +** initialization of pParse. The result will be contained in +** pParse->aBlob and pParse->nBlob. pParse->aBlob might be dynamically +** allocated (if pParse->nBlobAlloc is greater than zero) in which case +** the caller is responsible for freeing the space allocated to pParse->aBlob +** when it has finished with it. Or pParse->aBlob might be a static string +** or a value obtained from sqlite3_value_blob(pArg). +** +** If the argument is a BLOB that is clearly not a JSONB, then this +** function might set an error message in ctx and return non-zero. +** It might also set an error message and return non-zero on an OOM error. +*/ +static int jsonFunctionArgToBlob( + sqlite3_context *ctx, + sqlite3_value *pArg, + JsonParse *pParse +){ + int eType = sqlite3_value_type(pArg); + static u8 aNull[] = { 0x00 }; + memset(pParse, 0, sizeof(pParse[0])); + pParse->db = sqlite3_context_db_handle(ctx); + switch( eType ){ + default: { + pParse->aBlob = aNull; + pParse->nBlob = 1; + return 0; + } + case SQLITE_BLOB: { + if( jsonFuncArgMightBeBinary(pArg) ){ + pParse->aBlob = (u8*)sqlite3_value_blob(pArg); + pParse->nBlob = sqlite3_value_bytes(pArg); + }else{ + sqlite3_result_error(ctx, "JSON cannot hold BLOB values", -1); + return 1; + } + break; + } + case SQLITE_TEXT: { + const char *zJson = (const char*)sqlite3_value_text(pArg); + int nJson = sqlite3_value_bytes(pArg); + if( zJson==0 ) return 1; + if( sqlite3_value_subtype(pArg)==JSON_SUBTYPE ){ + pParse->zJson = (char*)zJson; + pParse->nJson = nJson; + if( jsonConvertTextToBlob(pParse, ctx) ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + sqlite3DbFree(pParse->db, pParse->aBlob); + memset(pParse, 0, sizeof(pParse[0])); + return 1; + } + }else{ + jsonBlobAppendNode(pParse, JSONB_TEXTRAW, nJson, zJson); + } + break; + } + case SQLITE_FLOAT: { + double r = sqlite3_value_double(pArg); + if( NEVER(sqlite3IsNaN(r)) ){ + jsonBlobAppendNode(pParse, JSONB_NULL, 0, 0); + }else{ + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + if( z[0]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 5, "9e999"); + }else if( z[0]=='-' && z[1]=='I' ){ + jsonBlobAppendNode(pParse, JSONB_FLOAT, 6, "-9e999"); + }else{ + jsonBlobAppendNode(pParse, JSONB_FLOAT, n, z); + } + } + break; + } + case SQLITE_INTEGER: { + int n = sqlite3_value_bytes(pArg); + const char *z = (const char*)sqlite3_value_text(pArg); + if( z==0 ) return 1; + jsonBlobAppendNode(pParse, JSONB_INT, n, z); + break; + } + } + if( pParse->oom ){ + sqlite3_result_error_nomem(ctx); + return 1; + }else{ + return 0; + } +} + +/* +** Generate a bad path error. +** +** If ctx is not NULL then push the error message into ctx and return NULL. +** If ctx is NULL, then return the text of the error message. +*/ +static char *jsonBadPathError( + sqlite3_context *ctx, /* The function call containing the error */ + const char *zPath /* The path with the problem */ +){ + char *zMsg = sqlite3_mprintf("bad JSON path: %Q", zPath); + if( ctx==0 ) return zMsg; + if( zMsg ){ + sqlite3_result_error(ctx, zMsg, -1); + sqlite3_free(zMsg); + }else{ + sqlite3_result_error_nomem(ctx); + } + return 0; +} + +/* argv[0] is a BLOB that seems likely to be a JSONB. Subsequent +** arguments come in parse where each pair contains a JSON path and +** content to insert or set at that patch. Do the updates +** and return the result. +** +** The specific operation is determined by eEdit, which can be one +** of JEDIT_INS, JEDIT_REPL, or JEDIT_SET. +*/ +static void jsonInsertIntoBlob( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv, + int eEdit /* JEDIT_INS, JEDIT_REPL, or JEDIT_SET */ +){ + int i; + u32 rc = 0; + const char *zPath = 0; + int flgs; + JsonParse *p; + JsonParse ax; + + assert( (argc&1)==1 ); + flgs = argc==1 ? 0 : JSON_EDITABLE; + p = jsonParseFuncArg(ctx, argv[0], flgs); + if( p==0 ) return; + for(i=1; inBlob, ax.aBlob, ax.nBlob); + } + rc = 0; + }else{ + p->eEdit = eEdit; + p->nIns = ax.nBlob; + p->aIns = ax.aBlob; + p->delta = 0; + rc = jsonLookupStep(p, 0, zPath+1, 0); + } + jsonParseReset(&ax); + if( rc==JSON_LOOKUP_NOTFOUND ) continue; + if( JSON_LOOKUP_ISERROR(rc) ) goto jsonInsertIntoBlob_patherror; + } + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +jsonInsertIntoBlob_patherror: + jsonParseFree(p); + if( rc==JSON_LOOKUP_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + }else{ + jsonBadPathError(ctx, zPath); + } + return; +} + +/* +** If pArg is a blob that seems like a JSONB blob, then initialize +** p to point to that JSONB and return TRUE. If pArg does not seem like +** a JSONB blob, then return FALSE; +** +** This routine is only called if it is already known that pArg is a +** blob. The only open question is whether or not the blob appears +** to be a JSONB blob. +*/ +static int jsonArgIsJsonb(sqlite3_value *pArg, JsonParse *p){ + u32 n, sz = 0; + p->aBlob = (u8*)sqlite3_value_blob(pArg); + p->nBlob = (u32)sqlite3_value_bytes(pArg); + if( p->nBlob==0 ){ + p->aBlob = 0; + return 0; + } + if( NEVER(p->aBlob==0) ){ + return 0; + } + if( (p->aBlob[0] & 0x0f)<=JSONB_OBJECT + && (n = jsonbPayloadSize(p, 0, &sz))>0 + && sz+n==p->nBlob + && ((p->aBlob[0] & 0x0f)>JSONB_FALSE || sz==0) + ){ + return 1; + } + p->aBlob = 0; + p->nBlob = 0; + return 0; +} + +/* +** Generate a JsonParse object, containing valid JSONB in aBlob and nBlob, +** from the SQL function argument pArg. Return a pointer to the new +** JsonParse object. +** +** Ownership of the new JsonParse object is passed to the caller. The +** caller should invoke jsonParseFree() on the return value when it +** has finished using it. +** +** If any errors are detected, an appropriate error messages is set +** using sqlite3_result_error() or the equivalent and this routine +** returns NULL. This routine also returns NULL if the pArg argument +** is an SQL NULL value, but no error message is set in that case. This +** is so that SQL functions that are given NULL arguments will return +** a NULL value. +*/ +static JsonParse *jsonParseFuncArg( + sqlite3_context *ctx, + sqlite3_value *pArg, + u32 flgs +){ + int eType; /* Datatype of pArg */ + JsonParse *p = 0; /* Value to be returned */ + JsonParse *pFromCache = 0; /* Value taken from cache */ + sqlite3 *db; /* The database connection */ + + assert( ctx!=0 ); + eType = sqlite3_value_type(pArg); + if( eType==SQLITE_NULL ){ + return 0; + } + pFromCache = jsonCacheSearch(ctx, pArg); + if( pFromCache ){ + pFromCache->nJPRef++; + if( (flgs & JSON_EDITABLE)==0 ){ + return pFromCache; + } + } + db = sqlite3_context_db_handle(ctx); +rebuild_from_cache: + p = sqlite3DbMallocZero(db, sizeof(*p)); + if( p==0 ) goto json_pfa_oom; + memset(p, 0, sizeof(*p)); + p->db = db; + p->nJPRef = 1; + if( pFromCache!=0 ){ + u32 nBlob = pFromCache->nBlob; + p->aBlob = sqlite3DbMallocRaw(db, nBlob); + if( p->aBlob==0 ) goto json_pfa_oom; + memcpy(p->aBlob, pFromCache->aBlob, nBlob); + p->nBlobAlloc = p->nBlob = nBlob; + p->hasNonstd = pFromCache->hasNonstd; + jsonParseFree(pFromCache); + return p; + } + if( eType==SQLITE_BLOB ){ + if( jsonArgIsJsonb(pArg,p) ){ + if( (flgs & JSON_EDITABLE)!=0 && jsonBlobMakeEditable(p, 0)==0 ){ + goto json_pfa_oom; + } + return p; + } + /* If the blob is not valid JSONB, fall through into trying to cast + ** the blob into text which is then interpreted as JSON. (tag-20240123-a) + ** + ** This goes against all historical documentation about how the SQLite + ** JSON functions were suppose to work. From the beginning, blob was + ** reserved for expansion and a blob value should have raised an error. + ** But it did not, due to a bug. And many applications came to depend + ** upon this buggy behavior, espeically when using the CLI and reading + ** JSON text using readfile(), which returns a blob. For this reason + ** we will continue to support the bug moving forward. + ** See for example https://sqlite.org/forum/forumpost/012136abd5292b8d + */ + } + p->zJson = (char*)sqlite3_value_text(pArg); + p->nJson = sqlite3_value_bytes(pArg); + if( db->mallocFailed ) goto json_pfa_oom; + if( p->nJson==0 ) goto json_pfa_malformed; + assert( p->zJson!=0 ); + if( jsonConvertTextToBlob(p, (flgs & JSON_KEEPERROR) ? 0 : ctx) ){ + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + return 0; + } + }else{ + int isRCStr = sqlite3ValueIsOfClass(pArg, sqlite3RCStrUnref); + int rc; + if( !isRCStr ){ + char *zNew = sqlite3RCStrNew( p->nJson ); + if( zNew==0 ) goto json_pfa_oom; + memcpy(zNew, p->zJson, p->nJson); + p->zJson = zNew; + p->zJson[p->nJson] = 0; + }else{ + sqlite3RCStrRef(p->zJson); + } + p->bJsonIsRCStr = 1; + rc = jsonCacheInsert(ctx, p); + if( rc==SQLITE_NOMEM ) goto json_pfa_oom; + if( flgs & JSON_EDITABLE ){ + pFromCache = p; + p = 0; + goto rebuild_from_cache; + } + } + return p; + +json_pfa_malformed: + if( flgs & JSON_KEEPERROR ){ + p->nErr = 1; + return p; + }else{ + jsonParseFree(p); + sqlite3_result_error(ctx, "malformed JSON", -1); + return 0; + } + +json_pfa_oom: + jsonParseFree(pFromCache); + jsonParseFree(p); + sqlite3_result_error_nomem(ctx); + return 0; +} + +/* +** Make the return value of a JSON function either the raw JSONB blob +** or make it JSON text, depending on whether the JSON_BLOB flag is +** set on the function. +*/ +static void jsonReturnParse( + sqlite3_context *ctx, + JsonParse *p +){ + int flgs; + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + return; + } + flgs = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( flgs & JSON_BLOB ){ + if( p->nBlobAlloc>0 && !p->bReadOnly ){ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_DYNAMIC); + p->nBlobAlloc = 0; + }else{ + sqlite3_result_blob(ctx, p->aBlob, p->nBlob, SQLITE_TRANSIENT); + } + }else{ + JsonString s; + jsonStringInit(&s, ctx); + p->delta = 0; + jsonTranslateBlobToText(p, 0, &s); + jsonReturnString(&s, p, ctx); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } +} + +/**************************************************************************** +** SQL functions used for testing and debugging +****************************************************************************/ + +#if SQLITE_DEBUG +/* +** Decode JSONB bytes in aBlob[] starting at iStart through but not +** including iEnd. Indent the +** content by nIndent spaces. +*/ +static void jsonDebugPrintBlob( + JsonParse *pParse, /* JSON content */ + u32 iStart, /* Start rendering here */ + u32 iEnd, /* Do not render this byte or any byte after this one */ + int nIndent, /* Indent by this many spaces */ + sqlite3_str *pOut /* Generate output into this sqlite3_str object */ +){ + while( iStartaBlob[iStart] & 0x0f; + u32 savedNBlob = pParse->nBlob; + sqlite3_str_appendf(pOut, "%5d:%*s", iStart, nIndent, ""); + if( pParse->nBlobAlloc>pParse->nBlob ){ + pParse->nBlob = pParse->nBlobAlloc; + } + nn = n = jsonbPayloadSize(pParse, iStart, &sz); + if( nn==0 ) nn = 1; + if( sz>0 && xaBlob[iStart+i]); + } + if( n==0 ){ + sqlite3_str_appendf(pOut, " ERROR invalid node size\n"); + iStart = n==0 ? iStart+1 : iEnd; + continue; + } + pParse->nBlob = savedNBlob; + if( iStart+n+sz>iEnd ){ + iEnd = iStart+n+sz; + if( iEnd>pParse->nBlob ){ + if( pParse->nBlobAlloc>0 && iEnd>pParse->nBlobAlloc ){ + iEnd = pParse->nBlobAlloc; + }else{ + iEnd = pParse->nBlob; + } + } + } + sqlite3_str_appendall(pOut," <-- "); + switch( x ){ + case JSONB_NULL: sqlite3_str_appendall(pOut,"null"); break; + case JSONB_TRUE: sqlite3_str_appendall(pOut,"true"); break; + case JSONB_FALSE: sqlite3_str_appendall(pOut,"false"); break; + case JSONB_INT: sqlite3_str_appendall(pOut,"int"); break; + case JSONB_INT5: sqlite3_str_appendall(pOut,"int5"); break; + case JSONB_FLOAT: sqlite3_str_appendall(pOut,"float"); break; + case JSONB_FLOAT5: sqlite3_str_appendall(pOut,"float5"); break; + case JSONB_TEXT: sqlite3_str_appendall(pOut,"text"); break; + case JSONB_TEXTJ: sqlite3_str_appendall(pOut,"textj"); break; + case JSONB_TEXT5: sqlite3_str_appendall(pOut,"text5"); break; + case JSONB_TEXTRAW: sqlite3_str_appendall(pOut,"textraw"); break; + case JSONB_ARRAY: { + sqlite3_str_appendf(pOut,"array, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); + showContent = 0; + break; + } + case JSONB_OBJECT: { + sqlite3_str_appendf(pOut, "object, %u bytes\n", sz); + jsonDebugPrintBlob(pParse, iStart+n, iStart+n+sz, nIndent+2, pOut); + showContent = 0; + break; + } + default: { + sqlite3_str_appendall(pOut, "ERROR: unknown node type\n"); + showContent = 0; + break; + } + } + if( showContent ){ + if( sz==0 && x<=JSONB_FALSE ){ + sqlite3_str_append(pOut, "\n", 1); + }else{ + u32 j; + sqlite3_str_appendall(pOut, ": \""); + for(j=iStart+n; jaBlob[j]; + if( c<0x20 || c>=0x7f ) c = '.'; + sqlite3_str_append(pOut, (char*)&c, 1); + } + sqlite3_str_append(pOut, "\"\n", 2); + } + } + iStart += n + sz; + } +} +static void jsonShowParse(JsonParse *pParse){ + sqlite3_str out; + char zBuf[1000]; + if( pParse==0 ){ + printf("NULL pointer\n"); + return; + }else{ + printf("nBlobAlloc = %u\n", pParse->nBlobAlloc); + printf("nBlob = %u\n", pParse->nBlob); + printf("delta = %d\n", pParse->delta); + if( pParse->nBlob==0 ) return; + printf("content (bytes 0..%u):\n", pParse->nBlob-1); + } + sqlite3StrAccumInit(&out, 0, zBuf, sizeof(zBuf), 1000000); + jsonDebugPrintBlob(pParse, 0, pParse->nBlob, 0, &out); + printf("%s", sqlite3_str_value(&out)); + sqlite3_str_reset(&out); +} +#endif /* SQLITE_DEBUG */ + +#ifdef SQLITE_DEBUG +/* +** SQL function: json_parse(JSON) +** +** Parse JSON using jsonParseFuncArg(). Return text that is a +** human-readable dump of the binary JSONB for the input parameter. +*/ +static void jsonParseFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + sqlite3_str out; + + assert( argc>=1 ); + sqlite3StrAccumInit(&out, 0, 0, 0, 1000000); + p = jsonParseFuncArg(ctx, argv[0], 0); + if( p==0 ) return; + if( argc==1 ){ + jsonDebugPrintBlob(p, 0, p->nBlob, 0, &out); + sqlite3_result_text64(ctx,out.zText,out.nChar,SQLITE_TRANSIENT,SQLITE_UTF8); + }else{ + jsonShowParse(p); + } + jsonParseFree(p); + sqlite3_str_reset(&out); +} +#endif /* SQLITE_DEBUG */ + +/**************************************************************************** +** Scalar SQL function implementations +****************************************************************************/ + +/* +** Implementation of the json_quote(VALUE) function. Return a JSON value +** corresponding to the SQL value input. Mostly this means putting +** double-quotes around strings and returning the unquoted string "null" +** when given a NULL input. +*/ +static void jsonQuoteFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString jx; + UNUSED_PARAMETER(argc); + + jsonStringInit(&jx, ctx); + jsonAppendSqlValue(&jx, argv[0]); + jsonReturnString(&jx, 0, 0); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} + +/* +** Implementation of the json_array(VALUE,...) function. Return a JSON +** array that contains all values given in arguments. Or if any argument +** is a BLOB, throw an error. +*/ +static void jsonArrayFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + + jsonStringInit(&jx, ctx); + jsonAppendChar(&jx, '['); + for(i=0; iaBlob[i] & 0x0f)==JSONB_ARRAY ){ + cnt = jsonbArrayCount(p, i); + } + if( !eErr ) sqlite3_result_int64(ctx, cnt); + jsonParseFree(p); +} + +/* True if the string is all alphanumerics and underscores */ +static int jsonAllAlphanum(const char *z, int n){ + int i; + for(i=0; i"(JSON,PATH) +** "->>"(JSON,PATH) +** +** Return the element described by PATH. Return NULL if that PATH element +** is not found. +** +** If JSON_JSON is set or if more that one PATH argument is supplied then +** always return a JSON representation of the result. If JSON_SQL is set, +** then always return an SQL representation of the result. If neither flag +** is present and argc==2, then return JSON for objects and arrays and SQL +** for all other values. +** +** When multiple PATH arguments are supplied, the result is a JSON array +** containing the result of each PATH. +** +** Abbreviated JSON path expressions are allows if JSON_ABPATH, for +** compatibility with PG. +*/ +static void jsonExtractFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p = 0; /* The parse */ + int flags; /* Flags associated with the function */ + int i; /* Loop counter */ + JsonString jx; /* String for array result */ + + if( argc<2 ) return; + p = jsonParseFuncArg(ctx, argv[0], 0); + if( p==0 ) return; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + jsonStringInit(&jx, ctx); + if( argc>2 ){ + jsonAppendChar(&jx, '['); + } + for(i=1; i and ->> operators accept abbreviated PATH arguments. This + ** is mostly for compatibility with PostgreSQL, but also for + ** convenience. + ** + ** NUMBER ==> $[NUMBER] // PG compatible + ** LABEL ==> $.LABEL // PG compatible + ** [NUMBER] ==> $[NUMBER] // Not PG. Purely for convenience + ** + ** Updated 2024-05-27: If the NUMBER is negative, then PG counts from + ** the right of the array. Hence for negative NUMBER: + ** + ** NUMBER ==> $[#NUMBER] // PG compatible + */ + jsonStringInit(&jx, ctx); + if( sqlite3_value_type(argv[i])==SQLITE_INTEGER ){ + jsonAppendRawNZ(&jx, "[", 1); + if( zPath[0]=='-' ) jsonAppendRawNZ(&jx,"#",1); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "]", 2); + }else if( jsonAllAlphanum(zPath, nPath) ){ + jsonAppendRawNZ(&jx, ".", 1); + jsonAppendRaw(&jx, zPath, nPath); + }else if( zPath[0]=='[' && nPath>=3 && zPath[nPath-1]==']' ){ + jsonAppendRaw(&jx, zPath, nPath); + }else{ + jsonAppendRawNZ(&jx, ".\"", 2); + jsonAppendRaw(&jx, zPath, nPath); + jsonAppendRawNZ(&jx, "\"", 1); + } + jsonStringTerminate(&jx); + j = jsonLookupStep(p, 0, jx.zBuf, 0); + jsonStringReset(&jx); + }else{ + jsonBadPathError(ctx, zPath); + goto json_extract_error; + } + if( jnBlob ){ + if( argc==2 ){ + if( flags & JSON_JSON ){ + jsonStringInit(&jx, ctx); + jsonTranslateBlobToText(p, j, &jx); + jsonReturnString(&jx, 0, 0); + jsonStringReset(&jx); + assert( (flags & JSON_BLOB)==0 ); + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + }else{ + jsonReturnFromBlob(p, j, ctx, 0); + if( (flags & (JSON_SQL|JSON_BLOB))==0 + && (p->aBlob[j]&0x0f)>=JSONB_ARRAY + ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + } + }else{ + jsonAppendSeparator(&jx); + jsonTranslateBlobToText(p, j, &jx); + } + }else if( j==JSON_LOOKUP_NOTFOUND ){ + if( argc==2 ){ + goto json_extract_error; /* Return NULL if not found */ + }else{ + jsonAppendSeparator(&jx); + jsonAppendRawNZ(&jx, "null", 4); + } + }else if( j==JSON_LOOKUP_ERROR ){ + sqlite3_result_error(ctx, "malformed JSON", -1); + goto json_extract_error; + }else{ + jsonBadPathError(ctx, zPath); + goto json_extract_error; + } + } + if( argc>2 ){ + jsonAppendChar(&jx, ']'); + jsonReturnString(&jx, 0, 0); + if( (flags & JSON_BLOB)==0 ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + } +json_extract_error: + jsonStringReset(&jx); + jsonParseFree(p); + return; +} + +/* +** Return codes for jsonMergePatch() +*/ +#define JSON_MERGE_OK 0 /* Success */ +#define JSON_MERGE_BADTARGET 1 /* Malformed TARGET blob */ +#define JSON_MERGE_BADPATCH 2 /* Malformed PATCH blob */ +#define JSON_MERGE_OOM 3 /* Out-of-memory condition */ + +/* +** RFC-7396 MergePatch for two JSONB blobs. +** +** pTarget is the target. pPatch is the patch. The target is updated +** in place. The patch is read-only. +** +** The original RFC-7396 algorithm is this: +** +** define MergePatch(Target, Patch): +** if Patch is an Object: +** if Target is not an Object: +** Target = {} # Ignore the contents and set it to an empty Object +** for each Name/Value pair in Patch: +** if Value is null: +** if Name exists in Target: +** remove the Name/Value pair from Target +** else: +** Target[Name] = MergePatch(Target[Name], Value) +** return Target +** else: +** return Patch +** +** Here is an equivalent algorithm restructured to show the actual +** implementation: +** +** 01 define MergePatch(Target, Patch): +** 02 if Patch is not an Object: +** 03 return Patch +** 04 else: // if Patch is an Object +** 05 if Target is not an Object: +** 06 Target = {} +** 07 for each Name/Value pair in Patch: +** 08 if Name exists in Target: +** 09 if Value is null: +** 10 remove the Name/Value pair from Target +** 11 else +** 12 Target[name] = MergePatch(Target[Name], Value) +** 13 else if Value is not NULL: +** 14 if Value is not an Object: +** 15 Target[name] = Value +** 16 else: +** 17 Target[name] = MergePatch('{}',value) +** 18 return Target +** | +** ^---- Line numbers referenced in comments in the implementation +*/ +static int jsonMergePatch( + JsonParse *pTarget, /* The JSON parser that contains the TARGET */ + u32 iTarget, /* Index of TARGET in pTarget->aBlob[] */ + const JsonParse *pPatch, /* The PATCH */ + u32 iPatch /* Index of PATCH in pPatch->aBlob[] */ +){ + u8 x; /* Type of a single node */ + u32 n, sz=0; /* Return values from jsonbPayloadSize() */ + u32 iTCursor; /* Cursor position while scanning the target object */ + u32 iTStart; /* First label in the target object */ + u32 iTEndBE; /* Original first byte past end of target, before edit */ + u32 iTEnd; /* Current first byte past end of target */ + u8 eTLabel; /* Node type of the target label */ + u32 iTLabel = 0; /* Index of the label */ + u32 nTLabel = 0; /* Header size in bytes for the target label */ + u32 szTLabel = 0; /* Size of the target label payload */ + u32 iTValue = 0; /* Index of the target value */ + u32 nTValue = 0; /* Header size of the target value */ + u32 szTValue = 0; /* Payload size for the target value */ + + u32 iPCursor; /* Cursor position while scanning the patch */ + u32 iPEnd; /* First byte past the end of the patch */ + u8 ePLabel; /* Node type of the patch label */ + u32 iPLabel; /* Start of patch label */ + u32 nPLabel; /* Size of header on the patch label */ + u32 szPLabel; /* Payload size of the patch label */ + u32 iPValue; /* Start of patch value */ + u32 nPValue; /* Header size for the patch value */ + u32 szPValue; /* Payload size of the patch value */ + + assert( iTarget>=0 && iTargetnBlob ); + assert( iPatch>=0 && iPatchnBlob ); + x = pPatch->aBlob[iPatch] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 02 */ + u32 szPatch; /* Total size of the patch, header+payload */ + u32 szTarget; /* Total size of the target, header+payload */ + n = jsonbPayloadSize(pPatch, iPatch, &sz); + szPatch = n+sz; + sz = 0; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + szTarget = n+sz; + jsonBlobEdit(pTarget, iTarget, szTarget, pPatch->aBlob+iPatch, szPatch); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; /* Line 03 */ + } + x = pTarget->aBlob[iTarget] & 0x0f; + if( x!=JSONB_OBJECT ){ /* Algorithm line 05 */ + n = jsonbPayloadSize(pTarget, iTarget, &sz); + jsonBlobEdit(pTarget, iTarget+n, sz, 0, 0); + x = pTarget->aBlob[iTarget]; + pTarget->aBlob[iTarget] = (x & 0xf0) | JSONB_OBJECT; + } + n = jsonbPayloadSize(pPatch, iPatch, &sz); + if( NEVER(n==0) ) return JSON_MERGE_BADPATCH; + iPCursor = iPatch+n; + iPEnd = iPCursor+sz; + n = jsonbPayloadSize(pTarget, iTarget, &sz); + if( NEVER(n==0) ) return JSON_MERGE_BADTARGET; + iTStart = iTarget+n; + iTEndBE = iTStart+sz; + + while( iPCursoraBlob[iPCursor] & 0x0f; + if( ePLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADPATCH; + } + nPLabel = jsonbPayloadSize(pPatch, iPCursor, &szPLabel); + if( nPLabel==0 ) return JSON_MERGE_BADPATCH; + iPValue = iPCursor + nPLabel + szPLabel; + if( iPValue>=iPEnd ) return JSON_MERGE_BADPATCH; + nPValue = jsonbPayloadSize(pPatch, iPValue, &szPValue); + if( nPValue==0 ) return JSON_MERGE_BADPATCH; + iPCursor = iPValue + nPValue + szPValue; + if( iPCursor>iPEnd ) return JSON_MERGE_BADPATCH; + + iTCursor = iTStart; + iTEnd = iTEndBE + pTarget->delta; + while( iTCursoraBlob[iTCursor] & 0x0f; + if( eTLabelJSONB_TEXTRAW ){ + return JSON_MERGE_BADTARGET; + } + nTLabel = jsonbPayloadSize(pTarget, iTCursor, &szTLabel); + if( nTLabel==0 ) return JSON_MERGE_BADTARGET; + iTValue = iTLabel + nTLabel + szTLabel; + if( iTValue>=iTEnd ) return JSON_MERGE_BADTARGET; + nTValue = jsonbPayloadSize(pTarget, iTValue, &szTValue); + if( nTValue==0 ) return JSON_MERGE_BADTARGET; + if( iTValue + nTValue + szTValue > iTEnd ) return JSON_MERGE_BADTARGET; + isEqual = jsonLabelCompare( + (const char*)&pPatch->aBlob[iPLabel+nPLabel], + szPLabel, + (ePLabel==JSONB_TEXT || ePLabel==JSONB_TEXTRAW), + (const char*)&pTarget->aBlob[iTLabel+nTLabel], + szTLabel, + (eTLabel==JSONB_TEXT || eTLabel==JSONB_TEXTRAW)); + if( isEqual ) break; + iTCursor = iTValue + nTValue + szTValue; + } + x = pPatch->aBlob[iPValue] & 0x0f; + if( iTCursoroom) ) return JSON_MERGE_OOM; + }else{ + /* Algorithm line 12 */ + int rc, savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatch(pTarget, iTValue, pPatch, iPValue); + if( rc ) return rc; + pTarget->delta += savedDelta; + } + }else if( x>0 ){ /* Algorithm line 13 */ + /* No match and patch value is not NULL */ + u32 szNew = szPLabel+nPLabel; + if( (pPatch->aBlob[iPValue] & 0x0f)!=JSONB_OBJECT ){ /* Line 14 */ + jsonBlobEdit(pTarget, iTEnd, 0, 0, szPValue+nPValue+szNew); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + memcpy(&pTarget->aBlob[iTEnd+szNew], + &pPatch->aBlob[iPValue], szPValue+nPValue); + }else{ + int rc, savedDelta; + jsonBlobEdit(pTarget, iTEnd, 0, 0, szNew+1); + if( pTarget->oom ) return JSON_MERGE_OOM; + memcpy(&pTarget->aBlob[iTEnd], &pPatch->aBlob[iPLabel], szNew); + pTarget->aBlob[iTEnd+szNew] = 0x00; + savedDelta = pTarget->delta; + pTarget->delta = 0; + rc = jsonMergePatch(pTarget, iTEnd+szNew,pPatch,iPValue); + if( rc ) return rc; + pTarget->delta += savedDelta; + } + } + } + if( pTarget->delta ) jsonAfterEditSizeAdjust(pTarget, iTarget); + return pTarget->oom ? JSON_MERGE_OOM : JSON_MERGE_OK; +} + + +/* +** Implementation of the json_mergepatch(JSON1,JSON2) function. Return a JSON +** object that is the result of running the RFC 7396 MergePatch() algorithm +** on the two arguments. +*/ +static void jsonPatchFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *pTarget; /* The TARGET */ + JsonParse *pPatch; /* The PATCH */ + int rc; /* Result code */ + + UNUSED_PARAMETER(argc); + assert( argc==2 ); + pTarget = jsonParseFuncArg(ctx, argv[0], JSON_EDITABLE); + if( pTarget==0 ) return; + pPatch = jsonParseFuncArg(ctx, argv[1], 0); + if( pPatch ){ + rc = jsonMergePatch(pTarget, 0, pPatch, 0); + if( rc==JSON_MERGE_OK ){ + jsonReturnParse(ctx, pTarget); + }else if( rc==JSON_MERGE_OOM ){ + sqlite3_result_error_nomem(ctx); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + jsonParseFree(pPatch); + } + jsonParseFree(pTarget); +} + + +/* +** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON +** object that contains all name/value given in arguments. Or if any name +** is not a string or if any value is a BLOB, throw an error. +*/ +static void jsonObjectFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + int i; + JsonString jx; + const char *z; + u32 n; + + if( argc&1 ){ + sqlite3_result_error(ctx, "json_object() requires an even number " + "of arguments", -1); + return; + } + jsonStringInit(&jx, ctx); + jsonAppendChar(&jx, '{'); + for(i=0; i1 ? JSON_EDITABLE : 0); + if( p==0 ) return; + for(i=1; ieEdit = JEDIT_DEL; + p->delta = 0; + rc = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(rc) ){ + if( rc==JSON_LOOKUP_NOTFOUND ){ + continue; /* No-op */ + }else if( rc==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_remove_done; + } + } + jsonReturnParse(ctx, p); + jsonParseFree(p); + return; + +json_remove_patherror: + jsonBadPathError(ctx, zPath); + +json_remove_done: + jsonParseFree(p); + return; +} + +/* +** json_replace(JSON, PATH, VALUE, ...) +** +** Replace the value at PATH with VALUE. If PATH does not already exist, +** this routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonReplaceFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, "replace"); + return; + } + jsonInsertIntoBlob(ctx, argc, argv, JEDIT_REPL); +} + + +/* +** json_set(JSON, PATH, VALUE, ...) +** +** Set the value at PATH to VALUE. Create the PATH if it does not already +** exist. Overwrite existing values that do exist. +** If JSON or PATH is malformed, throw an error. +** +** json_insert(JSON, PATH, VALUE, ...) +** +** Create PATH and initialize it to VALUE. If PATH already exists, this +** routine is a no-op. If JSON or PATH is malformed, throw an error. +*/ +static void jsonSetFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + + int flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + int bIsSet = (flags&JSON_ISSET)!=0; + + if( argc<1 ) return; + if( (argc&1)==0 ) { + jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); + return; + } + jsonInsertIntoBlob(ctx, argc, argv, bIsSet ? JEDIT_SET : JEDIT_INS); +} + +/* +** json_type(JSON) +** json_type(JSON, PATH) +** +** Return the top-level "type" of a JSON string. json_type() raises an +** error if either the JSON or PATH inputs are not well-formed. +*/ +static void jsonTypeFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + const char *zPath = 0; + u32 i; + + p = jsonParseFuncArg(ctx, argv[0], 0); + if( p==0 ) return; + if( argc==2 ){ + zPath = (const char*)sqlite3_value_text(argv[1]); + if( zPath==0 ) goto json_type_done; + if( zPath[0]!='$' ){ + jsonBadPathError(ctx, zPath); + goto json_type_done; + } + i = jsonLookupStep(p, 0, zPath+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + /* no-op */ + }else if( i==JSON_LOOKUP_PATHERROR ){ + jsonBadPathError(ctx, zPath); + }else{ + sqlite3_result_error(ctx, "malformed JSON", -1); + } + goto json_type_done; + } + }else{ + i = 0; + } + sqlite3_result_text(ctx, jsonbType[p->aBlob[i]&0x0f], -1, SQLITE_STATIC); +json_type_done: + jsonParseFree(p); +} + +/* +** json_pretty(JSON) +** json_pretty(JSON, INDENT) +** +** Return text that is a pretty-printed rendering of the input JSON. +** If the argument is not valid JSON, return NULL. +** +** The INDENT argument is text that is used for indentation. If omitted, +** it defaults to four spaces (the same as PostgreSQL). +*/ +static void jsonPrettyFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString s; /* The output string */ + JsonPretty x; /* Pretty printing context */ + + memset(&x, 0, sizeof(x)); + x.pParse = jsonParseFuncArg(ctx, argv[0], 0); + if( x.pParse==0 ) return; + x.pOut = &s; + jsonStringInit(&s, ctx); + if( argc==1 || (x.zIndent = (const char*)sqlite3_value_text(argv[1]))==0 ){ + x.zIndent = " "; + x.szIndent = 4; + }else{ + x.szIndent = (u32)strlen(x.zIndent); + } + jsonTranslateBlobToPrettyText(&x, 0); + jsonReturnString(&s, 0, 0); + jsonParseFree(x.pParse); +} + +/* +** json_valid(JSON) +** json_valid(JSON, FLAGS) +** +** Check the JSON argument to see if it is well-formed. The FLAGS argument +** encodes the various constraints on what is meant by "well-formed": +** +** 0x01 Canonical RFC-8259 JSON text +** 0x02 JSON text with optional JSON-5 extensions +** 0x04 Superficially appears to be JSONB +** 0x08 Strictly well-formed JSONB +** +** If the FLAGS argument is omitted, it defaults to 1. Useful values for +** FLAGS include: +** +** 1 Strict canonical JSON text +** 2 JSON text perhaps with JSON-5 extensions +** 4 Superficially appears to be JSONB +** 5 Canonical JSON text or superficial JSONB +** 6 JSON-5 text or superficial JSONB +** 8 Strict JSONB +** 9 Canonical JSON text or strict JSONB +** 10 JSON-5 text or strict JSONB +** +** Other flag combinations are redundant. For example, every canonical +** JSON text is also well-formed JSON-5 text, so FLAG values 2 and 3 +** are the same. Similarly, any input that passes a strict JSONB validation +** will also pass the superficial validation so 12 through 15 are the same +** as 8 through 11 respectively. +** +** This routine runs in linear time to validate text and when doing strict +** JSONB validation. Superficial JSONB validation is constant time, +** assuming the BLOB is already in memory. The performance advantage +** of superficial JSONB validation is why that option is provided. +** Application developers can choose to do fast superficial validation or +** slower strict validation, according to their specific needs. +** +** Only the lower four bits of the FLAGS argument are currently used. +** Higher bits are reserved for future expansion. To facilitate +** compatibility, the current implementation raises an error if any bit +** in FLAGS is set other than the lower four bits. +** +** The original circa 2015 implementation of the JSON routines in +** SQLite only supported canonical RFC-8259 JSON text and the json_valid() +** function only accepted one argument. That is why the default value +** for the FLAGS argument is 1, since FLAGS=1 causes this routine to only +** recognize canonical RFC-8259 JSON text as valid. The extra FLAGS +** argument was added when the JSON routines were extended to support +** JSON5-like extensions and binary JSONB stored in BLOBs. +** +** Return Values: +** +** * Raise an error if FLAGS is outside the range of 1 to 15. +** * Return NULL if the input is NULL +** * Return 1 if the input is well-formed. +** * Return 0 if the input is not well-formed. +*/ +static void jsonValidFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonParse *p; /* The parse */ + u8 flags = 1; + u8 res = 0; + if( argc==2 ){ + i64 f = sqlite3_value_int64(argv[1]); + if( f<1 || f>15 ){ + sqlite3_result_error(ctx, "FLAGS parameter to json_valid() must be" + " between 1 and 15", -1); + return; + } + flags = f & 0x0f; + } + switch( sqlite3_value_type(argv[0]) ){ + case SQLITE_NULL: { +#ifdef SQLITE_LEGACY_JSON_VALID + /* Incorrect legacy behavior was to return FALSE for a NULL input */ + sqlite3_result_int(ctx, 0); +#endif + return; + } + case SQLITE_BLOB: { + if( jsonFuncArgMightBeBinary(argv[0]) ){ + if( flags & 0x04 ){ + /* Superficial checking only - accomplished by the + ** jsonFuncArgMightBeBinary() call above. */ + res = 1; + }else if( flags & 0x08 ){ + /* Strict checking. Check by translating BLOB->TEXT->BLOB. If + ** no errors occur, call that a "strict check". */ + JsonParse px; + u32 iErr; + memset(&px, 0, sizeof(px)); + px.aBlob = (u8*)sqlite3_value_blob(argv[0]); + px.nBlob = sqlite3_value_bytes(argv[0]); + iErr = jsonbValidityCheck(&px, 0, px.nBlob, 1); + res = iErr==0; + } + break; + } + /* Fall through into interpreting the input as text. See note + ** above at tag-20240123-a. */ + /* no break */ deliberate_fall_through + } + default: { + JsonParse px; + if( (flags & 0x3)==0 ) break; + memset(&px, 0, sizeof(px)); + + p = jsonParseFuncArg(ctx, argv[0], JSON_KEEPERROR); + if( p ){ + if( p->oom ){ + sqlite3_result_error_nomem(ctx); + }else if( p->nErr ){ + /* no-op */ + }else if( (flags & 0x02)!=0 || p->hasNonstd==0 ){ + res = 1; + } + jsonParseFree(p); + }else{ + sqlite3_result_error_nomem(ctx); + } + break; + } + } + sqlite3_result_int(ctx, res); +} + +/* +** json_error_position(JSON) +** +** If the argument is NULL, return NULL +** +** If the argument is BLOB, do a full validity check and return non-zero +** if the check fails. The return value is the approximate 1-based offset +** to the byte of the element that contains the first error. +** +** Otherwise interpret the argument is TEXT (even if it is numeric) and +** return the 1-based character position for where the parser first recognized +** that the input was not valid JSON, or return 0 if the input text looks +** ok. JSON-5 extensions are accepted. +*/ +static void jsonErrorFunc( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + i64 iErrPos = 0; /* Error position to be returned */ + JsonParse s; + + assert( argc==1 ); + UNUSED_PARAMETER(argc); + memset(&s, 0, sizeof(s)); + s.db = sqlite3_context_db_handle(ctx); + if( jsonFuncArgMightBeBinary(argv[0]) ){ + s.aBlob = (u8*)sqlite3_value_blob(argv[0]); + s.nBlob = sqlite3_value_bytes(argv[0]); + iErrPos = (i64)jsonbValidityCheck(&s, 0, s.nBlob, 1); + }else{ + s.zJson = (char*)sqlite3_value_text(argv[0]); + if( s.zJson==0 ) return; /* NULL input or OOM */ + s.nJson = sqlite3_value_bytes(argv[0]); + if( jsonConvertTextToBlob(&s,0) ){ + if( s.oom ){ + iErrPos = -1; + }else{ + /* Convert byte-offset s.iErr into a character offset */ + u32 k; + assert( s.zJson!=0 ); /* Because s.oom is false */ + for(k=0; kzBuf==0 ){ + jsonStringInit(pStr, ctx); + jsonAppendChar(pStr, '['); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + } + pStr->pCtx = ctx; + jsonAppendSqlValue(pStr, argv[0]); + } +} +static void jsonArrayCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + int flags; + pStr->pCtx = ctx; + jsonAppendChar(pStr, ']'); + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr, 0, 0); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); + }else{ + jsonStringTrimOneChar(pStr); + } + return; + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + jsonStringTrimOneChar(pStr); + } + }else{ + sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonArrayValue(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 0); +} +static void jsonArrayFinal(sqlite3_context *ctx){ + jsonArrayCompute(ctx, 1); +} + +#ifndef SQLITE_OMIT_WINDOWFUNC +/* +** This method works for both json_group_array() and json_group_object(). +** It works by removing the first element of the group by searching forward +** to the first comma (",") that is not within a string and deleting all +** text through that comma. +*/ +static void jsonGroupInverse( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + unsigned int i; + int inStr = 0; + int nNest = 0; + char *z; + char c; + JsonString *pStr; + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(argv); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); +#ifdef NEVER + /* pStr is always non-NULL since jsonArrayStep() or jsonObjectStep() will + ** always have been called to initialize it */ + if( NEVER(!pStr) ) return; +#endif + z = pStr->zBuf; + for(i=1; inUsed && ((c = z[i])!=',' || inStr || nNest); i++){ + if( c=='"' ){ + inStr = !inStr; + }else if( c=='\\' ){ + i++; + }else if( !inStr ){ + if( c=='{' || c=='[' ) nNest++; + if( c=='}' || c==']' ) nNest--; + } + } + if( inUsed ){ + pStr->nUsed -= i; + memmove(&z[1], &z[i+1], (size_t)pStr->nUsed-1); + z[pStr->nUsed] = 0; + }else{ + pStr->nUsed = 1; + } +} +#else +# define jsonGroupInverse 0 +#endif + + +/* +** json_group_obj(NAME,VALUE) +** +** Return a JSON object composed of all names and values in the aggregate. +*/ +static void jsonObjectStep( + sqlite3_context *ctx, + int argc, + sqlite3_value **argv +){ + JsonString *pStr; + const char *z; + u32 n; + UNUSED_PARAMETER(argc); + pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); + if( pStr ){ + if( pStr->zBuf==0 ){ + jsonStringInit(pStr, ctx); + jsonAppendChar(pStr, '{'); + }else if( pStr->nUsed>1 ){ + jsonAppendChar(pStr, ','); + } + pStr->pCtx = ctx; + z = (const char*)sqlite3_value_text(argv[0]); + n = sqlite3Strlen30(z); + jsonAppendString(pStr, z, n); + jsonAppendChar(pStr, ':'); + jsonAppendSqlValue(pStr, argv[1]); + } +} +static void jsonObjectCompute(sqlite3_context *ctx, int isFinal){ + JsonString *pStr; + pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); + if( pStr ){ + int flags; + jsonAppendChar(pStr, '}'); + pStr->pCtx = ctx; + flags = SQLITE_PTR_TO_INT(sqlite3_user_data(ctx)); + if( pStr->eErr ){ + jsonReturnString(pStr, 0, 0); + return; + }else if( flags & JSON_BLOB ){ + jsonReturnStringAsBlob(pStr); + if( isFinal ){ + if( !pStr->bStatic ) sqlite3RCStrUnref(pStr->zBuf); + }else{ + jsonStringTrimOneChar(pStr); + } + return; + }else if( isFinal ){ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, + pStr->bStatic ? SQLITE_TRANSIENT : + sqlite3RCStrUnref); + pStr->bStatic = 1; + }else{ + sqlite3_result_text(ctx, pStr->zBuf, (int)pStr->nUsed, SQLITE_TRANSIENT); + jsonStringTrimOneChar(pStr); + } + }else{ + sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); + } + sqlite3_result_subtype(ctx, JSON_SUBTYPE); +} +static void jsonObjectValue(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 0); +} +static void jsonObjectFinal(sqlite3_context *ctx){ + jsonObjectCompute(ctx, 1); +} + + + +#ifndef SQLITE_OMIT_VIRTUALTABLE +/**************************************************************************** +** The json_each virtual table +****************************************************************************/ +typedef struct JsonParent JsonParent; +struct JsonParent { + u32 iHead; /* Start of object or array */ + u32 iValue; /* Start of the value */ + u32 iEnd; /* First byte past the end */ + u32 nPath; /* Length of path */ + i64 iKey; /* Key for JSONB_ARRAY */ +}; + +typedef struct JsonEachCursor JsonEachCursor; +struct JsonEachCursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + u32 iRowid; /* The rowid */ + u32 i; /* Index in sParse.aBlob[] of current row */ + u32 iEnd; /* EOF when i equals or exceeds this value */ + u32 nRoot; /* Size of the root path in bytes */ + u8 eType; /* Type of the container for element i */ + u8 bRecursive; /* True for json_tree(). False for json_each() */ + u32 nParent; /* Current nesting depth */ + u32 nParentAlloc; /* Space allocated for aParent[] */ + JsonParent *aParent; /* Parent elements of i */ + sqlite3 *db; /* Database connection */ + JsonString path; /* Current path */ + JsonParse sParse; /* Parse of the input JSON */ +}; +typedef struct JsonEachConnection JsonEachConnection; +struct JsonEachConnection { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection */ +}; + + +/* Constructor for the json_each virtual table */ +static int jsonEachConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + JsonEachConnection *pNew; + int rc; + +/* Column numbers */ +#define JEACH_KEY 0 +#define JEACH_VALUE 1 +#define JEACH_TYPE 2 +#define JEACH_ATOM 3 +#define JEACH_ID 4 +#define JEACH_PARENT 5 +#define JEACH_FULLKEY 6 +#define JEACH_PATH 7 +/* The xBestIndex method assumes that the JSON and ROOT columns are +** the last two columns in the table. Should this ever changes, be +** sure to update the xBestIndex method. */ +#define JEACH_JSON 8 +#define JEACH_ROOT 9 + + UNUSED_PARAMETER(pzErr); + UNUSED_PARAMETER(argv); + UNUSED_PARAMETER(argc); + UNUSED_PARAMETER(pAux); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," + "json HIDDEN,root HIDDEN)"); + if( rc==SQLITE_OK ){ + pNew = (JsonEachConnection*)sqlite3DbMallocZero(db, sizeof(*pNew)); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + pNew->db = db; + } + return rc; +} + +/* destructor for json_each virtual table */ +static int jsonEachDisconnect(sqlite3_vtab *pVtab){ + JsonEachConnection *p = (JsonEachConnection*)pVtab; + sqlite3DbFree(p->db, pVtab); + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_each(). */ +static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + JsonEachConnection *pVtab = (JsonEachConnection*)p; + JsonEachCursor *pCur; + + UNUSED_PARAMETER(p); + pCur = sqlite3DbMallocZero(pVtab->db, sizeof(*pCur)); + if( pCur==0 ) return SQLITE_NOMEM; + pCur->db = pVtab->db; + jsonStringZero(&pCur->path); + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +/* constructor for a JsonEachCursor object for json_tree(). */ +static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + int rc = jsonEachOpenEach(p, ppCursor); + if( rc==SQLITE_OK ){ + JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; + pCur->bRecursive = 1; + } + return rc; +} + +/* Reset a JsonEachCursor back to its original state. Free any memory +** held. */ +static void jsonEachCursorReset(JsonEachCursor *p){ + jsonParseReset(&p->sParse); + jsonStringReset(&p->path); + sqlite3DbFree(p->db, p->aParent); + p->iRowid = 0; + p->i = 0; + p->aParent = 0; + p->nParent = 0; + p->nParentAlloc = 0; + p->iEnd = 0; + p->eType = 0; +} + +/* Destructor for a jsonEachCursor object */ +static int jsonEachClose(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + jsonEachCursorReset(p); + + sqlite3DbFree(p->db, cur); + return SQLITE_OK; +} + +/* Return TRUE if the jsonEachCursor object has been advanced off the end +** of the JSON object */ +static int jsonEachEof(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + return p->i >= p->iEnd; +} + +/* +** If the cursor is currently pointing at the label of a object entry, +** then return the index of the value. For all other cases, return the +** current pointer position, which is the value. +*/ +static int jsonSkipLabel(JsonEachCursor *p){ + if( p->eType==JSONB_OBJECT ){ + u32 sz = 0; + u32 n = jsonbPayloadSize(&p->sParse, p->i, &sz); + return p->i + n + sz; + }else{ + return p->i; + } +} + +/* +** Append the path name for the current element. +*/ +static void jsonAppendPathName(JsonEachCursor *p){ + assert( p->nParent>0 ); + assert( p->eType==JSONB_ARRAY || p->eType==JSONB_OBJECT ); + if( p->eType==JSONB_ARRAY ){ + jsonPrintf(30, &p->path, "[%lld]", p->aParent[p->nParent-1].iKey); + }else{ + u32 n, sz = 0, k, i; + const char *z; + int needQuote = 0; + n = jsonbPayloadSize(&p->sParse, p->i, &sz); + k = p->i + n; + z = (const char*)&p->sParse.aBlob[k]; + if( sz==0 || !sqlite3Isalpha(z[0]) ){ + needQuote = 1; + }else{ + for(i=0; ipath,".\"%.*s\"", sz, z); + }else{ + jsonPrintf(sz+2,&p->path,".%.*s", sz, z); + } + } +} + +/* Advance the cursor to the next element for json_tree() */ +static int jsonEachNext(sqlite3_vtab_cursor *cur){ + JsonEachCursor *p = (JsonEachCursor*)cur; + int rc = SQLITE_OK; + if( p->bRecursive ){ + u8 x; + u8 levelChange = 0; + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + x = p->sParse.aBlob[i] & 0x0f; + n = jsonbPayloadSize(&p->sParse, i, &sz); + if( x==JSONB_OBJECT || x==JSONB_ARRAY ){ + JsonParent *pParent; + if( p->nParent>=p->nParentAlloc ){ + JsonParent *pNew; + u64 nNew; + nNew = p->nParentAlloc*2 + 3; + pNew = sqlite3DbRealloc(p->db, p->aParent, sizeof(JsonParent)*nNew); + if( pNew==0 ) return SQLITE_NOMEM; + p->nParentAlloc = (u32)nNew; + p->aParent = pNew; + } + levelChange = 1; + pParent = &p->aParent[p->nParent]; + pParent->iHead = p->i; + pParent->iValue = i; + pParent->iEnd = i + n + sz; + pParent->iKey = -1; + pParent->nPath = (u32)p->path.nUsed; + if( p->eType && p->nParent ){ + jsonAppendPathName(p); + if( p->path.eErr ) rc = SQLITE_NOMEM; + } + p->nParent++; + p->i = i + n; + }else{ + p->i = i + n + sz; + } + while( p->nParent>0 && p->i >= p->aParent[p->nParent-1].iEnd ){ + p->nParent--; + p->path.nUsed = p->aParent[p->nParent].nPath; + levelChange = 1; + } + if( levelChange ){ + if( p->nParent>0 ){ + JsonParent *pParent = &p->aParent[p->nParent-1]; + u32 iVal = pParent->iValue; + p->eType = p->sParse.aBlob[iVal] & 0x0f; + }else{ + p->eType = 0; + } + } + }else{ + u32 n, sz = 0; + u32 i = jsonSkipLabel(p); + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->i = i + n + sz; + } + if( p->eType==JSONB_ARRAY && p->nParent ){ + p->aParent[p->nParent-1].iKey++; + } + p->iRowid++; + return rc; +} + +/* Length of the path for rowid==0 in bRecursive mode. +*/ +static int jsonEachPathLength(JsonEachCursor *p){ + u32 n = p->path.nUsed; + char *z = p->path.zBuf; + if( p->iRowid==0 && p->bRecursive && n>=2 ){ + while( n>1 ){ + n--; + if( z[n]=='[' || z[n]=='.' ){ + u32 x, sz = 0; + char cSaved = z[n]; + z[n] = 0; + assert( p->sParse.eEdit==0 ); + x = jsonLookupStep(&p->sParse, 0, z+1, 0); + z[n] = cSaved; + if( JSON_LOOKUP_ISERROR(x) ) continue; + if( x + jsonbPayloadSize(&p->sParse, x, &sz) == p->i ) break; + } + } + } + return n; +} + +/* Return the value of a column */ +static int jsonEachColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int iColumn /* Which column to return */ +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + switch( iColumn ){ + case JEACH_KEY: { + if( p->nParent==0 ){ + u32 n, j; + if( p->nRoot==1 ) break; + j = jsonEachPathLength(p); + n = p->nRoot - j; + if( n==0 ){ + break; + }else if( p->path.zBuf[j]=='[' ){ + i64 x; + sqlite3Atoi64(&p->path.zBuf[j+1], &x, n-1, SQLITE_UTF8); + sqlite3_result_int64(ctx, x); + }else if( p->path.zBuf[j+1]=='"' ){ + sqlite3_result_text(ctx, &p->path.zBuf[j+2], n-3, SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(ctx, &p->path.zBuf[j+1], n-1, SQLITE_TRANSIENT); + } + break; + } + if( p->eType==JSONB_OBJECT ){ + jsonReturnFromBlob(&p->sParse, p->i, ctx, 1); + }else{ + assert( p->eType==JSONB_ARRAY ); + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iKey); + } + break; + } + case JEACH_VALUE: { + u32 i = jsonSkipLabel(p); + jsonReturnFromBlob(&p->sParse, i, ctx, 1); + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY ){ + sqlite3_result_subtype(ctx, JSON_SUBTYPE); + } + break; + } + case JEACH_TYPE: { + u32 i = jsonSkipLabel(p); + u8 eType = p->sParse.aBlob[i] & 0x0f; + sqlite3_result_text(ctx, jsonbType[eType], -1, SQLITE_STATIC); + break; + } + case JEACH_ATOM: { + u32 i = jsonSkipLabel(p); + if( (p->sParse.aBlob[i] & 0x0f)sParse, i, ctx, 1); + } + break; + } + case JEACH_ID: { + sqlite3_result_int64(ctx, (sqlite3_int64)p->i); + break; + } + case JEACH_PARENT: { + if( p->nParent>0 && p->bRecursive ){ + sqlite3_result_int64(ctx, p->aParent[p->nParent-1].iHead); + } + break; + } + case JEACH_FULLKEY: { + u64 nBase = p->path.nUsed; + if( p->nParent ) jsonAppendPathName(p); + sqlite3_result_text64(ctx, p->path.zBuf, p->path.nUsed, + SQLITE_TRANSIENT, SQLITE_UTF8); + p->path.nUsed = nBase; + break; + } + case JEACH_PATH: { + u32 n = jsonEachPathLength(p); + sqlite3_result_text64(ctx, p->path.zBuf, n, + SQLITE_TRANSIENT, SQLITE_UTF8); + break; + } + default: { + sqlite3_result_text(ctx, p->path.zBuf, p->nRoot, SQLITE_STATIC); + break; + } + case JEACH_JSON: { + if( p->sParse.zJson==0 ){ + sqlite3_result_blob(ctx, p->sParse.aBlob, p->sParse.nBlob, + SQLITE_TRANSIENT); + }else{ + sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_TRANSIENT); + } + break; + } + } + return SQLITE_OK; +} + +/* Return the current rowid value */ +static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + JsonEachCursor *p = (JsonEachCursor*)cur; + *pRowid = p->iRowid; + return SQLITE_OK; +} + +/* The query strategy is to look for an equality constraint on the json +** column. Without such a constraint, the table cannot operate. idxNum is +** 1 if the constraint is found, 3 if the constraint and zRoot are found, +** and 0 otherwise. +*/ +static int jsonEachBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + int i; /* Loop counter or computed array index */ + int aIdx[2]; /* Index of constraints for JSON and ROOT */ + int unusableMask = 0; /* Mask of unusable JSON and ROOT constraints */ + int idxMask = 0; /* Mask of usable == constraints JSON and ROOT */ + const struct sqlite3_index_constraint *pConstraint; + + /* This implementation assumes that JSON and ROOT are the last two + ** columns in the table */ + assert( JEACH_ROOT == JEACH_JSON+1 ); + UNUSED_PARAMETER(tab); + aIdx[0] = aIdx[1] = -1; + pConstraint = pIdxInfo->aConstraint; + for(i=0; inConstraint; i++, pConstraint++){ + int iCol; + int iMask; + if( pConstraint->iColumn < JEACH_JSON ) continue; + iCol = pConstraint->iColumn - JEACH_JSON; + assert( iCol==0 || iCol==1 ); + testcase( iCol==0 ); + iMask = 1 << iCol; + if( pConstraint->usable==0 ){ + unusableMask |= iMask; + }else if( pConstraint->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + aIdx[iCol] = i; + idxMask |= iMask; + } + } + if( pIdxInfo->nOrderBy>0 + && pIdxInfo->aOrderBy[0].iColumn<0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + + if( (unusableMask & ~idxMask)!=0 ){ + /* If there are any unusable constraints on JSON or ROOT, then reject + ** this entire plan */ + return SQLITE_CONSTRAINT; + } + if( aIdx[0]<0 ){ + /* No JSON input. Leave estimatedCost at the huge value that it was + ** initialized to to discourage the query planner from selecting this + ** plan. */ + pIdxInfo->idxNum = 0; + }else{ + pIdxInfo->estimatedCost = 1.0; + i = aIdx[0]; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + if( aIdx[1]<0 ){ + pIdxInfo->idxNum = 1; /* Only JSON supplied. Plan 1 */ + }else{ + i = aIdx[1]; + pIdxInfo->aConstraintUsage[i].argvIndex = 2; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->idxNum = 3; /* Both JSON and ROOT are supplied. Plan 3 */ + } + } + return SQLITE_OK; +} + +/* Start a search on a new JSON string */ +static int jsonEachFilter( + sqlite3_vtab_cursor *cur, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + JsonEachCursor *p = (JsonEachCursor*)cur; + const char *zRoot = 0; + u32 i, n, sz; + + UNUSED_PARAMETER(idxStr); + UNUSED_PARAMETER(argc); + jsonEachCursorReset(p); + if( idxNum==0 ) return SQLITE_OK; + memset(&p->sParse, 0, sizeof(p->sParse)); + p->sParse.nJPRef = 1; + p->sParse.db = p->db; + if( jsonFuncArgMightBeBinary(argv[0]) ){ + p->sParse.nBlob = sqlite3_value_bytes(argv[0]); + p->sParse.aBlob = (u8*)sqlite3_value_blob(argv[0]); + }else{ + p->sParse.zJson = (char*)sqlite3_value_text(argv[0]); + p->sParse.nJson = sqlite3_value_bytes(argv[0]); + if( p->sParse.zJson==0 ){ + p->i = p->iEnd = 0; + return SQLITE_OK; + } + if( jsonConvertTextToBlob(&p->sParse, 0) ){ + if( p->sParse.oom ){ + return SQLITE_NOMEM; + } + goto json_each_malformed_input; + } + } + if( idxNum==3 ){ + zRoot = (const char*)sqlite3_value_text(argv[1]); + if( zRoot==0 ) return SQLITE_OK; + if( zRoot[0]!='$' ){ + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + p->nRoot = sqlite3Strlen30(zRoot); + if( zRoot[1]==0 ){ + i = p->i = 0; + p->eType = 0; + }else{ + i = jsonLookupStep(&p->sParse, 0, zRoot+1, 0); + if( JSON_LOOKUP_ISERROR(i) ){ + if( i==JSON_LOOKUP_NOTFOUND ){ + p->i = 0; + p->eType = 0; + p->iEnd = 0; + return SQLITE_OK; + } + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = jsonBadPathError(0, zRoot); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; + } + if( p->sParse.iLabel ){ + p->i = p->sParse.iLabel; + p->eType = JSONB_OBJECT; + }else{ + p->i = i; + p->eType = JSONB_ARRAY; + } + } + jsonAppendRaw(&p->path, zRoot, p->nRoot); + }else{ + i = p->i = 0; + p->eType = 0; + p->nRoot = 1; + jsonAppendRaw(&p->path, "$", 1); + } + p->nParent = 0; + n = jsonbPayloadSize(&p->sParse, i, &sz); + p->iEnd = i+n+sz; + if( (p->sParse.aBlob[i] & 0x0f)>=JSONB_ARRAY && !p->bRecursive ){ + p->i = i + n; + p->eType = p->sParse.aBlob[i] & 0x0f; + p->aParent = sqlite3DbMallocZero(p->db, sizeof(JsonParent)); + if( p->aParent==0 ) return SQLITE_NOMEM; + p->nParent = 1; + p->nParentAlloc = 1; + p->aParent[0].iKey = 0; + p->aParent[0].iEnd = p->iEnd; + p->aParent[0].iHead = p->i; + p->aParent[0].iValue = i; + } + return SQLITE_OK; + +json_each_malformed_input: + sqlite3_free(cur->pVtab->zErrMsg); + cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); + jsonEachCursorReset(p); + return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; +} + +/* The methods of the json_each virtual table */ +static sqlite3_module jsonEachModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenEach, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +/* The methods of the json_tree virtual table. */ +static sqlite3_module jsonTreeModule = { + 0, /* iVersion */ + 0, /* xCreate */ + jsonEachConnect, /* xConnect */ + jsonEachBestIndex, /* xBestIndex */ + jsonEachDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + jsonEachOpenTree, /* xOpen - open a cursor */ + jsonEachClose, /* xClose - close a cursor */ + jsonEachFilter, /* xFilter - configure scan constraints */ + jsonEachNext, /* xNext - advance a cursor */ + jsonEachEof, /* xEof - check for end of scan */ + jsonEachColumn, /* xColumn - read data */ + jsonEachRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; +#endif /* SQLITE_OMIT_VIRTUALTABLE */ +#endif /* !defined(SQLITE_OMIT_JSON) */ + +/* +** Register JSON functions. +*/ +SQLITE_PRIVATE void sqlite3RegisterJsonFunctions(void){ +#ifndef SQLITE_OMIT_JSON + static FuncDef aJsonFunc[] = { + /* sqlite3_result_subtype() ----, ,--- sqlite3_value_subtype() */ + /* | | */ + /* Uses cache ------, | | ,---- Returns JSONB */ + /* | | | | */ + /* Number of arguments ---, | | | | ,--- Flags */ + /* | | | | | | */ + JFUNCTION(json, 1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb, 1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_array, -1,0,1, 1,0,0, jsonArrayFunc), + JFUNCTION(jsonb_array, -1,0,1, 1,1,0, jsonArrayFunc), + JFUNCTION(json_array_length, 1,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_array_length, 2,1,0, 0,0,0, jsonArrayLengthFunc), + JFUNCTION(json_error_position,1,1,0, 0,0,0, jsonErrorFunc), + JFUNCTION(json_extract, -1,1,1, 0,0,0, jsonExtractFunc), + JFUNCTION(jsonb_extract, -1,1,0, 0,1,0, jsonExtractFunc), + JFUNCTION(->, 2,1,1, 0,0,JSON_JSON, jsonExtractFunc), + JFUNCTION(->>, 2,1,0, 0,0,JSON_SQL, jsonExtractFunc), + JFUNCTION(json_insert, -1,1,1, 1,0,0, jsonSetFunc), + JFUNCTION(jsonb_insert, -1,1,0, 1,1,0, jsonSetFunc), + JFUNCTION(json_object, -1,0,1, 1,0,0, jsonObjectFunc), + JFUNCTION(jsonb_object, -1,0,1, 1,1,0, jsonObjectFunc), + JFUNCTION(json_patch, 2,1,1, 0,0,0, jsonPatchFunc), + JFUNCTION(jsonb_patch, 2,1,0, 0,1,0, jsonPatchFunc), + JFUNCTION(json_pretty, 1,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_pretty, 2,1,0, 0,0,0, jsonPrettyFunc), + JFUNCTION(json_quote, 1,0,1, 1,0,0, jsonQuoteFunc), + JFUNCTION(json_remove, -1,1,1, 0,0,0, jsonRemoveFunc), + JFUNCTION(jsonb_remove, -1,1,0, 0,1,0, jsonRemoveFunc), + JFUNCTION(json_replace, -1,1,1, 1,0,0, jsonReplaceFunc), + JFUNCTION(jsonb_replace, -1,1,0, 1,1,0, jsonReplaceFunc), + JFUNCTION(json_set, -1,1,1, 1,0,JSON_ISSET, jsonSetFunc), + JFUNCTION(jsonb_set, -1,1,0, 1,1,JSON_ISSET, jsonSetFunc), + JFUNCTION(json_type, 1,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_type, 2,1,0, 0,0,0, jsonTypeFunc), + JFUNCTION(json_valid, 1,1,0, 0,0,0, jsonValidFunc), + JFUNCTION(json_valid, 2,1,0, 0,0,0, jsonValidFunc), +#if SQLITE_DEBUG + JFUNCTION(json_parse, 1,1,0, 0,0,0, jsonParseFunc), +#endif + WAGGREGATE(json_group_array, 1, 0, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_array, 1, JSON_BLOB, 0, + jsonArrayStep, jsonArrayFinal, jsonArrayValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(json_group_object, 2, 0, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8|SQLITE_DETERMINISTIC), + WAGGREGATE(jsonb_group_object,2, JSON_BLOB, 0, + jsonObjectStep, jsonObjectFinal, jsonObjectValue, jsonGroupInverse, + SQLITE_SUBTYPE|SQLITE_RESULT_SUBTYPE|SQLITE_UTF8| + SQLITE_DETERMINISTIC) + }; + sqlite3InsertBuiltinFuncs(aJsonFunc, ArraySize(aJsonFunc)); +#endif +} + +#if !defined(SQLITE_OMIT_VIRTUALTABLE) && !defined(SQLITE_OMIT_JSON) +/* +** Register the JSON table-valued functions +*/ +SQLITE_PRIVATE int sqlite3JsonTableFunctions(sqlite3 *db){ + int rc = SQLITE_OK; + static const struct { + const char *zName; + sqlite3_module *pModule; + } aMod[] = { + { "json_each", &jsonEachModule }, + { "json_tree", &jsonTreeModule }, + }; + unsigned int i; + for(i=0; i */ -/* #include */ -/* #include */ - -#ifndef SQLITE_AMALGAMATION +/* +** If building separately, we will need some setup that is normally +** found in sqliteInt.h +*/ +#if !defined(SQLITE_AMALGAMATION) #include "sqlite3rtree.h" typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; typedef unsigned char u8; typedef unsigned short u16; typedef unsigned int u32; +#if !defined(NDEBUG) && !defined(SQLITE_DEBUG) +# define NDEBUG 1 #endif +#if defined(NDEBUG) && defined(SQLITE_DEBUG) +# undef NDEBUG +#endif +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif +#endif /* !defined(SQLITE_AMALGAMATION) */ + +/* Macro to check for 4-byte alignment. Only used inside of assert() */ +#ifdef SQLITE_DEBUG +# define FOUR_BYTE_ALIGNED(X) ((((char*)(X) - (char*)0) & 3)==0) +#endif + +/* #include */ +/* #include */ +/* #include */ +/* #include */ /* The following macro is used to suppress compiler warnings. */ @@ -159175,8 +211771,11 @@ typedef struct RtreeSearchPoint RtreeSearchPoint; /* The rtree may have between 1 and RTREE_MAX_DIMENSIONS dimensions. */ #define RTREE_MAX_DIMENSIONS 5 +/* Maximum number of auxiliary columns */ +#define RTREE_MAX_AUX_COLUMN 100 + /* Size of hash table Rtree.aHash. This hash table is not expected to -** ever contain very many entries, so a fixed number of buckets is +** ever contain very many entries, so a fixed number of buckets is ** used. */ #define HASHSIZE 97 @@ -159185,13 +211784,13 @@ typedef struct RtreeSearchPoint RtreeSearchPoint; ** the number of rows in the virtual table to calculate the costs of ** various strategies. If possible, this estimate is loaded from the ** sqlite_stat1 table (with RTREE_MIN_ROWEST as a hard-coded minimum). -** Otherwise, if no sqlite_stat1 entry is available, use +** Otherwise, if no sqlite_stat1 entry is available, use ** RTREE_DEFAULT_ROWEST. */ #define RTREE_DEFAULT_ROWEST 1048576 #define RTREE_MIN_ROWEST 100 -/* +/* ** An rtree virtual-table object. */ struct Rtree { @@ -159199,24 +211798,38 @@ struct Rtree { sqlite3 *db; /* Host database connection */ int iNodeSize; /* Size in bytes of each node in the node table */ u8 nDim; /* Number of dimensions */ + u8 nDim2; /* Twice the number of dimensions */ u8 eCoordType; /* RTREE_COORD_REAL32 or RTREE_COORD_INT32 */ u8 nBytesPerCell; /* Bytes consumed per cell */ + u8 inWrTrans; /* True if inside write transaction */ + u8 nAux; /* # of auxiliary columns in %_rowid */ +#ifdef SQLITE_ENABLE_GEOPOLY + u8 nAuxNotNull; /* Number of initial not-null aux columns */ +#endif +#ifdef SQLITE_DEBUG + u8 bCorrupt; /* Shadow table corruption detected */ +#endif int iDepth; /* Current depth of the r-tree structure */ char *zDb; /* Name of database containing r-tree table */ - char *zName; /* Name of r-tree table */ - int nBusy; /* Current number of users of this structure */ + char *zName; /* Name of r-tree table */ + char *zNodeName; /* Name of the %_node table */ + u32 nBusy; /* Current number of users of this structure */ i64 nRowEst; /* Estimated number of rows in this table */ + u32 nCursor; /* Number of open cursors */ + u32 nNodeRef; /* Number RtreeNodes with positive nRef */ + char *zReadAuxSql; /* SQL for statement to read aux data */ /* List of nodes removed during a CondenseTree operation. List is ** linked together via the pointer normally used for hash chains - - ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree + ** RtreeNode.pNext. RtreeNode.iNode stores the depth of the sub-tree ** headed by the node (leaf nodes have RtreeNode.iNode==0). */ RtreeNode *pDeleted; - int iReinsertHeight; /* Height of sub-trees Reinsert() has run on */ + + /* Blob I/O on xxx_node */ + sqlite3_blob *pNodeBlob; /* Statements to read/write/delete a record from xxx_node */ - sqlite3_stmt *pReadNode; sqlite3_stmt *pWriteNode; sqlite3_stmt *pDeleteNode; @@ -159230,7 +211843,10 @@ struct Rtree { sqlite3_stmt *pWriteParent; sqlite3_stmt *pDeleteParent; - RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ + /* Statement for writing to the "aux:" fields, if there are any */ + sqlite3_stmt *pWriteAux; + + RtreeNode *aHash[HASHSIZE]; /* Hash table of in-memory nodes. */ }; /* Possible values for Rtree.eCoordType: */ @@ -159252,6 +211868,15 @@ struct Rtree { # define RTREE_ZERO 0.0 #endif +/* +** Set the Rtree.bCorrupt flag +*/ +#ifdef SQLITE_DEBUG +# define RTREE_IS_CORRUPT(X) ((X)->bCorrupt = 1) +#else +# define RTREE_IS_CORRUPT(X) +#endif + /* ** When doing a search of an r-tree, instances of the following structure ** record intermediate results from the tree walk. @@ -159270,7 +211895,7 @@ struct RtreeSearchPoint { }; /* -** The minimum number of cells allowed for a node is a third of the +** The minimum number of cells allowed for a node is a third of the ** maximum. In Gutman's notation: ** ** m = M/3 @@ -159285,8 +211910,8 @@ struct RtreeSearchPoint { /* ** The smallest possible node-size is (512-64)==448 bytes. And the largest ** supported cell size is 48 bytes (8 byte rowid + ten 4 byte coordinates). -** Therefore all non-root nodes must contain at least 3 entries. Since -** 2^40 is greater than 2^64, an r-tree structure always has a depth of +** Therefore all non-root nodes must contain at least 3 entries. Since +** 3^40 is greater than 2^64, an r-tree structure always has a depth of ** 40 or less. */ #define RTREE_MAX_DEPTH 40 @@ -159299,13 +211924,14 @@ struct RtreeSearchPoint { */ #define RTREE_CACHE_SZ 5 -/* +/* ** An rtree cursor object. */ struct RtreeCursor { sqlite3_vtab_cursor base; /* Base class. Must be first */ u8 atEOF; /* True if at end of search */ u8 bPoint; /* True if sPoint is valid */ + u8 bAuxValid; /* True if pReadAux is valid */ int iStrategy; /* Copy of idxNum search parameter */ int nConstraint; /* Number of entries in aConstraint */ RtreeConstraint *aConstraint; /* Search constraints. */ @@ -159313,6 +211939,7 @@ struct RtreeCursor { int nPoint; /* Number of slots used in aPoint[] */ int mxLevel; /* iLevel value for root of the tree */ RtreeSearchPoint *aPoint; /* Priority queue for search points */ + sqlite3_stmt *pReadAux; /* Statement to read aux-data */ RtreeSearchPoint sPoint; /* Cached next search point */ RtreeNode *aNode[RTREE_CACHE_SZ]; /* Rtree node cache */ u32 anQueue[RTREE_MAX_DEPTH+1]; /* Number of queued entries by iLevel */ @@ -159370,8 +211997,14 @@ struct RtreeConstraint { #define RTREE_MATCH 0x46 /* F: Old-style sqlite3_rtree_geometry_callback() */ #define RTREE_QUERY 0x47 /* G: New-style sqlite3_rtree_query_callback() */ +/* Special operators available only on cursors. Needs to be consecutive +** with the normal values above, but must be less than RTREE_MATCH. These +** are used in the cursor for contraints such as x=NULL (RTREE_FALSE) or +** x<'xyz' (RTREE_TRUE) */ +#define RTREE_TRUE 0x3f /* ? */ +#define RTREE_FALSE 0x40 /* @ */ -/* +/* ** An rtree structure node. */ struct RtreeNode { @@ -159386,7 +212019,7 @@ struct RtreeNode { /* Return the number of cells in a node */ #define NCELL(pNode) readInt16(&(pNode)->zData[2]) -/* +/* ** A single cell from a node, deserialized */ struct RtreeCell { @@ -159401,11 +212034,11 @@ struct RtreeCell { ** sqlite3_rtree_query_callback() and which appear on the right of MATCH ** operators in order to constrain a search. ** -** xGeom and xQueryFunc are the callback functions. Exactly one of +** xGeom and xQueryFunc are the callback functions. Exactly one of ** xGeom and xQueryFunc fields is non-NULL, depending on whether the ** SQL function was created using sqlite3_rtree_geometry_callback() or ** sqlite3_rtree_query_callback(). -** +** ** This object is deleted automatically by the destructor mechanism in ** sqlite3_create_function_v2(). */ @@ -159416,14 +212049,6 @@ struct RtreeGeomCallback { void *pContext; }; - -/* -** Value for the first field of every RtreeMatchArg object. The MATCH -** operator tests that the first field of a blob operand matches this -** value to avoid operating on invalid blobs (which could cause a segfault). -*/ -#define RTREE_GEOMETRY_MAGIC 0x891245AB - /* ** An instance of this structure (in the form of a BLOB) is returned by ** the SQL functions that sqlite3_rtree_geometry_callback() and @@ -159431,7 +212056,7 @@ struct RtreeGeomCallback { ** operand to the MATCH operator of an R-Tree. */ struct RtreeMatchArg { - u32 magic; /* Always RTREE_GEOMETRY_MAGIC */ + u32 iSize; /* Size of this object */ RtreeGeomCallback cb; /* Info about the callback functions */ int nParam; /* Number of parameters to the SQL function */ sqlite3_value **apSqlParam; /* Original SQL parameter values */ @@ -159445,6 +212070,86 @@ struct RtreeMatchArg { # define MIN(x,y) ((x) > (y) ? (y) : (x)) #endif +/* What version of GCC is being used. 0 means GCC is not being used . +** Note that the GCC_VERSION macro will also be set correctly when using +** clang, since clang works hard to be gcc compatible. So the gcc +** optimizations will also work when compiling with clang. +*/ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif + +/* The testcase() macro should already be defined in the amalgamation. If +** it is not, make it a no-op. +*/ +#ifndef SQLITE_AMALGAMATION +# if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_DEBUG) + unsigned int sqlite3RtreeTestcase = 0; +# define testcase(X) if( X ){ sqlite3RtreeTestcase += __LINE__; } +# else +# define testcase(X) +# endif +#endif + +/* +** Make sure that the compiler intrinsics we desire are enabled when +** compiling with an appropriate version of MSVC unless prevented by +** the SQLITE_DISABLE_INTRINSIC define. +*/ +#if !defined(SQLITE_DISABLE_INTRINSIC) +# if defined(_MSC_VER) && _MSC_VER>=1400 +# if !defined(_WIN32_WCE) +/* # include */ +# pragma intrinsic(_byteswap_ulong) +# pragma intrinsic(_byteswap_uint64) +# else +/* # include */ +# endif +# endif +#endif + +/* +** Macros to determine whether the machine is big or little endian, +** and whether or not that determination is run-time or compile-time. +** +** For best performance, an attempt is made to guess at the byte-order +** using C-preprocessor macros. If that is unsuccessful, or if +** -DSQLITE_RUNTIME_BYTEORDER=1 is set, then byte-order is determined +** at run-time. +*/ +#ifndef SQLITE_BYTEORDER /* Replicate changes at tag-20230904a */ +# if defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +# define SQLITE_BYTEORDER 4321 +# elif defined(__BYTE_ORDER__) && __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ +# define SQLITE_BYTEORDER 1234 +# elif defined(__BIG_ENDIAN__) && __BIG_ENDIAN__==1 +# define SQLITE_BYTEORDER 4321 +# elif defined(i386) || defined(__i386__) || defined(_M_IX86) || \ + defined(__x86_64) || defined(__x86_64__) || defined(_M_X64) || \ + defined(_M_AMD64) || defined(_M_ARM) || defined(__x86) || \ + defined(__ARMEL__) || defined(__AARCH64EL__) || defined(_M_ARM64) +# define SQLITE_BYTEORDER 1234 +# elif defined(sparc) || defined(__ARMEB__) || defined(__AARCH64EB__) +# define SQLITE_BYTEORDER 4321 +# else +# define SQLITE_BYTEORDER 0 +# endif +#endif + + +/* What version of MSVC is being used. 0 means MSVC is not being used */ +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + /* ** Functions to deserialize a 16 bit integer, 32 bit real number and ** 64 bit integer. The deserialized value is returned. @@ -159453,24 +212158,47 @@ static int readInt16(u8 *p){ return (p[0]<<8) + p[1]; } static void readCoord(u8 *p, RtreeCoord *pCoord){ + assert( FOUR_BYTE_ALIGNED(p) ); +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + pCoord->u = _byteswap_ulong(*(u32*)p); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + pCoord->u = __builtin_bswap32(*(u32*)p); +#elif SQLITE_BYTEORDER==4321 + pCoord->u = *(u32*)p; +#else pCoord->u = ( - (((u32)p[0]) << 24) + - (((u32)p[1]) << 16) + - (((u32)p[2]) << 8) + + (((u32)p[0]) << 24) + + (((u32)p[1]) << 16) + + (((u32)p[2]) << 8) + (((u32)p[3]) << 0) ); +#endif } static i64 readInt64(u8 *p){ - return ( - (((i64)p[0]) << 56) + - (((i64)p[1]) << 48) + - (((i64)p[2]) << 40) + - (((i64)p[3]) << 32) + - (((i64)p[4]) << 24) + - (((i64)p[5]) << 16) + - (((i64)p[6]) << 8) + - (((i64)p[7]) << 0) +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + u64 x; + memcpy(&x, p, 8); + return (i64)_byteswap_uint64(x); +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + u64 x; + memcpy(&x, p, 8); + return (i64)__builtin_bswap64(x); +#elif SQLITE_BYTEORDER==4321 + i64 x; + memcpy(&x, p, 8); + return x; +#else + return (i64)( + (((u64)p[0]) << 56) + + (((u64)p[1]) << 48) + + (((u64)p[2]) << 40) + + (((u64)p[3]) << 32) + + (((u64)p[4]) << 24) + + (((u64)p[5]) << 16) + + (((u64)p[6]) << 8) + + (((u64)p[7]) << 0) ); +#endif } /* @@ -159478,23 +212206,43 @@ static i64 readInt64(u8 *p){ ** 64 bit integer. The value returned is the number of bytes written ** to the argument buffer (always 2, 4 and 8 respectively). */ -static int writeInt16(u8 *p, int i){ +static void writeInt16(u8 *p, int i){ p[0] = (i>> 8)&0xFF; p[1] = (i>> 0)&0xFF; - return 2; } static int writeCoord(u8 *p, RtreeCoord *pCoord){ u32 i; + assert( FOUR_BYTE_ALIGNED(p) ); assert( sizeof(RtreeCoord)==4 ); assert( sizeof(u32)==4 ); +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = __builtin_bswap32(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = _byteswap_ulong(pCoord->u); + memcpy(p, &i, 4); +#elif SQLITE_BYTEORDER==4321 + i = pCoord->u; + memcpy(p, &i, 4); +#else i = pCoord->u; p[0] = (i>>24)&0xFF; p[1] = (i>>16)&0xFF; p[2] = (i>> 8)&0xFF; p[3] = (i>> 0)&0xFF; +#endif return 4; } static int writeInt64(u8 *p, i64 i){ +#if SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 + i = (i64)__builtin_bswap64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 + i = (i64)_byteswap_uint64((u64)i); + memcpy(p, &i, 8); +#elif SQLITE_BYTEORDER==4321 + memcpy(p, &i, 8); +#else p[0] = (i>>56)&0xFF; p[1] = (i>>48)&0xFF; p[2] = (i>>40)&0xFF; @@ -159503,6 +212251,7 @@ static int writeInt64(u8 *p, i64 i){ p[5] = (i>>16)&0xFF; p[6] = (i>> 8)&0xFF; p[7] = (i>> 0)&0xFF; +#endif return 8; } @@ -159511,6 +212260,7 @@ static int writeInt64(u8 *p, i64 i){ */ static void nodeReference(RtreeNode *p){ if( p ){ + assert( p->nRef>0 ); p->nRef++; } } @@ -159527,8 +212277,8 @@ static void nodeZero(Rtree *pRtree, RtreeNode *p){ ** Given a node number iNode, return the corresponding key to use ** in the Rtree.aHash table. */ -static int nodeHash(i64 iNode){ - return iNode % HASHSIZE; +static unsigned int nodeHash(i64 iNode){ + return ((unsigned)iNode) % HASHSIZE; } /* @@ -159573,11 +212323,12 @@ static void nodeHashDelete(Rtree *pRtree, RtreeNode *pNode){ */ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ RtreeNode *pNode; - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode) + pRtree->iNodeSize); + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode) + pRtree->iNodeSize); if( pNode ){ memset(pNode, 0, sizeof(RtreeNode) + pRtree->iNodeSize); pNode->zData = (u8 *)&pNode[1]; pNode->nRef = 1; + pRtree->nNodeRef++; pNode->pParent = pParent; pNode->isDirty = 1; nodeReference(pParent); @@ -159585,6 +212336,15 @@ static RtreeNode *nodeNew(Rtree *pRtree, RtreeNode *pParent){ return pNode; } +/* +** Clear the Rtree.pNodeBlob object +*/ +static void nodeBlobReset(Rtree *pRtree){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + sqlite3_blob_close(pBlob); +} + /* ** Obtain a reference to an r-tree node. */ @@ -159594,46 +212354,61 @@ static int nodeAcquire( RtreeNode *pParent, /* Either the parent node or NULL */ RtreeNode **ppNode /* OUT: Acquired node */ ){ - int rc; - int rc2 = SQLITE_OK; - RtreeNode *pNode; + int rc = SQLITE_OK; + RtreeNode *pNode = 0; /* Check if the requested node is already in the hash table. If so, ** increase its reference count and return it. */ - if( (pNode = nodeHashLookup(pRtree, iNode)) ){ - assert( !pParent || !pNode->pParent || pNode->pParent==pParent ); - if( pParent && !pNode->pParent ){ - nodeReference(pParent); - pNode->pParent = pParent; + if( (pNode = nodeHashLookup(pRtree, iNode))!=0 ){ + if( pParent && ALWAYS(pParent!=pNode->pParent) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; } pNode->nRef++; *ppNode = pNode; return SQLITE_OK; } - sqlite3_bind_int64(pRtree->pReadNode, 1, iNode); - rc = sqlite3_step(pRtree->pReadNode); - if( rc==SQLITE_ROW ){ - const u8 *zBlob = sqlite3_column_blob(pRtree->pReadNode, 0); - if( pRtree->iNodeSize==sqlite3_column_bytes(pRtree->pReadNode, 0) ){ - pNode = (RtreeNode *)sqlite3_malloc(sizeof(RtreeNode)+pRtree->iNodeSize); - if( !pNode ){ - rc2 = SQLITE_NOMEM; - }else{ - pNode->pParent = pParent; - pNode->zData = (u8 *)&pNode[1]; - pNode->nRef = 1; - pNode->iNode = iNode; - pNode->isDirty = 0; - pNode->pNext = 0; - memcpy(pNode->zData, zBlob, pRtree->iNodeSize); - nodeReference(pParent); - } + if( pRtree->pNodeBlob ){ + sqlite3_blob *pBlob = pRtree->pNodeBlob; + pRtree->pNodeBlob = 0; + rc = sqlite3_blob_reopen(pBlob, iNode); + pRtree->pNodeBlob = pBlob; + if( rc ){ + nodeBlobReset(pRtree); + if( rc==SQLITE_NOMEM ) return SQLITE_NOMEM; + } + } + if( pRtree->pNodeBlob==0 ){ + rc = sqlite3_blob_open(pRtree->db, pRtree->zDb, pRtree->zNodeName, + "data", iNode, 0, + &pRtree->pNodeBlob); + } + if( rc ){ + *ppNode = 0; + /* If unable to open an sqlite3_blob on the desired row, that can only + ** be because the shadow tables hold erroneous data. */ + if( rc==SQLITE_ERROR ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + } + }else if( pRtree->iNodeSize==sqlite3_blob_bytes(pRtree->pNodeBlob) ){ + pNode = (RtreeNode *)sqlite3_malloc64(sizeof(RtreeNode)+pRtree->iNodeSize); + if( !pNode ){ + rc = SQLITE_NOMEM; + }else{ + pNode->pParent = pParent; + pNode->zData = (u8 *)&pNode[1]; + pNode->nRef = 1; + pRtree->nNodeRef++; + pNode->iNode = iNode; + pNode->isDirty = 0; + pNode->pNext = 0; + rc = sqlite3_blob_read(pRtree->pNodeBlob, pNode->zData, + pRtree->iNodeSize, 0); } } - rc = sqlite3_reset(pRtree->pReadNode); - if( rc==SQLITE_OK ) rc = rc2; /* If the root node was just loaded, set pRtree->iDepth to the height ** of the r-tree structure. A height of zero means all data is stored on @@ -159641,32 +212416,40 @@ static int nodeAcquire( ** are the leaves, and so on. If the depth as specified on the root node ** is greater than RTREE_MAX_DEPTH, the r-tree structure must be corrupt. */ - if( pNode && iNode==1 ){ + if( rc==SQLITE_OK && pNode && iNode==1 ){ pRtree->iDepth = readInt16(pNode->zData); if( pRtree->iDepth>RTREE_MAX_DEPTH ){ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); } } /* If no error has occurred so far, check if the "number of entries" - ** field on the node is too large. If so, set the return code to + ** field on the node is too large. If so, set the return code to ** SQLITE_CORRUPT_VTAB. */ if( pNode && rc==SQLITE_OK ){ if( NCELL(pNode)>((pRtree->iNodeSize-4)/pRtree->nBytesPerCell) ){ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); } } if( rc==SQLITE_OK ){ if( pNode!=0 ){ + nodeReference(pParent); nodeHashInsert(pRtree, pNode); }else{ rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); } *ppNode = pNode; }else{ - sqlite3_free(pNode); + nodeBlobReset(pRtree); + if( pNode ){ + pRtree->nNodeRef--; + sqlite3_free(pNode); + } *ppNode = 0; } @@ -159685,7 +212468,7 @@ static void nodeOverwriteCell( int ii; u8 *p = &pNode->zData[4 + pRtree->nBytesPerCell*iCell]; p += writeInt64(p, pCell->iRowid); - for(ii=0; ii<(pRtree->nDim*2); ii++){ + for(ii=0; iinDim2; ii++){ p += writeCoord(p, &pCell->aCoord[ii]); } pNode->isDirty = 1; @@ -159746,6 +212529,7 @@ static int nodeWrite(Rtree *pRtree, RtreeNode *pNode){ sqlite3_step(p); pNode->isDirty = 0; rc = sqlite3_reset(p); + sqlite3_bind_null(p, 2); if( pNode->iNode==0 && rc==SQLITE_OK ){ pNode->iNode = sqlite3_last_insert_rowid(pRtree->db); nodeHashInsert(pRtree, pNode); @@ -159762,8 +212546,10 @@ static int nodeRelease(Rtree *pRtree, RtreeNode *pNode){ int rc = SQLITE_OK; if( pNode ){ assert( pNode->nRef>0 ); + assert( pRtree->nNodeRef>0 ); pNode->nRef--; if( pNode->nRef==0 ){ + pRtree->nNodeRef--; if( pNode->iNode==1 ){ pRtree->iDepth = -1; } @@ -159804,6 +212590,7 @@ static void nodeGetCoord( int iCoord, /* Which coordinate to extract */ RtreeCoord *pCoord /* OUT: Space to write result to */ ){ + assert( iCellzData[12 + pRtree->nBytesPerCell*iCell + 4*iCoord], pCoord); } @@ -159819,13 +212606,16 @@ static void nodeGetCell( ){ u8 *pData; RtreeCoord *pCoord; - int ii; + int ii = 0; pCell->iRowid = nodeGetRowid(pRtree, pNode, iCell); pData = pNode->zData + (12 + pRtree->nBytesPerCell*iCell); pCoord = pCell->aCoord; - for(ii=0; iinDim*2; ii++){ - readCoord(&pData[ii*4], &pCoord[ii]); - } + do{ + readCoord(pData, &pCoord[ii]); + readCoord(pData+4, &pCoord[ii+1]); + pData += 8; + ii += 2; + }while( iinDim2 ); } @@ -159836,7 +212626,7 @@ static int rtreeInit( sqlite3 *, void *, int, const char *const*, sqlite3_vtab **, char **, int ); -/* +/* ** Rtree virtual table module xCreate method. */ static int rtreeCreate( @@ -159849,7 +212639,7 @@ static int rtreeCreate( return rtreeInit(db, pAux, argc, argv, ppVtab, pzErr, 1); } -/* +/* ** Rtree virtual table module xConnect method. */ static int rtreeConnect( @@ -159876,7 +212666,10 @@ static void rtreeReference(Rtree *pRtree){ static void rtreeRelease(Rtree *pRtree){ pRtree->nBusy--; if( pRtree->nBusy==0 ){ - sqlite3_finalize(pRtree->pReadNode); + pRtree->inWrTrans = 0; + assert( pRtree->nCursor==0 ); + nodeBlobReset(pRtree); + assert( pRtree->nNodeRef==0 || pRtree->bCorrupt ); sqlite3_finalize(pRtree->pWriteNode); sqlite3_finalize(pRtree->pDeleteNode); sqlite3_finalize(pRtree->pReadRowid); @@ -159885,11 +212678,13 @@ static void rtreeRelease(Rtree *pRtree){ sqlite3_finalize(pRtree->pReadParent); sqlite3_finalize(pRtree->pWriteParent); sqlite3_finalize(pRtree->pDeleteParent); + sqlite3_finalize(pRtree->pWriteAux); + sqlite3_free(pRtree->zReadAuxSql); sqlite3_free(pRtree); } } -/* +/* ** Rtree virtual table module xDisconnect method. */ static int rtreeDisconnect(sqlite3_vtab *pVtab){ @@ -159897,7 +212692,7 @@ static int rtreeDisconnect(sqlite3_vtab *pVtab){ return SQLITE_OK; } -/* +/* ** Rtree virtual table module xDestroy method. */ static int rtreeDestroy(sqlite3_vtab *pVtab){ @@ -159907,13 +212702,14 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ "DROP TABLE '%q'.'%q_node';" "DROP TABLE '%q'.'%q_rowid';" "DROP TABLE '%q'.'%q_parent';", - pRtree->zDb, pRtree->zName, + pRtree->zDb, pRtree->zName, pRtree->zDb, pRtree->zName, pRtree->zDb, pRtree->zName ); if( !zCreate ){ rc = SQLITE_NOMEM; }else{ + nodeBlobReset(pRtree); rc = sqlite3_exec(pRtree->db, zCreate, 0, 0, 0); sqlite3_free(zCreate); } @@ -159924,18 +212720,20 @@ static int rtreeDestroy(sqlite3_vtab *pVtab){ return rc; } -/* +/* ** Rtree virtual table module xOpen method. */ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ int rc = SQLITE_NOMEM; + Rtree *pRtree = (Rtree *)pVTab; RtreeCursor *pCsr; - pCsr = (RtreeCursor *)sqlite3_malloc(sizeof(RtreeCursor)); + pCsr = (RtreeCursor *)sqlite3_malloc64(sizeof(RtreeCursor)); if( pCsr ){ memset(pCsr, 0, sizeof(RtreeCursor)); pCsr->base.pVtab = pVTab; rc = SQLITE_OK; + pRtree->nCursor++; } *ppCursor = (sqlite3_vtab_cursor *)pCsr; @@ -159944,9 +212742,12 @@ static int rtreeOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ /* -** Free the RtreeCursor.aConstraint[] array and its contents. +** Reset a cursor back to its initial state. */ -static void freeCursorConstraints(RtreeCursor *pCsr){ +static void resetCursor(RtreeCursor *pCsr){ + Rtree *pRtree = (Rtree *)(pCsr->base.pVtab); + int ii; + sqlite3_stmt *pStmt; if( pCsr->aConstraint ){ int i; /* Used to iterate through constraint array */ for(i=0; inConstraint; i++){ @@ -159959,26 +212760,36 @@ static void freeCursorConstraints(RtreeCursor *pCsr){ sqlite3_free(pCsr->aConstraint); pCsr->aConstraint = 0; } + for(ii=0; iiaNode[ii]); + sqlite3_free(pCsr->aPoint); + pStmt = pCsr->pReadAux; + memset(pCsr, 0, sizeof(RtreeCursor)); + pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + pCsr->pReadAux = pStmt; + } -/* +/* ** Rtree virtual table module xClose method. */ static int rtreeClose(sqlite3_vtab_cursor *cur){ Rtree *pRtree = (Rtree *)(cur->pVtab); - int ii; RtreeCursor *pCsr = (RtreeCursor *)cur; - freeCursorConstraints(pCsr); - sqlite3_free(pCsr->aPoint); - for(ii=0; iiaNode[ii]); + assert( pRtree->nCursor>0 ); + resetCursor(pCsr); + sqlite3_finalize(pCsr->pReadAux); sqlite3_free(pCsr); + pRtree->nCursor--; + if( pRtree->nCursor==0 && pRtree->inWrTrans==0 ){ + nodeBlobReset(pRtree); + } return SQLITE_OK; } /* ** Rtree virtual table module xEof method. ** -** Return non-zero if the cursor does not currently point to a valid +** Return non-zero if the cursor does not currently point to a valid ** record (i.e if the scan has finished), or zero otherwise. */ static int rtreeEof(sqlite3_vtab_cursor *cur){ @@ -159994,15 +212805,22 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ** false. a[] is the four bytes of the on-disk record to be decoded. ** Store the results in "r". ** -** There are three versions of this macro, one each for little-endian and -** big-endian processors and a third generic implementation. The endian- -** specific implementations are much faster and are preferred if the -** processor endianness is known at compile-time. The SQLITE_BYTEORDER -** macro is part of sqliteInt.h and hence the endian-specific -** implementation will only be used if this module is compiled as part -** of the amalgamation. +** There are five versions of this macro. The last one is generic. The +** other four are various architectures-specific optimizations. */ -#if defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==1234 +#if SQLITE_BYTEORDER==1234 && MSVC_VERSION>=1300 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = _byteswap_ulong(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 && GCC_VERSION>=4003000 +#define RTREE_DECODE_COORD(eInt, a, r) { \ + RtreeCoord c; /* Coordinate decoded */ \ + c.u = __builtin_bswap32(*(u32*)a); \ + r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ +} +#elif SQLITE_BYTEORDER==1234 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -160010,7 +212828,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ ((c.u&0xff)<<24)|((c.u&0xff00)<<8); \ r = eInt ? (sqlite3_rtree_dbl)c.i : (sqlite3_rtree_dbl)c.f; \ } -#elif defined(SQLITE_BYTEORDER) && SQLITE_BYTEORDER==4321 +#elif SQLITE_BYTEORDER==4321 #define RTREE_DECODE_COORD(eInt, a, r) { \ RtreeCoord c; /* Coordinate decoded */ \ memcpy(&c.u,a,4); \ @@ -160027,7 +212845,7 @@ static int rtreeEof(sqlite3_vtab_cursor *cur){ /* ** Check the RTree node or entry given by pCellData and p against the MATCH -** constraint pConstraint. +** constraint pConstraint. */ static int rtreeCallbackConstraint( RtreeConstraint *pConstraint, /* The constraint to test */ @@ -160037,10 +212855,10 @@ static int rtreeCallbackConstraint( sqlite3_rtree_dbl *prScore, /* OUT: score for the cell */ int *peWithin /* OUT: visibility of the cell */ ){ - int i; /* Loop counter */ sqlite3_rtree_query_info *pInfo = pConstraint->pInfo; /* Callback info */ int nCoord = pInfo->nCoord; /* No. of coordinates */ int rc; /* Callback return code */ + RtreeCoord c; /* Translator union */ sqlite3_rtree_dbl aCoord[RTREE_MAX_DIMENSIONS*2]; /* Decoded coordinates */ assert( pConstraint->op==RTREE_MATCH || pConstraint->op==RTREE_QUERY ); @@ -160050,13 +212868,41 @@ static int rtreeCallbackConstraint( pInfo->iRowid = readInt64(pCellData); } pCellData += 8; - for(i=0; iop==RTREE_MATCH ){ + int eWithin = 0; rc = pConstraint->u.xGeom((sqlite3_rtree_geometry*)pInfo, - nCoord, aCoord, &i); - if( i==0 ) *peWithin = NOT_WITHIN; + nCoord, aCoord, &eWithin); + if( eWithin==0 ) *peWithin = NOT_WITHIN; *prScore = RTREE_ZERO; }else{ pInfo->aCoord = aCoord; @@ -160072,7 +212918,7 @@ static int rtreeCallbackConstraint( return rc; } -/* +/* ** Check the internal RTree node given by pCellData against constraint p. ** If this constraint cannot be satisfied by any child within the node, ** set *peWithin to NOT_WITHIN. @@ -160090,23 +212936,36 @@ static void rtreeNonleafConstraint( */ pCellData += 8 + 4*(p->iCoord&0xfe); - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ ); + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); + assert( FOUR_BYTE_ALIGNED(pCellData) ); switch( p->op ){ - case RTREE_LE: - case RTREE_LT: + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ case RTREE_EQ: + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the lower bound of the coordinate pair */ + if( p->u.rValue>=val ){ + pCellData += 4; + RTREE_DECODE_COORD(eInt, pCellData, val); + /* val now holds the upper bound of the coordinate pair */ + if( p->u.rValue<=val ) return; + } + break; + case RTREE_LE: + case RTREE_LT: RTREE_DECODE_COORD(eInt, pCellData, val); /* val now holds the lower bound of the coordinate pair */ if( p->u.rValue>=val ) return; - if( p->op!=RTREE_EQ ) break; /* RTREE_LE and RTREE_LT end here */ - /* Fall through for the RTREE_EQ case */ + break; - default: /* RTREE_GT or RTREE_GE, or fallthrough of RTREE_EQ */ + default: pCellData += 4; RTREE_DECODE_COORD(eInt, pCellData, val); /* val now holds the upper bound of the coordinate pair */ if( p->u.rValue<=val ) return; + break; } *peWithin = NOT_WITHIN; } @@ -160129,27 +212988,31 @@ static void rtreeLeafConstraint( ){ RtreeDValue xN; /* Coordinate value converted to a double */ - assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE - || p->op==RTREE_GT || p->op==RTREE_EQ ); + assert(p->op==RTREE_LE || p->op==RTREE_LT || p->op==RTREE_GE + || p->op==RTREE_GT || p->op==RTREE_EQ || p->op==RTREE_TRUE + || p->op==RTREE_FALSE ); pCellData += 8 + p->iCoord*4; + assert( FOUR_BYTE_ALIGNED(pCellData) ); RTREE_DECODE_COORD(eInt, pCellData, xN); switch( p->op ){ - case RTREE_LE: if( xN <= p->u.rValue ) return; break; - case RTREE_LT: if( xN < p->u.rValue ) return; break; - case RTREE_GE: if( xN >= p->u.rValue ) return; break; - case RTREE_GT: if( xN > p->u.rValue ) return; break; - default: if( xN == p->u.rValue ) return; break; + case RTREE_TRUE: return; /* Always satisfied */ + case RTREE_FALSE: break; /* Never satisfied */ + case RTREE_LE: if( xN <= p->u.rValue ) return; break; + case RTREE_LT: if( xN < p->u.rValue ) return; break; + case RTREE_GE: if( xN >= p->u.rValue ) return; break; + case RTREE_GT: if( xN > p->u.rValue ) return; break; + default: if( xN == p->u.rValue ) return; break; } *peWithin = NOT_WITHIN; } /* -** One of the cells in node pNode is guaranteed to have a 64-bit +** One of the cells in node pNode is guaranteed to have a 64-bit ** integer value equal to iRowid. Return the index of this cell. */ static int nodeRowidIndex( - Rtree *pRtree, - RtreeNode *pNode, + Rtree *pRtree, + RtreeNode *pNode, i64 iRowid, int *piIndex ){ @@ -160162,6 +213025,7 @@ static int nodeRowidIndex( return SQLITE_OK; } } + RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -160171,11 +213035,12 @@ static int nodeRowidIndex( */ static int nodeParentIndex(Rtree *pRtree, RtreeNode *pNode, int *piIndex){ RtreeNode *pParent = pNode->pParent; - if( pParent ){ + if( ALWAYS(pParent) ){ return nodeRowidIndex(pRtree, pParent, pNode->iNode, piIndex); + }else{ + *piIndex = -1; + return SQLITE_OK; } - *piIndex = -1; - return SQLITE_OK; } /* @@ -160200,7 +213065,7 @@ static int rtreeSearchPointCompare( } /* -** Interchange to search points in a cursor. +** Interchange two search points in a cursor. */ static void rtreeSearchPointSwap(RtreeCursor *p, int i, int j){ RtreeSearchPoint t = p->aPoint[i]; @@ -160255,7 +213120,7 @@ static RtreeSearchPoint *rtreeEnqueue( RtreeSearchPoint *pNew; if( pCur->nPoint>=pCur->nPointAlloc ){ int nNew = pCur->nPointAlloc*2 + 8; - pNew = sqlite3_realloc(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); + pNew = sqlite3_realloc64(pCur->aPoint, nNew*sizeof(pCur->aPoint[0])); if( pNew==0 ) return 0; pCur->aPoint = pNew; pCur->nPointAlloc = nNew; @@ -160290,7 +213155,7 @@ static RtreeSearchPoint *rtreeSearchPointNew( pFirst = rtreeSearchPointFirst(pCur); pCur->anQueue[iLevel]++; if( pFirst==0 - || pFirst->rScore>rScore + || pFirst->rScore>rScore || (pFirst->rScore==rScore && pFirst->iLevel>iLevel) ){ if( pCur->bPoint ){ @@ -160298,10 +213163,11 @@ static RtreeSearchPoint *rtreeSearchPointNew( pNew = rtreeEnqueue(pCur, rScore, iLevel); if( pNew==0 ) return 0; ii = (int)(pNew - pCur->aPoint) + 1; - if( iiaNode[ii]==0 ); pCur->aNode[ii] = pCur->aNode[0]; - }else{ + }else{ nodeRelease(RTREE_OF_CURSOR(pCur), pCur->aNode[0]); } pCur->aNode[0] = 0; @@ -160359,7 +213225,7 @@ static void rtreeSearchPointPop(RtreeCursor *p){ if( p->bPoint ){ p->anQueue[p->sPoint.iLevel]--; p->bPoint = 0; - }else if( p->nPoint ){ + }else if( ALWAYS(p->nPoint) ){ p->anQueue[p->aPoint[0].iLevel]--; n = --p->nPoint; p->aPoint[0] = p->aPoint[n]; @@ -160410,13 +213276,14 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ eInt = pRtree->eCoordType==RTREE_COORD_INT32; while( (p = rtreeSearchPointFirst(pCur))!=0 && p->iLevel>0 ){ + u8 *pCellData; pNode = rtreeNodeOfFirstSearchPoint(pCur, &rc); if( rc ) return rc; nCell = NCELL(pNode); assert( nCell<200 ); + pCellData = pNode->zData + (4+pRtree->nBytesPerCell*p->iCell); while( p->iCellzData + (4+pRtree->nBytesPerCell*p->iCell); eWithin = FULLY_WITHIN; for(ii=0; iiaConstraint + ii; @@ -160429,13 +213296,23 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ }else{ rtreeNonleafConstraint(pConstraint, eInt, pCellData, &eWithin); } - if( eWithin==NOT_WITHIN ) break; + if( eWithin==NOT_WITHIN ){ + p->iCell++; + pCellData += pRtree->nBytesPerCell; + break; + } } - p->iCell++; if( eWithin==NOT_WITHIN ) continue; + p->iCell++; x.iLevel = p->iLevel - 1; if( x.iLevel ){ x.id = readInt64(pCellData); + for(ii=0; iinPoint; ii++){ + if( pCur->aPoint[ii].id==x.id ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + } x.iCell = 0; }else{ x.id = p->id; @@ -160448,7 +213325,7 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ if( rScoreeWithin = eWithin; + p->eWithin = (u8)eWithin; p->id = x.id; p->iCell = x.iCell; RTREE_QUEUE_TRACE(pCur, "PUSH-S:"); @@ -160463,7 +213340,7 @@ static int rtreeStepToLeaf(RtreeCursor *pCur){ return SQLITE_OK; } -/* +/* ** Rtree virtual table module xNext method. */ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ @@ -160472,12 +213349,16 @@ static int rtreeNext(sqlite3_vtab_cursor *pVtabCursor){ /* Move to the next entry that matches the configured constraints. */ RTREE_QUEUE_TRACE(pCsr, "POP-Nx:"); + if( pCsr->bAuxValid ){ + pCsr->bAuxValid = 0; + sqlite3_reset(pCsr->pReadAux); + } rtreeSearchPointPop(pCsr); rc = rtreeStepToLeaf(pCsr); return rc; } -/* +/* ** Rtree virtual table module xRowid method. */ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ @@ -160485,13 +213366,17 @@ static int rtreeRowid(sqlite3_vtab_cursor *pVtabCursor, sqlite_int64 *pRowid){ RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); int rc = SQLITE_OK; RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); - if( rc==SQLITE_OK && p ){ - *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + if( rc==SQLITE_OK && ALWAYS(p) ){ + if( p->iCell>=NCELL(pNode) ){ + rc = SQLITE_ABORT; + }else{ + *pRowid = nodeGetRowid(RTREE_OF_CURSOR(pCsr), pNode, p->iCell); + } } return rc; } -/* +/* ** Rtree virtual table module xColumn method. */ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ @@ -160503,11 +213388,11 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); if( rc ) return rc; - if( p==0 ) return SQLITE_OK; + if( NEVER(p==0) ) return SQLITE_OK; + if( p->iCell>=NCELL(pNode) ) return SQLITE_ABORT; if( i==0 ){ sqlite3_result_int64(ctx, nodeGetRowid(pRtree, pNode, p->iCell)); - }else{ - if( rc ) return rc; + }else if( i<=pRtree->nDim2 ){ nodeGetCoord(pRtree, pNode, p->iCell, i-1, &c); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ @@ -160518,12 +213403,32 @@ static int rtreeColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ assert( pRtree->eCoordType==RTREE_COORD_INT32 ); sqlite3_result_int(ctx, c.i); } + }else{ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, + sqlite3_column_value(pCsr->pReadAux, i - pRtree->nDim2 + 1)); } return SQLITE_OK; } -/* -** Use nodeAcquire() to obtain the leaf node containing the record with +/* +** Use nodeAcquire() to obtain the leaf node containing the record with ** rowid iRowid. If successful, set *ppLeaf to point to the node and ** return SQLITE_OK. If there is no such record in the table, set ** *ppLeaf to 0 and return SQLITE_OK. If an error occurs, set *ppLeaf @@ -160556,33 +213461,17 @@ static int findLeafNode( ** operator. */ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ - RtreeMatchArg *pBlob; /* BLOB returned by geometry function */ + RtreeMatchArg *pBlob, *pSrc; /* BLOB returned by geometry function */ sqlite3_rtree_query_info *pInfo; /* Callback information */ - int nBlob; /* Size of the geometry function blob */ - int nExpected; /* Expected size of the BLOB */ - /* Check that value is actually a blob. */ - if( sqlite3_value_type(pValue)!=SQLITE_BLOB ) return SQLITE_ERROR; - - /* Check that the blob is roughly the right size. */ - nBlob = sqlite3_value_bytes(pValue); - if( nBlob<(int)sizeof(RtreeMatchArg) ){ - return SQLITE_ERROR; - } - - pInfo = (sqlite3_rtree_query_info*)sqlite3_malloc( sizeof(*pInfo)+nBlob ); + pSrc = sqlite3_value_pointer(pValue, "RtreeMatchArg"); + if( pSrc==0 ) return SQLITE_ERROR; + pInfo = (sqlite3_rtree_query_info*) + sqlite3_malloc64( sizeof(*pInfo)+pSrc->iSize ); if( !pInfo ) return SQLITE_NOMEM; memset(pInfo, 0, sizeof(*pInfo)); pBlob = (RtreeMatchArg*)&pInfo[1]; - - memcpy(pBlob, sqlite3_value_blob(pValue), nBlob); - nExpected = (int)(sizeof(RtreeMatchArg) + - pBlob->nParam*sizeof(sqlite3_value*) + - (pBlob->nParam-1)*sizeof(RtreeDValue)); - if( pBlob->magic!=RTREE_GEOMETRY_MAGIC || nBlob!=nExpected ){ - sqlite3_free(pInfo); - return SQLITE_ERROR; - } + memcpy(pBlob, pSrc, pSrc->iSize); pInfo->pContext = pBlob->cb.pContext; pInfo->nParam = pBlob->nParam; pInfo->aParam = pBlob->aParam; @@ -160598,11 +213487,13 @@ static int deserializeGeometry(sqlite3_value *pValue, RtreeConstraint *pCons){ return SQLITE_OK; } -/* +SQLITE_PRIVATE int sqlite3IntFloatCompare(i64,double); + +/* ** Rtree virtual table module xFilter method. */ static int rtreeFilter( - sqlite3_vtab_cursor *pVtabCursor, + sqlite3_vtab_cursor *pVtabCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ @@ -160616,19 +213507,25 @@ static int rtreeFilter( rtreeReference(pRtree); /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ - freeCursorConstraints(pCsr); - sqlite3_free(pCsr->aPoint); - memset(pCsr, 0, sizeof(RtreeCursor)); - pCsr->base.pVtab = (sqlite3_vtab*)pRtree; + resetCursor(pCsr); pCsr->iStrategy = idxNum; if( idxNum==1 ){ /* Special case - lookup by rowid. */ RtreeNode *pLeaf; /* Leaf on which the required cell resides */ - RtreeSearchPoint *p; /* Search point for the the leaf */ + RtreeSearchPoint *p; /* Search point for the leaf */ i64 iRowid = sqlite3_value_int64(argv[0]); i64 iNode = 0; - rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + int eType = sqlite3_value_numeric_type(argv[0]); + if( eType==SQLITE_INTEGER + || (eType==SQLITE_FLOAT + && 0==sqlite3IntFloatCompare(iRowid,sqlite3_value_double(argv[0]))) + ){ + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + }else{ + rc = SQLITE_OK; + pLeaf = 0; + } if( rc==SQLITE_OK && pLeaf!=0 ){ p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); assert( p!=0 ); /* Always returns pCsr->sPoint */ @@ -160636,18 +213533,18 @@ static int rtreeFilter( p->id = iNode; p->eWithin = PARTLY_WITHIN; rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); - p->iCell = iCell; + p->iCell = (u8)iCell; RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); }else{ pCsr->atEOF = 1; } }else{ - /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array - ** with the configured constraints. + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. */ rc = nodeAcquire(pRtree, 1, 0, &pRoot); if( rc==SQLITE_OK && argc>0 ){ - pCsr->aConstraint = sqlite3_malloc(sizeof(RtreeConstraint)*argc); + pCsr->aConstraint = sqlite3_malloc64(sizeof(RtreeConstraint)*argc); pCsr->nConstraint = argc; if( !pCsr->aConstraint ){ rc = SQLITE_NOMEM; @@ -160658,6 +213555,7 @@ static int rtreeFilter( || (idxStr && (int)strlen(idxStr)==argc*2) ); for(ii=0; iiaConstraint[ii]; + int eType = sqlite3_value_numeric_type(argv[ii]); p->op = idxStr[ii*2]; p->iCoord = idxStr[ii*2+1]-'0'; if( p->op>=RTREE_MATCH ){ @@ -160669,23 +213567,48 @@ static int rtreeFilter( if( rc!=SQLITE_OK ){ break; } - p->pInfo->nCoord = pRtree->nDim*2; + p->pInfo->nCoord = pRtree->nDim2; p->pInfo->anQueue = pCsr->anQueue; p->pInfo->mxLevel = pRtree->iDepth + 1; - }else{ + }else if( eType==SQLITE_INTEGER ){ + sqlite3_int64 iVal = sqlite3_value_int64(argv[ii]); +#ifdef SQLITE_RTREE_INT_ONLY + p->u.rValue = iVal; +#else + p->u.rValue = (double)iVal; + if( iVal>=((sqlite3_int64)1)<<48 + || iVal<=-(((sqlite3_int64)1)<<48) + ){ + if( p->op==RTREE_LT ) p->op = RTREE_LE; + if( p->op==RTREE_GT ) p->op = RTREE_GE; + } +#endif + }else if( eType==SQLITE_FLOAT ){ #ifdef SQLITE_RTREE_INT_ONLY p->u.rValue = sqlite3_value_int64(argv[ii]); #else p->u.rValue = sqlite3_value_double(argv[ii]); #endif + }else{ + p->u.rValue = RTREE_ZERO; + if( eType==SQLITE_NULL ){ + p->op = RTREE_FALSE; + }else if( p->op==RTREE_LT || p->op==RTREE_LE ){ + p->op = RTREE_TRUE; + }else{ + p->op = RTREE_FALSE; + } } } } } if( rc==SQLITE_OK ){ RtreeSearchPoint *pNew; - pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, pRtree->iDepth+1); - if( pNew==0 ) return SQLITE_NOMEM; + assert( pCsr->bPoint==0 ); /* Due to the resetCursor() call above */ + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( NEVER(pNew==0) ){ /* Because pCsr->bPoint was FALSE */ + return SQLITE_NOMEM; + } pNew->id = 1; pNew->iCell = 0; pNew->eWithin = PARTLY_WITHIN; @@ -160702,22 +213625,9 @@ static int rtreeFilter( return rc; } -/* -** Set the pIdxInfo->estimatedRows variable to nRow. Unless this -** extension is currently being used by a version of SQLite too old to -** support estimatedRows. In that case this function is a no-op. -*/ -static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ -#if SQLITE_VERSION_NUMBER>=3008002 - if( sqlite3_libversion_number()>=3008002 ){ - pIdxInfo->estimatedRows = nRow; - } -#endif -} - /* ** Rtree virtual table module xBestIndex method. There are three -** table scan strategies to choose from (in order from most to +** table scan strategies to choose from (in order from most to ** least desirable): ** ** idxNum idxStr Strategy @@ -160727,8 +213637,8 @@ static void setEstimatedRows(sqlite3_index_info *pIdxInfo, i64 nRow){ ** ------------------------------------------------ ** ** If strategy 1 is used, then idxStr is not meaningful. If strategy -** 2 is used, idxStr is formatted to contain 2 bytes for each -** constraint used. The first two bytes of idxStr correspond to +** 2 is used, idxStr is formatted to contain 2 bytes for each +** constraint used. The first two bytes of idxStr correspond to ** the constraint in sqlite3_index_info.aConstraintUsage[] with ** (argvIndex==1) etc. ** @@ -160774,8 +213684,8 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ for(ii=0; iinConstraint && iIdx<(int)(sizeof(zIdxStr)-1); ii++){ struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; - if( bMatch==0 && p->usable - && p->iColumn==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ + if( bMatch==0 && p->usable + && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ /* We have an equality constraint on the rowid. Use strategy 1. */ int jj; @@ -160788,45 +213698,54 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ pIdxInfo->aConstraintUsage[jj].omit = 1; /* This strategy involves a two rowid lookups on an B-Tree structures - ** and then a linear search of an R-Tree node. This should be - ** considered almost as quick as a direct rowid lookup (for which + ** and then a linear search of an R-Tree node. This should be + ** considered almost as quick as a direct rowid lookup (for which ** sqlite uses an internal cost of 0.0). It is expected to return ** a single row. - */ + */ pIdxInfo->estimatedCost = 30.0; - setEstimatedRows(pIdxInfo, 1); + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; return SQLITE_OK; } - if( p->usable && (p->iColumn>0 || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) ){ + if( p->usable + && ((p->iColumn>0 && p->iColumn<=pRtree->nDim2) + || p->op==SQLITE_INDEX_CONSTRAINT_MATCH) + ){ u8 op; + u8 doOmit = 1; switch( p->op ){ - case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; break; - case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; break; - case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; - case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; break; - case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; - default: - assert( p->op==SQLITE_INDEX_CONSTRAINT_MATCH ); - op = RTREE_MATCH; - break; + case SQLITE_INDEX_CONSTRAINT_EQ: op = RTREE_EQ; doOmit = 0; break; + case SQLITE_INDEX_CONSTRAINT_GT: op = RTREE_GT; doOmit = 0; break; + case SQLITE_INDEX_CONSTRAINT_LE: op = RTREE_LE; break; + case SQLITE_INDEX_CONSTRAINT_LT: op = RTREE_LT; doOmit = 0; break; + case SQLITE_INDEX_CONSTRAINT_GE: op = RTREE_GE; break; + case SQLITE_INDEX_CONSTRAINT_MATCH: op = RTREE_MATCH; break; + default: op = 0; break; + } + if( op ){ + zIdxStr[iIdx++] = op; + zIdxStr[iIdx++] = (char)(p->iColumn - 1 + '0'); + pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); + pIdxInfo->aConstraintUsage[ii].omit = doOmit; } - zIdxStr[iIdx++] = op; - zIdxStr[iIdx++] = p->iColumn - 1 + '0'; - pIdxInfo->aConstraintUsage[ii].argvIndex = (iIdx/2); - pIdxInfo->aConstraintUsage[ii].omit = 1; } } pIdxInfo->idxNum = 2; pIdxInfo->needToFreeIdxStr = 1; - if( iIdx>0 && 0==(pIdxInfo->idxStr = sqlite3_mprintf("%s", zIdxStr)) ){ - return SQLITE_NOMEM; + if( iIdx>0 ){ + pIdxInfo->idxStr = sqlite3_malloc( iIdx+1 ); + if( pIdxInfo->idxStr==0 ){ + return SQLITE_NOMEM; + } + memcpy(pIdxInfo->idxStr, zIdxStr, iIdx+1); } nRow = pRtree->nRowEst >> (iIdx/2); pIdxInfo->estimatedCost = (double)6.0 * (double)nRow; - setEstimatedRows(pIdxInfo, nRow); + pIdxInfo->estimatedRows = nRow; return rc; } @@ -160836,9 +213755,26 @@ static int rtreeBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ */ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ RtreeDValue area = (RtreeDValue)1; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - area = (area * (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii]))); + assert( pRtree->nDim>=1 && pRtree->nDim<=5 ); +#ifndef SQLITE_RTREE_INT_ONLY + if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ + switch( pRtree->nDim ){ + case 5: area = p->aCoord[9].f - p->aCoord[8].f; + case 4: area *= p->aCoord[7].f - p->aCoord[6].f; + case 3: area *= p->aCoord[5].f - p->aCoord[4].f; + case 2: area *= p->aCoord[3].f - p->aCoord[2].f; + default: area *= p->aCoord[1].f - p->aCoord[0].f; + } + }else +#endif + { + switch( pRtree->nDim ){ + case 5: area = (i64)p->aCoord[9].i - (i64)p->aCoord[8].i; + case 4: area *= (i64)p->aCoord[7].i - (i64)p->aCoord[6].i; + case 3: area *= (i64)p->aCoord[5].i - (i64)p->aCoord[4].i; + case 2: area *= (i64)p->aCoord[3].i - (i64)p->aCoord[2].i; + default: area *= (i64)p->aCoord[1].i - (i64)p->aCoord[0].i; + } } return area; } @@ -160848,11 +213784,12 @@ static RtreeDValue cellArea(Rtree *pRtree, RtreeCell *p){ ** of the objects size in each dimension. */ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ - RtreeDValue margin = (RtreeDValue)0; - int ii; - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + RtreeDValue margin = 0; + int ii = pRtree->nDim2 - 2; + do{ margin += (DCOORD(p->aCoord[ii+1]) - DCOORD(p->aCoord[ii])); - } + ii -= 2; + }while( ii>=0 ); return margin; } @@ -160860,17 +213797,19 @@ static RtreeDValue cellMargin(Rtree *pRtree, RtreeCell *p){ ** Store the union of cells p1 and p2 in p1. */ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ - int ii; + int ii = 0; if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].f = MIN(p1->aCoord[ii].f, p2->aCoord[ii].f); p1->aCoord[ii+1].f = MAX(p1->aCoord[ii+1].f, p2->aCoord[ii+1].f); - } + ii += 2; + }while( iinDim2 ); }else{ - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ + do{ p1->aCoord[ii].i = MIN(p1->aCoord[ii].i, p2->aCoord[ii].i); p1->aCoord[ii+1].i = MAX(p1->aCoord[ii+1].i, p2->aCoord[ii+1].i); - } + ii += 2; + }while( iinDim2 ); } } @@ -160880,35 +213819,26 @@ static void cellUnion(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ */ static int cellContains(Rtree *pRtree, RtreeCell *p1, RtreeCell *p2){ int ii; - int isInt = (pRtree->eCoordType==RTREE_COORD_INT32); - for(ii=0; ii<(pRtree->nDim*2); ii+=2){ - RtreeCoord *a1 = &p1->aCoord[ii]; - RtreeCoord *a2 = &p2->aCoord[ii]; - if( (!isInt && (a2[0].fa1[1].f)) - || ( isInt && (a2[0].ia1[1].i)) - ){ - return 0; + if( pRtree->eCoordType==RTREE_COORD_INT32 ){ + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( a2[0].ia1[1].i ) return 0; + } + }else{ + for(ii=0; iinDim2; ii+=2){ + RtreeCoord *a1 = &p1->aCoord[ii]; + RtreeCoord *a2 = &p2->aCoord[ii]; + if( a2[0].fa1[1].f ) return 0; } } return 1; } -/* -** Return the amount cell p would grow by if it were unioned with pCell. -*/ -static RtreeDValue cellGrowth(Rtree *pRtree, RtreeCell *p, RtreeCell *pCell){ - RtreeDValue area; - RtreeCell cell; - memcpy(&cell, p, sizeof(RtreeCell)); - area = cellArea(pRtree, &cell); - cellUnion(pRtree, &cell, pCell); - return (cellArea(pRtree, &cell)-area); -} - static RtreeDValue cellOverlap( - Rtree *pRtree, - RtreeCell *p, - RtreeCell *aCell, + Rtree *pRtree, + RtreeCell *p, + RtreeCell *aCell, int nCell ){ int ii; @@ -160916,7 +213846,7 @@ static RtreeDValue cellOverlap( for(ii=0; iinDim*2); jj+=2){ + for(jj=0; jjnDim2; jj+=2){ RtreeDValue x1, x2; x1 = MAX(DCOORD(p->aCoord[jj]), DCOORD(aCell[ii].aCoord[jj])); x2 = MIN(DCOORD(p->aCoord[jj+1]), DCOORD(aCell[ii].aCoord[jj+1])); @@ -160945,44 +213875,58 @@ static int ChooseLeaf( ){ int rc; int ii; - RtreeNode *pNode; + RtreeNode *pNode = 0; rc = nodeAcquire(pRtree, 1, 0, &pNode); for(ii=0; rc==SQLITE_OK && ii<(pRtree->iDepth-iHeight); ii++){ int iCell; sqlite3_int64 iBest = 0; - + int bFound = 0; RtreeDValue fMinGrowth = RTREE_ZERO; RtreeDValue fMinArea = RTREE_ZERO; - int nCell = NCELL(pNode); - RtreeCell cell; - RtreeNode *pChild; + RtreeNode *pChild = 0; - RtreeCell *aCell = 0; - - /* Select the child node which will be enlarged the least if pCell - ** is inserted into it. Resolve ties by choosing the entry with - ** the smallest area. + /* First check to see if there is are any cells in pNode that completely + ** contains pCell. If two or more cells in pNode completely contain pCell + ** then pick the smallest. */ for(iCell=0; iCellpParent ){ RtreeNode *pParent = p->pParent; RtreeCell cell; int iCell; - if( nodeParentIndex(pRtree, p, &iCell) ){ + cnt++; + if( NEVER(cnt>100) ){ + RTREE_IS_CORRUPT(pRtree); + return SQLITE_CORRUPT_VTAB; + } + rc = nodeParentIndex(pRtree, p, &iCell); + if( NEVER(rc!=SQLITE_OK) ){ + RTREE_IS_CORRUPT(pRtree); return SQLITE_CORRUPT_VTAB; } @@ -161017,7 +213970,7 @@ static int AdjustTree( cellUnion(pRtree, &cell, pCell); nodeOverwriteCell(pRtree, pParent, &cell, iCell); } - + p = pParent; } return SQLITE_OK; @@ -161046,81 +213999,10 @@ static int parentWrite(Rtree *pRtree, sqlite3_int64 iNode, sqlite3_int64 iPar){ static int rtreeInsertCell(Rtree *, RtreeNode *, RtreeCell *, int); -/* -** Arguments aIdx, aDistance and aSpare all point to arrays of size -** nIdx. The aIdx array contains the set of integers from 0 to -** (nIdx-1) in no particular order. This function sorts the values -** in aIdx according to the indexed values in aDistance. For -** example, assuming the inputs: -** -** aIdx = { 0, 1, 2, 3 } -** aDistance = { 5.0, 2.0, 7.0, 6.0 } -** -** this function sets the aIdx array to contain: -** -** aIdx = { 0, 1, 2, 3 } -** -** The aSpare array is used as temporary working space by the -** sorting algorithm. -*/ -static void SortByDistance( - int *aIdx, - int nIdx, - RtreeDValue *aDistance, - int *aSpare -){ - if( nIdx>1 ){ - int iLeft = 0; - int iRight = 0; - - int nLeft = nIdx/2; - int nRight = nIdx-nLeft; - int *aLeft = aIdx; - int *aRight = &aIdx[nLeft]; - - SortByDistance(aLeft, nLeft, aDistance, aSpare); - SortByDistance(aRight, nRight, aDistance, aSpare); - - memcpy(aSpare, aLeft, sizeof(int)*nLeft); - aLeft = aSpare; - - while( iLeft1 ){ @@ -161205,9 +214087,9 @@ static int splitNodeStartree( int iBestSplit = 0; RtreeDValue fBestMargin = RTREE_ZERO; - int nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); + sqlite3_int64 nByte = (pRtree->nDim+1)*(sizeof(int*)+nCell*sizeof(int)); - aaSorted = (int **)sqlite3_malloc(nByte); + aaSorted = (int **)sqlite3_malloc64(nByte); if( !aaSorted ){ return SQLITE_NOMEM; } @@ -161231,8 +214113,8 @@ static int splitNodeStartree( int nLeft; for( - nLeft=RTREE_MINCELLS(pRtree); - nLeft<=(nCell-RTREE_MINCELLS(pRtree)); + nLeft=RTREE_MINCELLS(pRtree); + nLeft<=(nCell-RTREE_MINCELLS(pRtree)); nLeft++ ){ RtreeCell left; @@ -161287,21 +214169,26 @@ static int splitNodeStartree( static int updateMapping( - Rtree *pRtree, - i64 iRowid, - RtreeNode *pNode, + Rtree *pRtree, + i64 iRowid, + RtreeNode *pNode, int iHeight ){ int (*xSetMapping)(Rtree *, sqlite3_int64, sqlite3_int64); xSetMapping = ((iHeight==0)?rowidWrite:parentWrite); if( iHeight>0 ){ RtreeNode *pChild = nodeHashLookup(pRtree, iRowid); + RtreeNode *p; + for(p=pNode; p; p=p->pParent){ + if( p==pChild ) return SQLITE_CORRUPT_VTAB; + } if( pChild ){ nodeRelease(pRtree, pChild->pParent); nodeReference(pNode); pChild->pParent = pNode; } } + if( NEVER(pNode==0) ) return SQLITE_ERROR; return xSetMapping(pRtree, iRowid, pNode->iNode); } @@ -161325,10 +214212,10 @@ static int SplitNode( RtreeCell leftbbox; RtreeCell rightbbox; - /* Allocate an array and populate it with a copy of pCell and + /* Allocate an array and populate it with a copy of pCell and ** all cells from node pLeft. Then zero the original node. */ - aCell = sqlite3_malloc((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); + aCell = sqlite3_malloc64((sizeof(RtreeCell)+sizeof(int))*(nCell+1)); if( !aCell ){ rc = SQLITE_NOMEM; goto splitnode_out; @@ -161351,7 +214238,7 @@ static int SplitNode( }else{ pLeft = pNode; pRight = nodeNew(pRtree, pLeft->pParent); - nodeReference(pLeft); + pLeft->nRef++; } if( !pLeft || !pRight ){ @@ -161391,11 +214278,12 @@ static int SplitNode( RtreeNode *pParent = pLeft->pParent; int iCell; rc = nodeParentIndex(pRtree, pLeft, &iCell); - if( rc==SQLITE_OK ){ + if( ALWAYS(rc==SQLITE_OK) ){ nodeOverwriteCell(pRtree, pParent, &leftbbox, iCell); rc = AdjustTree(pRtree, pParent, &leftbbox); + assert( rc==SQLITE_OK ); } - if( rc!=SQLITE_OK ){ + if( NEVER(rc!=SQLITE_OK) ){ goto splitnode_out; } } @@ -161442,14 +214330,14 @@ splitnode_out: } /* -** If node pLeaf is not the root of the r-tree and its pParent pointer is +** If node pLeaf is not the root of the r-tree and its pParent pointer is ** still NULL, load all ancestor nodes of pLeaf into memory and populate ** the pLeaf->pParent chain all the way up to the root node. ** ** This operation is required when a row is deleted (or updated - an update ** is implemented as a delete followed by an insert). SQLite provides the ** rowid of the row to delete, which can be used to find the leaf on which -** the entry resides (argument pLeaf). Once the leaf is located, this +** the entry resides (argument pLeaf). Once the leaf is located, this ** function is called to determine its ancestry. */ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ @@ -161470,13 +214358,16 @@ static int fixLeafParent(Rtree *pRtree, RtreeNode *pLeaf){ */ iNode = sqlite3_column_int64(pRtree->pReadParent, 0); for(pTest=pLeaf; pTest && pTest->iNode!=iNode; pTest=pTest->pParent); - if( !pTest ){ + if( pTest==0 ){ rc2 = nodeAcquire(pRtree, iNode, 0, &pChild->pParent); } } rc = sqlite3_reset(pRtree->pReadParent); if( rc==SQLITE_OK ) rc = rc2; - if( rc==SQLITE_OK && !pChild->pParent ) rc = SQLITE_CORRUPT_VTAB; + if( rc==SQLITE_OK && !pChild->pParent ){ + RTREE_IS_CORRUPT(pRtree); + rc = SQLITE_CORRUPT_VTAB; + } pChild = pChild->pParent; } return rc; @@ -161498,6 +214389,7 @@ static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ pParent = pNode->pParent; pNode->pParent = 0; rc = deleteCell(pRtree, pParent, iCell, iHeight+1); + testcase( rc!=SQLITE_OK ); } rc2 = nodeRelease(pRtree, pParent); if( rc==SQLITE_OK ){ @@ -161520,7 +214412,7 @@ static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ if( SQLITE_OK!=(rc = sqlite3_reset(pRtree->pDeleteParent)) ){ return rc; } - + /* Remove the node from the in-memory hash table and link it into ** the Rtree.pDeleted list. Its contents will be re-inserted later on. */ @@ -161535,9 +214427,9 @@ static int removeNode(Rtree *pRtree, RtreeNode *pNode, int iHeight){ static int fixBoundingBox(Rtree *pRtree, RtreeNode *pNode){ RtreeNode *pParent = pNode->pParent; - int rc = SQLITE_OK; + int rc = SQLITE_OK; if( pParent ){ - int ii; + int ii; int nCell = NCELL(pNode); RtreeCell box; /* Bounding box for pNode */ nodeGetCell(pRtree, pNode, 0, &box); @@ -161591,109 +214483,8 @@ static int deleteCell(Rtree *pRtree, RtreeNode *pNode, int iCell, int iHeight){ return rc; } -static int Reinsert( - Rtree *pRtree, - RtreeNode *pNode, - RtreeCell *pCell, - int iHeight -){ - int *aOrder; - int *aSpare; - RtreeCell *aCell; - RtreeDValue *aDistance; - int nCell; - RtreeDValue aCenterCoord[RTREE_MAX_DIMENSIONS]; - int iDim; - int ii; - int rc = SQLITE_OK; - int n; - - memset(aCenterCoord, 0, sizeof(RtreeDValue)*RTREE_MAX_DIMENSIONS); - - nCell = NCELL(pNode)+1; - n = (nCell+1)&(~1); - - /* Allocate the buffers used by this operation. The allocation is - ** relinquished before this function returns. - */ - aCell = (RtreeCell *)sqlite3_malloc(n * ( - sizeof(RtreeCell) + /* aCell array */ - sizeof(int) + /* aOrder array */ - sizeof(int) + /* aSpare array */ - sizeof(RtreeDValue) /* aDistance array */ - )); - if( !aCell ){ - return SQLITE_NOMEM; - } - aOrder = (int *)&aCell[n]; - aSpare = (int *)&aOrder[n]; - aDistance = (RtreeDValue *)&aSpare[n]; - - for(ii=0; iinDim; iDim++){ - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2]); - aCenterCoord[iDim] += DCOORD(aCell[ii].aCoord[iDim*2+1]); - } - } - for(iDim=0; iDimnDim; iDim++){ - aCenterCoord[iDim] = (aCenterCoord[iDim]/(nCell*(RtreeDValue)2)); - } - - for(ii=0; iinDim; iDim++){ - RtreeDValue coord = (DCOORD(aCell[ii].aCoord[iDim*2+1]) - - DCOORD(aCell[ii].aCoord[iDim*2])); - aDistance[ii] += (coord-aCenterCoord[iDim])*(coord-aCenterCoord[iDim]); - } - } - - SortByDistance(aOrder, nCell, aDistance, aSpare); - nodeZero(pRtree, pNode); - - for(ii=0; rc==SQLITE_OK && ii<(nCell-(RTREE_MINCELLS(pRtree)+1)); ii++){ - RtreeCell *p = &aCell[aOrder[ii]]; - nodeInsertCell(pRtree, pNode, p); - if( p->iRowid==pCell->iRowid ){ - if( iHeight==0 ){ - rc = rowidWrite(pRtree, p->iRowid, pNode->iNode); - }else{ - rc = parentWrite(pRtree, p->iRowid, pNode->iNode); - } - } - } - if( rc==SQLITE_OK ){ - rc = fixBoundingBox(pRtree, pNode); - } - for(; rc==SQLITE_OK && iiiNode currently contains - ** the height of the sub-tree headed by the cell. - */ - RtreeNode *pInsert; - RtreeCell *p = &aCell[aOrder[ii]]; - rc = ChooseLeaf(pRtree, p, iHeight, &pInsert); - if( rc==SQLITE_OK ){ - int rc2; - rc = rtreeInsertCell(pRtree, pInsert, p, iHeight); - rc2 = nodeRelease(pRtree, pInsert); - if( rc==SQLITE_OK ){ - rc = rc2; - } - } - } - - sqlite3_free(aCell); - return rc; -} - /* -** Insert cell pCell into node pNode. Node pNode is the head of a +** Insert cell pCell into node pNode. Node pNode is the head of a ** subtree iHeight high (leaf nodes have iHeight==0). */ static int rtreeInsertCell( @@ -161712,15 +214503,10 @@ static int rtreeInsertCell( } } if( nodeInsertCell(pRtree, pNode, pCell) ){ - if( iHeight<=pRtree->iReinsertHeight || pNode->iNode==1){ - rc = SplitNode(pRtree, pNode, pCell, iHeight); - }else{ - pRtree->iReinsertHeight = iHeight; - rc = Reinsert(pRtree, pNode, pCell, iHeight); - } + rc = SplitNode(pRtree, pNode, pCell, iHeight); }else{ rc = AdjustTree(pRtree, pNode, pCell); - if( rc==SQLITE_OK ){ + if( ALWAYS(rc==SQLITE_OK) ){ if( iHeight==0 ){ rc = rowidWrite(pRtree, pCell->iRowid, pNode->iNode); }else{ @@ -161760,7 +214546,7 @@ static int reinsertNodeContent(Rtree *pRtree, RtreeNode *pNode){ /* ** Select a currently unused rowid for a new r-tree record. */ -static int newRowid(Rtree *pRtree, i64 *piRowid){ +static int rtreeNewRowid(Rtree *pRtree, i64 *piRowid){ int rc; sqlite3_bind_null(pRtree->pWriteRowid, 1); sqlite3_bind_null(pRtree->pWriteRowid, 2); @@ -161777,21 +214563,25 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ int rc; /* Return code */ RtreeNode *pLeaf = 0; /* Leaf node containing record iDelete */ int iCell; /* Index of iDelete cell in pLeaf */ - RtreeNode *pRoot; /* Root node of rtree structure */ + RtreeNode *pRoot = 0; /* Root node of rtree structure */ /* Obtain a reference to the root node to initialize Rtree.iDepth */ rc = nodeAcquire(pRtree, 1, 0, &pRoot); - /* Obtain a reference to the leaf node that contains the entry - ** about to be deleted. + /* Obtain a reference to the leaf node that contains the entry + ** about to be deleted. */ if( rc==SQLITE_OK ){ rc = findLeafNode(pRtree, iDelete, &pLeaf, 0); } +#ifdef CORRUPT_DB + assert( pLeaf!=0 || rc!=SQLITE_OK || CORRUPT_DB ); +#endif + /* Delete the cell in question from the leaf node. */ - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && pLeaf ){ int rc2; rc = nodeRowidIndex(pRtree, pLeaf, iDelete, &iCell); if( rc==SQLITE_OK ){ @@ -161811,18 +214601,18 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ } /* Check if the root node now has exactly one child. If so, remove - ** it, schedule the contents of the child for reinsertion and + ** it, schedule the contents of the child for reinsertion and ** reduce the tree height by one. ** ** This is equivalent to copying the contents of the child into - ** the root node (the operation that Gutman's paper says to perform + ** the root node (the operation that Gutman's paper says to perform ** in this scenario). */ if( rc==SQLITE_OK && pRtree->iDepth>0 && NCELL(pRoot)==1 ){ int rc2; - RtreeNode *pChild; + RtreeNode *pChild = 0; i64 iChild = nodeGetRowid(pRtree, pRoot, 0); - rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); + rc = nodeAcquire(pRtree, iChild, pRoot, &pChild); /* tag-20210916a */ if( rc==SQLITE_OK ){ rc = removeNode(pRtree, pChild, pRtree->iDepth-1); } @@ -161841,6 +214631,7 @@ static int rtreeDeleteRowid(Rtree *pRtree, sqlite3_int64 iDelete){ rc = reinsertNodeContent(pRtree, pLeaf); } pRtree->pDeleted = pLeaf->pNext; + pRtree->nNodeRef--; sqlite3_free(pLeaf); } @@ -161883,14 +214674,61 @@ static RtreeValue rtreeValueUp(sqlite3_value *v){ } #endif /* !defined(SQLITE_RTREE_INT_ONLY) */ +/* +** A constraint has failed while inserting a row into an rtree table. +** Assuming no OOM error occurs, this function sets the error message +** (at pRtree->base.zErrMsg) to an appropriate value and returns +** SQLITE_CONSTRAINT. +** +** Parameter iCol is the index of the leftmost column involved in the +** constraint failure. If it is 0, then the constraint that failed is +** the unique constraint on the id column. Otherwise, it is the rtree +** (c1<=c2) constraint on columns iCol and iCol+1 that has failed. +** +** If an OOM occurs, SQLITE_NOMEM is returned instead of SQLITE_CONSTRAINT. +*/ +static int rtreeConstraintError(Rtree *pRtree, int iCol){ + sqlite3_stmt *pStmt = 0; + char *zSql; + int rc; + + assert( iCol==0 || iCol%2 ); + zSql = sqlite3_mprintf("SELECT * FROM %Q.%Q", pRtree->zDb, pRtree->zName); + if( zSql ){ + rc = sqlite3_prepare_v2(pRtree->db, zSql, -1, &pStmt, 0); + }else{ + rc = SQLITE_NOMEM; + } + sqlite3_free(zSql); + + if( rc==SQLITE_OK ){ + if( iCol==0 ){ + const char *zCol = sqlite3_column_name(pStmt, 0); + pRtree->base.zErrMsg = sqlite3_mprintf( + "UNIQUE constraint failed: %s.%s", pRtree->zName, zCol + ); + }else{ + const char *zCol1 = sqlite3_column_name(pStmt, iCol); + const char *zCol2 = sqlite3_column_name(pStmt, iCol+1); + pRtree->base.zErrMsg = sqlite3_mprintf( + "rtree constraint failed: %s.(%s<=%s)", pRtree->zName, zCol1, zCol2 + ); + } + } + + sqlite3_finalize(pStmt); + return (rc==SQLITE_OK ? SQLITE_CONSTRAINT : rc); +} + + /* ** The xUpdate method for rtree module virtual tables. */ static int rtreeUpdate( - sqlite3_vtab *pVtab, - int nData, - sqlite3_value **azData, + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, sqlite_int64 *pRowid ){ Rtree *pRtree = (Rtree *)pVtab; @@ -161898,10 +214736,16 @@ static int rtreeUpdate( RtreeCell cell; /* New cell to insert if nData>1 */ int bHaveRowid = 0; /* Set to 1 after new rowid is determined */ + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } rtreeReference(pRtree); assert(nData>=1); - cell.iRowid = 0; /* Used only to suppress a compiler warning */ + memset(&cell, 0, sizeof(cell)); /* Constraint handling. A write operation on an r-tree table may return ** SQLITE_CONSTRAINT for two reasons: @@ -161916,8 +214760,10 @@ static int rtreeUpdate( */ if( nData>1 ){ int ii; + int nn = nData - 4; - /* Populate the cell.aCoord[] array. The first coordinate is azData[3]. + if( nn > pRtree->nDim2 ) nn = pRtree->nDim2; + /* Populate the cell.aCoord[] array. The first coordinate is aData[3]. ** ** NB: nData can only be less than nDim*2+3 if the rtree is mis-declared ** with "column" that are interpreted as table constraints. @@ -161925,37 +214771,36 @@ static int rtreeUpdate( ** This problem was discovered after years of use, so we silently ignore ** these kinds of misdeclared tables to avoid breaking any legacy. */ - assert( nData<=(pRtree->nDim*2 + 3) ); #ifndef SQLITE_RTREE_INT_ONLY if( pRtree->eCoordType==RTREE_COORD_REAL32 ){ - for(ii=0; iicell.aCoord[ii+1].f ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } }else #endif { - for(ii=0; iicell.aCoord[ii+1].i ){ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, ii+1); goto constraint; } } } - /* If a rowid value was supplied, check if it is already present in + /* If a rowid value was supplied, check if it is already present in ** the table. If so, the constraint has failed. */ - if( sqlite3_value_type(azData[2])!=SQLITE_NULL ){ - cell.iRowid = sqlite3_value_int64(azData[2]); - if( sqlite3_value_type(azData[0])==SQLITE_NULL - || sqlite3_value_int64(azData[0])!=cell.iRowid + if( sqlite3_value_type(aData[2])!=SQLITE_NULL ){ + cell.iRowid = sqlite3_value_int64(aData[2]); + if( sqlite3_value_type(aData[0])==SQLITE_NULL + || sqlite3_value_int64(aData[0])!=cell.iRowid ){ int steprc; sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); @@ -161965,7 +214810,7 @@ static int rtreeUpdate( if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ rc = rtreeDeleteRowid(pRtree, cell.iRowid); }else{ - rc = SQLITE_CONSTRAINT; + rc = rtreeConstraintError(pRtree, 0); goto constraint; } } @@ -161974,16 +214819,16 @@ static int rtreeUpdate( } } - /* If azData[0] is not an SQL NULL value, it is the rowid of a + /* If aData[0] is not an SQL NULL value, it is the rowid of a ** record to delete from the r-tree table. The following block does ** just that. */ - if( sqlite3_value_type(azData[0])!=SQLITE_NULL ){ - rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(azData[0])); + if( sqlite3_value_type(aData[0])!=SQLITE_NULL ){ + rc = rtreeDeleteRowid(pRtree, sqlite3_value_int64(aData[0])); } - /* If the azData[] array contains more than one element, elements - ** (azData[2]..azData[argc-1]) contain a new record to insert into + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into ** the r-tree structure. */ if( rc==SQLITE_OK && nData>1 ){ @@ -161992,7 +214837,7 @@ static int rtreeUpdate( /* Figure out the rowid of the new row. */ if( bHaveRowid==0 ){ - rc = newRowid(pRtree, &cell.iRowid); + rc = rtreeNewRowid(pRtree, &cell.iRowid); } *pRowid = cell.iRowid; @@ -162001,13 +214846,22 @@ static int rtreeUpdate( } if( rc==SQLITE_OK ){ int rc2; - pRtree->iReinsertHeight = -1; rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); rc2 = nodeRelease(pRtree, pLeaf); if( rc==SQLITE_OK ){ rc = rc2; } } + if( rc==SQLITE_OK && pRtree->nAux ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + sqlite3_bind_int64(pUp, 1, *pRowid); + for(jj=0; jjnAux; jj++){ + sqlite3_bind_value(pUp, jj+2, aData[pRtree->nDim2+3+jj]); + } + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); + } } constraint: @@ -162015,6 +214869,30 @@ constraint: return rc; } +/* +** Called when a transaction starts. +*/ +static int rtreeBeginTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + assert( pRtree->inWrTrans==0 ); + pRtree->inWrTrans = 1; + return SQLITE_OK; +} + +/* +** Called when a transaction completes (either by COMMIT or ROLLBACK). +** The sqlite3_blob object should be released at this point. +*/ +static int rtreeEndTransaction(sqlite3_vtab *pVtab){ + Rtree *pRtree = (Rtree *)pVtab; + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + return SQLITE_OK; +} +static int rtreeRollback(sqlite3_vtab *pVtab){ + return rtreeEndTransaction(pVtab); +} + /* ** The xRename method for rtree module virtual tables. */ @@ -162025,17 +214903,42 @@ static int rtreeRename(sqlite3_vtab *pVtab, const char *zNewName){ "ALTER TABLE %Q.'%q_node' RENAME TO \"%w_node\";" "ALTER TABLE %Q.'%q_parent' RENAME TO \"%w_parent\";" "ALTER TABLE %Q.'%q_rowid' RENAME TO \"%w_rowid\";" - , pRtree->zDb, pRtree->zName, zNewName - , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName + , pRtree->zDb, pRtree->zName, zNewName , pRtree->zDb, pRtree->zName, zNewName ); if( zSql ){ + nodeBlobReset(pRtree); rc = sqlite3_exec(pRtree->db, zSql, 0, 0, 0); sqlite3_free(zSql); } return rc; } +/* +** The xSavepoint method. +** +** This module does not need to do anything to support savepoints. However, +** it uses this hook to close any open blob handle. This is done because a +** DROP TABLE command - which fortunately always opens a savepoint - cannot +** succeed if there are any open blob handles. i.e. if the blob handle were +** not closed here, the following would fail: +** +** BEGIN; +** INSERT INTO rtree... +** DROP TABLE ; -- Would fail with SQLITE_LOCKED +** COMMIT; +*/ +static int rtreeSavepoint(sqlite3_vtab *pVtab, int iSavepoint){ + Rtree *pRtree = (Rtree *)pVtab; + u8 iwt = pRtree->inWrTrans; + UNUSED_PARAMETER(iSavepoint); + pRtree->inWrTrans = 0; + nodeBlobReset(pRtree); + pRtree->inWrTrans = iwt; + return SQLITE_OK; +} + /* ** This function populates the pRtree->nRowEst variable with an estimate ** of the number of rows in the virtual table. If possible, this is based @@ -162046,8 +214949,15 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ char *zSql; sqlite3_stmt *p; int rc; - i64 nRow = 0; + i64 nRow = RTREE_MIN_ROWEST; + rc = sqlite3_table_column_metadata( + db, pRtree->zDb, "sqlite_stat1",0,0,0,0,0,0 + ); + if( rc!=SQLITE_OK ){ + pRtree->nRowEst = RTREE_DEFAULT_ROWEST; + return rc==SQLITE_ERROR ? SQLITE_OK : rc; + } zSql = sqlite3_mprintf(zFmt, pRtree->zDb, pRtree->zName); if( zSql==0 ){ rc = SQLITE_NOMEM; @@ -162056,25 +214966,34 @@ static int rtreeQueryStat1(sqlite3 *db, Rtree *pRtree){ if( rc==SQLITE_OK ){ if( sqlite3_step(p)==SQLITE_ROW ) nRow = sqlite3_column_int64(p, 0); rc = sqlite3_finalize(p); - }else if( rc!=SQLITE_NOMEM ){ - rc = SQLITE_OK; - } - - if( rc==SQLITE_OK ){ - if( nRow==0 ){ - pRtree->nRowEst = RTREE_DEFAULT_ROWEST; - }else{ - pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); - } } sqlite3_free(zSql); } - + pRtree->nRowEst = MAX(nRow, RTREE_MIN_ROWEST); return rc; } + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int rtreeShadowName(const char *zName){ + static const char *azName[] = { + "node", "parent", "rowid" + }; + unsigned int i; + for(i=0; idb = db; if( isCreate ){ - char *zCreate = sqlite3_mprintf( -"CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY, data BLOB);" -"CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY, nodeno INTEGER);" -"CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY," - " parentnode INTEGER);" -"INSERT INTO '%q'.'%q_node' VALUES(1, zeroblob(%d))", - zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, zDb, zPrefix, pRtree->iNodeSize - ); + char *zCreate; + sqlite3_str *p = sqlite3_str_new(db); + int ii; + sqlite3_str_appendf(p, + "CREATE TABLE \"%w\".\"%w_rowid\"(rowid INTEGER PRIMARY KEY,nodeno", + zDb, zPrefix); + for(ii=0; iinAux; ii++){ + sqlite3_str_appendf(p,",a%d",ii); + } + sqlite3_str_appendf(p, + ");CREATE TABLE \"%w\".\"%w_node\"(nodeno INTEGER PRIMARY KEY,data);", + zDb, zPrefix); + sqlite3_str_appendf(p, + "CREATE TABLE \"%w\".\"%w_parent\"(nodeno INTEGER PRIMARY KEY,parentnode);", + zDb, zPrefix); + sqlite3_str_appendf(p, + "INSERT INTO \"%w\".\"%w_node\"VALUES(1,zeroblob(%d))", + zDb, zPrefix, pRtree->iNodeSize); + zCreate = sqlite3_str_finish(p); if( !zCreate ){ return SQLITE_NOMEM; } @@ -162149,26 +215081,67 @@ static int rtreeSqlInit( } } - appStmt[0] = &pRtree->pReadNode; - appStmt[1] = &pRtree->pWriteNode; - appStmt[2] = &pRtree->pDeleteNode; - appStmt[3] = &pRtree->pReadRowid; - appStmt[4] = &pRtree->pWriteRowid; - appStmt[5] = &pRtree->pDeleteRowid; - appStmt[6] = &pRtree->pReadParent; - appStmt[7] = &pRtree->pWriteParent; - appStmt[8] = &pRtree->pDeleteParent; + appStmt[0] = &pRtree->pWriteNode; + appStmt[1] = &pRtree->pDeleteNode; + appStmt[2] = &pRtree->pReadRowid; + appStmt[3] = &pRtree->pWriteRowid; + appStmt[4] = &pRtree->pDeleteRowid; + appStmt[5] = &pRtree->pReadParent; + appStmt[6] = &pRtree->pWriteParent; + appStmt[7] = &pRtree->pDeleteParent; rc = rtreeQueryStat1(db, pRtree); for(i=0; inAux==0 ){ + zFormat = azSql[i]; + }else { + /* An UPSERT is very slightly slower than REPLACE, but it is needed + ** if there are auxiliary columns */ + zFormat = "INSERT INTO\"%w\".\"%w_rowid\"(rowid,nodeno)VALUES(?1,?2)" + "ON CONFLICT(rowid)DO UPDATE SET nodeno=excluded.nodeno"; + } + zSql = sqlite3_mprintf(zFormat, zDb, zPrefix); if( zSql ){ - rc = sqlite3_prepare_v2(db, zSql, -1, appStmt[i], 0); + rc = sqlite3_prepare_v3(db, zSql, -1, f, appStmt[i], 0); }else{ rc = SQLITE_NOMEM; } sqlite3_free(zSql); } + if( pRtree->nAux && rc!=SQLITE_NOMEM ){ + pRtree->zReadAuxSql = sqlite3_mprintf( + "SELECT * FROM \"%w\".\"%w_rowid\" WHERE rowid=?1", + zDb, zPrefix); + if( pRtree->zReadAuxSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + sqlite3_str *p = sqlite3_str_new(db); + int ii; + char *zSql; + sqlite3_str_appendf(p, "UPDATE \"%w\".\"%w_rowid\"SET ", zDb, zPrefix); + for(ii=0; iinAux; ii++){ + if( ii ) sqlite3_str_append(p, ",", 1); +#ifdef SQLITE_ENABLE_GEOPOLY + if( iinAuxNotNull ){ + sqlite3_str_appendf(p,"a%d=coalesce(?%d,a%d)",ii,ii+2,ii); + }else +#endif + { + sqlite3_str_appendf(p,"a%d=?%d",ii,ii+2); + } + } + sqlite3_str_appendf(p, " WHERE rowid=?1"); + zSql = sqlite3_str_finish(p); + if( zSql==0 ){ + rc = SQLITE_NOMEM; + }else{ + rc = sqlite3_prepare_v3(db, zSql, -1, f, &pRtree->pWriteAux, 0); + sqlite3_free(zSql); + } + } + } return rc; } @@ -162205,9 +215178,9 @@ static int getIntFromStmt(sqlite3 *db, const char *zSql, int *piVal){ ** table already exists. In this case the node-size is determined by inspecting ** the root node of the tree. ** -** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. -** This ensures that each node is stored on a single database page. If the -** database page-size is so large that more than RTREE_MAXCELLS entries +** Otherwise, for an xCreate(), use 64 bytes less than the database page-size. +** This ensures that each node is stored on a single database page. If the +** database page-size is so large that more than RTREE_MAXCELLS entries ** would fit in a single node, use a smaller node-size. */ static int getNodeSize( @@ -162238,6 +215211,11 @@ static int getNodeSize( rc = getIntFromStmt(db, zSql, &pRtree->iNodeSize); if( rc!=SQLITE_OK ){ *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + }else if( pRtree->iNodeSize<(512-64) ){ + rc = SQLITE_CORRUPT_VTAB; + RTREE_IS_CORRUPT(pRtree); + *pzErr = sqlite3_mprintf("undersize RTree blobs in \"%q_node\"", + pRtree->zName); } } @@ -162245,7 +215223,15 @@ static int getNodeSize( return rc; } -/* +/* +** Return the length of a token +*/ +static int rtreeTokenLength(const char *z){ + int dummy = 0; + return sqlite3GetToken((const unsigned char*)z,&dummy); +} + +/* ** This function is the implementation of both the xConnect and xCreate ** methods of the r-tree virtual table. ** @@ -162267,80 +215253,115 @@ static int rtreeInit( int nDb; /* Length of string argv[1] */ int nName; /* Length of string argv[2] */ int eCoordType = (pAux ? RTREE_COORD_INT32 : RTREE_COORD_REAL32); + sqlite3_str *pSql; + char *zSql; + int ii = 4; + int iErr; const char *aErrMsg[] = { 0, /* 0 */ "Wrong number of columns for an rtree table", /* 1 */ "Too few columns for an rtree table", /* 2 */ - "Too many columns for an rtree table" /* 3 */ + "Too many columns for an rtree table", /* 3 */ + "Auxiliary rtree columns must be last" /* 4 */ }; - int iErr = (argc<6) ? 2 : argc>(RTREE_MAX_DIMENSIONS*2+4) ? 3 : argc%2; - if( aErrMsg[iErr] ){ - *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + assert( RTREE_MAX_AUX_COLUMN<256 ); /* Aux columns counted by a u8 */ + if( argc<6 || argc>RTREE_MAX_AUX_COLUMN+3 ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[2 + (argc>=6)]); return SQLITE_ERROR; } sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + /* Allocate the sqlite3_vtab structure */ nDb = (int)strlen(argv[1]); nName = (int)strlen(argv[2]); - pRtree = (Rtree *)sqlite3_malloc(sizeof(Rtree)+nDb+nName+2); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8); if( !pRtree ){ return SQLITE_NOMEM; } - memset(pRtree, 0, sizeof(Rtree)+nDb+nName+2); + memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8); pRtree->nBusy = 1; pRtree->base.pModule = &rtreeModule; pRtree->zDb = (char *)&pRtree[1]; pRtree->zName = &pRtree->zDb[nDb+1]; - pRtree->nDim = (argc-4)/2; - pRtree->nBytesPerCell = 8 + pRtree->nDim*4*2; - pRtree->eCoordType = eCoordType; + pRtree->zNodeName = &pRtree->zName[nName+1]; + pRtree->eCoordType = (u8)eCoordType; memcpy(pRtree->zDb, argv[1], nDb); memcpy(pRtree->zName, argv[2], nName); + memcpy(pRtree->zNodeName, argv[2], nName); + memcpy(&pRtree->zNodeName[nName], "_node", 6); - /* Figure out the node size to use. */ - rc = getNodeSize(db, pRtree, isCreate, pzErr); /* Create/Connect to the underlying relational database schema. If ** that is successful, call sqlite3_declare_vtab() to configure ** the r-tree table schema. */ - if( rc==SQLITE_OK ){ - if( (rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate)) ){ - *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(%.*s INT", + rtreeTokenLength(argv[3]), argv[3]); + for(ii=4; iinAux++; + sqlite3_str_appendf(pSql, ",%.*s", rtreeTokenLength(zArg+1), zArg+1); + }else if( pRtree->nAux>0 ){ + break; }else{ - char *zSql = sqlite3_mprintf("CREATE TABLE x(%s", argv[3]); - char *zTmp; - int ii; - for(ii=4; zSql && iinDim2++; + sqlite3_str_appendf(pSql, azFormat[eCoordType], + rtreeTokenLength(zArg), zArg); } } - - if( rc==SQLITE_OK ){ - *ppVtab = (sqlite3_vtab *)pRtree; - }else{ - assert( *ppVtab==0 ); - assert( pRtree->nBusy==1 ); - rtreeRelease(pRtree); + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( iinDim = pRtree->nDim2/2; + if( pRtree->nDim<1 ){ + iErr = 2; + }else if( pRtree->nDim2>RTREE_MAX_DIMENSIONS*2 ){ + iErr = 3; + }else if( pRtree->nDim2 % 2 ){ + iErr = 1; + }else{ + iErr = 0; + } + if( iErr ){ + *pzErr = sqlite3_mprintf("%s", aErrMsg[iErr]); + goto rtreeInit_fail; + } + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto rtreeInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto rtreeInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +rtreeInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); return rc; } @@ -162358,52 +215379,50 @@ static int rtreeInit( ** ** The human readable string takes the form of a Tcl list with one ** entry for each cell in the r-tree node. Each entry is itself a -** list, containing the 8-byte rowid/pageno followed by the +** list, containing the 8-byte rowid/pageno followed by the ** *2 coordinates. */ static void rtreenode(sqlite3_context *ctx, int nArg, sqlite3_value **apArg){ - char *zText = 0; RtreeNode node; Rtree tree; int ii; + int nData; + int errCode; + sqlite3_str *pOut; UNUSED_PARAMETER(nArg); memset(&node, 0, sizeof(RtreeNode)); memset(&tree, 0, sizeof(Rtree)); - tree.nDim = sqlite3_value_int(apArg[0]); + tree.nDim = (u8)sqlite3_value_int(apArg[0]); + if( tree.nDim<1 || tree.nDim>5 ) return; + tree.nDim2 = tree.nDim*2; tree.nBytesPerCell = 8 + 8 * tree.nDim; node.zData = (u8 *)sqlite3_value_blob(apArg[1]); + if( node.zData==0 ) return; + nData = sqlite3_value_bytes(apArg[1]); + if( nData<4 ) return; + if( nData0 ) sqlite3_str_append(pOut, " ", 1); + sqlite3_str_appendf(pOut, "{%lld", cell.iRowid); + for(jj=0; jjrc==SQLITE_OK ) pCheck->rc = rc; +} + +/* +** The second and subsequent arguments to this function are a format string +** and printf style arguments. This function formats the string and attempts +** to compile it as an SQL statement. +** +** If successful, a pointer to the new SQL statement is returned. Otherwise, +** NULL is returned and an error code left in RtreeCheck.rc. +*/ +static sqlite3_stmt *rtreeCheckPrepare( + RtreeCheck *pCheck, /* RtreeCheck object */ + const char *zFmt, ... /* Format string and trailing args */ +){ + va_list ap; + char *z; + sqlite3_stmt *pRet = 0; + + va_start(ap, zFmt); + z = sqlite3_vmprintf(zFmt, ap); + + if( pCheck->rc==SQLITE_OK ){ + if( z==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + pCheck->rc = sqlite3_prepare_v2(pCheck->db, z, -1, &pRet, 0); + } + } + + sqlite3_free(z); + va_end(ap); + return pRet; +} + +/* +** The second and subsequent arguments to this function are a printf() +** style format string and arguments. This function formats the string and +** appends it to the report being accumuated in pCheck. +*/ +static void rtreeCheckAppendMsg(RtreeCheck *pCheck, const char *zFmt, ...){ + va_list ap; + va_start(ap, zFmt); + if( pCheck->rc==SQLITE_OK && pCheck->nErrrc = SQLITE_NOMEM; + }else{ + pCheck->zReport = sqlite3_mprintf("%z%s%z", + pCheck->zReport, (pCheck->zReport ? "\n" : ""), z + ); + if( pCheck->zReport==0 ){ + pCheck->rc = SQLITE_NOMEM; + } + } + pCheck->nErr++; + } + va_end(ap); +} + +/* +** This function is a no-op if there is already an error code stored +** in the RtreeCheck object indicated by the first argument. NULL is +** returned in this case. +** +** Otherwise, the contents of rtree table node iNode are loaded from +** the database and copied into a buffer obtained from sqlite3_malloc(). +** If no error occurs, a pointer to the buffer is returned and (*pnNode) +** is set to the size of the buffer in bytes. +** +** Or, if an error does occur, NULL is returned and an error code left +** in the RtreeCheck object. The final value of *pnNode is undefined in +** this case. +*/ +static u8 *rtreeCheckGetNode(RtreeCheck *pCheck, i64 iNode, int *pnNode){ + u8 *pRet = 0; /* Return value */ + + if( pCheck->rc==SQLITE_OK && pCheck->pGetNode==0 ){ + pCheck->pGetNode = rtreeCheckPrepare(pCheck, + "SELECT data FROM %Q.'%q_node' WHERE nodeno=?", + pCheck->zDb, pCheck->zTab + ); + } + + if( pCheck->rc==SQLITE_OK ){ + sqlite3_bind_int64(pCheck->pGetNode, 1, iNode); + if( sqlite3_step(pCheck->pGetNode)==SQLITE_ROW ){ + int nNode = sqlite3_column_bytes(pCheck->pGetNode, 0); + const u8 *pNode = (const u8*)sqlite3_column_blob(pCheck->pGetNode, 0); + pRet = sqlite3_malloc64(nNode); + if( pRet==0 ){ + pCheck->rc = SQLITE_NOMEM; + }else{ + memcpy(pRet, pNode, nNode); + *pnNode = nNode; + } + } + rtreeCheckReset(pCheck, pCheck->pGetNode); + if( pCheck->rc==SQLITE_OK && pRet==0 ){ + rtreeCheckAppendMsg(pCheck, "Node %lld missing from database", iNode); + } + } + + return pRet; +} + +/* +** This function is used to check that the %_parent (if bLeaf==0) or %_rowid +** (if bLeaf==1) table contains a specified entry. The schemas of the +** two tables are: +** +** CREATE TABLE %_parent(nodeno INTEGER PRIMARY KEY, parentnode INTEGER) +** CREATE TABLE %_rowid(rowid INTEGER PRIMARY KEY, nodeno INTEGER, ...) +** +** In both cases, this function checks that there exists an entry with +** IPK value iKey and the second column set to iVal. +** +*/ +static void rtreeCheckMapping( + RtreeCheck *pCheck, /* RtreeCheck object */ + int bLeaf, /* True for a leaf cell, false for interior */ + i64 iKey, /* Key for mapping */ + i64 iVal /* Expected value for mapping */ +){ + int rc; + sqlite3_stmt *pStmt; + const char *azSql[2] = { + "SELECT parentnode FROM %Q.'%q_parent' WHERE nodeno=?1", + "SELECT nodeno FROM %Q.'%q_rowid' WHERE rowid=?1" + }; + + assert( bLeaf==0 || bLeaf==1 ); + if( pCheck->aCheckMapping[bLeaf]==0 ){ + pCheck->aCheckMapping[bLeaf] = rtreeCheckPrepare(pCheck, + azSql[bLeaf], pCheck->zDb, pCheck->zTab + ); + } + if( pCheck->rc!=SQLITE_OK ) return; + + pStmt = pCheck->aCheckMapping[bLeaf]; + sqlite3_bind_int64(pStmt, 1, iKey); + rc = sqlite3_step(pStmt); + if( rc==SQLITE_DONE ){ + rtreeCheckAppendMsg(pCheck, "Mapping (%lld -> %lld) missing from %s table", + iKey, iVal, (bLeaf ? "%_rowid" : "%_parent") + ); + }else if( rc==SQLITE_ROW ){ + i64 ii = sqlite3_column_int64(pStmt, 0); + if( ii!=iVal ){ + rtreeCheckAppendMsg(pCheck, + "Found (%lld -> %lld) in %s table, expected (%lld -> %lld)", + iKey, ii, (bLeaf ? "%_rowid" : "%_parent"), iKey, iVal + ); + } + } + rtreeCheckReset(pCheck, pStmt); +} + +/* +** Argument pCell points to an array of coordinates stored on an rtree page. +** This function checks that the coordinates are internally consistent (no +** x1>x2 conditions) and adds an error message to the RtreeCheck object +** if they are not. +** +** Additionally, if pParent is not NULL, then it is assumed to point to +** the array of coordinates on the parent page that bound the page +** containing pCell. In this case it is also verified that the two +** sets of coordinates are mutually consistent and an error message added +** to the RtreeCheck object if they are not. +*/ +static void rtreeCheckCellCoord( + RtreeCheck *pCheck, + i64 iNode, /* Node id to use in error messages */ + int iCell, /* Cell number to use in error messages */ + u8 *pCell, /* Pointer to cell coordinates */ + u8 *pParent /* Pointer to parent coordinates */ +){ + RtreeCoord c1, c2; + RtreeCoord p1, p2; + int i; + + for(i=0; inDim; i++){ + readCoord(&pCell[4*2*i], &c1); + readCoord(&pCell[4*(2*i + 1)], &c2); + + /* printf("%e, %e\n", c1.u.f, c2.u.f); */ + if( pCheck->bInt ? c1.i>c2.i : c1.f>c2.f ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt", i, iCell, iNode + ); + } + + if( pParent ){ + readCoord(&pParent[4*2*i], &p1); + readCoord(&pParent[4*(2*i + 1)], &p2); + + if( (pCheck->bInt ? c1.ibInt ? c2.i>p2.i : c2.f>p2.f) + ){ + rtreeCheckAppendMsg(pCheck, + "Dimension %d of cell %d on node %lld is corrupt relative to parent" + , i, iCell, iNode + ); + } + } + } +} + +/* +** Run rtreecheck() checks on node iNode, which is at depth iDepth within +** the r-tree structure. Argument aParent points to the array of coordinates +** that bound node iNode on the parent node. +** +** If any problems are discovered, an error message is appended to the +** report accumulated in the RtreeCheck object. +*/ +static void rtreeCheckNode( + RtreeCheck *pCheck, + int iDepth, /* Depth of iNode (0==leaf) */ + u8 *aParent, /* Buffer containing parent coords */ + i64 iNode /* Node to check */ +){ + u8 *aNode = 0; + int nNode = 0; + + assert( iNode==1 || aParent!=0 ); + assert( pCheck->nDim>0 ); + + aNode = rtreeCheckGetNode(pCheck, iNode, &nNode); + if( aNode ){ + if( nNode<4 ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small (%d bytes)", iNode, nNode + ); + }else{ + int nCell; /* Number of cells on page */ + int i; /* Used to iterate through cells */ + if( aParent==0 ){ + iDepth = readInt16(aNode); + if( iDepth>RTREE_MAX_DEPTH ){ + rtreeCheckAppendMsg(pCheck, "Rtree depth out of range (%d)", iDepth); + sqlite3_free(aNode); + return; + } + } + nCell = readInt16(&aNode[2]); + if( (4 + nCell*(8 + pCheck->nDim*2*4))>nNode ){ + rtreeCheckAppendMsg(pCheck, + "Node %lld is too small for cell count of %d (%d bytes)", + iNode, nCell, nNode + ); + }else{ + for(i=0; inDim*2*4)]; + i64 iVal = readInt64(pCell); + rtreeCheckCellCoord(pCheck, iNode, i, &pCell[8], aParent); + + if( iDepth>0 ){ + rtreeCheckMapping(pCheck, 0, iVal, iNode); + rtreeCheckNode(pCheck, iDepth-1, &pCell[8], iVal); + pCheck->nNonLeaf++; + }else{ + rtreeCheckMapping(pCheck, 1, iVal, iNode); + pCheck->nLeaf++; + } + } + } + } + sqlite3_free(aNode); + } +} + +/* +** The second argument to this function must be either "_rowid" or +** "_parent". This function checks that the number of entries in the +** %_rowid or %_parent table is exactly nExpect. If not, it adds +** an error message to the report in the RtreeCheck object indicated +** by the first argument. +*/ +static void rtreeCheckCount(RtreeCheck *pCheck, const char *zTbl, i64 nExpect){ + if( pCheck->rc==SQLITE_OK ){ + sqlite3_stmt *pCount; + pCount = rtreeCheckPrepare(pCheck, "SELECT count(*) FROM %Q.'%q%s'", + pCheck->zDb, pCheck->zTab, zTbl + ); + if( pCount ){ + if( sqlite3_step(pCount)==SQLITE_ROW ){ + i64 nActual = sqlite3_column_int64(pCount, 0); + if( nActual!=nExpect ){ + rtreeCheckAppendMsg(pCheck, "Wrong number of entries in %%%s table" + " - expected %lld, actual %lld" , zTbl, nExpect, nActual + ); + } + } + pCheck->rc = sqlite3_finalize(pCount); + } + } +} + +/* +** This function does the bulk of the work for the rtree integrity-check. +** It is called by rtreecheck(), which is the SQL function implementation. +*/ +static int rtreeCheckTable( + sqlite3 *db, /* Database handle to access db through */ + const char *zDb, /* Name of db ("main", "temp" etc.) */ + const char *zTab, /* Name of rtree table to check */ + char **pzReport /* OUT: sqlite3_malloc'd report text */ +){ + RtreeCheck check; /* Common context for various routines */ + sqlite3_stmt *pStmt = 0; /* Used to find column count of rtree table */ + int nAux = 0; /* Number of extra columns. */ + + /* Initialize the context object */ + memset(&check, 0, sizeof(check)); + check.db = db; + check.zDb = zDb; + check.zTab = zTab; + + /* Find the number of auxiliary columns */ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.'%q_rowid'", zDb, zTab); + if( pStmt ){ + nAux = sqlite3_column_count(pStmt) - 2; + sqlite3_finalize(pStmt); + }else + if( check.rc!=SQLITE_NOMEM ){ + check.rc = SQLITE_OK; + } + + /* Find number of dimensions in the rtree table. */ + pStmt = rtreeCheckPrepare(&check, "SELECT * FROM %Q.%Q", zDb, zTab); + if( pStmt ){ + int rc; + check.nDim = (sqlite3_column_count(pStmt) - 1 - nAux) / 2; + if( check.nDim<1 ){ + rtreeCheckAppendMsg(&check, "Schema corrupt or not an rtree"); + }else if( SQLITE_ROW==sqlite3_step(pStmt) ){ + check.bInt = (sqlite3_column_type(pStmt, 1)==SQLITE_INTEGER); + } + rc = sqlite3_finalize(pStmt); + if( rc!=SQLITE_CORRUPT ) check.rc = rc; + } + + /* Do the actual integrity-check */ + if( check.nDim>=1 ){ + if( check.rc==SQLITE_OK ){ + rtreeCheckNode(&check, 0, 0, 1); + } + rtreeCheckCount(&check, "_rowid", check.nLeaf); + rtreeCheckCount(&check, "_parent", check.nNonLeaf); + } + + /* Finalize SQL statements used by the integrity-check */ + sqlite3_finalize(check.pGetNode); + sqlite3_finalize(check.aCheckMapping[0]); + sqlite3_finalize(check.aCheckMapping[1]); + + *pzReport = check.zReport; + return check.rc; +} + +/* +** Implementation of the xIntegrity method for Rtree. +*/ +static int rtreeIntegrity( + sqlite3_vtab *pVtab, /* The virtual table to check */ + const char *zSchema, /* Schema in which the virtual table lives */ + const char *zName, /* Name of the virtual table */ + int isQuick, /* True for a quick_check */ + char **pzErr /* Write results here */ +){ + Rtree *pRtree = (Rtree*)pVtab; + int rc; + assert( pzErr!=0 && *pzErr==0 ); + UNUSED_PARAMETER(zSchema); + UNUSED_PARAMETER(zName); + UNUSED_PARAMETER(isQuick); + rc = rtreeCheckTable(pRtree->db, pRtree->zDb, pRtree->zName, pzErr); + if( rc==SQLITE_OK && *pzErr ){ + *pzErr = sqlite3_mprintf("In RTree %s.%s:\n%z", + pRtree->zDb, pRtree->zName, *pzErr); + if( (*pzErr)==0 ) rc = SQLITE_NOMEM; + } + return rc; +} + +/* +** Usage: +** +** rtreecheck(); +** rtreecheck(, ); +** +** Invoking this SQL function runs an integrity-check on the named rtree +** table. The integrity-check verifies the following: +** +** 1. For each cell in the r-tree structure (%_node table), that: +** +** a) for each dimension, (coord1 <= coord2). +** +** b) unless the cell is on the root node, that the cell is bounded +** by the parent cell on the parent node. +** +** c) for leaf nodes, that there is an entry in the %_rowid +** table corresponding to the cell's rowid value that +** points to the correct node. +** +** d) for cells on non-leaf nodes, that there is an entry in the +** %_parent table mapping from the cell's child node to the +** node that it resides on. +** +** 2. That there are the same number of entries in the %_rowid table +** as there are leaf cells in the r-tree structure, and that there +** is a leaf cell that corresponds to each entry in the %_rowid table. +** +** 3. That there are the same number of entries in the %_parent table +** as there are non-leaf cells in the r-tree structure, and that +** there is a non-leaf cell that corresponds to each entry in the +** %_parent table. +*/ +static void rtreecheck( + sqlite3_context *ctx, + int nArg, + sqlite3_value **apArg +){ + if( nArg!=1 && nArg!=2 ){ + sqlite3_result_error(ctx, + "wrong number of arguments to function rtreecheck()", -1 + ); + }else{ + int rc; + char *zReport = 0; + const char *zDb = (const char*)sqlite3_value_text(apArg[0]); + const char *zTab; + if( nArg==1 ){ + zTab = zDb; + zDb = "main"; + }else{ + zTab = (const char*)sqlite3_value_text(apArg[1]); + } + rc = rtreeCheckTable(sqlite3_context_db_handle(ctx), zDb, zTab, &zReport); + if( rc==SQLITE_OK ){ + sqlite3_result_text(ctx, zReport ? zReport : "ok", -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(ctx, rc); + } + sqlite3_free(zReport); + } +} + +/* Conditionally include the geopoly code */ +#ifdef SQLITE_ENABLE_GEOPOLY +/************** Include geopoly.c in the middle of rtree.c *******************/ +/************** Begin file geopoly.c *****************************************/ +/* +** 2018-05-25 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file implements an alternative R-Tree virtual table that +** uses polygons to express the boundaries of 2-dimensional objects. +** +** This file is #include-ed onto the end of "rtree.c" so that it has +** access to all of the R-Tree internals. +*/ +/* #include */ + +/* Enable -DGEOPOLY_ENABLE_DEBUG for debugging facilities */ +#ifdef GEOPOLY_ENABLE_DEBUG + static int geo_debug = 0; +# define GEODEBUG(X) if(geo_debug)printf X +#else +# define GEODEBUG(X) +#endif + +/* Character class routines */ +#ifdef sqlite3Isdigit + /* Use the SQLite core versions if this routine is part of the + ** SQLite amalgamation */ +# define safe_isdigit(x) sqlite3Isdigit(x) +# define safe_isalnum(x) sqlite3Isalnum(x) +# define safe_isxdigit(x) sqlite3Isxdigit(x) +#else + /* Use the standard library for separate compilation */ +#include /* amalgamator: keep */ +# define safe_isdigit(x) isdigit((unsigned char)(x)) +# define safe_isalnum(x) isalnum((unsigned char)(x)) +# define safe_isxdigit(x) isxdigit((unsigned char)(x)) +#endif + +#ifndef JSON_NULL /* The following stuff repeats things found in json1 */ +/* +** Growing our own isspace() routine this way is twice as fast as +** the library isspace() function. +*/ +static const char geopolyIsSpace[] = { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +}; +#define fast_isspace(x) (geopolyIsSpace[(unsigned char)x]) +#endif /* JSON NULL - back to original code */ + +/* Compiler and version */ +#ifndef GCC_VERSION +#if defined(__GNUC__) && !defined(SQLITE_DISABLE_INTRINSIC) +# define GCC_VERSION (__GNUC__*1000000+__GNUC_MINOR__*1000+__GNUC_PATCHLEVEL__) +#else +# define GCC_VERSION 0 +#endif +#endif +#ifndef MSVC_VERSION +#if defined(_MSC_VER) && !defined(SQLITE_DISABLE_INTRINSIC) +# define MSVC_VERSION _MSC_VER +#else +# define MSVC_VERSION 0 +#endif +#endif + +/* Datatype for coordinates +*/ +typedef float GeoCoord; + +/* +** Internal representation of a polygon. +** +** The polygon consists of a sequence of vertexes. There is a line +** segment between each pair of vertexes, and one final segment from +** the last vertex back to the first. (This differs from the GeoJSON +** standard in which the final vertex is a repeat of the first.) +** +** The polygon follows the right-hand rule. The area to the right of +** each segment is "outside" and the area to the left is "inside". +** +** The on-disk representation consists of a 4-byte header followed by +** the values. The 4-byte header is: +** +** encoding (1 byte) 0=big-endian, 1=little-endian +** nvertex (3 bytes) Number of vertexes as a big-endian integer +** +** Enough space is allocated for 4 coordinates, to work around over-zealous +** warnings coming from some compiler (notably, clang). In reality, the size +** of each GeoPoly memory allocate is adjusted as necessary so that the +** GeoPoly.a[] array at the end is the appropriate size. +*/ +typedef struct GeoPoly GeoPoly; +struct GeoPoly { + int nVertex; /* Number of vertexes */ + unsigned char hdr[4]; /* Header for on-disk representation */ + GeoCoord a[8]; /* 2*nVertex values. X (longitude) first, then Y */ +}; + +/* The size of a memory allocation needed for a GeoPoly object sufficient +** to hold N coordinate pairs. +*/ +#define GEOPOLY_SZ(N) (sizeof(GeoPoly) + sizeof(GeoCoord)*2*((N)-4)) + +/* Macros to access coordinates of a GeoPoly. +** We have to use these macros, rather than just say p->a[i] in order +** to silence (incorrect) UBSAN warnings if the array index is too large. +*/ +#define GeoX(P,I) (((GeoCoord*)(P)->a)[(I)*2]) +#define GeoY(P,I) (((GeoCoord*)(P)->a)[(I)*2+1]) + + +/* +** State of a parse of a GeoJSON input. +*/ +typedef struct GeoParse GeoParse; +struct GeoParse { + const unsigned char *z; /* Unparsed input */ + int nVertex; /* Number of vertexes in a[] */ + int nAlloc; /* Space allocated to a[] */ + int nErr; /* Number of errors encountered */ + GeoCoord *a; /* Array of vertexes. From sqlite3_malloc64() */ +}; + +/* Do a 4-byte byte swap */ +static void geopolySwab32(unsigned char *a){ + unsigned char t = a[0]; + a[0] = a[3]; + a[3] = t; + t = a[1]; + a[1] = a[2]; + a[2] = t; +} + +/* Skip whitespace. Return the next non-whitespace character. */ +static char geopolySkipSpace(GeoParse *p){ + while( fast_isspace(p->z[0]) ) p->z++; + return p->z[0]; +} + +/* Parse out a number. Write the value into *pVal if pVal!=0. +** return non-zero on success and zero if the next token is not a number. +*/ +static int geopolyParseNumber(GeoParse *p, GeoCoord *pVal){ + char c = geopolySkipSpace(p); + const unsigned char *z = p->z; + int j = 0; + int seenDP = 0; + int seenE = 0; + if( c=='-' ){ + j = 1; + c = z[j]; + } + if( c=='0' && z[j+1]>='0' && z[j+1]<='9' ) return 0; + for(;; j++){ + c = z[j]; + if( safe_isdigit(c) ) continue; + if( c=='.' ){ + if( z[j-1]=='-' ) return 0; + if( seenDP ) return 0; + seenDP = 1; + continue; + } + if( c=='e' || c=='E' ){ + if( z[j-1]<'0' ) return 0; + if( seenE ) return -1; + seenDP = seenE = 1; + c = z[j+1]; + if( c=='+' || c=='-' ){ + j++; + c = z[j+1]; + } + if( c<'0' || c>'9' ) return 0; + continue; + } + break; + } + if( z[j-1]<'0' ) return 0; + if( pVal ){ +#ifdef SQLITE_AMALGAMATION + /* The sqlite3AtoF() routine is much much faster than atof(), if it + ** is available */ + double r; + (void)sqlite3AtoF((const char*)p->z, &r, j, SQLITE_UTF8); + *pVal = r; +#else + *pVal = (GeoCoord)atof((const char*)p->z); +#endif + } + p->z += j; + return 1; +} + +/* +** If the input is a well-formed JSON array of coordinates with at least +** four coordinates and where each coordinate is itself a two-value array, +** then convert the JSON into a GeoPoly object and return a pointer to +** that object. +** +** If any error occurs, return NULL. +*/ +static GeoPoly *geopolyParseJson(const unsigned char *z, int *pRc){ + GeoParse s; + int rc = SQLITE_OK; + memset(&s, 0, sizeof(s)); + s.z = z; + if( geopolySkipSpace(&s)=='[' ){ + s.z++; + while( geopolySkipSpace(&s)=='[' ){ + int ii = 0; + char c; + s.z++; + if( s.nVertex>=s.nAlloc ){ + GeoCoord *aNew; + s.nAlloc = s.nAlloc*2 + 16; + aNew = sqlite3_realloc64(s.a, s.nAlloc*sizeof(GeoCoord)*2 ); + if( aNew==0 ){ + rc = SQLITE_NOMEM; + s.nErr++; + break; + } + s.a = aNew; + } + while( geopolyParseNumber(&s, ii<=1 ? &s.a[s.nVertex*2+ii] : 0) ){ + ii++; + if( ii==2 ) s.nVertex++; + c = geopolySkipSpace(&s); + s.z++; + if( c==',' ) continue; + if( c==']' && ii>=2 ) break; + s.nErr++; + rc = SQLITE_ERROR; + goto parse_json_err; + } + if( geopolySkipSpace(&s)==',' ){ + s.z++; + continue; + } + break; + } + if( geopolySkipSpace(&s)==']' + && s.nVertex>=4 + && s.a[0]==s.a[s.nVertex*2-2] + && s.a[1]==s.a[s.nVertex*2-1] + && (s.z++, geopolySkipSpace(&s)==0) + ){ + GeoPoly *pOut; + int x = 1; + s.nVertex--; /* Remove the redundant vertex at the end */ + pOut = sqlite3_malloc64( GEOPOLY_SZ((sqlite3_int64)s.nVertex) ); + x = 1; + if( pOut==0 ) goto parse_json_err; + pOut->nVertex = s.nVertex; + memcpy(pOut->a, s.a, s.nVertex*2*sizeof(GeoCoord)); + pOut->hdr[0] = *(unsigned char*)&x; + pOut->hdr[1] = (s.nVertex>>16)&0xff; + pOut->hdr[2] = (s.nVertex>>8)&0xff; + pOut->hdr[3] = s.nVertex&0xff; + sqlite3_free(s.a); + if( pRc ) *pRc = SQLITE_OK; + return pOut; + }else{ + s.nErr++; + rc = SQLITE_ERROR; + } + } +parse_json_err: + if( pRc ) *pRc = rc; + sqlite3_free(s.a); + return 0; +} + +/* +** Given a function parameter, try to interpret it as a polygon, either +** in the binary format or JSON text. Compute a GeoPoly object and +** return a pointer to that object. Or if the input is not a well-formed +** polygon, put an error message in sqlite3_context and return NULL. +*/ +static GeoPoly *geopolyFuncParam( + sqlite3_context *pCtx, /* Context for error messages */ + sqlite3_value *pVal, /* The value to decode */ + int *pRc /* Write error here */ +){ + GeoPoly *p = 0; + int nByte; + testcase( pCtx==0 ); + if( sqlite3_value_type(pVal)==SQLITE_BLOB + && (nByte = sqlite3_value_bytes(pVal))>=(int)(4+6*sizeof(GeoCoord)) + ){ + const unsigned char *a = sqlite3_value_blob(pVal); + int nVertex; + if( a==0 ){ + if( pCtx ) sqlite3_result_error_nomem(pCtx); + return 0; + } + nVertex = (a[1]<<16) + (a[2]<<8) + a[3]; + if( (a[0]==0 || a[0]==1) + && (nVertex*2*sizeof(GeoCoord) + 4)==(unsigned int)nByte + ){ + p = sqlite3_malloc64( sizeof(*p) + (nVertex-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + if( pCtx ) sqlite3_result_error_nomem(pCtx); + }else{ + int x = 1; + p->nVertex = nVertex; + memcpy(p->hdr, a, nByte); + if( a[0] != *(unsigned char*)&x ){ + int ii; + for(ii=0; iihdr[0] ^= 1; + } + } + } + if( pRc ) *pRc = SQLITE_OK; + return p; + }else if( sqlite3_value_type(pVal)==SQLITE_TEXT ){ + const unsigned char *zJson = sqlite3_value_text(pVal); + if( zJson==0 ){ + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + return geopolyParseJson(zJson, pRc); + }else{ + if( pRc ) *pRc = SQLITE_ERROR; + return 0; + } +} + +/* +** Implementation of the geopoly_blob(X) function. +** +** If the input is a well-formed Geopoly BLOB or JSON string +** then return the BLOB representation of the polygon. Otherwise +** return NULL. +*/ +static void geopolyBlobFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_json(X) +** +** Interpret X as a polygon and render it as a JSON array +** of coordinates. Or, if X is not a valid polygon, return NULL. +*/ +static void geopolyJsonFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + sqlite3_str_append(x, "[", 1); + for(i=0; inVertex; i++){ + sqlite3_str_appendf(x, "[%!g,%!g],", GeoX(p,i), GeoY(p,i)); + } + sqlite3_str_appendf(x, "[%!g,%!g]]", GeoX(p,0), GeoY(p,0)); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL function: geopoly_svg(X, ....) +** +** Interpret X as a polygon and render it as a SVG . +** Additional arguments are added as attributes to the . +*/ +static void geopolySvgFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p; + if( argc<1 ) return; + p = geopolyFuncParam(context, argv[0], 0); + if( p ){ + sqlite3 *db = sqlite3_context_db_handle(context); + sqlite3_str *x = sqlite3_str_new(db); + int i; + char cSep = '\''; + sqlite3_str_appendf(x, ""); + sqlite3_result_text(context, sqlite3_str_finish(x), -1, sqlite3_free); + sqlite3_free(p); + } +} + +/* +** SQL Function: geopoly_xform(poly, A, B, C, D, E, F) +** +** Transform and/or translate a polygon as follows: +** +** x1 = A*x0 + B*y0 + E +** y1 = C*x0 + D*y0 + F +** +** For a translation: +** +** geopoly_xform(poly, 1, 0, 0, 1, x-offset, y-offset) +** +** Rotate by R around the point (0,0): +** +** geopoly_xform(poly, cos(R), sin(R), -sin(R), cos(R), 0, 0) +*/ +static void geopolyXformFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + double A = sqlite3_value_double(argv[1]); + double B = sqlite3_value_double(argv[2]); + double C = sqlite3_value_double(argv[3]); + double D = sqlite3_value_double(argv[4]); + double E = sqlite3_value_double(argv[5]); + double F = sqlite3_value_double(argv[6]); + GeoCoord x1, y1, x0, y0; + int ii; + (void)argc; + if( p ){ + for(ii=0; iinVertex; ii++){ + x0 = GeoX(p,ii); + y0 = GeoY(p,ii); + x1 = (GeoCoord)(A*x0 + B*y0 + E); + y1 = (GeoCoord)(C*x0 + D*y0 + F); + GeoX(p,ii) = x1; + GeoY(p,ii) = y1; + } + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** Compute the area enclosed by the polygon. +** +** This routine can also be used to detect polygons that rotate in +** the wrong direction. Polygons are suppose to be counter-clockwise (CCW). +** This routine returns a negative value for clockwise (CW) polygons. +*/ +static double geopolyArea(GeoPoly *p){ + double rArea = 0.0; + int ii; + for(ii=0; iinVertex-1; ii++){ + rArea += (GeoX(p,ii) - GeoX(p,ii+1)) /* (x0 - x1) */ + * (GeoY(p,ii) + GeoY(p,ii+1)) /* (y0 + y1) */ + * 0.5; + } + rArea += (GeoX(p,ii) - GeoX(p,0)) /* (xN - x0) */ + * (GeoY(p,ii) + GeoY(p,0)) /* (yN + y0) */ + * 0.5; + return rArea; +} + +/* +** Implementation of the geopoly_area(X) function. +** +** If the input is a well-formed Geopoly BLOB then return the area +** enclosed by the polygon. If the polygon circulates clockwise instead +** of counterclockwise (as it should) then return the negative of the +** enclosed area. Otherwise return NULL. +*/ +static void geopolyAreaFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; + if( p ){ + sqlite3_result_double(context, geopolyArea(p)); + sqlite3_free(p); + } +} + +/* +** Implementation of the geopoly_ccw(X) function. +** +** If the rotation of polygon X is clockwise (incorrect) instead of +** counter-clockwise (the correct winding order according to RFC7946) +** then reverse the order of the vertexes in polygon X. +** +** In other words, this routine returns a CCW polygon regardless of the +** winding order of its input. +** +** Use this routine to sanitize historical inputs that that sometimes +** contain polygons that wind in the wrong direction. +*/ +static void geopolyCcwFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyFuncParam(context, argv[0], 0); + (void)argc; + if( p ){ + if( geopolyArea(p)<0.0 ){ + int ii, jj; + for(ii=1, jj=p->nVertex-1; iihdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +#define GEOPOLY_PI 3.1415926535897932385 + +/* Fast approximation for sine(X) for X between -0.5*pi and 2*pi +*/ +static double geopolySine(double r){ + assert( r>=-0.5*GEOPOLY_PI && r<=2.0*GEOPOLY_PI ); + if( r>=1.5*GEOPOLY_PI ){ + r -= 2.0*GEOPOLY_PI; + } + if( r>=0.5*GEOPOLY_PI ){ + return -geopolySine(r-GEOPOLY_PI); + }else{ + double r2 = r*r; + double r3 = r2*r; + double r5 = r3*r2; + return 0.9996949*r - 0.1656700*r3 + 0.0075134*r5; + } +} + +/* +** Function: geopoly_regular(X,Y,R,N) +** +** Construct a simple, convex, regular polygon centered at X, Y +** with circumradius R and with N sides. +*/ +static void geopolyRegularFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + double x = sqlite3_value_double(argv[0]); + double y = sqlite3_value_double(argv[1]); + double r = sqlite3_value_double(argv[2]); + int n = sqlite3_value_int(argv[3]); + int i; + GeoPoly *p; + (void)argc; + + if( n<3 || r<=0.0 ) return; + if( n>1000 ) n = 1000; + p = sqlite3_malloc64( sizeof(*p) + (n-1)*2*sizeof(GeoCoord) ); + if( p==0 ){ + sqlite3_result_error_nomem(context); + return; + } + i = 1; + p->hdr[0] = *(unsigned char*)&i; + p->hdr[1] = 0; + p->hdr[2] = (n>>8)&0xff; + p->hdr[3] = n&0xff; + for(i=0; ihdr, 4+8*n, SQLITE_TRANSIENT); + sqlite3_free(p); +} + +/* +** If pPoly is a polygon, compute its bounding box. Then: +** +** (1) if aCoord!=0 store the bounding box in aCoord, returning NULL +** (2) otherwise, compute a GeoPoly for the bounding box and return the +** new GeoPoly +** +** If pPoly is NULL but aCoord is not NULL, then compute a new GeoPoly from +** the bounding box in aCoord and return a pointer to that GeoPoly. +*/ +static GeoPoly *geopolyBBox( + sqlite3_context *context, /* For recording the error */ + sqlite3_value *pPoly, /* The polygon */ + RtreeCoord *aCoord, /* Results here */ + int *pRc /* Error code here */ +){ + GeoPoly *pOut = 0; + GeoPoly *p; + float mnX, mxX, mnY, mxY; + if( pPoly==0 && aCoord!=0 ){ + p = 0; + mnX = aCoord[0].f; + mxX = aCoord[1].f; + mnY = aCoord[2].f; + mxY = aCoord[3].f; + goto geopolyBboxFill; + }else{ + p = geopolyFuncParam(context, pPoly, pRc); + } + if( p ){ + int ii; + mnX = mxX = GeoX(p,0); + mnY = mxY = GeoY(p,0); + for(ii=1; iinVertex; ii++){ + double r = GeoX(p,ii); + if( rmxX ) mxX = (float)r; + r = GeoY(p,ii); + if( rmxY ) mxY = (float)r; + } + if( pRc ) *pRc = SQLITE_OK; + if( aCoord==0 ){ + geopolyBboxFill: + pOut = sqlite3_realloc64(p, GEOPOLY_SZ(4)); + if( pOut==0 ){ + sqlite3_free(p); + if( context ) sqlite3_result_error_nomem(context); + if( pRc ) *pRc = SQLITE_NOMEM; + return 0; + } + pOut->nVertex = 4; + ii = 1; + pOut->hdr[0] = *(unsigned char*)ⅈ + pOut->hdr[1] = 0; + pOut->hdr[2] = 0; + pOut->hdr[3] = 4; + GeoX(pOut,0) = mnX; + GeoY(pOut,0) = mnY; + GeoX(pOut,1) = mxX; + GeoY(pOut,1) = mnY; + GeoX(pOut,2) = mxX; + GeoY(pOut,2) = mxY; + GeoX(pOut,3) = mnX; + GeoY(pOut,3) = mxY; + }else{ + sqlite3_free(p); + aCoord[0].f = mnX; + aCoord[1].f = mxX; + aCoord[2].f = mnY; + aCoord[3].f = mxY; + } + }else if( aCoord ){ + memset(aCoord, 0, sizeof(RtreeCoord)*4); + } + return pOut; +} + +/* +** Implementation of the geopoly_bbox(X) SQL function. +*/ +static void geopolyBBoxFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p = geopolyBBox(context, argv[0], 0, 0); + (void)argc; + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + +/* +** State vector for the geopoly_group_bbox() aggregate function. +*/ +typedef struct GeoBBox GeoBBox; +struct GeoBBox { + int isInit; + RtreeCoord a[4]; +}; + + +/* +** Implementation of the geopoly_group_bbox(X) aggregate SQL function. +*/ +static void geopolyBBoxStep( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + RtreeCoord a[4]; + int rc = SQLITE_OK; + (void)argc; + (void)geopolyBBox(context, argv[0], a, &rc); + if( rc==SQLITE_OK ){ + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, sizeof(*pBBox)); + if( pBBox==0 ) return; + if( pBBox->isInit==0 ){ + pBBox->isInit = 1; + memcpy(pBBox->a, a, sizeof(RtreeCoord)*4); + }else{ + if( a[0].f < pBBox->a[0].f ) pBBox->a[0] = a[0]; + if( a[1].f > pBBox->a[1].f ) pBBox->a[1] = a[1]; + if( a[2].f < pBBox->a[2].f ) pBBox->a[2] = a[2]; + if( a[3].f > pBBox->a[3].f ) pBBox->a[3] = a[3]; + } + } +} +static void geopolyBBoxFinal( + sqlite3_context *context +){ + GeoPoly *p; + GeoBBox *pBBox; + pBBox = (GeoBBox*)sqlite3_aggregate_context(context, 0); + if( pBBox==0 ) return; + p = geopolyBBox(context, 0, pBBox->a, 0); + if( p ){ + sqlite3_result_blob(context, p->hdr, + 4+8*p->nVertex, SQLITE_TRANSIENT); + sqlite3_free(p); + } +} + + +/* +** Determine if point (x0,y0) is beneath line segment (x1,y1)->(x2,y2). +** Returns: +** +** +2 x0,y0 is on the line segement +** +** +1 x0,y0 is beneath line segment +** +** 0 x0,y0 is not on or beneath the line segment or the line segment +** is vertical and x0,y0 is not on the line segment +** +** The left-most coordinate min(x1,x2) is not considered to be part of +** the line segment for the purposes of this analysis. +*/ +static int pointBeneathLine( + double x0, double y0, + double x1, double y1, + double x2, double y2 +){ + double y; + if( x0==x1 && y0==y1 ) return 2; + if( x1x2 ) return 0; + }else if( x1>x2 ){ + if( x0<=x2 || x0>x1 ) return 0; + }else{ + /* Vertical line segment */ + if( x0!=x1 ) return 0; + if( y0y1 && y0>y2 ) return 0; + return 2; + } + y = y1 + (y2-y1)*(x0-x1)/(x2-x1); + if( y0==y ) return 2; + if( y0nVertex-1; ii++){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,ii+1),GeoY(p1,ii+1)); + if( v==2 ) break; + cnt += v; + } + if( v!=2 ){ + v = pointBeneathLine(x0,y0,GeoX(p1,ii), GeoY(p1,ii), + GeoX(p1,0), GeoY(p1,0)); + } + if( v==2 ){ + sqlite3_result_int(context, 1); + }else if( ((v+cnt)&1)==0 ){ + sqlite3_result_int(context, 0); + }else{ + sqlite3_result_int(context, 2); + } + sqlite3_free(p1); +} + +/* Forward declaration */ +static int geopolyOverlap(GeoPoly *p1, GeoPoly *p2); + +/* +** SQL function: geopoly_within(P1,P2) +** +** Return +2 if P1 and P2 are the same polygon +** Return +1 if P2 is contained within P1 +** Return 0 if any part of P2 is on the outside of P1 +** +*/ +static void geopolyWithinFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x==2 ? 1 : x==4 ? 2 : 0); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* Objects used by the overlap algorihm. */ +typedef struct GeoEvent GeoEvent; +typedef struct GeoSegment GeoSegment; +typedef struct GeoOverlap GeoOverlap; +struct GeoEvent { + double x; /* X coordinate at which event occurs */ + int eType; /* 0 for ADD, 1 for REMOVE */ + GeoSegment *pSeg; /* The segment to be added or removed */ + GeoEvent *pNext; /* Next event in the sorted list */ +}; +struct GeoSegment { + double C, B; /* y = C*x + B */ + double y; /* Current y value */ + float y0; /* Initial y value */ + unsigned char side; /* 1 for p1, 2 for p2 */ + unsigned int idx; /* Which segment within the side */ + GeoSegment *pNext; /* Next segment in a list sorted by y */ +}; +struct GeoOverlap { + GeoEvent *aEvent; /* Array of all events */ + GeoSegment *aSegment; /* Array of all segments */ + int nEvent; /* Number of events */ + int nSegment; /* Number of segments */ +}; + +/* +** Add a single segment and its associated events. +*/ +static void geopolyAddOneSegment( + GeoOverlap *p, + GeoCoord x0, + GeoCoord y0, + GeoCoord x1, + GeoCoord y1, + unsigned char side, + unsigned int idx +){ + GeoSegment *pSeg; + GeoEvent *pEvent; + if( x0==x1 ) return; /* Ignore vertical segments */ + if( x0>x1 ){ + GeoCoord t = x0; + x0 = x1; + x1 = t; + t = y0; + y0 = y1; + y1 = t; + } + pSeg = p->aSegment + p->nSegment; + p->nSegment++; + pSeg->C = (y1-y0)/(x1-x0); + pSeg->B = y1 - x1*pSeg->C; + pSeg->y0 = y0; + pSeg->side = side; + pSeg->idx = idx; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x0; + pEvent->eType = 0; + pEvent->pSeg = pSeg; + pEvent = p->aEvent + p->nEvent; + p->nEvent++; + pEvent->x = x1; + pEvent->eType = 1; + pEvent->pSeg = pSeg; +} + + + +/* +** Insert all segments and events for polygon pPoly. +*/ +static void geopolyAddSegments( + GeoOverlap *p, /* Add segments to this Overlap object */ + GeoPoly *pPoly, /* Take all segments from this polygon */ + unsigned char side /* The side of pPoly */ +){ + unsigned int i; + GeoCoord *x; + for(i=0; i<(unsigned)pPoly->nVertex-1; i++){ + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], x[2], x[3], side, i); + } + x = &GeoX(pPoly,i); + geopolyAddOneSegment(p, x[0], x[1], pPoly->a[0], pPoly->a[1], side, i); +} + +/* +** Merge two lists of sorted events by X coordinate +*/ +static GeoEvent *geopolyEventMerge(GeoEvent *pLeft, GeoEvent *pRight){ + GeoEvent head, *pLast; + head.pNext = 0; + pLast = &head; + while( pRight && pLeft ){ + if( pRight->x <= pLeft->x ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort an array of nEvent event objects into a list. +*/ +static GeoEvent *geopolySortEventsByX(GeoEvent *aEvent, int nEvent){ + int mx = 0; + int i, j; + GeoEvent *p; + GeoEvent *a[50]; + for(i=0; ipNext = 0; + for(j=0; j=mx ) mx = j+1; + } + p = 0; + for(i=0; iy - pLeft->y; + if( r==0.0 ) r = pRight->C - pLeft->C; + if( r<0.0 ){ + pLast->pNext = pRight; + pLast = pRight; + pRight = pRight->pNext; + }else{ + pLast->pNext = pLeft; + pLast = pLeft; + pLeft = pLeft->pNext; + } + } + pLast->pNext = pRight ? pRight : pLeft; + return head.pNext; +} + +/* +** Sort a list of GeoSegments in order of increasing Y and in the event of +** a tie, increasing C (slope). +*/ +static GeoSegment *geopolySortSegmentsByYAndC(GeoSegment *pList){ + int mx = 0; + int i; + GeoSegment *p; + GeoSegment *a[50]; + while( pList ){ + p = pList; + pList = pList->pNext; + p->pNext = 0; + for(i=0; i=mx ) mx = i+1; + } + p = 0; + for(i=0; inVertex + p2->nVertex + 2; + GeoOverlap *p; + sqlite3_int64 nByte; + GeoEvent *pThisEvent; + double rX; + int rc = 0; + int needSort = 0; + GeoSegment *pActive = 0; + GeoSegment *pSeg; + unsigned char aOverlap[4]; + + nByte = sizeof(GeoEvent)*nVertex*2 + + sizeof(GeoSegment)*nVertex + + sizeof(GeoOverlap); + p = sqlite3_malloc64( nByte ); + if( p==0 ) return -1; + p->aEvent = (GeoEvent*)&p[1]; + p->aSegment = (GeoSegment*)&p->aEvent[nVertex*2]; + p->nEvent = p->nSegment = 0; + geopolyAddSegments(p, p1, 1); + geopolyAddSegments(p, p2, 2); + pThisEvent = geopolySortEventsByX(p->aEvent, p->nEvent); + rX = pThisEvent && pThisEvent->x==0.0 ? -1.0 : 0.0; + memset(aOverlap, 0, sizeof(aOverlap)); + while( pThisEvent ){ + if( pThisEvent->x!=rX ){ + GeoSegment *pPrev = 0; + int iMask = 0; + GEODEBUG(("Distinct X: %g\n", pThisEvent->x)); + rX = pThisEvent->x; + if( needSort ){ + GEODEBUG(("SORT\n")); + pActive = geopolySortSegmentsByYAndC(pActive); + needSort = 0; + } + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pPrev ){ + if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + pPrev = 0; + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + double y = pSeg->C*rX + pSeg->B; + GEODEBUG(("Segment %d.%d %g->%g\n", pSeg->side, pSeg->idx, pSeg->y, y)); + pSeg->y = y; + if( pPrev ){ + if( pPrev->y>pSeg->y && pPrev->side!=pSeg->side ){ + rc = 1; + GEODEBUG(("Crossing: %d.%d and %d.%d\n", + pPrev->side, pPrev->idx, + pSeg->side, pSeg->idx)); + goto geopolyOverlapDone; + }else if( pPrev->y!=pSeg->y ){ + GEODEBUG(("MASK: %d\n", iMask)); + aOverlap[iMask] = 1; + } + } + iMask ^= pSeg->side; + pPrev = pSeg; + } + } + GEODEBUG(("%s %d.%d C=%g B=%g\n", + pThisEvent->eType ? "RM " : "ADD", + pThisEvent->pSeg->side, pThisEvent->pSeg->idx, + pThisEvent->pSeg->C, + pThisEvent->pSeg->B)); + if( pThisEvent->eType==0 ){ + /* Add a segment */ + pSeg = pThisEvent->pSeg; + pSeg->y = pSeg->y0; + pSeg->pNext = pActive; + pActive = pSeg; + needSort = 1; + }else{ + /* Remove a segment */ + if( pActive==pThisEvent->pSeg ){ + pActive = ALWAYS(pActive) ? pActive->pNext : 0; + }else{ + for(pSeg=pActive; pSeg; pSeg=pSeg->pNext){ + if( pSeg->pNext==pThisEvent->pSeg ){ + pSeg->pNext = ALWAYS(pSeg->pNext) ? pSeg->pNext->pNext : 0; + break; + } + } + } + } + pThisEvent = pThisEvent->pNext; + } + if( aOverlap[3]==0 ){ + rc = 0; + }else if( aOverlap[1]!=0 && aOverlap[2]==0 ){ + rc = 3; + }else if( aOverlap[1]==0 && aOverlap[2]!=0 ){ + rc = 2; + }else if( aOverlap[1]==0 && aOverlap[2]==0 ){ + rc = 4; + }else{ + rc = 1; + } + +geopolyOverlapDone: + sqlite3_free(p); + return rc; +} + +/* +** SQL function: geopoly_overlap(P1,P2) +** +** Determine whether or not P1 and P2 overlap. Return value: +** +** 0 The two polygons are disjoint +** 1 They overlap +** 2 P1 is completely contained within P2 +** 3 P2 is completely contained within P1 +** 4 P1 and P2 are the same polygon +** NULL Either P1 or P2 or both are not valid polygons +*/ +static void geopolyOverlapFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + GeoPoly *p1 = geopolyFuncParam(context, argv[0], 0); + GeoPoly *p2 = geopolyFuncParam(context, argv[1], 0); + (void)argc; + if( p1 && p2 ){ + int x = geopolyOverlap(p1, p2); + if( x<0 ){ + sqlite3_result_error_nomem(context); + }else{ + sqlite3_result_int(context, x); + } + } + sqlite3_free(p1); + sqlite3_free(p2); +} + +/* +** Enable or disable debugging output +*/ +static void geopolyDebugFunc( + sqlite3_context *context, + int argc, + sqlite3_value **argv +){ + (void)context; + (void)argc; +#ifdef GEOPOLY_ENABLE_DEBUG + geo_debug = sqlite3_value_int(argv[0]); +#else + (void)argv; +#endif +} + +/* +** This function is the implementation of both the xConnect and xCreate +** methods of the geopoly virtual table. +** +** argv[0] -> module name +** argv[1] -> database name +** argv[2] -> table name +** argv[...] -> column names... +*/ +static int geopolyInit( + sqlite3 *db, /* Database connection */ + void *pAux, /* One of the RTREE_COORD_* constants */ + int argc, const char *const*argv, /* Parameters to CREATE TABLE statement */ + sqlite3_vtab **ppVtab, /* OUT: New virtual table */ + char **pzErr, /* OUT: Error message, if any */ + int isCreate /* True for xCreate, false for xConnect */ +){ + int rc = SQLITE_OK; + Rtree *pRtree; + sqlite3_int64 nDb; /* Length of string argv[1] */ + sqlite3_int64 nName; /* Length of string argv[2] */ + sqlite3_str *pSql; + char *zSql; + int ii; + (void)pAux; + + sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, 1); + sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + + /* Allocate the sqlite3_vtab structure */ + nDb = strlen(argv[1]); + nName = strlen(argv[2]); + pRtree = (Rtree *)sqlite3_malloc64(sizeof(Rtree)+nDb+nName*2+8); + if( !pRtree ){ + return SQLITE_NOMEM; + } + memset(pRtree, 0, sizeof(Rtree)+nDb+nName*2+8); + pRtree->nBusy = 1; + pRtree->base.pModule = &rtreeModule; + pRtree->zDb = (char *)&pRtree[1]; + pRtree->zName = &pRtree->zDb[nDb+1]; + pRtree->zNodeName = &pRtree->zName[nName+1]; + pRtree->eCoordType = RTREE_COORD_REAL32; + pRtree->nDim = 2; + pRtree->nDim2 = 4; + memcpy(pRtree->zDb, argv[1], nDb); + memcpy(pRtree->zName, argv[2], nName); + memcpy(pRtree->zNodeName, argv[2], nName); + memcpy(&pRtree->zNodeName[nName], "_node", 6); + + + /* Create/Connect to the underlying relational database schema. If + ** that is successful, call sqlite3_declare_vtab() to configure + ** the r-tree table schema. + */ + pSql = sqlite3_str_new(db); + sqlite3_str_appendf(pSql, "CREATE TABLE x(_shape"); + pRtree->nAux = 1; /* Add one for _shape */ + pRtree->nAuxNotNull = 1; /* The _shape column is always not-null */ + for(ii=3; iinAux++; + sqlite3_str_appendf(pSql, ",%s", argv[ii]); + } + sqlite3_str_appendf(pSql, ");"); + zSql = sqlite3_str_finish(pSql); + if( !zSql ){ + rc = SQLITE_NOMEM; + }else if( SQLITE_OK!=(rc = sqlite3_declare_vtab(db, zSql)) ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + } + sqlite3_free(zSql); + if( rc ) goto geopolyInit_fail; + pRtree->nBytesPerCell = 8 + pRtree->nDim2*4; + + /* Figure out the node size to use. */ + rc = getNodeSize(db, pRtree, isCreate, pzErr); + if( rc ) goto geopolyInit_fail; + rc = rtreeSqlInit(pRtree, db, argv[1], argv[2], isCreate); + if( rc ){ + *pzErr = sqlite3_mprintf("%s", sqlite3_errmsg(db)); + goto geopolyInit_fail; + } + + *ppVtab = (sqlite3_vtab *)pRtree; + return SQLITE_OK; + +geopolyInit_fail: + if( rc==SQLITE_OK ) rc = SQLITE_ERROR; + assert( *ppVtab==0 ); + assert( pRtree->nBusy==1 ); + rtreeRelease(pRtree); + return rc; +} + + +/* +** GEOPOLY virtual table module xCreate method. +*/ +static int geopolyCreate( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 1); +} + +/* +** GEOPOLY virtual table module xConnect method. +*/ +static int geopolyConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + return geopolyInit(db, pAux, argc, argv, ppVtab, pzErr, 0); +} + + +/* +** GEOPOLY virtual table module xFilter method. +** +** Query plans: +** +** 1 rowid lookup +** 2 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 3 search for objects overlapping the same bounding box +** that contains polygon argv[0] +** 4 full table scan +*/ +static int geopolyFilter( + sqlite3_vtab_cursor *pVtabCursor, /* The cursor to initialize */ + int idxNum, /* Query plan */ + const char *idxStr, /* Not Used */ + int argc, sqlite3_value **argv /* Parameters to the query plan */ +){ + Rtree *pRtree = (Rtree *)pVtabCursor->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)pVtabCursor; + RtreeNode *pRoot = 0; + int rc = SQLITE_OK; + int iCell = 0; + (void)idxStr; + + rtreeReference(pRtree); + + /* Reset the cursor to the same state as rtreeOpen() leaves it in. */ + resetCursor(pCsr); + + pCsr->iStrategy = idxNum; + if( idxNum==1 ){ + /* Special case - lookup by rowid. */ + RtreeNode *pLeaf; /* Leaf on which the required cell resides */ + RtreeSearchPoint *p; /* Search point for the leaf */ + i64 iRowid = sqlite3_value_int64(argv[0]); + i64 iNode = 0; + rc = findLeafNode(pRtree, iRowid, &pLeaf, &iNode); + if( rc==SQLITE_OK && pLeaf!=0 ){ + p = rtreeSearchPointNew(pCsr, RTREE_ZERO, 0); + assert( p!=0 ); /* Always returns pCsr->sPoint */ + pCsr->aNode[0] = pLeaf; + p->id = iNode; + p->eWithin = PARTLY_WITHIN; + rc = nodeRowidIndex(pRtree, pLeaf, iRowid, &iCell); + p->iCell = (u8)iCell; + RTREE_QUEUE_TRACE(pCsr, "PUSH-F1:"); + }else{ + pCsr->atEOF = 1; + } + }else{ + /* Normal case - r-tree scan. Set up the RtreeCursor.aConstraint array + ** with the configured constraints. + */ + rc = nodeAcquire(pRtree, 1, 0, &pRoot); + if( rc==SQLITE_OK && idxNum<=3 ){ + RtreeCoord bbox[4]; + RtreeConstraint *p; + assert( argc==1 ); + assert( argv[0]!=0 ); + geopolyBBox(0, argv[0], bbox, &rc); + if( rc ){ + goto geopoly_filter_end; + } + pCsr->aConstraint = p = sqlite3_malloc(sizeof(RtreeConstraint)*4); + pCsr->nConstraint = 4; + if( p==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pCsr->aConstraint, 0, sizeof(RtreeConstraint)*4); + memset(pCsr->anQueue, 0, sizeof(u32)*(pRtree->iDepth + 1)); + if( idxNum==2 ){ + /* Overlap query */ + p->op = 'B'; + p->iCoord = 0; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 1; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 2; + p->u.rValue = bbox[3].f; + p++; + p->op = 'D'; + p->iCoord = 3; + p->u.rValue = bbox[2].f; + }else{ + /* Within query */ + p->op = 'D'; + p->iCoord = 0; + p->u.rValue = bbox[0].f; + p++; + p->op = 'B'; + p->iCoord = 1; + p->u.rValue = bbox[1].f; + p++; + p->op = 'D'; + p->iCoord = 2; + p->u.rValue = bbox[2].f; + p++; + p->op = 'B'; + p->iCoord = 3; + p->u.rValue = bbox[3].f; + } + } + } + if( rc==SQLITE_OK ){ + RtreeSearchPoint *pNew; + pNew = rtreeSearchPointNew(pCsr, RTREE_ZERO, (u8)(pRtree->iDepth+1)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + goto geopoly_filter_end; + } + pNew->id = 1; + pNew->iCell = 0; + pNew->eWithin = PARTLY_WITHIN; + assert( pCsr->bPoint==1 ); + pCsr->aNode[0] = pRoot; + pRoot = 0; + RTREE_QUEUE_TRACE(pCsr, "PUSH-Fm:"); + rc = rtreeStepToLeaf(pCsr); + } + } + +geopoly_filter_end: + nodeRelease(pRtree, pRoot); + rtreeRelease(pRtree); + return rc; +} + +/* +** Rtree virtual table module xBestIndex method. There are three +** table scan strategies to choose from (in order from most to +** least desirable): +** +** idxNum idxStr Strategy +** ------------------------------------------------ +** 1 "rowid" Direct lookup by rowid. +** 2 "rtree" R-tree overlap query using geopoly_overlap() +** 3 "rtree" R-tree within query using geopoly_within() +** 4 "fullscan" full-table scan. +** ------------------------------------------------ +*/ +static int geopolyBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int ii; + int iRowidTerm = -1; + int iFuncTerm = -1; + int idxNum = 0; + (void)tab; + + for(ii=0; iinConstraint; ii++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[ii]; + if( !p->usable ) continue; + if( p->iColumn<0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + iRowidTerm = ii; + break; + } + if( p->iColumn==0 && p->op>=SQLITE_INDEX_CONSTRAINT_FUNCTION ){ + /* p->op==SQLITE_INDEX_CONSTRAINT_FUNCTION for geopoly_overlap() + ** p->op==(SQLITE_INDEX_CONTRAINT_FUNCTION+1) for geopoly_within(). + ** See geopolyFindFunction() */ + iFuncTerm = ii; + idxNum = p->op - SQLITE_INDEX_CONSTRAINT_FUNCTION + 2; + } + } + + if( iRowidTerm>=0 ){ + pIdxInfo->idxNum = 1; + pIdxInfo->idxStr = "rowid"; + pIdxInfo->aConstraintUsage[iRowidTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iRowidTerm].omit = 1; + pIdxInfo->estimatedCost = 30.0; + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + return SQLITE_OK; + } + if( iFuncTerm>=0 ){ + pIdxInfo->idxNum = idxNum; + pIdxInfo->idxStr = "rtree"; + pIdxInfo->aConstraintUsage[iFuncTerm].argvIndex = 1; + pIdxInfo->aConstraintUsage[iFuncTerm].omit = 0; + pIdxInfo->estimatedCost = 300.0; + pIdxInfo->estimatedRows = 10; + return SQLITE_OK; + } + pIdxInfo->idxNum = 4; + pIdxInfo->idxStr = "fullscan"; + pIdxInfo->estimatedCost = 3000000.0; + pIdxInfo->estimatedRows = 100000; + return SQLITE_OK; +} + + +/* +** GEOPOLY virtual table module xColumn method. +*/ +static int geopolyColumn(sqlite3_vtab_cursor *cur, sqlite3_context *ctx, int i){ + Rtree *pRtree = (Rtree *)cur->pVtab; + RtreeCursor *pCsr = (RtreeCursor *)cur; + RtreeSearchPoint *p = rtreeSearchPointFirst(pCsr); + int rc = SQLITE_OK; + RtreeNode *pNode = rtreeNodeOfFirstSearchPoint(pCsr, &rc); + + if( rc ) return rc; + if( p==0 ) return SQLITE_OK; + if( i==0 && sqlite3_vtab_nochange(ctx) ) return SQLITE_OK; + if( i<=pRtree->nAux ){ + if( !pCsr->bAuxValid ){ + if( pCsr->pReadAux==0 ){ + rc = sqlite3_prepare_v3(pRtree->db, pRtree->zReadAuxSql, -1, 0, + &pCsr->pReadAux, 0); + if( rc ) return rc; + } + sqlite3_bind_int64(pCsr->pReadAux, 1, + nodeGetRowid(pRtree, pNode, p->iCell)); + rc = sqlite3_step(pCsr->pReadAux); + if( rc==SQLITE_ROW ){ + pCsr->bAuxValid = 1; + }else{ + sqlite3_reset(pCsr->pReadAux); + if( rc==SQLITE_DONE ) rc = SQLITE_OK; + return rc; + } + } + sqlite3_result_value(ctx, sqlite3_column_value(pCsr->pReadAux, i+2)); + } + return SQLITE_OK; +} + + +/* +** The xUpdate method for GEOPOLY module virtual tables. +** +** For DELETE: +** +** argv[0] = the rowid to be deleted +** +** For INSERT: +** +** argv[0] = SQL NULL +** argv[1] = rowid to insert, or an SQL NULL to select automatically +** argv[2] = _shape column +** argv[3] = first application-defined column.... +** +** For UPDATE: +** +** argv[0] = rowid to modify. Never NULL +** argv[1] = rowid after the change. Never NULL +** argv[2] = new value for _shape +** argv[3] = new value for first application-defined column.... +*/ +static int geopolyUpdate( + sqlite3_vtab *pVtab, + int nData, + sqlite3_value **aData, + sqlite_int64 *pRowid +){ + Rtree *pRtree = (Rtree *)pVtab; + int rc = SQLITE_OK; + RtreeCell cell; /* New cell to insert if nData>1 */ + i64 oldRowid; /* The old rowid */ + int oldRowidValid; /* True if oldRowid is valid */ + i64 newRowid; /* The new rowid */ + int newRowidValid; /* True if newRowid is valid */ + int coordChange = 0; /* Change in coordinates */ + + if( pRtree->nNodeRef ){ + /* Unable to write to the btree while another cursor is reading from it, + ** since the write might do a rebalance which would disrupt the read + ** cursor. */ + return SQLITE_LOCKED_VTAB; + } + rtreeReference(pRtree); + assert(nData>=1); + + oldRowidValid = sqlite3_value_type(aData[0])!=SQLITE_NULL;; + oldRowid = oldRowidValid ? sqlite3_value_int64(aData[0]) : 0; + newRowidValid = nData>1 && sqlite3_value_type(aData[1])!=SQLITE_NULL; + newRowid = newRowidValid ? sqlite3_value_int64(aData[1]) : 0; + cell.iRowid = newRowid; + + if( nData>1 /* not a DELETE */ + && (!oldRowidValid /* INSERT */ + || !sqlite3_value_nochange(aData[2]) /* UPDATE _shape */ + || oldRowid!=newRowid) /* Rowid change */ + ){ + assert( aData[2]!=0 ); + geopolyBBox(0, aData[2], cell.aCoord, &rc); + if( rc ){ + if( rc==SQLITE_ERROR ){ + pVtab->zErrMsg = + sqlite3_mprintf("_shape does not contain a valid polygon"); + } + goto geopoly_update_end; + } + coordChange = 1; + + /* If a rowid value was supplied, check if it is already present in + ** the table. If so, the constraint has failed. */ + if( newRowidValid && (!oldRowidValid || oldRowid!=newRowid) ){ + int steprc; + sqlite3_bind_int64(pRtree->pReadRowid, 1, cell.iRowid); + steprc = sqlite3_step(pRtree->pReadRowid); + rc = sqlite3_reset(pRtree->pReadRowid); + if( SQLITE_ROW==steprc ){ + if( sqlite3_vtab_on_conflict(pRtree->db)==SQLITE_REPLACE ){ + rc = rtreeDeleteRowid(pRtree, cell.iRowid); + }else{ + rc = rtreeConstraintError(pRtree, 0); + } + } + } + } + + /* If aData[0] is not an SQL NULL value, it is the rowid of a + ** record to delete from the r-tree table. The following block does + ** just that. + */ + if( rc==SQLITE_OK && (nData==1 || (coordChange && oldRowidValid)) ){ + rc = rtreeDeleteRowid(pRtree, oldRowid); + } + + /* If the aData[] array contains more than one element, elements + ** (aData[2]..aData[argc-1]) contain a new record to insert into + ** the r-tree structure. + */ + if( rc==SQLITE_OK && nData>1 && coordChange ){ + /* Insert the new record into the r-tree */ + RtreeNode *pLeaf = 0; + if( !newRowidValid ){ + rc = rtreeNewRowid(pRtree, &cell.iRowid); + } + *pRowid = cell.iRowid; + if( rc==SQLITE_OK ){ + rc = ChooseLeaf(pRtree, &cell, 0, &pLeaf); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = rtreeInsertCell(pRtree, pLeaf, &cell, 0); + rc2 = nodeRelease(pRtree, pLeaf); + if( rc==SQLITE_OK ){ + rc = rc2; + } + } + } + + /* Change the data */ + if( rc==SQLITE_OK && nData>1 ){ + sqlite3_stmt *pUp = pRtree->pWriteAux; + int jj; + int nChange = 0; + sqlite3_bind_int64(pUp, 1, cell.iRowid); + assert( pRtree->nAux>=1 ); + if( sqlite3_value_nochange(aData[2]) ){ + sqlite3_bind_null(pUp, 2); + }else{ + GeoPoly *p = 0; + if( sqlite3_value_type(aData[2])==SQLITE_TEXT + && (p = geopolyFuncParam(0, aData[2], &rc))!=0 + && rc==SQLITE_OK + ){ + sqlite3_bind_blob(pUp, 2, p->hdr, 4+8*p->nVertex, SQLITE_TRANSIENT); + }else{ + sqlite3_bind_value(pUp, 2, aData[2]); + } + sqlite3_free(p); + nChange = 1; + } + for(jj=1; jjmagic = RTREE_GEOMETRY_MAGIC; + pBlob->iSize = nBlob; pBlob->cb = pGeomCtx[0]; pBlob->apSqlParam = (sqlite3_value**)&pBlob->aParam[nArg]; pBlob->nParam = nArg; @@ -162525,7 +217882,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ sqlite3_result_error_nomem(ctx); rtreeMatchArgFree(pBlob); }else{ - sqlite3_result_blob(ctx, pBlob, nBlob, rtreeMatchArgFree); + sqlite3_result_pointer(ctx, pBlob, "RtreeMatchArg", rtreeMatchArgFree); } } } @@ -162533,7 +217890,7 @@ static void geomCallback(sqlite3_context *ctx, int nArg, sqlite3_value **aArg){ /* ** Register a new geometry function for use with the r-tree MATCH operator. */ -SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( +SQLITE_API int sqlite3_rtree_geometry_callback( sqlite3 *db, /* Register SQL function on this connection */ const char *zGeom, /* Name of the new SQL function */ int (*xGeom)(sqlite3_rtree_geometry*,int,RtreeDValue*,int*), /* Callback */ @@ -162548,7 +217905,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( pGeomCtx->xQueryFunc = 0; pGeomCtx->xDestructor = 0; pGeomCtx->pContext = pContext; - return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, + return sqlite3_create_function_v2(db, zGeom, -1, SQLITE_ANY, (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } @@ -162557,7 +217914,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_geometry_callback( ** Register a new 2nd-generation geometry function for use with the ** r-tree MATCH operator. */ -SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( +SQLITE_API int sqlite3_rtree_query_callback( sqlite3 *db, /* Register SQL function on this connection */ const char *zQueryFunc, /* Name of new SQL function */ int (*xQueryFunc)(sqlite3_rtree_query_info*), /* Callback */ @@ -162568,12 +217925,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( /* Allocate and populate the context object. */ pGeomCtx = (RtreeGeomCallback *)sqlite3_malloc(sizeof(RtreeGeomCallback)); - if( !pGeomCtx ) return SQLITE_NOMEM; + if( !pGeomCtx ){ + if( xDestructor ) xDestructor(pContext); + return SQLITE_NOMEM; + } pGeomCtx->xGeom = 0; pGeomCtx->xQueryFunc = xQueryFunc; pGeomCtx->xDestructor = xDestructor; pGeomCtx->pContext = pContext; - return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, + return sqlite3_create_function_v2(db, zQueryFunc, -1, SQLITE_ANY, (void *)pGeomCtx, geomCallback, 0, 0, rtreeFreeCallback ); } @@ -162582,7 +217942,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_query_callback( #ifdef _WIN32 __declspec(dllexport) #endif -SQLITE_API int SQLITE_STDCALL sqlite3_rtree_init( +SQLITE_API int sqlite3_rtree_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -162609,9 +217969,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_init( ************************************************************************* ** $Id: icu.c,v 1.7 2007/12/13 21:54:11 drh Exp $ ** -** This file implements an integration between the ICU library -** ("International Components for Unicode", an open-source library -** for handling unicode data) and SQLite. The integration uses +** This file implements an integration between the ICU library +** ("International Components for Unicode", an open-source library +** for handling unicode data) and SQLite. The integration uses ** ICU to provide the following to SQLite: ** ** * An implementation of the SQL regexp() function (and hence REGEXP @@ -162622,11 +217982,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_init( ** ** * Integration of ICU and SQLite collation sequences. ** -** * An implementation of the LIKE operator that uses ICU to +** * An implementation of the LIKE operator that uses ICU to ** provide case-independent matching. */ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) +#if !defined(SQLITE_CORE) \ + || defined(SQLITE_ENABLE_ICU) \ + || defined(SQLITE_ENABLE_ICU_COLLATIONS) /* Include ICU headers */ #include @@ -162643,6 +218005,26 @@ SQLITE_API int SQLITE_STDCALL sqlite3_rtree_init( /* #include "sqlite3.h" */ #endif +/* +** This function is called when an ICU function called from within +** the implementation of an SQL scalar function returns an error. +** +** The scalar function context passed as the first argument is +** loaded with an error message based on the following two args. +*/ +static void icuFunctionError( + sqlite3_context *pCtx, /* SQLite scalar function context */ + const char *zName, /* Name of ICU function that failed */ + UErrorCode e /* Error code returned by ICU function */ +){ + char zBuf[128]; + sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); + zBuf[127] = '\0'; + sqlite3_result_error(pCtx, zBuf, -1); +} + +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) + /* ** Maximum length (in bytes) of the pattern in a LIKE or GLOB ** operator. @@ -162692,7 +218074,7 @@ static const unsigned char icuUtf8Trans1[] = { /* ** Compare two UTF-8 strings for equality where the first string is -** a "LIKE" expression. Return true (1) if they are the same and +** a "LIKE" expression. Return true (1) if they are the same and ** false (0) if they are different. */ static int icuLikeCompare( @@ -162700,15 +218082,15 @@ static int icuLikeCompare( const uint8_t *zString, /* The UTF-8 string to compare against */ const UChar32 uEsc /* The escape character */ ){ - static const int MATCH_ONE = (UChar32)'_'; - static const int MATCH_ALL = (UChar32)'%'; + static const uint32_t MATCH_ONE = (uint32_t)'_'; + static const uint32_t MATCH_ALL = (uint32_t)'%'; int prevEscape = 0; /* True if the previous character was uEsc */ while( 1 ){ /* Read (and consume) the next character from the input pattern. */ - UChar32 uPattern; + uint32_t uPattern; SQLITE_ICU_READ_UTF8(zPattern, uPattern); if( uPattern==0 ) break; @@ -162719,12 +218101,12 @@ static int icuLikeCompare( ** 3. uPattern is an unescaped escape character, or ** 4. uPattern is to be handled as an ordinary character */ - if( !prevEscape && uPattern==MATCH_ALL ){ + if( uPattern==MATCH_ALL && !prevEscape && uPattern!=(uint32_t)uEsc ){ /* Case 1. */ uint8_t c; /* Skip any MATCH_ALL or MATCH_ONE characters that follow a - ** MATCH_ALL. For each MATCH_ONE, skip one character in the + ** MATCH_ALL. For each MATCH_ONE, skip one character in the ** test string. */ while( (c=*zPattern) == MATCH_ALL || c == MATCH_ONE ){ @@ -162745,21 +218127,21 @@ static int icuLikeCompare( } return 0; - }else if( !prevEscape && uPattern==MATCH_ONE ){ + }else if( uPattern==MATCH_ONE && !prevEscape && uPattern!=(uint32_t)uEsc ){ /* Case 2. */ if( *zString==0 ) return 0; SQLITE_ICU_SKIP_UTF8(zString); - }else if( !prevEscape && uPattern==uEsc){ + }else if( uPattern==(uint32_t)uEsc && !prevEscape ){ /* Case 3. */ prevEscape = 1; }else{ /* Case 4. */ - UChar32 uString; + uint32_t uString; SQLITE_ICU_READ_UTF8(zString, uString); - uString = u_foldCase(uString, U_FOLD_CASE_DEFAULT); - uPattern = u_foldCase(uPattern, U_FOLD_CASE_DEFAULT); + uString = (uint32_t)u_foldCase((UChar32)uString, U_FOLD_CASE_DEFAULT); + uPattern = (uint32_t)u_foldCase((UChar32)uPattern, U_FOLD_CASE_DEFAULT); if( uString!=uPattern ){ return 0; } @@ -162777,15 +218159,15 @@ static int icuLikeCompare( ** ** A LIKE B ** -** is implemented as like(B, A). If there is an escape character E, +** is implemented as like(B, A). If there is an escape character E, ** ** A LIKE B ESCAPE E ** ** is mapped to like(B, A, E). */ static void icuLikeFunc( - sqlite3_context *context, - int argc, + sqlite3_context *context, + int argc, sqlite3_value **argv ){ const unsigned char *zA = sqlite3_value_text(argv[0]); @@ -162811,7 +218193,7 @@ static void icuLikeFunc( if( zE==0 ) return; U8_NEXT(zE, i, nE, uEsc); if( i!=nE){ - sqlite3_result_error(context, + sqlite3_result_error(context, "ESCAPE expression must be a single character", -1); return; } @@ -162822,24 +218204,6 @@ static void icuLikeFunc( } } -/* -** This function is called when an ICU function called from within -** the implementation of an SQL scalar function returns an error. -** -** The scalar function context passed as the first argument is -** loaded with an error message based on the following two args. -*/ -static void icuFunctionError( - sqlite3_context *pCtx, /* SQLite scalar function context */ - const char *zName, /* Name of ICU function that failed */ - UErrorCode e /* Error code returned by ICU function */ -){ - char zBuf[128]; - sqlite3_snprintf(128, zBuf, "ICU error: %s(): %s", zName, u_errorName(e)); - zBuf[127] = '\0'; - sqlite3_result_error(pCtx, zBuf, -1); -} - /* ** Function to delete compiled regexp objects. Registered as ** a destructor function with sqlite3_set_auxdata(). @@ -162852,7 +218216,7 @@ static void icuRegexpDelete(void *p){ /* ** Implementation of SQLite REGEXP operator. This scalar function takes ** two arguments. The first is a regular expression pattern to compile -** the second is a string to match against that pattern. If either +** the second is a string to match against that pattern. If either ** argument is an SQL NULL, then NULL Is returned. Otherwise, the result ** is 1 if the string matches the pattern, or 0 otherwise. ** @@ -162876,8 +218240,8 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ (void)nArg; /* Unused parameter */ - /* If the left hand side of the regexp operator is NULL, - ** then the result is also NULL. + /* If the left hand side of the regexp operator is NULL, + ** then the result is also NULL. */ if( !zString ){ return; @@ -162893,8 +218257,9 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ if( U_SUCCESS(status) ){ sqlite3_set_auxdata(p, 0, pExpr, icuRegexpDelete); - }else{ - assert(!pExpr); + pExpr = sqlite3_get_auxdata(p, 0); + } + if( !pExpr ){ icuFunctionError(p, "uregex_open", status); return; } @@ -162915,7 +218280,7 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ } /* Set the text that the regular expression operates on to a NULL - ** pointer. This is not really necessary, but it is tidier than + ** pointer. This is not really necessary, but it is tidier than ** leaving the regular expression object configured with an invalid ** pointer after this function returns. */ @@ -162926,7 +218291,7 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ } /* -** Implementations of scalar functions for case mapping - upper() and +** Implementations of scalar functions for case mapping - upper() and ** lower(). Function upper() converts its input to upper-case (ABC). ** Function lower() converts to lower-case (abc). ** @@ -162934,7 +218299,7 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ** "language specific". Refer to ICU documentation for the differences ** between the two. ** -** To utilise "general" case mapping, the upper() or lower() scalar +** To utilise "general" case mapping, the upper() or lower() scalar ** functions are invoked with one argument: ** ** upper('ABC') -> 'abc' @@ -162947,7 +218312,7 @@ static void icuRegexpFunc(sqlite3_context *p, int nArg, sqlite3_value **apArg){ ** of upper() or lower(). ** ** lower('I', 'en_us') -> 'i' -** lower('I', 'tr_tr') -> 'ı' (small dotless i) +** lower('I', 'tr_tr') -> '\u131' (small dotless i) ** ** http://www.icu-project.org/userguide/posix.html#case_mappings */ @@ -163005,6 +218370,8 @@ static void icuCaseFunc16(sqlite3_context *p, int nArg, sqlite3_value **apArg){ assert( 0 ); /* Unreachable */ } +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_ICU) */ + /* ** Collation sequence destructor function. The pCtx argument points to ** a UCollator structure previously allocated using ucol_open(). @@ -163040,7 +218407,7 @@ static int icuCollationColl( /* ** Implementation of the scalar function icu_load_collation(). ** -** This scalar function is used to add ICU collation based collation +** This scalar function is used to add ICU collation based collation ** types to an SQLite database connection. It is intended to be called ** as follows: ** @@ -163051,8 +218418,8 @@ static int icuCollationColl( ** collation sequence to create. */ static void icuLoadCollation( - sqlite3_context *p, - int nArg, + sqlite3_context *p, + int nArg, sqlite3_value **apArg ){ sqlite3 *db = (sqlite3 *)sqlite3_user_data(p); @@ -163062,7 +218429,7 @@ static void icuLoadCollation( UCollator *pUCollator; /* ICU library collation object */ int rc; /* Return code from sqlite3_create_collation_x() */ - assert(nArg==2); + assert(nArg==2 || nArg==3); (void)nArg; /* Unused parameter */ zLocale = (const char *)sqlite3_value_text(apArg[0]); zName = (const char *)sqlite3_value_text(apArg[1]); @@ -163077,8 +218444,40 @@ static void icuLoadCollation( return; } assert(p); - - rc = sqlite3_create_collation_v2(db, zName, SQLITE_UTF16, (void *)pUCollator, + if(nArg==3){ + const char *zOption = (const char*)sqlite3_value_text(apArg[2]); + static const struct { + const char *zName; + UColAttributeValue val; + } aStrength[] = { + { "PRIMARY", UCOL_PRIMARY }, + { "SECONDARY", UCOL_SECONDARY }, + { "TERTIARY", UCOL_TERTIARY }, + { "DEFAULT", UCOL_DEFAULT_STRENGTH }, + { "QUARTERNARY", UCOL_QUATERNARY }, + { "IDENTICAL", UCOL_IDENTICAL }, + }; + unsigned int i; + for(i=0; i=sizeof(aStrength)/sizeof(aStrength[0]) ){ + sqlite3_str *pStr = sqlite3_str_new(sqlite3_context_db_handle(p)); + sqlite3_str_appendf(pStr, + "unknown collation strength \"%s\" - should be one of:", + zOption); + for(i=0; izName, p->nArg, p->enc, p->pContext, p->xFunc, 0, 0 + db, p->zName, p->nArg, p->enc, + p->iContext ? (void*)db : (void*)0, + p->xFunc, 0, 0 ); } @@ -163133,8 +218533,8 @@ SQLITE_PRIVATE int sqlite3IcuInit(sqlite3 *db){ #ifdef _WIN32 __declspec(dllexport) #endif -SQLITE_API int SQLITE_STDCALL sqlite3_icu_init( - sqlite3 *db, +SQLITE_API int sqlite3_icu_init( + sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi ){ @@ -163209,7 +218609,7 @@ static int icuCreate( if( argc>0 ){ n = strlen(argv[0])+1; } - p = (IcuTokenizer *)sqlite3_malloc(sizeof(IcuTokenizer)+n); + p = (IcuTokenizer *)sqlite3_malloc64(sizeof(IcuTokenizer)+n); if( !p ){ return SQLITE_NOMEM; } @@ -163237,7 +218637,7 @@ static int icuDestroy(sqlite3_tokenizer *pTokenizer){ /* ** Prepare to begin tokenizing a particular string. The input ** string to be tokenized is pInput[0..nBytes-1]. A cursor -** used to incrementally tokenize this string is returned in +** used to incrementally tokenize this string is returned in ** *ppCursor. */ static int icuOpen( @@ -163266,7 +218666,7 @@ static int icuOpen( nInput = strlen(zInput); } nChar = nInput+1; - pCsr = (IcuCursor *)sqlite3_malloc( + pCsr = (IcuCursor *)sqlite3_malloc64( sizeof(IcuCursor) + /* IcuCursor */ ((nChar+3)&~3) * sizeof(UChar) + /* IcuCursor.aChar[] */ (nChar+1) * sizeof(int) /* IcuCursor.aOffset[] */ @@ -163279,7 +218679,7 @@ static int icuOpen( pCsr->aOffset = (int *)&pCsr->aChar[(nChar+3)&~3]; pCsr->aOffset[iOut] = iInput; - U8_NEXT(zInput, iInput, nInput, c); + U8_NEXT(zInput, iInput, nInput, c); while( c>0 ){ int isError = 0; c = u_foldCase(c, opt); @@ -163425,7 +218825,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ************************************************************************* ** ** -** OVERVIEW +** OVERVIEW ** ** The RBU extension requires that the RBU update be packaged as an ** SQLite database. The tables it expects to find are described in @@ -163433,34 +218833,34 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** that the user wishes to write to, a corresponding data_xyz table is ** created in the RBU database and populated with one row for each row to ** update, insert or delete from the target table. -** +** ** The update proceeds in three stages: -** +** ** 1) The database is updated. The modified database pages are written ** to a *-oal file. A *-oal file is just like a *-wal file, except ** that it is named "-oal" instead of "-wal". ** Because regular SQLite clients do not look for file named ** "-oal", they go on using the original database in ** rollback mode while the *-oal file is being generated. -** +** ** During this stage RBU does not update the database by writing ** directly to the target tables. Instead it creates "imposter" ** tables using the SQLITE_TESTCTRL_IMPOSTER interface that it uses ** to update each b-tree individually. All updates required by each ** b-tree are completed before moving on to the next, and all ** updates are done in sorted key order. -** +** ** 2) The "-oal" file is moved to the equivalent "-wal" ** location using a call to rename(2). Before doing this the RBU ** module takes an EXCLUSIVE lock on the database file, ensuring ** that there are no other active readers. -** +** ** Once the EXCLUSIVE lock is released, any other database readers ** detect the new *-wal file and read the database in wal mode. At ** this point they see the new version of the database - including ** the updates made as part of the RBU update. -** -** 3) The new *-wal file is checkpointed. This proceeds in the same way +** +** 3) The new *-wal file is checkpointed. This proceeds in the same way ** as a regular database checkpoint, except that a single frame is ** checkpointed each time sqlite3rbu_step() is called. If the RBU ** handle is closed before the entire *-wal file is checkpointed, @@ -163469,7 +218869,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** the future. ** ** POTENTIAL PROBLEMS -** +** ** The rename() call might not be portable. And RBU is not currently ** syncing the directory after renaming the file. ** @@ -163491,7 +218891,7 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** fields are collected. This means we're probably writing a lot more ** data to disk when saving the state of an ongoing update to the RBU ** update database than is strictly necessary. -** +** */ /* #include */ @@ -163515,42 +218915,42 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ************************************************************************* ** -** This file contains the public interface for the RBU extension. +** This file contains the public interface for the RBU extension. */ /* ** SUMMARY ** -** Writing a transaction containing a large number of operations on +** Writing a transaction containing a large number of operations on ** b-tree indexes that are collectively larger than the available cache -** memory can be very inefficient. +** memory can be very inefficient. ** ** The problem is that in order to update a b-tree, the leaf page (at least) ** containing the entry being inserted or deleted must be modified. If the -** working set of leaves is larger than the available cache memory, then a -** single leaf that is modified more than once as part of the transaction +** working set of leaves is larger than the available cache memory, then a +** single leaf that is modified more than once as part of the transaction ** may be loaded from or written to the persistent media multiple times. ** Additionally, because the index updates are likely to be applied in -** random order, access to pages within the database is also likely to be in +** random order, access to pages within the database is also likely to be in ** random order, which is itself quite inefficient. ** ** One way to improve the situation is to sort the operations on each index ** by index key before applying them to the b-tree. This leads to an IO ** pattern that resembles a single linear scan through the index b-tree, -** and all but guarantees each modified leaf page is loaded and stored +** and all but guarantees each modified leaf page is loaded and stored ** exactly once. SQLite uses this trick to improve the performance of ** CREATE INDEX commands. This extension allows it to be used to improve ** the performance of large transactions on existing databases. ** -** Additionally, this extension allows the work involved in writing the -** large transaction to be broken down into sub-transactions performed -** sequentially by separate processes. This is useful if the system cannot -** guarantee that a single update process will run for long enough to apply -** the entire update, for example because the update is being applied on a -** mobile device that is frequently rebooted. Even after the writer process +** Additionally, this extension allows the work involved in writing the +** large transaction to be broken down into sub-transactions performed +** sequentially by separate processes. This is useful if the system cannot +** guarantee that a single update process will run for long enough to apply +** the entire update, for example because the update is being applied on a +** mobile device that is frequently rebooted. Even after the writer process ** has committed one or more sub-transactions, other database clients continue -** to read from the original database snapshot. In other words, partially -** applied transactions are not visible to other clients. +** to read from the original database snapshot. In other words, partially +** applied transactions are not visible to other clients. ** ** "RBU" stands for "Resumable Bulk Update". As in a large database update ** transmitted via a wireless network to a mobile device. A transaction @@ -163566,9 +218966,9 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** * INSERT statements may not use any default values. ** -** * UPDATE and DELETE statements must identify their target rows by +** * UPDATE and DELETE statements must identify their target rows by ** non-NULL PRIMARY KEY values. Rows with NULL values stored in PRIMARY -** KEY fields may not be updated or deleted. If the table being written +** KEY fields may not be updated or deleted. If the table being written ** has no PRIMARY KEY, affected rows must be identified by rowid. ** ** * UPDATE statements may not modify PRIMARY KEY columns. @@ -163585,10 +218985,10 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** PREPARATION ** ** An "RBU update" is stored as a separate SQLite database. A database -** containing an RBU update is an "RBU database". For each table in the +** containing an RBU update is an "RBU database". For each table in the ** target database to be updated, the RBU database should contain a table ** named "data_" containing the same set of columns as the -** target table, and one more - "rbu_control". The data_% table should +** target table, and one more - "rbu_control". The data_% table should ** have no PRIMARY KEY or UNIQUE constraints, but each column should have ** the same type as the corresponding column in the target database. ** The "rbu_control" column should have no type at all. For example, if @@ -163603,22 +219003,22 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** The order of the columns in the data_% table does not matter. ** ** Instead of a regular table, the RBU database may also contain virtual -** tables or view named using the data_ naming scheme. +** tables or views named using the data_ naming scheme. ** -** Instead of the plain data_ naming scheme, RBU database tables +** Instead of the plain data_ naming scheme, RBU database tables ** may also be named data_, where is any sequence ** of zero or more numeric characters (0-9). This can be significant because -** tables within the RBU database are always processed in order sorted by -** name. By judicious selection of the the portion of the names +** tables within the RBU database are always processed in order sorted by +** name. By judicious selection of the portion of the names ** of the RBU tables the user can therefore control the order in which they ** are processed. This can be useful, for example, to ensure that "external ** content" FTS4 tables are updated before their underlying content tables. ** ** If the target database table is a virtual table or a table that has no -** PRIMARY KEY declaration, the data_% table must also contain a column -** named "rbu_rowid". This column is mapped to the tables implicit primary -** key column - "rowid". Virtual tables for which the "rowid" column does -** not function like a primary key value cannot be updated using RBU. For +** PRIMARY KEY declaration, the data_% table must also contain a column +** named "rbu_rowid". This column is mapped to the table's implicit primary +** key column - "rowid". Virtual tables for which the "rowid" column does +** not function like a primary key value cannot be updated using RBU. For ** example, if the target db contains either of the following: ** ** CREATE VIRTUAL TABLE x1 USING fts3(a, b); @@ -163641,35 +219041,35 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** CREATE TABLE data_ft1(a, b, langid, rbu_rowid, rbu_control); ** CREATE TABLE data_ft1(a, b, rbu_rowid, rbu_control); ** -** For each row to INSERT into the target database as part of the RBU +** For each row to INSERT into the target database as part of the RBU ** update, the corresponding data_% table should contain a single record ** with the "rbu_control" column set to contain integer value 0. The -** other columns should be set to the values that make up the new record -** to insert. +** other columns should be set to the values that make up the new record +** to insert. ** -** If the target database table has an INTEGER PRIMARY KEY, it is not -** possible to insert a NULL value into the IPK column. Attempting to +** If the target database table has an INTEGER PRIMARY KEY, it is not +** possible to insert a NULL value into the IPK column. Attempting to ** do so results in an SQLITE_MISMATCH error. ** -** For each row to DELETE from the target database as part of the RBU +** For each row to DELETE from the target database as part of the RBU ** update, the corresponding data_% table should contain a single record ** with the "rbu_control" column set to contain integer value 1. The ** real primary key values of the row to delete should be stored in the ** corresponding columns of the data_% table. The values stored in the ** other columns are not used. ** -** For each row to UPDATE from the target database as part of the RBU +** For each row to UPDATE from the target database as part of the RBU ** update, the corresponding data_% table should contain a single record ** with the "rbu_control" column set to contain a value of type text. -** The real primary key values identifying the row to update should be +** The real primary key values identifying the row to update should be ** stored in the corresponding columns of the data_% table row, as should -** the new values of all columns being update. The text value in the +** the new values of all columns being update. The text value in the ** "rbu_control" column must contain the same number of characters as ** there are columns in the target database table, and must consist entirely -** of 'x' and '.' characters (or in some special cases 'd' - see below). For +** of 'x' and '.' characters (or in some special cases 'd' - see below). For ** each column that is being updated, the corresponding character is set to ** 'x'. For those that remain as they are, the corresponding character of the -** rbu_control value should be set to '.'. For example, given the tables +** rbu_control value should be set to '.'. For example, given the tables ** above, the update statement: ** ** UPDATE t1 SET c = 'usa' WHERE a = 4; @@ -163683,30 +219083,30 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** target table with the value stored in the corresponding data_% column, the ** user-defined SQL function "rbu_delta()" is invoked and the result stored in ** the target table column. rbu_delta() is invoked with two arguments - the -** original value currently stored in the target table column and the +** original value currently stored in the target table column and the ** value specified in the data_xxx table. ** ** For example, this row: ** ** INSERT INTO data_t1(a, b, c, rbu_control) VALUES(4, NULL, 'usa', '..d'); ** -** is similar to an UPDATE statement such as: +** is similar to an UPDATE statement such as: ** ** UPDATE t1 SET c = rbu_delta(c, 'usa') WHERE a = 4; ** -** Finally, if an 'f' character appears in place of a 'd' or 's' in an +** Finally, if an 'f' character appears in place of a 'd' or 's' in an ** ota_control string, the contents of the data_xxx table column is assumed ** to be a "fossil delta" - a patch to be applied to a blob value in the ** format used by the fossil source-code management system. In this case -** the existing value within the target database table must be of type BLOB. +** the existing value within the target database table must be of type BLOB. ** It is replaced by the result of applying the specified fossil delta to ** itself. ** ** If the target database table is a virtual table or a table with no PRIMARY -** KEY, the rbu_control value should not include a character corresponding +** KEY, the rbu_control value should not include a character corresponding ** to the rbu_rowid value. For example, this: ** -** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) +** INSERT INTO data_ft1(a, b, rbu_rowid, rbu_control) ** VALUES(NULL, 'usa', 12, '.x'); ** ** causes a result similar to: @@ -163716,14 +219116,14 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** The data_xxx tables themselves should have no PRIMARY KEY declarations. ** However, RBU is more efficient if reading the rows in from each data_xxx ** table in "rowid" order is roughly the same as reading them sorted by -** the PRIMARY KEY of the corresponding target database table. In other -** words, rows should be sorted using the destination table PRIMARY KEY +** the PRIMARY KEY of the corresponding target database table. In other +** words, rows should be sorted using the destination table PRIMARY KEY ** fields before they are inserted into the data_xxx tables. ** ** USAGE ** -** The API declared below allows an application to apply an RBU update -** stored on disk to an existing target database. Essentially, the +** The API declared below allows an application to apply an RBU update +** stored on disk to an existing target database. Essentially, the ** application: ** ** 1) Opens an RBU handle using the sqlite3rbu_open() function. @@ -163734,24 +219134,24 @@ SQLITE_PRIVATE void sqlite3Fts3IcuTokenizerModule( ** ** 3) Calls the sqlite3rbu_step() function one or more times on ** the new handle. Each call to sqlite3rbu_step() performs a single -** b-tree operation, so thousands of calls may be required to apply +** b-tree operation, so thousands of calls may be required to apply ** a complete update. ** ** 4) Calls sqlite3rbu_close() to close the RBU update handle. If ** sqlite3rbu_step() has been called enough times to completely ** apply the update to the target database, then the RBU database -** is marked as fully applied. Otherwise, the state of the RBU -** update application is saved in the RBU database for later +** is marked as fully applied. Otherwise, the state of the RBU +** update application is saved in the RBU database for later ** resumption. ** ** See comments below for more detail on APIs. ** ** If an update is only partially applied to the target database by the -** time sqlite3rbu_close() is called, various state information is saved +** time sqlite3rbu_close() is called, various state information is saved ** within the RBU database. This allows subsequent processes to automatically ** resume the RBU update from where it left off. ** -** To remove all RBU extension state information, returning an RBU database +** To remove all RBU extension state information, returning an RBU database ** to its original contents, it is sufficient to drop all tables that begin ** with the prefix "rbu_" ** @@ -163787,21 +219187,21 @@ typedef struct sqlite3rbu sqlite3rbu; ** the path to the RBU database. Each call to this function must be matched ** by a call to sqlite3rbu_close(). When opening the databases, RBU passes ** the SQLITE_CONFIG_URI flag to sqlite3_open_v2(). So if either zTarget -** or zRbu begin with "file:", it will be interpreted as an SQLite +** or zRbu begin with "file:", it will be interpreted as an SQLite ** database URI, not a regular file name. ** -** If the zState argument is passed a NULL value, the RBU extension stores -** the current state of the update (how many rows have been updated, which +** If the zState argument is passed a NULL value, the RBU extension stores +** the current state of the update (how many rows have been updated, which ** indexes are yet to be updated etc.) within the RBU database itself. This ** can be convenient, as it means that the RBU application does not need to -** organize removing a separate state file after the update is concluded. -** Or, if zState is non-NULL, it must be a path to a database file in which +** organize removing a separate state file after the update is concluded. +** Or, if zState is non-NULL, it must be a path to a database file in which ** the RBU extension can store the state of the update. ** ** When resuming an RBU update, the zState argument must be passed the same ** value as when the RBU update was started. ** -** Once the RBU update is finished, the RBU extension does not +** Once the RBU update is finished, the RBU extension does not ** automatically remove any zState database file, even if it created it. ** ** By default, RBU uses the default VFS to access the files on disk. To @@ -163813,8 +219213,8 @@ typedef struct sqlite3rbu sqlite3rbu; ** not work out of the box with zipvfs. Refer to the comment describing ** the zipvfs_create_vfs() API below for details on using RBU with zipvfs. */ -SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( - const char *zTarget, +SQLITE_API sqlite3rbu *sqlite3rbu_open( + const char *zTarget, const char *zRbu, const char *zState ); @@ -163824,35 +219224,67 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( ** An RBU vacuum is similar to SQLite's built-in VACUUM command, except ** that it can be suspended and resumed like an RBU update. ** -** The second argument to this function, which may not be NULL, identifies -** a database in which to store the state of the RBU vacuum operation if -** it is suspended. The first time sqlite3rbu_vacuum() is called, to start -** an RBU vacuum operation, the state database should either not exist or -** be empty (contain no tables). If an RBU vacuum is suspended by calling +** The second argument to this function identifies a database in which +** to store the state of the RBU vacuum operation if it is suspended. The +** first time sqlite3rbu_vacuum() is called, to start an RBU vacuum +** operation, the state database should either not exist or be empty +** (contain no tables). If an RBU vacuum is suspended by calling ** sqlite3rbu_close() on the RBU handle before sqlite3rbu_step() has -** returned SQLITE_DONE, the vacuum state is stored in the state database. +** returned SQLITE_DONE, the vacuum state is stored in the state database. ** The vacuum can be resumed by calling this function to open a new RBU ** handle specifying the same target and state databases. ** +** If the second argument passed to this function is NULL, then the +** name of the state database is "-vacuum", where +** is the name of the target database file. In this case, on UNIX, if the +** state database is not already present in the file-system, it is created +** with the same permissions as the target db is made. +** +** With an RBU vacuum, it is an SQLITE_MISUSE error if the name of the +** state database ends with "-vactmp". This name is reserved for internal +** use. +** ** This function does not delete the state database after an RBU vacuum ** is completed, even if it created it. However, if the call to ** sqlite3rbu_close() returns any value other than SQLITE_OK, the contents ** of the state tables within the state database are zeroed. This way, -** the next call to sqlite3rbu_vacuum() opens a handle that starts a +** the next call to sqlite3rbu_vacuum() opens a handle that starts a ** new RBU vacuum operation. ** ** As with sqlite3rbu_open(), Zipvfs users should rever to the comment -** describing the sqlite3rbu_create_vfs() API function below for -** a description of the complications associated with using RBU with +** describing the sqlite3rbu_create_vfs() API function below for +** a description of the complications associated with using RBU with ** zipvfs databases. */ -SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_vacuum( - const char *zTarget, +SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, const char *zState ); /* -** Internally, each RBU connection uses a separate SQLite database +** Configure a limit for the amount of temp space that may be used by +** the RBU handle passed as the first argument. The new limit is specified +** in bytes by the second parameter. If it is positive, the limit is updated. +** If the second parameter to this function is passed zero, then the limit +** is removed entirely. If the second parameter is negative, the limit is +** not modified (this is useful for querying the current limit). +** +** In all cases the returned value is the current limit in bytes (zero +** indicates unlimited). +** +** If the temp space limit is exceeded during operation, an SQLITE_FULL +** error is returned. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu*, sqlite3_int64); + +/* +** Return the current amount of temp file space, in bytes, currently used by +** the RBU handle passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu*); + +/* +** Internally, each RBU connection uses a separate SQLite database ** connection to access the target and rbu update databases. This ** API allows the application direct access to these database handles. ** @@ -163863,10 +219295,10 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_vacuum( ** following scenarios: ** ** * If any target tables are virtual tables, it may be necessary to -** call sqlite3_create_module() on the target database handle to +** call sqlite3_create_module() on the target database handle to ** register the required virtual table implementations. ** -** * If the data_xxx tables in the RBU source database are virtual +** * If the data_xxx tables in the RBU source database are virtual ** tables, the application may need to call sqlite3_create_module() on ** the rbu update db handle to any required virtual table ** implementations. @@ -163882,36 +219314,36 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_vacuum( ** Database handles returned by this function remain valid until the next ** call to any sqlite3rbu_xxx() function other than sqlite3rbu_db(). */ -SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu*, int bRbu); +SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu*, int bRbu); /* -** Do some work towards applying the RBU update to the target db. +** Do some work towards applying the RBU update to the target db. ** -** Return SQLITE_DONE if the update has been completely applied, or +** Return SQLITE_DONE if the update has been completely applied, or ** SQLITE_OK if no error occurs but there remains work to do to apply -** the RBU update. If an error does occur, some other error code is -** returned. +** the RBU update. If an error does occur, some other error code is +** returned. ** ** Once a call to sqlite3rbu_step() has returned a value other than ** SQLITE_OK, all subsequent calls on the same RBU handle are no-ops ** that immediately return the same value. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *pRbu); +SQLITE_API int sqlite3rbu_step(sqlite3rbu *pRbu); /* ** Force RBU to save its state to disk. ** ** If a power failure or application crash occurs during an update, following ** system recovery RBU may resume the update from the point at which the state -** was last saved. In other words, from the most recent successful call to +** was last saved. In other words, from the most recent successful call to ** sqlite3rbu_close() or this function. ** ** SQLITE_OK is returned if successful, or an SQLite error code otherwise. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *pRbu); +SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *pRbu); /* -** Close an RBU handle. +** Close an RBU handle. ** ** If the RBU update has been completely applied, mark the RBU database ** as fully applied. Otherwise, assuming no error has occurred, save the @@ -163919,26 +219351,26 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *pRbu); ** ** If an error has already occurred as part of an sqlite3rbu_step() ** or sqlite3rbu_open() call, or if one occurs within this function, an -** SQLite error code is returned. Additionally, *pzErrmsg may be set to -** point to a buffer containing a utf-8 formatted English language error -** message. It is the responsibility of the caller to eventually free any -** such buffer using sqlite3_free(). +** SQLite error code is returned. Additionally, if pzErrmsg is not NULL, +** *pzErrmsg may be set to point to a buffer containing a utf-8 formatted +** English language error message. It is the responsibility of the caller to +** eventually free any such buffer using sqlite3_free(). ** ** Otherwise, if no error occurs, this function returns SQLITE_OK if the -** update has been partially applied, or SQLITE_DONE if it has been +** update has been partially applied, or SQLITE_DONE if it has been ** completely applied. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); +SQLITE_API int sqlite3rbu_close(sqlite3rbu *pRbu, char **pzErrmsg); /* -** Return the total number of key-value operations (inserts, deletes or +** Return the total number of key-value operations (inserts, deletes or ** updates) that have been performed on the target database since the ** current RBU update was started. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu); +SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu); /* -** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) +** Obtain permyriadage (permyriadage is to 10000 as percentage is to 100) ** progress indications for the two stages of an RBU update. This API may ** be useful for driving GUI progress indicators and similar. ** @@ -163951,16 +219383,16 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu); ** The update is visible to non-RBU clients during stage 2. During stage 1 ** non-RBU reader clients may see the original database. ** -** If this API is called during stage 2 of the update, output variable +** If this API is called during stage 2 of the update, output variable ** (*pnOne) is set to 10000 to indicate that stage 1 has finished and (*pnTwo) ** to a value between 0 and 10000 to indicate the permyriadage progress of -** stage 2. A value of 5000 indicates that stage 2 is half finished, +** stage 2. A value of 5000 indicates that stage 2 is half finished, ** 9000 indicates that it is 90% finished, and so on. ** -** If this API is called during stage 1 of the update, output variable +** If this API is called during stage 1 of the update, output variable ** (*pnTwo) is set to 0 to indicate that stage 2 has not yet started. The -** value to which (*pnOne) is set depends on whether or not the RBU -** database contains an "rbu_count" table. The rbu_count table, if it +** value to which (*pnOne) is set depends on whether or not the RBU +** database contains an "rbu_count" table. The rbu_count table, if it ** exists, must contain the same columns as the following: ** ** CREATE TABLE rbu_count(tbl TEXT PRIMARY KEY, cnt INTEGER) WITHOUT ROWID; @@ -163977,24 +219409,90 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu); ** table exists but is not correctly populated, the value of the *pnOne ** output variable during stage 1 is undefined. */ -SQLITE_API void SQLITE_STDCALL sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int *pnTwo); +SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnOne, int*pnTwo); + +/* +** Obtain an indication as to the current stage of an RBU update or vacuum. +** This function always returns one of the SQLITE_RBU_STATE_XXX constants +** defined in this file. Return values should be interpreted as follows: +** +** SQLITE_RBU_STATE_OAL: +** RBU is currently building a *-oal file. The next call to sqlite3rbu_step() +** may either add further data to the *-oal file, or compute data that will +** be added by a subsequent call. +** +** SQLITE_RBU_STATE_MOVE: +** RBU has finished building the *-oal file. The next call to sqlite3rbu_step() +** will move the *-oal file to the equivalent *-wal path. If the current +** operation is an RBU update, then the updated version of the database +** file will become visible to ordinary SQLite clients following the next +** call to sqlite3rbu_step(). +** +** SQLITE_RBU_STATE_CHECKPOINT: +** RBU is currently performing an incremental checkpoint. The next call to +** sqlite3rbu_step() will copy a page of data from the *-wal file into +** the target database file. +** +** SQLITE_RBU_STATE_DONE: +** The RBU operation has finished. Any subsequent calls to sqlite3rbu_step() +** will immediately return SQLITE_DONE. +** +** SQLITE_RBU_STATE_ERROR: +** An error has occurred. Any subsequent calls to sqlite3rbu_step() will +** immediately return the SQLite error code associated with the error. +*/ +#define SQLITE_RBU_STATE_OAL 1 +#define SQLITE_RBU_STATE_MOVE 2 +#define SQLITE_RBU_STATE_CHECKPOINT 3 +#define SQLITE_RBU_STATE_DONE 4 +#define SQLITE_RBU_STATE_ERROR 5 + +SQLITE_API int sqlite3rbu_state(sqlite3rbu *pRbu); + +/* +** As part of applying an RBU update or performing an RBU vacuum operation, +** the system must at one point move the *-oal file to the equivalent *-wal +** path. Normally, it does this by invoking POSIX function rename(2) directly. +** Except on WINCE platforms, where it uses win32 API MoveFileW(). This +** function may be used to register a callback that the RBU module will invoke +** instead of one of these APIs. +** +** If a callback is registered with an RBU handle, it invokes it instead +** of rename(2) when it needs to move a file within the file-system. The +** first argument passed to the xRename() callback is a copy of the second +** argument (pArg) passed to this function. The second is the full path +** to the file to move and the third the full path to which it should be +** moved. The callback function should return SQLITE_OK to indicate +** success. If an error occurs, it should return an SQLite error code. +** In this case the RBU operation will be abandoned and the error returned +** to the RBU user. +** +** Passing a NULL pointer in place of the xRename argument to this function +** restores the default behaviour. +*/ +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +); + /* ** Create an RBU VFS named zName that accesses the underlying file-system -** via existing VFS zParent. Or, if the zParent parameter is passed NULL, +** via existing VFS zParent. Or, if the zParent parameter is passed NULL, ** then the new RBU VFS uses the default system VFS to access the file-system. -** The new object is registered as a non-default VFS with SQLite before +** The new object is registered as a non-default VFS with SQLite before ** returning. ** ** Part of the RBU implementation uses a custom VFS object. Usually, this -** object is created and deleted automatically by RBU. +** object is created and deleted automatically by RBU. ** ** The exception is for applications that also use zipvfs. In this case, ** the custom VFS must be explicitly created by the user before the RBU ** handle is opened. The RBU VFS should be installed so that the zipvfs -** VFS uses the RBU VFS, which in turn uses any other VFS layers in use +** VFS uses the RBU VFS, which in turn uses any other VFS layers in use ** (for example multiplexor) to access the file-system. For example, -** to assemble an RBU enabled VFS stack that uses both zipvfs and +** to assemble an RBU enabled VFS stack that uses both zipvfs and ** multiplexor (error checking omitted): ** ** // Create a VFS named "multiplex" (not the default). @@ -164016,12 +219514,12 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_bp_progress(sqlite3rbu *pRbu, int *pnO ** may be used by RBU clients. Attempting to use RBU with a zipvfs VFS stack ** that does not include the RBU layer results in an error. ** -** The overhead of adding the "rbu" VFS to the system is negligible for -** non-RBU users. There is no harm in an application accessing the -** file-system via "rbu" all the time, even if it only uses RBU functionality +** The overhead of adding the "rbu" VFS to the system is negligible for +** non-RBU users. There is no harm in an application accessing the +** file-system via "rbu" all the time, even if it only uses RBU functionality ** occasionally. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const char *zParent); +SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent); /* ** Deregister and destroy an RBU vfs created by an earlier call to @@ -164031,7 +219529,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha ** before all database handles that use it have been closed, the results ** are undefined. */ -SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); +SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName); #if 0 } /* end of the 'extern "C"' block */ @@ -164049,6 +219547,13 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); /* Maximum number of prepared UPDATE statements held by this module */ #define SQLITE_RBU_UPDATE_CACHESIZE 16 +/* Delta checksums disabled by default. Compile with -DRBU_ENABLE_DELTA_CKSUM +** to enable checksum verification. +*/ +#ifndef RBU_ENABLE_DELTA_CKSUM +# define RBU_ENABLE_DELTA_CKSUM 0 +#endif + /* ** Swap two objects of type TYPE. */ @@ -164056,6 +219561,13 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); # define SWAP(TYPE,A,B) {TYPE t=A; A=B; B=t;} #endif +/* +** Name of the URI option that causes RBU to take an exclusive lock as +** part of the incremental checkpoint operation. +*/ +#define RBU_EXCLUSIVE_CHECKPOINT "rbu_exclusive_checkpoint" + + /* ** The rbu_state table is used to save the state of a partially applied ** update so that it can be resumed later. The table consists of integer @@ -164064,17 +219576,17 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); ** RBU_STATE_STAGE: ** May be set to integer values 1, 2, 4 or 5. As follows: ** 1: the *-rbu file is currently under construction. -** 2: the *-rbu file has been constructed, but not yet moved +** 2: the *-rbu file has been constructed, but not yet moved ** to the *-wal path. ** 4: the checkpoint is underway. ** 5: the rbu update has been checkpointed. ** ** RBU_STATE_TBL: -** Only valid if STAGE==1. The target database name of the table +** Only valid if STAGE==1. The target database name of the table ** currently being written. ** ** RBU_STATE_IDX: -** Only valid if STAGE==1. The target database name of the index +** Only valid if STAGE==1. The target database name of the index ** currently being written, or NULL if the main table is currently being ** updated. ** @@ -164094,11 +219606,15 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); ** be continued if this happens). ** ** RBU_STATE_COOKIE: -** Valid if STAGE==1. The current change-counter cookie value in the +** Valid if STAGE==1. The current change-counter cookie value in the ** target db file. ** ** RBU_STATE_OALSZ: ** Valid if STAGE==1. The size in bytes of the *-oal file. +** +** RBU_STATE_DATATBL: +** Only valid if STAGE==1. The RBU database name of the table +** currently being read. */ #define RBU_STATE_STAGE 1 #define RBU_STATE_TBL 2 @@ -164109,6 +219625,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); #define RBU_STATE_COOKIE 7 #define RBU_STATE_OALSZ 8 #define RBU_STATE_PHASEONESTEP 9 +#define RBU_STATE_DATATBL 10 #define RBU_STAGE_OAL 1 #define RBU_STAGE_MOVE 2 @@ -164123,6 +219640,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName); typedef struct RbuFrame RbuFrame; typedef struct RbuObjIter RbuObjIter; typedef struct RbuState RbuState; +typedef struct RbuSpan RbuSpan; typedef struct rbu_vfs rbu_vfs; typedef struct rbu_file rbu_file; typedef struct RbuUpdateStmt RbuUpdateStmt; @@ -164132,6 +219650,7 @@ typedef unsigned int u32; typedef unsigned short u16; typedef unsigned char u8; typedef sqlite3_int64 i64; +typedef sqlite3_uint64 u64; #endif /* @@ -164151,6 +219670,7 @@ typedef sqlite3_int64 i64; struct RbuState { int eStage; char *zTbl; + char *zDataTbl; char *zIdx; i64 iWalCksum; int nRow; @@ -164166,12 +219686,17 @@ struct RbuUpdateStmt { RbuUpdateStmt *pNext; }; +struct RbuSpan { + const char *zSpan; + int nSpan; +}; + /* ** An iterator of this type is used to iterate through all objects in ** the target database that require updating. For each such table, the ** iterator visits, in order: ** -** * the table itself, +** * the table itself, ** * each index of the table (zero or more points to visit), and ** * a special "cleanup table" state. ** @@ -164180,7 +219705,12 @@ struct RbuUpdateStmt { ** it points to an array of flags nTblCol elements in size. The flag is ** set for each column that is either a part of the PK or a part of an ** index. Or clear otherwise. -** +** +** If there are one or more partial indexes on the table, all fields of +** this array set set to 1. This is because in that case, the module has +** no way to tell which fields will be required to add and remove entries +** from the partial indexes. +** */ struct RbuObjIter { sqlite3_stmt *pTblIter; /* Iterate through tables */ @@ -164210,6 +219740,9 @@ struct RbuObjIter { sqlite3_stmt *pInsert; /* Statement for INSERT operations */ sqlite3_stmt *pDelete; /* Statement for DELETE ops */ sqlite3_stmt *pTmpInsert; /* Insert into rbu_tmp_$zDataTbl */ + int nIdxCol; + RbuSpan *aIdxCol; + char *zIdxSql; /* Last UPDATE used (for PK b-tree updates only), or NULL. */ RbuUpdateStmt *pRbuUpdate; @@ -164254,12 +219787,33 @@ struct RbuFrame { u32 iWalFrame; }; +#ifndef UNUSED_PARAMETER +/* +** The following macros are used to suppress compiler warnings and to +** make it clear to human readers when a function parameter is deliberately +** left unused within the body of a function. This usually happens when +** a function is called via a function pointer. For example the +** implementation of an SQL aggregate step callback may not use the +** parameter indicating the number of arguments passed to the aggregate, +** if it knows that this is enforced elsewhere. +** +** When a function parameter is not used at all within the body of a function, +** it is generally named "NotUsed" or "NotUsed2" to make things even clearer. +** However, these macros may also be used to suppress warnings related to +** parameters that may or may not be used depending on compilation options. +** For example those parameters only used in assert() statements. In these +** cases the parameters are named as per the usual conventions. +*/ +#define UNUSED_PARAMETER(x) (void)(x) +#define UNUSED_PARAMETER2(x,y) UNUSED_PARAMETER(x),UNUSED_PARAMETER(y) +#endif + /* ** RBU handle. ** ** nPhaseOneStep: ** If the RBU database contains an rbu_count table, this value is set to -** a running estimate of the number of b-tree operations required to +** a running estimate of the number of b-tree operations required to ** finish populating the *-oal file. This allows the sqlite3_bp_progress() ** API to calculate the permyriadage progress of populating the *-oal file ** using the formula: @@ -164279,7 +219833,7 @@ struct RbuFrame { ** ** * the RBU update contains any UPDATE operations. If the PK specified ** for an UPDATE operation does not exist in the target table, then -** no b-tree operations are required on index b-trees. Or if the +** no b-tree operations are required on index b-trees. Or if the ** specified PK does exist, then (nIndex*2) such operations are ** required (one delete and one insert on each index b-tree). ** @@ -164305,12 +219859,15 @@ struct sqlite3rbu { int rc; /* Value returned by last rbu_step() call */ char *zErrmsg; /* Error message if rc!=SQLITE_OK */ int nStep; /* Rows processed for current object */ - int nProgress; /* Rows processed for all objects */ + sqlite3_int64 nProgress; /* Rows processed for all objects */ RbuObjIter objiter; /* Iterator for skipping through tbl/idx */ const char *zVfsName; /* Name of automatically created rbu vfs */ rbu_file *pTargetFd; /* File handle open on target db */ + int nPagePerSector; /* Pages per sector for pTargetFd */ i64 iOalSz; i64 nPhaseOneStep; + void *pRenameArg; + int (*xRename)(void*, const char*, const char*); /* The following state variables are used as part of the incremental ** checkpoint stage (eStage==RBU_STAGE_CKPT). See comments surrounding @@ -164323,6 +219880,8 @@ struct sqlite3rbu { int pgsz; u8 *aBuf; i64 iWalCksum; + i64 szTemp; /* Current size of all temp files in use */ + i64 szTempLimit; /* Total size limit for temp files */ /* Used in RBU vacuum mode only */ int nRbu; /* Number of RBU VFS in the stack */ @@ -164331,23 +219890,34 @@ struct sqlite3rbu { /* ** An rbu VFS is implemented using an instance of this structure. +** +** Variable pRbu is only non-NULL for automatically created RBU VFS objects. +** It is NULL for RBU VFS objects created explicitly using +** sqlite3rbu_create_vfs(). It is used to track the total amount of temp +** space used by the RBU handle. */ struct rbu_vfs { sqlite3_vfs base; /* rbu VFS shim methods */ sqlite3_vfs *pRealVfs; /* Underlying VFS */ sqlite3_mutex *mutex; /* Mutex to protect pMain */ - rbu_file *pMain; /* Linked list of main db files */ + sqlite3rbu *pRbu; /* Owner RBU object */ + rbu_file *pMain; /* List of main db files */ + rbu_file *pMainRbu; /* List of main db files with pRbu!=0 */ }; /* ** Each file opened by an rbu VFS is represented by an instance of ** the following structure. +** +** If this is a temporary file (pRbu!=0 && flags&DELETE_ON_CLOSE), variable +** "sz" is set to the current size of the database file. */ struct rbu_file { sqlite3_file base; /* sqlite3_file methods */ sqlite3_file *pReal; /* Underlying file handle */ rbu_vfs *pRbuVfs; /* Pointer to the rbu_vfs object */ sqlite3rbu *pRbu; /* Pointer to rbu object (rbu target only) */ + i64 sz; /* Size of file in bytes (temp only) */ int openFlags; /* Flags this file was opened with */ u32 iCookie; /* Cookie value for main db files */ @@ -164361,6 +219931,7 @@ struct rbu_file { const char *zWal; /* Wal filename for this main db file */ rbu_file *pWalFd; /* Wal file descriptor for this main db */ rbu_file *pMainNext; /* Next MAIN_DB file */ + rbu_file *pMainRbuNext; /* Next MAIN_DB file with pRbu!=0 */ }; /* @@ -164405,11 +219976,12 @@ static unsigned int rbuDeltaGetInt(const char **pz, int *pLen){ v = (v<<6) + c; } z--; - *pLen -= z - zStart; + *pLen -= (int)(z - zStart); *pz = (char*)z; return v; } +#if RBU_ENABLE_DELTA_CKSUM /* ** Compute a 32-bit checksum on the N-byte buffer. Return the result. */ @@ -164444,6 +220016,7 @@ static unsigned int rbuDeltaChecksum(const char *zIn, size_t N){ } return sum3; } +#endif /* ** Apply a delta. @@ -164474,7 +220047,7 @@ static int rbuDeltaApply( ){ unsigned int limit; unsigned int total = 0; -#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST +#if RBU_ENABLE_DELTA_CKSUM char *zOrigOut = zOut; #endif @@ -164529,7 +220102,7 @@ static int rbuDeltaApply( case ';': { zDelta++; lenDelta--; zOut[0] = 0; -#ifndef FOSSIL_OMIT_DELTA_CKSUM_TEST +#if RBU_ENABLE_DELTA_CKSUM if( cnt!=rbuDeltaChecksum(zOrigOut, total) ){ /* ERROR: bad checksum */ return -1; @@ -164588,6 +220161,7 @@ static void rbuFossilDeltaFunc( char *aOut; assert( argc==2 ); + UNUSED_PARAMETER(argc); nOrig = sqlite3_value_bytes(argv[0]); aOrig = (const char*)sqlite3_value_blob(argv[0]); @@ -164607,6 +220181,7 @@ static void rbuFossilDeltaFunc( }else{ nOut2 = rbuDeltaApply(aOrig, nOrig, aDelta, nDelta, aOut); if( nOut2!=nOut ){ + sqlite3_free(aOut); sqlite3_result_error(context, "corrupt fossil delta", -1); }else{ sqlite3_result_blob(context, aOut, nOut, sqlite3_free); @@ -164618,7 +220193,7 @@ static void rbuFossilDeltaFunc( /* ** Prepare the SQL statement in buffer zSql against database handle db. ** If successful, set *ppStmt to point to the new statement and return -** SQLITE_OK. +** SQLITE_OK. ** ** Otherwise, if an error does occur, set *ppStmt to NULL and return ** an SQLite error code. Additionally, set output variable *pzErrmsg to @@ -164626,7 +220201,7 @@ static void rbuFossilDeltaFunc( ** of the caller to (eventually) free this buffer using sqlite3_free(). */ static int prepareAndCollectError( - sqlite3 *db, + sqlite3 *db, sqlite3_stmt **ppStmt, char **pzErrmsg, const char *zSql @@ -164658,9 +220233,9 @@ static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ /* ** Unless it is NULL, argument zSql points to a buffer allocated using ** sqlite3_malloc containing an SQL statement. This function prepares the SQL -** statement against database db and frees the buffer. If statement -** compilation is successful, *ppStmt is set to point to the new statement -** handle and SQLITE_OK is returned. +** statement against database db and frees the buffer. If statement +** compilation is successful, *ppStmt is set to point to the new statement +** handle and SQLITE_OK is returned. ** ** Otherwise, if an error occurs, *ppStmt is set to NULL and an error code ** returned. In this case, *pzErrmsg may also be set to point to an error @@ -164671,7 +220246,7 @@ static int resetAndCollectError(sqlite3_stmt *pStmt, char **pzErrmsg){ ** In this case SQLITE_NOMEM is returned and *ppStmt set to NULL. */ static int prepareFreeAndCollectError( - sqlite3 *db, + sqlite3 *db, sqlite3_stmt **ppStmt, char **pzErrmsg, char *zSql @@ -164726,13 +220301,18 @@ static void rbuObjIterClearStatements(RbuObjIter *pIter){ sqlite3_free(pUp); pUp = pTmp; } - + sqlite3_free(pIter->aIdxCol); + sqlite3_free(pIter->zIdxSql); + pIter->pSelect = 0; pIter->pInsert = 0; pIter->pDelete = 0; pIter->pRbuUpdate = 0; pIter->pTmpInsert = 0; pIter->nCol = 0; + pIter->nIdxCol = 0; + pIter->aIdxCol = 0; + pIter->zIdxSql = 0; } /* @@ -164750,16 +220330,16 @@ static void rbuObjIterFinalize(RbuObjIter *pIter){ /* ** Advance the iterator to the next position. ** -** If no error occurs, SQLITE_OK is returned and the iterator is left -** pointing to the next entry. Otherwise, an error code and message is -** left in the RBU handle passed as the first argument. A copy of the +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the next entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the ** error code is returned. */ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ int rc = p->rc; if( rc==SQLITE_OK ){ - /* Free any SQLite statements used while processing the previous object */ + /* Free any SQLite statements used while processing the previous object */ rbuObjIterClearStatements(pIter); if( pIter->zIdx==0 ){ rc = sqlite3_exec(p->dbMain, @@ -164779,6 +220359,7 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ if( rc!=SQLITE_ROW ){ rc = resetAndCollectError(pIter->pTblIter, &p->zErrmsg); pIter->zTbl = 0; + pIter->zDataTbl = 0; }else{ pIter->zTbl = (const char*)sqlite3_column_text(pIter->pTblIter, 0); pIter->zDataTbl = (const char*)sqlite3_column_text(pIter->pTblIter,1); @@ -164818,7 +220399,7 @@ static int rbuObjIterNext(sqlite3rbu *p, RbuObjIter *pIter){ ** The implementation of the rbu_target_name() SQL function. This function ** accepts one or two arguments. The first argument is the name of a table - ** the name of a table in the RBU database. The second, if it is present, is 1 -** for a view or 0 for a table. +** for a view or 0 for a table. ** ** For a non-vacuum RBU handle, if the table name matches the pattern: ** @@ -164847,6 +220428,7 @@ static void rbuTargetNameFunc( zIn = (const char*)sqlite3_value_text(argv[0]); if( zIn ){ if( rbuIsVacuum(p) ){ + assert( argc==2 || argc==1 ); if( argc==1 || 0==sqlite3_value_int(argv[1]) ){ sqlite3_result_text(pCtx, zIn, -1, SQLITE_STATIC); } @@ -164865,26 +220447,28 @@ static void rbuTargetNameFunc( /* ** Initialize the iterator structure passed as the second argument. ** -** If no error occurs, SQLITE_OK is returned and the iterator is left -** pointing to the first entry. Otherwise, an error code and message is -** left in the RBU handle passed as the first argument. A copy of the +** If no error occurs, SQLITE_OK is returned and the iterator is left +** pointing to the first entry. Otherwise, an error code and message is +** left in the RBU handle passed as the first argument. A copy of the ** error code is returned. */ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ int rc; memset(pIter, 0, sizeof(RbuObjIter)); - rc = prepareAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pTblIter, &p->zErrmsg, + sqlite3_mprintf( "SELECT rbu_target_name(name, type='view') AS target, name " - "FROM sqlite_master " + "FROM sqlite_schema " "WHERE type IN ('table', 'view') AND target IS NOT NULL " + " %s " "ORDER BY name" - ); + , rbuIsVacuum(p) ? "AND rootpage!=0 AND rootpage IS NOT NULL" : "")); if( rc==SQLITE_OK ){ rc = prepareAndCollectError(p->dbMain, &pIter->pIdxIter, &p->zErrmsg, "SELECT name, rootpage, sql IS NULL OR substr(8, 6)=='UNIQUE' " - " FROM main.sqlite_master " + " FROM main.sqlite_schema " " WHERE type='index' AND tbl_name = ?" ); } @@ -164900,7 +220484,7 @@ static int rbuObjIterFirst(sqlite3rbu *p, RbuObjIter *pIter){ ** ** If an error has already occurred (p->rc is already set to something other ** than SQLITE_OK), then this function returns NULL without modifying the -** stored error code. In this case it still calls sqlite3_free() on any +** stored error code. In this case it still calls sqlite3_free() on any ** printf() parameters associated with %z conversions. */ static char *rbuMPrintf(sqlite3rbu *p, const char *zFmt, ...){ @@ -164946,16 +220530,16 @@ static int rbuMPrintfExec(sqlite3rbu *p, sqlite3 *db, const char *zFmt, ...){ } /* -** Attempt to allocate and return a pointer to a zeroed block of nByte -** bytes. +** Attempt to allocate and return a pointer to a zeroed block of nByte +** bytes. ** -** If an error (i.e. an OOM condition) occurs, return NULL and leave an -** error code in the rbu handle passed as the first argument. Or, if an -** error has already occurred when this function is called, return NULL +** If an error (i.e. an OOM condition) occurs, return NULL and leave an +** error code in the rbu handle passed as the first argument. Or, if an +** error has already occurred when this function is called, return NULL ** immediately without attempting the allocation or modifying the stored ** error code. */ -static void *rbuMalloc(sqlite3rbu *p, int nByte){ +static void *rbuMalloc(sqlite3rbu *p, sqlite3_int64 nByte){ void *pRet = 0; if( p->rc==SQLITE_OK ){ assert( nByte>0 ); @@ -164976,7 +220560,7 @@ static void *rbuMalloc(sqlite3rbu *p, int nByte){ ** error code in the RBU handle passed as the first argument. */ static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ - int nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; + sqlite3_int64 nByte = (2*sizeof(char*) + sizeof(int) + 3*sizeof(u8)) * nCol; char **azNew; azNew = (char**)rbuMalloc(p, nByte); @@ -165003,14 +220587,15 @@ static void rbuAllocateIterArrays(sqlite3rbu *p, RbuObjIter *pIter, int nCol){ static char *rbuStrndup(const char *zStr, int *pRc){ char *zRet = 0; - assert( *pRc==SQLITE_OK ); - if( zStr ){ - size_t nCopy = strlen(zStr) + 1; - zRet = (char*)sqlite3_malloc64(nCopy); - if( zRet ){ - memcpy(zRet, zStr, nCopy); - }else{ - *pRc = SQLITE_NOMEM; + if( *pRc==SQLITE_OK ){ + if( zStr ){ + size_t nCopy = strlen(zStr) + 1; + zRet = (char*)sqlite3_malloc64(nCopy); + if( zRet ){ + memcpy(zRet, zStr, nCopy); + }else{ + *pRc = SQLITE_NOMEM; + } } } @@ -165047,7 +220632,7 @@ static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ ** RBU_PK_VTAB: Table is a virtual table. ** ** Argument *piPk is also of type (int*), and also points to an output -** parameter. Unless the table has an external primary key index +** parameter. Unless the table has an external primary key index ** (i.e. unless *peType is set to 3), then *piPk is set to zero. Or, ** if the table does have an external primary key index, then *piPk ** is set to the root page number of the primary key index before @@ -165055,12 +220640,12 @@ static void rbuFinalize(sqlite3rbu *p, sqlite3_stmt *pStmt){ ** ** ALGORITHM: ** -** if( no entry exists in sqlite_master ){ +** if( no entry exists in sqlite_schema ){ ** return RBU_PK_NOTABLE ** }else if( sql for the entry starts with "CREATE VIRTUAL" ){ ** return RBU_PK_VTAB ** }else if( "PRAGMA index_list()" for the table contains a "pk" index ){ -** if( the index that is the pk exists in sqlite_master ){ +** if( the index that is the pk exists in sqlite_schema ){ ** *piPK = rootpage of that index. ** return RBU_PK_EXTERNAL ** }else{ @@ -165080,9 +220665,9 @@ static void rbuTableType( int *piPk ){ /* - ** 0) SELECT count(*) FROM sqlite_master where name=%Q AND IsVirtual(%Q) + ** 0) SELECT count(*) FROM sqlite_schema where name=%Q AND IsVirtual(%Q) ** 1) PRAGMA index_list = ? - ** 2) SELECT count(*) FROM sqlite_master where name=%Q + ** 2) SELECT count(*) FROM sqlite_schema where name=%Q ** 3) PRAGMA table_info = ? */ sqlite3_stmt *aStmt[4] = {0, 0, 0, 0}; @@ -165091,10 +220676,12 @@ static void rbuTableType( *piPk = 0; assert( p->rc==SQLITE_OK ); - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[0], &p->zErrmsg, sqlite3_mprintf( - "SELECT (sql LIKE 'create virtual%%'), rootpage" - " FROM sqlite_master" + "SELECT " + " (sql COLLATE nocase BETWEEN 'CREATE VIRTUAL' AND 'CREATE VIRTUAM')," + " rootpage" + " FROM sqlite_schema" " WHERE name=%Q", zTab )); if( p->rc!=SQLITE_OK || sqlite3_step(aStmt[0])!=SQLITE_ROW ){ @@ -165107,7 +220694,7 @@ static void rbuTableType( } *piTnum = sqlite3_column_int(aStmt[0], 1); - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[1], &p->zErrmsg, sqlite3_mprintf("PRAGMA index_list=%Q",zTab) ); if( p->rc ) goto rbuTableType_end; @@ -165115,9 +220702,9 @@ static void rbuTableType( const u8 *zOrig = sqlite3_column_text(aStmt[1], 3); const u8 *zIdx = sqlite3_column_text(aStmt[1], 1); if( zOrig && zIdx && zOrig[0]=='p' ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[2], &p->zErrmsg, sqlite3_mprintf( - "SELECT rootpage FROM sqlite_master WHERE name = %Q", zIdx + "SELECT rootpage FROM sqlite_schema WHERE name = %Q", zIdx )); if( p->rc==SQLITE_OK ){ if( sqlite3_step(aStmt[2])==SQLITE_ROW ){ @@ -165131,7 +220718,7 @@ static void rbuTableType( } } - p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbMain, &aStmt[3], &p->zErrmsg, sqlite3_mprintf("PRAGMA table_info=%Q",zTab) ); if( p->rc==SQLITE_OK ){ @@ -165170,14 +220757,21 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ pIter->nIndex = 0; while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pList) ){ const char *zIdx = (const char*)sqlite3_column_text(pList, 1); + int bPartial = sqlite3_column_int(pList, 4); sqlite3_stmt *pXInfo = 0; if( zIdx==0 ) break; + if( bPartial ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", zIdx) ); while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ int iCid = sqlite3_column_int(pXInfo, 1); if( iCid>=0 ) pIter->abIndexed[iCid] = 1; + if( iCid==-2 ){ + memset(pIter->abIndexed, 0x01, sizeof(u8)*pIter->nTblCol); + } } rbuFinalize(p, pXInfo); bIndex = 1; @@ -165200,7 +220794,7 @@ static void rbuObjIterCacheIndexedCols(sqlite3rbu *p, RbuObjIter *pIter){ ** the table (not index) that the iterator currently points to. ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. If -** an error does occur, an error code and error message are also left in +** an error does occur, an error code and error message are also left in ** the RBU handle. */ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ @@ -165222,7 +220816,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ if( p->rc ) return p->rc; if( pIter->zIdx==0 ) pIter->iTnum = iTnum; - assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK + assert( pIter->eType==RBU_PK_NONE || pIter->eType==RBU_PK_IPK || pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_WITHOUT_ROWID || pIter->eType==RBU_PK_VTAB ); @@ -165230,7 +220824,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ /* Populate the azTblCol[] and nTblCol variables based on the columns ** of the input table. Ignore any input table columns that begin with ** "rbu_". */ - p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, sqlite3_mprintf("SELECT * FROM '%q'", pIter->zDataTbl) ); if( p->rc==SQLITE_OK ){ @@ -165266,7 +220860,7 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ ** present in the input table. Populate the abTblPk[], azTblType[] and ** aiTblOrder[] arrays at the same time. */ if( p->rc==SQLITE_OK ){ - p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, sqlite3_mprintf("PRAGMA table_info(%Q)", pIter->zTbl) ); } @@ -165292,7 +220886,8 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ } pIter->azTblType[iOrder] = rbuStrndup(zType, &p->rc); - pIter->abTblPk[iOrder] = (iPk!=0); + assert( iPk>=0 ); + pIter->abTblPk[iOrder] = (u8)iPk; pIter->abNotNull[iOrder] = (u8)bNotNull || (iPk!=0); iOrder++; } @@ -165308,8 +220903,8 @@ static int rbuObjIterCacheTableInfo(sqlite3rbu *p, RbuObjIter *pIter){ } /* -** This function constructs and returns a pointer to a nul-terminated -** string containing some SQL clause or list based on one or more of the +** This function constructs and returns a pointer to a nul-terminated +** string containing some SQL clause or list based on one or more of the ** column names currently stored in the pIter->azTblCol[] array. */ static char *rbuObjIterGetCollist( @@ -165328,23 +220923,232 @@ static char *rbuObjIterGetCollist( } /* -** This function is used to create a SELECT list (the list of SQL -** expressions that follows a SELECT keyword) for a SELECT statement -** used to read from an data_xxx or rbu_tmp_xxx table while updating the -** index object currently indicated by the iterator object passed as the -** second argument. A "PRAGMA index_xinfo = " statement is used +** Return a comma separated list of the quoted PRIMARY KEY column names, +** in order, for the current table. Before each column name, add the text +** zPre. After each column name, add the zPost text. Use zSeparator as +** the separator text (usually ", "). +*/ +static char *rbuObjIterGetPkList( + sqlite3rbu *p, /* RBU object */ + RbuObjIter *pIter, /* Object iterator for column names */ + const char *zPre, /* Before each quoted column name */ + const char *zSeparator, /* Separator to use between columns */ + const char *zPost /* After each quoted column name */ +){ + int iPk = 1; + char *zRet = 0; + const char *zSep = ""; + while( 1 ){ + int i; + for(i=0; inTblCol; i++){ + if( (int)pIter->abTblPk[i]==iPk ){ + const char *zCol = pIter->azTblCol[i]; + zRet = rbuMPrintf(p, "%z%s%s\"%w\"%s", zRet, zSep, zPre, zCol, zPost); + zSep = zSeparator; + break; + } + } + if( i==pIter->nTblCol ) break; + iPk++; + } + return zRet; +} + +/* +** This function is called as part of restarting an RBU vacuum within +** stage 1 of the process (while the *-oal file is being built) while +** updating a table (not an index). The table may be a rowid table or +** a WITHOUT ROWID table. It queries the target database to find the +** largest key that has already been written to the target table and +** constructs a WHERE clause that can be used to extract the remaining +** rows from the source table. For a rowid table, the WHERE clause +** is of the form: +** +** "WHERE _rowid_ > ?" +** +** and for WITHOUT ROWID tables: +** +** "WHERE (key1, key2) > (?, ?)" +** +** Instead of "?" placeholders, the actual WHERE clauses created by +** this function contain literal SQL values. +*/ +static char *rbuVacuumTableStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter, /* RBU iterator object */ + int bRowid, /* True for a rowid table */ + const char *zWrite /* Target table name prefix */ +){ + sqlite3_stmt *pMax = 0; + char *zRet = 0; + if( bRowid ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT max(_rowid_) FROM \"%s%w\"", zWrite, pIter->zTbl + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + sqlite3_int64 iMax = sqlite3_column_int64(pMax, 0); + zRet = rbuMPrintf(p, " WHERE _rowid_ > %lld ", iMax); + } + rbuFinalize(p, pMax); + }else{ + char *zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", " DESC"); + char *zSelect = rbuObjIterGetPkList(p, pIter, "quote(", "||','||", ")"); + char *zList = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbMain, &pMax, &p->zErrmsg, + sqlite3_mprintf( + "SELECT %s FROM \"%s%w\" ORDER BY %s LIMIT 1", + zSelect, zWrite, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pMax) ){ + const char *zVal = (const char*)sqlite3_column_text(pMax, 0); + zRet = rbuMPrintf(p, " WHERE (%s) > (%s) ", zList, zVal); + } + rbuFinalize(p, pMax); + } + + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zList); + } + return zRet; +} + +/* +** This function is called as part of restating an RBU vacuum when the +** current operation is writing content to an index. If possible, it +** queries the target index b-tree for the largest key already written to +** it, then composes and returns an expression that can be used in a WHERE +** clause to select the remaining required rows from the source table. +** It is only possible to return such an expression if: +** +** * The index contains no DESC columns, and +** * The last key written to the index before the operation was +** suspended does not contain any NULL values. +** +** The expression is of the form: +** +** (index-field1, index-field2, ...) > (?, ?, ...) +** +** except that the "?" placeholders are replaced with literal values. +** +** If the expression cannot be created, NULL is returned. In this case, +** the caller has to use an OFFSET clause to extract only the required +** rows from the sourct table, just as it does for an RBU update operation. +*/ +static char *rbuVacuumIndexStart( + sqlite3rbu *p, /* RBU handle */ + RbuObjIter *pIter /* RBU iterator object */ +){ + char *zOrder = 0; + char *zLhs = 0; + char *zSelect = 0; + char *zVector = 0; + char *zRet = 0; + int bFailed = 0; + const char *zSep = ""; + int iCol = 0; + sqlite3_stmt *pXInfo = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pXInfo, &p->zErrmsg, + sqlite3_mprintf("PRAGMA main.index_xinfo = %Q", pIter->zIdx) + ); + while( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pXInfo) ){ + int iCid = sqlite3_column_int(pXInfo, 1); + const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); + const char *zCol; + if( sqlite3_column_int(pXInfo, 3) ){ + bFailed = 1; + break; + } + + if( iCid<0 ){ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else{ + zCol = "_rowid_"; + } + }else{ + zCol = pIter->azTblCol[iCid]; + } + + zLhs = rbuMPrintf(p, "%z%s \"%w\" COLLATE %Q", + zLhs, zSep, zCol, zCollate + ); + zOrder = rbuMPrintf(p, "%z%s \"rbu_imp_%d%w\" COLLATE %Q DESC", + zOrder, zSep, iCol, zCol, zCollate + ); + zSelect = rbuMPrintf(p, "%z%s quote(\"rbu_imp_%d%w\")", + zSelect, zSep, iCol, zCol + ); + zSep = ", "; + iCol++; + } + rbuFinalize(p, pXInfo); + if( bFailed ) goto index_start_out; + + if( p->rc==SQLITE_OK ){ + sqlite3_stmt *pSel = 0; + + p->rc = prepareFreeAndCollectError(p->dbMain, &pSel, &p->zErrmsg, + sqlite3_mprintf("SELECT %s FROM \"rbu_imp_%w\" ORDER BY %s LIMIT 1", + zSelect, pIter->zTbl, zOrder + ) + ); + if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pSel) ){ + zSep = ""; + for(iCol=0; iColnCol; iCol++){ + const char *zQuoted = (const char*)sqlite3_column_text(pSel, iCol); + if( zQuoted==0 ){ + p->rc = SQLITE_NOMEM; + }else if( zQuoted[0]=='N' ){ + bFailed = 1; + break; + } + zVector = rbuMPrintf(p, "%z%s%s", zVector, zSep, zQuoted); + zSep = ", "; + } + + if( !bFailed ){ + zRet = rbuMPrintf(p, "(%s) > (%s)", zLhs, zVector); + } + } + rbuFinalize(p, pSel); + } + + index_start_out: + sqlite3_free(zOrder); + sqlite3_free(zSelect); + sqlite3_free(zVector); + sqlite3_free(zLhs); + return zRet; +} + +/* +** This function is used to create a SELECT list (the list of SQL +** expressions that follows a SELECT keyword) for a SELECT statement +** used to read from an data_xxx or rbu_tmp_xxx table while updating the +** index object currently indicated by the iterator object passed as the +** second argument. A "PRAGMA index_xinfo = " statement is used ** to obtain the required information. ** ** If the index is of the following form: ** ** CREATE INDEX i1 ON t1(c, b COLLATE nocase); ** -** and "t1" is a table with an explicit INTEGER PRIMARY KEY column +** and "t1" is a table with an explicit INTEGER PRIMARY KEY column ** "ipk", the returned string is: ** ** "`c` COLLATE 'BINARY', `b` COLLATE 'NOCASE', `ipk` COLLATE 'BINARY'" ** -** As well as the returned string, three other malloc'd strings are +** As well as the returned string, three other malloc'd strings are ** returned via output parameters. As follows: ** ** pzImposterCols: ... @@ -165381,36 +221185,44 @@ static char *rbuObjIterGetIndexCols( int iCid = sqlite3_column_int(pXInfo, 1); int bDesc = sqlite3_column_int(pXInfo, 3); const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - const char *zCol; + const char *zCol = 0; const char *zType; - if( iCid<0 ){ - /* An integer primary key. If the table has an explicit IPK, use - ** its name. Otherwise, use "rbu_rowid". */ - if( pIter->eType==RBU_PK_IPK ){ - int i; - for(i=0; pIter->abTblPk[i]==0; i++); - assert( inTblCol ); - zCol = pIter->azTblCol[i]; - }else if( rbuIsVacuum(p) ){ - zCol = "_rowid_"; + if( iCid==-2 ){ + int iSeq = sqlite3_column_int(pXInfo, 0); + zRet = sqlite3_mprintf("%z%s(%.*s) COLLATE %Q", zRet, zCom, + pIter->aIdxCol[iSeq].nSpan, pIter->aIdxCol[iSeq].zSpan, zCollate + ); + zType = ""; + }else { + if( iCid<0 ){ + /* An integer primary key. If the table has an explicit IPK, use + ** its name. Otherwise, use "rbu_rowid". */ + if( pIter->eType==RBU_PK_IPK ){ + int i; + for(i=0; pIter->abTblPk[i]==0; i++); + assert( inTblCol ); + zCol = pIter->azTblCol[i]; + }else if( rbuIsVacuum(p) ){ + zCol = "_rowid_"; + }else{ + zCol = "rbu_rowid"; + } + zType = "INTEGER"; }else{ - zCol = "rbu_rowid"; + zCol = pIter->azTblCol[iCid]; + zType = pIter->azTblType[iCid]; } - zType = "INTEGER"; - }else{ - zCol = pIter->azTblCol[iCid]; - zType = pIter->azTblType[iCid]; + zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom,zCol,zCollate); } - zRet = sqlite3_mprintf("%z%s\"%w\" COLLATE %Q", zRet, zCom, zCol, zCollate); if( pIter->bUnique==0 || sqlite3_column_int(pXInfo, 5) ){ const char *zOrder = (bDesc ? " DESC" : ""); - zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", + zImpPK = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\"%s", zImpPK, zCom, nBind, zCol, zOrder ); } - zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", + zImpCols = sqlite3_mprintf("%z%s\"rbu_imp_%d%w\" %s COLLATE %Q", zImpCols, zCom, nBind, zCol, zType, zCollate ); zWhere = sqlite3_mprintf( @@ -165456,7 +221268,7 @@ static char *rbuObjIterGetIndexCols( ** the text ", old._rowid_" to the returned value. */ static char *rbuObjIterGetOldlist( - sqlite3rbu *p, + sqlite3rbu *p, RbuObjIter *pIter, const char *zObj ){ @@ -165497,7 +221309,7 @@ static char *rbuObjIterGetOldlist( ** "b = ?1 AND c = ?2" */ static char *rbuObjIterGetWhere( - sqlite3rbu *p, + sqlite3rbu *p, RbuObjIter *pIter ){ char *zList = 0; @@ -165512,7 +221324,7 @@ static char *rbuObjIterGetWhere( zSep = " AND "; } } - zList = rbuMPrintf(p, + zList = rbuMPrintf(p, "_rowid_ = (SELECT id FROM rbu_imposter2 WHERE %z)", zList ); @@ -165552,7 +221364,7 @@ static void rbuBadControlError(sqlite3rbu *p){ ** ** The memory for the returned string is obtained from sqlite3_malloc(). ** It is the responsibility of the caller to eventually free it using -** sqlite3_free(). +** sqlite3_free(). ** ** If an OOM error is encountered when allocating space for the new ** string, an error code is left in the rbu handle passed as the first @@ -165576,19 +221388,19 @@ static char *rbuObjIterGetSetlist( for(i=0; inTblCol; i++){ char c = zMask[pIter->aiSrcOrder[i]]; if( c=='x' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", + zList = rbuMPrintf(p, "%z%s\"%w\"=?%d", zList, zSep, pIter->azTblCol[i], i+1 ); zSep = ", "; } else if( c=='d' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_delta(\"%w\", ?%d)", zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 ); zSep = ", "; } else if( c=='f' ){ - zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", + zList = rbuMPrintf(p, "%z%s\"%w\"=rbu_fossil_delta(\"%w\", ?%d)", zList, zSep, pIter->azTblCol[i], pIter->azTblCol[i], i+1 ); zSep = ", "; @@ -165606,7 +221418,7 @@ static char *rbuObjIterGetSetlist( ** ** The memory for the returned string is obtained from sqlite3_malloc(). ** It is the responsibility of the caller to eventually free it using -** sqlite3_free(). +** sqlite3_free(). ** ** If an OOM error is encountered when allocating space for the new ** string, an error code is left in the rbu handle passed as the first @@ -165616,7 +221428,7 @@ static char *rbuObjIterGetSetlist( */ static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ char *zRet = 0; - int nByte = nBind*2 + 1; + sqlite3_int64 nByte = 2*(sqlite3_int64)nBind + 1; zRet = (char*)rbuMalloc(p, nByte); if( zRet ){ @@ -165630,8 +221442,8 @@ static char *rbuObjIterGetBindlist(sqlite3rbu *p, int nBind){ } /* -** The iterator currently points to a table (not index) of type -** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY +** The iterator currently points to a table (not index) of type +** RBU_PK_WITHOUT_ROWID. This function creates the PRIMARY KEY ** declaration for the corresponding imposter table. For example, ** if the iterator points to a table created as: ** @@ -165648,7 +221460,7 @@ static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){ const char *zSep = "PRIMARY KEY("; sqlite3_stmt *pXList = 0; /* PRAGMA index_list = (pIter->zTbl) */ sqlite3_stmt *pXInfo = 0; /* PRAGMA index_xinfo = */ - + p->rc = prepareFreeAndCollectError(p->dbMain, &pXList, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.index_list = %Q", pIter->zTbl) ); @@ -165686,7 +221498,7 @@ static char *rbuWithoutRowidPK(sqlite3rbu *p, RbuObjIter *pIter){ ** a table b-tree where the table has an external primary key. If the ** iterator passed as the second argument does not currently point to ** a table (not index) with an external primary key, this function is a -** no-op. +** no-op. ** ** Assuming the iterator does point to a table with an external PK, this ** function creates a WITHOUT ROWID imposter table named "rbu_imposter2" @@ -165713,8 +221525,8 @@ static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ /* Figure out the name of the primary key index for the current table. ** This is needed for the argument to "PRAGMA index_xinfo". Set ** zIdx to point to a nul-terminated string containing this name. */ - p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, - "SELECT name FROM sqlite_master WHERE rootpage = ?" + p->rc = prepareAndCollectError(p->dbMain, &pQuery, &p->zErrmsg, + "SELECT name FROM sqlite_schema WHERE rootpage = ?" ); if( p->rc==SQLITE_OK ){ sqlite3_bind_int(pQuery, 1, tnum); @@ -165735,7 +221547,7 @@ static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ int iCid = sqlite3_column_int(pXInfo, 1); int bDesc = sqlite3_column_int(pXInfo, 3); const char *zCollate = (const char*)sqlite3_column_text(pXInfo, 4); - zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %s", zCols, zComma, + zCols = rbuMPrintf(p, "%z%sc%d %s COLLATE %Q", zCols, zComma, iCid, pIter->azTblType[iCid], zCollate ); zPk = rbuMPrintf(p, "%z%sc%d%s", zPk, zComma, iCid, bDesc?" DESC":""); @@ -165747,7 +221559,7 @@ static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); rbuMPrintfExec(p, p->dbMain, - "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", + "CREATE TABLE rbu_imposter2(%z, PRIMARY KEY(%z)) WITHOUT ROWID", zCols, zPk ); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); @@ -165755,7 +221567,7 @@ static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ } /* -** If an error has already occurred when this function is called, it +** If an error has already occurred when this function is called, it ** immediately returns zero (without doing any work). Or, if an error ** occurs during the execution of this function, it sets the error code ** in the sqlite3rbu object indicated by the first argument and returns @@ -165768,9 +221580,9 @@ static void rbuCreateImposterTable2(sqlite3rbu *p, RbuObjIter *pIter){ ** an imposter table are created, or zero otherwise. ** ** An imposter table is required in all cases except RBU_PK_VTAB. Only -** virtual tables are written to directly. The imposter table has the -** same schema as the actual target table (less any UNIQUE constraints). -** More precisely, the "same schema" means the same columns, types, +** virtual tables are written to directly. The imposter table has the +** same schema as the actual target table (less any UNIQUE constraints). +** More precisely, the "same schema" means the same columns, types, ** collation sequences. For tables that do not have an external PRIMARY ** KEY, it also means the same PRIMARY KEY declaration. */ @@ -165796,7 +221608,7 @@ static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ ** "PRIMARY KEY" to the imposter table column declaration. */ zPk = "PRIMARY KEY "; } - zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %s%s", + zSql = rbuMPrintf(p, "%z%s\"%w\" %s %sCOLLATE %Q%s", zSql, zComma, zCol, pIter->azTblType[iCol], zPk, zColl, (pIter->abNotNull[iCol] ? " NOT NULL" : "") ); @@ -165811,8 +221623,8 @@ static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ } sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 1, tnum); - rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", - pIter->zTbl, zSql, + rbuMPrintfExec(p, p->dbMain, "CREATE TABLE \"rbu_imp_%w\"(%z)%s", + pIter->zTbl, zSql, (pIter->eType==RBU_PK_WITHOUT_ROWID ? " WITHOUT ROWID" : "") ); sqlite3_test_control(SQLITE_TESTCTRL_IMPOSTER, p->dbMain, "main", 0, 0); @@ -165826,12 +221638,12 @@ static void rbuCreateImposterTable(sqlite3rbu *p, RbuObjIter *pIter){ ** INSERT INTO rbu_tmp_xxx VALUES(?, ?, ? ...); ** ** The number of bound variables is equal to the number of columns in -** the target table, plus one (for the rbu_control column), plus one more -** (for the rbu_rowid column) if the target table is an implicit IPK or +** the target table, plus one (for the rbu_control column), plus one more +** (for the rbu_rowid column) if the target table is an implicit IPK or ** virtual table. */ static void rbuObjIterPrepareTmpInsert( - sqlite3rbu *p, + sqlite3rbu *p, RbuObjIter *pIter, const char *zCollist, const char *zRbuRowid @@ -165842,14 +221654,14 @@ static void rbuObjIterPrepareTmpInsert( assert( pIter->pTmpInsert==0 ); p->rc = prepareFreeAndCollectError( p->dbRbu, &pIter->pTmpInsert, &p->zErrmsg, sqlite3_mprintf( - "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", + "INSERT INTO %s.'rbu_tmp_%q'(rbu_control,%s%s) VALUES(%z)", p->zStateDb, pIter->zDataTbl, zCollist, zRbuRowid, zBind )); } } static void rbuTmpInsertFunc( - sqlite3_context *pCtx, + sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ @@ -165858,8 +221670,8 @@ static void rbuTmpInsertFunc( int i; assert( sqlite3_value_int(apVal[0])!=0 - || p->objiter.eType==RBU_PK_EXTERNAL - || p->objiter.eType==RBU_PK_NONE + || p->objiter.eType==RBU_PK_EXTERNAL + || p->objiter.eType==RBU_PK_NONE ); if( sqlite3_value_int(apVal[0])!=0 ){ p->nPhaseOneStep += p->objiter.nIndex; @@ -165878,13 +221690,108 @@ static void rbuTmpInsertFunc( } } +static char *rbuObjIterGetIndexWhere(sqlite3rbu *p, RbuObjIter *pIter){ + sqlite3_stmt *pStmt = 0; + int rc = p->rc; + char *zRet = 0; + + assert( pIter->zIdxSql==0 && pIter->nIdxCol==0 && pIter->aIdxCol==0 ); + + if( rc==SQLITE_OK ){ + rc = prepareAndCollectError(p->dbMain, &pStmt, &p->zErrmsg, + "SELECT trim(sql) FROM sqlite_schema WHERE type='index' AND name=?" + ); + } + if( rc==SQLITE_OK ){ + int rc2; + rc = sqlite3_bind_text(pStmt, 1, pIter->zIdx, -1, SQLITE_STATIC); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + char *zSql = (char*)sqlite3_column_text(pStmt, 0); + if( zSql ){ + pIter->zIdxSql = zSql = rbuStrndup(zSql, &rc); + } + if( zSql ){ + int nParen = 0; /* Number of open parenthesis */ + int i; + int iIdxCol = 0; + int nIdxAlloc = 0; + for(i=0; zSql[i]; i++){ + char c = zSql[i]; + + /* If necessary, grow the pIter->aIdxCol[] array */ + if( iIdxCol==nIdxAlloc ){ + RbuSpan *aIdxCol = (RbuSpan*)sqlite3_realloc( + pIter->aIdxCol, (nIdxAlloc+16)*sizeof(RbuSpan) + ); + if( aIdxCol==0 ){ + rc = SQLITE_NOMEM; + break; + } + pIter->aIdxCol = aIdxCol; + nIdxAlloc += 16; + } + + if( c=='(' ){ + if( nParen==0 ){ + assert( iIdxCol==0 ); + pIter->aIdxCol[0].zSpan = &zSql[i+1]; + } + nParen++; + } + else if( c==')' ){ + nParen--; + if( nParen==0 ){ + int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan); + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + i++; + break; + } + }else if( c==',' && nParen==1 ){ + int nSpan = (int)(&zSql[i] - pIter->aIdxCol[iIdxCol].zSpan); + pIter->aIdxCol[iIdxCol++].nSpan = nSpan; + pIter->aIdxCol[iIdxCol].zSpan = &zSql[i+1]; + }else if( c=='"' || c=='\'' || c=='`' ){ + for(i++; 1; i++){ + if( zSql[i]==c ){ + if( zSql[i+1]!=c ) break; + i++; + } + } + }else if( c=='[' ){ + for(i++; 1; i++){ + if( zSql[i]==']' ) break; + } + }else if( c=='-' && zSql[i+1]=='-' ){ + for(i=i+2; zSql[i] && zSql[i]!='\n'; i++); + if( zSql[i]=='\0' ) break; + }else if( c=='/' && zSql[i+1]=='*' ){ + for(i=i+2; zSql[i] && (zSql[i]!='*' || zSql[i+1]!='/'); i++); + if( zSql[i]=='\0' ) break; + i++; + } + } + if( zSql[i] ){ + zRet = rbuStrndup(&zSql[i], &rc); + } + pIter->nIdxCol = iIdxCol; + } + } + + rc2 = sqlite3_finalize(pStmt); + if( rc==SQLITE_OK ) rc = rc2; + } + + p->rc = rc; + return zRet; +} + /* -** Ensure that the SQLite statement handles required to update the -** target database object currently indicated by the iterator passed +** Ensure that the SQLite statement handles required to update the +** target database object currently indicated by the iterator passed ** as the second argument are available. */ static int rbuObjIterPrepareAll( - sqlite3rbu *p, + sqlite3rbu *p, RbuObjIter *pIter, int nOffset /* Add "LIMIT -1 OFFSET $nOffset" to SELECT */ ){ @@ -165907,9 +221814,11 @@ static int rbuObjIterPrepareAll( char *zImposterPK = 0; /* Primary key declaration for imposter */ char *zWhere = 0; /* WHERE clause on PK columns */ char *zBind = 0; + char *zPart = 0; int nBind = 0; assert( pIter->eType!=RBU_PK_VTAB ); + zPart = rbuObjIterGetIndexWhere(p, pIter); zCollist = rbuObjIterGetIndexCols( p, pIter, &zImposterCols, &zImposterPK, &zWhere, &nBind ); @@ -165945,39 +221854,58 @@ static int rbuObjIterPrepareAll( if( p->rc==SQLITE_OK ){ char *zSql; if( rbuIsVacuum(p) ){ + char *zStart = 0; + if( nOffset ){ + zStart = rbuVacuumIndexStart(p, pIter); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + zSql = sqlite3_mprintf( - "SELECT %s, 0 AS rbu_control FROM '%q' ORDER BY %s%s", - zCollist, + "SELECT %s, 0 AS rbu_control FROM '%q' %s %s %s ORDER BY %s%s", + zCollist, pIter->zDataTbl, + zPart, + (zStart ? (zPart ? "AND" : "WHERE") : ""), zStart, zCollist, zLimit ); + sqlite3_free(zStart); }else if( pIter->eType==RBU_PK_EXTERNAL || pIter->eType==RBU_PK_NONE ){ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' ORDER BY %s%s", + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s ORDER BY %s%s", zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, zLimit + zPart, zCollist, zLimit ); }else{ zSql = sqlite3_mprintf( - "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' " + "SELECT %s, rbu_control FROM %s.'rbu_tmp_%q' %s " "UNION ALL " "SELECT %s, rbu_control FROM '%q' " - "WHERE typeof(rbu_control)='integer' AND rbu_control!=1 " + "%s %s typeof(rbu_control)='integer' AND rbu_control!=1 " "ORDER BY %s%s", - zCollist, p->zStateDb, pIter->zDataTbl, - zCollist, pIter->zDataTbl, + zCollist, p->zStateDb, pIter->zDataTbl, zPart, + zCollist, pIter->zDataTbl, + zPart, + (zPart ? "AND" : "WHERE"), zCollist, zLimit ); } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, zSql); + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu,&pIter->pSelect,pz,zSql); + }else{ + sqlite3_free(zSql); + } } sqlite3_free(zImposterCols); sqlite3_free(zImposterPK); sqlite3_free(zWhere); sqlite3_free(zBind); + sqlite3_free(zPart); }else{ int bRbuRowid = (pIter->eType==RBU_PK_VTAB) ||(pIter->eType==RBU_PK_NONE) @@ -166002,7 +221930,7 @@ static int rbuObjIterPrepareAll( if( p->rc==SQLITE_OK ){ p->rc = prepareFreeAndCollectError(p->dbMain, &pIter->pInsert, pz, sqlite3_mprintf( - "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", + "INSERT INTO \"%s%w\"(%s%s) VALUES(%s)", zWrite, zTbl, zCollist, (bRbuRowid ? ", _rowid_" : ""), zBindings ) ); @@ -166070,18 +221998,42 @@ static int rbuObjIterPrepareAll( /* Create the SELECT statement to read keys from data_xxx */ if( p->rc==SQLITE_OK ){ const char *zRbuRowid = ""; + char *zStart = 0; + char *zOrder = 0; if( bRbuRowid ){ zRbuRowid = rbuIsVacuum(p) ? ",_rowid_ " : ",rbu_rowid"; } - p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, - sqlite3_mprintf( - "SELECT %s,%s rbu_control%s FROM '%q'%s", - zCollist, - (rbuIsVacuum(p) ? "0 AS " : ""), - zRbuRowid, - pIter->zDataTbl, zLimit - ) - ); + + if( rbuIsVacuum(p) ){ + if( nOffset ){ + zStart = rbuVacuumTableStart(p, pIter, bRbuRowid, zWrite); + if( zStart ){ + sqlite3_free(zLimit); + zLimit = 0; + } + } + if( bRbuRowid ){ + zOrder = rbuMPrintf(p, "_rowid_"); + }else{ + zOrder = rbuObjIterGetPkList(p, pIter, "", ", ", ""); + } + } + + if( p->rc==SQLITE_OK ){ + p->rc = prepareFreeAndCollectError(p->dbRbu, &pIter->pSelect, pz, + sqlite3_mprintf( + "SELECT %s,%s rbu_control%s FROM '%q'%s %s %s %s", + zCollist, + (rbuIsVacuum(p) ? "0 AS " : ""), + zRbuRowid, + pIter->zDataTbl, (zStart ? zStart : ""), + (zOrder ? "ORDER BY" : ""), zOrder, + zLimit + ) + ); + } + sqlite3_free(zStart); + sqlite3_free(zOrder); } sqlite3_free(zWhere); @@ -166092,16 +222044,16 @@ static int rbuObjIterPrepareAll( sqlite3_free(zCollist); sqlite3_free(zLimit); } - + return p->rc; } /* ** Set output variable *ppStmt to point to an UPDATE statement that may ** be used to update the imposter table for the main table b-tree of the -** table object that pIter currently points to, assuming that the +** table object that pIter currently points to, assuming that the ** rbu_control column of the data_xyz table contains zMask. -** +** ** If the zMask string does not specify any columns to update, then this ** is not an error. Output variable *ppStmt is set to NULL in this case. */ @@ -166128,7 +222080,7 @@ static int rbuGetUpdateStmt( *pp = pUp->pNext; pUp->pNext = pIter->pRbuUpdate; pIter->pRbuUpdate = pUp; - *ppStmt = pUp->pUpdate; + *ppStmt = pUp->pUpdate; return SQLITE_OK; } nUp++; @@ -166158,7 +222110,7 @@ static int rbuGetUpdateStmt( const char *zPrefix = ""; if( pIter->eType!=RBU_PK_VTAB ) zPrefix = "rbu_imp_"; - zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", + zUpdate = sqlite3_mprintf("UPDATE \"%s%w\" SET %s WHERE %s", zPrefix, pIter->zTbl, zSet, zWhere ); p->rc = prepareFreeAndCollectError( @@ -166174,8 +222126,8 @@ static int rbuGetUpdateStmt( } static sqlite3 *rbuOpenDbhandle( - sqlite3rbu *p, - const char *zName, + sqlite3rbu *p, + const char *zName, int bUseVfs ){ sqlite3 *db = 0; @@ -166197,14 +222149,15 @@ static sqlite3 *rbuOpenDbhandle( static void rbuFreeState(RbuState *p){ if( p ){ sqlite3_free(p->zTbl); + sqlite3_free(p->zDataTbl); sqlite3_free(p->zIdx); sqlite3_free(p); } } /* -** Allocate an RbuState object and load the contents of the rbu_state -** table into it. Return a pointer to the new object. It is the +** Allocate an RbuState object and load the contents of the rbu_state +** table into it. Return a pointer to the new object. It is the ** responsibility of the caller to eventually free the object using ** sqlite3_free(). ** @@ -166220,7 +222173,7 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ pRet = (RbuState*)rbuMalloc(p, sizeof(RbuState)); if( pRet==0 ) return 0; - rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, + rc = prepareFreeAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, sqlite3_mprintf("SELECT k, v FROM %s.rbu_state", p->zStateDb) ); while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ @@ -166260,13 +222213,17 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ break; case RBU_STATE_OALSZ: - pRet->iOalSz = (u32)sqlite3_column_int64(pStmt, 1); + pRet->iOalSz = sqlite3_column_int64(pStmt, 1); break; case RBU_STATE_PHASEONESTEP: pRet->nPhaseOneStep = sqlite3_column_int64(pStmt, 1); break; + case RBU_STATE_DATATBL: + pRet->zDataTbl = rbuStrndup((char*)sqlite3_column_text(pStmt, 1), &rc); + break; + default: rc = SQLITE_CORRUPT; break; @@ -166283,17 +222240,26 @@ static RbuState *rbuLoadState(sqlite3rbu *p){ /* ** Open the database handle and attach the RBU database as "rbu". If an ** error occurs, leave an error code and message in the RBU handle. +** +** If argument dbMain is not NULL, then it is a database handle already +** open on the target database. Use this handle instead of opening a new +** one. */ -static void rbuOpenDatabase(sqlite3rbu *p){ - assert( p->rc==SQLITE_OK ); - assert( p->dbMain==0 && p->dbRbu==0 ); - assert( rbuIsVacuum(p) || p->zTarget!=0 ); +static void rbuOpenDatabase(sqlite3rbu *p, sqlite3 *dbMain, int *pbRetry){ + assert( p->rc || (p->dbMain==0 && p->dbRbu==0) ); + assert( p->rc || rbuIsVacuum(p) || p->zTarget!=0 ); + assert( dbMain==0 || rbuIsVacuum(p)==0 ); /* Open the RBU database */ p->dbRbu = rbuOpenDbhandle(p, p->zRbu, 1); + p->dbMain = dbMain; if( p->rc==SQLITE_OK && rbuIsVacuum(p) ){ sqlite3_file_control(p->dbRbu, "main", SQLITE_FCNTL_RBUCNT, (void*)p); + if( p->zState==0 ){ + const char *zFile = sqlite3_db_filename(p->dbRbu, "main"); + p->zState = rbuMPrintf(p, "file:///%s-vacuum?modeof=%s", zFile, zFile); + } } /* If using separate RBU and state databases, attach the state database to @@ -166321,9 +222287,9 @@ static void rbuOpenDatabase(sqlite3rbu *p){ int bOk = 0; sqlite3_stmt *pCnt = 0; p->rc = prepareAndCollectError(p->dbRbu, &pCnt, &p->zErrmsg, - "SELECT count(*) FROM stat.sqlite_master" + "SELECT count(*) FROM stat.sqlite_schema" ); - if( p->rc==SQLITE_OK + if( p->rc==SQLITE_OK && sqlite3_step(pCnt)==SQLITE_ROW && 1==sqlite3_column_int(pCnt, 0) ){ @@ -166336,7 +222302,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("invalid state database"); } - + if( p->rc==SQLITE_OK ){ p->rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); } @@ -166356,7 +222322,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ }else{ RbuState *pState = rbuLoadState(p); if( pState ){ - bOpen = (pState->eStage>RBU_STAGE_MOVE); + bOpen = (pState->eStage>=RBU_STAGE_MOVE); rbuFreeState(pState); } } @@ -166368,6 +222334,15 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( !rbuIsVacuum(p) ){ p->dbMain = rbuOpenDbhandle(p, p->zTarget, 1); }else if( p->pRbuFd->pWalFd ){ + if( pbRetry ){ + p->pRbuFd->bNolock = 0; + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + *pbRetry = 1; + return; + } p->rc = SQLITE_ERROR; p->zErrmsg = sqlite3_mprintf("cannot vacuum wal mode database"); }else{ @@ -166381,7 +222356,7 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( *zExtra=='\0' ) zExtra = 0; } - zTarget = sqlite3_mprintf("file:%s-vacuum?rbu_memory=1%s%s", + zTarget = sqlite3_mprintf("file:%s-vactmp?rbu_memory=1%s%s", sqlite3_db_filename(p->dbRbu, "main"), (zExtra==0 ? "" : "&"), (zExtra==0 ? "" : zExtra) ); @@ -166396,19 +222371,19 @@ static void rbuOpenDatabase(sqlite3rbu *p){ } if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbMain, + p->rc = sqlite3_create_function(p->dbMain, "rbu_tmp_insert", -1, SQLITE_UTF8, (void*)p, rbuTmpInsertFunc, 0, 0 ); } if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbMain, + p->rc = sqlite3_create_function(p->dbMain, "rbu_fossil_delta", 2, SQLITE_UTF8, 0, rbuFossilDeltaFunc, 0, 0 ); } if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_create_function(p->dbRbu, + p->rc = sqlite3_create_function(p->dbRbu, "rbu_target_name", -1, SQLITE_UTF8, (void*)p, rbuTargetNameFunc, 0, 0 ); } @@ -166416,9 +222391,9 @@ static void rbuOpenDatabase(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ p->rc = sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_RBU, (void*)p); } - rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_master"); + rbuMPrintfExec(p, p->dbMain, "SELECT * FROM sqlite_schema"); - /* Mark the database file just opened as an RBU target database. If + /* Mark the database file just opened as an RBU target database. If ** this call returns SQLITE_NOTFOUND, then the RBU vfs is not in use. ** This is an error. */ if( p->rc==SQLITE_OK ){ @@ -166458,18 +222433,20 @@ static void rbuFileSuffix3(const char *zBase, char *z){ #endif { int i, sz; - sz = sqlite3Strlen30(z); + sz = (int)strlen(z)&0xffffff; for(i=sz-1; i>0 && z[i]!='/' && z[i]!='.'; i--){} - if( z[i]=='.' && ALWAYS(sz>i+4) ) memmove(&z[i+1], &z[sz-3], 4); + if( z[i]=='.' && sz>i+4 ) memmove(&z[i+1], &z[sz-3], 4); } +#else + UNUSED_PARAMETER2(zBase,z); #endif } /* -** Return the current wal-index header checksum for the target database +** Return the current wal-index header checksum for the target database ** as a 64-bit integer. ** -** The checksum is store in the first page of xShmMap memory as an 8-byte +** The checksum is store in the first page of xShmMap memory as an 8-byte ** blob starting at byte offset 40. */ static i64 rbuShmChecksum(sqlite3rbu *p){ @@ -166479,7 +222456,7 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ u32 volatile *ptr; p->rc = pDb->pMethods->xShmMap(pDb, 0, 32*1024, 0, (void volatile**)&ptr); if( p->rc==SQLITE_OK ){ - iRet = ((i64)ptr[10] << 32) + ptr[11]; + iRet = (i64)(((u64)ptr[10] << 32) + ptr[11]); } } return iRet; @@ -166487,11 +222464,11 @@ static i64 rbuShmChecksum(sqlite3rbu *p){ /* ** This function is called as part of initializing or reinitializing an -** incremental checkpoint. +** incremental checkpoint. ** -** It populates the sqlite3rbu.aFrame[] array with the set of -** (wal frame -> db page) copy operations required to checkpoint the -** current wal file, and obtains the set of shm locks required to safely +** It populates the sqlite3rbu.aFrame[] array with the set of +** (wal frame -> db page) copy operations required to checkpoint the +** current wal file, and obtains the set of shm locks required to safely ** perform the copy operations directly on the file-system. ** ** If argument pState is not NULL, then the incremental checkpoint is @@ -166509,7 +222486,7 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ if( pState==0 ){ p->eStage = 0; if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_master", 0, 0, 0); + p->rc = sqlite3_exec(p->dbMain, "SELECT * FROM sqlite_schema", 0, 0, 0); } } @@ -166526,38 +222503,57 @@ static void rbuSetupCheckpoint(sqlite3rbu *p, RbuState *pState){ ** would be read/written are recorded in the sqlite3rbu.aFrame[] ** array. ** - ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, + ** * Calls to xShmLock(UNLOCK) to release the exclusive shm WRITER, ** READ0 and CHECKPOINT locks taken as part of the checkpoint are ** no-ops. These locks will not be released until the connection ** is closed. ** - ** * Attempting to xSync() the database file causes an SQLITE_INTERNAL + ** * Attempting to xSync() the database file causes an SQLITE_NOTICE ** error. ** ** As a result, unless an error (i.e. OOM or SQLITE_BUSY) occurs, the - ** checkpoint below fails with SQLITE_INTERNAL, and leaves the aFrame[] - ** array populated with a set of (frame -> page) mappings. Because the - ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy - ** data from the wal file into the database file according to the + ** checkpoint below fails with SQLITE_NOTICE, and leaves the aFrame[] + ** array populated with a set of (frame -> page) mappings. Because the + ** WRITER, CHECKPOINT and READ0 locks are still held, it is safe to copy + ** data from the wal file into the database file according to the ** contents of aFrame[]. */ if( p->rc==SQLITE_OK ){ int rc2; p->eStage = RBU_STAGE_CAPTURE; rc2 = sqlite3_exec(p->dbMain, "PRAGMA main.wal_checkpoint=restart", 0, 0,0); - if( rc2!=SQLITE_INTERNAL ) p->rc = rc2; + if( rc2!=SQLITE_NOTICE ) p->rc = rc2; } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && p->nFrame>0 ){ p->eStage = RBU_STAGE_CKPT; p->nStep = (pState ? pState->nRow : 0); p->aBuf = rbuMalloc(p, p->pgsz); p->iWalCksum = rbuShmChecksum(p); } - if( p->rc==SQLITE_OK && pState && pState->iWalCksum!=p->iWalCksum ){ - p->rc = SQLITE_DONE; - p->eStage = RBU_STAGE_DONE; + if( p->rc==SQLITE_OK ){ + if( p->nFrame==0 || (pState && pState->iWalCksum!=p->iWalCksum) ){ + p->rc = SQLITE_DONE; + p->eStage = RBU_STAGE_DONE; + }else{ + int nSectorSize; + sqlite3_file *pDb = p->pTargetFd->pReal; + sqlite3_file *pWal = p->pTargetFd->pWalFd->pReal; + assert( p->nPagePerSector==0 ); + nSectorSize = pDb->pMethods->xSectorSize(pDb); + if( nSectorSize>p->pgsz ){ + p->nPagePerSector = nSectorSize / p->pgsz; + }else{ + p->nPagePerSector = 1; + } + + /* Call xSync() on the wal file. This causes SQLite to sync the + ** directory in which the target database and the wal file reside, in + ** case it has not been synced since the rename() call in + ** rbuMoveOalFile(). */ + p->rc = pWal->pMethods->xSync(pWal, SQLITE_SYNC_NORMAL); + } } } @@ -166572,7 +222568,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ if( pRbu->mLock!=mReq ){ pRbu->rc = SQLITE_BUSY; - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } pRbu->pgsz = iAmt; @@ -166595,7 +222591,7 @@ static int rbuCaptureWalRead(sqlite3rbu *pRbu, i64 iOff, int iAmt){ /* ** Called when a page of data is written to offset iOff of the database -** file while the rbu handle is in capture mode. Record the page number +** file while the rbu handle is in capture mode. Record the page number ** of the page being written in the aFrame[] array. */ static int rbuCaptureDbWrite(sqlite3rbu *pRbu, i64 iOff){ @@ -166622,17 +222618,49 @@ static void rbuCheckpointFrame(sqlite3rbu *p, RbuFrame *pFrame){ p->rc = pDb->pMethods->xWrite(pDb, p->aBuf, p->pgsz, iOff); } +/* +** This value is copied from the definition of ZIPVFS_CTRL_FILE_POINTER +** in zipvfs.h. +*/ +#define RBU_ZIPVFS_CTRL_FILE_POINTER 230439 /* -** Take an EXCLUSIVE lock on the database file. +** Take an EXCLUSIVE lock on the database file. Return SQLITE_OK if +** successful, or an SQLite error code otherwise. */ -static void rbuLockDatabase(sqlite3rbu *p){ - sqlite3_file *pReal = p->pTargetFd->pReal; - assert( p->rc==SQLITE_OK ); - p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_SHARED); - if( p->rc==SQLITE_OK ){ - p->rc = pReal->pMethods->xLock(pReal, SQLITE_LOCK_EXCLUSIVE); +static int rbuLockDatabase(sqlite3 *db){ + int rc = SQLITE_OK; + sqlite3_file *fd = 0; + + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + if( fd ){ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = fd->pMethods->xUnlock(fd, SQLITE_LOCK_NONE); + } + sqlite3_file_control(db, "main", RBU_ZIPVFS_CTRL_FILE_POINTER, &fd); + }else{ + sqlite3_file_control(db, "main", SQLITE_FCNTL_FILE_POINTER, &fd); } + + if( rc==SQLITE_OK && fd->pMethods ){ + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_SHARED); + if( rc==SQLITE_OK ){ + rc = fd->pMethods->xLock(fd, SQLITE_LOCK_EXCLUSIVE); + } + } + return rc; +} + +/* +** Return true if the database handle passed as the only argument +** was opened with the rbu_exclusive_checkpoint=1 URI parameter +** specified. Or false otherwise. +*/ +static int rbuExclusiveCheckpoint(sqlite3 *db){ + const char *zUri = sqlite3_db_filename(db, 0); + return sqlite3_uri_boolean(zUri, RBU_EXCLUSIVE_CHECKPOINT, 0); } #if defined(_WIN32_WCE) @@ -166663,7 +222691,7 @@ static LPWSTR rbuWinUtf8ToUnicode(const char *zFilename){ ** The RBU handle is currently in RBU_STAGE_OAL state, with a SHARED lock ** on the database file. This proc moves the *-oal file to the *-wal path, ** then reopens the database file (this time in vanilla, non-oal, WAL mode). -** If an error occurs, leave an error code and error message in the rbu +** If an error occurs, leave an error code and error message in the rbu ** handle. */ static void rbuMoveOalFile(sqlite3rbu *p){ @@ -166685,54 +222713,43 @@ static void rbuMoveOalFile(sqlite3rbu *p){ }else{ /* Move the *-oal file to *-wal. At this point connection p->db is ** holding a SHARED lock on the target database file (because it is - ** in WAL mode). So no other connection may be writing the db. + ** in WAL mode). So no other connection may be writing the db. ** ** In order to ensure that there are no database readers, an EXCLUSIVE ** lock is obtained here before the *-oal is moved to *-wal. */ - rbuLockDatabase(p); + sqlite3 *dbMain = 0; + rbuFileSuffix3(zBase, zWal); + rbuFileSuffix3(zBase, zOal); + + /* Re-open the databases. */ + rbuObjIterFinalize(&p->objiter); + sqlite3_close(p->dbRbu); + sqlite3_close(p->dbMain); + p->dbMain = 0; + p->dbRbu = 0; + + dbMain = rbuOpenDbhandle(p, p->zTarget, 1); + if( dbMain ){ + assert( p->rc==SQLITE_OK ); + p->rc = rbuLockDatabase(dbMain); + } + if( p->rc==SQLITE_OK ){ - rbuFileSuffix3(zBase, zWal); - rbuFileSuffix3(zBase, zOal); + p->rc = p->xRename(p->pRenameArg, zOal, zWal); + } - /* Re-open the databases. */ - rbuObjIterFinalize(&p->objiter); - sqlite3_close(p->dbRbu); - sqlite3_close(p->dbMain); - p->dbMain = 0; - p->dbRbu = 0; + if( p->rc!=SQLITE_OK + || rbuIsVacuum(p) + || rbuExclusiveCheckpoint(dbMain)==0 + ){ + sqlite3_close(dbMain); + dbMain = 0; + } -#if defined(_WIN32_WCE) - { - LPWSTR zWideOal; - LPWSTR zWideWal; - - zWideOal = rbuWinUtf8ToUnicode(zOal); - if( zWideOal ){ - zWideWal = rbuWinUtf8ToUnicode(zWal); - if( zWideWal ){ - if( MoveFileW(zWideOal, zWideWal) ){ - p->rc = SQLITE_OK; - }else{ - p->rc = SQLITE_IOERR; - } - sqlite3_free(zWideWal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - sqlite3_free(zWideOal); - }else{ - p->rc = SQLITE_IOERR_NOMEM; - } - } -#else - p->rc = rename(zOal, zWal) ? SQLITE_IOERR : SQLITE_OK; -#endif - - if( p->rc==SQLITE_OK ){ - rbuOpenDatabase(p); - rbuSetupCheckpoint(p, 0); - } + if( p->rc==SQLITE_OK ){ + rbuOpenDatabase(p, dbMain, 0); + rbuSetupCheckpoint(p, 0); } } @@ -166843,8 +222860,8 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ /* If this is an INSERT into a table b-tree and the table has an ** explicit INTEGER PRIMARY KEY, check that this is not an attempt ** to write a NULL into the IPK column. That is not permitted. */ - if( eType==RBU_INSERT - && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i] + if( eType==RBU_INSERT + && pIter->zIdx==0 && pIter->eType==RBU_PK_IPK && pIter->abTblPk[i] && sqlite3_column_type(pIter->pSelect, i)==SQLITE_NULL ){ p->rc = SQLITE_MISMATCH; @@ -166861,18 +222878,18 @@ static void rbuStepOneOp(sqlite3rbu *p, int eType){ if( p->rc ) return; } if( pIter->zIdx==0 ){ - if( pIter->eType==RBU_PK_VTAB - || pIter->eType==RBU_PK_NONE - || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) + if( pIter->eType==RBU_PK_VTAB + || pIter->eType==RBU_PK_NONE + || (pIter->eType==RBU_PK_EXTERNAL && rbuIsVacuum(p)) ){ - /* For a virtual table, or a table with no primary key, the + /* For a virtual table, or a table with no primary key, the ** SELECT statement is: ** ** SELECT , rbu_control, rbu_rowid FROM .... ** ** Hence column_value(pIter->nCol+1). */ - assertColumnName(pIter->pSelect, pIter->nCol+1, + assertColumnName(pIter->pSelect, pIter->nCol+1, rbuIsVacuum(p) ? "rowid" : "rbu_rowid" ); pVal = sqlite3_column_value(pIter->pSelect, pIter->nCol+1); @@ -166936,8 +222953,8 @@ static int rbuStep(sqlite3rbu *p){ p->rc = sqlite3_bind_value(pUpdate, i+1, pVal); } } - if( p->rc==SQLITE_OK - && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) + if( p->rc==SQLITE_OK + && (pIter->eType==RBU_PK_VTAB || pIter->eType==RBU_PK_NONE) ){ /* Bind the rbu_rowid value to column _rowid_ */ assertColumnName(pIter->pSelect, pIter->nCol+1, "rbu_rowid"); @@ -166967,7 +222984,7 @@ static void rbuIncrSchemaCookie(sqlite3rbu *p){ int iCookie = 1000000; sqlite3_stmt *pStmt; - p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, + p->rc = prepareAndCollectError(dbread, &pStmt, &p->zErrmsg, "PRAGMA schema_version" ); if( p->rc==SQLITE_OK ){ @@ -166999,28 +223016,30 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ int rc; assert( p->zErrmsg==0 ); - rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg, + rc = prepareFreeAndCollectError(p->dbRbu, &pInsert, &p->zErrmsg, sqlite3_mprintf( "INSERT OR REPLACE INTO %s.rbu_state(k, v) VALUES " "(%d, %d), " "(%d, %Q), " "(%d, %Q), " "(%d, %d), " - "(%d, %d), " "(%d, %lld), " "(%d, %lld), " "(%d, %lld), " - "(%d, %lld) ", + "(%d, %lld), " + "(%d, %lld), " + "(%d, %Q) ", p->zStateDb, RBU_STATE_STAGE, eStage, - RBU_STATE_TBL, p->objiter.zTbl, - RBU_STATE_IDX, p->objiter.zIdx, - RBU_STATE_ROW, p->nStep, + RBU_STATE_TBL, p->objiter.zTbl, + RBU_STATE_IDX, p->objiter.zIdx, + RBU_STATE_ROW, p->nStep, RBU_STATE_PROGRESS, p->nProgress, RBU_STATE_CKPT, p->iWalCksum, RBU_STATE_COOKIE, (i64)pFd->iCookie, RBU_STATE_OALSZ, p->iOalSz, - RBU_STATE_PHASEONESTEP, p->nPhaseOneStep + RBU_STATE_PHASEONESTEP, p->nPhaseOneStep, + RBU_STATE_DATATBL, p->objiter.zDataTbl ) ); assert( pInsert==0 || rc==SQLITE_OK ); @@ -167035,7 +223054,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ /* -** The second argument passed to this function is the name of a PRAGMA +** The second argument passed to this function is the name of a PRAGMA ** setting - "page_size", "auto_vacuum", "user_version" or "application_id". ** This function executes the following on sqlite3rbu.dbRbu: ** @@ -167054,7 +223073,7 @@ static void rbuSaveState(sqlite3rbu *p, int eStage){ static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ if( p->rc==SQLITE_OK ){ sqlite3_stmt *pPragma = 0; - p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, + p->rc = prepareFreeAndCollectError(p->dbRbu, &pPragma, &p->zErrmsg, sqlite3_mprintf("PRAGMA main.%s", zPragma) ); if( p->rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pPragma) ){ @@ -167067,7 +223086,7 @@ static void rbuCopyPragma(sqlite3rbu *p, const char *zPragma){ } /* -** The RBU handle passed as the only argument has just been opened and +** The RBU handle passed as the only argument has just been opened and ** the state database is empty. If this RBU handle was opened for an ** RBU vacuum operation, create the schema in the target db. */ @@ -167078,8 +223097,8 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ assert( rbuIsVacuum(p) ); p->rc = sqlite3_exec(p->dbMain, "PRAGMA writable_schema=1", 0,0, &p->zErrmsg); if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, - "SELECT sql FROM sqlite_master WHERE sql!='' AND rootpage!=0" + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT sql FROM sqlite_schema WHERE sql!='' AND rootpage!=0" " AND name!='sqlite_sequence' " " ORDER BY type DESC" ); @@ -167093,14 +223112,14 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ if( p->rc!=SQLITE_OK ) return; if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, - "SELECT * FROM sqlite_master WHERE rootpage=0 OR rootpage IS NULL" + p->rc = prepareAndCollectError(p->dbRbu, &pSql, &p->zErrmsg, + "SELECT * FROM sqlite_schema WHERE rootpage=0 OR rootpage IS NULL" ); } if( p->rc==SQLITE_OK ){ - p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, - "INSERT INTO sqlite_master VALUES(?,?,?,?,?)" + p->rc = prepareAndCollectError(p->dbMain, &pInsert, &p->zErrmsg, + "INSERT INTO sqlite_schema VALUES(?,?,?,?,?)" ); } @@ -167123,7 +223142,7 @@ static void rbuCreateTargetSchema(sqlite3rbu *p){ /* ** Step the RBU object. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *p){ +SQLITE_API int sqlite3rbu_step(sqlite3rbu *p){ if( p ){ switch( p->eStage ){ case RBU_STAGE_OAL: { @@ -167140,11 +223159,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *p){ while( p->rc==SQLITE_OK && pIter->zTbl ){ if( pIter->bCleanup ){ - /* Clean up the rbu_tmp_xxx table for the previous table. It + /* Clean up the rbu_tmp_xxx table for the previous table. It ** cannot be dropped as there are currently active SQL statements. ** But the contents can be deleted. */ if( rbuIsVacuum(p)==0 && pIter->abIndexed ){ - rbuMPrintfExec(p, p->dbRbu, + rbuMPrintfExec(p, p->dbRbu, "DELETE FROM %s.'rbu_tmp_%q'", p->zStateDb, pIter->zDataTbl ); } @@ -167194,10 +223213,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *p){ if( p->rc==SQLITE_OK ){ if( p->nStep>=p->nFrame ){ sqlite3_file *pDb = p->pTargetFd->pReal; - + /* Sync the db file */ p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); - + /* Update nBackfill */ if( p->rc==SQLITE_OK ){ void volatile *ptr; @@ -167206,15 +223225,32 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_step(sqlite3rbu *p){ ((u32 volatile*)ptr)[24] = p->iMaxFrame; } } - + if( p->rc==SQLITE_OK ){ p->eStage = RBU_STAGE_DONE; p->rc = SQLITE_DONE; } }else{ - RbuFrame *pFrame = &p->aFrame[p->nStep]; - rbuCheckpointFrame(p, pFrame); - p->nStep++; + /* At one point the following block copied a single frame from the + ** wal file to the database file. So that one call to sqlite3rbu_step() + ** checkpointed a single frame. + ** + ** However, if the sector-size is larger than the page-size, and the + ** application calls sqlite3rbu_savestate() or close() immediately + ** after this step, then rbu_step() again, then a power failure occurs, + ** then the database page written here may be damaged. Work around + ** this by checkpointing frames until the next page in the aFrame[] + ** lies on a different disk sector to the current one. */ + u32 iSector; + do{ + RbuFrame *pFrame = &p->aFrame[p->nStep]; + iSector = (pFrame->iDbPage-1) / p->nPagePerSector; + rbuCheckpointFrame(p, pFrame); + p->nStep++; + }while( p->nStepnFrame + && iSector==((p->aFrame[p->nStep].iDbPage-1) / p->nPagePerSector) + && p->rc==SQLITE_OK + ); } p->nProgress++; } @@ -167257,9 +223293,10 @@ static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){ RbuObjIter *pIter = &p->objiter; int rc = SQLITE_OK; - while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup + while( rc==SQLITE_OK && pIter->zTbl && (pIter->bCleanup || rbuStrCompare(pIter->zIdx, pState->zIdx) - || rbuStrCompare(pIter->zTbl, pState->zTbl) + || (pState->zDataTbl==0 && rbuStrCompare(pIter->zTbl, pState->zTbl)) + || (pState->zDataTbl && rbuStrCompare(pIter->zDataTbl, pState->zDataTbl)) )){ rc = rbuObjIterNext(p, pIter); } @@ -167286,7 +223323,8 @@ static void rbuSetupOal(sqlite3rbu *p, RbuState *pState){ static void rbuDeleteOalFile(sqlite3rbu *p){ char *zOal = rbuMPrintf(p, "%s-oal", p->zTarget); if( zOal ){ - sqlite3_vfs *pVfs = sqlite3_vfs_find(0); + sqlite3_vfs *pVfs = 0; + sqlite3_file_control(p->dbMain, "main", SQLITE_FCNTL_VFS_POINTER, &pVfs); assert( pVfs && p->rc==SQLITE_OK && p->zErrmsg==0 ); pVfs->xDelete(pVfs, zOal, 0); sqlite3_free(zOal); @@ -167311,6 +223349,7 @@ static void rbuCreateVfs(sqlite3rbu *p){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zRnd); assert( pVfs ); p->zVfsName = pVfs->zName; + ((rbu_vfs*)pVfs)->pRbu = p; } } @@ -167331,7 +223370,7 @@ static void rbuDeleteVfs(sqlite3rbu *p){ ** the number of auxilliary indexes on the table. */ static void rbuIndexCntFunc( - sqlite3_context *pCtx, + sqlite3_context *pCtx, int nVal, sqlite3_value **apVal ){ @@ -167339,11 +223378,13 @@ static void rbuIndexCntFunc( sqlite3_stmt *pStmt = 0; char *zErrmsg = 0; int rc; + sqlite3 *db = (rbuIsVacuum(p) ? p->dbRbu : p->dbMain); assert( nVal==1 ); - - rc = prepareFreeAndCollectError(p->dbMain, &pStmt, &zErrmsg, - sqlite3_mprintf("SELECT count(*) FROM sqlite_master " + UNUSED_PARAMETER(nVal); + + rc = prepareFreeAndCollectError(db, &pStmt, &zErrmsg, + sqlite3_mprintf("SELECT count(*) FROM sqlite_schema " "WHERE type='index' AND tbl_name = %Q", sqlite3_value_text(apVal[0])) ); if( rc!=SQLITE_OK ){ @@ -167357,7 +223398,7 @@ static void rbuIndexCntFunc( if( rc==SQLITE_OK ){ sqlite3_result_int(pCtx, nIndex); }else{ - sqlite3_result_error(pCtx, sqlite3_errmsg(p->dbMain), -1); + sqlite3_result_error(pCtx, sqlite3_errmsg(db), -1); } } @@ -167376,7 +223417,7 @@ static void rbuIndexCntFunc( ** and the cnt column the number of rows it contains. ** ** sqlite3rbu.nPhaseOneStep is initialized to the sum of (1 + nIndex) * cnt -** for all rows in the rbu_count table, where nIndex is the number of +** for all rows in the rbu_count table, where nIndex is the number of ** indexes on the corresponding target database table. */ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ @@ -167386,15 +223427,15 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ p->nPhaseOneStep = -1; - p->rc = sqlite3_create_function(p->dbRbu, + p->rc = sqlite3_create_function(p->dbRbu, "rbu_index_cnt", 1, SQLITE_UTF8, (void*)p, rbuIndexCntFunc, 0, 0 ); - + /* Check for the rbu_count table. If it does not exist, or if an error ** occurs, nPhaseOneStep will be left set to -1. */ if( p->rc==SQLITE_OK ){ p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, - "SELECT 1 FROM sqlite_master WHERE tbl_name = 'rbu_count'" + "SELECT 1 FROM sqlite_schema WHERE tbl_name = 'rbu_count'" ); } if( p->rc==SQLITE_OK ){ @@ -167403,7 +223444,7 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ } p->rc = sqlite3_finalize(pStmt); } - + if( p->rc==SQLITE_OK && bExists ){ p->rc = prepareAndCollectError(p->dbRbu, &pStmt, &p->zErrmsg, "SELECT sum(cnt * (1 + rbu_index_cnt(rbu_target_name(tbl))))" @@ -167421,15 +223462,14 @@ static void rbuInitPhaseOneSteps(sqlite3rbu *p){ static sqlite3rbu *openRbuHandle( - const char *zTarget, + const char *zTarget, const char *zRbu, const char *zState ){ sqlite3rbu *p; size_t nTarget = zTarget ? strlen(zTarget) : 0; size_t nRbu = strlen(zRbu); - size_t nState = zState ? strlen(zState) : 0; - size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1+ nState+1; + size_t nByte = sizeof(sqlite3rbu) + nTarget+1 + nRbu+1; p = (sqlite3rbu*)sqlite3_malloc64(nByte); if( p ){ @@ -167437,11 +223477,13 @@ static sqlite3rbu *openRbuHandle( /* Create the custom VFS. */ memset(p, 0, sizeof(sqlite3rbu)); + sqlite3rbu_rename_handler(p, 0, 0); rbuCreateVfs(p); /* Open the target, RBU and state databases */ if( p->rc==SQLITE_OK ){ char *pCsr = (char*)&p[1]; + int bRetry = 0; if( zTarget ){ p->zTarget = pCsr; memcpy(p->zTarget, zTarget, nTarget+1); @@ -167451,10 +223493,20 @@ static sqlite3rbu *openRbuHandle( memcpy(p->zRbu, zRbu, nRbu+1); pCsr += nRbu+1; if( zState ){ - p->zState = pCsr; - memcpy(p->zState, zState, nState+1); + p->zState = rbuMPrintf(p, "%s", zState); + } + + /* If the first attempt to open the database file fails and the bRetry + ** flag it set, this means that the db was not opened because it seemed + ** to be a wal-mode db. But, this may have happened due to an earlier + ** RBU vacuum operation leaving an old wal file in the directory. + ** If this is the case, it will have been checkpointed and deleted + ** when the handle was closed and a second attempt to open the + ** database may succeed. */ + rbuOpenDatabase(p, 0, &bRetry); + if( bRetry ){ + rbuOpenDatabase(p, 0, 0); } - rbuOpenDatabase(p); } if( p->rc==SQLITE_OK ){ @@ -167462,7 +223514,7 @@ static sqlite3rbu *openRbuHandle( assert( pState || p->rc!=SQLITE_OK ); if( p->rc==SQLITE_OK ){ - if( pState->eStage==0 ){ + if( pState->eStage==0 ){ rbuDeleteOalFile(p); rbuInitPhaseOneSteps(p); p->eStage = RBU_STAGE_OAL; @@ -167486,15 +223538,15 @@ static sqlite3rbu *openRbuHandle( } } - if( p->rc==SQLITE_OK + if( p->rc==SQLITE_OK && (p->eStage==RBU_STAGE_OAL || p->eStage==RBU_STAGE_MOVE) && pState->eStage!=0 ){ rbu_file *pFd = (rbuIsVacuum(p) ? p->pRbuFd : p->pTargetFd); - if( pFd->iCookie!=pState->iCookie ){ + if( pFd->iCookie!=pState->iCookie ){ /* At this point (pTargetFd->iCookie) contains the value of the - ** change-counter cookie (the thing that gets incremented when a - ** transaction is committed in rollback mode) currently stored on + ** change-counter cookie (the thing that gets incremented when a + ** transaction is committed in rollback mode) currently stored on ** page 1 of the database file. */ p->rc = SQLITE_BUSY; p->zErrmsg = sqlite3_mprintf("database modified during rbu %s", @@ -167506,30 +223558,7 @@ static sqlite3rbu *openRbuHandle( if( p->rc==SQLITE_OK ){ if( p->eStage==RBU_STAGE_OAL ){ sqlite3 *db = p->dbMain; - - if( pState->eStage==0 && rbuIsVacuum(p) ){ - rbuCopyPragma(p, "page_size"); - rbuCopyPragma(p, "auto_vacuum"); - } - - /* Open transactions both databases. The *-oal file is opened or - ** created at this point. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); - } - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); - } - - /* Check if the main database is a zipvfs db. If it is, set the upper - ** level pager to use "journal_mode=off". This prevents it from - ** generating a large journal using a temp file. */ - if( p->rc==SQLITE_OK ){ - int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); - if( frc==SQLITE_OK ){ - p->rc = sqlite3_exec(db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); - } - } + p->rc = sqlite3_exec(p->dbRbu, "BEGIN", 0, 0, &p->zErrmsg); /* Point the object iterator at the first object */ if( p->rc==SQLITE_OK ){ @@ -167540,15 +223569,45 @@ static sqlite3rbu *openRbuHandle( ** update finished. */ if( p->rc==SQLITE_OK && p->objiter.zTbl==0 ){ p->rc = SQLITE_DONE; - } + p->eStage = RBU_STAGE_DONE; + }else{ + if( p->rc==SQLITE_OK && pState->eStage==0 && rbuIsVacuum(p) ){ + rbuCopyPragma(p, "page_size"); + rbuCopyPragma(p, "auto_vacuum"); + } - if( p->rc==SQLITE_OK ){ - rbuSetupOal(p, pState); - } + /* Open transactions both databases. The *-oal file is opened or + ** created at this point. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3_exec(db, "BEGIN IMMEDIATE", 0, 0, &p->zErrmsg); + } + /* Check if the main database is a zipvfs db. If it is, set the upper + ** level pager to use "journal_mode=off". This prevents it from + ** generating a large journal using a temp file. */ + if( p->rc==SQLITE_OK ){ + int frc = sqlite3_file_control(db, "main", SQLITE_FCNTL_ZIPVFS, 0); + if( frc==SQLITE_OK ){ + p->rc = sqlite3_exec( + db, "PRAGMA journal_mode=off",0,0,&p->zErrmsg); + } + } + + if( p->rc==SQLITE_OK ){ + rbuSetupOal(p, pState); + } + } }else if( p->eStage==RBU_STAGE_MOVE ){ /* no-op */ }else if( p->eStage==RBU_STAGE_CKPT ){ + if( !rbuIsVacuum(p) && rbuExclusiveCheckpoint(p->dbMain) ){ + /* If the rbu_exclusive_checkpoint=1 URI parameter was specified + ** and an incremental checkpoint is being resumed, attempt an + ** exclusive lock on the db file. If this fails, so be it. */ + p->eStage = RBU_STAGE_DONE; + rbuLockDatabase(p->dbMain); + p->eStage = RBU_STAGE_CKPT; + } rbuSetupCheckpoint(p, pState); }else if( p->eStage==RBU_STAGE_DONE ){ p->rc = SQLITE_DONE; @@ -167564,24 +223623,45 @@ static sqlite3rbu *openRbuHandle( } /* -** Open and return a new RBU handle. +** Allocate and return an RBU handle with all fields zeroed except for the +** error code, which is set to SQLITE_MISUSE. */ -SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_open( - const char *zTarget, +static sqlite3rbu *rbuMisuseError(void){ + sqlite3rbu *pRet; + pRet = sqlite3_malloc64(sizeof(sqlite3rbu)); + if( pRet ){ + memset(pRet, 0, sizeof(sqlite3rbu)); + pRet->rc = SQLITE_MISUSE; + } + return pRet; +} + +/* +** Open and return a new RBU handle. +*/ +SQLITE_API sqlite3rbu *sqlite3rbu_open( + const char *zTarget, const char *zRbu, const char *zState ){ - /* TODO: Check that zTarget and zRbu are non-NULL */ + if( zTarget==0 || zRbu==0 ){ return rbuMisuseError(); } return openRbuHandle(zTarget, zRbu, zState); } /* ** Open a handle to begin or resume an RBU VACUUM operation. */ -SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_vacuum( - const char *zTarget, +SQLITE_API sqlite3rbu *sqlite3rbu_vacuum( + const char *zTarget, const char *zState ){ + if( zTarget==0 ){ return rbuMisuseError(); } + if( zState ){ + size_t n = strlen(zState); + if( n>=7 && 0==memcmp("-vactmp", &zState[n-7], 7) ){ + return rbuMisuseError(); + } + } /* TODO: Check that both arguments are non-NULL */ return openRbuHandle(0, zTarget, zState); } @@ -167589,7 +223669,7 @@ SQLITE_API sqlite3rbu *SQLITE_STDCALL sqlite3rbu_vacuum( /* ** Return the database handle used by pRbu. */ -SQLITE_API sqlite3 *SQLITE_STDCALL sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ +SQLITE_API sqlite3 *sqlite3rbu_db(sqlite3rbu *pRbu, int bRbu){ sqlite3 *db = 0; if( pRbu ){ db = (bRbu ? pRbu->dbRbu : pRbu->dbMain); @@ -167621,7 +223701,7 @@ static void rbuEditErrmsg(sqlite3rbu *p){ /* ** Close the RBU handle. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ +SQLITE_API int sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ int rc; if( p ){ @@ -167630,6 +223710,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ p->rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, &p->zErrmsg); } + /* Sync the db file if currently doing an incremental checkpoint */ + if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + p->rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + rbuSaveState(p, p->eStage); if( p->rc==SQLITE_OK && p->eStage==RBU_STAGE_OAL ){ @@ -167640,8 +223726,8 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ rbuObjIterFinalize(&p->objiter); /* If this is an RBU vacuum handle and the vacuum has either finished - ** successfully or encountered an error, delete the contents of the - ** state table. This causes the next call to sqlite3rbu_vacuum() + ** successfully or encountered an error, delete the contents of the + ** state table. This causes the next call to sqlite3rbu_vacuum() ** specifying the current target and state databases to start a new ** vacuum from scratch. */ if( rbuIsVacuum(p) && p->rc!=SQLITE_OK && p->dbRbu ){ @@ -167652,13 +223738,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ /* Close the open database handle and VFS object. */ sqlite3_close(p->dbRbu); sqlite3_close(p->dbMain); + assert( p->szTemp==0 ); rbuDeleteVfs(p); sqlite3_free(p->aBuf); sqlite3_free(p->aFrame); rbuEditErrmsg(p); rc = p->rc; - *pzErrmsg = p->zErrmsg; + if( pzErrmsg ){ + *pzErrmsg = p->zErrmsg; + }else{ + sqlite3_free(p->zErrmsg); + } + sqlite3_free(p->zState); sqlite3_free(p); }else{ rc = SQLITE_NOMEM; @@ -167668,11 +223760,11 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_close(sqlite3rbu *p, char **pzErrmsg){ } /* -** Return the total number of key-value operations (inserts, deletes or +** Return the total number of key-value operations (inserts, deletes or ** updates) that have been performed on the target database since the ** current RBU update was started. */ -SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu){ +SQLITE_API sqlite3_int64 sqlite3rbu_progress(sqlite3rbu *pRbu){ return pRbu->nProgress; } @@ -167680,7 +223772,7 @@ SQLITE_API sqlite3_int64 SQLITE_STDCALL sqlite3rbu_progress(sqlite3rbu *pRbu){ ** Return permyriadage progress indications for the two main stages of ** an RBU update. */ -SQLITE_API void SQLITE_STDCALL sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ +SQLITE_API void sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, int *pnTwo){ const int MAX_PROGRESS = 10000; switch( p->eStage ){ case RBU_STAGE_OAL: @@ -167712,9 +223804,39 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_bp_progress(sqlite3rbu *p, int *pnOne, } } -SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ +/* +** Return the current state of the RBU vacuum or update operation. +*/ +SQLITE_API int sqlite3rbu_state(sqlite3rbu *p){ + int aRes[] = { + 0, SQLITE_RBU_STATE_OAL, SQLITE_RBU_STATE_MOVE, + 0, SQLITE_RBU_STATE_CHECKPOINT, SQLITE_RBU_STATE_DONE + }; + + assert( RBU_STAGE_OAL==1 ); + assert( RBU_STAGE_MOVE==2 ); + assert( RBU_STAGE_CKPT==4 ); + assert( RBU_STAGE_DONE==5 ); + assert( aRes[RBU_STAGE_OAL]==SQLITE_RBU_STATE_OAL ); + assert( aRes[RBU_STAGE_MOVE]==SQLITE_RBU_STATE_MOVE ); + assert( aRes[RBU_STAGE_CKPT]==SQLITE_RBU_STATE_CHECKPOINT ); + assert( aRes[RBU_STAGE_DONE]==SQLITE_RBU_STATE_DONE ); + + if( p->rc!=SQLITE_OK && p->rc!=SQLITE_DONE ){ + return SQLITE_RBU_STATE_ERROR; + }else{ + assert( p->rc!=SQLITE_DONE || p->eStage==RBU_STAGE_DONE ); + assert( p->eStage==RBU_STAGE_OAL + || p->eStage==RBU_STAGE_MOVE + || p->eStage==RBU_STAGE_CKPT + || p->eStage==RBU_STAGE_DONE + ); + return aRes[p->eStage]; + } +} + +SQLITE_API int sqlite3rbu_savestate(sqlite3rbu *p){ int rc = p->rc; - if( rc==SQLITE_DONE ) return SQLITE_OK; assert( p->eStage>=RBU_STAGE_OAL && p->eStage<=RBU_STAGE_DONE ); @@ -167723,6 +223845,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "COMMIT", 0, 0, 0); } + /* Sync the db file */ + if( rc==SQLITE_OK && p->eStage==RBU_STAGE_CKPT ){ + sqlite3_file *pDb = p->pTargetFd->pReal; + rc = pDb->pMethods->xSync(pDb, SQLITE_SYNC_NORMAL); + } + p->rc = rc; rbuSaveState(p, p->eStage); rc = p->rc; @@ -167730,7 +223858,10 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ if( p->eStage==RBU_STAGE_OAL ){ assert( rc!=SQLITE_DONE ); if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "COMMIT", 0, 0, 0); - if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbRbu, "BEGIN IMMEDIATE", 0, 0, 0); + if( rc==SQLITE_OK ){ + const char *zBegin = rbuIsVacuum(p) ? "BEGIN" : "BEGIN IMMEDIATE"; + rc = sqlite3_exec(p->dbRbu, zBegin, 0, 0, 0); + } if( rc==SQLITE_OK ) rc = sqlite3_exec(p->dbMain, "BEGIN IMMEDIATE", 0, 0,0); } @@ -167738,11 +223869,60 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ return rc; } +/* +** Default xRename callback for RBU. +*/ +static int xDefaultRename(void *pArg, const char *zOld, const char *zNew){ + int rc = SQLITE_OK; + UNUSED_PARAMETER(pArg); +#if defined(_WIN32_WCE) + { + LPWSTR zWideOld; + LPWSTR zWideNew; + + zWideOld = rbuWinUtf8ToUnicode(zOld); + if( zWideOld ){ + zWideNew = rbuWinUtf8ToUnicode(zNew); + if( zWideNew ){ + if( MoveFileW(zWideOld, zWideNew) ){ + rc = SQLITE_OK; + }else{ + rc = SQLITE_IOERR; + } + sqlite3_free(zWideNew); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + sqlite3_free(zWideOld); + }else{ + rc = SQLITE_IOERR_NOMEM; + } + } +#else + rc = rename(zOld, zNew) ? SQLITE_IOERR : SQLITE_OK; +#endif + return rc; +} + +SQLITE_API void sqlite3rbu_rename_handler( + sqlite3rbu *pRbu, + void *pArg, + int (*xRename)(void *pArg, const char *zOld, const char *zNew) +){ + if( xRename ){ + pRbu->xRename = xRename; + pRbu->pRenameArg = pArg; + }else{ + pRbu->xRename = xDefaultRename; + pRbu->pRenameArg = 0; + } +} + /************************************************************************** ** Beginning of RBU VFS shim methods. The VFS shim modifies the behaviour ** of a standard VFS in the following ways: ** -** 1. Whenever the first page of a main database file is read or +** 1. Whenever the first page of a main database file is read or ** written, the value of the change-counter cookie is stored in ** rbu_file.iCookie. Similarly, the value of the "write-version" ** database header field is stored in rbu_file.iWriteVer. This ensures @@ -167750,15 +223930,15 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ ** ** 2. Whenever an SQLITE_OPEN_WAL file is opened, the (rbu_file.pWalFd) ** member variable of the associated database file descriptor is set -** to point to the new file. A mutex protected linked list of all main -** db fds opened using a particular RBU VFS is maintained at +** to point to the new file. A mutex protected linked list of all main +** db fds opened using a particular RBU VFS is maintained at ** rbu_vfs.pMain to facilitate this. ** -** 3. Using a new file-control "SQLITE_FCNTL_RBU", a main db rbu_file +** 3. Using a new file-control "SQLITE_FCNTL_RBU", a main db rbu_file ** object can be marked as the target database of an RBU update. This ** turns on the following extra special behaviour: ** -** 3a. If xAccess() is called to check if there exists a *-wal file +** 3a. If xAccess() is called to check if there exists a *-wal file ** associated with an RBU target database currently in RBU_STAGE_OAL ** stage (preparing the *-oal file), the following special handling ** applies: @@ -167771,33 +223951,34 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_savestate(sqlite3rbu *p){ ** ** Then, when xOpen() is called to open the *-wal file associated with ** the RBU target in RBU_STAGE_OAL stage, instead of opening the *-wal -** file, the rbu vfs opens the corresponding *-oal file instead. +** file, the rbu vfs opens the corresponding *-oal file instead. ** ** 3b. The *-shm pages returned by xShmMap() for a target db file in ** RBU_STAGE_OAL mode are actually stored in heap memory. This is to ** avoid creating a *-shm file on disk. Additionally, xShmLock() calls ** are no-ops on target database files in RBU_STAGE_OAL mode. This is -** because assert() statements in some VFS implementations fail if +** because assert() statements in some VFS implementations fail if ** xShmLock() is called before xShmMap(). ** ** 3c. If an EXCLUSIVE lock is attempted on a target database file in any -** mode except RBU_STAGE_DONE (all work completed and checkpointed), it +** mode except RBU_STAGE_DONE (all work completed and checkpointed), it ** fails with an SQLITE_BUSY error. This is to stop RBU connections ** from automatically checkpointing a *-wal (or *-oal) file from within ** sqlite3_close(). ** ** 3d. In RBU_STAGE_CAPTURE mode, all xRead() calls on the wal file, and -** all xWrite() calls on the target database file perform no IO. +** all xWrite() calls on the target database file perform no IO. ** Instead the frame and page numbers that would be read and written ** are recorded. Additionally, successful attempts to obtain exclusive -** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target +** xShmLock() WRITER, CHECKPOINTER and READ0 locks on the target ** database file are recorded. xShmLock() calls to unlock the same ** locks are no-ops (so that once obtained, these locks are never ** relinquished). Finally, calls to xSync() on the target database -** file fail with SQLITE_INTERNAL errors. +** file fail with SQLITE_NOTICE errors. */ static void rbuUnlockShm(rbu_file *p){ + assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); if( p->pRbu ){ int (*xShmLock)(sqlite3_file*,int,int,int) = p->pReal->pMethods->xShmLock; int i; @@ -167810,6 +223991,81 @@ static void rbuUnlockShm(rbu_file *p){ } } +/* +*/ +static int rbuUpdateTempSize(rbu_file *pFd, sqlite3_int64 nNew){ + sqlite3rbu *pRbu = pFd->pRbu; + i64 nDiff = nNew - pFd->sz; + pRbu->szTemp += nDiff; + pFd->sz = nNew; + assert( pRbu->szTemp>=0 ); + if( pRbu->szTempLimit && pRbu->szTemp>pRbu->szTempLimit ) return SQLITE_FULL; + return SQLITE_OK; +} + +/* +** Add an item to the main-db lists, if it is not already present. +** +** There are two main-db lists. One for all file descriptors, and one +** for all file descriptors with rbu_file.pDb!=0. If the argument has +** rbu_file.pDb!=0, then it is assumed to already be present on the +** main list and is only added to the pDb!=0 list. +*/ +static void rbuMainlistAdd(rbu_file *p){ + rbu_vfs *pRbuVfs = p->pRbuVfs; + rbu_file *pIter; + assert( (p->openFlags & SQLITE_OPEN_MAIN_DB) ); + sqlite3_mutex_enter(pRbuVfs->mutex); + if( p->pRbu==0 ){ + for(pIter=pRbuVfs->pMain; pIter; pIter=pIter->pMainNext); + p->pMainNext = pRbuVfs->pMain; + pRbuVfs->pMain = p; + }else{ + for(pIter=pRbuVfs->pMainRbu; pIter && pIter!=p; pIter=pIter->pMainRbuNext){} + if( pIter==0 ){ + p->pMainRbuNext = pRbuVfs->pMainRbu; + pRbuVfs->pMainRbu = p; + } + } + sqlite3_mutex_leave(pRbuVfs->mutex); +} + +/* +** Remove an item from the main-db lists. +*/ +static void rbuMainlistRemove(rbu_file *p){ + rbu_file **pp; + sqlite3_mutex_enter(p->pRbuVfs->mutex); + for(pp=&p->pRbuVfs->pMain; *pp && *pp!=p; pp=&((*pp)->pMainNext)){} + if( *pp ) *pp = p->pMainNext; + p->pMainNext = 0; + for(pp=&p->pRbuVfs->pMainRbu; *pp && *pp!=p; pp=&((*pp)->pMainRbuNext)){} + if( *pp ) *pp = p->pMainRbuNext; + p->pMainRbuNext = 0; + sqlite3_mutex_leave(p->pRbuVfs->mutex); +} + +/* +** Given that zWal points to a buffer containing a wal file name passed to +** either the xOpen() or xAccess() VFS method, search the main-db list for +** a file-handle opened by the same database connection on the corresponding +** database file. +** +** If parameter bRbu is true, only search for file-descriptors with +** rbu_file.pDb!=0. +*/ +static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal, int bRbu){ + rbu_file *pDb; + sqlite3_mutex_enter(pRbuVfs->mutex); + if( bRbu ){ + for(pDb=pRbuVfs->pMainRbu; pDb && pDb->zWal!=zWal; pDb=pDb->pMainRbuNext){} + }else{ + for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){} + } + sqlite3_mutex_leave(pRbuVfs->mutex); + return pDb; +} + /* ** Close an rbu file. */ @@ -167827,14 +224083,17 @@ static int rbuVfsClose(sqlite3_file *pFile){ sqlite3_free(p->zDel); if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ - rbu_file **pp; - sqlite3_mutex_enter(p->pRbuVfs->mutex); - for(pp=&p->pRbuVfs->pMain; *pp!=p; pp=&((*pp)->pMainNext)); - *pp = p->pMainNext; - sqlite3_mutex_leave(p->pRbuVfs->mutex); + const sqlite3_io_methods *pMeth = p->pReal->pMethods; + rbuMainlistRemove(p); rbuUnlockShm(p); - p->pReal->pMethods->xShmUnmap(p->pReal, 0); + if( pMeth->iVersion>1 && pMeth->xShmUnmap ){ + pMeth->xShmUnmap(p->pReal, 0); + } } + else if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ + rbuUpdateTempSize(p, 0); + } + assert( p->pMainNext==0 && p->pRbuVfs->pMain!=p ); /* Close the underlying file handle */ rc = p->pReal->pMethods->xClose(p->pReal); @@ -167843,7 +224102,7 @@ static int rbuVfsClose(sqlite3_file *pFile){ /* -** Read and return an unsigned 32-bit big-endian integer from the buffer +** Read and return an unsigned 32-bit big-endian integer from the buffer ** passed as the only argument. */ static u32 rbuGetU32(u8 *aBuf){ @@ -167873,9 +224132,9 @@ static void rbuPutU16(u8 *aBuf, u16 iVal){ ** Read data from an rbuVfs-file. */ static int rbuVfsRead( - sqlite3_file *pFile, - void *zBuf, - int iAmt, + sqlite3_file *pFile, + void *zBuf, + int iAmt, sqlite_int64 iOfst ){ rbu_file *p = (rbu_file*)pFile; @@ -167886,20 +224145,20 @@ static int rbuVfsRead( assert( p->openFlags & SQLITE_OPEN_WAL ); rc = rbuCaptureWalRead(p->pRbu, iOfst, iAmt); }else{ - if( pRbu && pRbu->eStage==RBU_STAGE_OAL - && (p->openFlags & SQLITE_OPEN_WAL) - && iOfst>=pRbu->iOalSz + if( pRbu && pRbu->eStage==RBU_STAGE_OAL + && (p->openFlags & SQLITE_OPEN_WAL) + && iOfst>=pRbu->iOalSz ){ rc = SQLITE_OK; memset(zBuf, 0, iAmt); }else{ rc = p->pReal->pMethods->xRead(p->pReal, zBuf, iAmt, iOfst); #if 1 - /* If this is being called to read the first page of the target - ** database as part of an rbu vacuum operation, synthesize the + /* If this is being called to read the first page of the target + ** database as part of an rbu vacuum operation, synthesize the ** contents of the first page if it does not yet exist. Otherwise, ** SQLite will not check for a *-wal file. */ - if( pRbu && rbuIsVacuum(pRbu) + if( pRbu && rbuIsVacuum(pRbu) && rc==SQLITE_IOERR_SHORT_READ && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) && pRbu->rc==SQLITE_OK @@ -167939,9 +224198,9 @@ static int rbuVfsRead( ** Write data to an rbuVfs-file. */ static int rbuVfsWrite( - sqlite3_file *pFile, - const void *zBuf, - int iAmt, + sqlite3_file *pFile, + const void *zBuf, + int iAmt, sqlite_int64 iOfst ){ rbu_file *p = (rbu_file*)pFile; @@ -167952,11 +224211,19 @@ static int rbuVfsWrite( assert( p->openFlags & SQLITE_OPEN_MAIN_DB ); rc = rbuCaptureDbWrite(p->pRbu, iOfst); }else{ - if( pRbu && pRbu->eStage==RBU_STAGE_OAL - && (p->openFlags & SQLITE_OPEN_WAL) - && iOfst>=pRbu->iOalSz - ){ - pRbu->iOalSz = iAmt + iOfst; + if( pRbu ){ + if( pRbu->eStage==RBU_STAGE_OAL + && (p->openFlags & SQLITE_OPEN_WAL) + && iOfst>=pRbu->iOalSz + ){ + pRbu->iOalSz = iAmt + iOfst; + }else if( p->openFlags & SQLITE_OPEN_DELETEONCLOSE ){ + i64 szNew = iAmt+iOfst; + if( szNew>p->sz ){ + rc = rbuUpdateTempSize(p, szNew); + if( rc!=SQLITE_OK ) return rc; + } + } } rc = p->pReal->pMethods->xWrite(p->pReal, zBuf, iAmt, iOfst); if( rc==SQLITE_OK && iOfst==0 && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ @@ -167975,6 +224242,10 @@ static int rbuVfsWrite( */ static int rbuVfsTruncate(sqlite3_file *pFile, sqlite_int64 size){ rbu_file *p = (rbu_file*)pFile; + if( (p->openFlags & SQLITE_OPEN_DELETEONCLOSE) && p->pRbu ){ + int rc = rbuUpdateTempSize(p, size); + if( rc!=SQLITE_OK ) return rc; + } return p->pReal->pMethods->xTruncate(p->pReal, size); } @@ -167985,7 +224256,7 @@ static int rbuVfsSync(sqlite3_file *pFile, int flags){ rbu_file *p = (rbu_file *)pFile; if( p->pRbu && p->pRbu->eStage==RBU_STAGE_CAPTURE ){ if( p->openFlags & SQLITE_OPEN_MAIN_DB ){ - return SQLITE_INTERNAL; + return SQLITE_NOTICE_RBU; } return SQLITE_OK; } @@ -168002,10 +224273,10 @@ static int rbuVfsFileSize(sqlite3_file *pFile, sqlite_int64 *pSize){ /* If this is an RBU vacuum operation and this is the target database, ** pretend that it has at least one page. Otherwise, SQLite will not - ** check for the existance of a *-wal file. rbuVfsRead() contains + ** check for the existance of a *-wal file. rbuVfsRead() contains ** similar logic. */ - if( rc==SQLITE_OK && *pSize==0 - && p->pRbu && rbuIsVacuum(p->pRbu) + if( rc==SQLITE_OK && *pSize==0 + && p->pRbu && rbuIsVacuum(p->pRbu) && (p->openFlags & SQLITE_OPEN_MAIN_DB) ){ *pSize = 1024; @@ -168022,10 +224293,10 @@ static int rbuVfsLock(sqlite3_file *pFile, int eLock){ int rc = SQLITE_OK; assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( eLock==SQLITE_LOCK_EXCLUSIVE + if( eLock==SQLITE_LOCK_EXCLUSIVE && (p->bNolock || (pRbu && pRbu->eStage!=RBU_STAGE_DONE)) ){ - /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this + /* Do not allow EXCLUSIVE locks. Preventing SQLite from taking this ** prevents it from checkpointing the database from sqlite3_close(). */ rc = SQLITE_BUSY; }else{ @@ -168081,6 +224352,7 @@ static int rbuVfsFileControl(sqlite3_file *pFile, int op, void *pArg){ }else if( rc==SQLITE_NOTFOUND ){ pRbu->pTargetFd = p; p->pRbu = pRbu; + rbuMainlistAdd(p); if( p->pWalFd ) p->pWalFd->pRbu = pRbu; rc = SQLITE_OK; } @@ -168135,25 +224407,24 @@ static int rbuVfsShmLock(sqlite3_file *pFile, int ofst, int n, int flags){ #endif assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( pRbu && (pRbu->eStage==RBU_STAGE_OAL || pRbu->eStage==RBU_STAGE_MOVE) ){ - /* Magic number 1 is the WAL_CKPT_LOCK lock. Preventing SQLite from - ** taking this lock also prevents any checkpoints from occurring. - ** todo: really, it's not clear why this might occur, as - ** wal_autocheckpoint ought to be turned off. */ + if( pRbu && ( + pRbu->eStage==RBU_STAGE_OAL + || pRbu->eStage==RBU_STAGE_MOVE + || pRbu->eStage==RBU_STAGE_DONE + )){ + /* Prevent SQLite from taking a shm-lock on the target file when it + ** is supplying heap memory to the upper layer in place of *-shm + ** segments. */ if( ofst==WAL_LOCK_CKPT && n==1 ) rc = SQLITE_BUSY; }else{ int bCapture = 0; - if( n==1 && (flags & SQLITE_SHM_EXCLUSIVE) - && pRbu && pRbu->eStage==RBU_STAGE_CAPTURE - && (ofst==WAL_LOCK_WRITE || ofst==WAL_LOCK_CKPT || ofst==WAL_LOCK_READ0) - ){ + if( pRbu && pRbu->eStage==RBU_STAGE_CAPTURE ){ bCapture = 1; } - if( bCapture==0 || 0==(flags & SQLITE_SHM_UNLOCK) ){ rc = p->pReal->pMethods->xShmLock(p->pReal, ofst, n, flags); if( bCapture && rc==SQLITE_OK ){ - pRbu->mLock |= (1 << ofst); + pRbu->mLock |= ((1<pRbu ? p->pRbu->eStage : 0); /* If not in RBU_STAGE_OAL, allow this call to pass through. Or, if this - ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space + ** rbu is in the RBU_STAGE_OAL state, use heap memory for *-shm space ** instead of a file on disk. */ assert( p->openFlags & (SQLITE_OPEN_MAIN_DB|SQLITE_OPEN_TEMP_DB) ); - if( eStage==RBU_STAGE_OAL || eStage==RBU_STAGE_MOVE ){ - if( iRegion<=p->nShm ){ - int nByte = (iRegion+1) * sizeof(char*); - char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); - if( apNew==0 ){ - rc = SQLITE_NOMEM; - }else{ - memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); - p->apShm = apNew; - p->nShm = iRegion+1; - } + if( eStage==RBU_STAGE_OAL ){ + sqlite3_int64 nByte = (iRegion+1) * sizeof(char*); + char **apNew = (char**)sqlite3_realloc64(p->apShm, nByte); + + /* This is an RBU connection that uses its own heap memory for the + ** pages of the *-shm file. Since no other process can have run + ** recovery, the connection must request *-shm pages in order + ** from start to finish. */ + assert( iRegion==p->nShm ); + if( apNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(&apNew[p->nShm], 0, sizeof(char*) * (1 + iRegion - p->nShm)); + p->apShm = apNew; + p->nShm = iRegion+1; } - if( rc==SQLITE_OK && p->apShm[iRegion]==0 ){ + if( rc==SQLITE_OK ){ char *pNew = (char*)sqlite3_malloc64(szRegion); if( pNew==0 ){ rc = SQLITE_NOMEM; @@ -168242,47 +224517,6 @@ static int rbuVfsShmUnmap(sqlite3_file *pFile, int delFlag){ return rc; } -/* -** Given that zWal points to a buffer containing a wal file name passed to -** either the xOpen() or xAccess() VFS method, return a pointer to the -** file-handle opened by the same database connection on the corresponding -** database file. -*/ -static rbu_file *rbuFindMaindb(rbu_vfs *pRbuVfs, const char *zWal){ - rbu_file *pDb; - sqlite3_mutex_enter(pRbuVfs->mutex); - for(pDb=pRbuVfs->pMain; pDb && pDb->zWal!=zWal; pDb=pDb->pMainNext){} - sqlite3_mutex_leave(pRbuVfs->mutex); - return pDb; -} - -/* -** A main database named zName has just been opened. The following -** function returns a pointer to a buffer owned by SQLite that contains -** the name of the *-wal file this db connection will use. SQLite -** happens to pass a pointer to this buffer when using xAccess() -** or xOpen() to operate on the *-wal file. -*/ -static const char *rbuMainToWal(const char *zName, int flags){ - int n = (int)strlen(zName); - const char *z = &zName[n]; - if( flags & SQLITE_OPEN_URI ){ - int odd = 0; - while( 1 ){ - if( z[0]==0 ){ - odd = 1 - odd; - if( odd && z[1]==0 ) break; - } - z++; - } - z += 2; - }else{ - while( *z==0 ) z++; - } - z += (n + 8 + 1); - return z; -} - /* ** Open an rbu file handle. */ @@ -168313,6 +224547,25 @@ static int rbuVfsOpen( rbuVfsShmUnmap, /* xShmUnmap */ 0, 0 /* xFetch, xUnfetch */ }; + static sqlite3_io_methods rbuvfs_io_methods1 = { + 1, /* iVersion */ + rbuVfsClose, /* xClose */ + rbuVfsRead, /* xRead */ + rbuVfsWrite, /* xWrite */ + rbuVfsTruncate, /* xTruncate */ + rbuVfsSync, /* xSync */ + rbuVfsFileSize, /* xFileSize */ + rbuVfsLock, /* xLock */ + rbuVfsUnlock, /* xUnlock */ + rbuVfsCheckReservedLock, /* xCheckReservedLock */ + rbuVfsFileControl, /* xFileControl */ + rbuVfsSectorSize, /* xSectorSize */ + rbuVfsDeviceCharacteristics, /* xDeviceCharacteristics */ + 0, 0, 0, 0, 0, 0 + }; + + + rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs; sqlite3_vfs *pRealVfs = pRbuVfs->pRealVfs; rbu_file *pFd = (rbu_file *)pFile; @@ -168331,43 +224584,31 @@ static int rbuVfsOpen( ** the name of the *-wal file this db connection will use. SQLite ** happens to pass a pointer to this buffer when using xAccess() ** or xOpen() to operate on the *-wal file. */ - pFd->zWal = rbuMainToWal(zName, flags); + pFd->zWal = sqlite3_filename_wal(zName); } else if( flags & SQLITE_OPEN_WAL ){ - rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName); + rbu_file *pDb = rbuFindMaindb(pRbuVfs, zName, 0); if( pDb ){ if( pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ - /* This call is to open a *-wal file. Intead, open the *-oal. This - ** code ensures that the string passed to xOpen() is terminated by a - ** pair of '\0' bytes in case the VFS attempts to extract a URI - ** parameter from it. */ - const char *zBase = zName; - size_t nCopy; - char *zCopy; + /* This call is to open a *-wal file. Intead, open the *-oal. */ + size_t nOpen; if( rbuIsVacuum(pDb->pRbu) ){ - zBase = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); - zBase = rbuMainToWal(zBase, SQLITE_OPEN_URI); - } - nCopy = strlen(zBase); - zCopy = sqlite3_malloc64(nCopy+2); - if( zCopy ){ - memcpy(zCopy, zBase, nCopy); - zCopy[nCopy-3] = 'o'; - zCopy[nCopy] = '\0'; - zCopy[nCopy+1] = '\0'; - zOpen = (const char*)(pFd->zDel = zCopy); - }else{ - rc = SQLITE_NOMEM; + zOpen = sqlite3_db_filename(pDb->pRbu->dbRbu, "main"); + zOpen = sqlite3_filename_wal(zOpen); } + nOpen = strlen(zOpen); + ((char*)zOpen)[nOpen-3] = 'o'; pFd->pRbu = pDb->pRbu; } pDb->pWalFd = pFd; } } + }else{ + pFd->pRbu = pRbuVfs->pRbu; } - if( oflags & SQLITE_OPEN_MAIN_DB - && sqlite3_uri_boolean(zName, "rbu_memory", 0) + if( oflags & SQLITE_OPEN_MAIN_DB + && sqlite3_uri_boolean(zName, "rbu_memory", 0) ){ assert( oflags & SQLITE_OPEN_MAIN_DB ); oflags = SQLITE_OPEN_TEMP_DB | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | @@ -168379,15 +224620,17 @@ static int rbuVfsOpen( rc = pRealVfs->xOpen(pRealVfs, zOpen, pFd->pReal, oflags, pOutFlags); } if( pFd->pReal->pMethods ){ + const sqlite3_io_methods *pMeth = pFd->pReal->pMethods; /* The xOpen() operation has succeeded. Set the sqlite3_file.pMethods ** pointer and, if the file is a main database file, link it into the ** mutex protected linked list of all such files. */ - pFile->pMethods = &rbuvfs_io_methods; + if( pMeth->iVersion<2 || pMeth->xShmLock==0 ){ + pFile->pMethods = &rbuvfs_io_methods1; + }else{ + pFile->pMethods = &rbuvfs_io_methods; + } if( flags & SQLITE_OPEN_MAIN_DB ){ - sqlite3_mutex_enter(pRbuVfs->mutex); - pFd->pMainNext = pRbuVfs->pMain; - pRbuVfs->pMain = pFd; - sqlite3_mutex_leave(pRbuVfs->mutex); + rbuMainlistAdd(pFd); } }else{ sqlite3_free(pFd->zDel); @@ -168409,9 +224652,9 @@ static int rbuVfsDelete(sqlite3_vfs *pVfs, const char *zPath, int dirSync){ ** is available, or false otherwise. */ static int rbuVfsAccess( - sqlite3_vfs *pVfs, - const char *zPath, - int flags, + sqlite3_vfs *pVfs, + const char *zPath, + int flags, int *pResOut ){ rbu_vfs *pRbuVfs = (rbu_vfs*)pVfs; @@ -168427,7 +224670,7 @@ static int rbuVfsAccess( ** a) if the *-wal file does exist, return SQLITE_CANTOPEN. This ** ensures that the RBU extension never tries to update a database ** in wal mode, even if the first page of the database file has - ** been damaged. + ** been damaged. ** ** b) if the *-wal file does not exist, claim that it does anyway, ** causing SQLite to call xOpen() to open it. This call will also @@ -168435,12 +224678,15 @@ static int rbuVfsAccess( ** file opened instead. */ if( rc==SQLITE_OK && flags==SQLITE_ACCESS_EXISTS ){ - rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath); - if( pDb && pDb->pRbu && pDb->pRbu->eStage==RBU_STAGE_OAL ){ + rbu_file *pDb = rbuFindMaindb(pRbuVfs, zPath, 1); + if( pDb && pDb->pRbu->eStage==RBU_STAGE_OAL ){ + assert( pDb->pRbu ); if( *pResOut ){ rc = SQLITE_CANTOPEN; }else{ - *pResOut = 1; + sqlite3_int64 sz = 0; + rc = rbuVfsFileSize(&pDb->base, &sz); + *pResOut = (sz>0); } } } @@ -168454,9 +224700,9 @@ static int rbuVfsAccess( ** of at least (DEVSYM_MAX_PATHNAME+1) bytes. */ static int rbuVfsFullPathname( - sqlite3_vfs *pVfs, - const char *zPath, - int nOut, + sqlite3_vfs *pVfs, + const char *zPath, + int nOut, char *zOut ){ sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs; @@ -168474,7 +224720,7 @@ static void *rbuVfsDlOpen(sqlite3_vfs *pVfs, const char *zPath){ /* ** Populate the buffer zErrMsg (size nByte bytes) with a human readable -** utf-8 string describing the most recent error encountered associated +** utf-8 string describing the most recent error encountered associated ** with dynamic libraries. */ static void rbuVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ @@ -168486,8 +224732,8 @@ static void rbuVfsDlError(sqlite3_vfs *pVfs, int nByte, char *zErrMsg){ ** Return a pointer to the symbol zSymbol in the dynamic library pHandle. */ static void (*rbuVfsDlSym( - sqlite3_vfs *pVfs, - void *pArg, + sqlite3_vfs *pVfs, + void *pArg, const char *zSym ))(void){ sqlite3_vfs *pRealVfs = ((rbu_vfs*)pVfs)->pRealVfs; @@ -168504,7 +224750,7 @@ static void rbuVfsDlClose(sqlite3_vfs *pVfs, void *pHandle){ #endif /* SQLITE_OMIT_LOAD_EXTENSION */ /* -** Populate the buffer pointed to by zBufOut with nByte bytes of +** Populate the buffer pointed to by zBufOut with nByte bytes of ** random data. */ static int rbuVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ @@ -168513,7 +224759,7 @@ static int rbuVfsRandomness(sqlite3_vfs *pVfs, int nByte, char *zBufOut){ } /* -** Sleep for nMicro microseconds. Return the number of microseconds +** Sleep for nMicro microseconds. Return the number of microseconds ** actually slept. */ static int rbuVfsSleep(sqlite3_vfs *pVfs, int nMicro){ @@ -168533,6 +224779,9 @@ static int rbuVfsCurrentTime(sqlite3_vfs *pVfs, double *pTimeOut){ ** No-op. */ static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ + UNUSED_PARAMETER(pVfs); + UNUSED_PARAMETER(a); + UNUSED_PARAMETER(b); return 0; } @@ -168540,7 +224789,7 @@ static int rbuVfsGetLastError(sqlite3_vfs *pVfs, int a, char *b){ ** Deregister and destroy an RBU vfs created by an earlier call to ** sqlite3rbu_create_vfs(). */ -SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName){ +SQLITE_API void sqlite3rbu_destroy_vfs(const char *zName){ sqlite3_vfs *pVfs = sqlite3_vfs_find(zName); if( pVfs && pVfs->xOpen==rbuVfsOpen ){ sqlite3_mutex_free(((rbu_vfs*)pVfs)->mutex); @@ -168554,7 +224803,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3rbu_destroy_vfs(const char *zName){ ** via existing VFS zParent. The new object is registered as a non-default ** VFS with SQLite before returning. */ -SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const char *zParent){ +SQLITE_API int sqlite3rbu_create_vfs(const char *zName, const char *zParent){ /* Template for VFS */ static sqlite3_vfs vfs_template = { @@ -168629,6 +224878,20 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha return rc; } +/* +** Configure the aggregate temp file size limit for this RBU handle. +*/ +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size_limit(sqlite3rbu *pRbu, sqlite3_int64 n){ + if( n>=0 ){ + pRbu->szTempLimit = n; + } + return pRbu->szTempLimit; +} + +SQLITE_API sqlite3_int64 sqlite3rbu_temp_size(sqlite3rbu *pRbu){ + return pRbu->szTemp; +} + /**************************************************************************/ @@ -168650,7 +224913,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha ** ** This file contains an implementation of the "dbstat" virtual table. ** -** The dbstat virtual table is used to extract low-level formatting +** The dbstat virtual table is used to extract low-level storage ** information from an SQLite database in order to implement the ** "sqlite3_analyzer" utility. See the ../tool/spaceanal.tcl script ** for an example implementation. @@ -168663,24 +224926,33 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha #if (defined(SQLITE_ENABLE_DBSTAT_VTAB) || defined(SQLITE_TEST)) \ && !defined(SQLITE_OMIT_VIRTUALTABLE) +/* +** The pager and btree modules arrange objects in memory so that there are +** always approximately 200 bytes of addressable memory following each page +** buffer. This way small buffer overreads caused by corrupt database pages +** do not cause undefined behaviour. This module pads each page buffer +** by the following number of bytes for the same purpose. +*/ +#define DBSTAT_PAGE_PADDING_BYTES 256 + /* ** Page paths: -** -** The value of the 'path' column describes the path taken from the -** root-node of the b-tree structure to each page. The value of the +** +** The value of the 'path' column describes the path taken from the +** root-node of the b-tree structure to each page. The value of the ** root-node path is '/'. ** ** The value of the path for the left-most child page of the root of ** a b-tree is '/000/'. (Btrees store content ordered from left to right ** so the pages to the left have smaller keys than the pages to the right.) ** The next to left-most child of the root page is -** '/001', and so on, each sibling page identified by a 3-digit hex +** '/001', and so on, each sibling page identified by a 3-digit hex ** value. The children of the 451st left-most sibling have paths such ** as '/1c2/000/, '/1c2/001/' etc. ** -** Overflow pages are specified by appending a '+' character and a +** Overflow pages are specified by appending a '+' character and a ** six-digit hexadecimal value to the path to the cell they are linked -** from. For example, the three overflow pages in a chain linked from +** from. For example, the three overflow pages in a chain linked from ** the left-most cell of the 450th child of the root page are identified ** by the paths: ** @@ -168694,27 +224966,30 @@ SQLITE_API int SQLITE_STDCALL sqlite3rbu_create_vfs(const char *zName, const cha ** ** '/1c2/000/' // Left-most child of 451st child of root */ -#define VTAB_SCHEMA \ - "CREATE TABLE xx( " \ - " name STRING, /* Name of table or index */" \ - " path INTEGER, /* Path to page from root */" \ - " pageno INTEGER, /* Page number */" \ - " pagetype STRING, /* 'internal', 'leaf' or 'overflow' */" \ - " ncell INTEGER, /* Cells on page (0 for overflow) */" \ - " payload INTEGER, /* Bytes of payload on this page */" \ - " unused INTEGER, /* Bytes of unused space on this page */" \ - " mx_payload INTEGER, /* Largest payload size of all cells */" \ - " pgoffset INTEGER, /* Offset of page in file */" \ - " pgsize INTEGER, /* Size of the page */" \ - " schema TEXT HIDDEN /* Database schema being analyzed */" \ - ");" - +static const char zDbstatSchema[] = + "CREATE TABLE x(" + " name TEXT," /* 0 Name of table or index */ + " path TEXT," /* 1 Path to page from root (NULL for agg) */ + " pageno INTEGER," /* 2 Page number (page count for aggregates) */ + " pagetype TEXT," /* 3 'internal', 'leaf', 'overflow', or NULL */ + " ncell INTEGER," /* 4 Cells on page (0 for overflow) */ + " payload INTEGER," /* 5 Bytes of payload on this page */ + " unused INTEGER," /* 6 Bytes of unused space on this page */ + " mx_payload INTEGER," /* 7 Largest payload size of all cells */ + " pgoffset INTEGER," /* 8 Offset of page in file (NULL for agg) */ + " pgsize INTEGER," /* 9 Size of the page (sum for aggregate) */ + " schema TEXT HIDDEN," /* 10 Database schema being analyzed */ + " aggregate BOOLEAN HIDDEN" /* 11 aggregate info for each table */ + ")" +; +/* Forward reference to data structured used in this module */ typedef struct StatTable StatTable; typedef struct StatCursor StatCursor; typedef struct StatPage StatPage; typedef struct StatCell StatCell; +/* Size information for a single cell within a btree page */ struct StatCell { int nLocal; /* Bytes of local payload */ u32 iChildPg; /* Child node (or 0 if this is a leaf) */ @@ -168724,11 +224999,11 @@ struct StatCell { int iOvfl; /* Iterates through aOvfl[] */ }; +/* Size information for a single btree page */ struct StatPage { - u32 iPgno; - DbPage *pPg; - int iCell; - + u32 iPgno; /* Page number */ + u8 *aPg; /* Page buffer from sqlite3_malloc() */ + int iCell; /* Current cell */ char *zPath; /* Path to this page */ /* Variables populated by statDecodePage(): */ @@ -168737,34 +225012,38 @@ struct StatPage { int nUnused; /* Number of unused bytes on page */ StatCell *aCell; /* Array of parsed cells */ u32 iRightChildPg; /* Right-child page number (or 0) */ - int nMxPayload; /* Largest payload of any cell on this page */ + int nMxPayload; /* Largest payload of any cell on the page */ }; +/* The cursor for scanning the dbstat virtual table */ struct StatCursor { - sqlite3_vtab_cursor base; + sqlite3_vtab_cursor base; /* base class. MUST BE FIRST! */ sqlite3_stmt *pStmt; /* Iterates through set of root pages */ - int isEof; /* After pStmt has returned SQLITE_DONE */ + u8 isEof; /* After pStmt has returned SQLITE_DONE */ + u8 isAgg; /* Aggregate results for each table */ int iDb; /* Schema used for this query */ - StatPage aPage[32]; + StatPage aPage[32]; /* Pages in path to current page */ int iPage; /* Current entry in aPage[] */ /* Values to return. */ + u32 iPageno; /* Value of 'pageno' column */ char *zName; /* Value of 'name' column */ char *zPath; /* Value of 'path' column */ - u32 iPageno; /* Value of 'pageno' column */ char *zPagetype; /* Value of 'pagetype' column */ + int nPage; /* Number of pages in current btree */ int nCell; /* Value of 'ncell' column */ - int nPayload; /* Value of 'payload' column */ - int nUnused; /* Value of 'unused' column */ int nMxPayload; /* Value of 'mx_payload' column */ + i64 nUnused; /* Value of 'unused' column */ + i64 nPayload; /* Value of 'payload' column */ i64 iOffset; /* Value of 'pgOffset' column */ - int szPage; /* Value of 'pgSize' column */ + i64 szPage; /* Value of 'pgSize' column */ }; +/* An instance of the DBSTAT virtual table */ struct StatTable { - sqlite3_vtab base; - sqlite3 *db; + sqlite3_vtab base; /* base class. MUST BE FIRST! */ + sqlite3 *db; /* Database connection that owns this vtab */ int iDb; /* Index of database to analyze */ }; @@ -168773,7 +225052,7 @@ struct StatTable { #endif /* -** Connect to or create a statvfs virtual table. +** Connect to or create a new DBSTAT virtual table. */ static int statConnect( sqlite3 *db, @@ -168785,6 +225064,7 @@ static int statConnect( StatTable *pTab = 0; int rc = SQLITE_OK; int iDb; + (void)pAux; if( argc>=4 ){ Token nm; @@ -168797,7 +225077,8 @@ static int statConnect( }else{ iDb = 0; } - rc = sqlite3_declare_vtab(db, VTAB_SCHEMA); + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + rc = sqlite3_declare_vtab(db, zDbstatSchema); if( rc==SQLITE_OK ){ pTab = (StatTable *)sqlite3_malloc64(sizeof(StatTable)); if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; @@ -168815,7 +225096,7 @@ static int statConnect( } /* -** Disconnect from or destroy a statvfs virtual table. +** Disconnect from or destroy the DBSTAT virtual table. */ static int statDisconnect(sqlite3_vtab *pVtab){ sqlite3_free(pVtab); @@ -168823,16 +225104,21 @@ static int statDisconnect(sqlite3_vtab *pVtab){ } /* -** There is no "best-index". This virtual table always does a linear -** scan. However, a schema=? constraint should cause this table to -** operate on a different database schema, so check for it. +** Compute the best query strategy and return the result in idxNum. ** -** idxNum is normally 0, but will be 1 if a schema=? constraint exists. +** idxNum-Bit Meaning +** ---------- ---------------------------------------------- +** 0x01 There is a schema=? term in the WHERE clause +** 0x02 There is a name=? term in the WHERE clause +** 0x04 There is an aggregate=? term in the WHERE clause +** 0x08 Output should be ordered by name and path */ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ int i; - - pIdxInfo->estimatedCost = 1.0e6; /* Initial cost estimate */ + int iSchema = -1; + int iName = -1; + int iAgg = -1; + (void)tab; /* Look for a valid schema=? constraint. If found, change the idxNum to ** 1 and request the value of that constraint be sent to xFilter. And @@ -168840,19 +225126,44 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ** used. */ for(i=0; inConstraint; i++){ - if( pIdxInfo->aConstraint[i].usable==0 ) continue; if( pIdxInfo->aConstraint[i].op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - if( pIdxInfo->aConstraint[i].iColumn!=10 ) continue; - pIdxInfo->idxNum = 1; - pIdxInfo->estimatedCost = 1.0; - pIdxInfo->aConstraintUsage[i].argvIndex = 1; - pIdxInfo->aConstraintUsage[i].omit = 1; - break; + if( pIdxInfo->aConstraint[i].usable==0 ){ + /* Force DBSTAT table should always be the right-most table in a join */ + return SQLITE_CONSTRAINT; + } + switch( pIdxInfo->aConstraint[i].iColumn ){ + case 0: { /* name */ + iName = i; + break; + } + case 10: { /* schema */ + iSchema = i; + break; + } + case 11: { /* aggregate */ + iAgg = i; + break; + } + } } + i = 0; + if( iSchema>=0 ){ + pIdxInfo->aConstraintUsage[iSchema].argvIndex = ++i; + pIdxInfo->aConstraintUsage[iSchema].omit = 1; + pIdxInfo->idxNum |= 0x01; + } + if( iName>=0 ){ + pIdxInfo->aConstraintUsage[iName].argvIndex = ++i; + pIdxInfo->idxNum |= 0x02; + } + if( iAgg>=0 ){ + pIdxInfo->aConstraintUsage[iAgg].argvIndex = ++i; + pIdxInfo->idxNum |= 0x04; + } + pIdxInfo->estimatedCost = 1.0; - - /* Records are always returned in ascending order of (name, path). - ** If this will satisfy the client, set the orderByConsumed flag so that + /* Records are always returned in ascending order of (name, path). + ** If this will satisfy the client, set the orderByConsumed flag so that ** SQLite does not do an external sort. */ if( ( pIdxInfo->nOrderBy==1 @@ -168867,13 +225178,15 @@ static int statBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ ) ){ pIdxInfo->orderByConsumed = 1; + pIdxInfo->idxNum |= 0x08; } + pIdxInfo->idxFlags |= SQLITE_INDEX_SCAN_HEX; return SQLITE_OK; } /* -** Open a new statvfs cursor. +** Open a new DBSTAT cursor. */ static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ StatTable *pTab = (StatTable *)pVTab; @@ -168892,7 +225205,7 @@ static int statOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ return SQLITE_OK; } -static void statClearPage(StatPage *p){ +static void statClearCells(StatPage *p){ int i; if( p->aCell ){ for(i=0; inCell; i++){ @@ -168900,25 +225213,48 @@ static void statClearPage(StatPage *p){ } sqlite3_free(p->aCell); } - sqlite3PagerUnref(p->pPg); + p->nCell = 0; + p->aCell = 0; +} + +static void statClearPage(StatPage *p){ + u8 *aPg = p->aPg; + statClearCells(p); sqlite3_free(p->zPath); memset(p, 0, sizeof(StatPage)); + p->aPg = aPg; } static void statResetCsr(StatCursor *pCsr){ int i; - sqlite3_reset(pCsr->pStmt); + /* In some circumstances, specifically if an OOM has occurred, the call + ** to sqlite3_reset() may cause the pager to be reset (emptied). It is + ** important that statClearPage() is called to free any page refs before + ** this happens. dbsqlfuzz 9ed3e4e3816219d3509d711636c38542bf3f40b1. */ for(i=0; iaPage); i++){ statClearPage(&pCsr->aPage[i]); + sqlite3_free(pCsr->aPage[i].aPg); + pCsr->aPage[i].aPg = 0; } + sqlite3_reset(pCsr->pStmt); pCsr->iPage = 0; sqlite3_free(pCsr->zPath); pCsr->zPath = 0; pCsr->isEof = 0; } +/* Resize the space-used counters inside of the cursor */ +static void statResetCounts(StatCursor *pCsr){ + pCsr->nCell = 0; + pCsr->nMxPayload = 0; + pCsr->nUnused = 0; + pCsr->nPayload = 0; + pCsr->szPage = 0; + pCsr->nPage = 0; +} + /* -** Close a statvfs cursor. +** Close a DBSTAT cursor. */ static int statClose(sqlite3_vtab_cursor *pCursor){ StatCursor *pCsr = (StatCursor *)pCursor; @@ -168928,16 +225264,20 @@ static int statClose(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } -static void getLocalPayload( +/* +** For a single cell on a btree page, compute the number of bytes of +** content (payload) stored on that page. That is to say, compute the +** number of bytes of content not found on overflow pages. +*/ +static int getLocalPayload( int nUsable, /* Usable bytes per page */ u8 flags, /* Page flags */ - int nTotal, /* Total record (payload) size */ - int *pnLocal /* OUT: Bytes stored locally */ + int nTotal /* Total record (payload) size */ ){ int nLocal; int nMinLocal; int nMaxLocal; - + if( flags==0x0D ){ /* Table leaf node */ nMinLocal = (nUsable - 12) * 32 / 255 - 23; nMaxLocal = nUsable - 35; @@ -168948,9 +225288,12 @@ static void getLocalPayload( nLocal = nMinLocal + (nTotal - nMinLocal) % (nUsable - 4); if( nLocal>nMaxLocal ) nLocal = nMinLocal; - *pnLocal = nLocal; + return nLocal; } +/* Populate the StatPage object with information about the all +** cells found on the page currently under analysis. +*/ static int statDecodePage(Btree *pBt, StatPage *p){ int nUnused; int iOff; @@ -168958,26 +225301,37 @@ static int statDecodePage(Btree *pBt, StatPage *p){ int isLeaf; int szPage; - u8 *aData = sqlite3PagerGetData(p->pPg); + u8 *aData = p->aPg; u8 *aHdr = &aData[p->iPgno==1 ? 100 : 0]; p->flags = aHdr[0]; + if( p->flags==0x0A || p->flags==0x0D ){ + isLeaf = 1; + nHdr = 8; + }else if( p->flags==0x05 || p->flags==0x02 ){ + isLeaf = 0; + nHdr = 12; + }else{ + goto statPageIsCorrupt; + } + if( p->iPgno==1 ) nHdr += 100; p->nCell = get2byte(&aHdr[3]); p->nMxPayload = 0; - - isLeaf = (p->flags==0x0A || p->flags==0x0D); - nHdr = 12 - isLeaf*4 + (p->iPgno==1)*100; + szPage = sqlite3BtreeGetPageSize(pBt); nUnused = get2byte(&aHdr[5]) - nHdr - 2*p->nCell; nUnused += (int)aHdr[7]; iOff = get2byte(&aHdr[1]); while( iOff ){ + int iNext; + if( iOff>=szPage ) goto statPageIsCorrupt; nUnused += get2byte(&aData[iOff+2]); - iOff = get2byte(&aData[iOff]); + iNext = get2byte(&aData[iOff]); + if( iNext0 ) goto statPageIsCorrupt; + iOff = iNext; } p->nUnused = nUnused; p->iRightChildPg = isLeaf ? 0 : sqlite3Get4byte(&aHdr[8]); - szPage = sqlite3BtreeGetPageSize(pBt); if( p->nCell ){ int i; /* Used to iterate through cells */ @@ -168994,6 +225348,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){ StatCell *pCell = &p->aCell[i]; iOff = get2byte(&aData[nHdr+i*2]); + if( iOff=szPage ) goto statPageIsCorrupt; if( !isLeaf ){ pCell->iChildPg = sqlite3Get4byte(&aData[iOff]); iOff += 4; @@ -169009,14 +225364,17 @@ static int statDecodePage(Btree *pBt, StatPage *p){ iOff += sqlite3GetVarint(&aData[iOff], &dummy); } if( nPayload>(u32)p->nMxPayload ) p->nMxPayload = nPayload; - getLocalPayload(nUsable, p->flags, nPayload, &nLocal); + nLocal = getLocalPayload(nUsable, p->flags, nPayload); + if( nLocal<0 ) goto statPageIsCorrupt; pCell->nLocal = nLocal; - assert( nLocal>=0 ); assert( nPayload>=(u32)nLocal ); assert( nLocal<=(nUsable-35) ); if( nPayload>(u32)nLocal ){ int j; int nOvfl = ((nPayload - nLocal) + nUsable-4 - 1) / (nUsable - 4); + if( iOff+nLocal+4>nUsable || nPayload>0x7fffffff ){ + goto statPageIsCorrupt; + } pCell->nLastOvfl = (nPayload-nLocal) - (nOvfl-1) * (nUsable-4); pCell->nOvfl = nOvfl; pCell->aOvfl = sqlite3_malloc64(sizeof(u32)*nOvfl); @@ -169030,7 +225388,7 @@ static int statDecodePage(Btree *pBt, StatPage *p){ if( rc!=SQLITE_OK ){ assert( pPg==0 ); return rc; - } + } pCell->aOvfl[j] = sqlite3Get4byte(sqlite3PagerGetData(pPg)); sqlite3PagerUnref(pPg); } @@ -169040,6 +225398,11 @@ static int statDecodePage(Btree *pBt, StatPage *p){ } return SQLITE_OK; + +statPageIsCorrupt: + p->flags = 0; + statClearCells(p); + return SQLITE_OK; } /* @@ -169053,23 +225416,57 @@ static void statSizeAndOffset(StatCursor *pCsr){ sqlite3_file *fd; sqlite3_int64 x[2]; - /* The default page size and offset */ - pCsr->szPage = sqlite3BtreeGetPageSize(pBt); - pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1); - - /* If connected to a ZIPVFS backend, override the page size and - ** offset with actual values obtained from ZIPVFS. + /* If connected to a ZIPVFS backend, find the page size and + ** offset from ZIPVFS. */ fd = sqlite3PagerFile(pPager); x[0] = pCsr->iPageno; - if( fd->pMethods!=0 && sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ + if( sqlite3OsFileControl(fd, 230440, &x)==SQLITE_OK ){ pCsr->iOffset = x[0]; - pCsr->szPage = (int)x[1]; + pCsr->szPage += x[1]; + }else{ + /* Not ZIPVFS: The default page size and offset */ + pCsr->szPage += sqlite3BtreeGetPageSize(pBt); + pCsr->iOffset = (i64)pCsr->szPage * (pCsr->iPageno - 1); } } /* -** Move a statvfs cursor to the next entry in the file. +** Load a copy of the page data for page iPg into the buffer belonging +** to page object pPg. Allocate the buffer if necessary. Return SQLITE_OK +** if successful, or an SQLite error code otherwise. +*/ +static int statGetPage( + Btree *pBt, /* Load page from this b-tree */ + u32 iPg, /* Page number to load */ + StatPage *pPg /* Load page into this object */ +){ + int pgsz = sqlite3BtreeGetPageSize(pBt); + DbPage *pDbPage = 0; + int rc; + + if( pPg->aPg==0 ){ + pPg->aPg = (u8*)sqlite3_malloc(pgsz + DBSTAT_PAGE_PADDING_BYTES); + if( pPg->aPg==0 ){ + return SQLITE_NOMEM_BKPT; + } + memset(&pPg->aPg[pgsz], 0, DBSTAT_PAGE_PADDING_BYTES); + } + + rc = sqlite3PagerGet(sqlite3BtreePager(pBt), iPg, &pDbPage, 0); + if( rc==SQLITE_OK ){ + const u8 *a = sqlite3PagerGetData(pDbPage); + memcpy(pPg->aPg, a, pgsz); + sqlite3PagerUnref(pDbPage); + } + + return rc; +} + +/* +** Move a DBSTAT cursor to the next entry. Normally, the next +** entry will be the next page, but in aggregated mode (pCsr->isAgg!=0), +** the next entry is the next btree. */ static int statNext(sqlite3_vtab_cursor *pCursor){ int rc; @@ -169084,7 +225481,9 @@ static int statNext(sqlite3_vtab_cursor *pCursor){ pCsr->zPath = 0; statNextRestart: - if( pCsr->aPage[0].pPg==0 ){ + if( pCsr->iPage<0 ){ + /* Start measuring space on the next btree */ + statResetCounts(pCsr); rc = sqlite3_step(pCsr->pStmt); if( rc==SQLITE_ROW ){ int nPage; @@ -169094,47 +225493,50 @@ statNextRestart: pCsr->isEof = 1; return sqlite3_reset(pCsr->pStmt); } - rc = sqlite3PagerGet(pPager, iRoot, &pCsr->aPage[0].pPg, 0); + rc = statGetPage(pBt, iRoot, &pCsr->aPage[0]); pCsr->aPage[0].iPgno = iRoot; pCsr->aPage[0].iCell = 0; - pCsr->aPage[0].zPath = z = sqlite3_mprintf("/"); + if( !pCsr->isAgg ){ + pCsr->aPage[0].zPath = z = sqlite3_mprintf("/"); + if( z==0 ) rc = SQLITE_NOMEM_BKPT; + } pCsr->iPage = 0; - if( z==0 ) rc = SQLITE_NOMEM_BKPT; + pCsr->nPage = 1; }else{ pCsr->isEof = 1; return sqlite3_reset(pCsr->pStmt); } }else{ - - /* Page p itself has already been visited. */ + /* Continue analyzing the btree previously started */ StatPage *p = &pCsr->aPage[pCsr->iPage]; - + if( !pCsr->isAgg ) statResetCounts(pCsr); while( p->iCellnCell ){ StatCell *pCell = &p->aCell[p->iCell]; - if( pCell->iOvflnOvfl ){ - int nUsable; + while( pCell->iOvflnOvfl ){ + int nUsable, iOvfl; sqlite3BtreeEnter(pBt); - nUsable = sqlite3BtreeGetPageSize(pBt) - + nUsable = sqlite3BtreeGetPageSize(pBt) - sqlite3BtreeGetReserveNoMutex(pBt); sqlite3BtreeLeave(pBt); - pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); - pCsr->iPageno = pCell->aOvfl[pCell->iOvfl]; - pCsr->zPagetype = "overflow"; - pCsr->nCell = 0; - pCsr->nMxPayload = 0; - pCsr->zPath = z = sqlite3_mprintf( - "%s%.3x+%.6x", p->zPath, p->iCell, pCell->iOvfl - ); - if( pCell->iOvflnOvfl-1 ){ - pCsr->nUnused = 0; - pCsr->nPayload = nUsable - 4; - }else{ - pCsr->nPayload = pCell->nLastOvfl; - pCsr->nUnused = nUsable - 4 - pCsr->nPayload; - } - pCell->iOvfl++; + pCsr->nPage++; statSizeAndOffset(pCsr); - return z==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; + if( pCell->iOvflnOvfl-1 ){ + pCsr->nPayload += nUsable - 4; + }else{ + pCsr->nPayload += pCell->nLastOvfl; + pCsr->nUnused += nUsable - 4 - pCell->nLastOvfl; + } + iOvfl = pCell->iOvfl; + pCell->iOvfl++; + if( !pCsr->isAgg ){ + pCsr->zName = (char *)sqlite3_column_text(pCsr->pStmt, 0); + pCsr->iPageno = pCell->aOvfl[iOvfl]; + pCsr->zPagetype = "overflow"; + pCsr->zPath = z = sqlite3_mprintf( + "%s%.3x+%.6x", p->zPath, p->iCell, iOvfl + ); + return z==0 ? SQLITE_NOMEM_BKPT : SQLITE_OK; + } } if( p->iRightChildPg ) break; p->iCell++; @@ -169142,11 +225544,19 @@ statNextRestart: if( !p->iRightChildPg || p->iCell>p->nCell ){ statClearPage(p); - if( pCsr->iPage==0 ) return statNext(pCursor); pCsr->iPage--; + if( pCsr->isAgg && pCsr->iPage<0 ){ + /* label-statNext-done: When computing aggregate space usage over + ** an entire btree, this is the exit point from this function */ + return SQLITE_OK; + } goto statNextRestart; /* Tail recursion */ } pCsr->iPage++; + if( pCsr->iPage>=ArraySize(pCsr->aPage) ){ + statResetCsr(pCsr); + return SQLITE_CORRUPT_BKPT; + } assert( p==&pCsr->aPage[pCsr->iPage-1] ); if( p->iCell==p->nCell ){ @@ -169154,11 +225564,14 @@ statNextRestart: }else{ p[1].iPgno = p->aCell[p->iCell].iChildPg; } - rc = sqlite3PagerGet(pPager, p[1].iPgno, &p[1].pPg, 0); + rc = statGetPage(pBt, p[1].iPgno, &p[1]); + pCsr->nPage++; p[1].iCell = 0; - p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); + if( !pCsr->isAgg ){ + p[1].zPath = z = sqlite3_mprintf("%s%.3x/", p->zPath, p->iCell); + if( z==0 ) rc = SQLITE_NOMEM_BKPT; + } p->iCell++; - if( z==0 ) rc = SQLITE_NOMEM_BKPT; } @@ -169188,16 +225601,23 @@ statNextRestart: pCsr->zPagetype = "corrupted"; break; } - pCsr->nCell = p->nCell; - pCsr->nUnused = p->nUnused; - pCsr->nMxPayload = p->nMxPayload; - pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath); - if( z==0 ) rc = SQLITE_NOMEM_BKPT; + pCsr->nCell += p->nCell; + pCsr->nUnused += p->nUnused; + if( p->nMxPayload>pCsr->nMxPayload ) pCsr->nMxPayload = p->nMxPayload; + if( !pCsr->isAgg ){ + pCsr->zPath = z = sqlite3_mprintf("%s", p->zPath); + if( z==0 ) rc = SQLITE_NOMEM_BKPT; + } nPayload = 0; for(i=0; inCell; i++){ nPayload += p->aCell[i].nLocal; } - pCsr->nPayload = nPayload; + pCsr->nPayload += nPayload; + + /* If computing aggregate space usage by btree, continue with the + ** next page. The loop will exit via the return at label-statNext-done + */ + if( pCsr->isAgg ) goto statNextRestart; } } @@ -169209,38 +225629,65 @@ static int statEof(sqlite3_vtab_cursor *pCursor){ return pCsr->isEof; } +/* Initialize a cursor according to the query plan idxNum using the +** arguments in argv[0]. See statBestIndex() for a description of the +** meaning of the bits in idxNum. +*/ static int statFilter( - sqlite3_vtab_cursor *pCursor, + sqlite3_vtab_cursor *pCursor, int idxNum, const char *idxStr, int argc, sqlite3_value **argv ){ StatCursor *pCsr = (StatCursor *)pCursor; StatTable *pTab = (StatTable*)(pCursor->pVtab); - char *zSql; - int rc = SQLITE_OK; - char *zMaster; + sqlite3_str *pSql; /* Query of btrees to analyze */ + char *zSql; /* String value of pSql */ + int iArg = 0; /* Count of argv[] parameters used so far */ + int rc = SQLITE_OK; /* Result of this operation */ + const char *zName = 0; /* Only provide analysis of this table */ + (void)argc; + (void)idxStr; - if( idxNum==1 ){ - const char *zDbase = (const char*)sqlite3_value_text(argv[0]); + statResetCsr(pCsr); + sqlite3_finalize(pCsr->pStmt); + pCsr->pStmt = 0; + if( idxNum & 0x01 ){ + /* schema=? constraint is present. Get its value */ + const char *zDbase = (const char*)sqlite3_value_text(argv[iArg++]); pCsr->iDb = sqlite3FindDbName(pTab->db, zDbase); if( pCsr->iDb<0 ){ - sqlite3_free(pCursor->pVtab->zErrMsg); - pCursor->pVtab->zErrMsg = sqlite3_mprintf("no such schema: %s", zDbase); - return pCursor->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM_BKPT; + pCsr->iDb = 0; + pCsr->isEof = 1; + return SQLITE_OK; } }else{ pCsr->iDb = pTab->iDb; } - statResetCsr(pCsr); - sqlite3_finalize(pCsr->pStmt); - pCsr->pStmt = 0; - zMaster = pCsr->iDb==1 ? "sqlite_temp_master" : "sqlite_master"; - zSql = sqlite3_mprintf( - "SELECT 'sqlite_master' AS name, 1 AS rootpage, 'table' AS type" - " UNION ALL " - "SELECT name, rootpage, type" - " FROM \"%w\".%s WHERE rootpage!=0" - " ORDER BY name", pTab->db->aDb[pCsr->iDb].zName, zMaster); + if( idxNum & 0x02 ){ + /* name=? constraint is present */ + zName = (const char*)sqlite3_value_text(argv[iArg++]); + } + if( idxNum & 0x04 ){ + /* aggregate=? constraint is present */ + pCsr->isAgg = sqlite3_value_double(argv[iArg++])!=0.0; + }else{ + pCsr->isAgg = 0; + } + pSql = sqlite3_str_new(pTab->db); + sqlite3_str_appendf(pSql, + "SELECT * FROM (" + "SELECT 'sqlite_schema' AS name,1 AS rootpage,'table' AS type" + " UNION ALL " + "SELECT name,rootpage,type" + " FROM \"%w\".sqlite_schema WHERE rootpage!=0)", + pTab->db->aDb[pCsr->iDb].zDbSName); + if( zName ){ + sqlite3_str_appendf(pSql, "WHERE name=%Q", zName); + } + if( idxNum & 0x08 ){ + sqlite3_str_appendf(pSql, " ORDER BY name"); + } + zSql = sqlite3_str_finish(pSql); if( zSql==0 ){ return SQLITE_NOMEM_BKPT; }else{ @@ -169249,14 +225696,15 @@ static int statFilter( } if( rc==SQLITE_OK ){ + pCsr->iPage = -1; rc = statNext(pCursor); } return rc; } static int statColumn( - sqlite3_vtab_cursor *pCursor, - sqlite3_context *ctx, + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, int i ){ StatCursor *pCsr = (StatCursor *)pCursor; @@ -169265,36 +225713,50 @@ static int statColumn( sqlite3_result_text(ctx, pCsr->zName, -1, SQLITE_TRANSIENT); break; case 1: /* path */ - sqlite3_result_text(ctx, pCsr->zPath, -1, SQLITE_TRANSIENT); + if( !pCsr->isAgg ){ + sqlite3_result_text(ctx, pCsr->zPath, -1, SQLITE_TRANSIENT); + } break; case 2: /* pageno */ - sqlite3_result_int64(ctx, pCsr->iPageno); + if( pCsr->isAgg ){ + sqlite3_result_int64(ctx, pCsr->nPage); + }else{ + sqlite3_result_int64(ctx, pCsr->iPageno); + } break; case 3: /* pagetype */ - sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC); + if( !pCsr->isAgg ){ + sqlite3_result_text(ctx, pCsr->zPagetype, -1, SQLITE_STATIC); + } break; case 4: /* ncell */ - sqlite3_result_int(ctx, pCsr->nCell); + sqlite3_result_int64(ctx, pCsr->nCell); break; case 5: /* payload */ - sqlite3_result_int(ctx, pCsr->nPayload); + sqlite3_result_int64(ctx, pCsr->nPayload); break; case 6: /* unused */ - sqlite3_result_int(ctx, pCsr->nUnused); + sqlite3_result_int64(ctx, pCsr->nUnused); break; case 7: /* mx_payload */ - sqlite3_result_int(ctx, pCsr->nMxPayload); + sqlite3_result_int64(ctx, pCsr->nMxPayload); break; case 8: /* pgoffset */ - sqlite3_result_int64(ctx, pCsr->iOffset); + if( !pCsr->isAgg ){ + sqlite3_result_int64(ctx, pCsr->iOffset); + } break; case 9: /* pgsize */ - sqlite3_result_int(ctx, pCsr->szPage); + sqlite3_result_int64(ctx, pCsr->szPage); break; - default: { /* schema */ + case 10: { /* schema */ sqlite3 *db = sqlite3_context_db_handle(ctx); int iDb = pCsr->iDb; - sqlite3_result_text(ctx, db->aDb[iDb].zName, -1, SQLITE_STATIC); + sqlite3_result_text(ctx, db->aDb[iDb].zDbSName, -1, SQLITE_STATIC); + break; + } + default: { /* aggregate */ + sqlite3_result_int(ctx, pCsr->isAgg); break; } } @@ -169332,6 +225794,11 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ 0, /* xRollback */ 0, /* xFindMethod */ 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ }; return sqlite3_create_module(db, "dbstat", &dbstat_module, 0); } @@ -169340,6 +225807,488 @@ SQLITE_PRIVATE int sqlite3DbstatRegister(sqlite3 *db){ return SQLITE_OK; } #endif /* SQLITE_ENABLE_DBSTAT_VTAB */ /************** End of dbstat.c **********************************************/ +/************** Begin file dbpage.c ******************************************/ +/* +** 2017-10-11 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** This file contains an implementation of the "sqlite_dbpage" virtual table. +** +** The sqlite_dbpage virtual table is used to read or write whole raw +** pages of the database file. The pager interface is used so that +** uncommitted changes and changes recorded in the WAL file are correctly +** retrieved. +** +** Usage example: +** +** SELECT data FROM sqlite_dbpage('aux1') WHERE pgno=123; +** +** This is an eponymous virtual table so it does not need to be created before +** use. The optional argument to the sqlite_dbpage() table name is the +** schema for the database file that is to be read. The default schema is +** "main". +** +** The data field of sqlite_dbpage table can be updated. The new +** value must be a BLOB which is the correct page size, otherwise the +** update fails. INSERT operations also work, and operate as if they +** where REPLACE. The size of the database can be extended by INSERT-ing +** new pages on the end. +** +** Rows may not be deleted. However, doing an INSERT to page number N +** with NULL page data causes the N-th page and all subsequent pages to be +** deleted and the database to be truncated. +*/ + +/* #include "sqliteInt.h" ** Requires access to internal data structures ** */ +#if (defined(SQLITE_ENABLE_DBPAGE_VTAB) || defined(SQLITE_TEST)) \ + && !defined(SQLITE_OMIT_VIRTUALTABLE) + +typedef struct DbpageTable DbpageTable; +typedef struct DbpageCursor DbpageCursor; + +struct DbpageCursor { + sqlite3_vtab_cursor base; /* Base class. Must be first */ + int pgno; /* Current page number */ + int mxPgno; /* Last page to visit on this scan */ + Pager *pPager; /* Pager being read/written */ + DbPage *pPage1; /* Page 1 of the database */ + int iDb; /* Index of database to analyze */ + int szPage; /* Size of each page in bytes */ +}; + +struct DbpageTable { + sqlite3_vtab base; /* Base class. Must be first */ + sqlite3 *db; /* The database */ + int iDbTrunc; /* Database to truncate */ + Pgno pgnoTrunc; /* Size to truncate to */ +}; + +/* Columns */ +#define DBPAGE_COLUMN_PGNO 0 +#define DBPAGE_COLUMN_DATA 1 +#define DBPAGE_COLUMN_SCHEMA 2 + + +/* +** Connect to or create a dbpagevfs virtual table. +*/ +static int dbpageConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + DbpageTable *pTab = 0; + int rc = SQLITE_OK; + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; + + sqlite3_vtab_config(db, SQLITE_VTAB_DIRECTONLY); + sqlite3_vtab_config(db, SQLITE_VTAB_USES_ALL_SCHEMAS); + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(pgno INTEGER PRIMARY KEY, data BLOB, schema HIDDEN)"); + if( rc==SQLITE_OK ){ + pTab = (DbpageTable *)sqlite3_malloc64(sizeof(DbpageTable)); + if( pTab==0 ) rc = SQLITE_NOMEM_BKPT; + } + + assert( rc==SQLITE_OK || pTab==0 ); + if( rc==SQLITE_OK ){ + memset(pTab, 0, sizeof(DbpageTable)); + pTab->db = db; + } + + *ppVtab = (sqlite3_vtab*)pTab; + return rc; +} + +/* +** Disconnect from or destroy a dbpagevfs virtual table. +*/ +static int dbpageDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** idxNum: +** +** 0 schema=main, full table scan +** 1 schema=main, pgno=?1 +** 2 schema=?1, full table scan +** 3 schema=?1, pgno=?2 +*/ +static int dbpageBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo){ + int i; + int iPlan = 0; + (void)tab; + + /* If there is a schema= constraint, it must be honored. Report a + ** ridiculously large estimated cost if the schema= constraint is + ** unavailable + */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->iColumn!=DBPAGE_COLUMN_SCHEMA ) continue; + if( p->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; + if( !p->usable ){ + /* No solution. */ + return SQLITE_CONSTRAINT; + } + iPlan = 2; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + break; + } + + /* If we reach this point, it means that either there is no schema= + ** constraint (in which case we use the "main" schema) or else the + ** schema constraint was accepted. Lower the estimated cost accordingly + */ + pIdxInfo->estimatedCost = 1.0e6; + + /* Check for constraints against pgno */ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pIdxInfo->aConstraint[i]; + if( p->usable && p->iColumn<=0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ ){ + pIdxInfo->estimatedRows = 1; + pIdxInfo->idxFlags = SQLITE_INDEX_SCAN_UNIQUE; + pIdxInfo->estimatedCost = 1.0; + pIdxInfo->aConstraintUsage[i].argvIndex = iPlan ? 2 : 1; + pIdxInfo->aConstraintUsage[i].omit = 1; + iPlan |= 1; + break; + } + } + pIdxInfo->idxNum = iPlan; + + if( pIdxInfo->nOrderBy>=1 + && pIdxInfo->aOrderBy[0].iColumn<=0 + && pIdxInfo->aOrderBy[0].desc==0 + ){ + pIdxInfo->orderByConsumed = 1; + } + return SQLITE_OK; +} + +/* +** Open a new dbpagevfs cursor. +*/ +static int dbpageOpen(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCursor){ + DbpageCursor *pCsr; + + pCsr = (DbpageCursor *)sqlite3_malloc64(sizeof(DbpageCursor)); + if( pCsr==0 ){ + return SQLITE_NOMEM_BKPT; + }else{ + memset(pCsr, 0, sizeof(DbpageCursor)); + pCsr->base.pVtab = pVTab; + pCsr->pgno = -1; + } + + *ppCursor = (sqlite3_vtab_cursor *)pCsr; + return SQLITE_OK; +} + +/* +** Close a dbpagevfs cursor. +*/ +static int dbpageClose(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1); + sqlite3_free(pCsr); + return SQLITE_OK; +} + +/* +** Move a dbpagevfs cursor to the next entry in the file. +*/ +static int dbpageNext(sqlite3_vtab_cursor *pCursor){ + int rc = SQLITE_OK; + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + pCsr->pgno++; + return rc; +} + +static int dbpageEof(sqlite3_vtab_cursor *pCursor){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + return pCsr->pgno > pCsr->mxPgno; +} + +/* +** idxNum: +** +** 0 schema=main, full table scan +** 1 schema=main, pgno=?1 +** 2 schema=?1, full table scan +** 3 schema=?1, pgno=?2 +** +** idxStr is not used +*/ +static int dbpageFilter( + sqlite3_vtab_cursor *pCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + DbpageTable *pTab = (DbpageTable *)pCursor->pVtab; + int rc; + sqlite3 *db = pTab->db; + Btree *pBt; + + (void)idxStr; + + /* Default setting is no rows of result */ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + + if( idxNum & 2 ){ + const char *zSchema; + assert( argc>=1 ); + zSchema = (const char*)sqlite3_value_text(argv[0]); + pCsr->iDb = sqlite3FindDbName(db, zSchema); + if( pCsr->iDb<0 ) return SQLITE_OK; + }else{ + pCsr->iDb = 0; + } + pBt = db->aDb[pCsr->iDb].pBt; + if( NEVER(pBt==0) ) return SQLITE_OK; + pCsr->pPager = sqlite3BtreePager(pBt); + pCsr->szPage = sqlite3BtreeGetPageSize(pBt); + pCsr->mxPgno = sqlite3BtreeLastPage(pBt); + if( idxNum & 1 ){ + assert( argc>(idxNum>>1) ); + pCsr->pgno = sqlite3_value_int(argv[idxNum>>1]); + if( pCsr->pgno<1 || pCsr->pgno>pCsr->mxPgno ){ + pCsr->pgno = 1; + pCsr->mxPgno = 0; + }else{ + pCsr->mxPgno = pCsr->pgno; + } + }else{ + assert( pCsr->pgno==1 ); + } + if( pCsr->pPage1 ) sqlite3PagerUnrefPageOne(pCsr->pPage1); + rc = sqlite3PagerGet(pCsr->pPager, 1, &pCsr->pPage1, 0); + return rc; +} + +static int dbpageColumn( + sqlite3_vtab_cursor *pCursor, + sqlite3_context *ctx, + int i +){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + int rc = SQLITE_OK; + switch( i ){ + case 0: { /* pgno */ + sqlite3_result_int(ctx, pCsr->pgno); + break; + } + case 1: { /* data */ + DbPage *pDbPage = 0; + if( pCsr->pgno==((PENDING_BYTE/pCsr->szPage)+1) ){ + /* The pending byte page. Assume it is zeroed out. Attempting to + ** request this page from the page is an SQLITE_CORRUPT error. */ + sqlite3_result_zeroblob(ctx, pCsr->szPage); + }else{ + rc = sqlite3PagerGet(pCsr->pPager, pCsr->pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + sqlite3_result_blob(ctx, sqlite3PagerGetData(pDbPage), pCsr->szPage, + SQLITE_TRANSIENT); + } + sqlite3PagerUnref(pDbPage); + } + break; + } + default: { /* schema */ + sqlite3 *db = sqlite3_context_db_handle(ctx); + sqlite3_result_text(ctx, db->aDb[pCsr->iDb].zDbSName, -1, SQLITE_STATIC); + break; + } + } + return rc; +} + +static int dbpageRowid(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ + DbpageCursor *pCsr = (DbpageCursor *)pCursor; + *pRowid = pCsr->pgno; + return SQLITE_OK; +} + +static int dbpageUpdate( + sqlite3_vtab *pVtab, + int argc, + sqlite3_value **argv, + sqlite_int64 *pRowid +){ + DbpageTable *pTab = (DbpageTable *)pVtab; + Pgno pgno; + DbPage *pDbPage = 0; + int rc = SQLITE_OK; + char *zErr = 0; + int iDb; + Btree *pBt; + Pager *pPager; + int szPage; + int isInsert; + + (void)pRowid; + if( pTab->db->flags & SQLITE_Defensive ){ + zErr = "read-only"; + goto update_fail; + } + if( argc==1 ){ + zErr = "cannot delete"; + goto update_fail; + } + if( sqlite3_value_type(argv[0])==SQLITE_NULL ){ + pgno = (Pgno)sqlite3_value_int(argv[2]); + isInsert = 1; + }else{ + pgno = sqlite3_value_int(argv[0]); + if( (Pgno)sqlite3_value_int(argv[1])!=pgno ){ + zErr = "cannot insert"; + goto update_fail; + } + isInsert = 0; + } + if( sqlite3_value_type(argv[4])==SQLITE_NULL ){ + iDb = 0; + }else{ + const char *zSchema = (const char*)sqlite3_value_text(argv[4]); + iDb = sqlite3FindDbName(pTab->db, zSchema); + if( iDb<0 ){ + zErr = "no such schema"; + goto update_fail; + } + } + pBt = pTab->db->aDb[iDb].pBt; + if( pgno<1 || NEVER(pBt==0) ){ + zErr = "bad page number"; + goto update_fail; + } + szPage = sqlite3BtreeGetPageSize(pBt); + if( sqlite3_value_type(argv[3])!=SQLITE_BLOB + || sqlite3_value_bytes(argv[3])!=szPage + ){ + if( sqlite3_value_type(argv[3])==SQLITE_NULL && isInsert && pgno>1 ){ + /* "INSERT INTO dbpage($PGNO,NULL)" causes page number $PGNO and + ** all subsequent pages to be deleted. */ + pTab->iDbTrunc = iDb; + pgno--; + pTab->pgnoTrunc = pgno; + }else{ + zErr = "bad page value"; + goto update_fail; + } + } + pPager = sqlite3BtreePager(pBt); + rc = sqlite3PagerGet(pPager, pgno, (DbPage**)&pDbPage, 0); + if( rc==SQLITE_OK ){ + const void *pData = sqlite3_value_blob(argv[3]); + if( (rc = sqlite3PagerWrite(pDbPage))==SQLITE_OK && pData ){ + unsigned char *aPage = sqlite3PagerGetData(pDbPage); + memcpy(aPage, pData, szPage); + pTab->pgnoTrunc = 0; + } + } + sqlite3PagerUnref(pDbPage); + return rc; + +update_fail: + sqlite3_free(pVtab->zErrMsg); + pVtab->zErrMsg = sqlite3_mprintf("%s", zErr); + return SQLITE_ERROR; +} + +/* Since we do not know in advance which database files will be +** written by the sqlite_dbpage virtual table, start a write transaction +** on them all. +*/ +static int dbpageBegin(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + sqlite3 *db = pTab->db; + int i; + for(i=0; inDb; i++){ + Btree *pBt = db->aDb[i].pBt; + if( pBt ) (void)sqlite3BtreeBeginTrans(pBt, 1, 0); + } + pTab->pgnoTrunc = 0; + return SQLITE_OK; +} + +/* Invoke sqlite3PagerTruncate() as necessary, just prior to COMMIT +*/ +static int dbpageSync(sqlite3_vtab *pVtab){ + DbpageTable *pTab = (DbpageTable *)pVtab; + if( pTab->pgnoTrunc>0 ){ + Btree *pBt = pTab->db->aDb[pTab->iDbTrunc].pBt; + Pager *pPager = sqlite3BtreePager(pBt); + sqlite3PagerTruncateImage(pPager, pTab->pgnoTrunc); + } + pTab->pgnoTrunc = 0; + return SQLITE_OK; +} + +/* Cancel any pending truncate. +*/ +static int dbpageRollbackTo(sqlite3_vtab *pVtab, int notUsed1){ + DbpageTable *pTab = (DbpageTable *)pVtab; + pTab->pgnoTrunc = 0; + (void)notUsed1; + return SQLITE_OK; +} + +/* +** Invoke this routine to register the "dbpage" virtual table module +*/ +SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ + static sqlite3_module dbpage_module = { + 0, /* iVersion */ + dbpageConnect, /* xCreate */ + dbpageConnect, /* xConnect */ + dbpageBestIndex, /* xBestIndex */ + dbpageDisconnect, /* xDisconnect */ + dbpageDisconnect, /* xDestroy */ + dbpageOpen, /* xOpen - open a cursor */ + dbpageClose, /* xClose - close a cursor */ + dbpageFilter, /* xFilter - configure scan constraints */ + dbpageNext, /* xNext - advance a cursor */ + dbpageEof, /* xEof - check for end of scan */ + dbpageColumn, /* xColumn - read data */ + dbpageRowid, /* xRowid - read data */ + dbpageUpdate, /* xUpdate */ + dbpageBegin, /* xBegin */ + dbpageSync, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + dbpageRollbackTo, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + return sqlite3_create_module(db, "sqlite_dbpage", &dbpage_module, 0); +} +#elif defined(SQLITE_ENABLE_DBPAGE_VTAB) +SQLITE_PRIVATE int sqlite3DbpageRegister(sqlite3 *db){ return SQLITE_OK; } +#endif /* SQLITE_ENABLE_DBSTAT_VTAB */ + +/************** End of dbpage.c **********************************************/ /************** Begin file sqlite3session.c **********************************/ #if defined(SQLITE_ENABLE_SESSION) && defined(SQLITE_ENABLE_PREUPDATE_HOOK) @@ -169368,6 +226317,10 @@ typedef struct SessionInput SessionInput; # endif #endif +#define SESSIONS_ROWID "_rowid_" + +static int sessions_strm_chunk_size = SESSIONS_STRM_CHUNK_SIZE; + typedef struct SessionHook SessionHook; struct SessionHook { void *pCtx; @@ -169383,12 +226336,17 @@ struct SessionHook { struct sqlite3_session { sqlite3 *db; /* Database handle session is attached to */ char *zDb; /* Name of database session is attached to */ + int bEnableSize; /* True if changeset_size() enabled */ int bEnable; /* True if currently recording */ int bIndirect; /* True if all changes are indirect */ int bAutoAttach; /* True to auto-attach tables */ + int bImplicitPK; /* True to handle tables with implicit PK */ int rc; /* Non-zero if an error has occurred */ void *pFilterCtx; /* First argument to pass to xTableFilter */ int (*xTableFilter)(void *pCtx, const char *zTab); + i64 nMalloc; /* Number of bytes of data allocated */ + i64 nMaxChangesetSize; + sqlite3_value *pZeroBlob; /* Value containing X'' */ sqlite3_session *pNext; /* Next session object on same db. */ SessionTable *pTable; /* List of attached tables */ SessionHook hook; /* APIs to grab new and old data with */ @@ -169404,13 +226362,17 @@ struct SessionBuffer { }; /* -** An object of this type is used internally as an abstraction for +** An object of this type is used internally as an abstraction for ** input data. Input data may be supplied either as a single large buffer ** (e.g. sqlite3changeset_start()) or using a stream function (e.g. ** sqlite3changeset_start_strm()). +** +** bNoDiscard: +** If true, then the only time data is discarded is as a result of explicit +** sessionDiscardData() calls. Not within every sessionInputBuffer() call. */ struct SessionInput { - int bNoDiscard; /* If true, discard no data */ + int bNoDiscard; /* If true, do not discard in InputBuffer() */ int iCurrent; /* Offset in aData[] of current change */ int iNext; /* Offset in aData[] of next change */ u8 *aData; /* Pointer to buffer containing changeset */ @@ -169429,6 +226391,8 @@ struct sqlite3_changeset_iter { SessionInput in; /* Input buffer or stream */ SessionBuffer tblhdr; /* Buffer to hold apValue/zTab/abPK/ */ int bPatchset; /* True if this is a patchset */ + int bInvert; /* True to invert changeset */ + int bSkipEmpty; /* Skip noop UPDATE changes */ int rc; /* Iterator error code */ sqlite3_stmt *pConflict; /* Points to conflicting row, if any */ char *zTab; /* Current table */ @@ -169451,23 +226415,39 @@ struct sqlite3_changeset_iter { ** The data associated with each hash-table entry is a structure containing ** a subset of the initial values that the modified row contained at the ** start of the session. Or no initial values if the row was inserted. +** +** pDfltStmt: +** This is only used by the sqlite3changegroup_xxx() APIs, not by +** regular sqlite3_session objects. It is a SELECT statement that +** selects the default value for each table column. For example, +** if the table is +** +** CREATE TABLE xx(a DEFAULT 1, b, c DEFAULT 'abc') +** +** then this variable is the compiled version of: +** +** SELECT 1, NULL, 'abc' */ struct SessionTable { SessionTable *pNext; char *zName; /* Local name of table */ int nCol; /* Number of columns in table zName */ + int bStat1; /* True if this is sqlite_stat1 */ + int bRowid; /* True if this table uses rowid for PK */ const char **azCol; /* Column names */ + const char **azDflt; /* Default value expressions */ u8 *abPK; /* Array of primary key flags */ int nEntry; /* Total number of entries in hash table */ int nChange; /* Size of apChange[] array */ SessionChange **apChange; /* Hash table buckets */ + sqlite3_stmt *pDfltStmt; }; -/* +/* ** RECORD FORMAT: ** -** The following record format is similar to (but not compatible with) that -** used in SQLite database files. This format is used as part of the +** The following record format is similar to (but not compatible with) that +** used in SQLite database files. This format is used as part of the ** change-set binary format, and so must be architecture independent. ** ** Unlike the SQLite database record format, each field is self-contained - @@ -169501,7 +226481,7 @@ struct SessionTable { ** Real values: ** An 8-byte big-endian IEEE 754-2008 real value. ** -** Varint values are encoded in the same way as varints in the SQLite +** Varint values are encoded in the same way as varints in the SQLite ** record format. ** ** CHANGESET FORMAT: @@ -169533,7 +226513,7 @@ struct SessionTable { ** ** The new.* record that is part of each INSERT change contains the values ** that make up the new row. Similarly, the old.* record that is part of each -** DELETE change contains the values that made up the row that was deleted +** DELETE change contains the values that made up the row that was deleted ** from the database. In the changeset format, the records that are part ** of INSERT or DELETE changes never contain any undefined (type byte 0x00) ** fields. @@ -169542,8 +226522,8 @@ struct SessionTable { ** associated with table columns that are not PRIMARY KEY columns and are ** not modified by the UPDATE change are set to "undefined". Other fields ** are set to the values that made up the row before the UPDATE that the -** change records took place. Within the new.* record, fields associated -** with table columns modified by the UPDATE change contain the new +** change records took place. Within the new.* record, fields associated +** with table columns modified by the UPDATE change contain the new ** values. Fields associated with table columns that are not modified ** are set to "undefined". ** @@ -169569,12 +226549,12 @@ struct SessionTable { ** ** As in the changeset format, each field of the single record that is part ** of a patchset change is associated with the correspondingly positioned -** table column, counting from left to right within the CREATE TABLE +** table column, counting from left to right within the CREATE TABLE ** statement. ** ** For a DELETE change, all fields within the record except those associated -** with PRIMARY KEY columns are set to "undefined". The PRIMARY KEY fields -** contain the values identifying the row to delete. +** with PRIMARY KEY columns are omitted. The PRIMARY KEY fields contain the +** values identifying the row to delete. ** ** For an UPDATE change, all fields except those associated with PRIMARY KEY ** columns and columns that are modified by the UPDATE are set to "undefined". @@ -169584,6 +226564,42 @@ struct SessionTable { ** The records associated with INSERT changes are in the same format as for ** changesets. It is not possible for a record associated with an INSERT ** change to contain a field set to "undefined". +** +** REBASE BLOB FORMAT: +** +** A rebase blob may be output by sqlite3changeset_apply_v2() and its +** streaming equivalent for use with the sqlite3_rebaser APIs to rebase +** existing changesets. A rebase blob contains one entry for each conflict +** resolved using either the OMIT or REPLACE strategies within the apply_v2() +** call. +** +** The format used for a rebase blob is very similar to that used for +** changesets. All entries related to a single table are grouped together. +** +** Each group of entries begins with a table header in changeset format: +** +** 1 byte: Constant 0x54 (capital 'T') +** Varint: Number of columns in the table. +** nCol bytes: 0x01 for PK columns, 0x00 otherwise. +** N bytes: Unqualified table name (encoded using UTF-8). Nul-terminated. +** +** Followed by one or more entries associated with the table. +** +** 1 byte: Either SQLITE_INSERT (0x12), DELETE (0x09). +** 1 byte: Flag. 0x01 for REPLACE, 0x00 for OMIT. +** record: (in the record format defined above). +** +** In a rebase blob, the first field is set to SQLITE_INSERT if the change +** that caused the conflict was an INSERT or UPDATE, or to SQLITE_DELETE if +** it was a DELETE. The second field is set to 0x01 if the conflict +** resolution strategy was REPLACE, or 0x00 if it was OMIT. +** +** If the change that caused the conflict was a DELETE, then the single +** record is a copy of the old.* record from the original changeset. If it +** was an INSERT, then the single record is a copy of the new.* record. If +** the conflicting change was an UPDATE, then the single record is a copy +** of the new.* record with the PK fields filled in based on the original +** old.* record. */ /* @@ -169591,15 +226607,17 @@ struct SessionTable { ** this structure stored in a SessionTable.aChange[] hash table. */ struct SessionChange { - int op; /* One of UPDATE, DELETE, INSERT */ - int bIndirect; /* True if this change is "indirect" */ + u8 op; /* One of UPDATE, DELETE, INSERT */ + u8 bIndirect; /* True if this change is "indirect" */ + u16 nRecordField; /* Number of fields in aRecord[] */ + int nMaxSize; /* Max size of eventual changeset record */ int nRecord; /* Number of bytes in buffer aRecord[] */ u8 *aRecord; /* Buffer containing old.* record */ SessionChange *pNext; /* For hash-table collisions */ }; /* -** Write a varint with value iVal into the buffer at aBuf. Return the +** Write a varint with value iVal into the buffer at aBuf. Return the ** number of bytes written. */ static int sessionVarintPut(u8 *aBuf, int iVal){ @@ -169614,10 +226632,10 @@ static int sessionVarintLen(int iVal){ } /* -** Read a varint value from aBuf[] into *piVal. Return the number of +** Read a varint value from aBuf[] into *piVal. Return the number of ** bytes read. */ -static int sessionVarintGet(u8 *aBuf, int *piVal){ +static int sessionVarintGet(const u8 *aBuf, int *piVal){ return getVarint32(aBuf, *piVal); } @@ -169653,34 +226671,34 @@ static void sessionPutI64(u8 *aBuf, sqlite3_int64 i){ ** This function is used to serialize the contents of value pValue (see ** comment titled "RECORD FORMAT" above). ** -** If it is non-NULL, the serialized form of the value is written to +** If it is non-NULL, the serialized form of the value is written to ** buffer aBuf. *pnWrite is set to the number of bytes written before ** returning. Or, if aBuf is NULL, the only thing this function does is ** set *pnWrite. ** ** If no error occurs, SQLITE_OK is returned. Or, if an OOM error occurs -** within a call to sqlite3_value_text() (may fail if the db is utf-16)) +** within a call to sqlite3_value_text() (may fail if the db is utf-16)) ** SQLITE_NOMEM is returned. */ static int sessionSerializeValue( u8 *aBuf, /* If non-NULL, write serialized value here */ sqlite3_value *pValue, /* Value to serialize */ - int *pnWrite /* IN/OUT: Increment by bytes written */ + sqlite3_int64 *pnWrite /* IN/OUT: Increment by bytes written */ ){ int nByte; /* Size of serialized value in bytes */ if( pValue ){ int eType; /* Value type (SQLITE_NULL, TEXT etc.) */ - + eType = sqlite3_value_type(pValue); if( aBuf ) aBuf[0] = eType; - + switch( eType ){ - case SQLITE_NULL: + case SQLITE_NULL: nByte = 1; break; - - case SQLITE_INTEGER: + + case SQLITE_INTEGER: case SQLITE_FLOAT: if( aBuf ){ /* TODO: SQLite does something special to deal with mixed-endian @@ -169697,14 +226715,14 @@ static int sessionSerializeValue( } sessionPutI64(&aBuf[1], i); } - nByte = 9; + nByte = 9; break; - + default: { u8 *z; int n; int nVarint; - + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); if( eType==SQLITE_TEXT ){ z = (u8 *)sqlite3_value_text(pValue); @@ -169714,14 +226732,12 @@ static int sessionSerializeValue( n = sqlite3_value_bytes(pValue); if( z==0 && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; nVarint = sessionVarintLen(n); - + if( aBuf ){ sessionVarintPut(&aBuf[1], n); - memcpy(&aBuf[nVarint + 1], eType==SQLITE_TEXT ? - sqlite3_value_text(pValue) : sqlite3_value_blob(pValue), n - ); + if( n>0 ) memcpy(&aBuf[nVarint + 1], z, n); } - + nByte = 1 + nVarint + n; break; } @@ -169735,6 +226751,26 @@ static int sessionSerializeValue( return SQLITE_OK; } +/* +** Allocate and return a pointer to a buffer nByte bytes in size. If +** pSession is not NULL, increase the sqlite3_session.nMalloc variable +** by the number of bytes allocated. +*/ +static void *sessionMalloc64(sqlite3_session *pSession, i64 nByte){ + void *pRet = sqlite3_malloc64(nByte); + if( pSession ) pSession->nMalloc += sqlite3_msize(pRet); + return pRet; +} + +/* +** Free buffer pFree, which must have been allocated by an earlier +** call to sessionMalloc64(). If pSession is not NULL, decrease the +** sqlite3_session.nMalloc counter by the number of bytes freed. +*/ +static void sessionFree(sqlite3_session *pSession, void *pFree){ + if( pSession ) pSession->nMalloc -= sqlite3_msize(pFree); + sqlite3_free(pFree); +} /* ** This macro is used to calculate hash key values for data structures. In @@ -169763,7 +226799,7 @@ static unsigned int sessionHashAppendI64(unsigned int h, i64 i){ } /* -** Append the hash of the blob passed via the second and third arguments to +** Append the hash of the blob passed via the second and third arguments to ** the hash-key value passed as the first. Return the new hash-key value. */ static unsigned int sessionHashAppendBlob(unsigned int h, int n, const u8 *z){ @@ -169782,7 +226818,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ /* ** This function may only be called from within a pre-update callback. -** It calculates a hash based on the primary key values of the old.* or +** It calculates a hash based on the primary key values of the old.* or ** new.* row currently available and, assuming no error occurs, writes it to ** *piHash before returning. If the primary key contains one or more NULL ** values, *pbNullPK is set to true before returning. @@ -169793,6 +226829,7 @@ static unsigned int sessionHashAppendType(unsigned int h, int eType){ */ static int sessionPreupdateHash( sqlite3_session *pSession, /* Session object that owns pTab */ + i64 iRowid, SessionTable *pTab, /* Session table handle */ int bNew, /* True to hash the new.* PK */ int *piHash, /* OUT: Hash value */ @@ -169801,47 +226838,53 @@ static int sessionPreupdateHash( unsigned int h = 0; /* Hash value to return */ int i; /* Used to iterate through columns */ - assert( *pbNullPK==0 ); - assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); - for(i=0; inCol; i++){ - if( pTab->abPK[i] ){ - int rc; - int eType; - sqlite3_value *pVal; + if( pTab->bRowid ){ + assert( pTab->nCol-1==pSession->hook.xCount(pSession->hook.pCtx) ); + h = sessionHashAppendI64(h, iRowid); + }else{ + assert( *pbNullPK==0 ); + assert( pTab->nCol==pSession->hook.xCount(pSession->hook.pCtx) ); + for(i=0; inCol; i++){ + if( pTab->abPK[i] ){ + int rc; + int eType; + sqlite3_value *pVal; - if( bNew ){ - rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); - }else{ - rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); - } - if( rc!=SQLITE_OK ) return rc; + if( bNew ){ + rc = pSession->hook.xNew(pSession->hook.pCtx, i, &pVal); + }else{ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &pVal); + } + if( rc!=SQLITE_OK ) return rc; - eType = sqlite3_value_type(pVal); - h = sessionHashAppendType(h, eType); - if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - i64 iVal; - if( eType==SQLITE_INTEGER ){ - iVal = sqlite3_value_int64(pVal); + eType = sqlite3_value_type(pVal); + h = sessionHashAppendType(h, eType); + if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_value_int64(pVal); + }else{ + double rVal = sqlite3_value_double(pVal); + assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); + memcpy(&iVal, &rVal, 8); + } + h = sessionHashAppendI64(h, iVal); + }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ + const u8 *z; + int n; + if( eType==SQLITE_TEXT ){ + z = (const u8 *)sqlite3_value_text(pVal); + }else{ + z = (const u8 *)sqlite3_value_blob(pVal); + } + n = sqlite3_value_bytes(pVal); + if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; + h = sessionHashAppendBlob(h, n, z); }else{ - double rVal = sqlite3_value_double(pVal); - assert( sizeof(iVal)==8 && sizeof(rVal)==8 ); - memcpy(&iVal, &rVal, 8); + assert( eType==SQLITE_NULL ); + assert( pTab->bStat1==0 || i!=1 ); + *pbNullPK = 1; } - h = sessionHashAppendI64(h, iVal); - }else if( eType==SQLITE_TEXT || eType==SQLITE_BLOB ){ - const u8 *z; - int n; - if( eType==SQLITE_TEXT ){ - z = (const u8 *)sqlite3_value_text(pVal); - }else{ - z = (const u8 *)sqlite3_value_blob(pVal); - } - n = sqlite3_value_bytes(pVal); - if( !z && (eType!=SQLITE_BLOB || n>0) ) return SQLITE_NOMEM; - h = sessionHashAppendBlob(h, n, z); - }else{ - assert( eType==SQLITE_NULL ); - *pbNullPK = 1; } } } @@ -169855,10 +226898,12 @@ static int sessionPreupdateHash( ** Return the number of bytes of space occupied by the value (including ** the type byte). */ -static int sessionSerialLen(u8 *a){ - int e = *a; +static int sessionSerialLen(const u8 *a){ + int e; int n; - if( e==0 ) return 1; + assert( a!=0 ); + e = *a; + if( e==0 || e==0xFF ) return 1; if( e==SQLITE_NULL ) return 1; if( e==SQLITE_INTEGER || e==SQLITE_FLOAT ) return 9; return sessionVarintGet(&a[1], &n) + 1 + n; @@ -169888,12 +226933,12 @@ static unsigned int sessionChangeHash( int isPK = pTab->abPK[i]; if( bPkOnly && isPK==0 ) continue; - /* It is not possible for eType to be SQLITE_NULL here. The session + /* It is not possible for eType to be SQLITE_NULL here. The session ** module does not record changes for rows with NULL values stored in ** primary key columns. */ - assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT - || eType==SQLITE_TEXT || eType==SQLITE_BLOB - || eType==SQLITE_NULL || eType==0 + assert( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT + || eType==SQLITE_TEXT || eType==SQLITE_BLOB + || eType==SQLITE_NULL || eType==0 ); assert( !isPK || (eType!=0 && eType!=SQLITE_NULL) ); @@ -169904,7 +226949,7 @@ static unsigned int sessionChangeHash( h = sessionHashAppendI64(h, sessionGetI64(a)); a += 8; }else{ - int n; + int n; a += sessionVarintGet(a, &n); h = sessionHashAppendBlob(h, n, a); a += n; @@ -169919,7 +226964,7 @@ static unsigned int sessionChangeHash( /* ** Arguments aLeft and aRight are pointers to change records for table pTab. ** This function returns true if the two records apply to the same row (i.e. -** have the same values stored in the primary key columns), or false +** have the same values stored in the primary key columns), or false ** otherwise. */ static int sessionChangeEqual( @@ -169938,7 +226983,7 @@ static int sessionChangeEqual( int n1 = sessionSerialLen(a1); int n2 = sessionSerialLen(a2); - if( pTab->abPK[iCol] && (n1!=n2 || memcmp(a1, a2, n1)) ){ + if( n1!=n2 || memcmp(a1, a2, n1) ){ return 0; } a1 += n1; @@ -169956,17 +227001,17 @@ static int sessionChangeEqual( ** Arguments aLeft and aRight both point to buffers containing change ** records with nCol columns. This function "merges" the two records into ** a single records which is written to the buffer at *paOut. *paOut is -** then set to point to one byte after the last byte written before +** then set to point to one byte after the last byte written before ** returning. ** -** The merging of records is done as follows: For each column, if the +** The merging of records is done as follows: For each column, if the ** aRight record contains a value for the column, copy the value from ** their. Otherwise, if aLeft contains a value, copy it. If neither ** record contains a value for a given column, then neither does the ** output record. */ static void sessionMergeRecord( - u8 **paOut, + u8 **paOut, int nCol, u8 *aLeft, u8 *aRight @@ -169996,13 +227041,13 @@ static void sessionMergeRecord( /* ** This is a helper function used by sessionMergeUpdate(). ** -** When this function is called, both *paOne and *paTwo point to a value -** within a change record. Before it returns, both have been advanced so +** When this function is called, both *paOne and *paTwo point to a value +** within a change record. Before it returns, both have been advanced so ** as to point to the next value in the record. ** ** If, when this function is called, *paTwo points to a valid value (i.e. ** *paTwo[0] is not 0x00 - the "no value" placeholder), a copy of the *paTwo -** pointer is returned and *pnVal is set to the number of bytes in the +** pointer is returned and *pnVal is set to the number of bytes in the ** serialized value. Otherwise, a copy of *paOne is returned and *pnVal ** set to the number of bytes in the value at *paOne. If *paOne points ** to the "no value" placeholder, *pnVal is set to 1. In other words: @@ -170101,8 +227146,8 @@ static int sessionMergeUpdate( aOld = sessionMergeValue(&aOld1, &aOld2, &nOld); aNew = sessionMergeValue(&aNew1, &aNew2, &nNew); - if( bPatchset==0 - && (pTab->abPK[i] || (nOld==nNew && 0==memcmp(aOld, aNew, nNew))) + if( bPatchset==0 + && (pTab->abPK[i] || (nOld==nNew && 0==memcmp(aOld, aNew, nNew))) ){ *(aOut++) = '\0'; }else{ @@ -170124,6 +227169,7 @@ static int sessionMergeUpdate( */ static int sessionPreupdateEqual( sqlite3_session *pSession, /* Session object that owns SessionTable */ + i64 iRowid, /* Rowid value if pTab->bRowid */ SessionTable *pTab, /* Table associated with change */ SessionChange *pChange, /* Change to compare to */ int op /* Current pre-update operation */ @@ -170131,6 +227177,11 @@ static int sessionPreupdateEqual( int iCol; /* Used to iterate through columns */ u8 *a = pChange->aRecord; /* Cursor used to scan change record */ + if( pTab->bRowid ){ + if( a[0]!=SQLITE_INTEGER ) return 0; + return sessionGetI64(&a[1])==iRowid; + } + assert( op==SQLITE_INSERT || op==SQLITE_UPDATE || op==SQLITE_DELETE ); for(iCol=0; iColnCol; iCol++){ if( !pTab->abPK[iCol] ){ @@ -170153,6 +227204,7 @@ static int sessionPreupdateEqual( rc = pSession->hook.xOld(pSession->hook.pCtx, iCol, &pVal); } assert( rc==SQLITE_OK ); + (void)rc; /* Suppress warning about unused variable */ if( sqlite3_value_type(pVal)!=eType ) return 0; /* A SessionChange object never has a NULL value in a PK column */ @@ -170181,9 +227233,8 @@ static int sessionPreupdateEqual( }else{ z = sqlite3_value_blob(pVal); } - if( memcmp(a, z, n) ) return 0; + if( n>0 && memcmp(a, z, n) ) return 0; a += n; - break; } } } @@ -170192,7 +227243,7 @@ static int sessionPreupdateEqual( } /* -** If required, grow the hash table used to store changes on table pTab +** If required, grow the hash table used to store changes on table pTab ** (part of the session pSession). If a fatal OOM error occurs, set the ** session object to failed and return SQLITE_ERROR. Otherwise, return ** SQLITE_OK. @@ -170202,13 +227253,19 @@ static int sessionPreupdateEqual( ** Growing the hash table in this case is a performance optimization only, ** it is not required for correct operation. */ -static int sessionGrowHash(int bPatchset, SessionTable *pTab){ +static int sessionGrowHash( + sqlite3_session *pSession, /* For memory accounting. May be NULL */ + int bPatchset, + SessionTable *pTab +){ if( pTab->nChange==0 || pTab->nEntry>=(pTab->nChange/2) ){ int i; SessionChange **apNew; - int nNew = (pTab->nChange ? pTab->nChange : 128) * 2; + sqlite3_int64 nNew = 2*(sqlite3_int64)(pTab->nChange ? pTab->nChange : 128); - apNew = (SessionChange **)sqlite3_malloc(sizeof(SessionChange *) * nNew); + apNew = (SessionChange**)sessionMalloc64( + pSession, sizeof(SessionChange*) * nNew + ); if( apNew==0 ){ if( pTab->nChange==0 ){ return SQLITE_ERROR; @@ -170229,7 +227286,7 @@ static int sessionGrowHash(int bPatchset, SessionTable *pTab){ } } - sqlite3_free(pTab->apChange); + sessionFree(pSession, pTab->apChange); pTab->nChange = nNew; pTab->apChange = apNew; } @@ -170239,9 +227296,7 @@ static int sessionGrowHash(int bPatchset, SessionTable *pTab){ /* ** This function queries the database for the names of the columns of table -** zThis, in schema zDb. It is expected that the table has nCol columns. If -** not, SQLITE_SCHEMA is returned and none of the output variables are -** populated. +** zThis, in schema zDb. ** ** Otherwise, if they are not NULL, variable *pnCol is set to the number ** of columns in the database table and variable *pzTab is set to point to a @@ -170252,67 +227307,106 @@ static int sessionGrowHash(int bPatchset, SessionTable *pTab){ ** ** For example, if the table is declared as: ** -** CREATE TABLE tbl1(w, x, y, z, PRIMARY KEY(w, z)); +** CREATE TABLE tbl1(w, x DEFAULT 'abc', y, z, PRIMARY KEY(w, z)); ** -** Then the four output variables are populated as follows: +** Then the five output variables are populated as follows: ** ** *pnCol = 4 ** *pzTab = "tbl1" ** *pazCol = {"w", "x", "y", "z"} +** *pazDflt = {NULL, 'abc', NULL, NULL} ** *pabPK = {1, 0, 0, 1} ** ** All returned buffers are part of the same single allocation, which must -** be freed using sqlite3_free() by the caller. If pazCol was not NULL, then -** pointer *pazCol should be freed to release all memory. Otherwise, pointer -** *pabPK. It is illegal for both pazCol and pabPK to be NULL. +** be freed using sqlite3_free() by the caller */ static int sessionTableInfo( + sqlite3_session *pSession, /* For memory accounting. May be NULL */ sqlite3 *db, /* Database connection */ const char *zDb, /* Name of attached database (e.g. "main") */ const char *zThis, /* Table name */ int *pnCol, /* OUT: number of columns */ const char **pzTab, /* OUT: Copy of zThis */ const char ***pazCol, /* OUT: Array of column names for table */ - u8 **pabPK /* OUT: Array of booleans - true for PK col */ + const char ***pazDflt, /* OUT: Array of default value expressions */ + u8 **pabPK, /* OUT: Array of booleans - true for PK col */ + int *pbRowid /* OUT: True if only PK is a rowid */ ){ char *zPragma; sqlite3_stmt *pStmt; int rc; - int nByte; + sqlite3_int64 nByte; int nDbCol = 0; int nThis; int i; u8 *pAlloc = 0; char **azCol = 0; + char **azDflt = 0; u8 *abPK = 0; + int bRowid = 0; /* Set to true to use rowid as PK */ assert( pazCol && pabPK ); + *pazCol = 0; + *pabPK = 0; + *pnCol = 0; + if( pzTab ) *pzTab = 0; + if( pazDflt ) *pazDflt = 0; + nThis = sqlite3Strlen30(zThis); - zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); - if( !zPragma ) return SQLITE_NOMEM; + if( nThis==12 && 0==sqlite3_stricmp("sqlite_stat1", zThis) ){ + rc = sqlite3_table_column_metadata(db, zDb, zThis, 0, 0, 0, 0, 0, 0); + if( rc==SQLITE_OK ){ + /* For sqlite_stat1, pretend that (tbl,idx) is the PRIMARY KEY. */ + zPragma = sqlite3_mprintf( + "SELECT 0, 'tbl', '', 0, '', 1 UNION ALL " + "SELECT 1, 'idx', '', 0, '', 2 UNION ALL " + "SELECT 2, 'stat', '', 0, '', 0" + ); + }else if( rc==SQLITE_ERROR ){ + zPragma = sqlite3_mprintf(""); + }else{ + return rc; + } + }else{ + zPragma = sqlite3_mprintf("PRAGMA '%q'.table_info('%q')", zDb, zThis); + } + if( !zPragma ){ + return SQLITE_NOMEM; + } rc = sqlite3_prepare_v2(db, zPragma, -1, &pStmt, 0); sqlite3_free(zPragma); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + return rc; + } nByte = nThis + 1; + bRowid = (pbRowid!=0); while( SQLITE_ROW==sqlite3_step(pStmt) ){ - nByte += sqlite3_column_bytes(pStmt, 1); + nByte += sqlite3_column_bytes(pStmt, 1); /* name */ + nByte += sqlite3_column_bytes(pStmt, 4); /* dflt_value */ nDbCol++; + if( sqlite3_column_int(pStmt, 5) ) bRowid = 0; /* pk */ } + if( nDbCol==0 ) bRowid = 0; + nDbCol += bRowid; + nByte += strlen(SESSIONS_ROWID); rc = sqlite3_reset(pStmt); if( rc==SQLITE_OK ){ - nByte += nDbCol * (sizeof(const char *) + sizeof(u8) + 1); - pAlloc = sqlite3_malloc(nByte); + nByte += nDbCol * (sizeof(const char *)*2 + sizeof(u8) + 1 + 1); + pAlloc = sessionMalloc64(pSession, nByte); if( pAlloc==0 ){ rc = SQLITE_NOMEM; + }else{ + memset(pAlloc, 0, nByte); } } if( rc==SQLITE_OK ){ azCol = (char **)pAlloc; - pAlloc = (u8 *)&azCol[nDbCol]; + azDflt = (char**)&azCol[nDbCol]; + pAlloc = (u8 *)&azDflt[nDbCol]; abPK = (u8 *)pAlloc; pAlloc = &abPK[nDbCol]; if( pzTab ){ @@ -170320,60 +227414,82 @@ static int sessionTableInfo( *pzTab = (char *)pAlloc; pAlloc += nThis+1; } - + i = 0; + if( bRowid ){ + size_t nName = strlen(SESSIONS_ROWID); + memcpy(pAlloc, SESSIONS_ROWID, nName+1); + azCol[i] = (char*)pAlloc; + pAlloc += nName+1; + abPK[i] = 1; + i++; + } while( SQLITE_ROW==sqlite3_step(pStmt) ){ int nName = sqlite3_column_bytes(pStmt, 1); + int nDflt = sqlite3_column_bytes(pStmt, 4); const unsigned char *zName = sqlite3_column_text(pStmt, 1); + const unsigned char *zDflt = sqlite3_column_text(pStmt, 4); + if( zName==0 ) break; memcpy(pAlloc, zName, nName+1); azCol[i] = (char *)pAlloc; pAlloc += nName+1; + if( zDflt ){ + memcpy(pAlloc, zDflt, nDflt+1); + azDflt[i] = (char *)pAlloc; + pAlloc += nDflt+1; + }else{ + azDflt[i] = 0; + } abPK[i] = sqlite3_column_int(pStmt, 5); i++; } rc = sqlite3_reset(pStmt); - } /* If successful, populate the output variables. Otherwise, zero them and ** free any allocation made. An error code will be returned in this case. */ if( rc==SQLITE_OK ){ - *pazCol = (const char **)azCol; + *pazCol = (const char**)azCol; + if( pazDflt ) *pazDflt = (const char**)azDflt; *pabPK = abPK; *pnCol = nDbCol; }else{ - *pazCol = 0; - *pabPK = 0; - *pnCol = 0; - if( pzTab ) *pzTab = 0; - sqlite3_free(azCol); + sessionFree(pSession, azCol); } + if( pbRowid ) *pbRowid = bRowid; sqlite3_finalize(pStmt); return rc; } /* -** This function is only called from within a pre-update handler for a -** write to table pTab, part of session pSession. If this is the first -** write to this table, initalize the SessionTable.nCol, azCol[] and -** abPK[] arrays accordingly. +** This function is called to initialize the SessionTable.nCol, azCol[] +** abPK[] and azDflt[] members of SessionTable object pTab. If these +** fields are already initilialized, this function is a no-op. ** ** If an error occurs, an error code is stored in sqlite3_session.rc and ** non-zero returned. Or, if no error occurs but the table has no primary ** key, sqlite3_session.rc is left set to SQLITE_OK and non-zero returned to -** indicate that updates on this table should be ignored. SessionTable.abPK +** indicate that updates on this table should be ignored. SessionTable.abPK ** is set to NULL in this case. */ -static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ +static int sessionInitTable( + sqlite3_session *pSession, /* Optional session handle */ + SessionTable *pTab, /* Table object to initialize */ + sqlite3 *db, /* Database handle to read schema from */ + const char *zDb /* Name of db - "main", "temp" etc. */ +){ + int rc = SQLITE_OK; + if( pTab->nCol==0 ){ u8 *abPK; assert( pTab->azCol==0 || pTab->abPK==0 ); - pSession->rc = sessionTableInfo(pSession->db, pSession->zDb, - pTab->zName, &pTab->nCol, 0, &pTab->azCol, &abPK + rc = sessionTableInfo(pSession, db, zDb, + pTab->zName, &pTab->nCol, 0, &pTab->azCol, &pTab->azDflt, &abPK, + ((pSession==0 || pSession->bImplicitPK) ? &pTab->bRowid : 0) ); - if( pSession->rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ int i; for(i=0; inCol; i++){ if( abPK[i] ){ @@ -170381,13 +227497,472 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ break; } } + if( 0==sqlite3_stricmp("sqlite_stat1", pTab->zName) ){ + pTab->bStat1 = 1; + } + + if( pSession && pSession->bEnableSize ){ + pSession->nMaxChangesetSize += ( + 1 + sessionVarintLen(pTab->nCol) + pTab->nCol + strlen(pTab->zName)+1 + ); + } } } - return (pSession->rc || pTab->abPK==0); + + if( pSession ){ + pSession->rc = rc; + return (rc || pTab->abPK==0); + } + return rc; } /* -** This function is only called from with a pre-update-hook reporting a +** Re-initialize table object pTab. +*/ +static int sessionReinitTable(sqlite3_session *pSession, SessionTable *pTab){ + int nCol = 0; + const char **azCol = 0; + const char **azDflt = 0; + u8 *abPK = 0; + int bRowid = 0; + + assert( pSession->rc==SQLITE_OK ); + + pSession->rc = sessionTableInfo(pSession, pSession->db, pSession->zDb, + pTab->zName, &nCol, 0, &azCol, &azDflt, &abPK, + (pSession->bImplicitPK ? &bRowid : 0) + ); + if( pSession->rc==SQLITE_OK ){ + if( pTab->nCol>nCol || pTab->bRowid!=bRowid ){ + pSession->rc = SQLITE_SCHEMA; + }else{ + int ii; + int nOldCol = pTab->nCol; + for(ii=0; iinCol ){ + if( pTab->abPK[ii]!=abPK[ii] ){ + pSession->rc = SQLITE_SCHEMA; + } + }else if( abPK[ii] ){ + pSession->rc = SQLITE_SCHEMA; + } + } + + if( pSession->rc==SQLITE_OK ){ + const char **a = pTab->azCol; + pTab->azCol = azCol; + pTab->nCol = nCol; + pTab->azDflt = azDflt; + pTab->abPK = abPK; + azCol = a; + } + if( pSession->bEnableSize ){ + pSession->nMaxChangesetSize += (nCol - nOldCol); + pSession->nMaxChangesetSize += sessionVarintLen(nCol); + pSession->nMaxChangesetSize -= sessionVarintLen(nOldCol); + } + } + } + + sqlite3_free((char*)azCol); + return pSession->rc; +} + +/* +** Session-change object (*pp) contains an old.* record with fewer than +** nCol fields. This function updates it with the default values for +** the missing fields. +*/ +static void sessionUpdateOneChange( + sqlite3_session *pSession, /* For memory accounting */ + int *pRc, /* IN/OUT: Error code */ + SessionChange **pp, /* IN/OUT: Change object to update */ + int nCol, /* Number of columns now in table */ + sqlite3_stmt *pDflt /* SELECT */ +){ + SessionChange *pOld = *pp; + + while( pOld->nRecordFieldnRecordField; + int eType = sqlite3_column_type(pDflt, iField); + switch( eType ){ + case SQLITE_NULL: + nIncr = 1; + break; + case SQLITE_INTEGER: + case SQLITE_FLOAT: + nIncr = 9; + break; + default: { + int n = sqlite3_column_bytes(pDflt, iField); + nIncr = 1 + sessionVarintLen(n) + n; + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + break; + } + } + + nByte = nIncr + (sizeof(SessionChange) + pOld->nRecord); + pNew = sessionMalloc64(pSession, nByte); + if( pNew==0 ){ + *pRc = SQLITE_NOMEM; + return; + }else{ + memcpy(pNew, pOld, sizeof(SessionChange)); + pNew->aRecord = (u8*)&pNew[1]; + memcpy(pNew->aRecord, pOld->aRecord, pOld->nRecord); + pNew->aRecord[pNew->nRecord++] = (u8)eType; + switch( eType ){ + case SQLITE_INTEGER: { + i64 iVal = sqlite3_column_int64(pDflt, iField); + sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + pNew->nRecord += 8; + break; + } + + case SQLITE_FLOAT: { + double rVal = sqlite3_column_double(pDflt, iField); + i64 iVal = 0; + memcpy(&iVal, &rVal, sizeof(rVal)); + sessionPutI64(&pNew->aRecord[pNew->nRecord], iVal); + pNew->nRecord += 8; + break; + } + + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pDflt, iField); + const char *z = (const char*)sqlite3_column_text(pDflt, iField); + pNew->nRecord += sessionVarintPut(&pNew->aRecord[pNew->nRecord], n); + memcpy(&pNew->aRecord[pNew->nRecord], z, n); + pNew->nRecord += n; + break; + } + + case SQLITE_BLOB: { + int n = sqlite3_column_bytes(pDflt, iField); + const u8 *z = (const u8*)sqlite3_column_blob(pDflt, iField); + pNew->nRecord += sessionVarintPut(&pNew->aRecord[pNew->nRecord], n); + memcpy(&pNew->aRecord[pNew->nRecord], z, n); + pNew->nRecord += n; + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + + sessionFree(pSession, pOld); + *pp = pOld = pNew; + pNew->nRecordField++; + pNew->nMaxSize += nIncr; + if( pSession ){ + pSession->nMaxChangesetSize += nIncr; + } + } + } +} + +/* +** Ensure that there is room in the buffer to append nByte bytes of data. +** If not, use sqlite3_realloc() to grow the buffer so that there is. +** +** If successful, return zero. Otherwise, if an OOM condition is encountered, +** set *pRc to SQLITE_NOMEM and return non-zero. +*/ +static int sessionBufferGrow(SessionBuffer *p, i64 nByte, int *pRc){ +#define SESSION_MAX_BUFFER_SZ (0x7FFFFF00 - 1) + i64 nReq = p->nBuf + nByte; + if( *pRc==SQLITE_OK && nReq>p->nAlloc ){ + u8 *aNew; + i64 nNew = p->nAlloc ? p->nAlloc : 128; + + do { + nNew = nNew*2; + }while( nNewSESSION_MAX_BUFFER_SZ ){ + nNew = SESSION_MAX_BUFFER_SZ; + if( nNewaBuf, nNew); + if( 0==aNew ){ + *pRc = SQLITE_NOMEM; + }else{ + p->aBuf = aNew; + p->nAlloc = nNew; + } + } + return (*pRc!=SQLITE_OK); +} + + +/* +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a string to the buffer. All bytes in the string +** up to (but not including) the nul-terminator are written to the buffer. +** +** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before +** returning. +*/ +static void sessionAppendStr( + SessionBuffer *p, + const char *zStr, + int *pRc +){ + int nStr = sqlite3Strlen30(zStr); + if( 0==sessionBufferGrow(p, nStr+1, pRc) ){ + memcpy(&p->aBuf[p->nBuf], zStr, nStr); + p->nBuf += nStr; + p->aBuf[p->nBuf] = 0x00; + } +} + +/* +** Format a string using printf() style formatting and then append it to the +** buffer using sessionAppendString(). +*/ +static void sessionAppendPrintf( + SessionBuffer *p, /* Buffer to append to */ + int *pRc, + const char *zFmt, + ... +){ + if( *pRc==SQLITE_OK ){ + char *zApp = 0; + va_list ap; + va_start(ap, zFmt); + zApp = sqlite3_vmprintf(zFmt, ap); + if( zApp==0 ){ + *pRc = SQLITE_NOMEM; + }else{ + sessionAppendStr(p, zApp, pRc); + } + va_end(ap); + sqlite3_free(zApp); + } +} + +/* +** Prepare a statement against database handle db that SELECTs a single +** row containing the default values for each column in table pTab. For +** example, if pTab is declared as: +** +** CREATE TABLE pTab(a PRIMARY KEY, b DEFAULT 123, c DEFAULT 'abcd'); +** +** Then this function prepares and returns the SQL statement: +** +** SELECT NULL, 123, 'abcd'; +*/ +static int sessionPrepareDfltStmt( + sqlite3 *db, /* Database handle */ + SessionTable *pTab, /* Table to prepare statement for */ + sqlite3_stmt **ppStmt /* OUT: Statement handle */ +){ + SessionBuffer sql = {0,0,0}; + int rc = SQLITE_OK; + const char *zSep = " "; + int ii = 0; + + *ppStmt = 0; + sessionAppendPrintf(&sql, &rc, "SELECT"); + for(ii=0; iinCol; ii++){ + const char *zDflt = pTab->azDflt[ii] ? pTab->azDflt[ii] : "NULL"; + sessionAppendPrintf(&sql, &rc, "%s%s", zSep, zDflt); + zSep = ", "; + } + if( rc==SQLITE_OK ){ + rc = sqlite3_prepare_v2(db, (const char*)sql.aBuf, -1, ppStmt, 0); + } + sqlite3_free(sql.aBuf); + + return rc; +} + +/* +** Table pTab has one or more existing change-records with old.* records +** with fewer than pTab->nCol columns. This function updates all such +** change-records with the default values for the missing columns. +*/ +static int sessionUpdateChanges(sqlite3_session *pSession, SessionTable *pTab){ + sqlite3_stmt *pStmt = 0; + int rc = pSession->rc; + + rc = sessionPrepareDfltStmt(pSession->db, pTab, &pStmt); + if( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pStmt) ){ + int ii = 0; + SessionChange **pp = 0; + for(ii=0; iinChange; ii++){ + for(pp=&pTab->apChange[ii]; *pp; pp=&((*pp)->pNext)){ + if( (*pp)->nRecordField!=pTab->nCol ){ + sessionUpdateOneChange(pSession, &rc, pp, pTab->nCol, pStmt); + } + } + } + } + + pSession->rc = rc; + rc = sqlite3_finalize(pStmt); + if( pSession->rc==SQLITE_OK ) pSession->rc = rc; + return pSession->rc; +} + +/* +** Versions of the four methods in object SessionHook for use with the +** sqlite_stat1 table. The purpose of this is to substitute a zero-length +** blob each time a NULL value is read from the "idx" column of the +** sqlite_stat1 table. +*/ +typedef struct SessionStat1Ctx SessionStat1Ctx; +struct SessionStat1Ctx { + SessionHook hook; + sqlite3_session *pSession; +}; +static int sessionStat1Old(void *pCtx, int iCol, sqlite3_value **ppVal){ + SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx; + sqlite3_value *pVal = 0; + int rc = p->hook.xOld(p->hook.pCtx, iCol, &pVal); + if( rc==SQLITE_OK && iCol==1 && sqlite3_value_type(pVal)==SQLITE_NULL ){ + pVal = p->pSession->pZeroBlob; + } + *ppVal = pVal; + return rc; +} +static int sessionStat1New(void *pCtx, int iCol, sqlite3_value **ppVal){ + SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx; + sqlite3_value *pVal = 0; + int rc = p->hook.xNew(p->hook.pCtx, iCol, &pVal); + if( rc==SQLITE_OK && iCol==1 && sqlite3_value_type(pVal)==SQLITE_NULL ){ + pVal = p->pSession->pZeroBlob; + } + *ppVal = pVal; + return rc; +} +static int sessionStat1Count(void *pCtx){ + SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx; + return p->hook.xCount(p->hook.pCtx); +} +static int sessionStat1Depth(void *pCtx){ + SessionStat1Ctx *p = (SessionStat1Ctx*)pCtx; + return p->hook.xDepth(p->hook.pCtx); +} + +static int sessionUpdateMaxSize( + int op, + sqlite3_session *pSession, /* Session object pTab is attached to */ + SessionTable *pTab, /* Table that change applies to */ + SessionChange *pC /* Update pC->nMaxSize */ +){ + i64 nNew = 2; + if( pC->op==SQLITE_INSERT ){ + if( pTab->bRowid ) nNew += 9; + if( op!=SQLITE_DELETE ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *p = 0; + pSession->hook.xNew(pSession->hook.pCtx, ii, &p); + sessionSerializeValue(0, p, &nNew); + } + } + }else if( op==SQLITE_DELETE ){ + nNew += pC->nRecord; + if( sqlite3_preupdate_blobwrite(pSession->db)>=0 ){ + nNew += pC->nRecord; + } + }else{ + int ii; + u8 *pCsr = pC->aRecord; + if( pTab->bRowid ){ + nNew += 9 + 1; + pCsr += 9; + } + for(ii=pTab->bRowid; iinCol; ii++){ + int bChanged = 1; + int nOld = 0; + int eType; + sqlite3_value *p = 0; + pSession->hook.xNew(pSession->hook.pCtx, ii-pTab->bRowid, &p); + if( p==0 ){ + return SQLITE_NOMEM; + } + + eType = *pCsr++; + switch( eType ){ + case SQLITE_NULL: + bChanged = sqlite3_value_type(p)!=SQLITE_NULL; + break; + + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + if( eType==sqlite3_value_type(p) ){ + sqlite3_int64 iVal = sessionGetI64(pCsr); + if( eType==SQLITE_INTEGER ){ + bChanged = (iVal!=sqlite3_value_int64(p)); + }else{ + double dVal; + memcpy(&dVal, &iVal, 8); + bChanged = (dVal!=sqlite3_value_double(p)); + } + } + nOld = 8; + pCsr += 8; + break; + } + + default: { + int nByte; + nOld = sessionVarintGet(pCsr, &nByte); + pCsr += nOld; + nOld += nByte; + assert( eType==SQLITE_TEXT || eType==SQLITE_BLOB ); + if( eType==sqlite3_value_type(p) + && nByte==sqlite3_value_bytes(p) + && (nByte==0 || 0==memcmp(pCsr, sqlite3_value_blob(p), nByte)) + ){ + bChanged = 0; + } + pCsr += nByte; + break; + } + } + + if( bChanged && pTab->abPK[ii] ){ + nNew = pC->nRecord + 2; + break; + } + + if( bChanged ){ + nNew += 1 + nOld; + sessionSerializeValue(0, p, &nNew); + }else if( pTab->abPK[ii] ){ + nNew += 2 + nOld; + }else{ + nNew += 2; + } + } + } + + if( nNew>pC->nMaxSize ){ + int nIncr = nNew - pC->nMaxSize; + pC->nMaxSize = nNew; + pSession->nMaxChangesetSize += nIncr; + } + return SQLITE_OK; +} + +/* +** This function is only called from with a pre-update-hook reporting a ** change on table pTab (attached to session pSession). The type of change ** (UPDATE, INSERT, DELETE) is specified by the first argument. ** @@ -170396,127 +227971,176 @@ static int sessionInitTable(sqlite3_session *pSession, SessionTable *pTab){ */ static void sessionPreupdateOneChange( int op, /* One of SQLITE_UPDATE, INSERT, DELETE */ + i64 iRowid, sqlite3_session *pSession, /* Session object pTab is attached to */ SessionTable *pTab /* Table that change applies to */ ){ - int iHash; - int bNull = 0; + int iHash; + int bNull = 0; int rc = SQLITE_OK; + int nExpect = 0; + SessionStat1Ctx stat1 = {{0,0,0,0,0},0}; if( pSession->rc ) return; /* Load table details if required */ - if( sessionInitTable(pSession, pTab) ) return; + if( sessionInitTable(pSession, pTab, pSession->db, pSession->zDb) ) return; - /* Check the number of columns in this xPreUpdate call matches the + /* Check the number of columns in this xPreUpdate call matches the ** number of columns in the table. */ - if( pTab->nCol!=pSession->hook.xCount(pSession->hook.pCtx) ){ + nExpect = pSession->hook.xCount(pSession->hook.pCtx); + if( (pTab->nCol-pTab->bRowid)nCol-pTab->bRowid)!=nExpect ){ pSession->rc = SQLITE_SCHEMA; return; } /* Grow the hash table if required */ - if( sessionGrowHash(0, pTab) ){ + if( sessionGrowHash(pSession, 0, pTab) ){ pSession->rc = SQLITE_NOMEM; return; } + if( pTab->bStat1 ){ + stat1.hook = pSession->hook; + stat1.pSession = pSession; + pSession->hook.pCtx = (void*)&stat1; + pSession->hook.xNew = sessionStat1New; + pSession->hook.xOld = sessionStat1Old; + pSession->hook.xCount = sessionStat1Count; + pSession->hook.xDepth = sessionStat1Depth; + if( pSession->pZeroBlob==0 ){ + sqlite3_value *p = sqlite3ValueNew(0); + if( p==0 ){ + rc = SQLITE_NOMEM; + goto error_out; + } + sqlite3ValueSetStr(p, 0, "", 0, SQLITE_STATIC); + pSession->pZeroBlob = p; + } + } + /* Calculate the hash-key for this change. If the primary key of the row ** includes a NULL value, exit early. Such changes are ignored by the ** session module. */ - rc = sessionPreupdateHash(pSession, pTab, op==SQLITE_INSERT, &iHash, &bNull); + rc = sessionPreupdateHash( + pSession, iRowid, pTab, op==SQLITE_INSERT, &iHash, &bNull + ); if( rc!=SQLITE_OK ) goto error_out; if( bNull==0 ){ /* Search the hash table for an existing record for this row. */ SessionChange *pC; for(pC=pTab->apChange[iHash]; pC; pC=pC->pNext){ - if( sessionPreupdateEqual(pSession, pTab, pC, op) ) break; + if( sessionPreupdateEqual(pSession, iRowid, pTab, pC, op) ) break; } if( pC==0 ){ /* Create a new change object containing all the old values (if ** this is an SQLITE_UPDATE or SQLITE_DELETE), or just the PK ** values (if this is an INSERT). */ - SessionChange *pChange; /* New change object */ - int nByte; /* Number of bytes to allocate */ + sqlite3_int64 nByte; /* Number of bytes to allocate */ int i; /* Used to iterate through columns */ - + assert( rc==SQLITE_OK ); pTab->nEntry++; - + /* Figure out how large an allocation is required */ nByte = sizeof(SessionChange); - for(i=0; inCol; i++){ + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ - TESTONLY(int trc = ) pSession->hook.xOld(pSession->hook.pCtx, i, &p); - assert( trc==SQLITE_OK ); + /* This may fail if the column has a non-NULL default and was added + ** using ALTER TABLE ADD COLUMN after this record was created. */ + rc = pSession->hook.xOld(pSession->hook.pCtx, i, &p); }else if( pTab->abPK[i] ){ TESTONLY(int trc = ) pSession->hook.xNew(pSession->hook.pCtx, i, &p); assert( trc==SQLITE_OK ); } - /* This may fail if SQLite value p contains a utf-16 string that must - ** be converted to utf-8 and an OOM error occurs while doing so. */ - rc = sessionSerializeValue(0, p, &nByte); + if( rc==SQLITE_OK ){ + /* This may fail if SQLite value p contains a utf-16 string that must + ** be converted to utf-8 and an OOM error occurs while doing so. */ + rc = sessionSerializeValue(0, p, &nByte); + } if( rc!=SQLITE_OK ) goto error_out; } - + if( pTab->bRowid ){ + nByte += 9; /* Size of rowid field - an integer */ + } + /* Allocate the change object */ - pChange = (SessionChange *)sqlite3_malloc(nByte); - if( !pChange ){ + pC = (SessionChange*)sessionMalloc64(pSession, nByte); + if( !pC ){ rc = SQLITE_NOMEM; goto error_out; }else{ - memset(pChange, 0, sizeof(SessionChange)); - pChange->aRecord = (u8 *)&pChange[1]; + memset(pC, 0, sizeof(SessionChange)); + pC->aRecord = (u8 *)&pC[1]; } - + /* Populate the change object. None of the preupdate_old(), ** preupdate_new() or SerializeValue() calls below may fail as all ** required values and encodings have already been cached in memory. ** It is not possible for an OOM to occur in this block. */ nByte = 0; - for(i=0; inCol; i++){ + if( pTab->bRowid ){ + pC->aRecord[0] = SQLITE_INTEGER; + sessionPutI64(&pC->aRecord[1], iRowid); + nByte = 9; + } + for(i=0; i<(pTab->nCol-pTab->bRowid); i++){ sqlite3_value *p = 0; if( op!=SQLITE_INSERT ){ pSession->hook.xOld(pSession->hook.pCtx, i, &p); }else if( pTab->abPK[i] ){ pSession->hook.xNew(pSession->hook.pCtx, i, &p); } - sessionSerializeValue(&pChange->aRecord[nByte], p, &nByte); + sessionSerializeValue(&pC->aRecord[nByte], p, &nByte); } /* Add the change to the hash-table */ if( pSession->bIndirect || pSession->hook.xDepth(pSession->hook.pCtx) ){ - pChange->bIndirect = 1; + pC->bIndirect = 1; } - pChange->nRecord = nByte; - pChange->op = op; - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; + pC->nRecordField = pTab->nCol; + pC->nRecord = nByte; + pC->op = op; + pC->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pC; }else if( pC->bIndirect ){ /* If the existing change is considered "indirect", but this current ** change is "direct", mark the change object as direct. */ - if( pSession->hook.xDepth(pSession->hook.pCtx)==0 - && pSession->bIndirect==0 + if( pSession->hook.xDepth(pSession->hook.pCtx)==0 + && pSession->bIndirect==0 ){ pC->bIndirect = 0; } } + + assert( rc==SQLITE_OK ); + if( pSession->bEnableSize ){ + rc = sessionUpdateMaxSize(op, pSession, pTab, pC); + } } + /* If an error has occurred, mark the session object as failed. */ error_out: + if( pTab->bStat1 ){ + pSession->hook = stat1.hook; + } if( rc!=SQLITE_OK ){ pSession->rc = rc; } } static int sessionFindTable( - sqlite3_session *pSession, + sqlite3_session *pSession, const char *zName, SessionTable **ppTab ){ @@ -170533,11 +228157,15 @@ static int sessionFindTable( /* If there is a table-filter configured, invoke it. If it returns 0, ** do not automatically add the new table. */ if( pSession->xTableFilter==0 - || pSession->xTableFilter(pSession->pFilterCtx, zName) + || pSession->xTableFilter(pSession->pFilterCtx, zName) ){ rc = sqlite3session_attach(pSession, zName); if( rc==SQLITE_OK ){ - for(pRet=pSession->pTable; pRet->pNext; pRet=pRet->pNext); + pRet = pSession->pTable; + while( ALWAYS(pRet) && pRet->pNext ){ + pRet = pRet->pNext; + } + assert( pRet!=0 ); assert( 0==sqlite3_strnicmp(pRet->zName, zName, nName+1) ); } } @@ -170564,12 +228192,14 @@ static void xPreUpdate( int nDb = sqlite3Strlen30(zDb); assert( sqlite3_mutex_held(db->mutex) ); + (void)iKey1; + (void)iKey2; for(pSession=(sqlite3_session *)pCtx; pSession; pSession=pSession->pNext){ SessionTable *pTab; - /* If this session is attached to a different database ("main", "temp" - ** etc.), or if it is not currently enabled, there is nothing to do. Skip + /* If this session is attached to a different database ("main", "temp" + ** etc.), or if it is not currently enabled, there is nothing to do. Skip ** to the next session object attached to this database. */ if( pSession->bEnable==0 ) continue; if( pSession->rc ) continue; @@ -170578,9 +228208,10 @@ static void xPreUpdate( pSession->rc = sessionFindTable(pSession, zName, &pTab); if( pTab ){ assert( pSession->rc==SQLITE_OK ); - sessionPreupdateOneChange(op, pSession, pTab); + assert( op==SQLITE_UPDATE || iKey1==iKey2 ); + sessionPreupdateOneChange(op, iKey1, pSession, pTab); if( op==SQLITE_UPDATE ){ - sessionPreupdateOneChange(SQLITE_INSERT, pSession, pTab); + sessionPreupdateOneChange(SQLITE_INSERT, iKey2, pSession, pTab); } } } @@ -170619,6 +228250,7 @@ static void sessionPreupdateHooks( typedef struct SessionDiffCtx SessionDiffCtx; struct SessionDiffCtx { sqlite3_stmt *pStmt; + int bRowid; int nOldOff; }; @@ -170627,19 +228259,20 @@ struct SessionDiffCtx { */ static int sessionDiffOld(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->nOldOff+p->bRowid); return SQLITE_OK; } static int sessionDiffNew(void *pCtx, int iVal, sqlite3_value **ppVal){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - *ppVal = sqlite3_column_value(p->pStmt, iVal); + *ppVal = sqlite3_column_value(p->pStmt, iVal+p->bRowid); return SQLITE_OK; } static int sessionDiffCount(void *pCtx){ SessionDiffCtx *p = (SessionDiffCtx*)pCtx; - return p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt); + return (p->nOldOff ? p->nOldOff : sqlite3_column_count(p->pStmt)) - p->bRowid; } static int sessionDiffDepth(void *pCtx){ + (void)pCtx; return 0; } @@ -170660,7 +228293,7 @@ static void sessionDiffHooks( static char *sessionExprComparePK( int nCol, - const char *zDb1, const char *zDb2, + const char *zDb1, const char *zDb2, const char *zTab, const char **azCol, u8 *abPK ){ @@ -170683,7 +228316,7 @@ static char *sessionExprComparePK( static char *sessionExprCompareOther( int nCol, - const char *zDb1, const char *zDb2, + const char *zDb1, const char *zDb2, const char *zTab, const char **azCol, u8 *abPK ){ @@ -170713,17 +228346,18 @@ static char *sessionExprCompareOther( } static char *sessionSelectFindNew( - int nCol, const char *zDb1, /* Pick rows in this db only */ const char *zDb2, /* But not in this one */ + int bRowid, const char *zTbl, /* Table name */ const char *zExpr ){ + const char *zSel = (bRowid ? SESSIONS_ROWID ", *" : "*"); char *zRet = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\" WHERE NOT EXISTS (" + "SELECT %s FROM \"%w\".\"%w\" WHERE NOT EXISTS (" " SELECT 1 FROM \"%w\".\"%w\" WHERE %s" ")", - zDb1, zTbl, zDb2, zTbl, zExpr + zSel, zDb1, zTbl, zDb2, zTbl, zExpr ); return zRet; } @@ -170737,7 +228371,9 @@ static int sessionDiffFindNew( char *zExpr ){ int rc = SQLITE_OK; - char *zStmt = sessionSelectFindNew(pTab->nCol, zDb1, zDb2, pTab->zName,zExpr); + char *zStmt = sessionSelectFindNew( + zDb1, zDb2, pTab->bRowid, pTab->zName, zExpr + ); if( zStmt==0 ){ rc = SQLITE_NOMEM; @@ -170748,8 +228384,10 @@ static int sessionDiffFindNew( SessionDiffCtx *pDiffCtx = (SessionDiffCtx*)pSession->hook.pCtx; pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = 0; + pDiffCtx->bRowid = pTab->bRowid; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(op, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(op, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } @@ -170759,10 +228397,31 @@ static int sessionDiffFindNew( return rc; } +/* +** Return a comma-separated list of the fully-qualified (with both database +** and table name) column names from table pTab. e.g. +** +** "main"."t1"."a", "main"."t1"."b", "main"."t1"."c" +*/ +static char *sessionAllCols( + const char *zDb, + SessionTable *pTab +){ + int ii; + char *zRet = 0; + for(ii=0; iinCol; ii++){ + zRet = sqlite3_mprintf("%z%s\"%w\".\"%w\".\"%w\"", + zRet, (zRet ? ", " : ""), zDb, pTab->zName, pTab->azCol[ii] + ); + if( !zRet ) break; + } + return zRet; +} + static int sessionDiffFindModified( - sqlite3_session *pSession, - SessionTable *pTab, - const char *zFrom, + sqlite3_session *pSession, + SessionTable *pTab, + const char *zFrom, const char *zExpr ){ int rc = SQLITE_OK; @@ -170773,11 +228432,13 @@ static int sessionDiffFindModified( if( zExpr2==0 ){ rc = SQLITE_NOMEM; }else{ + char *z1 = sessionAllCols(pSession->zDb, pTab); + char *z2 = sessionAllCols(zFrom, pTab); char *zStmt = sqlite3_mprintf( - "SELECT * FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", - pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 + "SELECT %s,%s FROM \"%w\".\"%w\", \"%w\".\"%w\" WHERE %s AND (%z)", + z1, z2, pSession->zDb, pTab->zName, zFrom, pTab->zName, zExpr, zExpr2 ); - if( zStmt==0 ){ + if( zStmt==0 || z1==0 || z2==0 ){ rc = SQLITE_NOMEM; }else{ sqlite3_stmt *pStmt; @@ -170788,18 +228449,21 @@ static int sessionDiffFindModified( pDiffCtx->pStmt = pStmt; pDiffCtx->nOldOff = pTab->nCol; while( SQLITE_ROW==sqlite3_step(pStmt) ){ - sessionPreupdateOneChange(SQLITE_UPDATE, pSession, pTab); + i64 iRowid = (pTab->bRowid ? sqlite3_column_int64(pStmt, 0) : 0); + sessionPreupdateOneChange(SQLITE_UPDATE, iRowid, pSession, pTab); } rc = sqlite3_finalize(pStmt); } - sqlite3_free(zStmt); } + sqlite3_free(zStmt); + sqlite3_free(z1); + sqlite3_free(z2); } return rc; } -SQLITE_API int SQLITE_STDCALL sqlite3session_diff( +SQLITE_API int sqlite3session_diff( sqlite3_session *pSession, const char *zFrom, const char *zTbl, @@ -170822,7 +228486,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_diff( /* Locate and if necessary initialize the target table object */ rc = sessionFindTable(pSession, zTbl, &pTo); if( pTo==0 ) goto diff_out; - if( sessionInitTable(pSession, pTo) ){ + if( sessionInitTable(pSession, pTo, pSession->db, pSession->zDb) ){ rc = pSession->rc; goto diff_out; } @@ -170832,9 +228496,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_diff( int bHasPk = 0; int bMismatch = 0; int nCol; /* Columns in zFrom.zTbl */ + int bRowid = 0; u8 *abPK; const char **azCol = 0; - rc = sessionTableInfo(db, zFrom, zTbl, &nCol, 0, &azCol, &abPK); + rc = sessionTableInfo(0, db, zFrom, zTbl, &nCol, 0, &azCol, 0, &abPK, + pSession->bImplicitPK ? &bRowid : 0 + ); if( rc==SQLITE_OK ){ if( pTo->nCol!=nCol ){ bMismatch = 1; @@ -170846,11 +228513,12 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_diff( if( abPK[i] ) bHasPk = 1; } } - } sqlite3_free((char*)azCol); if( bMismatch ){ - *pzErrMsg = sqlite3_mprintf("table schemas do not match"); + if( pzErrMsg ){ + *pzErrMsg = sqlite3_mprintf("table schemas do not match"); + } rc = SQLITE_SCHEMA; } if( bHasPk==0 ){ @@ -170860,7 +228528,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_diff( } if( rc==SQLITE_OK ){ - zExpr = sessionExprComparePK(pTo->nCol, + zExpr = sessionExprComparePK(pTo->nCol, zDb, zFrom, pTo->zName, pTo->azCol, pTo->abPK ); } @@ -170893,7 +228561,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_diff( ** Create a session object. This session object will record changes to ** database zDb attached to connection db. */ -SQLITE_API int SQLITE_STDCALL sqlite3session_create( +SQLITE_API int sqlite3session_create( sqlite3 *db, /* Database handle */ const char *zDb, /* Name of db (e.g. "main") */ sqlite3_session **ppSession /* OUT: New session object */ @@ -170906,7 +228574,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_create( *ppSession = 0; /* Allocate and populate the new session object. */ - pNew = (sqlite3_session *)sqlite3_malloc(sizeof(sqlite3_session) + nDb + 1); + pNew = (sqlite3_session *)sqlite3_malloc64(sizeof(sqlite3_session) + nDb + 1); if( !pNew ) return SQLITE_NOMEM; memset(pNew, 0, sizeof(sqlite3_session)); pNew->db = db; @@ -170915,7 +228583,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_create( memcpy(pNew->zDb, zDb, nDb+1); sessionPreupdateHooks(pNew); - /* Add the new session object to the linked list of session objects + /* Add the new session object to the linked list of session objects ** attached to database handle $db. Do this under the cover of the db ** handle mutex. */ sqlite3_mutex_enter(sqlite3_db_mutex(db)); @@ -170931,7 +228599,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_create( ** Free the list of table objects passed as the first argument. The contents ** of the changed-rows hash tables are also deleted. */ -static void sessionDeleteTable(SessionTable *pList){ +static void sessionDeleteTable(sqlite3_session *pSession, SessionTable *pList){ SessionTable *pNext; SessionTable *pTab; @@ -170943,19 +228611,20 @@ static void sessionDeleteTable(SessionTable *pList){ SessionChange *pNextChange; for(p=pTab->apChange[i]; p; p=pNextChange){ pNextChange = p->pNext; - sqlite3_free(p); + sessionFree(pSession, p); } } - sqlite3_free((char*)pTab->azCol); /* cast works around VC++ bug */ - sqlite3_free(pTab->apChange); - sqlite3_free(pTab); + sqlite3_finalize(pTab->pDfltStmt); + sessionFree(pSession, (char*)pTab->azCol); /* cast works around VC++ bug */ + sessionFree(pSession, pTab->apChange); + sessionFree(pSession, pTab); } } /* ** Delete a session object previously allocated using sqlite3session_create(). */ -SQLITE_API void SQLITE_STDCALL sqlite3session_delete(sqlite3_session *pSession){ +SQLITE_API void sqlite3session_delete(sqlite3_session *pSession){ sqlite3 *db = pSession->db; sqlite3_session *pHead; sqlite3_session **pp; @@ -170972,20 +228641,21 @@ SQLITE_API void SQLITE_STDCALL sqlite3session_delete(sqlite3_session *pSession){ } } sqlite3_mutex_leave(sqlite3_db_mutex(db)); + sqlite3ValueFree(pSession->pZeroBlob); - /* Delete all attached table objects. And the contents of their + /* Delete all attached table objects. And the contents of their ** associated hash-tables. */ - sessionDeleteTable(pSession->pTable); + sessionDeleteTable(pSession, pSession->pTable); - /* Free the session object itself. */ + /* Free the session object. */ sqlite3_free(pSession); } /* ** Set a table filter on a Session Object. */ -SQLITE_API void SQLITE_STDCALL sqlite3session_table_filter( - sqlite3_session *pSession, +SQLITE_API void sqlite3session_table_filter( + sqlite3_session *pSession, int(*xFilter)(void*, const char*), void *pCtx /* First argument passed to xFilter */ ){ @@ -171002,7 +228672,7 @@ SQLITE_API void SQLITE_STDCALL sqlite3session_table_filter( ** not matter if the PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) ** or not. */ -SQLITE_API int SQLITE_STDCALL sqlite3session_attach( +SQLITE_API int sqlite3session_attach( sqlite3_session *pSession, /* Session object */ const char *zName /* Table name */ ){ @@ -171024,12 +228694,13 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_attach( if( !pTab ){ /* Allocate new SessionTable object. */ - pTab = (SessionTable *)sqlite3_malloc(sizeof(SessionTable) + nName + 1); + int nByte = sizeof(SessionTable) + nName + 1; + pTab = (SessionTable*)sessionMalloc64(pSession, nByte); if( !pTab ){ rc = SQLITE_NOMEM; }else{ /* Populate the new SessionTable object and link it into the list. - ** The new object must be linked onto the end of the list, not + ** The new object must be linked onto the end of the list, not ** simply added to the start of it in order to ensure that tables ** appear in the correct order when a changeset or patchset is ** eventually generated. */ @@ -171047,32 +228718,6 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_attach( return rc; } -/* -** Ensure that there is room in the buffer to append nByte bytes of data. -** If not, use sqlite3_realloc() to grow the buffer so that there is. -** -** If successful, return zero. Otherwise, if an OOM condition is encountered, -** set *pRc to SQLITE_NOMEM and return non-zero. -*/ -static int sessionBufferGrow(SessionBuffer *p, int nByte, int *pRc){ - if( *pRc==SQLITE_OK && p->nAlloc-p->nBufnAlloc ? p->nAlloc : 128; - do { - nNew = nNew*2; - }while( nNew<(p->nBuf+nByte) ); - - aNew = (u8 *)sqlite3_realloc(p->aBuf, nNew); - if( 0==aNew ){ - *pRc = SQLITE_NOMEM; - }else{ - p->aBuf = aNew; - p->nAlloc = nNew; - } - } - return (*pRc!=SQLITE_OK); -} - /* ** Append the value passed as the second argument to the buffer passed ** as the first. @@ -171084,7 +228729,7 @@ static int sessionBufferGrow(SessionBuffer *p, int nByte, int *pRc){ static void sessionAppendValue(SessionBuffer *p, sqlite3_value *pVal, int *pRc){ int rc = *pRc; if( rc==SQLITE_OK ){ - int nByte = 0; + sqlite3_int64 nByte = 0; rc = sessionSerializeValue(0, pVal, &nByte); sessionBufferGrow(p, nByte, &rc); if( rc==SQLITE_OK ){ @@ -171097,8 +228742,8 @@ static void sessionAppendValue(SessionBuffer *p, sqlite3_value *pVal, int *pRc){ } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is -** called. Otherwise, append a single byte to the buffer. +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a single byte to the buffer. ** ** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before ** returning. @@ -171110,8 +228755,8 @@ static void sessionAppendByte(SessionBuffer *p, u8 v, int *pRc){ } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is -** called. Otherwise, append a single varint to the buffer. +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a single varint to the buffer. ** ** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before ** returning. @@ -171123,46 +228768,26 @@ static void sessionAppendVarint(SessionBuffer *p, int v, int *pRc){ } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is -** called. Otherwise, append a blob of data to the buffer. +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. Otherwise, append a blob of data to the buffer. ** ** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before ** returning. */ static void sessionAppendBlob( - SessionBuffer *p, - const u8 *aBlob, - int nBlob, + SessionBuffer *p, + const u8 *aBlob, + int nBlob, int *pRc ){ - if( 0==sessionBufferGrow(p, nBlob, pRc) ){ + if( nBlob>0 && 0==sessionBufferGrow(p, nBlob, pRc) ){ memcpy(&p->aBuf[p->nBuf], aBlob, nBlob); p->nBuf += nBlob; } } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is -** called. Otherwise, append a string to the buffer. All bytes in the string -** up to (but not including) the nul-terminator are written to the buffer. -** -** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before -** returning. -*/ -static void sessionAppendStr( - SessionBuffer *p, - const char *zStr, - int *pRc -){ - int nStr = sqlite3Strlen30(zStr); - if( 0==sessionBufferGrow(p, nStr, pRc) ){ - memcpy(&p->aBuf[p->nBuf], zStr, nStr); - p->nBuf += nStr; - } -} - -/* -** This function is a no-op if *pRc is other than SQLITE_OK when it is +** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string representation of integer iVal ** to the buffer. No nul-terminator is written. ** @@ -171180,9 +228805,9 @@ static void sessionAppendInteger( } /* -** This function is a no-op if *pRc is other than SQLITE_OK when it is +** This function is a no-op if *pRc is other than SQLITE_OK when it is ** called. Otherwise, append the string zStr enclosed in quotes (") and -** with any embedded quote characters escaped to the buffer. No +** with any embedded quote characters escaped to the buffer. No ** nul-terminator byte is written. ** ** If an OOM condition is encountered, set *pRc to SQLITE_NOMEM before @@ -171193,7 +228818,7 @@ static void sessionAppendIdent( const char *zStr, /* String to quote, escape and append */ int *pRc /* IN/OUT: Error code */ ){ - int nStr = sqlite3Strlen30(zStr)*2 + 2 + 1; + int nStr = sqlite3Strlen30(zStr)*2 + 2 + 2; if( 0==sessionBufferGrow(p, nStr, pRc) ){ char *zOut = (char *)&p->aBuf[p->nBuf]; const char *zIn = zStr; @@ -171204,6 +228829,7 @@ static void sessionAppendIdent( } *zOut++ = '"'; p->nBuf = (int)((u8 *)zOut - p->aBuf); + p->aBuf[p->nBuf] = 0x00; } } @@ -171255,8 +228881,8 @@ static void sessionAppendCol( /* ** -** This function appends an update change to the buffer (see the comments -** under "CHANGESET FORMAT" at the top of the file). An update change +** This function appends an update change to the buffer (see the comments +** under "CHANGESET FORMAT" at the top of the file). An update change ** consists of: ** ** 1 byte: SQLITE_UPDATE (0x17) @@ -171271,10 +228897,10 @@ static void sessionAppendCol( ** If all of the old.* values are equal to their corresponding new.* value ** (i.e. nothing has changed), then no data at all is appended to the buffer. ** -** Otherwise, the old.* record contains all primary key values and the -** original values of any fields that have been modified. The new.* record +** Otherwise, the old.* record contains all primary key values and the +** original values of any fields that have been modified. The new.* record ** contains the new values of only those fields that have been modified. -*/ +*/ static int sessionAppendUpdate( SessionBuffer *pBuf, /* Buffer to append to */ int bPatchset, /* True for "patchset", 0 for "changeset" */ @@ -171289,6 +228915,7 @@ static int sessionAppendUpdate( int i; /* Used to iterate through columns */ u8 *pCsr = p->aRecord; /* Used to iterate through old.* values */ + assert( abPK!=0 ); sessionAppendByte(pBuf, SQLITE_UPDATE, &rc); sessionAppendByte(pBuf, p->bIndirect, &rc); for(i=0; i FROM zDb.zTab WHERE (pk1, pk2,...) IS (?1, ?2,...) +** +** where is: +** +** 1 AND (?A OR ?1 IS ) AND ... +** +** for each non-pk . */ static int sessionSelectStmt( sqlite3 *db, /* Database handle */ + int bIgnoreNoop, const char *zDb, /* Database name */ const char *zTab, /* Table name */ + int bRowid, int nCol, /* Number of columns in table */ const char **azCol, /* Names of table columns */ u8 *abPK, /* PRIMARY KEY array */ sqlite3_stmt **ppStmt /* OUT: Prepared SELECT statement */ ){ int rc = SQLITE_OK; - int i; + char *zSql = 0; const char *zSep = ""; - SessionBuffer buf = {0, 0, 0}; + const char *zCols = bRowid ? SESSIONS_ROWID ", *" : "*"; + int nSql = -1; + int i; - sessionAppendStr(&buf, "SELECT * FROM ", &rc); - sessionAppendIdent(&buf, zDb, &rc); - sessionAppendStr(&buf, ".", &rc); - sessionAppendIdent(&buf, zTab, &rc); - sessionAppendStr(&buf, " WHERE ", &rc); - for(i=0; ipTable; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ if( pTab->nEntry ){ const char *zName = pTab->zName; - int nCol; /* Number of columns in table */ - u8 *abPK; /* Primary key array */ - const char **azCol = 0; /* Table columns */ int i; /* Used to iterate through hash buckets */ sqlite3_stmt *pSel = 0; /* SELECT statement to query table pTab */ int nRewind = buf.nBuf; /* Initial size of write buffer */ int nNoop; /* Size of buffer after writing tbl header */ + int nOldCol = pTab->nCol; /* Check the table schema is still Ok. */ - rc = sessionTableInfo(db, pSession->zDb, zName, &nCol, 0, &azCol, &abPK); - if( !rc && (pTab->nCol!=nCol || memcmp(abPK, pTab->abPK, nCol)) ){ - rc = SQLITE_SCHEMA; + rc = sessionReinitTable(pSession, pTab); + if( rc==SQLITE_OK && pTab->nCol!=nOldCol ){ + rc = sessionUpdateChanges(pSession, pTab); } /* Write a table header */ @@ -171617,8 +229311,9 @@ static int sessionGenerateChangeset( /* Build and compile a statement to execute: */ if( rc==SQLITE_OK ){ - rc = sessionSelectStmt( - db, pSession->zDb, zName, nCol, azCol, abPK, &pSel); + rc = sessionSelectStmt(db, 0, pSession->zDb, + zName, pTab->bRowid, pTab->nCol, pTab->azCol, pTab->abPK, &pSel + ); } nNoop = buf.nBuf; @@ -171626,32 +229321,33 @@ static int sessionGenerateChangeset( SessionChange *p; /* Used to iterate through changes */ for(p=pTab->apChange[i]; rc==SQLITE_OK && p; p=p->pNext){ - rc = sessionSelectBind(pSel, nCol, abPK, p); + rc = sessionSelectBind(pSel, pTab->nCol, pTab->abPK, p); if( rc!=SQLITE_OK ) continue; if( sqlite3_step(pSel)==SQLITE_ROW ){ if( p->op==SQLITE_INSERT ){ int iCol; sessionAppendByte(&buf, SQLITE_INSERT, &rc); sessionAppendByte(&buf, p->bIndirect, &rc); - for(iCol=0; iColnCol; iCol++){ sessionAppendCol(&buf, pSel, iCol, &rc); } }else{ - rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, abPK); + assert( pTab->abPK!=0 ); + rc = sessionAppendUpdate(&buf, bPatchset, pSel, p, pTab->abPK); } }else if( p->op!=SQLITE_INSERT ){ - rc = sessionAppendDelete(&buf, bPatchset, p, nCol, abPK); + rc = sessionAppendDelete(&buf, bPatchset, p, pTab->nCol,pTab->abPK); } if( rc==SQLITE_OK ){ rc = sqlite3_reset(pSel); } - /* If the buffer is now larger than SESSIONS_STRM_CHUNK_SIZE, pass + /* If the buffer is now larger than sessions_strm_chunk_size, pass ** its contents to the xOutput() callback. */ - if( xOutput - && rc==SQLITE_OK - && buf.nBuf>nNoop - && buf.nBuf>SESSIONS_STRM_CHUNK_SIZE + if( xOutput + && rc==SQLITE_OK + && buf.nBuf>nNoop + && buf.nBuf>sessions_strm_chunk_size ){ rc = xOutput(pOut, (void*)buf.aBuf, buf.nBuf); nNoop = -1; @@ -171665,7 +229361,6 @@ static int sessionGenerateChangeset( if( buf.nBuf==nNoop ){ buf.nBuf = nRewind; } - sqlite3_free((char*)azCol); /* cast works around VC++ bug */ } } @@ -171686,61 +229381,71 @@ static int sessionGenerateChangeset( } /* -** Obtain a changeset object containing all changes recorded by the +** Obtain a changeset object containing all changes recorded by the ** session object passed as the first argument. ** -** It is the responsibility of the caller to eventually free the buffer +** It is the responsibility of the caller to eventually free the buffer ** using sqlite3_free(). */ -SQLITE_API int SQLITE_STDCALL sqlite3session_changeset( +SQLITE_API int sqlite3session_changeset( sqlite3_session *pSession, /* Session object */ int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ void **ppChangeset /* OUT: Buffer containing changeset */ ){ - return sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); + int rc; + + if( pnChangeset==0 || ppChangeset==0 ) return SQLITE_MISUSE; + rc = sessionGenerateChangeset(pSession, 0, 0, 0, pnChangeset, ppChangeset); + assert( rc || pnChangeset==0 + || pSession->bEnableSize==0 || *pnChangeset<=pSession->nMaxChangesetSize + ); + return rc; } /* ** Streaming version of sqlite3session_changeset(). */ -SQLITE_API int SQLITE_STDCALL sqlite3session_changeset_strm( +SQLITE_API int sqlite3session_changeset_strm( sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ + if( xOutput==0 ) return SQLITE_MISUSE; return sessionGenerateChangeset(pSession, 0, xOutput, pOut, 0, 0); } /* ** Streaming version of sqlite3session_patchset(). */ -SQLITE_API int SQLITE_STDCALL sqlite3session_patchset_strm( +SQLITE_API int sqlite3session_patchset_strm( sqlite3_session *pSession, int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ + if( xOutput==0 ) return SQLITE_MISUSE; return sessionGenerateChangeset(pSession, 1, xOutput, pOut, 0, 0); } /* -** Obtain a patchset object containing all changes recorded by the +** Obtain a patchset object containing all changes recorded by the ** session object passed as the first argument. ** -** It is the responsibility of the caller to eventually free the buffer +** It is the responsibility of the caller to eventually free the buffer ** using sqlite3_free(). */ -SQLITE_API int SQLITE_STDCALL sqlite3session_patchset( +SQLITE_API int sqlite3session_patchset( sqlite3_session *pSession, /* Session object */ int *pnPatchset, /* OUT: Size of buffer at *ppChangeset */ void **ppPatchset /* OUT: Buffer containing changeset */ ){ + if( pnPatchset==0 || ppPatchset==0 ) return SQLITE_MISUSE; return sessionGenerateChangeset(pSession, 1, 0, 0, pnPatchset, ppPatchset); } /* ** Enable or disable the session object passed as the first argument. */ -SQLITE_API int SQLITE_STDCALL sqlite3session_enable(sqlite3_session *pSession, int bEnable){ +SQLITE_API int sqlite3session_enable(sqlite3_session *pSession, int bEnable){ int ret; sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); if( bEnable>=0 ){ @@ -171754,7 +229459,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_enable(sqlite3_session *pSession, i /* ** Enable or disable the session object passed as the first argument. */ -SQLITE_API int SQLITE_STDCALL sqlite3session_indirect(sqlite3_session *pSession, int bIndirect){ +SQLITE_API int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect){ int ret; sqlite3_mutex_enter(sqlite3_db_mutex(pSession->db)); if( bIndirect>=0 ){ @@ -171769,7 +229474,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_indirect(sqlite3_session *pSession, ** Return true if there have been no changes to monitored tables recorded ** by the session object passed as the only argument. */ -SQLITE_API int SQLITE_STDCALL sqlite3session_isempty(sqlite3_session *pSession){ +SQLITE_API int sqlite3session_isempty(sqlite3_session *pSession){ int ret = 0; SessionTable *pTab; @@ -171782,6 +229487,59 @@ SQLITE_API int SQLITE_STDCALL sqlite3session_isempty(sqlite3_session *pSession){ return (ret==0); } +/* +** Return the amount of heap memory in use. +*/ +SQLITE_API sqlite3_int64 sqlite3session_memory_used(sqlite3_session *pSession){ + return pSession->nMalloc; +} + +/* +** Configure the session object passed as the first argument. +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session *pSession, int op, void *pArg){ + int rc = SQLITE_OK; + switch( op ){ + case SQLITE_SESSION_OBJCONFIG_SIZE: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bEnableSize = (iArg!=0); + } + } + *(int*)pArg = pSession->bEnableSize; + break; + } + + case SQLITE_SESSION_OBJCONFIG_ROWID: { + int iArg = *(int*)pArg; + if( iArg>=0 ){ + if( pSession->pTable ){ + rc = SQLITE_MISUSE; + }else{ + pSession->bImplicitPK = (iArg!=0); + } + } + *(int*)pArg = pSession->bImplicitPK; + break; + } + + default: + rc = SQLITE_MISUSE; + } + + return rc; +} + +/* +** Return the maximum size of sqlite3session_changeset() output. +*/ +SQLITE_API sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession){ + return pSession->nMaxChangesetSize; +} + /* ** Do the work for either sqlite3changeset_start() or start_strm(). */ @@ -171790,7 +229548,9 @@ static int sessionChangesetStart( int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn, int nChangeset, /* Size of buffer pChangeset in bytes */ - void *pChangeset /* Pointer to buffer containing changeset */ + void *pChangeset, /* Pointer to buffer containing changeset */ + int bInvert, /* True to invert changeset */ + int bSkipEmpty /* True to skip empty UPDATE changes */ ){ sqlite3_changeset_iter *pRet; /* Iterator to return */ int nByte; /* Number of bytes to allocate for iterator */ @@ -171810,6 +229570,8 @@ static int sessionChangesetStart( pRet->in.xInput = xInput; pRet->in.pIn = pIn; pRet->in.bEof = (xInput ? 0 : 1); + pRet->bInvert = bInvert; + pRet->bSkipEmpty = bSkipEmpty; /* Populate the output variable and return success. */ *pp = pRet; @@ -171819,23 +229581,41 @@ static int sessionChangesetStart( /* ** Create an iterator used to iterate through the contents of a changeset. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_start( +SQLITE_API int sqlite3changeset_start( sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ int nChangeset, /* Size of buffer pChangeset in bytes */ void *pChangeset /* Pointer to buffer containing changeset */ ){ - return sessionChangesetStart(pp, 0, 0, nChangeset, pChangeset); + return sessionChangesetStart(pp, 0, 0, nChangeset, pChangeset, 0, 0); +} +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int nChangeset, /* Size of buffer pChangeset in bytes */ + void *pChangeset, /* Pointer to buffer containing changeset */ + int flags +){ + int bInvert = !!(flags & SQLITE_CHANGESETSTART_INVERT); + return sessionChangesetStart(pp, 0, 0, nChangeset, pChangeset, bInvert, 0); } /* ** Streaming version of sqlite3changeset_start(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_start_strm( +SQLITE_API int sqlite3changeset_start_strm( sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn ){ - return sessionChangesetStart(pp, xInput, pIn, 0, 0); + return sessionChangesetStart(pp, xInput, pIn, 0, 0, 0, 0); +} +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, /* OUT: Changeset iterator handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +){ + int bInvert = !!(flags & SQLITE_CHANGESETSTART_INVERT); + return sessionChangesetStart(pp, xInput, pIn, 0, 0, bInvert, 0); } /* @@ -171843,7 +229623,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_start_strm( ** object and the buffer is full, discard some data to free up space. */ static void sessionDiscardData(SessionInput *pIn){ - if( pIn->bEof && pIn->xInput && pIn->iNext>=SESSIONS_STRM_CHUNK_SIZE ){ + if( pIn->xInput && pIn->iNext>=sessions_strm_chunk_size ){ int nMove = pIn->buf.nBuf - pIn->iNext; assert( nMove>=0 ); if( nMove>0 ){ @@ -171866,7 +229646,7 @@ static int sessionInputBuffer(SessionInput *pIn, int nByte){ int rc = SQLITE_OK; if( pIn->xInput ){ while( !pIn->bEof && (pIn->iNext+nByte)>=pIn->nData && rc==SQLITE_OK ){ - int nNew = SESSIONS_STRM_CHUNK_SIZE; + int nNew = sessions_strm_chunk_size; if( pIn->bNoDiscard==0 ) sessionDiscardData(pIn); if( SQLITE_OK==sessionBufferGrow(&pIn->buf, nNew, &rc) ){ @@ -171912,7 +229692,7 @@ static void sessionSkipRecord( /* ** This function sets the value of the sqlite3_value object passed as the -** first argument to a copy of the string or blob held in the aData[] +** first argument to a copy of the string or blob held in the aData[] ** buffer. SQLITE_OK is returned if successful, or SQLITE_NOMEM if an OOM ** error occurs. */ @@ -171923,10 +229703,10 @@ static int sessionValueSetStr( u8 enc /* String encoding (0 for blobs) */ ){ /* In theory this code could just pass SQLITE_TRANSIENT as the final - ** argument to sqlite3ValueSetStr() and have the copy created + ** argument to sqlite3ValueSetStr() and have the copy created ** automatically. But doing so makes it difficult to detect any OOM ** error. Hence the code to create the copy externally. */ - u8 *aCopy = sqlite3_malloc(nData+1); + u8 *aCopy = sqlite3_malloc64((sqlite3_int64)nData+1); if( aCopy==0 ) return SQLITE_NOMEM; memcpy(aCopy, aData, nData); sqlite3ValueSetStr(pVal, nData, (char*)aCopy, enc, sqlite3_free); @@ -171961,23 +229741,30 @@ static int sessionReadRecord( SessionInput *pIn, /* Input data */ int nCol, /* Number of values in record */ u8 *abPK, /* Array of primary key flags, or NULL */ - sqlite3_value **apOut /* Write values to this array */ + sqlite3_value **apOut, /* Write values to this array */ + int *pbEmpty ){ int i; /* Used to iterate through columns */ int rc = SQLITE_OK; + assert( pbEmpty==0 || *pbEmpty==0 ); + if( pbEmpty ) *pbEmpty = 1; for(i=0; iaData[pIn->iNext++]; - } - - assert( apOut[i]==0 ); - if( eType ){ - apOut[i] = sqlite3ValueNew(0); - if( !apOut[i] ) rc = SQLITE_NOMEM; + if( pIn->iNext>=pIn->nData ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + eType = pIn->aData[pIn->iNext++]; + assert( apOut[i]==0 ); + if( eType ){ + if( pbEmpty ) *pbEmpty = 0; + apOut[i] = sqlite3ValueNew(0); + if( !apOut[i] ) rc = SQLITE_NOMEM; + } + } } if( rc==SQLITE_OK ){ @@ -171987,21 +229774,29 @@ static int sessionReadRecord( pIn->iNext += sessionVarintGet(aVal, &nByte); rc = sessionInputBuffer(pIn, nByte); if( rc==SQLITE_OK ){ - u8 enc = (eType==SQLITE_TEXT ? SQLITE_UTF8 : 0); - rc = sessionValueSetStr(apOut[i],&pIn->aData[pIn->iNext],nByte,enc); + if( nByte<0 || nByte>pIn->nData-pIn->iNext ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + u8 enc = (eType==SQLITE_TEXT ? SQLITE_UTF8 : 0); + rc = sessionValueSetStr(apOut[i],&pIn->aData[pIn->iNext],nByte,enc); + pIn->iNext += nByte; + } } - pIn->iNext += nByte; } if( eType==SQLITE_INTEGER || eType==SQLITE_FLOAT ){ - sqlite3_int64 v = sessionGetI64(aVal); - if( eType==SQLITE_INTEGER ){ - sqlite3VdbeMemSetInt64(apOut[i], v); + if( (pIn->nData-pIn->iNext)<8 ){ + rc = SQLITE_CORRUPT_BKPT; }else{ - double d; - memcpy(&d, &v, 8); - sqlite3VdbeMemSetDouble(apOut[i], d); + sqlite3_int64 v = sessionGetI64(aVal); + if( eType==SQLITE_INTEGER ){ + sqlite3VdbeMemSetInt64(apOut[i], v); + }else{ + double d; + memcpy(&d, &v, 8); + sqlite3VdbeMemSetDouble(apOut[i], d); + } + pIn->iNext += 8; } - pIn->iNext += 8; } } } @@ -172017,7 +229812,7 @@ static int sessionReadRecord( ** + array of PK flags (1 byte per column), ** + table name (nul terminated). ** -** This function ensures that all of the above is present in the input +** This function ensures that all of the above is present in the input ** buffer (i.e. that it can be accessed without any calls to xInput()). ** If successful, SQLITE_OK is returned. Otherwise, an SQLite error code. ** The input pointer is not moved. @@ -172030,8 +229825,19 @@ static int sessionChangesetBufferTblhdr(SessionInput *pIn, int *pnByte){ rc = sessionInputBuffer(pIn, 9); if( rc==SQLITE_OK ){ nRead += sessionVarintGet(&pIn->aData[pIn->iNext + nRead], &nCol); - rc = sessionInputBuffer(pIn, nRead+nCol+100); - nRead += nCol; + /* The hard upper limit for the number of columns in an SQLite + ** database table is, according to sqliteLimit.h, 32676. So + ** consider any table-header that purports to have more than 65536 + ** columns to be corrupt. This is convenient because otherwise, + ** if the (nCol>65536) condition below were omitted, a sufficiently + ** large value for nCol may cause nRead to wrap around and become + ** negative. Leading to a crash. */ + if( nCol<0 || nCol>65536 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ + rc = sessionInputBuffer(pIn, nRead+nCol+100); + nRead += nCol; + } } while( rc==SQLITE_OK ){ @@ -172089,8 +229895,8 @@ static int sessionChangesetBufferRecord( ** + array of PK flags (1 byte per column), ** + table name (nul terminated). ** -** This function decodes the table-header and populates the p->nCol, -** p->zTab and p->abPK[] variables accordingly. The p->apValue[] array is +** This function decodes the table-header and populates the p->nCol, +** p->zTab and p->abPK[] variables accordingly. The p->apValue[] array is ** also allocated or resized according to the new value of p->nCol. The ** input pointer is left pointing to the byte following the table header. ** @@ -172108,51 +229914,57 @@ static int sessionChangesetReadTblhdr(sqlite3_changeset_iter *p){ int nByte; int nVarint; nVarint = sessionVarintGet(&p->in.aData[p->in.iNext], &p->nCol); - nCopy -= nVarint; - p->in.iNext += nVarint; - nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; - p->tblhdr.nBuf = 0; - sessionBufferGrow(&p->tblhdr, nByte, &rc); + if( p->nCol>0 ){ + nCopy -= nVarint; + p->in.iNext += nVarint; + nByte = p->nCol * sizeof(sqlite3_value*) * 2 + nCopy; + p->tblhdr.nBuf = 0; + sessionBufferGrow(&p->tblhdr, nByte, &rc); + }else{ + rc = SQLITE_CORRUPT_BKPT; + } } if( rc==SQLITE_OK ){ - int iPK = sizeof(sqlite3_value*)*p->nCol*2; + size_t iPK = sizeof(sqlite3_value*)*p->nCol*2; memset(p->tblhdr.aBuf, 0, iPK); memcpy(&p->tblhdr.aBuf[iPK], &p->in.aData[p->in.iNext], nCopy); p->in.iNext += nCopy; } p->apValue = (sqlite3_value**)p->tblhdr.aBuf; - p->abPK = (u8*)&p->apValue[p->nCol*2]; - p->zTab = (char*)&p->abPK[p->nCol]; + if( p->apValue==0 ){ + p->abPK = 0; + p->zTab = 0; + }else{ + p->abPK = (u8*)&p->apValue[p->nCol*2]; + p->zTab = p->abPK ? (char*)&p->abPK[p->nCol] : 0; + } return (p->rc = rc); } /* -** Advance the changeset iterator to the next change. +** Advance the changeset iterator to the next change. The differences between +** this function and sessionChangesetNext() are that ** -** If both paRec and pnRec are NULL, then this function works like the public -** API sqlite3changeset_next(). If SQLITE_ROW is returned, then the -** sqlite3changeset_new() and old() APIs may be used to query for values. +** * If pbEmpty is not NULL and the change is a no-op UPDATE (an UPDATE +** that modifies no columns), this function sets (*pbEmpty) to 1. ** -** Otherwise, if paRec and pnRec are not NULL, then a pointer to the change -** record is written to *paRec before returning and the number of bytes in -** the record to *pnRec. -** -** Either way, this function returns SQLITE_ROW if the iterator is -** successfully advanced to the next change in the changeset, an SQLite -** error code if an error occurs, or SQLITE_DONE if there are no further -** changes in the changeset. +** * If the iterator is configured to skip no-op UPDATEs, +** sessionChangesetNext() does that. This function does not. */ -static int sessionChangesetNext( +static int sessionChangesetNextOne( sqlite3_changeset_iter *p, /* Changeset iterator */ u8 **paRec, /* If non-NULL, store record pointer here */ - int *pnRec /* If non-NULL, store size of record here */ + int *pnRec, /* If non-NULL, store size of record here */ + int *pbNew, /* If non-NULL, true if new table */ + int *pbEmpty ){ int i; u8 op; assert( (paRec==0 && pnRec==0) || (paRec && pnRec) ); + assert( pbEmpty==0 || *pbEmpty==0 ); /* If the iterator is in the error-state, return immediately. */ if( p->rc!=SQLITE_OK ) return p->rc; @@ -172172,30 +229984,39 @@ static int sessionChangesetNext( p->rc = sessionInputBuffer(&p->in, 2); if( p->rc!=SQLITE_OK ) return p->rc; + sessionDiscardData(&p->in); + p->in.iCurrent = p->in.iNext; + /* If the iterator is already at the end of the changeset, return DONE. */ if( p->in.iNext>=p->in.nData ){ return SQLITE_DONE; } - sessionDiscardData(&p->in); - p->in.iCurrent = p->in.iNext; - op = p->in.aData[p->in.iNext++]; - if( op=='T' || op=='P' ){ + while( op=='T' || op=='P' ){ + if( pbNew ) *pbNew = 1; p->bPatchset = (op=='P'); if( sessionChangesetReadTblhdr(p) ) return p->rc; if( (p->rc = sessionInputBuffer(&p->in, 2)) ) return p->rc; p->in.iCurrent = p->in.iNext; + if( p->in.iNext>=p->in.nData ) return SQLITE_DONE; op = p->in.aData[p->in.iNext++]; } + if( p->zTab==0 || (p->bPatchset && p->bInvert) ){ + /* The first record in the changeset is not a table header. Must be a + ** corrupt changeset. */ + assert( p->in.iNext==1 || p->zTab ); + return (p->rc = SQLITE_CORRUPT_BKPT); + } + p->op = op; p->bIndirect = p->in.aData[p->in.iNext++]; if( p->op!=SQLITE_UPDATE && p->op!=SQLITE_DELETE && p->op!=SQLITE_INSERT ){ return (p->rc = SQLITE_CORRUPT_BKPT); } - if( paRec ){ + if( paRec ){ int nVal; /* Number of values to buffer */ if( p->bPatchset==0 && op==SQLITE_UPDATE ){ nVal = p->nCol * 2; @@ -172210,39 +230031,92 @@ static int sessionChangesetNext( *paRec = &p->in.aData[p->in.iNext]; p->in.iNext += *pnRec; }else{ + sqlite3_value **apOld = (p->bInvert ? &p->apValue[p->nCol] : p->apValue); + sqlite3_value **apNew = (p->bInvert ? p->apValue : &p->apValue[p->nCol]); /* If this is an UPDATE or DELETE, read the old.* record. */ if( p->op!=SQLITE_INSERT && (p->bPatchset==0 || p->op==SQLITE_DELETE) ){ u8 *abPK = p->bPatchset ? p->abPK : 0; - p->rc = sessionReadRecord(&p->in, p->nCol, abPK, p->apValue); + p->rc = sessionReadRecord(&p->in, p->nCol, abPK, apOld, 0); if( p->rc!=SQLITE_OK ) return p->rc; } /* If this is an INSERT or UPDATE, read the new.* record. */ if( p->op!=SQLITE_DELETE ){ - p->rc = sessionReadRecord(&p->in, p->nCol, 0, &p->apValue[p->nCol]); + p->rc = sessionReadRecord(&p->in, p->nCol, 0, apNew, pbEmpty); if( p->rc!=SQLITE_OK ) return p->rc; } - if( p->bPatchset && p->op==SQLITE_UPDATE ){ + if( (p->bPatchset || p->bInvert) && p->op==SQLITE_UPDATE ){ /* If this is an UPDATE that is part of a patchset, then all PK and ** modified fields are present in the new.* record. The old.* record ** is currently completely empty. This block shifts the PK fields from ** new.* to old.*, to accommodate the code that reads these arrays. */ for(i=0; inCol; i++){ - assert( p->apValue[i]==0 ); - assert( p->abPK[i]==0 || p->apValue[i+p->nCol] ); + assert( p->bPatchset==0 || p->apValue[i]==0 ); if( p->abPK[i] ){ + assert( p->apValue[i]==0 ); p->apValue[i] = p->apValue[i+p->nCol]; + if( p->apValue[i]==0 ) return (p->rc = SQLITE_CORRUPT_BKPT); p->apValue[i+p->nCol] = 0; } } + }else if( p->bInvert ){ + if( p->op==SQLITE_INSERT ) p->op = SQLITE_DELETE; + else if( p->op==SQLITE_DELETE ) p->op = SQLITE_INSERT; + } + + /* If this is an UPDATE that is part of a changeset, then check that + ** there are no fields in the old.* record that are not (a) PK fields, + ** or (b) also present in the new.* record. + ** + ** Such records are technically corrupt, but the rebaser was at one + ** point generating them. Under most circumstances this is benign, but + ** can cause spurious SQLITE_RANGE errors when applying the changeset. */ + if( p->bPatchset==0 && p->op==SQLITE_UPDATE){ + for(i=0; inCol; i++){ + if( p->abPK[i]==0 && p->apValue[i+p->nCol]==0 ){ + sqlite3ValueFree(p->apValue[i]); + p->apValue[i] = 0; + } + } } } return SQLITE_ROW; } +/* +** Advance the changeset iterator to the next change. +** +** If both paRec and pnRec are NULL, then this function works like the public +** API sqlite3changeset_next(). If SQLITE_ROW is returned, then the +** sqlite3changeset_new() and old() APIs may be used to query for values. +** +** Otherwise, if paRec and pnRec are not NULL, then a pointer to the change +** record is written to *paRec before returning and the number of bytes in +** the record to *pnRec. +** +** Either way, this function returns SQLITE_ROW if the iterator is +** successfully advanced to the next change in the changeset, an SQLite +** error code if an error occurs, or SQLITE_DONE if there are no further +** changes in the changeset. +*/ +static int sessionChangesetNext( + sqlite3_changeset_iter *p, /* Changeset iterator */ + u8 **paRec, /* If non-NULL, store record pointer here */ + int *pnRec, /* If non-NULL, store size of record here */ + int *pbNew /* If non-NULL, true if new table */ +){ + int bEmpty; + int rc; + do { + bEmpty = 0; + rc = sessionChangesetNextOne(p, paRec, pnRec, pbNew, &bEmpty); + }while( rc==SQLITE_ROW && p->bSkipEmpty && bEmpty); + return rc; +} + /* ** Advance an iterator created by sqlite3changeset_start() to the next ** change in the changeset. This function may return SQLITE_ROW, SQLITE_DONE @@ -172251,8 +230125,8 @@ static int sessionChangesetNext( ** This function may not be called on iterators passed to a conflict handler ** callback by changeset_apply(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_next(sqlite3_changeset_iter *p){ - return sessionChangesetNext(p, 0, 0); +SQLITE_API int sqlite3changeset_next(sqlite3_changeset_iter *p){ + return sessionChangesetNext(p, 0, 0, 0); } /* @@ -172260,7 +230134,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_next(sqlite3_changeset_iter *p){ ** from a changeset iterator. It may only be called after changeset_next() ** has returned SQLITE_ROW. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_op( +SQLITE_API int sqlite3changeset_op( sqlite3_changeset_iter *pIter, /* Iterator handle */ const char **pzTab, /* OUT: Pointer to table name */ int *pnCol, /* OUT: Number of columns in table */ @@ -172280,7 +230154,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_op( ** to. This function may only be called after changeset_next() returns ** SQLITE_ROW. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_pk( +SQLITE_API int sqlite3changeset_pk( sqlite3_changeset_iter *pIter, /* Iterator object */ unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ int *pnCol /* OUT: Number of entries in output array */ @@ -172303,7 +230177,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_pk( ** If value iVal is out-of-range, SQLITE_RANGE is returned and *ppValue is ** not modified. Otherwise, SQLITE_OK. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_old( +SQLITE_API int sqlite3changeset_old( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int iVal, /* Index of old.* value to retrieve */ sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ @@ -172331,7 +230205,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_old( ** If value iVal is out-of-range, SQLITE_RANGE is returned and *ppValue is ** not modified. Otherwise, SQLITE_OK. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_new( +SQLITE_API int sqlite3changeset_new( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int iVal, /* Index of new.* value to retrieve */ sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ @@ -172356,7 +230230,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_new( /* ** This function may only be called with a changeset iterator that has been -** passed to an SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT +** passed to an SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT ** conflict-handler function. Otherwise, SQLITE_MISUSE is returned. ** ** If successful, *ppValue is set to point to an sqlite3_value structure @@ -172365,7 +230239,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_new( ** If value iVal is out-of-range or some other error occurs, an SQLite error ** code is returned. Otherwise, SQLITE_OK. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_conflict( +SQLITE_API int sqlite3changeset_conflict( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int iVal, /* Index of conflict record value to fetch */ sqlite3_value **ppValue /* OUT: Value from conflicting row */ @@ -172373,7 +230247,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_conflict( if( !pIter->pConflict ){ return SQLITE_MISUSE; } - if( iVal<0 || iVal>=sqlite3_column_count(pIter->pConflict) ){ + if( iVal<0 || iVal>=pIter->nCol ){ return SQLITE_RANGE; } *ppValue = sqlite3_column_value(pIter->pConflict, iVal); @@ -172388,7 +230262,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_conflict( ** ** In all other cases this function returns SQLITE_MISUSE. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_fk_conflicts( +SQLITE_API int sqlite3changeset_fk_conflicts( sqlite3_changeset_iter *pIter, /* Changeset iterator */ int *pnOut /* OUT: Number of FK violations */ ){ @@ -172406,7 +230280,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_fk_conflicts( ** This function may not be called on iterators passed to a conflict handler ** callback by changeset_apply(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_finalize(sqlite3_changeset_iter *p){ +SQLITE_API int sqlite3changeset_finalize(sqlite3_changeset_iter *p){ int rc = SQLITE_OK; if( p ){ int i; /* Used to iterate through p->apValue[] */ @@ -172501,7 +230375,7 @@ static int sessionChangesetInvert( int iCol; if( 0==apVal ){ - apVal = (sqlite3_value **)sqlite3_malloc(sizeof(apVal[0])*nCol*2); + apVal = (sqlite3_value **)sqlite3_malloc64(sizeof(apVal[0])*nCol*2); if( 0==apVal ){ rc = SQLITE_NOMEM; goto finished_invert; @@ -172515,9 +230389,9 @@ static int sessionChangesetInvert( /* Read the old.* and new.* records for the update change. */ pInput->iNext += 2; - rc = sessionReadRecord(pInput, nCol, 0, &apVal[0]); + rc = sessionReadRecord(pInput, nCol, 0, &apVal[0], 0); if( rc==SQLITE_OK ){ - rc = sessionReadRecord(pInput, nCol, 0, &apVal[nCol]); + rc = sessionReadRecord(pInput, nCol, 0, &apVal[nCol], 0); } /* Write the new old.* record. Consists of the PK columns from the @@ -172553,7 +230427,7 @@ static int sessionChangesetInvert( } assert( rc==SQLITE_OK ); - if( xOutput && sOut.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){ + if( xOutput && sOut.nBuf>=sessions_strm_chunk_size ){ rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); sOut.nBuf = 0; if( rc!=SQLITE_OK ) goto finished_invert; @@ -172561,11 +230435,11 @@ static int sessionChangesetInvert( } assert( rc==SQLITE_OK ); - if( pnInverted ){ + if( pnInverted && ALWAYS(ppInverted) ){ *pnInverted = sOut.nBuf; *ppInverted = sOut.aBuf; sOut.aBuf = 0; - }else if( sOut.nBuf>0 ){ + }else if( sOut.nBuf>0 && ALWAYS(xOutput!=0) ){ rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); } @@ -172580,7 +230454,7 @@ static int sessionChangesetInvert( /* ** Invert a changeset object. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_invert( +SQLITE_API int sqlite3changeset_invert( int nChangeset, /* Number of bytes in input */ const void *pChangeset, /* Input changeset */ int *pnInverted, /* OUT: Number of bytes in output changeset */ @@ -172599,7 +230473,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_invert( /* ** Streaming version of sqlite3changeset_invert(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_invert_strm( +SQLITE_API int sqlite3changeset_invert_strm( int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn, int (*xOutput)(void *pOut, const void *pData, int nData), @@ -172618,21 +230492,197 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_invert_strm( return rc; } + +typedef struct SessionUpdate SessionUpdate; +struct SessionUpdate { + sqlite3_stmt *pStmt; + u32 *aMask; + SessionUpdate *pNext; +}; + typedef struct SessionApplyCtx SessionApplyCtx; struct SessionApplyCtx { sqlite3 *db; sqlite3_stmt *pDelete; /* DELETE statement */ - sqlite3_stmt *pUpdate; /* UPDATE statement */ sqlite3_stmt *pInsert; /* INSERT statement */ sqlite3_stmt *pSelect; /* SELECT statement */ int nCol; /* Size of azCol[] and abPK[] arrays */ const char **azCol; /* Array of column names */ u8 *abPK; /* Boolean array - true if column is in PK */ - + u32 *aUpdateMask; /* Used by sessionUpdateFind */ + SessionUpdate *pUp; + int bStat1; /* True if table is sqlite_stat1 */ int bDeferConstraints; /* True to defer constraints */ + int bInvertConstraints; /* Invert when iterating constraints buffer */ SessionBuffer constraints; /* Deferred constraints are stored here */ + SessionBuffer rebase; /* Rebase information (if any) here */ + u8 bRebaseStarted; /* If table header is already in rebase */ + u8 bRebase; /* True to collect rebase information */ + u8 bIgnoreNoop; /* True to ignore no-op conflicts */ + int bRowid; }; +/* Number of prepared UPDATE statements to cache. */ +#define SESSION_UPDATE_CACHE_SZ 12 + +/* +** Find a prepared UPDATE statement suitable for the UPDATE step currently +** being visited by the iterator. The UPDATE is of the form: +** +** UPDATE tbl SET col = ?, col2 = ? WHERE pk1 IS ? AND pk2 IS ? +*/ +static int sessionUpdateFind( + sqlite3_changeset_iter *pIter, + SessionApplyCtx *p, + int bPatchset, + sqlite3_stmt **ppStmt +){ + int rc = SQLITE_OK; + SessionUpdate *pUp = 0; + int nCol = pIter->nCol; + int nU32 = (pIter->nCol+33)/32; + int ii; + + if( p->aUpdateMask==0 ){ + p->aUpdateMask = sqlite3_malloc(nU32*sizeof(u32)); + if( p->aUpdateMask==0 ){ + rc = SQLITE_NOMEM; + } + } + + if( rc==SQLITE_OK ){ + memset(p->aUpdateMask, 0, nU32*sizeof(u32)); + rc = SQLITE_CORRUPT; + for(ii=0; iinCol; ii++){ + if( sessionChangesetNew(pIter, ii) ){ + p->aUpdateMask[ii/32] |= (1<<(ii%32)); + rc = SQLITE_OK; + } + } + } + + if( rc==SQLITE_OK ){ + if( bPatchset ) p->aUpdateMask[nCol/32] |= (1<<(nCol%32)); + + if( p->pUp ){ + int nUp = 0; + SessionUpdate **pp = &p->pUp; + while( 1 ){ + nUp++; + if( 0==memcmp(p->aUpdateMask, (*pp)->aMask, nU32*sizeof(u32)) ){ + pUp = *pp; + *pp = pUp->pNext; + pUp->pNext = p->pUp; + p->pUp = pUp; + break; + } + + if( (*pp)->pNext ){ + pp = &(*pp)->pNext; + }else{ + if( nUp>=SESSION_UPDATE_CACHE_SZ ){ + sqlite3_finalize((*pp)->pStmt); + sqlite3_free(*pp); + *pp = 0; + } + break; + } + } + } + + if( pUp==0 ){ + int nByte = sizeof(SessionUpdate) * nU32*sizeof(u32); + int bStat1 = (sqlite3_stricmp(pIter->zTab, "sqlite_stat1")==0); + pUp = (SessionUpdate*)sqlite3_malloc(nByte); + if( pUp==0 ){ + rc = SQLITE_NOMEM; + }else{ + const char *zSep = ""; + SessionBuffer buf; + + memset(&buf, 0, sizeof(buf)); + pUp->aMask = (u32*)&pUp[1]; + memcpy(pUp->aMask, p->aUpdateMask, nU32*sizeof(u32)); + + sessionAppendStr(&buf, "UPDATE main.", &rc); + sessionAppendIdent(&buf, pIter->zTab, &rc); + sessionAppendStr(&buf, " SET ", &rc); + + /* Create the assignments part of the UPDATE */ + for(ii=0; iinCol; ii++){ + if( p->abPK[ii]==0 && sessionChangesetNew(pIter, ii) ){ + sessionAppendStr(&buf, zSep, &rc); + sessionAppendIdent(&buf, p->azCol[ii], &rc); + sessionAppendStr(&buf, " = ?", &rc); + sessionAppendInteger(&buf, ii*2+1, &rc); + zSep = ", "; + } + } + + /* Create the WHERE clause part of the UPDATE */ + zSep = ""; + sessionAppendStr(&buf, " WHERE ", &rc); + for(ii=0; iinCol; ii++){ + if( p->abPK[ii] || (bPatchset==0 && sessionChangesetOld(pIter, ii)) ){ + sessionAppendStr(&buf, zSep, &rc); + if( bStat1 && ii==1 ){ + assert( sqlite3_stricmp(p->azCol[ii], "idx")==0 ); + sessionAppendStr(&buf, + "idx IS CASE " + "WHEN length(?4)=0 AND typeof(?4)='blob' THEN NULL " + "ELSE ?4 END ", &rc + ); + }else{ + sessionAppendIdent(&buf, p->azCol[ii], &rc); + sessionAppendStr(&buf, " IS ?", &rc); + sessionAppendInteger(&buf, ii*2+2, &rc); + } + zSep = " AND "; + } + } + + if( rc==SQLITE_OK ){ + char *zSql = (char*)buf.aBuf; + rc = sqlite3_prepare_v2(p->db, zSql, buf.nBuf, &pUp->pStmt, 0); + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(pUp); + pUp = 0; + }else{ + pUp->pNext = p->pUp; + p->pUp = pUp; + } + sqlite3_free(buf.aBuf); + } + } + } + + assert( (rc==SQLITE_OK)==(pUp!=0) ); + if( pUp ){ + *ppStmt = pUp->pStmt; + }else{ + *ppStmt = 0; + } + return rc; +} + +/* +** Free all cached UPDATE statements. +*/ +static void sessionUpdateFree(SessionApplyCtx *p){ + SessionUpdate *pUp; + SessionUpdate *pNext; + for(pUp=p->pUp; pUp; pUp=pNext){ + pNext = pUp->pNext; + sqlite3_finalize(pUp->pStmt); + sqlite3_free(pUp); + } + p->pUp = 0; + sqlite3_free(p->aUpdateMask); + p->aUpdateMask = 0; +} + /* ** Formulate a statement to DELETE a row from database db. Assuming a table ** structure like this: @@ -172661,7 +230711,7 @@ static int sessionDeleteRow( SessionBuffer buf = {0, 0, 0}; int nPk = 0; - sessionAppendStr(&buf, "DELETE FROM ", &rc); + sessionAppendStr(&buf, "DELETE FROM main.", &rc); sessionAppendIdent(&buf, zTab, &rc); sessionAppendStr(&buf, " WHERE ", &rc); @@ -172702,102 +230752,6 @@ static int sessionDeleteRow( return rc; } -/* -** Formulate and prepare a statement to UPDATE a row from database db. -** Assuming a table structure like this: -** -** CREATE TABLE x(a, b, c, d, PRIMARY KEY(a, c)); -** -** The UPDATE statement looks like this: -** -** UPDATE x SET -** a = CASE WHEN ?2 THEN ?3 ELSE a END, -** b = CASE WHEN ?5 THEN ?6 ELSE b END, -** c = CASE WHEN ?8 THEN ?9 ELSE c END, -** d = CASE WHEN ?11 THEN ?12 ELSE d END -** WHERE a = ?1 AND c = ?7 AND (?13 OR -** (?5==0 OR b IS ?4) AND (?11==0 OR d IS ?10) AND -** ) -** -** For each column in the table, there are three variables to bind: -** -** ?(i*3+1) The old.* value of the column, if any. -** ?(i*3+2) A boolean flag indicating that the value is being modified. -** ?(i*3+3) The new.* value of the column, if any. -** -** Also, a boolean flag that, if set to true, causes the statement to update -** a row even if the non-PK values do not match. This is required if the -** conflict-handler is invoked with CHANGESET_DATA and returns -** CHANGESET_REPLACE. This is variable "?(nCol*3+1)". -** -** If successful, SQLITE_OK is returned and SessionApplyCtx.pUpdate is left -** pointing to the prepared version of the SQL statement. -*/ -static int sessionUpdateRow( - sqlite3 *db, /* Database handle */ - const char *zTab, /* Table name */ - SessionApplyCtx *p /* Session changeset-apply context */ -){ - int rc = SQLITE_OK; - int i; - const char *zSep = ""; - SessionBuffer buf = {0, 0, 0}; - - /* Append "UPDATE tbl SET " */ - sessionAppendStr(&buf, "UPDATE ", &rc); - sessionAppendIdent(&buf, zTab, &rc); - sessionAppendStr(&buf, " SET ", &rc); - - /* Append the assignments */ - for(i=0; inCol; i++){ - sessionAppendStr(&buf, zSep, &rc); - sessionAppendIdent(&buf, p->azCol[i], &rc); - sessionAppendStr(&buf, " = CASE WHEN ?", &rc); - sessionAppendInteger(&buf, i*3+2, &rc); - sessionAppendStr(&buf, " THEN ?", &rc); - sessionAppendInteger(&buf, i*3+3, &rc); - sessionAppendStr(&buf, " ELSE ", &rc); - sessionAppendIdent(&buf, p->azCol[i], &rc); - sessionAppendStr(&buf, " END", &rc); - zSep = ", "; - } - - /* Append the PK part of the WHERE clause */ - sessionAppendStr(&buf, " WHERE ", &rc); - for(i=0; inCol; i++){ - if( p->abPK[i] ){ - sessionAppendIdent(&buf, p->azCol[i], &rc); - sessionAppendStr(&buf, " = ?", &rc); - sessionAppendInteger(&buf, i*3+1, &rc); - sessionAppendStr(&buf, " AND ", &rc); - } - } - - /* Append the non-PK part of the WHERE clause */ - sessionAppendStr(&buf, " (?", &rc); - sessionAppendInteger(&buf, p->nCol*3+1, &rc); - sessionAppendStr(&buf, " OR 1", &rc); - for(i=0; inCol; i++){ - if( !p->abPK[i] ){ - sessionAppendStr(&buf, " AND (?", &rc); - sessionAppendInteger(&buf, i*3+2, &rc); - sessionAppendStr(&buf, "=0 OR ", &rc); - sessionAppendIdent(&buf, p->azCol[i], &rc); - sessionAppendStr(&buf, " IS ?", &rc); - sessionAppendInteger(&buf, i*3+1, &rc); - sessionAppendStr(&buf, ")", &rc); - } - } - sessionAppendStr(&buf, ")", &rc); - - if( rc==SQLITE_OK ){ - rc = sqlite3_prepare_v2(db, (char *)buf.aBuf, buf.nBuf, &p->pUpdate, 0); - } - sqlite3_free(buf.aBuf); - - return rc; -} - /* ** Formulate and prepare an SQL statement to query table zTab by primary ** key. Assuming the following table structure: @@ -172816,8 +230770,10 @@ static int sessionSelectRow( const char *zTab, /* Table name */ SessionApplyCtx *p /* Session changeset-apply context */ ){ - return sessionSelectStmt( - db, "main", zTab, p->nCol, p->azCol, p->abPK, &p->pSelect); + /* TODO */ + return sessionSelectStmt(db, p->bIgnoreNoop, + "main", zTab, p->bRowid, p->nCol, p->azCol, p->abPK, &p->pSelect + ); } /* @@ -172840,7 +230796,13 @@ static int sessionInsertRow( sessionAppendStr(&buf, "INSERT INTO main.", &rc); sessionAppendIdent(&buf, zTab, &rc); - sessionAppendStr(&buf, " VALUES(?", &rc); + sessionAppendStr(&buf, "(", &rc); + for(i=0; inCol; i++){ + if( i!=0 ) sessionAppendStr(&buf, ", ", &rc); + sessionAppendIdent(&buf, p->azCol[i], &rc); + } + + sessionAppendStr(&buf, ") VALUES(?", &rc); for(i=1; inCol; i++){ sessionAppendStr(&buf, ", ?", &rc); } @@ -172853,8 +230815,37 @@ static int sessionInsertRow( return rc; } +static int sessionPrepare(sqlite3 *db, sqlite3_stmt **pp, const char *zSql){ + return sqlite3_prepare_v2(db, zSql, -1, pp, 0); +} + /* -** A wrapper around sqlite3_bind_value() that detects an extra problem. +** Prepare statements for applying changes to the sqlite_stat1 table. +** These are similar to those created by sessionSelectRow(), +** sessionInsertRow(), sessionUpdateRow() and sessionDeleteRow() for +** other tables. +*/ +static int sessionStat1Sql(sqlite3 *db, SessionApplyCtx *p){ + int rc = sessionSelectRow(db, "sqlite_stat1", p); + if( rc==SQLITE_OK ){ + rc = sessionPrepare(db, &p->pInsert, + "INSERT INTO main.sqlite_stat1 VALUES(?1, " + "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END, " + "?3)" + ); + } + if( rc==SQLITE_OK ){ + rc = sessionPrepare(db, &p->pDelete, + "DELETE FROM main.sqlite_stat1 WHERE tbl=?1 AND idx IS " + "CASE WHEN length(?2)=0 AND typeof(?2)='blob' THEN NULL ELSE ?2 END " + "AND (?4 OR stat IS ?3)" + ); + } + return rc; +} + +/* +** A wrapper around sqlite3_bind_value() that detects an extra problem. ** See comments in the body of this function for details. */ static int sessionBindValue( @@ -172877,15 +230868,15 @@ static int sessionBindValue( } /* -** Iterator pIter must point to an SQLITE_INSERT entry. This function +** Iterator pIter must point to an SQLITE_INSERT entry. This function ** transfers new.* values from the current iterator entry to statement ** pStmt. The table being inserted into has nCol columns. ** -** New.* value $i from the iterator is bound to variable ($i+1) of +** New.* value $i from the iterator is bound to variable ($i+1) of ** statement pStmt. If parameter abPK is NULL, all values from 0 to (nCol-1) ** are transfered to the statement. Otherwise, if abPK is not NULL, it points -** to an array nCol elements in size. In this case only those values for -** which abPK[$i] is true are read from the iterator and bound to the +** to an array nCol elements in size. In this case only those values for +** which abPK[$i] is true are read from the iterator and bound to the ** statement. ** ** An SQLite error code is returned if an error occurs. Otherwise, SQLITE_OK. @@ -172901,16 +230892,22 @@ static int sessionBindRow( int rc = SQLITE_OK; /* Neither sqlite3changeset_old or sqlite3changeset_new can fail if the - ** argument iterator points to a suitable entry. Make sure that xValue - ** is one of these to guarantee that it is safe to ignore the return + ** argument iterator points to a suitable entry. Make sure that xValue + ** is one of these to guarantee that it is safe to ignore the return ** in the code below. */ assert( xValue==sqlite3changeset_old || xValue==sqlite3changeset_new ); for(i=0; rc==SQLITE_OK && ipSelect; int rc; /* Return code */ int nCol; /* Number of columns in table */ int op; /* Changset operation (SQLITE_UPDATE etc.) */ const char *zDummy; /* Unused */ + sqlite3_clear_bindings(pSelect); sqlite3changeset_op(pIter, &zDummy, &nCol, &op, 0); - rc = sessionBindRow(pIter, + rc = sessionBindRow(pIter, op==SQLITE_INSERT ? sqlite3changeset_new : sqlite3changeset_old, - nCol, abPK, pSelect + nCol, p->abPK, pSelect ); + if( op!=SQLITE_DELETE && p->bIgnoreNoop ){ + int ii; + for(ii=0; rc==SQLITE_OK && iiabPK[ii]==0 ){ + sqlite3_value *pVal = 0; + sqlite3changeset_new(pIter, ii, &pVal); + sqlite3_bind_int(pSelect, ii+1+nCol, (pVal==0)); + if( pVal ) rc = sessionBindValue(pSelect, ii+1, pVal); + } + } + } + if( rc==SQLITE_OK ){ rc = sqlite3_step(pSelect); if( rc!=SQLITE_ROW ) rc = sqlite3_reset(pSelect); @@ -172958,6 +230967,55 @@ static int sessionSeekToRow( return rc; } +/* +** This function is called from within sqlite3changeset_apply_v2() when +** a conflict is encountered and resolved using conflict resolution +** mode eType (either SQLITE_CHANGESET_OMIT or SQLITE_CHANGESET_REPLACE).. +** It adds a conflict resolution record to the buffer in +** SessionApplyCtx.rebase, which will eventually be returned to the caller +** of apply_v2() as the "rebase" buffer. +** +** Return SQLITE_OK if successful, or an SQLite error code otherwise. +*/ +static int sessionRebaseAdd( + SessionApplyCtx *p, /* Apply context */ + int eType, /* Conflict resolution (OMIT or REPLACE) */ + sqlite3_changeset_iter *pIter /* Iterator pointing at current change */ +){ + int rc = SQLITE_OK; + if( p->bRebase ){ + int i; + int eOp = pIter->op; + if( p->bRebaseStarted==0 ){ + /* Append a table-header to the rebase buffer */ + const char *zTab = pIter->zTab; + sessionAppendByte(&p->rebase, 'T', &rc); + sessionAppendVarint(&p->rebase, p->nCol, &rc); + sessionAppendBlob(&p->rebase, p->abPK, p->nCol, &rc); + sessionAppendBlob(&p->rebase, (u8*)zTab, (int)strlen(zTab)+1, &rc); + p->bRebaseStarted = 1; + } + + assert( eType==SQLITE_CHANGESET_REPLACE||eType==SQLITE_CHANGESET_OMIT ); + assert( eOp==SQLITE_DELETE || eOp==SQLITE_INSERT || eOp==SQLITE_UPDATE ); + + sessionAppendByte(&p->rebase, + (eOp==SQLITE_DELETE ? SQLITE_DELETE : SQLITE_INSERT), &rc + ); + sessionAppendByte(&p->rebase, (eType==SQLITE_CHANGESET_REPLACE), &rc); + for(i=0; inCol; i++){ + sqlite3_value *pVal = 0; + if( eOp==SQLITE_DELETE || (eOp==SQLITE_UPDATE && p->abPK[i]) ){ + sqlite3changeset_old(pIter, i, &pVal); + }else{ + sqlite3changeset_new(pIter, i, &pVal); + } + sessionAppendValue(&p->rebase, pVal, &rc); + } + } + return rc; +} + /* ** Invoke the conflict handler for the change that the changeset iterator ** currently points to. @@ -172989,7 +231047,7 @@ static int sessionSeekToRow( ** is set to non-zero before returning SQLITE_OK. ** ** If the conflict handler returns SQLITE_CHANGESET_ABORT, SQLITE_ABORT is -** returned. Or, if the conflict handler returns an invalid value, +** returned. Or, if the conflict handler returns an invalid value, ** SQLITE_MISUSE. If the conflict handler returns SQLITE_CHANGESET_OMIT, ** this function returns SQLITE_OK. */ @@ -173015,16 +231073,22 @@ static int sessionConflictHandler( /* Bind the new.* PRIMARY KEY values to the SELECT statement. */ if( pbReplace ){ - rc = sessionSeekToRow(p->db, pIter, p->abPK, p->pSelect); + rc = sessionSeekToRow(pIter, p); }else{ rc = SQLITE_OK; } if( rc==SQLITE_ROW ){ /* There exists another row with the new.* primary key. */ - pIter->pConflict = p->pSelect; - res = xConflict(pCtx, eType, pIter); - pIter->pConflict = 0; + if( p->bIgnoreNoop + && sqlite3_column_int(p->pSelect, sqlite3_column_count(p->pSelect)-1) + ){ + res = SQLITE_CHANGESET_OMIT; + }else{ + pIter->pConflict = p->pSelect; + res = xConflict(pCtx, eType, pIter); + pIter->pConflict = 0; + } rc = sqlite3_reset(p->pSelect); }else if( rc==SQLITE_OK ){ if( p->bDeferConstraints && eType==SQLITE_CHANGESET_CONFLICT ){ @@ -173033,7 +231097,7 @@ static int sessionConflictHandler( u8 *aBlob = &pIter->in.aData[pIter->in.iCurrent]; int nBlob = pIter->in.iNext - pIter->in.iCurrent; sessionAppendBlob(&p->constraints, aBlob, nBlob, &rc); - res = SQLITE_CHANGESET_OMIT; + return SQLITE_OK; }else{ /* No other row with the new.* primary key. */ res = xConflict(pCtx, eType+1, pIter); @@ -173059,6 +231123,9 @@ static int sessionConflictHandler( rc = SQLITE_MISUSE; break; } + if( rc==SQLITE_OK ){ + rc = sessionRebaseAdd(p, res, pIter); + } } return rc; @@ -173076,16 +231143,16 @@ static int sessionConflictHandler( ** to true before returning. In this case the caller will invoke this function ** again, this time with pbRetry set to NULL. ** -** If argument pbReplace is NULL and a CHANGESET_CONFLICT conflict is +** If argument pbReplace is NULL and a CHANGESET_CONFLICT conflict is ** encountered invoke the conflict handler with CHANGESET_CONSTRAINT instead. ** Or, if pbReplace is not NULL, invoke it with CHANGESET_CONFLICT. If such ** an invocation returns SQLITE_CHANGESET_REPLACE, set *pbReplace to true ** before retrying. In this case the caller attempts to remove the conflicting -** row before invoking this function again, this time with pbReplace set +** row before invoking this function again, this time with pbReplace set ** to NULL. ** ** If any conflict handler returns SQLITE_CHANGESET_ABORT, this function -** returns SQLITE_ABORT. Otherwise, if no error occurs, SQLITE_OK is +** returns SQLITE_ABORT. Otherwise, if no error occurs, SQLITE_OK is ** returned. */ static int sessionApplyOneOp( @@ -173101,7 +231168,7 @@ static int sessionApplyOneOp( int nCol; int rc = SQLITE_OK; - assert( p->pDelete && p->pUpdate && p->pInsert && p->pSelect ); + assert( p->pDelete && p->pInsert && p->pSelect ); assert( p->azCol && p->abPK ); assert( !pbReplace || *pbReplace==0 ); @@ -173129,7 +231196,7 @@ static int sessionApplyOneOp( sqlite3_step(p->pDelete); rc = sqlite3_reset(p->pDelete); - if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ + if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 && p->bIgnoreNoop==0 ){ rc = sessionConflictHandler( SQLITE_CHANGESET_DATA, p, pIter, xConflict, pCtx, pbRetry ); @@ -173141,29 +231208,28 @@ static int sessionApplyOneOp( }else if( op==SQLITE_UPDATE ){ int i; + sqlite3_stmt *pUp = 0; + int bPatchset = (pbRetry==0 || pIter->bPatchset); + + rc = sessionUpdateFind(pIter, p, bPatchset, &pUp); /* Bind values to the UPDATE statement. */ for(i=0; rc==SQLITE_OK && ipUpdate, i*3+2, !!pNew); - if( pOld ){ - rc = sessionBindValue(p->pUpdate, i*3+1, pOld); + if( p->abPK[i] || (bPatchset==0 && pOld) ){ + rc = sessionBindValue(pUp, i*2+2, pOld); } if( rc==SQLITE_OK && pNew ){ - rc = sessionBindValue(p->pUpdate, i*3+3, pNew); + rc = sessionBindValue(pUp, i*2+1, pNew); } } - if( rc==SQLITE_OK ){ - sqlite3_bind_int(p->pUpdate, nCol*3+1, pbRetry==0 || pIter->bPatchset); - } if( rc!=SQLITE_OK ) return rc; /* Attempt the UPDATE. In the case of a NOTFOUND or DATA conflict, ** the result will be SQLITE_OK with 0 rows modified. */ - sqlite3_step(p->pUpdate); - rc = sqlite3_reset(p->pUpdate); + sqlite3_step(pUp); + rc = sqlite3_reset(pUp); if( rc==SQLITE_OK && sqlite3_changes(p->db)==0 ){ /* A NOTFOUND or DATA error. Search the table to see if it contains @@ -173183,11 +231249,25 @@ static int sessionApplyOneOp( }else{ assert( op==SQLITE_INSERT ); - rc = sessionBindRow(pIter, sqlite3changeset_new, nCol, 0, p->pInsert); - if( rc!=SQLITE_OK ) return rc; + if( p->bStat1 ){ + /* Check if there is a conflicting row. For sqlite_stat1, this needs + ** to be done using a SELECT, as there is no PRIMARY KEY in the + ** database schema to throw an exception if a duplicate is inserted. */ + rc = sessionSeekToRow(pIter, p); + if( rc==SQLITE_ROW ){ + rc = SQLITE_CONSTRAINT; + sqlite3_reset(p->pSelect); + } + } + + if( rc==SQLITE_OK ){ + rc = sessionBindRow(pIter, sqlite3changeset_new, nCol, 0, p->pInsert); + if( rc!=SQLITE_OK ) return rc; + + sqlite3_step(p->pInsert); + rc = sqlite3_reset(p->pInsert); + } - sqlite3_step(p->pInsert); - rc = sqlite3_reset(p->pInsert); if( (rc&0xff)==SQLITE_CONSTRAINT ){ rc = sessionConflictHandler( SQLITE_CHANGESET_CONFLICT, p, pIter, xConflict, pCtx, pbReplace @@ -173204,7 +231284,7 @@ static int sessionApplyOneOp( ** the conflict handler callback. ** ** The difference between this function and sessionApplyOne() is that this -** function handles the case where the conflict-handler is invoked and +** function handles the case where the conflict-handler is invoked and ** returns SQLITE_CHANGESET_REPLACE - indicating that the change should be ** retried in some manner. */ @@ -173220,42 +231300,42 @@ static int sessionApplyOneWithRetry( int rc; rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, &bReplace, &bRetry); - assert( rc==SQLITE_OK || (bRetry==0 && bReplace==0) ); - - /* If the bRetry flag is set, the change has not been applied due to an - ** SQLITE_CHANGESET_DATA problem (i.e. this is an UPDATE or DELETE and - ** a row with the correct PK is present in the db, but one or more other - ** fields do not contain the expected values) and the conflict handler - ** returned SQLITE_CHANGESET_REPLACE. In this case retry the operation, - ** but pass NULL as the final argument so that sessionApplyOneOp() ignores - ** the SQLITE_CHANGESET_DATA problem. */ - if( bRetry ){ - assert( pIter->op==SQLITE_UPDATE || pIter->op==SQLITE_DELETE ); - rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); - } - - /* If the bReplace flag is set, the change is an INSERT that has not - ** been performed because the database already contains a row with the - ** specified primary key and the conflict handler returned - ** SQLITE_CHANGESET_REPLACE. In this case remove the conflicting row - ** before reattempting the INSERT. */ - else if( bReplace ){ - assert( pIter->op==SQLITE_INSERT ); - rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); - if( rc==SQLITE_OK ){ - rc = sessionBindRow(pIter, - sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete); - sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); - } - if( rc==SQLITE_OK ){ - sqlite3_step(pApply->pDelete); - rc = sqlite3_reset(pApply->pDelete); - } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK ){ + /* If the bRetry flag is set, the change has not been applied due to an + ** SQLITE_CHANGESET_DATA problem (i.e. this is an UPDATE or DELETE and + ** a row with the correct PK is present in the db, but one or more other + ** fields do not contain the expected values) and the conflict handler + ** returned SQLITE_CHANGESET_REPLACE. In this case retry the operation, + ** but pass NULL as the final argument so that sessionApplyOneOp() ignores + ** the SQLITE_CHANGESET_DATA problem. */ + if( bRetry ){ + assert( pIter->op==SQLITE_UPDATE || pIter->op==SQLITE_DELETE ); rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); } - if( rc==SQLITE_OK ){ - rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + + /* If the bReplace flag is set, the change is an INSERT that has not + ** been performed because the database already contains a row with the + ** specified primary key and the conflict handler returned + ** SQLITE_CHANGESET_REPLACE. In this case remove the conflicting row + ** before reattempting the INSERT. */ + else if( bReplace ){ + assert( pIter->op==SQLITE_INSERT ); + rc = sqlite3_exec(db, "SAVEPOINT replace_op", 0, 0, 0); + if( rc==SQLITE_OK ){ + rc = sessionBindRow(pIter, + sqlite3changeset_new, pApply->nCol, pApply->abPK, pApply->pDelete); + sqlite3_bind_int(pApply->pDelete, pApply->nCol+1, 1); + } + if( rc==SQLITE_OK ){ + sqlite3_step(pApply->pDelete); + rc = sqlite3_reset(pApply->pDelete); + } + if( rc==SQLITE_OK ){ + rc = sessionApplyOneOp(pIter, pApply, xConflict, pCtx, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_exec(db, "RELEASE replace_op", 0, 0, 0); + } } } @@ -173266,7 +231346,7 @@ static int sessionApplyOneWithRetry( ** Retry the changes accumulated in the pApply->constraints buffer. */ static int sessionRetryConstraints( - sqlite3 *db, + sqlite3 *db, int bPatchset, const char *zTab, SessionApplyCtx *pApply, @@ -173280,9 +231360,11 @@ static int sessionRetryConstraints( SessionBuffer cons = pApply->constraints; memset(&pApply->constraints, 0, sizeof(SessionBuffer)); - rc = sessionChangesetStart(&pIter2, 0, 0, cons.nBuf, cons.aBuf); + rc = sessionChangesetStart( + &pIter2, 0, 0, cons.nBuf, cons.aBuf, pApply->bInvertConstraints, 1 + ); if( rc==SQLITE_OK ){ - int nByte = 2*pApply->nCol*sizeof(sqlite3_value*); + size_t nByte = 2*pApply->nCol*sizeof(sqlite3_value*); int rc2; pIter2->bPatchset = bPatchset; pIter2->zTab = (char*)zTab; @@ -173314,7 +231396,7 @@ static int sessionRetryConstraints( /* ** Argument pIter is a changeset iterator that has been initialized, but -** not yet passed to sqlite3changeset_next(). This function applies the +** not yet passed to sqlite3changeset_next(). This function applies the ** changeset to the main database attached to handle "db". The supplied ** conflict handler callback is invoked to resolve any conflicts encountered ** while applying the change. @@ -173331,21 +231413,34 @@ static int sessionChangesetApply( int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ sqlite3_changeset_iter *p /* Handle describing change and conflict */ ), - void *pCtx /* First argument passed to xConflict */ + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase information */ + int flags /* SESSION_APPLY_XXX flags */ ){ int schemaMismatch = 0; - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ const char *zTab = 0; /* Name of current table */ int nTab = 0; /* Result of sqlite3Strlen30(zTab) */ SessionApplyCtx sApply; /* changeset_apply() context object */ int bPatchset; + u64 savedFlag = db->flags & SQLITE_FkNoAction; assert( xConflict!=0 ); + sqlite3_mutex_enter(sqlite3_db_mutex(db)); + if( flags & SQLITE_CHANGESETAPPLY_FKNOACTION ){ + db->flags |= ((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } + pIter->in.bNoDiscard = 1; memset(&sApply, 0, sizeof(sApply)); - sqlite3_mutex_enter(sqlite3_db_mutex(db)); - rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); + sApply.bRebase = (ppRebase && pnRebase); + sApply.bInvertConstraints = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + sApply.bIgnoreNoop = !!(flags & SQLITE_CHANGESETAPPLY_IGNORENOOP); + if( (flags & SQLITE_CHANGESETAPPLY_NOSAVEPOINT)==0 ){ + rc = sqlite3_exec(db, "SAVEPOINT changeset_apply", 0, 0, 0); + } if( rc==SQLITE_OK ){ rc = sqlite3_exec(db, "PRAGMA defer_foreign_keys = 1", 0, 0, 0); } @@ -173353,7 +231448,7 @@ static int sessionChangesetApply( int nCol; int op; const char *zNew; - + sqlite3changeset_op(pIter, &zNew, &nCol, &op, 0); if( zTab==0 || sqlite3_strnicmp(zNew, zTab, nTab+1) ){ @@ -173364,16 +231459,25 @@ static int sessionChangesetApply( ); if( rc!=SQLITE_OK ) break; + sessionUpdateFree(&sApply); sqlite3_free((char*)sApply.azCol); /* cast works around VC++ bug */ sqlite3_finalize(sApply.pDelete); - sqlite3_finalize(sApply.pUpdate); sqlite3_finalize(sApply.pInsert); sqlite3_finalize(sApply.pSelect); - memset(&sApply, 0, sizeof(sApply)); sApply.db = db; + sApply.pDelete = 0; + sApply.pInsert = 0; + sApply.pSelect = 0; + sApply.nCol = 0; + sApply.azCol = 0; + sApply.abPK = 0; + sApply.bStat1 = 0; sApply.bDeferConstraints = 1; + sApply.bRebaseStarted = 0; + sApply.bRowid = 0; + memset(&sApply.constraints, 0, sizeof(SessionBuffer)); - /* If an xFilter() callback was specified, invoke it now. If the + /* If an xFilter() callback was specified, invoke it now. If the ** xFilter callback returns zero, skip this table. If it returns ** non-zero, proceed. */ schemaMismatch = (xFilter && (0==xFilter(pCtx, zNew))); @@ -173386,38 +231490,54 @@ static int sessionChangesetApply( nTab = (int)strlen(zTab); sApply.azCol = (const char **)zTab; }else{ + int nMinCol = 0; + int i; + sqlite3changeset_pk(pIter, &abPK, 0); - rc = sessionTableInfo( - db, "main", zNew, &sApply.nCol, &zTab, &sApply.azCol, &sApply.abPK + rc = sessionTableInfo(0, db, "main", zNew, + &sApply.nCol, &zTab, &sApply.azCol, 0, &sApply.abPK, &sApply.bRowid ); if( rc!=SQLITE_OK ) break; - + for(i=0; iflags & SQLITE_FkNoAction ); + db->flags &= ~((u64)SQLITE_FkNoAction); + db->aDb[0].pSchema->schema_cookie -= 32; + } sqlite3_mutex_leave(sqlite3_db_mutex(db)); return rc; } +/* +** Apply the changeset passed via pChangeset/nChangeset to the main +** database attached to handle "db". +*/ +SQLITE_API int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int bInv = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + int rc = sessionChangesetStart(&pIter, 0, 0, nChangeset, pChangeset, bInv, 1); + + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply( + db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags + ); + } + + return rc; +} + /* ** Apply the changeset passed via pChangeset/nChangeset to the main database ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_apply( +SQLITE_API int sqlite3changeset_apply( sqlite3 *db, /* Apply change to "main" db of this handle */ int nChangeset, /* Size of changeset in bytes */ void *pChangeset, /* Changeset blob */ @@ -173494,12 +231663,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_apply( ), void *pCtx /* First argument passed to xConflict */ ){ - sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int rc = sqlite3changeset_start(&pIter, nChangeset, pChangeset); - if( rc==SQLITE_OK ){ - rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx); - } - return rc; + return sqlite3changeset_apply_v2( + db, nChangeset, pChangeset, xFilter, xConflict, pCtx, 0, 0, 0 + ); } /* @@ -173507,7 +231673,34 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_apply( ** attached to handle "db". Invoke the supplied conflict handler callback ** to resolve any conflicts encountered while applying the change. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_apply_strm( +SQLITE_API int sqlite3changeset_apply_v2_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +){ + sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ + int bInverse = !!(flags & SQLITE_CHANGESETAPPLY_INVERT); + int rc = sessionChangesetStart(&pIter, xInput, pIn, 0, 0, bInverse, 1); + if( rc==SQLITE_OK ){ + rc = sessionChangesetApply( + db, pIter, xFilter, xConflict, pCtx, ppRebase, pnRebase, flags + ); + } + return rc; +} +SQLITE_API int sqlite3changeset_apply_strm( sqlite3 *db, /* Apply change to "main" db of this handle */ int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ void *pIn, /* First arg for xInput */ @@ -173522,12 +231715,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_apply_strm( ), void *pCtx /* First argument passed to xConflict */ ){ - sqlite3_changeset_iter *pIter; /* Iterator to skip through changeset */ - int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); - if( rc==SQLITE_OK ){ - rc = sessionChangesetApply(db, pIter, xFilter, xConflict, pCtx); - } - return rc; + return sqlite3changeset_apply_v2_strm( + db, xInput, pIn, xFilter, xConflict, pCtx, 0, 0, 0 + ); } /* @@ -173537,6 +231727,10 @@ struct sqlite3_changegroup { int rc; /* Error code */ int bPatch; /* True to accumulate patchsets */ SessionTable *pList; /* List of tables in current patch */ + SessionBuffer rec; + + sqlite3 *db; /* Configured by changegroup_schema() */ + char *zDb; /* Configured by changegroup_schema() */ }; /* @@ -173546,6 +231740,7 @@ struct sqlite3_changegroup { */ static int sessionChangeMerge( SessionTable *pTab, /* Table structure */ + int bRebase, /* True for a rebase hash-table */ int bPatchset, /* True for patchsets */ SessionChange *pExist, /* Existing change */ int op2, /* Second change operation */ @@ -173555,22 +231750,81 @@ static int sessionChangeMerge( SessionChange **ppNew /* OUT: Merged change */ ){ SessionChange *pNew = 0; + int rc = SQLITE_OK; + assert( aRec!=0 ); if( !pExist ){ - pNew = (SessionChange *)sqlite3_malloc(sizeof(SessionChange) + nRec); + pNew = (SessionChange *)sqlite3_malloc64(sizeof(SessionChange) + nRec); if( !pNew ){ return SQLITE_NOMEM; } memset(pNew, 0, sizeof(SessionChange)); pNew->op = op2; pNew->bIndirect = bIndirect; - pNew->nRecord = nRec; pNew->aRecord = (u8*)&pNew[1]; - memcpy(pNew->aRecord, aRec, nRec); + if( bIndirect==0 || bRebase==0 ){ + pNew->nRecord = nRec; + memcpy(pNew->aRecord, aRec, nRec); + }else{ + int i; + u8 *pIn = aRec; + u8 *pOut = pNew->aRecord; + for(i=0; inCol; i++){ + int nIn = sessionSerialLen(pIn); + if( *pIn==0 ){ + *pOut++ = 0; + }else if( pTab->abPK[i]==0 ){ + *pOut++ = 0xFF; + }else{ + memcpy(pOut, pIn, nIn); + pOut += nIn; + } + pIn += nIn; + } + pNew->nRecord = pOut - pNew->aRecord; + } + }else if( bRebase ){ + if( pExist->op==SQLITE_DELETE && pExist->bIndirect ){ + *ppNew = pExist; + }else{ + sqlite3_int64 nByte = nRec + pExist->nRecord + sizeof(SessionChange); + pNew = (SessionChange*)sqlite3_malloc64(nByte); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + int i; + u8 *a1 = pExist->aRecord; + u8 *a2 = aRec; + u8 *pOut; + + memset(pNew, 0, nByte); + pNew->bIndirect = bIndirect || pExist->bIndirect; + pNew->op = op2; + pOut = pNew->aRecord = (u8*)&pNew[1]; + + for(i=0; inCol; i++){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); + if( *a1==0xFF || (pTab->abPK[i]==0 && bIndirect) ){ + *pOut++ = 0xFF; + }else if( *a2==0 ){ + memcpy(pOut, a1, n1); + pOut += n1; + }else{ + memcpy(pOut, a2, n2); + pOut += n2; + } + a1 += n1; + a2 += n2; + } + pNew->nRecord = pOut - pNew->aRecord; + } + sqlite3_free(pExist); + } }else{ int op1 = pExist->op; - /* + /* ** op1=INSERT, op2=INSERT -> Unsupported. Discard op2. ** op1=INSERT, op2=UPDATE -> INSERT. ** op1=INSERT, op2=DELETE -> (none) @@ -173582,7 +231836,7 @@ static int sessionChangeMerge( ** op1=DELETE, op2=INSERT -> UPDATE. ** op1=DELETE, op2=UPDATE -> Unsupported. Discard op2. ** op1=DELETE, op2=DELETE -> Unsupported. Discard op2. - */ + */ if( (op1==SQLITE_INSERT && op2==SQLITE_INSERT) || (op1==SQLITE_UPDATE && op2==SQLITE_INSERT) || (op1==SQLITE_DELETE && op2==SQLITE_UPDATE) @@ -173594,14 +231848,14 @@ static int sessionChangeMerge( assert( pNew==0 ); }else{ u8 *aExist = pExist->aRecord; - int nByte; + sqlite3_int64 nByte; u8 *aCsr; /* Allocate a new SessionChange object. Ensure that the aRecord[] ** buffer of the new object is large enough to hold any record that ** may be generated by combining the input records. */ nByte = sizeof(SessionChange) + pExist->nRecord + nRec; - pNew = (SessionChange *)sqlite3_malloc(nByte); + pNew = (SessionChange *)sqlite3_malloc64(nByte); if( !pNew ){ sqlite3_free(pExist); return SQLITE_NOMEM; @@ -173660,88 +231914,246 @@ static int sessionChangeMerge( } *ppNew = pNew; - return SQLITE_OK; + return rc; } /* -** Add all changes in the changeset traversed by the iterator passed as -** the first argument to the changegroup hash tables. +** Check if a changeset entry with nCol columns and the PK array passed +** as the final argument to this function is compatible with SessionTable +** pTab. If so, return 1. Otherwise, if they are incompatible in some way, +** return 0. */ -static int sessionChangesetToHash( - sqlite3_changeset_iter *pIter, /* Iterator to read from */ - sqlite3_changegroup *pGrp /* Changegroup object to add changeset to */ +static int sessionChangesetCheckCompat( + SessionTable *pTab, + int nCol, + u8 *abPK ){ - u8 *aRec; - int nRec; - int rc = SQLITE_OK; - SessionTable *pTab = 0; - - - while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec) ){ - const char *zNew; - int nCol; - int op; - int iHash; - int bIndirect; - SessionChange *pChange; - SessionChange *pExist = 0; - SessionChange **pp; - - if( pGrp->pList==0 ){ - pGrp->bPatch = pIter->bPatchset; - }else if( pIter->bPatchset!=pGrp->bPatch ){ - rc = SQLITE_ERROR; - break; + if( pTab->azCol && nColnCol ){ + int ii; + for(ii=0; iinCol; ii++){ + u8 bPK = (ii < nCol) ? abPK[ii] : 0; + if( pTab->abPK[ii]!=bPK ) return 0; } + return 1; + } + return (pTab->nCol==nCol && 0==memcmp(abPK, pTab->abPK, nCol)); +} - sqlite3changeset_op(pIter, &zNew, &nCol, &op, &bIndirect); - if( !pTab || sqlite3_stricmp(zNew, pTab->zName) ){ - /* Search the list for a matching table */ - int nNew = (int)strlen(zNew); - u8 *abPK; +static int sessionChangesetExtendRecord( + sqlite3_changegroup *pGrp, + SessionTable *pTab, + int nCol, + int op, + const u8 *aRec, + int nRec, + SessionBuffer *pOut +){ + int rc = SQLITE_OK; + int ii = 0; - sqlite3changeset_pk(pIter, &abPK, 0); - for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ - if( 0==sqlite3_strnicmp(pTab->zName, zNew, nNew+1) ) break; + assert( pTab->azCol ); + assert( nColnCol ); + + pOut->nBuf = 0; + if( op==SQLITE_INSERT || (op==SQLITE_DELETE && pGrp->bPatch==0) ){ + /* Append the missing default column values to the record. */ + sessionAppendBlob(pOut, aRec, nRec, &rc); + if( rc==SQLITE_OK && pTab->pDfltStmt==0 ){ + rc = sessionPrepareDfltStmt(pGrp->db, pTab, &pTab->pDfltStmt); + if( rc==SQLITE_OK && SQLITE_ROW!=sqlite3_step(pTab->pDfltStmt) ){ + rc = sqlite3_errcode(pGrp->db); } - if( !pTab ){ - SessionTable **ppTab; - - pTab = sqlite3_malloc(sizeof(SessionTable) + nCol + nNew+1); - if( !pTab ){ - rc = SQLITE_NOMEM; + } + for(ii=nCol; rc==SQLITE_OK && iinCol; ii++){ + int eType = sqlite3_column_type(pTab->pDfltStmt, ii); + sessionAppendByte(pOut, eType, &rc); + switch( eType ){ + case SQLITE_FLOAT: + case SQLITE_INTEGER: { + i64 iVal; + if( eType==SQLITE_INTEGER ){ + iVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + }else{ + double rVal = sqlite3_column_int64(pTab->pDfltStmt, ii); + memcpy(&iVal, &rVal, sizeof(i64)); + } + if( SQLITE_OK==sessionBufferGrow(pOut, 8, &rc) ){ + sessionPutI64(&pOut->aBuf[pOut->nBuf], iVal); + pOut->nBuf += 8; + } break; } - memset(pTab, 0, sizeof(SessionTable)); - pTab->nCol = nCol; - pTab->abPK = (u8*)&pTab[1]; - memcpy(pTab->abPK, abPK, nCol); - pTab->zName = (char*)&pTab->abPK[nCol]; - memcpy(pTab->zName, zNew, nNew+1); - /* The new object must be linked on to the end of the list, not - ** simply added to the start of it. This is to ensure that the - ** tables within the output of sqlite3changegroup_output() are in - ** the right order. */ - for(ppTab=&pGrp->pList; *ppTab; ppTab=&(*ppTab)->pNext); - *ppTab = pTab; - }else if( pTab->nCol!=nCol || memcmp(pTab->abPK, abPK, nCol) ){ - rc = SQLITE_SCHEMA; - break; + case SQLITE_BLOB: + case SQLITE_TEXT: { + int n = sqlite3_column_bytes(pTab->pDfltStmt, ii); + sessionAppendVarint(pOut, n, &rc); + if( eType==SQLITE_TEXT ){ + const u8 *z = (const u8*)sqlite3_column_text(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + }else{ + const u8 *z = (const u8*)sqlite3_column_blob(pTab->pDfltStmt, ii); + sessionAppendBlob(pOut, z, n, &rc); + } + break; + } + + default: + assert( eType==SQLITE_NULL ); + break; + } + } + }else if( op==SQLITE_UPDATE ){ + /* Append missing "undefined" entries to the old.* record. And, if this + ** is an UPDATE, to the new.* record as well. */ + int iOff = 0; + if( pGrp->bPatch==0 ){ + for(ii=0; iinCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); } } - if( sessionGrowHash(pIter->bPatchset, pTab) ){ - rc = SQLITE_NOMEM; - break; + sessionAppendBlob(pOut, &aRec[iOff], nRec-iOff, &rc); + for(ii=0; ii<(pTab->nCol-nCol); ii++){ + sessionAppendByte(pOut, 0x00, &rc); } + }else{ + assert( op==SQLITE_DELETE && pGrp->bPatch ); + sessionAppendBlob(pOut, aRec, nRec, &rc); + } + + return rc; +} + +/* +** Locate or create a SessionTable object that may be used to add the +** change currently pointed to by iterator pIter to changegroup pGrp. +** If successful, set output variable (*ppTab) to point to the table +** object and return SQLITE_OK. Otherwise, if some error occurs, return +** an SQLite error code and leave (*ppTab) set to NULL. +*/ +static int sessionChangesetFindTable( + sqlite3_changegroup *pGrp, + const char *zTab, + sqlite3_changeset_iter *pIter, + SessionTable **ppTab +){ + int rc = SQLITE_OK; + SessionTable *pTab = 0; + int nTab = (int)strlen(zTab); + u8 *abPK = 0; + int nCol = 0; + + *ppTab = 0; + sqlite3changeset_pk(pIter, &abPK, &nCol); + + /* Search the list for an existing table */ + for(pTab = pGrp->pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_strnicmp(pTab->zName, zTab, nTab+1) ) break; + } + + /* If one was not found above, create a new table now */ + if( !pTab ){ + SessionTable **ppNew; + + pTab = sqlite3_malloc64(sizeof(SessionTable) + nCol + nTab+1); + if( !pTab ){ + return SQLITE_NOMEM; + } + memset(pTab, 0, sizeof(SessionTable)); + pTab->nCol = nCol; + pTab->abPK = (u8*)&pTab[1]; + memcpy(pTab->abPK, abPK, nCol); + pTab->zName = (char*)&pTab->abPK[nCol]; + memcpy(pTab->zName, zTab, nTab+1); + + if( pGrp->db ){ + pTab->nCol = 0; + rc = sessionInitTable(0, pTab, pGrp->db, pGrp->zDb); + if( rc ){ + assert( pTab->azCol==0 ); + sqlite3_free(pTab); + return rc; + } + } + + /* The new object must be linked on to the end of the list, not + ** simply added to the start of it. This is to ensure that the + ** tables within the output of sqlite3changegroup_output() are in + ** the right order. */ + for(ppNew=&pGrp->pList; *ppNew; ppNew=&(*ppNew)->pNext); + *ppNew = pTab; + } + + /* Check that the table is compatible. */ + if( !sessionChangesetCheckCompat(pTab, nCol, abPK) ){ + rc = SQLITE_SCHEMA; + } + + *ppTab = pTab; + return rc; +} + +/* +** Add the change currently indicated by iterator pIter to the hash table +** belonging to changegroup pGrp. +*/ +static int sessionOneChangeToHash( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter, + int bRebase +){ + int rc = SQLITE_OK; + int nCol = 0; + int op = 0; + int iHash = 0; + int bIndirect = 0; + SessionChange *pChange = 0; + SessionChange *pExist = 0; + SessionChange **pp = 0; + SessionTable *pTab = 0; + u8 *aRec = &pIter->in.aData[pIter->in.iCurrent + 2]; + int nRec = (pIter->in.iNext - pIter->in.iCurrent) - 2; + + assert( nRec>0 ); + + /* Ensure that only changesets, or only patchsets, but not a mixture + ** of both, are being combined. It is an error to try to combine a + ** changeset and a patchset. */ + if( pGrp->pList==0 ){ + pGrp->bPatch = pIter->bPatchset; + }else if( pIter->bPatchset!=pGrp->bPatch ){ + rc = SQLITE_ERROR; + } + + if( rc==SQLITE_OK ){ + const char *zTab = 0; + sqlite3changeset_op(pIter, &zTab, &nCol, &op, &bIndirect); + rc = sessionChangesetFindTable(pGrp, zTab, pIter, &pTab); + } + + if( rc==SQLITE_OK && nColnCol ){ + SessionBuffer *pBuf = &pGrp->rec; + rc = sessionChangesetExtendRecord(pGrp, pTab, nCol, op, aRec, nRec, pBuf); + aRec = pBuf->aBuf; + nRec = pBuf->nBuf; + assert( pGrp->db ); + } + + if( rc==SQLITE_OK && sessionGrowHash(0, pIter->bPatchset, pTab) ){ + rc = SQLITE_NOMEM; + } + + if( rc==SQLITE_OK ){ + /* Search for existing entry. If found, remove it from the hash table. + ** Code below may link it back in. */ iHash = sessionChangeHash( pTab, (pIter->bPatchset && op==SQLITE_DELETE), aRec, pTab->nChange ); - - /* Search for existing entry. If found, remove it from the hash table. - ** Code below may link it back in. - */ for(pp=&pTab->apChange[iHash]; *pp; pp=&(*pp)->pNext){ int bPkOnly1 = 0; int bPkOnly2 = 0; @@ -173756,16 +232168,40 @@ static int sessionChangesetToHash( break; } } + } - rc = sessionChangeMerge(pTab, + if( rc==SQLITE_OK ){ + rc = sessionChangeMerge(pTab, bRebase, pIter->bPatchset, pExist, op, bIndirect, aRec, nRec, &pChange ); - if( rc ) break; - if( pChange ){ - pChange->pNext = pTab->apChange[iHash]; - pTab->apChange[iHash] = pChange; - pTab->nEntry++; - } + } + if( rc==SQLITE_OK && pChange ){ + pChange->pNext = pTab->apChange[iHash]; + pTab->apChange[iHash] = pChange; + pTab->nEntry++; + } + + if( rc==SQLITE_OK ) rc = pIter->rc; + return rc; +} + +/* +** Add all changes in the changeset traversed by the iterator passed as +** the first argument to the changegroup hash tables. +*/ +static int sessionChangesetToHash( + sqlite3_changeset_iter *pIter, /* Iterator to read from */ + sqlite3_changegroup *pGrp, /* Changegroup object to add changeset to */ + int bRebase /* True if hash table is for rebasing */ +){ + u8 *aRec; + int nRec; + int rc = SQLITE_OK; + + pIter->in.bNoDiscard = 1; + while( SQLITE_ROW==(sessionChangesetNext(pIter, &aRec, &nRec, 0)) ){ + rc = sessionOneChangeToHash(pGrp, pIter, bRebase); + if( rc!=SQLITE_OK ) break; } if( rc==SQLITE_OK ) rc = pIter->rc; @@ -173778,7 +232214,7 @@ static int sessionChangesetToHash( ** ** If xOutput is not NULL, then the changeset/patchset is returned to the ** user via one or more calls to xOutput, as with the other streaming -** interfaces. +** interfaces. ** ** Or, if xOutput is NULL, then (*ppOut) is populated with a pointer to a ** buffer containing the output changeset before this function returns. In @@ -173803,7 +232239,7 @@ static int sessionChangegroupOutput( assert( xOutput==0 || (ppOut==0 && pnOut==0) ); /* Create the serialized output changeset based on the contents of the - ** hash tables attached to the SessionTable objects in list p->pList. + ** hash tables attached to the SessionTable objects in list p->pList. */ for(pTab=pGrp->pList; rc==SQLITE_OK && pTab; pTab=pTab->pNext){ int i; @@ -173816,21 +232252,20 @@ static int sessionChangegroupOutput( sessionAppendByte(&buf, p->op, &rc); sessionAppendByte(&buf, p->bIndirect, &rc); sessionAppendBlob(&buf, p->aRecord, p->nRecord, &rc); + if( rc==SQLITE_OK && xOutput && buf.nBuf>=sessions_strm_chunk_size ){ + rc = xOutput(pOut, buf.aBuf, buf.nBuf); + buf.nBuf = 0; + } } } - - if( rc==SQLITE_OK && xOutput && buf.nBuf>=SESSIONS_STRM_CHUNK_SIZE ){ - rc = xOutput(pOut, buf.aBuf, buf.nBuf); - buf.nBuf = 0; - } } if( rc==SQLITE_OK ){ if( xOutput ){ if( buf.nBuf>0 ) rc = xOutput(pOut, buf.aBuf, buf.nBuf); - }else{ + }else if( ppOut ){ *ppOut = buf.aBuf; - *pnOut = buf.nBuf; + if( pnOut ) *pnOut = buf.nBuf; buf.aBuf = 0; } } @@ -173842,7 +232277,7 @@ static int sessionChangegroupOutput( /* ** Allocate a new, empty, sqlite3_changegroup. */ -SQLITE_API int SQLITE_STDCALL sqlite3changegroup_new(sqlite3_changegroup **pp){ +SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp){ int rc = SQLITE_OK; /* Return code */ sqlite3_changegroup *p; /* New object */ p = (sqlite3_changegroup*)sqlite3_malloc(sizeof(sqlite3_changegroup)); @@ -173855,27 +232290,69 @@ SQLITE_API int SQLITE_STDCALL sqlite3changegroup_new(sqlite3_changegroup **pp){ return rc; } +/* +** Provide a database schema to the changegroup object. +*/ +SQLITE_API int sqlite3changegroup_schema( + sqlite3_changegroup *pGrp, + sqlite3 *db, + const char *zDb +){ + int rc = SQLITE_OK; + + if( pGrp->pList || pGrp->db ){ + /* Cannot add a schema after one or more calls to sqlite3changegroup_add(), + ** or after sqlite3changegroup_schema() has already been called. */ + rc = SQLITE_MISUSE; + }else{ + pGrp->zDb = sqlite3_mprintf("%s", zDb); + if( pGrp->zDb==0 ){ + rc = SQLITE_NOMEM; + }else{ + pGrp->db = db; + } + } + return rc; +} + /* ** Add the changeset currently stored in buffer pData, size nData bytes, ** to changeset-group p. */ -SQLITE_API int SQLITE_STDCALL sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void *pData){ +SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup *pGrp, int nData, void *pData){ sqlite3_changeset_iter *pIter; /* Iterator opened on pData/nData */ int rc; /* Return code */ rc = sqlite3changeset_start(&pIter, nData, pData); if( rc==SQLITE_OK ){ - rc = sessionChangesetToHash(pIter, pGrp); + rc = sessionChangesetToHash(pIter, pGrp, 0); } sqlite3changeset_finalize(pIter); return rc; } +/* +** Add a single change to a changeset-group. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup *pGrp, + sqlite3_changeset_iter *pIter +){ + if( pIter->in.iCurrent==pIter->in.iNext + || pIter->rc!=SQLITE_OK + || pIter->bInvert + ){ + /* Iterator does not point to any valid entry or is an INVERT iterator. */ + return SQLITE_ERROR; + } + return sessionOneChangeToHash(pGrp, pIter, 0); +} + /* ** Obtain a buffer containing a changeset representing the concatenation ** of all changesets added to the group so far. */ -SQLITE_API int SQLITE_STDCALL sqlite3changegroup_output( +SQLITE_API int sqlite3changegroup_output( sqlite3_changegroup *pGrp, int *pnData, void **ppData @@ -173886,7 +232363,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changegroup_output( /* ** Streaming versions of changegroup_add(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changegroup_add_strm( +SQLITE_API int sqlite3changegroup_add_strm( sqlite3_changegroup *pGrp, int (*xInput)(void *pIn, void *pData, int *pnData), void *pIn @@ -173896,7 +232373,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changegroup_add_strm( rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); if( rc==SQLITE_OK ){ - rc = sessionChangesetToHash(pIter, pGrp); + rc = sessionChangesetToHash(pIter, pGrp, 0); } sqlite3changeset_finalize(pIter); return rc; @@ -173905,9 +232382,9 @@ SQLITE_API int SQLITE_STDCALL sqlite3changegroup_add_strm( /* ** Streaming versions of changegroup_output(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changegroup_output_strm( +SQLITE_API int sqlite3changegroup_output_strm( sqlite3_changegroup *pGrp, - int (*xOutput)(void *pOut, const void *pData, int nData), + int (*xOutput)(void *pOut, const void *pData, int nData), void *pOut ){ return sessionChangegroupOutput(pGrp, xOutput, pOut, 0, 0); @@ -173916,17 +232393,19 @@ SQLITE_API int SQLITE_STDCALL sqlite3changegroup_output_strm( /* ** Delete a changegroup object. */ -SQLITE_API void SQLITE_STDCALL sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ +SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup *pGrp){ if( pGrp ){ - sessionDeleteTable(pGrp->pList); + sqlite3_free(pGrp->zDb); + sessionDeleteTable(0, pGrp->pList); + sqlite3_free(pGrp->rec.aBuf); sqlite3_free(pGrp); } } -/* +/* ** Combine two changesets together. */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_concat( +SQLITE_API int sqlite3changeset_concat( int nLeft, /* Number of bytes in lhs input */ void *pLeft, /* Lhs input changeset */ int nRight /* Number of bytes in rhs input */, @@ -173955,7 +232434,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_concat( /* ** Streaming version of sqlite3changeset_concat(). */ -SQLITE_API int SQLITE_STDCALL sqlite3changeset_concat_strm( +SQLITE_API int sqlite3changeset_concat_strm( int (*xInputA)(void *pIn, void *pData, int *pnData), void *pInA, int (*xInputB)(void *pIn, void *pData, int *pnData), @@ -173981,2216 +232460,412 @@ SQLITE_API int SQLITE_STDCALL sqlite3changeset_concat_strm( return rc; } +/* +** Changeset rebaser handle. +*/ +struct sqlite3_rebaser { + sqlite3_changegroup grp; /* Hash table */ +}; + +/* +** Buffers a1 and a2 must both contain a sessions module record nCol +** fields in size. This function appends an nCol sessions module +** record to buffer pBuf that is a copy of a1, except that for +** each field that is undefined in a1[], swap in the field from a2[]. +*/ +static void sessionAppendRecordMerge( + SessionBuffer *pBuf, /* Buffer to append to */ + int nCol, /* Number of columns in each record */ + u8 *a1, int n1, /* Record 1 */ + u8 *a2, int n2, /* Record 2 */ + int *pRc /* IN/OUT: error code */ +){ + sessionBufferGrow(pBuf, n1+n2, pRc); + if( *pRc==SQLITE_OK ){ + int i; + u8 *pOut = &pBuf->aBuf[pBuf->nBuf]; + for(i=0; inBuf = pOut-pBuf->aBuf; + assert( pBuf->nBuf<=pBuf->nAlloc ); + } +} + +/* +** This function is called when rebasing a local UPDATE change against one +** or more remote UPDATE changes. The aRec/nRec buffer contains the current +** old.* and new.* records for the change. The rebase buffer (a single +** record) is in aChange/nChange. The rebased change is appended to buffer +** pBuf. +** +** Rebasing the UPDATE involves: +** +** * Removing any changes to fields for which the corresponding field +** in the rebase buffer is set to "replaced" (type 0xFF). If this +** means the UPDATE change updates no fields, nothing is appended +** to the output buffer. +** +** * For each field modified by the local change for which the +** corresponding field in the rebase buffer is not "undefined" (0x00) +** or "replaced" (0xFF), the old.* value is replaced by the value +** in the rebase buffer. +*/ +static void sessionAppendPartialUpdate( + SessionBuffer *pBuf, /* Append record here */ + sqlite3_changeset_iter *pIter, /* Iterator pointed at local change */ + u8 *aRec, int nRec, /* Local change */ + u8 *aChange, int nChange, /* Record to rebase against */ + int *pRc /* IN/OUT: Return Code */ +){ + sessionBufferGrow(pBuf, 2+nRec+nChange, pRc); + if( *pRc==SQLITE_OK ){ + int bData = 0; + u8 *pOut = &pBuf->aBuf[pBuf->nBuf]; + int i; + u8 *a1 = aRec; + u8 *a2 = aChange; + + *pOut++ = SQLITE_UPDATE; + *pOut++ = pIter->bIndirect; + for(i=0; inCol; i++){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); + if( pIter->abPK[i] || a2[0]==0 ){ + if( !pIter->abPK[i] && a1[0] ) bData = 1; + memcpy(pOut, a1, n1); + pOut += n1; + }else if( a2[0]!=0xFF && a1[0] ){ + bData = 1; + memcpy(pOut, a2, n2); + pOut += n2; + }else{ + *pOut++ = '\0'; + } + a1 += n1; + a2 += n2; + } + if( bData ){ + a2 = aChange; + for(i=0; inCol; i++){ + int n1 = sessionSerialLen(a1); + int n2 = sessionSerialLen(a2); + if( pIter->abPK[i] || a2[0]!=0xFF ){ + memcpy(pOut, a1, n1); + pOut += n1; + }else{ + *pOut++ = '\0'; + } + a1 += n1; + a2 += n2; + } + pBuf->nBuf = (pOut - pBuf->aBuf); + } + } +} + +/* +** pIter is configured to iterate through a changeset. This function rebases +** that changeset according to the current configuration of the rebaser +** object passed as the first argument. If no error occurs and argument xOutput +** is not NULL, then the changeset is returned to the caller by invoking +** xOutput zero or more times and SQLITE_OK returned. Or, if xOutput is NULL, +** then (*ppOut) is set to point to a buffer containing the rebased changeset +** before this function returns. In this case (*pnOut) is set to the size of +** the buffer in bytes. It is the responsibility of the caller to eventually +** free the (*ppOut) buffer using sqlite3_free(). +** +** If an error occurs, an SQLite error code is returned. If ppOut and +** pnOut are not NULL, then the two output parameters are set to 0 before +** returning. +*/ +static int sessionRebase( + sqlite3_rebaser *p, /* Rebaser hash table */ + sqlite3_changeset_iter *pIter, /* Input data */ + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut, /* Context for xOutput callback */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Inverse of pChangeset */ +){ + int rc = SQLITE_OK; + u8 *aRec = 0; + int nRec = 0; + int bNew = 0; + SessionTable *pTab = 0; + SessionBuffer sOut = {0,0,0}; + + while( SQLITE_ROW==sessionChangesetNext(pIter, &aRec, &nRec, &bNew) ){ + SessionChange *pChange = 0; + int bDone = 0; + + if( bNew ){ + const char *zTab = pIter->zTab; + for(pTab=p->grp.pList; pTab; pTab=pTab->pNext){ + if( 0==sqlite3_stricmp(pTab->zName, zTab) ) break; + } + bNew = 0; + + /* A patchset may not be rebased */ + if( pIter->bPatchset ){ + rc = SQLITE_ERROR; + } + + /* Append a table header to the output for this new table */ + sessionAppendByte(&sOut, pIter->bPatchset ? 'P' : 'T', &rc); + sessionAppendVarint(&sOut, pIter->nCol, &rc); + sessionAppendBlob(&sOut, pIter->abPK, pIter->nCol, &rc); + sessionAppendBlob(&sOut,(u8*)pIter->zTab,(int)strlen(pIter->zTab)+1,&rc); + } + + if( pTab && rc==SQLITE_OK ){ + int iHash = sessionChangeHash(pTab, 0, aRec, pTab->nChange); + + for(pChange=pTab->apChange[iHash]; pChange; pChange=pChange->pNext){ + if( sessionChangeEqual(pTab, 0, aRec, 0, pChange->aRecord) ){ + break; + } + } + } + + if( pChange ){ + assert( pChange->op==SQLITE_DELETE || pChange->op==SQLITE_INSERT ); + switch( pIter->op ){ + case SQLITE_INSERT: + if( pChange->op==SQLITE_INSERT ){ + bDone = 1; + if( pChange->bIndirect==0 ){ + sessionAppendByte(&sOut, SQLITE_UPDATE, &rc); + sessionAppendByte(&sOut, pIter->bIndirect, &rc); + sessionAppendBlob(&sOut, pChange->aRecord, pChange->nRecord, &rc); + sessionAppendBlob(&sOut, aRec, nRec, &rc); + } + } + break; + + case SQLITE_UPDATE: + bDone = 1; + if( pChange->op==SQLITE_DELETE ){ + if( pChange->bIndirect==0 ){ + u8 *pCsr = aRec; + sessionSkipRecord(&pCsr, pIter->nCol); + sessionAppendByte(&sOut, SQLITE_INSERT, &rc); + sessionAppendByte(&sOut, pIter->bIndirect, &rc); + sessionAppendRecordMerge(&sOut, pIter->nCol, + pCsr, nRec-(pCsr-aRec), + pChange->aRecord, pChange->nRecord, &rc + ); + } + }else{ + sessionAppendPartialUpdate(&sOut, pIter, + aRec, nRec, pChange->aRecord, pChange->nRecord, &rc + ); + } + break; + + default: + assert( pIter->op==SQLITE_DELETE ); + bDone = 1; + if( pChange->op==SQLITE_INSERT ){ + sessionAppendByte(&sOut, SQLITE_DELETE, &rc); + sessionAppendByte(&sOut, pIter->bIndirect, &rc); + sessionAppendRecordMerge(&sOut, pIter->nCol, + pChange->aRecord, pChange->nRecord, aRec, nRec, &rc + ); + } + break; + } + } + + if( bDone==0 ){ + sessionAppendByte(&sOut, pIter->op, &rc); + sessionAppendByte(&sOut, pIter->bIndirect, &rc); + sessionAppendBlob(&sOut, aRec, nRec, &rc); + } + if( rc==SQLITE_OK && xOutput && sOut.nBuf>sessions_strm_chunk_size ){ + rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); + sOut.nBuf = 0; + } + if( rc ) break; + } + + if( rc!=SQLITE_OK ){ + sqlite3_free(sOut.aBuf); + memset(&sOut, 0, sizeof(sOut)); + } + + if( rc==SQLITE_OK ){ + if( xOutput ){ + if( sOut.nBuf>0 ){ + rc = xOutput(pOut, sOut.aBuf, sOut.nBuf); + } + }else if( ppOut ){ + *ppOut = (void*)sOut.aBuf; + *pnOut = sOut.nBuf; + sOut.aBuf = 0; + } + } + sqlite3_free(sOut.aBuf); + return rc; +} + +/* +** Create a new rebaser object. +*/ +SQLITE_API int sqlite3rebaser_create(sqlite3_rebaser **ppNew){ + int rc = SQLITE_OK; + sqlite3_rebaser *pNew; + + pNew = sqlite3_malloc(sizeof(sqlite3_rebaser)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + memset(pNew, 0, sizeof(sqlite3_rebaser)); + } + *ppNew = pNew; + return rc; +} + +/* +** Call this one or more times to configure a rebaser. +*/ +SQLITE_API int sqlite3rebaser_configure( + sqlite3_rebaser *p, + int nRebase, const void *pRebase +){ + sqlite3_changeset_iter *pIter = 0; /* Iterator opened on pData/nData */ + int rc; /* Return code */ + rc = sqlite3changeset_start(&pIter, nRebase, (void*)pRebase); + if( rc==SQLITE_OK ){ + rc = sessionChangesetToHash(pIter, &p->grp, 1); + } + sqlite3changeset_finalize(pIter); + return rc; +} + +/* +** Rebase a changeset according to current rebaser configuration +*/ +SQLITE_API int sqlite3rebaser_rebase( + sqlite3_rebaser *p, + int nIn, const void *pIn, + int *pnOut, void **ppOut +){ + sqlite3_changeset_iter *pIter = 0; /* Iterator to skip through input */ + int rc = sqlite3changeset_start(&pIter, nIn, (void*)pIn); + + if( rc==SQLITE_OK ){ + rc = sessionRebase(p, pIter, 0, 0, pnOut, ppOut); + sqlite3changeset_finalize(pIter); + } + + return rc; +} + +/* +** Rebase a changeset according to current rebaser configuration +*/ +SQLITE_API int sqlite3rebaser_rebase_strm( + sqlite3_rebaser *p, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +){ + sqlite3_changeset_iter *pIter = 0; /* Iterator to skip through input */ + int rc = sqlite3changeset_start_strm(&pIter, xInput, pIn); + + if( rc==SQLITE_OK ){ + rc = sessionRebase(p, pIter, xOutput, pOut, 0, 0); + sqlite3changeset_finalize(pIter); + } + + return rc; +} + +/* +** Destroy a rebaser object +*/ +SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p){ + if( p ){ + sessionDeleteTable(0, p->grp.pList); + sqlite3_free(p->grp.rec.aBuf); + sqlite3_free(p); + } +} + +/* +** Global configuration +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg){ + int rc = SQLITE_OK; + switch( op ){ + case SQLITE_SESSION_CONFIG_STRMSIZE: { + int *pInt = (int*)pArg; + if( *pInt>0 ){ + sessions_strm_chunk_size = *pInt; + } + *pInt = sessions_strm_chunk_size; + break; + } + default: + rc = SQLITE_MISUSE; + break; + } + return rc; +} + #endif /* SQLITE_ENABLE_SESSION && SQLITE_ENABLE_PREUPDATE_HOOK */ /************** End of sqlite3session.c **************************************/ -/************** Begin file json1.c *******************************************/ -/* -** 2015-08-12 -** -** The author disclaims copyright to this source code. In place of -** a legal notice, here is a blessing: -** -** May you do good and not evil. -** May you find forgiveness for yourself and forgive others. -** May you share freely, never taking more than you give. -** -****************************************************************************** -** -** This SQLite extension implements JSON functions. The interface is -** modeled after MySQL JSON functions: -** -** https://dev.mysql.com/doc/refman/5.7/en/json.html -** -** For the time being, all JSON is stored as pure text. (We might add -** a JSONB type in the future which stores a binary encoding of JSON in -** a BLOB, but there is no support for JSONB in the current implementation. -** This implementation parses JSON text at 250 MB/s, so it is hard to see -** how JSONB might improve on that.) -*/ -#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_JSON1) -#if !defined(_SQLITEINT_H_) -/* #include "sqlite3ext.h" */ -#endif -SQLITE_EXTENSION_INIT1 -/* #include */ -/* #include */ -/* #include */ -/* #include */ - -/* Mark a function parameter as unused, to suppress nuisance compiler -** warnings. */ -#ifndef UNUSED_PARAM -# define UNUSED_PARAM(X) (void)(X) -#endif - -#ifndef LARGEST_INT64 -# define LARGEST_INT64 (0xffffffff|(((sqlite3_int64)0x7fffffff)<<32)) -# define SMALLEST_INT64 (((sqlite3_int64)-1) - LARGEST_INT64) -#endif - -/* -** Versions of isspace(), isalnum() and isdigit() to which it is safe -** to pass signed char values. -*/ -#ifdef sqlite3Isdigit - /* Use the SQLite core versions if this routine is part of the - ** SQLite amalgamation */ -# define safe_isdigit(x) sqlite3Isdigit(x) -# define safe_isalnum(x) sqlite3Isalnum(x) -#else - /* Use the standard library for separate compilation */ -#include /* amalgamator: keep */ -# define safe_isdigit(x) isdigit((unsigned char)(x)) -# define safe_isalnum(x) isalnum((unsigned char)(x)) -#endif - -/* -** Growing our own isspace() routine this way is twice as fast as -** the library isspace() function, resulting in a 7% overall performance -** increase for the parser. (Ubuntu14.10 gcc 4.8.4 x64 with -Os). -*/ -static const char jsonIsSpace[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -}; -#define safe_isspace(x) (jsonIsSpace[(unsigned char)x]) - -#ifndef SQLITE_AMALGAMATION - /* Unsigned integer types. These are already defined in the sqliteInt.h, - ** but the definitions need to be repeated for separate compilation. */ - typedef sqlite3_uint64 u64; - typedef unsigned int u32; - typedef unsigned char u8; -#endif - -/* Objects */ -typedef struct JsonString JsonString; -typedef struct JsonNode JsonNode; -typedef struct JsonParse JsonParse; - -/* An instance of this object represents a JSON string -** under construction. Really, this is a generic string accumulator -** that can be and is used to create strings other than JSON. -*/ -struct JsonString { - sqlite3_context *pCtx; /* Function context - put error messages here */ - char *zBuf; /* Append JSON content here */ - u64 nAlloc; /* Bytes of storage available in zBuf[] */ - u64 nUsed; /* Bytes of zBuf[] currently used */ - u8 bStatic; /* True if zBuf is static space */ - u8 bErr; /* True if an error has been encountered */ - char zSpace[100]; /* Initial static space */ -}; - -/* JSON type values -*/ -#define JSON_NULL 0 -#define JSON_TRUE 1 -#define JSON_FALSE 2 -#define JSON_INT 3 -#define JSON_REAL 4 -#define JSON_STRING 5 -#define JSON_ARRAY 6 -#define JSON_OBJECT 7 - -/* The "subtype" set for JSON values */ -#define JSON_SUBTYPE 74 /* Ascii for "J" */ - -/* -** Names of the various JSON types: -*/ -static const char * const jsonType[] = { - "null", "true", "false", "integer", "real", "text", "array", "object" -}; - -/* Bit values for the JsonNode.jnFlag field -*/ -#define JNODE_RAW 0x01 /* Content is raw, not JSON encoded */ -#define JNODE_ESCAPE 0x02 /* Content is text with \ escapes */ -#define JNODE_REMOVE 0x04 /* Do not output */ -#define JNODE_REPLACE 0x08 /* Replace with JsonNode.iVal */ -#define JNODE_APPEND 0x10 /* More ARRAY/OBJECT entries at u.iAppend */ -#define JNODE_LABEL 0x20 /* Is a label of an object */ - - -/* A single node of parsed JSON -*/ -struct JsonNode { - u8 eType; /* One of the JSON_ type values */ - u8 jnFlags; /* JNODE flags */ - u8 iVal; /* Replacement value when JNODE_REPLACE */ - u32 n; /* Bytes of content, or number of sub-nodes */ - union { - const char *zJContent; /* Content for INT, REAL, and STRING */ - u32 iAppend; /* More terms for ARRAY and OBJECT */ - u32 iKey; /* Key for ARRAY objects in json_tree() */ - } u; -}; - -/* A completely parsed JSON string -*/ -struct JsonParse { - u32 nNode; /* Number of slots of aNode[] used */ - u32 nAlloc; /* Number of slots of aNode[] allocated */ - JsonNode *aNode; /* Array of nodes containing the parse */ - const char *zJson; /* Original JSON string */ - u32 *aUp; /* Index of parent of each node */ - u8 oom; /* Set to true if out of memory */ - u8 nErr; /* Number of errors seen */ -}; - -/************************************************************************** -** Utility routines for dealing with JsonString objects -**************************************************************************/ - -/* Set the JsonString object to an empty string -*/ -static void jsonZero(JsonString *p){ - p->zBuf = p->zSpace; - p->nAlloc = sizeof(p->zSpace); - p->nUsed = 0; - p->bStatic = 1; -} - -/* Initialize the JsonString object -*/ -static void jsonInit(JsonString *p, sqlite3_context *pCtx){ - p->pCtx = pCtx; - p->bErr = 0; - jsonZero(p); -} - - -/* Free all allocated memory and reset the JsonString object back to its -** initial state. -*/ -static void jsonReset(JsonString *p){ - if( !p->bStatic ) sqlite3_free(p->zBuf); - jsonZero(p); -} - - -/* Report an out-of-memory (OOM) condition -*/ -static void jsonOom(JsonString *p){ - p->bErr = 1; - sqlite3_result_error_nomem(p->pCtx); - jsonReset(p); -} - -/* Enlarge pJson->zBuf so that it can hold at least N more bytes. -** Return zero on success. Return non-zero on an OOM error -*/ -static int jsonGrow(JsonString *p, u32 N){ - u64 nTotal = NnAlloc ? p->nAlloc*2 : p->nAlloc+N+10; - char *zNew; - if( p->bStatic ){ - if( p->bErr ) return 1; - zNew = sqlite3_malloc64(nTotal); - if( zNew==0 ){ - jsonOom(p); - return SQLITE_NOMEM; - } - memcpy(zNew, p->zBuf, (size_t)p->nUsed); - p->zBuf = zNew; - p->bStatic = 0; - }else{ - zNew = sqlite3_realloc64(p->zBuf, nTotal); - if( zNew==0 ){ - jsonOom(p); - return SQLITE_NOMEM; - } - p->zBuf = zNew; - } - p->nAlloc = nTotal; - return SQLITE_OK; -} - -/* Append N bytes from zIn onto the end of the JsonString string. -*/ -static void jsonAppendRaw(JsonString *p, const char *zIn, u32 N){ - if( (N+p->nUsed >= p->nAlloc) && jsonGrow(p,N)!=0 ) return; - memcpy(p->zBuf+p->nUsed, zIn, N); - p->nUsed += N; -} - -/* Append formatted text (not to exceed N bytes) to the JsonString. -*/ -static void jsonPrintf(int N, JsonString *p, const char *zFormat, ...){ - va_list ap; - if( (p->nUsed + N >= p->nAlloc) && jsonGrow(p, N) ) return; - va_start(ap, zFormat); - sqlite3_vsnprintf(N, p->zBuf+p->nUsed, zFormat, ap); - va_end(ap); - p->nUsed += (int)strlen(p->zBuf+p->nUsed); -} - -/* Append a single character -*/ -static void jsonAppendChar(JsonString *p, char c){ - if( p->nUsed>=p->nAlloc && jsonGrow(p,1)!=0 ) return; - p->zBuf[p->nUsed++] = c; -} - -/* Append a comma separator to the output buffer, if the previous -** character is not '[' or '{'. -*/ -static void jsonAppendSeparator(JsonString *p){ - char c; - if( p->nUsed==0 ) return; - c = p->zBuf[p->nUsed-1]; - if( c!='[' && c!='{' ) jsonAppendChar(p, ','); -} - -/* Append the N-byte string in zIn to the end of the JsonString string -** under construction. Enclose the string in "..." and escape -** any double-quotes or backslash characters contained within the -** string. -*/ -static void jsonAppendString(JsonString *p, const char *zIn, u32 N){ - u32 i; - if( (N+p->nUsed+2 >= p->nAlloc) && jsonGrow(p,N+2)!=0 ) return; - p->zBuf[p->nUsed++] = '"'; - for(i=0; inUsed+N+3-i > p->nAlloc) && jsonGrow(p,N+3-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - }else if( c<=0x1f ){ - static const char aSpecial[] = { - 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 - }; - assert( sizeof(aSpecial)==32 ); - assert( aSpecial['\b']=='b' ); - assert( aSpecial['\f']=='f' ); - assert( aSpecial['\n']=='n' ); - assert( aSpecial['\r']=='r' ); - assert( aSpecial['\t']=='t' ); - if( aSpecial[c] ){ - c = aSpecial[c]; - goto json_simple_escape; - } - if( (p->nUsed+N+7+i > p->nAlloc) && jsonGrow(p,N+7-i)!=0 ) return; - p->zBuf[p->nUsed++] = '\\'; - p->zBuf[p->nUsed++] = 'u'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0'; - p->zBuf[p->nUsed++] = '0' + (c>>4); - c = "0123456789abcdef"[c&0xf]; - } - p->zBuf[p->nUsed++] = c; - } - p->zBuf[p->nUsed++] = '"'; - assert( p->nUsednAlloc ); -} - -/* -** Append a function parameter value to the JSON string under -** construction. -*/ -static void jsonAppendValue( - JsonString *p, /* Append to this JSON string */ - sqlite3_value *pValue /* Value to append */ -){ - switch( sqlite3_value_type(pValue) ){ - case SQLITE_NULL: { - jsonAppendRaw(p, "null", 4); - break; - } - case SQLITE_INTEGER: - case SQLITE_FLOAT: { - const char *z = (const char*)sqlite3_value_text(pValue); - u32 n = (u32)sqlite3_value_bytes(pValue); - jsonAppendRaw(p, z, n); - break; - } - case SQLITE_TEXT: { - const char *z = (const char*)sqlite3_value_text(pValue); - u32 n = (u32)sqlite3_value_bytes(pValue); - if( sqlite3_value_subtype(pValue)==JSON_SUBTYPE ){ - jsonAppendRaw(p, z, n); - }else{ - jsonAppendString(p, z, n); - } - break; - } - default: { - if( p->bErr==0 ){ - sqlite3_result_error(p->pCtx, "JSON cannot hold BLOB values", -1); - p->bErr = 2; - jsonReset(p); - } - break; - } - } -} - - -/* Make the JSON in p the result of the SQL function. -*/ -static void jsonResult(JsonString *p){ - if( p->bErr==0 ){ - sqlite3_result_text64(p->pCtx, p->zBuf, p->nUsed, - p->bStatic ? SQLITE_TRANSIENT : sqlite3_free, - SQLITE_UTF8); - jsonZero(p); - } - assert( p->bStatic ); -} - -/************************************************************************** -** Utility routines for dealing with JsonNode and JsonParse objects -**************************************************************************/ - -/* -** Return the number of consecutive JsonNode slots need to represent -** the parsed JSON at pNode. The minimum answer is 1. For ARRAY and -** OBJECT types, the number might be larger. -** -** Appended elements are not counted. The value returned is the number -** by which the JsonNode counter should increment in order to go to the -** next peer value. -*/ -static u32 jsonNodeSize(JsonNode *pNode){ - return pNode->eType>=JSON_ARRAY ? pNode->n+1 : 1; -} - -/* -** Reclaim all memory allocated by a JsonParse object. But do not -** delete the JsonParse object itself. -*/ -static void jsonParseReset(JsonParse *pParse){ - sqlite3_free(pParse->aNode); - pParse->aNode = 0; - pParse->nNode = 0; - pParse->nAlloc = 0; - sqlite3_free(pParse->aUp); - pParse->aUp = 0; -} - -/* -** Convert the JsonNode pNode into a pure JSON string and -** append to pOut. Subsubstructure is also included. Return -** the number of JsonNode objects that are encoded. -*/ -static void jsonRenderNode( - JsonNode *pNode, /* The node to render */ - JsonString *pOut, /* Write JSON here */ - sqlite3_value **aReplace /* Replacement values */ -){ - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - jsonAppendRaw(pOut, "null", 4); - break; - } - case JSON_TRUE: { - jsonAppendRaw(pOut, "true", 4); - break; - } - case JSON_FALSE: { - jsonAppendRaw(pOut, "false", 5); - break; - } - case JSON_STRING: { - if( pNode->jnFlags & JNODE_RAW ){ - jsonAppendString(pOut, pNode->u.zJContent, pNode->n); - break; - } - /* Fall through into the next case */ - } - case JSON_REAL: - case JSON_INT: { - jsonAppendRaw(pOut, pNode->u.zJContent, pNode->n); - break; - } - case JSON_ARRAY: { - u32 j = 1; - jsonAppendChar(pOut, '['); - for(;;){ - while( j<=pNode->n ){ - if( pNode[j].jnFlags & (JNODE_REMOVE|JNODE_REPLACE) ){ - if( pNode[j].jnFlags & JNODE_REPLACE ){ - jsonAppendSeparator(pOut); - jsonAppendValue(pOut, aReplace[pNode[j].iVal]); - } - }else{ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - } - j += jsonNodeSize(&pNode[j]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, ']'); - break; - } - case JSON_OBJECT: { - u32 j = 1; - jsonAppendChar(pOut, '{'); - for(;;){ - while( j<=pNode->n ){ - if( (pNode[j+1].jnFlags & JNODE_REMOVE)==0 ){ - jsonAppendSeparator(pOut); - jsonRenderNode(&pNode[j], pOut, aReplace); - jsonAppendChar(pOut, ':'); - if( pNode[j+1].jnFlags & JNODE_REPLACE ){ - jsonAppendValue(pOut, aReplace[pNode[j+1].iVal]); - }else{ - jsonRenderNode(&pNode[j+1], pOut, aReplace); - } - } - j += 1 + jsonNodeSize(&pNode[j+1]); - } - if( (pNode->jnFlags & JNODE_APPEND)==0 ) break; - pNode = &pNode[pNode->u.iAppend]; - j = 1; - } - jsonAppendChar(pOut, '}'); - break; - } - } -} - -/* -** Return a JsonNode and all its descendents as a JSON string. -*/ -static void jsonReturnJson( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - JsonString s; - jsonInit(&s, pCtx); - jsonRenderNode(pNode, &s, aReplace); - jsonResult(&s); - sqlite3_result_subtype(pCtx, JSON_SUBTYPE); -} - -/* -** Make the JsonNode the return value of the function. -*/ -static void jsonReturn( - JsonNode *pNode, /* Node to return */ - sqlite3_context *pCtx, /* Return value for this function */ - sqlite3_value **aReplace /* Array of replacement values */ -){ - switch( pNode->eType ){ - default: { - assert( pNode->eType==JSON_NULL ); - sqlite3_result_null(pCtx); - break; - } - case JSON_TRUE: { - sqlite3_result_int(pCtx, 1); - break; - } - case JSON_FALSE: { - sqlite3_result_int(pCtx, 0); - break; - } - case JSON_INT: { - sqlite3_int64 i = 0; - const char *z = pNode->u.zJContent; - if( z[0]=='-' ){ z++; } - while( z[0]>='0' && z[0]<='9' ){ - unsigned v = *(z++) - '0'; - if( i>=LARGEST_INT64/10 ){ - if( i>LARGEST_INT64/10 ) goto int_as_real; - if( z[0]>='0' && z[0]<='9' ) goto int_as_real; - if( v==9 ) goto int_as_real; - if( v==8 ){ - if( pNode->u.zJContent[0]=='-' ){ - sqlite3_result_int64(pCtx, SMALLEST_INT64); - goto int_done; - }else{ - goto int_as_real; - } - } - } - i = i*10 + v; - } - if( pNode->u.zJContent[0]=='-' ){ i = -i; } - sqlite3_result_int64(pCtx, i); - int_done: - break; - int_as_real: /* fall through to real */; - } - case JSON_REAL: { - double r; -#ifdef SQLITE_AMALGAMATION - const char *z = pNode->u.zJContent; - sqlite3AtoF(z, &r, sqlite3Strlen30(z), SQLITE_UTF8); -#else - r = strtod(pNode->u.zJContent, 0); -#endif - sqlite3_result_double(pCtx, r); - break; - } - case JSON_STRING: { -#if 0 /* Never happens because JNODE_RAW is only set by json_set(), - ** json_insert() and json_replace() and those routines do not - ** call jsonReturn() */ - if( pNode->jnFlags & JNODE_RAW ){ - sqlite3_result_text(pCtx, pNode->u.zJContent, pNode->n, - SQLITE_TRANSIENT); - }else -#endif - assert( (pNode->jnFlags & JNODE_RAW)==0 ); - if( (pNode->jnFlags & JNODE_ESCAPE)==0 ){ - /* JSON formatted without any backslash-escapes */ - sqlite3_result_text(pCtx, pNode->u.zJContent+1, pNode->n-2, - SQLITE_TRANSIENT); - }else{ - /* Translate JSON formatted string into raw text */ - u32 i; - u32 n = pNode->n; - const char *z = pNode->u.zJContent; - char *zOut; - u32 j; - zOut = sqlite3_malloc( n+1 ); - if( zOut==0 ){ - sqlite3_result_error_nomem(pCtx); - break; - } - for(i=1, j=0; i='0' && c<='9' ) v = v*16 + c - '0'; - else if( c>='A' && c<='F' ) v = v*16 + c - 'A' + 10; - else if( c>='a' && c<='f' ) v = v*16 + c - 'a' + 10; - else break; - } - if( v==0 ) break; - if( v<=0x7f ){ - zOut[j++] = (char)v; - }else if( v<=0x7ff ){ - zOut[j++] = (char)(0xc0 | (v>>6)); - zOut[j++] = 0x80 | (v&0x3f); - }else{ - zOut[j++] = (char)(0xe0 | (v>>12)); - zOut[j++] = 0x80 | ((v>>6)&0x3f); - zOut[j++] = 0x80 | (v&0x3f); - } - }else{ - if( c=='b' ){ - c = '\b'; - }else if( c=='f' ){ - c = '\f'; - }else if( c=='n' ){ - c = '\n'; - }else if( c=='r' ){ - c = '\r'; - }else if( c=='t' ){ - c = '\t'; - } - zOut[j++] = c; - } - } - } - zOut[j] = 0; - sqlite3_result_text(pCtx, zOut, j, sqlite3_free); - } - break; - } - case JSON_ARRAY: - case JSON_OBJECT: { - jsonReturnJson(pNode, pCtx, aReplace); - break; - } - } -} - -/* Forward reference */ -static int jsonParseAddNode(JsonParse*,u32,u32,const char*); - -/* -** A macro to hint to the compiler that a function should not be -** inlined. -*/ -#if defined(__GNUC__) -# define JSON_NOINLINE __attribute__((noinline)) -#elif defined(_MSC_VER) && _MSC_VER>=1310 -# define JSON_NOINLINE __declspec(noinline) -#else -# define JSON_NOINLINE -#endif - - -static JSON_NOINLINE int jsonParseAddNodeExpand( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - u32 nNew; - JsonNode *pNew; - assert( pParse->nNode>=pParse->nAlloc ); - if( pParse->oom ) return -1; - nNew = pParse->nAlloc*2 + 10; - pNew = sqlite3_realloc(pParse->aNode, sizeof(JsonNode)*nNew); - if( pNew==0 ){ - pParse->oom = 1; - return -1; - } - pParse->nAlloc = nNew; - pParse->aNode = pNew; - assert( pParse->nNodenAlloc ); - return jsonParseAddNode(pParse, eType, n, zContent); -} - -/* -** Create a new JsonNode instance based on the arguments and append that -** instance to the JsonParse. Return the index in pParse->aNode[] of the -** new node, or -1 if a memory allocation fails. -*/ -static int jsonParseAddNode( - JsonParse *pParse, /* Append the node to this object */ - u32 eType, /* Node type */ - u32 n, /* Content size or sub-node count */ - const char *zContent /* Content */ -){ - JsonNode *p; - if( pParse->nNode>=pParse->nAlloc ){ - return jsonParseAddNodeExpand(pParse, eType, n, zContent); - } - p = &pParse->aNode[pParse->nNode]; - p->eType = (u8)eType; - p->jnFlags = 0; - p->iVal = 0; - p->n = n; - p->u.zJContent = zContent; - return pParse->nNode++; -} - -/* -** Parse a single JSON value which begins at pParse->zJson[i]. Return the -** index of the first character past the end of the value parsed. -** -** Return negative for a syntax error. Special cases: return -2 if the -** first non-whitespace character is '}' and return -3 if the first -** non-whitespace character is ']'. -*/ -static int jsonParseValue(JsonParse *pParse, u32 i){ - char c; - u32 j; - int iThis; - int x; - JsonNode *pNode; - while( safe_isspace(pParse->zJson[i]) ){ i++; } - if( (c = pParse->zJson[i])=='{' ){ - /* Parse object */ - iThis = jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - if( iThis<0 ) return -1; - for(j=i+1;;j++){ - while( safe_isspace(pParse->zJson[j]) ){ j++; } - x = jsonParseValue(pParse, j); - if( x<0 ){ - if( x==(-2) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; - } - if( pParse->oom ) return -1; - pNode = &pParse->aNode[pParse->nNode-1]; - if( pNode->eType!=JSON_STRING ) return -1; - pNode->jnFlags |= JNODE_LABEL; - j = x; - while( safe_isspace(pParse->zJson[j]) ){ j++; } - if( pParse->zJson[j]!=':' ) return -1; - j++; - x = jsonParseValue(pParse, j); - if( x<0 ) return -1; - j = x; - while( safe_isspace(pParse->zJson[j]) ){ j++; } - c = pParse->zJson[j]; - if( c==',' ) continue; - if( c!='}' ) return -1; - break; - } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - return j+1; - }else if( c=='[' ){ - /* Parse array */ - iThis = jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - if( iThis<0 ) return -1; - for(j=i+1;;j++){ - while( safe_isspace(pParse->zJson[j]) ){ j++; } - x = jsonParseValue(pParse, j); - if( x<0 ){ - if( x==(-3) && pParse->nNode==(u32)iThis+1 ) return j+1; - return -1; - } - j = x; - while( safe_isspace(pParse->zJson[j]) ){ j++; } - c = pParse->zJson[j]; - if( c==',' ) continue; - if( c!=']' ) return -1; - break; - } - pParse->aNode[iThis].n = pParse->nNode - (u32)iThis - 1; - return j+1; - }else if( c=='"' ){ - /* Parse string */ - u8 jnFlags = 0; - j = i+1; - for(;;){ - c = pParse->zJson[j]; - if( c==0 ) return -1; - if( c=='\\' ){ - c = pParse->zJson[++j]; - if( c==0 ) return -1; - jnFlags = JNODE_ESCAPE; - }else if( c=='"' ){ - break; - } - j++; - } - jsonParseAddNode(pParse, JSON_STRING, j+1-i, &pParse->zJson[i]); - if( !pParse->oom ) pParse->aNode[pParse->nNode-1].jnFlags = jnFlags; - return j+1; - }else if( c=='n' - && strncmp(pParse->zJson+i,"null",4)==0 - && !safe_isalnum(pParse->zJson[i+4]) ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return i+4; - }else if( c=='t' - && strncmp(pParse->zJson+i,"true",4)==0 - && !safe_isalnum(pParse->zJson[i+4]) ){ - jsonParseAddNode(pParse, JSON_TRUE, 0, 0); - return i+4; - }else if( c=='f' - && strncmp(pParse->zJson+i,"false",5)==0 - && !safe_isalnum(pParse->zJson[i+5]) ){ - jsonParseAddNode(pParse, JSON_FALSE, 0, 0); - return i+5; - }else if( c=='-' || (c>='0' && c<='9') ){ - /* Parse number */ - u8 seenDP = 0; - u8 seenE = 0; - j = i+1; - for(;; j++){ - c = pParse->zJson[j]; - if( c>='0' && c<='9' ) continue; - if( c=='.' ){ - if( pParse->zJson[j-1]=='-' ) return -1; - if( seenDP ) return -1; - seenDP = 1; - continue; - } - if( c=='e' || c=='E' ){ - if( pParse->zJson[j-1]<'0' ) return -1; - if( seenE ) return -1; - seenDP = seenE = 1; - c = pParse->zJson[j+1]; - if( c=='+' || c=='-' ){ - j++; - c = pParse->zJson[j+1]; - } - if( c<'0' || c>'9' ) return -1; - continue; - } - break; - } - if( pParse->zJson[j-1]<'0' ) return -1; - jsonParseAddNode(pParse, seenDP ? JSON_REAL : JSON_INT, - j - i, &pParse->zJson[i]); - return j; - }else if( c=='}' ){ - return -2; /* End of {...} */ - }else if( c==']' ){ - return -3; /* End of [...] */ - }else if( c==0 ){ - return 0; /* End of file */ - }else{ - return -1; /* Syntax error */ - } -} - -/* -** Parse a complete JSON string. Return 0 on success or non-zero if there -** are any errors. If an error occurs, free all memory associated with -** pParse. -** -** pParse is uninitialized when this routine is called. -*/ -static int jsonParse( - JsonParse *pParse, /* Initialize and fill this JsonParse object */ - sqlite3_context *pCtx, /* Report errors here */ - const char *zJson /* Input JSON text to be parsed */ -){ - int i; - memset(pParse, 0, sizeof(*pParse)); - if( zJson==0 ) return 1; - pParse->zJson = zJson; - i = jsonParseValue(pParse, 0); - if( pParse->oom ) i = -1; - if( i>0 ){ - while( safe_isspace(zJson[i]) ) i++; - if( zJson[i] ) i = -1; - } - if( i<=0 ){ - if( pCtx!=0 ){ - if( pParse->oom ){ - sqlite3_result_error_nomem(pCtx); - }else{ - sqlite3_result_error(pCtx, "malformed JSON", -1); - } - } - jsonParseReset(pParse); - return 1; - } - return 0; -} - -/* Mark node i of pParse as being a child of iParent. Call recursively -** to fill in all the descendants of node i. -*/ -static void jsonParseFillInParentage(JsonParse *pParse, u32 i, u32 iParent){ - JsonNode *pNode = &pParse->aNode[i]; - u32 j; - pParse->aUp[i] = iParent; - switch( pNode->eType ){ - case JSON_ARRAY: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j)){ - jsonParseFillInParentage(pParse, i+j, i); - } - break; - } - case JSON_OBJECT: { - for(j=1; j<=pNode->n; j += jsonNodeSize(pNode+j+1)+1){ - pParse->aUp[i+j] = i; - jsonParseFillInParentage(pParse, i+j+1, i); - } - break; - } - default: { - break; - } - } -} - -/* -** Compute the parentage of all nodes in a completed parse. -*/ -static int jsonParseFindParents(JsonParse *pParse){ - u32 *aUp; - assert( pParse->aUp==0 ); - aUp = pParse->aUp = sqlite3_malloc( sizeof(u32)*pParse->nNode ); - if( aUp==0 ){ - pParse->oom = 1; - return SQLITE_NOMEM; - } - jsonParseFillInParentage(pParse, 0, 0); - return SQLITE_OK; -} - -/* -** Compare the OBJECT label at pNode against zKey,nKey. Return true on -** a match. -*/ -static int jsonLabelCompare(JsonNode *pNode, const char *zKey, u32 nKey){ - if( pNode->jnFlags & JNODE_RAW ){ - if( pNode->n!=nKey ) return 0; - return strncmp(pNode->u.zJContent, zKey, nKey)==0; - }else{ - if( pNode->n!=nKey+2 ) return 0; - return strncmp(pNode->u.zJContent+1, zKey, nKey)==0; - } -} - -/* forward declaration */ -static JsonNode *jsonLookupAppend(JsonParse*,const char*,int*,const char**); - -/* -** Search along zPath to find the node specified. Return a pointer -** to that node, or NULL if zPath is malformed or if there is no such -** node. -** -** If pApnd!=0, then try to append new nodes to complete zPath if it is -** possible to do so and if no existing node corresponds to zPath. If -** new nodes are appended *pApnd is set to 1. -*/ -static JsonNode *jsonLookupStep( - JsonParse *pParse, /* The JSON to search */ - u32 iRoot, /* Begin the search at this node */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - const char **pzErr /* Make *pzErr point to any syntax error in zPath */ -){ - u32 i, j, nKey; - const char *zKey; - JsonNode *pRoot = &pParse->aNode[iRoot]; - if( zPath[0]==0 ) return pRoot; - if( zPath[0]=='.' ){ - if( pRoot->eType!=JSON_OBJECT ) return 0; - zPath++; - if( zPath[0]=='"' ){ - zKey = zPath + 1; - for(i=1; zPath[i] && zPath[i]!='"'; i++){} - nKey = i-1; - if( zPath[i] ){ - i++; - }else{ - *pzErr = zPath; - return 0; - } - }else{ - zKey = zPath; - for(i=0; zPath[i] && zPath[i]!='.' && zPath[i]!='['; i++){} - nKey = i; - } - if( nKey==0 ){ - *pzErr = zPath; - return 0; - } - j = 1; - for(;;){ - while( j<=pRoot->n ){ - if( jsonLabelCompare(pRoot+j, zKey, nKey) ){ - return jsonLookupStep(pParse, iRoot+j+1, &zPath[i], pApnd, pzErr); - } - j++; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( pApnd ){ - u32 iStart, iLabel; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_OBJECT, 2, 0); - iLabel = jsonParseAddNode(pParse, JSON_STRING, i, zPath); - zPath += i; - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - pParse->aNode[iLabel].jnFlags |= JNODE_RAW; - } - return pNode; - } - }else if( zPath[0]=='[' && safe_isdigit(zPath[1]) ){ - if( pRoot->eType!=JSON_ARRAY ) return 0; - i = 0; - j = 1; - while( safe_isdigit(zPath[j]) ){ - i = i*10 + zPath[j] - '0'; - j++; - } - if( zPath[j]!=']' ){ - *pzErr = zPath; - return 0; - } - zPath += j + 1; - j = 1; - for(;;){ - while( j<=pRoot->n && (i>0 || (pRoot[j].jnFlags & JNODE_REMOVE)!=0) ){ - if( (pRoot[j].jnFlags & JNODE_REMOVE)==0 ) i--; - j += jsonNodeSize(&pRoot[j]); - } - if( (pRoot->jnFlags & JNODE_APPEND)==0 ) break; - iRoot += pRoot->u.iAppend; - pRoot = &pParse->aNode[iRoot]; - j = 1; - } - if( j<=pRoot->n ){ - return jsonLookupStep(pParse, iRoot+j, zPath, pApnd, pzErr); - } - if( i==0 && pApnd ){ - u32 iStart; - JsonNode *pNode; - iStart = jsonParseAddNode(pParse, JSON_ARRAY, 1, 0); - pNode = jsonLookupAppend(pParse, zPath, pApnd, pzErr); - if( pParse->oom ) return 0; - if( pNode ){ - pRoot = &pParse->aNode[iRoot]; - pRoot->u.iAppend = iStart - iRoot; - pRoot->jnFlags |= JNODE_APPEND; - } - return pNode; - } - }else{ - *pzErr = zPath; - } - return 0; -} - -/* -** Append content to pParse that will complete zPath. Return a pointer -** to the inserted node, or return NULL if the append fails. -*/ -static JsonNode *jsonLookupAppend( - JsonParse *pParse, /* Append content to the JSON parse */ - const char *zPath, /* Description of content to append */ - int *pApnd, /* Set this flag to 1 */ - const char **pzErr /* Make this point to any syntax error */ -){ - *pApnd = 1; - if( zPath[0]==0 ){ - jsonParseAddNode(pParse, JSON_NULL, 0, 0); - return pParse->oom ? 0 : &pParse->aNode[pParse->nNode-1]; - } - if( zPath[0]=='.' ){ - jsonParseAddNode(pParse, JSON_OBJECT, 0, 0); - }else if( strncmp(zPath,"[0]",3)==0 ){ - jsonParseAddNode(pParse, JSON_ARRAY, 0, 0); - }else{ - return 0; - } - if( pParse->oom ) return 0; - return jsonLookupStep(pParse, pParse->nNode-1, zPath, pApnd, pzErr); -} - -/* -** Return the text of a syntax error message on a JSON path. Space is -** obtained from sqlite3_malloc(). -*/ -static char *jsonPathSyntaxError(const char *zErr){ - return sqlite3_mprintf("JSON path error near '%q'", zErr); -} - -/* -** Do a node lookup using zPath. Return a pointer to the node on success. -** Return NULL if not found or if there is an error. -** -** On an error, write an error message into pCtx and increment the -** pParse->nErr counter. -** -** If pApnd!=NULL then try to append missing nodes and set *pApnd = 1 if -** nodes are appended. -*/ -static JsonNode *jsonLookup( - JsonParse *pParse, /* The JSON to search */ - const char *zPath, /* The path to search */ - int *pApnd, /* Append nodes to complete path if not NULL */ - sqlite3_context *pCtx /* Report errors here, if not NULL */ -){ - const char *zErr = 0; - JsonNode *pNode = 0; - char *zMsg; - - if( zPath==0 ) return 0; - if( zPath[0]!='$' ){ - zErr = zPath; - goto lookup_err; - } - zPath++; - pNode = jsonLookupStep(pParse, 0, zPath, pApnd, &zErr); - if( zErr==0 ) return pNode; - -lookup_err: - pParse->nErr++; - assert( zErr!=0 && pCtx!=0 ); - zMsg = jsonPathSyntaxError(zErr); - if( zMsg ){ - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); - }else{ - sqlite3_result_error_nomem(pCtx); - } - return 0; -} - - -/* -** Report the wrong number of arguments for json_insert(), json_replace() -** or json_set(). -*/ -static void jsonWrongNumArgs( - sqlite3_context *pCtx, - const char *zFuncName -){ - char *zMsg = sqlite3_mprintf("json_%s() needs an odd number of arguments", - zFuncName); - sqlite3_result_error(pCtx, zMsg, -1); - sqlite3_free(zMsg); -} - - -/**************************************************************************** -** SQL functions used for testing and debugging -****************************************************************************/ - -#ifdef SQLITE_DEBUG -/* -** The json_parse(JSON) function returns a string which describes -** a parse of the JSON provided. Or it returns NULL if JSON is not -** well-formed. -*/ -static void jsonParseFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonString s; /* Output string - not real JSON */ - JsonParse x; /* The parse */ - u32 i; - - assert( argc==1 ); - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonParseFindParents(&x); - jsonInit(&s, ctx); - for(i=0; ieType==JSON_ARRAY ){ - assert( (pNode->jnFlags & JNODE_APPEND)==0 ); - for(i=1; i<=pNode->n; n++){ - i += jsonNodeSize(&pNode[i]); - } - } - if( x.nErr==0 ) sqlite3_result_int64(ctx, n); - jsonParseReset(&x); -} - -/* -** json_extract(JSON, PATH, ...) -** -** Return the element described by PATH. Return NULL if there is no -** PATH element. If there are multiple PATHs, then return a JSON array -** with the result from each path. Throw an error if the JSON or any PATH -** is malformed. -*/ -static void jsonExtractFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - JsonString jx; - int i; - - if( argc<2 ) return; - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '['); - for(i=1; i2 ){ - jsonAppendSeparator(&jx); - if( pNode ){ - jsonRenderNode(pNode, &jx, 0); - }else{ - jsonAppendRaw(&jx, "null", 4); - } - }else if( pNode ){ - jsonReturn(pNode, ctx, 0); - } - } - if( argc>2 && i==argc ){ - jsonAppendChar(&jx, ']'); - jsonResult(&jx); - sqlite3_result_subtype(ctx, JSON_SUBTYPE); - } - jsonReset(&jx); - jsonParseReset(&x); -} - -/* -** Implementation of the json_object(NAME,VALUE,...) function. Return a JSON -** object that contains all name/value given in arguments. Or if any name -** is not a string or if any value is a BLOB, throw an error. -*/ -static void jsonObjectFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - int i; - JsonString jx; - const char *z; - u32 n; - - if( argc&1 ){ - sqlite3_result_error(ctx, "json_object() requires an even number " - "of arguments", -1); - return; - } - jsonInit(&jx, ctx); - jsonAppendChar(&jx, '{'); - for(i=0; ijnFlags |= JNODE_REMOVE; - } - if( (x.aNode[0].jnFlags & JNODE_REMOVE)==0 ){ - jsonReturnJson(x.aNode, ctx, 0); - } -remove_done: - jsonParseReset(&x); -} - -/* -** json_replace(JSON, PATH, VALUE, ...) -** -** Replace the value at PATH with VALUE. If PATH does not already exist, -** this routine is a no-op. If JSON or PATH is malformed, throw an error. -*/ -static void jsonReplaceFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - - if( argc<1 ) return; - if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, "replace"); - return; - } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - pNode = jsonLookup(&x, zPath, 0, ctx); - if( x.nErr ) goto replace_err; - if( pNode ){ - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -replace_err: - jsonParseReset(&x); -} - -/* -** json_set(JSON, PATH, VALUE, ...) -** -** Set the value at PATH to VALUE. Create the PATH if it does not already -** exist. Overwrite existing values that do exist. -** If JSON or PATH is malformed, throw an error. -** -** json_insert(JSON, PATH, VALUE, ...) -** -** Create PATH and initialize it to VALUE. If PATH already exists, this -** routine is a no-op. If JSON or PATH is malformed, throw an error. -*/ -static void jsonSetFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - JsonNode *pNode; - const char *zPath; - u32 i; - int bApnd; - int bIsSet = *(int*)sqlite3_user_data(ctx); - - if( argc<1 ) return; - if( (argc&1)==0 ) { - jsonWrongNumArgs(ctx, bIsSet ? "set" : "insert"); - return; - } - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - for(i=1; i<(u32)argc; i+=2){ - zPath = (const char*)sqlite3_value_text(argv[i]); - bApnd = 0; - pNode = jsonLookup(&x, zPath, &bApnd, ctx); - if( x.oom ){ - sqlite3_result_error_nomem(ctx); - goto jsonSetDone; - }else if( x.nErr ){ - goto jsonSetDone; - }else if( pNode && (bApnd || bIsSet) ){ - pNode->jnFlags |= (u8)JNODE_REPLACE; - pNode->iVal = (u8)(i+1); - } - } - if( x.aNode[0].jnFlags & JNODE_REPLACE ){ - sqlite3_result_value(ctx, argv[x.aNode[0].iVal]); - }else{ - jsonReturnJson(x.aNode, ctx, argv); - } -jsonSetDone: - jsonParseReset(&x); -} - -/* -** json_type(JSON) -** json_type(JSON, PATH) -** -** Return the top-level "type" of a JSON string. Throw an error if -** either the JSON or PATH inputs are not well-formed. -*/ -static void jsonTypeFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - const char *zPath; - JsonNode *pNode; - - if( jsonParse(&x, ctx, (const char*)sqlite3_value_text(argv[0])) ) return; - assert( x.nNode ); - if( argc==2 ){ - zPath = (const char*)sqlite3_value_text(argv[1]); - pNode = jsonLookup(&x, zPath, 0, ctx); - }else{ - pNode = x.aNode; - } - if( pNode ){ - sqlite3_result_text(ctx, jsonType[pNode->eType], -1, SQLITE_STATIC); - } - jsonParseReset(&x); -} - -/* -** json_valid(JSON) -** -** Return 1 if JSON is a well-formed JSON string according to RFC-7159. -** Return 0 otherwise. -*/ -static void jsonValidFunc( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonParse x; /* The parse */ - int rc = 0; - - UNUSED_PARAM(argc); - if( jsonParse(&x, 0, (const char*)sqlite3_value_text(argv[0]))==0 ){ - rc = 1; - } - jsonParseReset(&x); - sqlite3_result_int(ctx, rc); -} - - -/**************************************************************************** -** Aggregate SQL function implementations -****************************************************************************/ -/* -** json_group_array(VALUE) -** -** Return a JSON array composed of all values in the aggregate. -*/ -static void jsonArrayStep( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonString *pStr; - UNUSED_PARAM(argc); - pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); - if( pStr ){ - if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); - jsonAppendChar(pStr, '['); - }else{ - jsonAppendChar(pStr, ','); - pStr->pCtx = ctx; - } - jsonAppendValue(pStr, argv[0]); - } -} -static void jsonArrayFinal(sqlite3_context *ctx){ - JsonString *pStr; - pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); - if( pStr ){ - pStr->pCtx = ctx; - jsonAppendChar(pStr, ']'); - if( pStr->bErr ){ - if( pStr->bErr==1 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); - }else{ - sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); - pStr->bStatic = 1; - } - }else{ - sqlite3_result_text(ctx, "[]", 2, SQLITE_STATIC); - } - sqlite3_result_subtype(ctx, JSON_SUBTYPE); -} - -/* -** json_group_obj(NAME,VALUE) -** -** Return a JSON object composed of all names and values in the aggregate. -*/ -static void jsonObjectStep( - sqlite3_context *ctx, - int argc, - sqlite3_value **argv -){ - JsonString *pStr; - const char *z; - u32 n; - UNUSED_PARAM(argc); - pStr = (JsonString*)sqlite3_aggregate_context(ctx, sizeof(*pStr)); - if( pStr ){ - if( pStr->zBuf==0 ){ - jsonInit(pStr, ctx); - jsonAppendChar(pStr, '{'); - }else{ - jsonAppendChar(pStr, ','); - pStr->pCtx = ctx; - } - z = (const char*)sqlite3_value_text(argv[0]); - n = (u32)sqlite3_value_bytes(argv[0]); - jsonAppendString(pStr, z, n); - jsonAppendChar(pStr, ':'); - jsonAppendValue(pStr, argv[1]); - } -} -static void jsonObjectFinal(sqlite3_context *ctx){ - JsonString *pStr; - pStr = (JsonString*)sqlite3_aggregate_context(ctx, 0); - if( pStr ){ - jsonAppendChar(pStr, '}'); - if( pStr->bErr ){ - if( pStr->bErr==0 ) sqlite3_result_error_nomem(ctx); - assert( pStr->bStatic ); - }else{ - sqlite3_result_text(ctx, pStr->zBuf, pStr->nUsed, - pStr->bStatic ? SQLITE_TRANSIENT : sqlite3_free); - pStr->bStatic = 1; - } - }else{ - sqlite3_result_text(ctx, "{}", 2, SQLITE_STATIC); - } - sqlite3_result_subtype(ctx, JSON_SUBTYPE); -} - - -#ifndef SQLITE_OMIT_VIRTUALTABLE -/**************************************************************************** -** The json_each virtual table -****************************************************************************/ -typedef struct JsonEachCursor JsonEachCursor; -struct JsonEachCursor { - sqlite3_vtab_cursor base; /* Base class - must be first */ - u32 iRowid; /* The rowid */ - u32 iBegin; /* The first node of the scan */ - u32 i; /* Index in sParse.aNode[] of current row */ - u32 iEnd; /* EOF when i equals or exceeds this value */ - u8 eType; /* Type of top-level element */ - u8 bRecursive; /* True for json_tree(). False for json_each() */ - char *zJson; /* Input JSON */ - char *zRoot; /* Path by which to filter zJson */ - JsonParse sParse; /* Parse of the input JSON */ -}; - -/* Constructor for the json_each virtual table */ -static int jsonEachConnect( - sqlite3 *db, - void *pAux, - int argc, const char *const*argv, - sqlite3_vtab **ppVtab, - char **pzErr -){ - sqlite3_vtab *pNew; - int rc; - -/* Column numbers */ -#define JEACH_KEY 0 -#define JEACH_VALUE 1 -#define JEACH_TYPE 2 -#define JEACH_ATOM 3 -#define JEACH_ID 4 -#define JEACH_PARENT 5 -#define JEACH_FULLKEY 6 -#define JEACH_PATH 7 -#define JEACH_JSON 8 -#define JEACH_ROOT 9 - - UNUSED_PARAM(pzErr); - UNUSED_PARAM(argv); - UNUSED_PARAM(argc); - UNUSED_PARAM(pAux); - rc = sqlite3_declare_vtab(db, - "CREATE TABLE x(key,value,type,atom,id,parent,fullkey,path," - "json HIDDEN,root HIDDEN)"); - if( rc==SQLITE_OK ){ - pNew = *ppVtab = sqlite3_malloc( sizeof(*pNew) ); - if( pNew==0 ) return SQLITE_NOMEM; - memset(pNew, 0, sizeof(*pNew)); - } - return rc; -} - -/* destructor for json_each virtual table */ -static int jsonEachDisconnect(sqlite3_vtab *pVtab){ - sqlite3_free(pVtab); - return SQLITE_OK; -} - -/* constructor for a JsonEachCursor object for json_each(). */ -static int jsonEachOpenEach(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - JsonEachCursor *pCur; - - UNUSED_PARAM(p); - pCur = sqlite3_malloc( sizeof(*pCur) ); - if( pCur==0 ) return SQLITE_NOMEM; - memset(pCur, 0, sizeof(*pCur)); - *ppCursor = &pCur->base; - return SQLITE_OK; -} - -/* constructor for a JsonEachCursor object for json_tree(). */ -static int jsonEachOpenTree(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ - int rc = jsonEachOpenEach(p, ppCursor); - if( rc==SQLITE_OK ){ - JsonEachCursor *pCur = (JsonEachCursor*)*ppCursor; - pCur->bRecursive = 1; - } - return rc; -} - -/* Reset a JsonEachCursor back to its original state. Free any memory -** held. */ -static void jsonEachCursorReset(JsonEachCursor *p){ - sqlite3_free(p->zJson); - sqlite3_free(p->zRoot); - jsonParseReset(&p->sParse); - p->iRowid = 0; - p->i = 0; - p->iEnd = 0; - p->eType = 0; - p->zJson = 0; - p->zRoot = 0; -} - -/* Destructor for a jsonEachCursor object */ -static int jsonEachClose(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - jsonEachCursorReset(p); - sqlite3_free(cur); - return SQLITE_OK; -} - -/* Return TRUE if the jsonEachCursor object has been advanced off the end -** of the JSON object */ -static int jsonEachEof(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - return p->i >= p->iEnd; -} - -/* Advance the cursor to the next element for json_tree() */ -static int jsonEachNext(sqlite3_vtab_cursor *cur){ - JsonEachCursor *p = (JsonEachCursor*)cur; - if( p->bRecursive ){ - if( p->sParse.aNode[p->i].jnFlags & JNODE_LABEL ) p->i++; - p->i++; - p->iRowid++; - if( p->iiEnd ){ - u32 iUp = p->sParse.aUp[p->i]; - JsonNode *pUp = &p->sParse.aNode[iUp]; - p->eType = pUp->eType; - if( pUp->eType==JSON_ARRAY ){ - if( iUp==p->i-1 ){ - pUp->u.iKey = 0; - }else{ - pUp->u.iKey++; - } - } - } - }else{ - switch( p->eType ){ - case JSON_ARRAY: { - p->i += jsonNodeSize(&p->sParse.aNode[p->i]); - p->iRowid++; - break; - } - case JSON_OBJECT: { - p->i += 1 + jsonNodeSize(&p->sParse.aNode[p->i+1]); - p->iRowid++; - break; - } - default: { - p->i = p->iEnd; - break; - } - } - } - return SQLITE_OK; -} - -/* Append the name of the path for element i to pStr -*/ -static void jsonEachComputePath( - JsonEachCursor *p, /* The cursor */ - JsonString *pStr, /* Write the path here */ - u32 i /* Path to this element */ -){ - JsonNode *pNode, *pUp; - u32 iUp; - if( i==0 ){ - jsonAppendChar(pStr, '$'); - return; - } - iUp = p->sParse.aUp[i]; - jsonEachComputePath(p, pStr, iUp); - pNode = &p->sParse.aNode[i]; - pUp = &p->sParse.aNode[iUp]; - if( pUp->eType==JSON_ARRAY ){ - jsonPrintf(30, pStr, "[%d]", pUp->u.iKey); - }else{ - assert( pUp->eType==JSON_OBJECT ); - if( (pNode->jnFlags & JNODE_LABEL)==0 ) pNode--; - assert( pNode->eType==JSON_STRING ); - assert( pNode->jnFlags & JNODE_LABEL ); - jsonPrintf(pNode->n+1, pStr, ".%.*s", pNode->n-2, pNode->u.zJContent+1); - } -} - -/* Return the value of a column */ -static int jsonEachColumn( - sqlite3_vtab_cursor *cur, /* The cursor */ - sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ - int i /* Which column to return */ -){ - JsonEachCursor *p = (JsonEachCursor*)cur; - JsonNode *pThis = &p->sParse.aNode[p->i]; - switch( i ){ - case JEACH_KEY: { - if( p->i==0 ) break; - if( p->eType==JSON_OBJECT ){ - jsonReturn(pThis, ctx, 0); - }else if( p->eType==JSON_ARRAY ){ - u32 iKey; - if( p->bRecursive ){ - if( p->iRowid==0 ) break; - iKey = p->sParse.aNode[p->sParse.aUp[p->i]].u.iKey; - }else{ - iKey = p->iRowid; - } - sqlite3_result_int64(ctx, (sqlite3_int64)iKey); - } - break; - } - case JEACH_VALUE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - jsonReturn(pThis, ctx, 0); - break; - } - case JEACH_TYPE: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - sqlite3_result_text(ctx, jsonType[pThis->eType], -1, SQLITE_STATIC); - break; - } - case JEACH_ATOM: { - if( pThis->jnFlags & JNODE_LABEL ) pThis++; - if( pThis->eType>=JSON_ARRAY ) break; - jsonReturn(pThis, ctx, 0); - break; - } - case JEACH_ID: { - sqlite3_result_int64(ctx, - (sqlite3_int64)p->i + ((pThis->jnFlags & JNODE_LABEL)!=0)); - break; - } - case JEACH_PARENT: { - if( p->i>p->iBegin && p->bRecursive ){ - sqlite3_result_int64(ctx, (sqlite3_int64)p->sParse.aUp[p->i]); - } - break; - } - case JEACH_FULLKEY: { - JsonString x; - jsonInit(&x, ctx); - if( p->bRecursive ){ - jsonEachComputePath(p, &x, p->i); - }else{ - if( p->zRoot ){ - jsonAppendRaw(&x, p->zRoot, (int)strlen(p->zRoot)); - }else{ - jsonAppendChar(&x, '$'); - } - if( p->eType==JSON_ARRAY ){ - jsonPrintf(30, &x, "[%d]", p->iRowid); - }else{ - jsonPrintf(pThis->n, &x, ".%.*s", pThis->n-2, pThis->u.zJContent+1); - } - } - jsonResult(&x); - break; - } - case JEACH_PATH: { - if( p->bRecursive ){ - JsonString x; - jsonInit(&x, ctx); - jsonEachComputePath(p, &x, p->sParse.aUp[p->i]); - jsonResult(&x); - break; - } - /* For json_each() path and root are the same so fall through - ** into the root case */ - } - case JEACH_ROOT: { - const char *zRoot = p->zRoot; - if( zRoot==0 ) zRoot = "$"; - sqlite3_result_text(ctx, zRoot, -1, SQLITE_STATIC); - break; - } - case JEACH_JSON: { - assert( i==JEACH_JSON ); - sqlite3_result_text(ctx, p->sParse.zJson, -1, SQLITE_STATIC); - break; - } - } - return SQLITE_OK; -} - -/* Return the current rowid value */ -static int jsonEachRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ - JsonEachCursor *p = (JsonEachCursor*)cur; - *pRowid = p->iRowid; - return SQLITE_OK; -} - -/* The query strategy is to look for an equality constraint on the json -** column. Without such a constraint, the table cannot operate. idxNum is -** 1 if the constraint is found, 3 if the constraint and zRoot are found, -** and 0 otherwise. -*/ -static int jsonEachBestIndex( - sqlite3_vtab *tab, - sqlite3_index_info *pIdxInfo -){ - int i; - int jsonIdx = -1; - int rootIdx = -1; - const struct sqlite3_index_constraint *pConstraint; - - UNUSED_PARAM(tab); - pConstraint = pIdxInfo->aConstraint; - for(i=0; inConstraint; i++, pConstraint++){ - if( pConstraint->usable==0 ) continue; - if( pConstraint->op!=SQLITE_INDEX_CONSTRAINT_EQ ) continue; - switch( pConstraint->iColumn ){ - case JEACH_JSON: jsonIdx = i; break; - case JEACH_ROOT: rootIdx = i; break; - default: /* no-op */ break; - } - } - if( jsonIdx<0 ){ - pIdxInfo->idxNum = 0; - pIdxInfo->estimatedCost = 1e99; - }else{ - pIdxInfo->estimatedCost = 1.0; - pIdxInfo->aConstraintUsage[jsonIdx].argvIndex = 1; - pIdxInfo->aConstraintUsage[jsonIdx].omit = 1; - if( rootIdx<0 ){ - pIdxInfo->idxNum = 1; - }else{ - pIdxInfo->aConstraintUsage[rootIdx].argvIndex = 2; - pIdxInfo->aConstraintUsage[rootIdx].omit = 1; - pIdxInfo->idxNum = 3; - } - } - return SQLITE_OK; -} - -/* Start a search on a new JSON string */ -static int jsonEachFilter( - sqlite3_vtab_cursor *cur, - int idxNum, const char *idxStr, - int argc, sqlite3_value **argv -){ - JsonEachCursor *p = (JsonEachCursor*)cur; - const char *z; - const char *zRoot = 0; - sqlite3_int64 n; - - UNUSED_PARAM(idxStr); - UNUSED_PARAM(argc); - jsonEachCursorReset(p); - if( idxNum==0 ) return SQLITE_OK; - z = (const char*)sqlite3_value_text(argv[0]); - if( z==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[0]); - p->zJson = sqlite3_malloc64( n+1 ); - if( p->zJson==0 ) return SQLITE_NOMEM; - memcpy(p->zJson, z, (size_t)n+1); - if( jsonParse(&p->sParse, 0, p->zJson) ){ - int rc = SQLITE_NOMEM; - if( p->sParse.oom==0 ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = sqlite3_mprintf("malformed JSON"); - if( cur->pVtab->zErrMsg ) rc = SQLITE_ERROR; - } - jsonEachCursorReset(p); - return rc; - }else if( p->bRecursive && jsonParseFindParents(&p->sParse) ){ - jsonEachCursorReset(p); - return SQLITE_NOMEM; - }else{ - JsonNode *pNode = 0; - if( idxNum==3 ){ - const char *zErr = 0; - zRoot = (const char*)sqlite3_value_text(argv[1]); - if( zRoot==0 ) return SQLITE_OK; - n = sqlite3_value_bytes(argv[1]); - p->zRoot = sqlite3_malloc64( n+1 ); - if( p->zRoot==0 ) return SQLITE_NOMEM; - memcpy(p->zRoot, zRoot, (size_t)n+1); - if( zRoot[0]!='$' ){ - zErr = zRoot; - }else{ - pNode = jsonLookupStep(&p->sParse, 0, p->zRoot+1, 0, &zErr); - } - if( zErr ){ - sqlite3_free(cur->pVtab->zErrMsg); - cur->pVtab->zErrMsg = jsonPathSyntaxError(zErr); - jsonEachCursorReset(p); - return cur->pVtab->zErrMsg ? SQLITE_ERROR : SQLITE_NOMEM; - }else if( pNode==0 ){ - return SQLITE_OK; - } - }else{ - pNode = p->sParse.aNode; - } - p->iBegin = p->i = (int)(pNode - p->sParse.aNode); - p->eType = pNode->eType; - if( p->eType>=JSON_ARRAY ){ - pNode->u.iKey = 0; - p->iEnd = p->i + pNode->n + 1; - if( p->bRecursive ){ - p->eType = p->sParse.aNode[p->sParse.aUp[p->i]].eType; - if( p->i>0 && (p->sParse.aNode[p->i-1].jnFlags & JNODE_LABEL)!=0 ){ - p->i--; - } - }else{ - p->i++; - } - }else{ - p->iEnd = p->i+1; - } - } - return SQLITE_OK; -} - -/* The methods of the json_each virtual table */ -static sqlite3_module jsonEachModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenEach, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ -}; - -/* The methods of the json_tree virtual table. */ -static sqlite3_module jsonTreeModule = { - 0, /* iVersion */ - 0, /* xCreate */ - jsonEachConnect, /* xConnect */ - jsonEachBestIndex, /* xBestIndex */ - jsonEachDisconnect, /* xDisconnect */ - 0, /* xDestroy */ - jsonEachOpenTree, /* xOpen - open a cursor */ - jsonEachClose, /* xClose - close a cursor */ - jsonEachFilter, /* xFilter - configure scan constraints */ - jsonEachNext, /* xNext - advance a cursor */ - jsonEachEof, /* xEof - check for end of scan */ - jsonEachColumn, /* xColumn - read data */ - jsonEachRowid, /* xRowid - read data */ - 0, /* xUpdate */ - 0, /* xBegin */ - 0, /* xSync */ - 0, /* xCommit */ - 0, /* xRollback */ - 0, /* xFindMethod */ - 0, /* xRename */ - 0, /* xSavepoint */ - 0, /* xRelease */ - 0 /* xRollbackTo */ -}; -#endif /* SQLITE_OMIT_VIRTUALTABLE */ - -/**************************************************************************** -** The following routines are the only publically visible identifiers in this -** file. Call the following routines in order to register the various SQL -** functions and the virtual table implemented by this file. -****************************************************************************/ - -SQLITE_PRIVATE int sqlite3Json1Init(sqlite3 *db){ - int rc = SQLITE_OK; - unsigned int i; - static const struct { - const char *zName; - int nArg; - int flag; - void (*xFunc)(sqlite3_context*,int,sqlite3_value**); - } aFunc[] = { - { "json", 1, 0, jsonRemoveFunc }, - { "json_array", -1, 0, jsonArrayFunc }, - { "json_array_length", 1, 0, jsonArrayLengthFunc }, - { "json_array_length", 2, 0, jsonArrayLengthFunc }, - { "json_extract", -1, 0, jsonExtractFunc }, - { "json_insert", -1, 0, jsonSetFunc }, - { "json_object", -1, 0, jsonObjectFunc }, - { "json_remove", -1, 0, jsonRemoveFunc }, - { "json_replace", -1, 0, jsonReplaceFunc }, - { "json_set", -1, 1, jsonSetFunc }, - { "json_type", 1, 0, jsonTypeFunc }, - { "json_type", 2, 0, jsonTypeFunc }, - { "json_valid", 1, 0, jsonValidFunc }, - -#if SQLITE_DEBUG - /* DEBUG and TESTING functions */ - { "json_parse", 1, 0, jsonParseFunc }, - { "json_test1", 1, 0, jsonTest1Func }, -#endif - }; - static const struct { - const char *zName; - int nArg; - void (*xStep)(sqlite3_context*,int,sqlite3_value**); - void (*xFinal)(sqlite3_context*); - } aAgg[] = { - { "json_group_array", 1, jsonArrayStep, jsonArrayFinal }, - { "json_group_object", 2, jsonObjectStep, jsonObjectFinal }, - }; -#ifndef SQLITE_OMIT_VIRTUALTABLE - static const struct { - const char *zName; - sqlite3_module *pModule; - } aMod[] = { - { "json_each", &jsonEachModule }, - { "json_tree", &jsonTreeModule }, - }; -#endif - for(i=0; i */ +#endif +#ifdef HAVE_INTTYPES_H +/* #include */ +#endif /* ** 2014 May 31 ** @@ -176203,7 +232878,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_json_init( ** ****************************************************************************** ** -** Interfaces to extend FTS5. Using the interfaces defined in this file, +** Interfaces to extend FTS5. Using the interfaces defined in this file, ** FTS5 may be extended with: ** ** * custom tokenizers, and @@ -176248,19 +232923,19 @@ struct Fts5PhraseIter { ** EXTENSION API FUNCTIONS ** ** xUserData(pFts): -** Return a copy of the context pointer the extension function was -** registered with. +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. ** ** xColumnTotalSize(pFts, iCol, pnToken): ** If parameter iCol is less than zero, set output variable *pnToken ** to the total number of tokens in the FTS5 table. Or, if iCol is ** non-negative but less than the number of columns in the table, return -** the total number of tokens in column iCol, considering all rows in +** the total number of tokens in column iCol, considering all rows in ** the FTS5 table. ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. -** an OOM condition or IO error), an appropriate SQLite error code is +** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** xColumnCount(pFts): @@ -176274,15 +232949,18 @@ struct Fts5PhraseIter { ** ** If parameter iCol is greater than or equal to the number of columns ** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. -** an OOM condition or IO error), an appropriate SQLite error code is +** an OOM condition or IO error), an appropriate SQLite error code is ** returned. ** ** This function may be quite inefficient if used with an FTS5 table ** created with the "columnsize=0" option. ** ** xColumnText: -** This function attempts to retrieve the text of column iCol of the -** current document. If successful, (*pz) is set to point to a buffer +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer ** containing the text in utf-8 encoding, (*pn) is set to the size in bytes ** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, ** if an error occurs, an SQLite error code is returned and the final values @@ -176292,8 +232970,10 @@ struct Fts5PhraseIter { ** Returns the number of phrases in the current query expression. ** ** xPhraseSize: -** Returns the number of tokens in phrase iPhrase of the query. Phrases -** are numbered starting from zero. +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. ** ** xInstCount: ** Set *pnInst to the total number of occurrences of all phrases within @@ -176301,27 +232981,24 @@ struct Fts5PhraseIter { ** an error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. If the FTS5 table is created -** with either "detail=none" or "detail=column" and "content=" option +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always returns 0. ** ** xInst: ** Query for the details of phrase match iIdx within the current row. ** Phrase matches are numbered starting from zero, so the iIdx argument ** should be greater than or equal to zero and smaller than the value -** output by xInstCount(). +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. ** -** Usually, output parameter *piPhrase is set to the phrase number, *piCol +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol ** to the column in which it occurs and *piOff the token offset of the -** first token of the phrase. The exception is if the table was created -** with the offsets=0 option specified. In this case *piOff is always -** set to -1. -** -** Returns SQLITE_OK if successful, or an error code (i.e. SQLITE_NOMEM) -** if an error occurs. +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. +** "detail=none" or "detail=column" option. ** ** xRowid: ** Returns the rowid of the current row. @@ -176337,13 +233014,17 @@ struct Fts5PhraseIter { ** ** with $p set to a phrase equivalent to the phrase iPhrase of the ** current query is executed. Any column filter that applies to -** phrase iPhrase of the current query is included in $p. For each -** row visited, the callback function passed as the fourth argument -** is invoked. The context and API objects passed to the callback +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback ** function may be used to access the properties of each matched row. -** Invoking Api.xUserData() returns a copy of the pointer passed as +** Invoking Api.xUserData() returns a copy of the pointer passed as ** the third argument to pUserData. ** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** ** If the callback function returns any value other than SQLITE_OK, the ** query is abandoned and the xQueryPhrase function returns immediately. ** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. @@ -176356,14 +233037,14 @@ struct Fts5PhraseIter { ** ** xSetAuxdata(pFts5, pAux, xDelete) ** -** Save the pointer passed as the second argument as the extension functions +** Save the pointer passed as the second argument as the extension function's ** "auxiliary data". The pointer may then be retrieved by the current or any ** future invocation of the same fts5 extension function made as part of -** of the same MATCH query using the xGetAuxdata() API. +** the same MATCH query using the xGetAuxdata() API. ** ** Each extension function is allocated a single auxiliary data slot for -** each FTS query (MATCH expression). If the extension function is invoked -** more than once for a single FTS query, then all invocations share a +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a ** single auxiliary data context. ** ** If there is already an auxiliary data pointer when this function is @@ -176374,7 +233055,7 @@ struct Fts5PhraseIter { ** The xDelete callback, if one is specified, is also invoked on the ** auxiliary data pointer after the FTS5 query has finished. ** -** If an error (e.g. an OOM condition) occurs within this function, an +** If an error (e.g. an OOM condition) occurs within this function, ** the auxiliary data is set to NULL and an error code returned. If the ** xDelete parameter was not NULL, it is invoked on the auxiliary data ** pointer before returning. @@ -176382,7 +233063,7 @@ struct Fts5PhraseIter { ** ** xGetAuxdata(pFts5, bClear) ** -** Returns the current auxiliary data pointer for the fts5 extension +** Returns the current auxiliary data pointer for the fts5 extension ** function. See the xSetAuxdata() method for details. ** ** If the bClear argument is non-zero, then the auxiliary data is cleared @@ -176402,7 +233083,7 @@ struct Fts5PhraseIter { ** method, to iterate through all instances of a single query phrase within ** the current row. This is the same information as is accessible via the ** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient -** to use, this API may be faster under some circumstances. To iterate +** to use, this API may be faster under some circumstances. To iterate ** through instances of phrase iPhrase, use the following code: ** ** Fts5PhraseIter iter; @@ -176420,11 +233101,15 @@ struct Fts5PhraseIter { ** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" or "detail=column" option. If the FTS5 table is created -** with either "detail=none" or "detail=column" and "content=" option +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option ** (i.e. if it is a contentless table), then this API always iterates ** through an empty set (all calls to xPhraseFirst() set iCol to -1). ** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** ** xPhraseNext() ** See xPhraseFirst above. ** @@ -176445,22 +233130,78 @@ struct Fts5PhraseIter { ** } ** ** This API can be quite slow if used with an FTS5 table created with the -** "detail=none" option. If the FTS5 table is created with either -** "detail=none" "content=" option (i.e. if it is a contentless table), -** then this API always iterates through an empty set (all calls to +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to ** xPhraseFirstColumn() set iCol to -1). ** ** The information accessed using this API and its companion ** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext ** (or xInst/xInstCount). The chief advantage of this API is that it is ** significantly more efficient than those alternatives when used with -** "detail=column" tables. +** "detail=column" tables. ** ** xPhraseNextColumn() ** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. */ struct Fts5ExtensionApi { - int iVersion; /* Currently always set to 3 */ + int iVersion; /* Currently always set to 4 */ void *(*xUserData)(Fts5Context*); @@ -176468,7 +233209,7 @@ struct Fts5ExtensionApi { int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); - int (*xTokenize)(Fts5Context*, + int (*xTokenize)(Fts5Context*, const char *pText, int nText, /* Text to tokenize */ void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ @@ -176495,37 +233236,53 @@ struct Fts5ExtensionApi { int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); }; -/* +/* ** CUSTOM AUXILIARY FUNCTIONS *************************************************************************/ /************************************************************************* ** CUSTOM TOKENIZERS ** -** Applications may also register custom tokenizer types. A tokenizer -** is registered by providing fts5 with a populated instance of the +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the ** following structure. All structure methods must be defined, setting ** any member of the fts5_tokenizer struct to NULL leads to undefined ** behaviour. The structure methods are expected to function as follows: ** ** xCreate: -** This function is used to allocate and inititalize a tokenizer instance. +** This function is used to allocate and initialize a tokenizer instance. ** A tokenizer instance is required to actually tokenize text. ** ** The first argument passed to this function is a copy of the (void*) -** pointer provided by the application when the fts5_tokenizer object -** was registered with FTS5 (the third argument to xCreateTokenizer()). +** pointer provided by the application when the fts5_tokenizer_v2 object +** was registered with FTS5 (the third argument to xCreateTokenizer()). ** The second and third arguments are an array of nul-terminated strings ** containing the tokenizer arguments, if any, specified following the ** tokenizer name as part of the CREATE VIRTUAL TABLE statement used ** to create the FTS5 table. ** -** The final argument is an output variable. If successful, (*ppOut) +** The final argument is an output variable. If successful, (*ppOut) ** should be set to point to the new tokenizer handle and SQLITE_OK ** returned. If an error occurs, some value other than SQLITE_OK should -** be returned. In this case, fts5 assumes that the final value of *ppOut +** be returned. In this case, fts5 assumes that the final value of *ppOut ** is undefined. ** ** xDelete: @@ -176534,12 +233291,12 @@ struct Fts5ExtensionApi { ** be invoked exactly once for each successful call to xCreate(). ** ** xTokenize: -** This function is expected to tokenize the nText byte string indicated +** This function is expected to tokenize the nText byte string indicated ** by argument pText. pText may or may not be nul-terminated. The first ** argument passed to this function is a pointer to an Fts5Tokenizer object ** returned by an earlier call to xCreate(). ** -** The second argument indicates the reason that FTS5 is requesting +** The third argument indicates the reason that FTS5 is requesting ** tokenization of the supplied text. This is always one of the following ** four values: ** @@ -176548,8 +233305,8 @@ struct Fts5ExtensionApi { ** determine the set of tokens to add to (or delete from) the ** FTS index. ** -**
          • FTS5_TOKENIZE_QUERY - A MATCH query is being executed -** against the FTS index. The tokenizer is being called to tokenize +**
          • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize ** a bareword or quoted string specified as part of the query. ** **
          • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as @@ -176557,12 +233314,19 @@ struct Fts5ExtensionApi { ** followed by a "*" character, indicating that the last token ** returned by the tokenizer will be treated as a token prefix. ** -**
          • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +**
          • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to ** satisfy an fts5_api.xTokenize() request made by an auxiliary ** function. Or an fts5_api.xColumnSize() request made by the same -** on a columnsize=0 database. +** on a columnsize=0 database. ** ** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** ** For each token in the input string, the supplied callback xToken() must ** be invoked. The first argument to it should be a copy of the pointer ** passed as the second argument to xTokenize(). The third and fourth @@ -176572,10 +233336,10 @@ struct Fts5ExtensionApi { ** which the token is derived within the input. ** ** The second argument passed to the xToken() callback ("tflags") should -** normally be set to 0. The exception is if the tokenizer supports +** normally be set to 0. The exception is if the tokenizer supports ** synonyms. In this case see the discussion below for details. ** -** FTS5 assumes the xToken() callback is invoked for each token in the +** FTS5 assumes the xToken() callback is invoked for each token in the ** order that they occur within the input text. ** ** If an xToken() callback returns any value other than SQLITE_OK, then @@ -176586,10 +233350,34 @@ struct Fts5ExtensionApi { ** may abandon the tokenization and return any error code other than ** SQLITE_OK or SQLITE_DONE. ** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
              +**
            • There is no "iVersion" field, and +**
            • The xTokenize() method does not take a locale argument. +**
            +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** ** SYNONYM SUPPORT ** ** Custom tokenizers may also support synonyms. Consider a case in which a -** user wishes to query for a phrase such as "first place". Using the +** user wishes to query for a phrase such as "first place". Using the ** built-in tokenizers, the FTS5 query 'first + place' will match instances ** of "first place" within the document set, but not alternative forms ** such as "1st place". In some applications, it would be better to match @@ -176598,8 +233386,8 @@ struct Fts5ExtensionApi { ** ** There are several ways to approach this in FTS5: ** -**
            1. By mapping all synonyms to a single token. In this case, the -** In the above example, this means that the tokenizer returns the +**
              1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the ** same token for inputs "first" and "1st". Say that token is in ** fact "first", so that when the user inserts the document "I won ** 1st place" entries are added to the index for tokens "i", "won", @@ -176607,37 +233395,37 @@ struct Fts5ExtensionApi { ** the tokenizer substitutes "first" for "1st" and the query works ** as expected. ** -**
              2. By adding multiple synonyms for a single term to the FTS index. -** In this case, when tokenizing query text, the tokenizer may -** provide multiple synonyms for a single term within the document. -** FTS5 then queries the index for each synonym individually. For -** example, faced with the query: +**
              3. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: ** ** ** ... MATCH 'first place' ** ** the tokenizer offers both "1st" and "first" as synonyms for the -** first token in the MATCH query and FTS5 effectively runs a query +** first token in the MATCH query and FTS5 effectively runs a query ** similar to: ** ** ** ... MATCH '(first OR 1st) place' ** ** except that, for the purposes of auxiliary functions, the query -** still appears to contain just two phrases - "(first OR 1st)" +** still appears to contain just two phrases - "(first OR 1st)" ** being treated as a single phrase. ** **
              4. By adding multiple synonyms for a single term to the FTS index. ** Using this method, when tokenizing document text, the tokenizer -** provides multiple synonyms for each token. So that when a +** provides multiple synonyms for each token. So that when a ** document such as "I won first place" is tokenized, entries are ** added to the FTS index for "i", "won", "first", "1st" and ** "place". ** ** This way, even if the tokenizer does not provide synonyms -** when tokenizing query text (it should not - to do would be -** inefficient), it doesn't matter if the user queries for -** 'first + place' or '1st + place', as there are entires in the +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the ** FTS index corresponding to both forms of the first token. **
              ** @@ -176657,15 +233445,15 @@ struct Fts5ExtensionApi { ** ** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time ** xToken() is called. Multiple synonyms may be specified for a single token -** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. ** There is no limit to the number of synonyms that may be provided for a ** single token. ** -** In many cases, method (1) above is the best approach. It does not add +** In many cases, method (1) above is the best approach. It does not add ** extra data to the FTS index or require FTS5 to query for multiple terms, ** so it is efficient in terms of disk space and query speed. However, it ** does not support prefix queries very well. If, as suggested above, the -** token "first" is subsituted for "1st" by the tokenizer, then the query: +** token "first" is substituted for "1st" by the tokenizer, then the query: ** ** ** ... MATCH '1s*' @@ -176673,35 +233461,38 @@ struct Fts5ExtensionApi { ** will not match documents that contain the token "1st" (as the tokenizer ** will probably not map "1s" to any prefix of "first"). ** -** For full prefix support, method (3) may be preferred. In this case, +** For full prefix support, method (3) may be preferred. In this case, ** because the index contains entries for both "first" and "1st", prefix ** queries such as 'fi*' or '1s*' will match correctly. However, because ** extra entries are added to the FTS index, this method uses more space ** within the database. ** ** Method (2) offers a midpoint between (1) and (3). Using this method, -** a query such as '1s*' will match documents that contain the literal +** a query such as '1s*' will match documents that contain the literal ** token "1st", but not "first" (assuming the tokenizer is not able to ** provide synonyms for prefixes). However, a non-prefix query like '1st' ** will match against "1st" and "first". This method does not require -** extra disk space, as no extra entries are added to the FTS index. +** extra disk space, as no extra entries are added to the FTS index. ** On the other hand, it may require more CPU cycles to run MATCH queries, ** as separate queries of the FTS index are required for each synonym. ** ** When using methods (2) or (3), it is important that the tokenizer only -** provide synonyms when tokenizing document text (method (2)) or query -** text (method (3)), not both. Doing so will not cause any errors, but is +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is ** inefficient. */ typedef struct Fts5Tokenizer Fts5Tokenizer; -typedef struct fts5_tokenizer fts5_tokenizer; -struct fts5_tokenizer { +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); void (*xDelete)(Fts5Tokenizer*); - int (*xTokenize)(Fts5Tokenizer*, + int (*xTokenize)(Fts5Tokenizer*, void *pCtx, int flags, /* Mask of FTS5_TOKENIZE_* flags */ - const char *pText, int nText, + const char *pText, int nText, + const char *pLocale, int nLocale, int (*xToken)( void *pCtx, /* Copy of 2nd argument to xTokenize() */ int tflags, /* Mask of FTS5_TOKEN_* flags */ @@ -176713,6 +233504,31 @@ struct fts5_tokenizer { ); }; +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + + /* Flags that may be passed as the third argument to xTokenize() */ #define FTS5_TOKENIZE_QUERY 0x0001 #define FTS5_TOKENIZE_PREFIX 0x0002 @@ -176732,13 +233548,13 @@ struct fts5_tokenizer { */ typedef struct fts5_api fts5_api; struct fts5_api { - int iVersion; /* Currently always set to 2 */ + int iVersion; /* Currently always set to 3 */ /* Create a new tokenizer */ int (*xCreateTokenizer)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_tokenizer *pTokenizer, void (*xDestroy)(void*) ); @@ -176747,7 +233563,7 @@ struct fts5_api { int (*xFindTokenizer)( fts5_api *pApi, const char *zName, - void **ppContext, + void **ppUserData, fts5_tokenizer *pTokenizer ); @@ -176755,10 +233571,29 @@ struct fts5_api { int (*xCreateFunction)( fts5_api *pApi, const char *zName, - void *pContext, + void *pUserData, fts5_extension_function xFunction, void (*xDestroy)(void*) ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); }; /* @@ -176771,7 +233606,6 @@ struct fts5_api { #endif /* _FTS5_H */ - /* ** 2014 May 31 ** @@ -176804,13 +233638,26 @@ typedef short i16; typedef sqlite3_int64 i64; typedef sqlite3_uint64 u64; -#define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#ifndef ArraySize +# define ArraySize(x) ((int)(sizeof(x) / sizeof(x[0]))) +#endif #define testcase(x) -#define ALWAYS(x) 1 -#define NEVER(x) 0 -# +#if defined(SQLITE_COVERAGE_TEST) || defined(SQLITE_MUTATION_TEST) +# define SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS 1 +#endif +#if defined(SQLITE_OMIT_AUXILIARY_SAFETY_CHECKS) +# define ALWAYS(X) (1) +# define NEVER(X) (0) +#elif !defined(NDEBUG) +# define ALWAYS(X) ((X)?1:(assert(0),0)) +# define NEVER(X) ((X)?(assert(0),1):0) +#else +# define ALWAYS(X) (X) +# define NEVER(X) (X) +#endif + #define MIN(x,y) (((x) < (y)) ? (x) : (y)) #define MAX(x,y) (((x) > (y)) ? (x) : (y)) @@ -176820,9 +233667,25 @@ typedef sqlite3_uint64 u64; # define LARGEST_INT64 (0xffffffff|(((i64)0x7fffffff)<<32)) # define SMALLEST_INT64 (((i64)-1) - LARGEST_INT64) +/* The uptr type is an unsigned integer large enough to hold a pointer +*/ +#if defined(HAVE_STDINT_H) + typedef uintptr_t uptr; +#elif SQLITE_PTRSIZE==4 + typedef u32 uptr; +#else + typedef u64 uptr; #endif -/* Truncate very long tokens to this many bytes. Hard limit is +#ifdef SQLITE_4_BYTE_ALIGNED_MALLOC +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&3)==0) +#else +# define EIGHT_BYTE_ALIGNMENT(X) ((((uptr)(X) - (uptr)0)&7)==0) +#endif + +#endif + +/* Truncate very long tokens to this many bytes. Hard limit is ** (65536-1-1-4-9)==65521 bytes. The limiting factor is the 16-bit offset ** field that occurs at the start of each leaf page (see fts5_index.c). */ #define FTS5_MAX_TOKEN_SIZE 32768 @@ -176834,6 +233697,11 @@ typedef sqlite3_uint64 u64; */ #define FTS5_MAX_PREFIX_INDEXES 31 +/* +** Maximum segments permitted in a single index +*/ +#define FTS5_MAX_SEGMENT 2000 + #define FTS5_DEFAULT_NEARDIST 10 #define FTS5_DEFAULT_RANK "bm25" @@ -176850,7 +233718,7 @@ static int sqlite3Fts5Corrupt(void); /* ** The assert_nc() macro is similar to the assert() macro, except that it -** is used for assert() conditions that are true only if it can be +** is used for assert() conditions that are true only if it can be ** guranteed that the database is not corrupt. */ #ifdef SQLITE_DEBUG @@ -176860,6 +233728,12 @@ SQLITE_API extern int sqlite3_fts5_may_be_corrupt; # define assert_nc(x) assert(x) #endif +/* +** A version of memcmp() that does not cause asan errors if one of the pointer +** parameters is NULL and the number of bytes to compare is zero. +*/ +#define fts5Memcmp(s1, s2, n) ((n)<=0 ? 0 : memcmp((s1), (s2), (n))) + /* Mark a function parameter as unused, to suppress nuisance compiler ** warnings. */ #ifndef UNUSED_PARAM @@ -176873,7 +233747,7 @@ SQLITE_API extern int sqlite3_fts5_may_be_corrupt; typedef struct Fts5Global Fts5Global; typedef struct Fts5Colset Fts5Colset; -/* If a NEAR() clump or phrase may only match a specific set of columns, +/* If a NEAR() clump or phrase may only match a specific set of columns, ** then an object of the following type is used to record the set of columns. ** Each entry in the aiCol[] array is a column that may be matched. ** @@ -176892,6 +233766,18 @@ struct Fts5Colset { */ typedef struct Fts5Config Fts5Config; +typedef struct Fts5TokenizerConfig Fts5TokenizerConfig; + +struct Fts5TokenizerConfig { + Fts5Tokenizer *pTok; + fts5_tokenizer_v2 *pApi2; + fts5_tokenizer *pApi1; + const char **azArg; + int nArg; + int ePattern; /* FTS_PATTERN_XXX constant */ + const char *pLocale; /* Current locale to use */ + int nLocale; /* Size of pLocale in bytes */ +}; /* ** An instance of the following structure encodes all information that can @@ -176901,20 +233787,24 @@ typedef struct Fts5Config Fts5Config; ** ** nAutomerge: ** The minimum number of segments that an auto-merge operation should -** attempt to merge together. A value of 1 sets the object to use the +** attempt to merge together. A value of 1 sets the object to use the ** compile time default. Zero disables auto-merge altogether. ** +** bContentlessDelete: +** True if the contentless_delete option was present in the CREATE +** VIRTUAL TABLE statement. +** ** zContent: ** ** zContentRowid: -** The value of the content_rowid= option, if one was specified. Or +** The value of the content_rowid= option, if one was specified. Or ** the string "rowid" otherwise. This text is not quoted - if it is ** used as part of an SQL statement it needs to be quoted appropriately. ** ** zContentExprlist: ** ** pzErrmsg: -** This exists in order to allow the fts5_index.c module to return a +** This exists in order to allow the fts5_index.c module to return a ** decent error message if it encounters a file-format version it does ** not understand. ** @@ -176927,9 +233817,12 @@ typedef struct Fts5Config Fts5Config; ** ** INSERT INTO tbl(tbl, rank) VALUES('prefix-index', $bPrefixIndex); ** +** bLocale: +** Set to true if locale=1 was specified when the table was created. */ struct Fts5Config { sqlite3 *db; /* Database handle */ + Fts5Global *pGlobal; /* Global fts5 object for handle db */ char *zDb; /* Database holding FTS index (e.g. "main") */ char *zName; /* Name of FTS index */ int nCol; /* Number of columns */ @@ -176938,15 +233831,21 @@ struct Fts5Config { int nPrefix; /* Number of prefix indexes */ int *aPrefix; /* Sizes in bytes of nPrefix prefix indexes */ int eContent; /* An FTS5_CONTENT value */ - char *zContent; /* content table */ - char *zContentRowid; /* "content_rowid=" option value */ + int bContentlessDelete; /* "contentless_delete=" option (dflt==0) */ + int bContentlessUnindexed; /* "contentless_unindexed=" option (dflt=0) */ + char *zContent; /* content table */ + char *zContentRowid; /* "content_rowid=" option value */ int bColumnsize; /* "columnsize=" option value (dflt==1) */ + int bTokendata; /* "tokendata=" option value (dflt==0) */ + int bLocale; /* "locale=" option value (dflt==0) */ int eDetail; /* FTS5_DETAIL_XXX value */ char *zContentExprlist; - Fts5Tokenizer *pTok; - fts5_tokenizer *pTokApi; + Fts5TokenizerConfig t; + int bLock; /* True when table is preparing statement */ + /* Values loaded from the %_config table */ + int iVersion; /* fts5 file format 'version' */ int iCookie; /* Incremented when %_config is modified */ int pgsz; /* Approximate page size used in %_data */ int nAutomerge; /* 'automerge' setting */ @@ -176955,6 +233854,8 @@ struct Fts5Config { int nHashSize; /* Bytes of memory for in-memory hash */ char *zRank; /* Name of rank function */ char *zRankArgs; /* Arguments to rank function */ + int bSecureDelete; /* 'secure-delete' */ + int nDeleteMerge; /* 'deletemerge' */ /* If non-NULL, points to sqlite3_vtab.base.zErrmsg. Often NULL. */ char **pzErrmsg; @@ -176964,18 +233865,24 @@ struct Fts5Config { #endif }; -/* Current expected value of %_config table 'version' field */ -#define FTS5_CURRENT_VERSION 4 +/* Current expected value of %_config table 'version' field. And +** the expected version if the 'secure-delete' option has ever been +** set on the table. */ +#define FTS5_CURRENT_VERSION 4 +#define FTS5_CURRENT_VERSION_SECUREDELETE 5 -#define FTS5_CONTENT_NORMAL 0 -#define FTS5_CONTENT_NONE 1 -#define FTS5_CONTENT_EXTERNAL 2 - -#define FTS5_DETAIL_FULL 0 -#define FTS5_DETAIL_NONE 1 -#define FTS5_DETAIL_COLUMNS 2 +#define FTS5_CONTENT_NORMAL 0 +#define FTS5_CONTENT_NONE 1 +#define FTS5_CONTENT_EXTERNAL 2 +#define FTS5_CONTENT_UNINDEXED 3 +#define FTS5_DETAIL_FULL 0 +#define FTS5_DETAIL_NONE 1 +#define FTS5_DETAIL_COLUMNS 2 +#define FTS5_PATTERN_NONE 0 +#define FTS5_PATTERN_LIKE 65 /* matches SQLITE_INDEX_CONSTRAINT_LIKE */ +#define FTS5_PATTERN_GLOB 66 /* matches SQLITE_INDEX_CONSTRAINT_GLOB */ static int sqlite3Fts5ConfigParse( Fts5Global*, sqlite3*, int, const char **, Fts5Config**, char** @@ -177002,6 +233909,8 @@ static int sqlite3Fts5ConfigSetValue(Fts5Config*, const char*, sqlite3_value*, i static int sqlite3Fts5ConfigParseRank(const char*, char**, char**); +static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...); + /* ** End of interface to code in fts5_config.c. **************************************************************************/ @@ -177032,7 +233941,7 @@ static void sqlite3Fts5BufferAppendPrintf(int *, Fts5Buffer*, char *zFmt, ...); static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); #define fts5BufferZero(x) sqlite3Fts5BufferZero(x) -#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,c) +#define fts5BufferAppendVarint(a,b,c) sqlite3Fts5BufferAppendVarint(a,b,(i64)c) #define fts5BufferFree(a) sqlite3Fts5BufferFree(a) #define fts5BufferAppendBlob(a,b,c,d) sqlite3Fts5BufferAppendBlob(a,b,c,d) #define fts5BufferSet(a,b,c,d) sqlite3Fts5BufferSet(a,b,c,d) @@ -177046,8 +233955,8 @@ static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...); static void sqlite3Fts5Put32(u8*, int); static int sqlite3Fts5Get32(const u8*); -#define FTS5_POS2COLUMN(iPos) (int)(iPos >> 32) -#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0xFFFFFFFF) +#define FTS5_POS2COLUMN(iPos) (int)((iPos >> 32) & 0x7FFFFFFF) +#define FTS5_POS2OFFSET(iPos) (int)(iPos & 0x7FFFFFFF) typedef struct Fts5PoslistReader Fts5PoslistReader; struct Fts5PoslistReader { @@ -177082,7 +233991,7 @@ static int sqlite3Fts5PoslistNext64( ); /* Malloc utility */ -static void *sqlite3Fts5MallocZero(int *pRc, int nByte); +static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte); static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn); /* Character set tests (like isspace(), isalpha() etc.) */ @@ -177119,16 +234028,19 @@ struct Fts5IndexIter { /* ** Values used as part of the flags argument passed to IndexQuery(). */ -#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ -#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ -#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ -#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ +#define FTS5INDEX_QUERY_PREFIX 0x0001 /* Prefix query */ +#define FTS5INDEX_QUERY_DESC 0x0002 /* Docs in descending rowid order */ +#define FTS5INDEX_QUERY_TEST_NOIDX 0x0004 /* Do not use prefix index */ +#define FTS5INDEX_QUERY_SCAN 0x0008 /* Scan query (fts5vocab) */ /* The following are used internally by the fts5_index.c module. They are ** defined here only to make it easier to avoid clashes with the flags ** above. */ -#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 -#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPEMPTY 0x0010 +#define FTS5INDEX_QUERY_NOOUTPUT 0x0020 +#define FTS5INDEX_QUERY_SKIPHASH 0x0040 +#define FTS5INDEX_QUERY_NOTOKENDATA 0x0080 +#define FTS5INDEX_QUERY_SCANONETERM 0x0100 /* ** Create/destroy an Fts5Index object. @@ -177140,27 +234052,27 @@ static int sqlite3Fts5IndexClose(Fts5Index *p); ** Return a simple checksum value based on the arguments. */ static u64 sqlite3Fts5IndexEntryCksum( - i64 iRowid, - int iCol, - int iPos, + i64 iRowid, + int iCol, + int iPos, int iIdx, const char *pTerm, int nTerm ); /* -** Argument p points to a buffer containing utf-8 text that is n bytes in +** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ static int sqlite3Fts5IndexCharlenToBytelen( - const char *p, - int nByte, + const char *p, + int nByte, int nChar ); /* -** Open a new iterator to iterate though all rowids that match the +** Open a new iterator to iterate though all rowids that match the ** specified token or token prefix. */ static int sqlite3Fts5IndexQuery( @@ -177183,15 +234095,27 @@ static int sqlite3Fts5IterNextFrom(Fts5IndexIter*, i64 iMatch); */ static void sqlite3Fts5IterClose(Fts5IndexIter*); +/* +** Close the reader blob handle, if it is open. +*/ +static void sqlite3Fts5IndexCloseReader(Fts5Index*); + /* ** This interface is used by the fts5vocab module. */ static const char *sqlite3Fts5IterTerm(Fts5IndexIter*, int*); static int sqlite3Fts5IterNextScan(Fts5IndexIter*); - +static void *sqlite3Fts5StructureRef(Fts5Index*); +static void sqlite3Fts5StructureRelease(void*); +static int sqlite3Fts5StructureTest(Fts5Index*, void*); /* -** Insert or remove data to or from the index. Each time a document is +** Used by xInstToken(): +*/ +static int sqlite3Fts5IterToken(Fts5IndexIter*, i64, int, int, const char**, int*); + +/* +** Insert or remove data to or from the index. Each time a document is ** added to or removed from the index, this function is called one or more ** times. ** @@ -177219,14 +234143,14 @@ static int sqlite3Fts5IndexBeginWrite( /* ** Flush any data stored in the in-memory hash tables to the database. -** If the bCommit flag is true, also close any open blob handles. +** Also close any open blob handles. */ -static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit); +static int sqlite3Fts5IndexSync(Fts5Index *p); /* ** Discard any data stored in the in-memory hash tables. Do not write it ** to the database. Additionally, assume that the contents of the %_data -** table may have changed on disk. So any in-memory caches of %_data +** table may have changed on disk. So any in-memory caches of %_data ** records must be invalidated. */ static int sqlite3Fts5IndexRollback(Fts5Index *p); @@ -177240,18 +234164,18 @@ static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8*, int); /* ** Functions called by the storage module as part of integrity-check. */ -static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum); +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index*, u64 cksum, int bUseCksum); -/* -** Called during virtual module initialization to register UDF -** fts5_decode() with SQLite +/* +** Called during virtual module initialization to register UDF +** fts5_decode() with SQLite */ static int sqlite3Fts5IndexInit(sqlite3*); static int sqlite3Fts5IndexSetCookie(Fts5Index*, int); /* -** Return the total number of entries read from the %_data table by +** Return the total number of entries read from the %_data table by ** this connection since it was created. */ static int sqlite3Fts5IndexReads(Fts5Index *p); @@ -177263,19 +234187,29 @@ static int sqlite3Fts5IndexReset(Fts5Index *p); static int sqlite3Fts5IndexLoadConfig(Fts5Index *p); +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin); +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid); + +static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter*); + +/* Used to populate hash tables for xInstToken in detail=none/column mode. */ +static int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter*, const char*, int, i64 iRowid, int iCol, int iOff +); + /* ** End of interface to code in fts5_index.c. **************************************************************************/ /************************************************************************** -** Interface to code in fts5_varint.c. +** Interface to code in fts5_varint.c. */ static int sqlite3Fts5GetVarint32(const unsigned char *p, u32 *v); static int sqlite3Fts5GetVarintLen(u32 iVal); static u8 sqlite3Fts5GetVarint(const unsigned char*, u64*); static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); -#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&b) +#define fts5GetVarint32(a,b) sqlite3Fts5GetVarint32(a,(u32*)&(b)) #define fts5GetVarint sqlite3Fts5GetVarint #define fts5FastGetVarint32(a, iOff, nVal) { \ @@ -177293,26 +234227,39 @@ static int sqlite3Fts5PutVarint(unsigned char *p, u64 v); /************************************************************************** -** Interface to code in fts5.c. +** Interface to code in fts5_main.c. */ -static int sqlite3Fts5GetTokenizer( - Fts5Global*, - const char **azArg, - int nArg, - Fts5Tokenizer**, - fts5_tokenizer**, - char **pzErr -); +/* +** Virtual-table object. +*/ +typedef struct Fts5Table Fts5Table; +struct Fts5Table { + sqlite3_vtab base; /* Base class used by SQLite core */ + Fts5Config *pConfig; /* Virtual table configuration */ + Fts5Index *pIndex; /* Full-text index */ +}; -static Fts5Index *sqlite3Fts5IndexFromCsrid(Fts5Global*, i64, Fts5Config **); +static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig); + +static Fts5Table *sqlite3Fts5TableFromCsrid(Fts5Global*, i64); + +static int sqlite3Fts5FlushToDisk(Fts5Table*); + +static void sqlite3Fts5ClearLocale(Fts5Config *pConfig); +static void sqlite3Fts5SetLocale(Fts5Config *pConfig, const char *pLoc, int nLoc); + +static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal); +static int sqlite3Fts5DecodeLocaleValue(sqlite3_value *pVal, + const char **ppText, int *pnText, const char **ppLoc, int *pnLoc +); /* ** End of interface to code in fts5.c. **************************************************************************/ /************************************************************************** -** Interface to code in fts5_hash.c. +** Interface to code in fts5_hash.c. */ typedef struct Fts5Hash Fts5Hash; @@ -177336,10 +234283,16 @@ static int sqlite3Fts5HashWrite( */ static void sqlite3Fts5HashClear(Fts5Hash*); +/* +** Return true if the hash is empty, false otherwise. +*/ +static int sqlite3Fts5HashIsEmpty(Fts5Hash*); + static int sqlite3Fts5HashQuery( Fts5Hash*, /* Hash table to query */ + int nPre, const char *pTerm, int nTerm, /* Query term */ - const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + void **ppObj, /* OUT: Pointer to doclist for pTerm */ int *pnDoclist /* OUT: Size of doclist in bytes */ ); @@ -177351,17 +234304,19 @@ static void sqlite3Fts5HashScanNext(Fts5Hash*); static int sqlite3Fts5HashScanEof(Fts5Hash*); static void sqlite3Fts5HashScanEntry(Fts5Hash *, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ); + /* ** End of interface to code in fts5_hash.c. **************************************************************************/ /************************************************************************** -** Interface to code in fts5_storage.c. fts5_storage.c contains contains +** Interface to code in fts5_storage.c. fts5_storage.c contains contains ** code to access the data stored in the %_content and %_docsize tables. */ @@ -177378,11 +234333,11 @@ static int sqlite3Fts5StorageRename(Fts5Storage*, const char *zName); static int sqlite3Fts5DropAll(Fts5Config*); static int sqlite3Fts5CreateTable(Fts5Config*, const char*, const char*, int, char **); -static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**); -static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, sqlite3_value**, i64*); +static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64, sqlite3_value**, int); +static int sqlite3Fts5StorageContentInsert(Fts5Storage *p, int, sqlite3_value**, i64*); static int sqlite3Fts5StorageIndexInsert(Fts5Storage *p, sqlite3_value**, i64); -static int sqlite3Fts5StorageIntegrity(Fts5Storage *p); +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg); static int sqlite3Fts5StorageStmt(Fts5Storage *p, int eStmt, sqlite3_stmt**, char**); static void sqlite3Fts5StorageStmtRelease(Fts5Storage *p, int eStmt, sqlite3_stmt*); @@ -177391,7 +234346,7 @@ static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol); static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnAvg); static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow); -static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit); +static int sqlite3Fts5StorageSync(Fts5Storage *p); static int sqlite3Fts5StorageRollback(Fts5Storage *p); static int sqlite3Fts5StorageConfigValue( @@ -177404,13 +234359,16 @@ static int sqlite3Fts5StorageOptimize(Fts5Storage *p); static int sqlite3Fts5StorageMerge(Fts5Storage *p, int nMerge); static int sqlite3Fts5StorageReset(Fts5Storage *p); +static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage*); +static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel); + /* ** End of interface to code in fts5_storage.c. **************************************************************************/ /************************************************************************** -** Interface to code in fts5_expr.c. +** Interface to code in fts5_expr.c. */ typedef struct Fts5Expr Fts5Expr; typedef struct Fts5ExprNode Fts5ExprNode; @@ -177426,11 +234384,20 @@ struct Fts5Token { /* Parse a MATCH expression. */ static int sqlite3Fts5ExprNew( - Fts5Config *pConfig, + Fts5Config *pConfig, + int bPhraseToAnd, + int iCol, /* Column on LHS of MATCH operator */ const char *zExpr, - Fts5Expr **ppNew, + Fts5Expr **ppNew, char **pzErr ); +static int sqlite3Fts5ExprPattern( + Fts5Config *pConfig, + int bGlob, + int iCol, + const char *zText, + Fts5Expr **pp +); /* ** for(rc = sqlite3Fts5ExprFirst(pExpr, pIdx, bDesc); @@ -177447,6 +234414,7 @@ static int sqlite3Fts5ExprEof(Fts5Expr*); static i64 sqlite3Fts5ExprRowid(Fts5Expr*); static void sqlite3Fts5ExprFree(Fts5Expr*); +static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2); /* Called during startup to register a UDF with SQLite */ static int sqlite3Fts5ExprInit(Fts5Global*, sqlite3*); @@ -177461,12 +234429,15 @@ static int sqlite3Fts5ExprPopulatePoslists( Fts5Config*, Fts5Expr*, Fts5PoslistPopulator*, int, const char*, int ); static void sqlite3Fts5ExprCheckPoslists(Fts5Expr*, i64); -static void sqlite3Fts5ExprClearEof(Fts5Expr*); static int sqlite3Fts5ExprClonePhrase(Fts5Expr*, int, Fts5Expr**); static int sqlite3Fts5ExprPhraseCollist(Fts5Expr *, int, const u8 **, int *); +static int sqlite3Fts5ExprQueryToken(Fts5Expr*, int, int, const char**, int*); +static int sqlite3Fts5ExprInstToken(Fts5Expr*, i64, int, int, int, int, const char**, int*); +static void sqlite3Fts5ExprClearTokens(Fts5Expr*); + /******************************************* ** The fts5_expr.c API above this point is used by the other hand-written ** C code in this module. The interfaces below this point are called by @@ -177489,21 +234460,23 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( ); static Fts5ExprPhrase *sqlite3Fts5ParseTerm( - Fts5Parse *pParse, - Fts5ExprPhrase *pPhrase, + Fts5Parse *pParse, + Fts5ExprPhrase *pPhrase, Fts5Token *pToken, int bPrefix ); +static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase*); + static Fts5ExprNearset *sqlite3Fts5ParseNearset( - Fts5Parse*, + Fts5Parse*, Fts5ExprNearset*, - Fts5ExprPhrase* + Fts5ExprPhrase* ); static Fts5Colset *sqlite3Fts5ParseColset( - Fts5Parse*, - Fts5Colset*, + Fts5Parse*, + Fts5Colset*, Fts5Token * ); @@ -177512,7 +234485,8 @@ static void sqlite3Fts5ParseNearsetFree(Fts5ExprNearset*); static void sqlite3Fts5ParseNodeFree(Fts5ExprNode*); static void sqlite3Fts5ParseSetDistance(Fts5Parse*, Fts5ExprNearset*, Fts5Token*); -static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNearset*, Fts5Colset*); +static void sqlite3Fts5ParseSetColset(Fts5Parse*, Fts5ExprNode*, Fts5Colset*); +static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse*, Fts5Colset*); static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p); static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); @@ -177523,7 +234497,7 @@ static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token*); /************************************************************************** -** Interface to code in fts5_aux.c. +** Interface to code in fts5_aux.c. */ static int sqlite3Fts5AuxInit(fts5_api*); @@ -177532,16 +234506,21 @@ static int sqlite3Fts5AuxInit(fts5_api*); **************************************************************************/ /************************************************************************** -** Interface to code in fts5_tokenizer.c. +** Interface to code in fts5_tokenizer.c. */ static int sqlite3Fts5TokenizerInit(fts5_api*); +static int sqlite3Fts5TokenizerPattern( + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), + Fts5Tokenizer *pTok +); +static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig*); /* ** End of interface to code in fts5_tokenizer.c. **************************************************************************/ /************************************************************************** -** Interface to code in fts5_vocab.c. +** Interface to code in fts5_vocab.c. */ static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); @@ -177552,11 +234531,14 @@ static int sqlite3Fts5VocabInit(Fts5Global*, sqlite3*); /************************************************************************** -** Interface to automatically generated code in fts5_unicode2.c. +** Interface to automatically generated code in fts5_unicode2.c. */ -static int sqlite3Fts5UnicodeIsalnum(int c); static int sqlite3Fts5UnicodeIsdiacritic(int c); static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); + +static int sqlite3Fts5UnicodeCatParse(const char*, u8*); +static int sqlite3Fts5UnicodeCategory(u32 iCode); +static void sqlite3Fts5UnicodeAscii(u8*, u8*); /* ** End of interface to code in fts5_unicode2.c. **************************************************************************/ @@ -177568,15 +234550,20 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); #define FTS5_NOT 3 #define FTS5_TERM 4 #define FTS5_COLON 5 -#define FTS5_LP 6 -#define FTS5_RP 7 -#define FTS5_LCP 8 -#define FTS5_RCP 9 -#define FTS5_STRING 10 -#define FTS5_COMMA 11 -#define FTS5_PLUS 12 -#define FTS5_STAR 13 +#define FTS5_MINUS 6 +#define FTS5_LCP 7 +#define FTS5_RCP 8 +#define FTS5_STRING 9 +#define FTS5_LP 10 +#define FTS5_RP 11 +#define FTS5_CARET 12 +#define FTS5_COMMA 13 +#define FTS5_PLUS 14 +#define FTS5_STAR 15 +/* This file is automatically generated by Lemon from input grammar +** source file "fts5parse.y". +*/ /* ** 2000-05-29 ** @@ -177601,7 +234588,6 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); ** The following is the concatenation of all %include directives from the ** input grammar file: */ -/* #include */ /************ Begin %include sections from the grammar ************************/ /* #include "fts5Int.h" */ @@ -177631,11 +234617,26 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); #define fts5YYMALLOCARGTYPE u64 /**************** End of %include directives **********************************/ -/* These constants specify the various numeric values for terminal symbols -** in a format understandable to "makeheaders". This section is blank unless -** "lemon" is run with the "-m" command-line option. -***************** Begin makeheaders token definitions *************************/ -/**************** End makeheaders token definitions ***************************/ +/* These constants specify the various numeric values for terminal symbols. +***************** Begin token definitions *************************************/ +#ifndef FTS5_OR +#define FTS5_OR 1 +#define FTS5_AND 2 +#define FTS5_NOT 3 +#define FTS5_TERM 4 +#define FTS5_COLON 5 +#define FTS5_MINUS 6 +#define FTS5_LCP 7 +#define FTS5_RCP 8 +#define FTS5_STRING 9 +#define FTS5_LP 10 +#define FTS5_RP 11 +#define FTS5_CARET 12 +#define FTS5_COMMA 13 +#define FTS5_PLUS 14 +#define FTS5_STAR 15 +#endif +/**************** End token definitions ***************************************/ /* The next sections is a series of control #defines. ** various aspects of the generated parser. @@ -177660,7 +234661,7 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); ** the minor type might be the name of the identifier. ** Each non-terminal can have a different minor type. ** Terminal symbols all have the same minor type, though. -** This macros defines the minor type for terminal +** This macros defines the minor type for terminal ** symbols. ** fts5YYMINORTYPE is the data type used for all minor types. ** This is typically a union of many types, one of @@ -177670,19 +234671,28 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); ** zero the stack is dynamically sized using realloc() ** sqlite3Fts5ParserARG_SDECL A static variable declaration for the %extra_argument ** sqlite3Fts5ParserARG_PDECL A parameter declaration for the %extra_argument +** sqlite3Fts5ParserARG_PARAM Code to pass %extra_argument as a subroutine parameter ** sqlite3Fts5ParserARG_STORE Code to store %extra_argument into fts5yypParser ** sqlite3Fts5ParserARG_FETCH Code to extract %extra_argument from fts5yypParser +** sqlite3Fts5ParserCTX_* As sqlite3Fts5ParserARG_ except for %extra_context +** fts5YYREALLOC Name of the realloc() function to use +** fts5YYFREE Name of the free() function to use +** fts5YYDYNSTACK True if stack space should be extended on heap ** fts5YYERRORSYMBOL is the code number of the error symbol. If not ** defined, then do no error processing. ** fts5YYNSTATE the combined number of states. ** fts5YYNRULE the number of rules in the grammar +** fts5YYNFTS5TOKEN Number of terminal symbols ** fts5YY_MAX_SHIFT Maximum value for shift actions ** fts5YY_MIN_SHIFTREDUCE Minimum value for shift-reduce actions ** fts5YY_MAX_SHIFTREDUCE Maximum value for shift-reduce actions -** fts5YY_MIN_REDUCE Maximum value for reduce actions ** fts5YY_ERROR_ACTION The fts5yy_action[] code for syntax error ** fts5YY_ACCEPT_ACTION The fts5yy_action[] code for accept ** fts5YY_NO_ACTION The fts5yy_action[] code for no-op +** fts5YY_MIN_REDUCE Minimum value for reduce actions +** fts5YY_MAX_REDUCE Maximum value for reduce actions +** fts5YY_MIN_DSTRCTR Minimum symbol value that has a destructor +** fts5YY_MAX_DSTRCTR Maximum symbol value that has a destructor */ #ifndef INTERFACE # define INTERFACE 1 @@ -177695,30 +234705,44 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic); typedef union { int fts5yyinit; sqlite3Fts5ParserFTS5TOKENTYPE fts5yy0; - Fts5Colset* fts5yy3; - Fts5ExprPhrase* fts5yy11; - Fts5ExprNode* fts5yy18; - int fts5yy20; - Fts5ExprNearset* fts5yy26; + int fts5yy4; + Fts5Colset* fts5yy11; + Fts5ExprNode* fts5yy24; + Fts5ExprNearset* fts5yy46; + Fts5ExprPhrase* fts5yy53; } fts5YYMINORTYPE; #ifndef fts5YYSTACKDEPTH #define fts5YYSTACKDEPTH 100 #endif #define sqlite3Fts5ParserARG_SDECL Fts5Parse *pParse; #define sqlite3Fts5ParserARG_PDECL ,Fts5Parse *pParse -#define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse = fts5yypParser->pParse -#define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse = pParse -#define fts5YYNSTATE 26 -#define fts5YYNRULE 24 -#define fts5YY_MAX_SHIFT 25 -#define fts5YY_MIN_SHIFTREDUCE 40 -#define fts5YY_MAX_SHIFTREDUCE 63 -#define fts5YY_MIN_REDUCE 64 -#define fts5YY_MAX_REDUCE 87 -#define fts5YY_ERROR_ACTION 88 -#define fts5YY_ACCEPT_ACTION 89 -#define fts5YY_NO_ACTION 90 +#define sqlite3Fts5ParserARG_PARAM ,pParse +#define sqlite3Fts5ParserARG_FETCH Fts5Parse *pParse=fts5yypParser->pParse; +#define sqlite3Fts5ParserARG_STORE fts5yypParser->pParse=pParse; +#define fts5YYREALLOC realloc +#define fts5YYFREE free +#define fts5YYDYNSTACK 0 +#define sqlite3Fts5ParserCTX_SDECL +#define sqlite3Fts5ParserCTX_PDECL +#define sqlite3Fts5ParserCTX_PARAM +#define sqlite3Fts5ParserCTX_FETCH +#define sqlite3Fts5ParserCTX_STORE +#define fts5YYNSTATE 35 +#define fts5YYNRULE 28 +#define fts5YYNRULE_WITH_ACTION 28 +#define fts5YYNFTS5TOKEN 16 +#define fts5YY_MAX_SHIFT 34 +#define fts5YY_MIN_SHIFTREDUCE 52 +#define fts5YY_MAX_SHIFTREDUCE 79 +#define fts5YY_ERROR_ACTION 80 +#define fts5YY_ACCEPT_ACTION 81 +#define fts5YY_NO_ACTION 82 +#define fts5YY_MIN_REDUCE 83 +#define fts5YY_MAX_REDUCE 110 +#define fts5YY_MIN_DSTRCTR 16 +#define fts5YY_MAX_DSTRCTR 24 /************* End control #defines *******************************************/ +#define fts5YY_NLOOKAHEAD ((int)(sizeof(fts5yy_lookahead)/sizeof(fts5yy_lookahead[0]))) /* Define the fts5yytestcase() macro to be a no-op if is not already defined ** otherwise. @@ -177732,11 +234756,27 @@ typedef union { # define fts5yytestcase(X) #endif +/* Macro to determine if stack space has the ability to grow using +** heap memory. +*/ +#if fts5YYSTACKDEPTH<=0 || fts5YYDYNSTACK +# define fts5YYGROWABLESTACK 1 +#else +# define fts5YYGROWABLESTACK 0 +#endif + +/* Guarantee a minimum number of initial stack slots. +*/ +#if fts5YYSTACKDEPTH<=0 +# undef fts5YYSTACKDEPTH +# define fts5YYSTACKDEPTH 2 /* Need a minimum stack size */ +#endif + /* Next are the tables used to determine what action to take based on the ** current state and lookahead token. These tables are used to implement ** functions that take a state number and lookahead value and return an -** action integer. +** action integer. ** ** Suppose the action integer is N. Then the action is determined as ** follows @@ -177747,9 +234787,6 @@ typedef union { ** N between fts5YY_MIN_SHIFTREDUCE Shift to an arbitrary state then ** and fts5YY_MAX_SHIFTREDUCE reduce by rule N-fts5YY_MIN_SHIFTREDUCE. ** -** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE -** and fts5YY_MAX_REDUCE - ** N == fts5YY_ERROR_ACTION A syntax error has occurred. ** ** N == fts5YY_ACCEPT_ACTION The parser accepts its input. @@ -177757,21 +234794,22 @@ typedef union { ** N == fts5YY_NO_ACTION No such action. Denotes unused ** slots in the fts5yy_action[] table. ** +** N between fts5YY_MIN_REDUCE Reduce by rule N-fts5YY_MIN_REDUCE +** and fts5YY_MAX_REDUCE +** ** The action table is constructed as a single large table named fts5yy_action[]. -** Given state S and lookahead X, the action is computed as +** Given state S and lookahead X, the action is computed as either: ** -** fts5yy_action[ fts5yy_shift_ofst[S] + X ] +** (A) N = fts5yy_action[ fts5yy_shift_ofst[S] + X ] +** (B) N = fts5yy_default[S] ** -** If the index value fts5yy_shift_ofst[S]+X is out of range or if the value -** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X or if fts5yy_shift_ofst[S] -** is equal to fts5YY_SHIFT_USE_DFLT, it means that the action is not in the table -** and that fts5yy_default[S] should be used instead. +** The (A) formula is preferred. The B formula is used instead if +** fts5yy_lookahead[fts5yy_shift_ofst[S]+X] is not equal to X. ** -** The formula above is for computing the action when the lookahead is +** The formulas above are for computing the action when the lookahead is ** a terminal symbol. If the lookahead is a non-terminal (as occurs after ** a reduce action) then the fts5yy_reduce_ofst[] array is used in place of -** the fts5yy_shift_ofst[] array and fts5YY_REDUCE_USE_DFLT is used in place of -** fts5YY_SHIFT_USE_DFLT. +** the fts5yy_shift_ofst[] array. ** ** The following are the tables generated in this section: ** @@ -177785,54 +234823,62 @@ typedef union { ** fts5yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define fts5YY_ACTTAB_COUNT (78) +#define fts5YY_ACTTAB_COUNT (105) static const fts5YYACTIONTYPE fts5yy_action[] = { - /* 0 */ 89, 15, 46, 5, 48, 24, 12, 19, 23, 14, - /* 10 */ 46, 5, 48, 24, 20, 21, 23, 43, 46, 5, - /* 20 */ 48, 24, 6, 18, 23, 17, 46, 5, 48, 24, - /* 30 */ 75, 7, 23, 25, 46, 5, 48, 24, 62, 47, - /* 40 */ 23, 48, 24, 7, 11, 23, 9, 3, 4, 2, - /* 50 */ 62, 50, 52, 44, 64, 3, 4, 2, 49, 4, - /* 60 */ 2, 1, 23, 11, 16, 9, 12, 2, 10, 61, - /* 70 */ 53, 59, 62, 60, 22, 13, 55, 8, + /* 0 */ 81, 20, 96, 6, 28, 99, 98, 26, 26, 18, + /* 10 */ 96, 6, 28, 17, 98, 56, 26, 19, 96, 6, + /* 20 */ 28, 14, 98, 14, 26, 31, 92, 96, 6, 28, + /* 30 */ 108, 98, 25, 26, 21, 96, 6, 28, 78, 98, + /* 40 */ 58, 26, 29, 96, 6, 28, 107, 98, 22, 26, + /* 50 */ 24, 16, 12, 11, 1, 13, 13, 24, 16, 23, + /* 60 */ 11, 33, 34, 13, 97, 8, 27, 32, 98, 7, + /* 70 */ 26, 3, 4, 5, 3, 4, 5, 3, 83, 4, + /* 80 */ 5, 3, 63, 5, 3, 62, 12, 2, 86, 13, + /* 90 */ 9, 30, 10, 10, 54, 57, 75, 78, 78, 53, + /* 100 */ 57, 15, 82, 82, 71, }; static const fts5YYCODETYPE fts5yy_lookahead[] = { - /* 0 */ 15, 16, 17, 18, 19, 20, 10, 11, 23, 16, - /* 10 */ 17, 18, 19, 20, 23, 24, 23, 16, 17, 18, - /* 20 */ 19, 20, 22, 23, 23, 16, 17, 18, 19, 20, - /* 30 */ 5, 6, 23, 16, 17, 18, 19, 20, 13, 17, - /* 40 */ 23, 19, 20, 6, 8, 23, 10, 1, 2, 3, - /* 50 */ 13, 9, 10, 7, 0, 1, 2, 3, 19, 2, - /* 60 */ 3, 6, 23, 8, 21, 10, 10, 3, 10, 25, - /* 70 */ 10, 10, 13, 25, 12, 10, 7, 5, + /* 0 */ 16, 17, 18, 19, 20, 22, 22, 24, 24, 17, + /* 10 */ 18, 19, 20, 7, 22, 9, 24, 17, 18, 19, + /* 20 */ 20, 9, 22, 9, 24, 13, 17, 18, 19, 20, + /* 30 */ 26, 22, 24, 24, 17, 18, 19, 20, 15, 22, + /* 40 */ 9, 24, 17, 18, 19, 20, 26, 22, 21, 24, + /* 50 */ 6, 7, 9, 9, 10, 12, 12, 6, 7, 21, + /* 60 */ 9, 24, 25, 12, 18, 5, 20, 14, 22, 5, + /* 70 */ 24, 3, 1, 2, 3, 1, 2, 3, 0, 1, + /* 80 */ 2, 3, 11, 2, 3, 11, 9, 10, 5, 12, + /* 90 */ 23, 24, 10, 10, 8, 9, 9, 15, 15, 8, + /* 100 */ 9, 9, 27, 27, 11, 27, 27, 27, 27, 27, + /* 110 */ 27, 27, 27, 27, 27, 27, 27, 27, 27, 27, + /* 120 */ 27, }; -#define fts5YY_SHIFT_USE_DFLT (-5) -#define fts5YY_SHIFT_COUNT (25) -#define fts5YY_SHIFT_MIN (-4) -#define fts5YY_SHIFT_MAX (72) -static const signed char fts5yy_shift_ofst[] = { - /* 0 */ 55, 55, 55, 55, 55, 36, -4, 56, 58, 25, - /* 10 */ 37, 60, 59, 59, 46, 54, 42, 57, 62, 61, - /* 20 */ 62, 69, 65, 62, 72, 64, +#define fts5YY_SHIFT_COUNT (34) +#define fts5YY_SHIFT_MIN (0) +#define fts5YY_SHIFT_MAX (93) +static const unsigned char fts5yy_shift_ofst[] = { + /* 0 */ 44, 44, 44, 44, 44, 44, 51, 77, 43, 12, + /* 10 */ 14, 83, 82, 14, 23, 23, 31, 31, 71, 74, + /* 20 */ 78, 81, 86, 91, 6, 53, 53, 60, 64, 68, + /* 30 */ 53, 87, 92, 53, 93, }; -#define fts5YY_REDUCE_USE_DFLT (-16) -#define fts5YY_REDUCE_COUNT (13) -#define fts5YY_REDUCE_MIN (-15) -#define fts5YY_REDUCE_MAX (48) +#define fts5YY_REDUCE_COUNT (17) +#define fts5YY_REDUCE_MIN (-17) +#define fts5YY_REDUCE_MAX (67) static const signed char fts5yy_reduce_ofst[] = { - /* 0 */ -15, -7, 1, 9, 17, 22, -9, 0, 39, 44, - /* 10 */ 44, 43, 44, 48, + /* 0 */ -16, -8, 0, 9, 17, 25, 46, -17, -17, 37, + /* 10 */ 67, 4, 4, 8, 4, 20, 27, 38, }; static const fts5YYACTIONTYPE fts5yy_default[] = { - /* 0 */ 88, 88, 88, 88, 88, 69, 82, 88, 88, 87, - /* 10 */ 87, 88, 87, 87, 88, 88, 88, 66, 80, 88, - /* 20 */ 81, 88, 88, 78, 88, 65, + /* 0 */ 80, 80, 80, 80, 80, 80, 95, 80, 80, 105, + /* 10 */ 80, 110, 110, 80, 110, 110, 80, 80, 80, 80, + /* 20 */ 80, 91, 80, 80, 80, 101, 100, 80, 80, 90, + /* 30 */ 103, 80, 80, 104, 80, }; /********** End of lemon-generated parsing tables *****************************/ -/* The next table maps tokens (terminal symbols) into fallback tokens. +/* The next table maps tokens (terminal symbols) into fallback tokens. ** If a construct like the following: -** +** ** %fallback ID X Y Z. ** ** appears in the grammar, then ID becomes a fallback token for X, Y, @@ -177877,23 +234923,22 @@ typedef struct fts5yyStackEntry fts5yyStackEntry; /* The state of the parser is completely contained in an instance of ** the following structure */ struct fts5yyParser { - int fts5yyidx; /* Index of top element in stack */ + fts5yyStackEntry *fts5yytos; /* Pointer to top element of the stack */ #ifdef fts5YYTRACKMAXSTACKDEPTH - int fts5yyidxMax; /* Maximum value of fts5yyidx */ + int fts5yyhwm; /* High-water mark of the stack */ #endif #ifndef fts5YYNOERRORRECOVERY int fts5yyerrcnt; /* Shifts left before out of the error */ #endif sqlite3Fts5ParserARG_SDECL /* A place to hold %extra_argument */ -#if fts5YYSTACKDEPTH<=0 - int fts5yystksz; /* Current side of the stack */ - fts5yyStackEntry *fts5yystack; /* The parser's stack */ -#else - fts5yyStackEntry fts5yystack[fts5YYSTACKDEPTH]; /* The parser's stack */ -#endif + sqlite3Fts5ParserCTX_SDECL /* A place to hold %extra_context */ + fts5yyStackEntry *fts5yystackEnd; /* Last entry in the stack */ + fts5yyStackEntry *fts5yystack; /* The parser stack */ + fts5yyStackEntry fts5yystk0[fts5YYSTACKDEPTH]; /* Initial stack space */ }; typedef struct fts5yyParser fts5yyParser; +/* #include */ #ifndef NDEBUG /* #include */ static FILE *fts5yyTraceFILE = 0; @@ -177901,10 +234946,10 @@ static char *fts5yyTracePrompt = 0; #endif /* NDEBUG */ #ifndef NDEBUG -/* +/* ** Turn parser tracing on by giving a stream to which to write the trace ** and a prompt to preface each trace message. Tracing is turned off -** by making either argument NULL +** by making either argument NULL ** ** Inputs: **
                @@ -177926,73 +234971,115 @@ static void sqlite3Fts5ParserTrace(FILE *TraceFILE, char *zTracePrompt){ } #endif /* NDEBUG */ -#ifndef NDEBUG +#if defined(fts5YYCOVERAGE) || !defined(NDEBUG) /* For tracing shifts, the names of all terminals and nonterminals ** are required. The following table supplies these names */ -static const char *const fts5yyTokenName[] = { - "$", "OR", "AND", "NOT", - "TERM", "COLON", "LP", "RP", - "LCP", "RCP", "STRING", "COMMA", - "PLUS", "STAR", "error", "input", - "expr", "cnearset", "exprlist", "nearset", - "colset", "colsetlist", "nearphrases", "phrase", - "neardist_opt", "star_opt", +static const char *const fts5yyTokenName[] = { + /* 0 */ "$", + /* 1 */ "OR", + /* 2 */ "AND", + /* 3 */ "NOT", + /* 4 */ "TERM", + /* 5 */ "COLON", + /* 6 */ "MINUS", + /* 7 */ "LCP", + /* 8 */ "RCP", + /* 9 */ "STRING", + /* 10 */ "LP", + /* 11 */ "RP", + /* 12 */ "CARET", + /* 13 */ "COMMA", + /* 14 */ "PLUS", + /* 15 */ "STAR", + /* 16 */ "input", + /* 17 */ "expr", + /* 18 */ "cnearset", + /* 19 */ "exprlist", + /* 20 */ "colset", + /* 21 */ "colsetlist", + /* 22 */ "nearset", + /* 23 */ "nearphrases", + /* 24 */ "phrase", + /* 25 */ "neardist_opt", + /* 26 */ "star_opt", }; -#endif /* NDEBUG */ +#endif /* defined(fts5YYCOVERAGE) || !defined(NDEBUG) */ #ifndef NDEBUG /* For tracing reduce actions, the names of all rules are required. */ static const char *const fts5yyRuleName[] = { /* 0 */ "input ::= expr", - /* 1 */ "expr ::= expr AND expr", - /* 2 */ "expr ::= expr OR expr", - /* 3 */ "expr ::= expr NOT expr", - /* 4 */ "expr ::= LP expr RP", - /* 5 */ "expr ::= exprlist", - /* 6 */ "exprlist ::= cnearset", - /* 7 */ "exprlist ::= exprlist cnearset", - /* 8 */ "cnearset ::= nearset", - /* 9 */ "cnearset ::= colset COLON nearset", - /* 10 */ "colset ::= LCP colsetlist RCP", - /* 11 */ "colset ::= STRING", - /* 12 */ "colsetlist ::= colsetlist STRING", - /* 13 */ "colsetlist ::= STRING", - /* 14 */ "nearset ::= phrase", - /* 15 */ "nearset ::= STRING LP nearphrases neardist_opt RP", - /* 16 */ "nearphrases ::= phrase", - /* 17 */ "nearphrases ::= nearphrases phrase", - /* 18 */ "neardist_opt ::=", - /* 19 */ "neardist_opt ::= COMMA STRING", - /* 20 */ "phrase ::= phrase PLUS STRING star_opt", - /* 21 */ "phrase ::= STRING star_opt", - /* 22 */ "star_opt ::= STAR", - /* 23 */ "star_opt ::=", + /* 1 */ "colset ::= MINUS LCP colsetlist RCP", + /* 2 */ "colset ::= LCP colsetlist RCP", + /* 3 */ "colset ::= STRING", + /* 4 */ "colset ::= MINUS STRING", + /* 5 */ "colsetlist ::= colsetlist STRING", + /* 6 */ "colsetlist ::= STRING", + /* 7 */ "expr ::= expr AND expr", + /* 8 */ "expr ::= expr OR expr", + /* 9 */ "expr ::= expr NOT expr", + /* 10 */ "expr ::= colset COLON LP expr RP", + /* 11 */ "expr ::= LP expr RP", + /* 12 */ "expr ::= exprlist", + /* 13 */ "exprlist ::= cnearset", + /* 14 */ "exprlist ::= exprlist cnearset", + /* 15 */ "cnearset ::= nearset", + /* 16 */ "cnearset ::= colset COLON nearset", + /* 17 */ "nearset ::= phrase", + /* 18 */ "nearset ::= CARET phrase", + /* 19 */ "nearset ::= STRING LP nearphrases neardist_opt RP", + /* 20 */ "nearphrases ::= phrase", + /* 21 */ "nearphrases ::= nearphrases phrase", + /* 22 */ "neardist_opt ::=", + /* 23 */ "neardist_opt ::= COMMA STRING", + /* 24 */ "phrase ::= phrase PLUS STRING star_opt", + /* 25 */ "phrase ::= STRING star_opt", + /* 26 */ "star_opt ::= STAR", + /* 27 */ "star_opt ::=", }; #endif /* NDEBUG */ -#if fts5YYSTACKDEPTH<=0 +#if fts5YYGROWABLESTACK /* -** Try to increase the size of the parser stack. +** Try to increase the size of the parser stack. Return the number +** of errors. Return 0 on success. */ -static void fts5yyGrowStack(fts5yyParser *p){ +static int fts5yyGrowStack(fts5yyParser *p){ + int oldSize = 1 + (int)(p->fts5yystackEnd - p->fts5yystack); int newSize; + int idx; fts5yyStackEntry *pNew; - newSize = p->fts5yystksz*2 + 100; - pNew = realloc(p->fts5yystack, newSize*sizeof(pNew[0])); - if( pNew ){ - p->fts5yystack = pNew; - p->fts5yystksz = newSize; -#ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sStack grows to %d entries!\n", - fts5yyTracePrompt, p->fts5yystksz); - } -#endif + newSize = oldSize*2 + 100; + idx = (int)(p->fts5yytos - p->fts5yystack); + if( p->fts5yystack==p->fts5yystk0 ){ + pNew = fts5YYREALLOC(0, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; + memcpy(pNew, p->fts5yystack, oldSize*sizeof(pNew[0])); + }else{ + pNew = fts5YYREALLOC(p->fts5yystack, newSize*sizeof(pNew[0])); + if( pNew==0 ) return 1; } + p->fts5yystack = pNew; + p->fts5yytos = &p->fts5yystack[idx]; +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sStack grows from %d to %d entries.\n", + fts5yyTracePrompt, oldSize, newSize); + } +#endif + p->fts5yystackEnd = &p->fts5yystack[newSize-1]; + return 0; } +#endif /* fts5YYGROWABLESTACK */ + +#if !fts5YYGROWABLESTACK +/* For builds that do no have a growable stack, fts5yyGrowStack always +** returns an error. +*/ +# define fts5yyGrowStack(X) 1 #endif /* Datatype of the argument to the memory allocated passed as the @@ -178004,7 +235091,26 @@ static void fts5yyGrowStack(fts5yyParser *p){ # define fts5YYMALLOCARGTYPE size_t #endif -/* +/* Initialize a new parser that has already been allocated. +*/ +static void sqlite3Fts5ParserInit(void *fts5yypRawParser sqlite3Fts5ParserCTX_PDECL){ + fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yypRawParser; + sqlite3Fts5ParserCTX_STORE +#ifdef fts5YYTRACKMAXSTACKDEPTH + fts5yypParser->fts5yyhwm = 0; +#endif + fts5yypParser->fts5yystack = fts5yypParser->fts5yystk0; + fts5yypParser->fts5yystackEnd = &fts5yypParser->fts5yystack[fts5YYSTACKDEPTH-1]; +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + fts5yypParser->fts5yytos = fts5yypParser->fts5yystack; + fts5yypParser->fts5yystack[0].stateno = 0; + fts5yypParser->fts5yystack[0].major = 0; +} + +#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK +/* ** This function allocates a new parser. ** The only argument is a pointer to a function which works like ** malloc. @@ -178016,27 +235122,22 @@ static void fts5yyGrowStack(fts5yyParser *p){ ** A pointer to a parser. This pointer is used in subsequent calls ** to sqlite3Fts5Parser and sqlite3Fts5ParserFree. */ -static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPE)){ - fts5yyParser *pParser; - pParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPE)sizeof(fts5yyParser) ); - if( pParser ){ - pParser->fts5yyidx = -1; -#ifdef fts5YYTRACKMAXSTACKDEPTH - pParser->fts5yyidxMax = 0; -#endif -#if fts5YYSTACKDEPTH<=0 - pParser->fts5yystack = NULL; - pParser->fts5yystksz = 0; - fts5yyGrowStack(pParser); -#endif +static void *sqlite3Fts5ParserAlloc(void *(*mallocProc)(fts5YYMALLOCARGTYPE) sqlite3Fts5ParserCTX_PDECL){ + fts5yyParser *fts5yypParser; + fts5yypParser = (fts5yyParser*)(*mallocProc)( (fts5YYMALLOCARGTYPE)sizeof(fts5yyParser) ); + if( fts5yypParser ){ + sqlite3Fts5ParserCTX_STORE + sqlite3Fts5ParserInit(fts5yypParser sqlite3Fts5ParserCTX_PARAM); } - return pParser; + return (void*)fts5yypParser; } +#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */ + /* The following function deletes the "minor type" or semantic value ** associated with a symbol. The symbol can be either a terminal ** or nonterminal. "fts5yymajor" is the symbol code, and "fts5yypminor" is -** a pointer to the value to be deleted. The code used to do the +** a pointer to the value to be deleted. The code used to do the ** deletions is derived from the %destructor and/or %token_destructor ** directives of the input grammar. */ @@ -178045,12 +235146,13 @@ static void fts5yy_destructor( fts5YYCODETYPE fts5yymajor, /* Type code for object to destroy */ fts5YYMINORTYPE *fts5yypminor /* The object to be destroyed */ ){ - sqlite3Fts5ParserARG_FETCH; + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH switch( fts5yymajor ){ /* Here is inserted the actions which take place when a ** terminal or non-terminal is destroyed. This can happen ** when the symbol is popped from the stack during a - ** reduce or during error processing or when a parser is + ** reduce or during error processing or when a parser is ** being destroyed before it is finished parsing. ** ** Note: during a reduce, the only symbols destroyed are those @@ -178058,33 +235160,33 @@ static void fts5yy_destructor( ** inside the C code. */ /********* Begin destructor definitions ***************************************/ - case 15: /* input */ + case 16: /* input */ { - (void)pParse; + (void)pParse; } break; - case 16: /* expr */ - case 17: /* cnearset */ - case 18: /* exprlist */ + case 17: /* expr */ + case 18: /* cnearset */ + case 19: /* exprlist */ { - sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy18)); -} - break; - case 19: /* nearset */ - case 22: /* nearphrases */ -{ - sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy26)); + sqlite3Fts5ParseNodeFree((fts5yypminor->fts5yy24)); } break; case 20: /* colset */ case 21: /* colsetlist */ { - sqlite3_free((fts5yypminor->fts5yy3)); + sqlite3_free((fts5yypminor->fts5yy11)); } break; - case 23: /* phrase */ + case 22: /* nearset */ + case 23: /* nearphrases */ { - sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy11)); + sqlite3Fts5ParseNearsetFree((fts5yypminor->fts5yy46)); +} + break; + case 24: /* phrase */ +{ + sqlite3Fts5ParsePhraseFree((fts5yypminor->fts5yy53)); } break; /********* End destructor definitions *****************************************/ @@ -178100,8 +235202,9 @@ static void fts5yy_destructor( */ static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ fts5yyStackEntry *fts5yytos; - assert( pParser->fts5yyidx>=0 ); - fts5yytos = &pParser->fts5yystack[pParser->fts5yyidx--]; + assert( pParser->fts5yytos!=0 ); + assert( pParser->fts5yytos > pParser->fts5yystack ); + fts5yytos = pParser->fts5yytos--; #ifndef NDEBUG if( fts5yyTraceFILE ){ fprintf(fts5yyTraceFILE,"%sPopping %s\n", @@ -178112,7 +235215,36 @@ static void fts5yy_pop_parser_stack(fts5yyParser *pParser){ fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); } -/* +/* +** Clear all secondary memory allocations from the parser +*/ +static void sqlite3Fts5ParserFinalize(void *p){ + fts5yyParser *pParser = (fts5yyParser*)p; + + /* In-lined version of calling fts5yy_pop_parser_stack() for each + ** element left in the stack */ + fts5yyStackEntry *fts5yytos = pParser->fts5yytos; + while( fts5yytos>pParser->fts5yystack ){ +#ifndef NDEBUG + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE,"%sPopping %s\n", + fts5yyTracePrompt, + fts5yyTokenName[fts5yytos->major]); + } +#endif + if( fts5yytos->major>=fts5YY_MIN_DSTRCTR ){ + fts5yy_destructor(pParser, fts5yytos->major, &fts5yytos->minor); + } + fts5yytos--; + } + +#if fts5YYGROWABLESTACK + if( pParser->fts5yystack!=pParser->fts5yystk0 ) fts5YYFREE(pParser->fts5yystack); +#endif +} + +#ifndef sqlite3Fts5Parser_ENGINEALWAYSONSTACK +/* ** Deallocate and destroy a parser. Destructors are called for ** all stack elements before shutting the parser down. ** @@ -178124,16 +235256,13 @@ static void sqlite3Fts5ParserFree( void *p, /* The parser to be deleted */ void (*freeProc)(void*) /* Function used to reclaim memory */ ){ - fts5yyParser *pParser = (fts5yyParser*)p; #ifndef fts5YYPARSEFREENEVERNULL - if( pParser==0 ) return; + if( p==0 ) return; #endif - while( pParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(pParser); -#if fts5YYSTACKDEPTH<=0 - free(pParser->fts5yystack); -#endif - (*freeProc)((void*)pParser); + sqlite3Fts5ParserFinalize(p); + (*freeProc)(p); } +#endif /* sqlite3Fts5Parser_ENGINEALWAYSONSTACK */ /* ** Return the peak depth of the stack for a parser. @@ -178141,7 +235270,44 @@ static void sqlite3Fts5ParserFree( #ifdef fts5YYTRACKMAXSTACKDEPTH static int sqlite3Fts5ParserStackPeak(void *p){ fts5yyParser *pParser = (fts5yyParser*)p; - return pParser->fts5yyidxMax; + return pParser->fts5yyhwm; +} +#endif + +/* This array of booleans keeps track of the parser statement +** coverage. The element fts5yycoverage[X][Y] is set when the parser +** is in state X and has a lookahead token Y. In a well-tested +** systems, every element of this matrix should end up being set. +*/ +#if defined(fts5YYCOVERAGE) +static unsigned char fts5yycoverage[fts5YYNSTATE][fts5YYNFTS5TOKEN]; +#endif + +/* +** Write into out a description of every state/lookahead combination that +** +** (1) has not been used by the parser, and +** (2) is not a syntax error. +** +** Return the number of missed state/lookahead combinations. +*/ +#if defined(fts5YYCOVERAGE) +static int sqlite3Fts5ParserCoverage(FILE *out){ + int stateno, iLookAhead, i; + int nMissed = 0; + for(stateno=0; statenofts5yystack[pParser->fts5yyidx].stateno; - - if( stateno>=fts5YY_MIN_REDUCE ) return stateno; + + if( stateno>fts5YY_MAX_SHIFT ) return stateno; assert( stateno <= fts5YY_SHIFT_COUNT ); +#if defined(fts5YYCOVERAGE) + fts5yycoverage[stateno][iLookAhead] = 1; +#endif do{ i = fts5yy_shift_ofst[stateno]; - if( i==fts5YY_SHIFT_USE_DFLT ) return fts5yy_default[stateno]; + assert( i>=0 ); + assert( i<=fts5YY_ACTTAB_COUNT ); + assert( i+fts5YYNFTS5TOKEN<=(int)fts5YY_NLOOKAHEAD ); assert( iLookAhead!=fts5YYNOCODE ); + assert( iLookAhead < fts5YYNFTS5TOKEN ); i += iLookAhead; - if( i<0 || i>=fts5YY_ACTTAB_COUNT || fts5yy_lookahead[i]!=iLookAhead ){ - if( iLookAhead>0 ){ + assert( i<(int)fts5YY_NLOOKAHEAD ); + if( fts5yy_lookahead[i]!=iLookAhead ){ #ifdef fts5YYFALLBACK - fts5YYCODETYPE iFallback; /* Fallback token */ - if( iLookAhead %s\n", - fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]); - } -#endif - assert( fts5yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ - iLookAhead = iFallback; - continue; + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE, "%sFALLBACK %s => %s\n", + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], fts5yyTokenName[iFallback]); } +#endif + assert( fts5yyFallback[iFallback]==0 ); /* Fallback loop must terminate */ + iLookAhead = iFallback; + continue; + } #endif #ifdef fts5YYWILDCARD - { - int j = i - iLookAhead + fts5YYWILDCARD; - if( -#if fts5YY_SHIFT_MIN+fts5YYWILDCARD<0 - j>=0 && -#endif -#if fts5YY_SHIFT_MAX+fts5YYWILDCARD>=fts5YY_ACTTAB_COUNT - j0 ){ #ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n", - fts5yyTracePrompt, fts5yyTokenName[iLookAhead], - fts5yyTokenName[fts5YYWILDCARD]); - } -#endif /* NDEBUG */ - return fts5yy_action[j]; + if( fts5yyTraceFILE ){ + fprintf(fts5yyTraceFILE, "%sWILDCARD %s => %s\n", + fts5yyTracePrompt, fts5yyTokenName[iLookAhead], + fts5yyTokenName[fts5YYWILDCARD]); } +#endif /* NDEBUG */ + return fts5yy_action[j]; } -#endif /* fts5YYWILDCARD */ } +#endif /* fts5YYWILDCARD */ return fts5yy_default[stateno]; }else{ + assert( i>=0 && i<(int)(sizeof(fts5yy_action)/sizeof(fts5yy_action[0])) ); return fts5yy_action[i]; } }while(1); @@ -178215,8 +235380,8 @@ static unsigned int fts5yy_find_shift_action( ** Find the appropriate action for a parser given the non-terminal ** look-ahead token iLookAhead. */ -static int fts5yy_find_reduce_action( - int stateno, /* Current state number */ +static fts5YYACTIONTYPE fts5yy_find_reduce_action( + fts5YYACTIONTYPE stateno, /* Current state number */ fts5YYCODETYPE iLookAhead /* The look-ahead token */ ){ int i; @@ -178228,7 +235393,6 @@ static int fts5yy_find_reduce_action( assert( stateno<=fts5YY_REDUCE_COUNT ); #endif i = fts5yy_reduce_ofst[stateno]; - assert( i!=fts5YY_REDUCE_USE_DFLT ); assert( iLookAhead!=fts5YYNOCODE ); i += iLookAhead; #ifdef fts5YYERRORSYMBOL @@ -178246,41 +235410,43 @@ static int fts5yy_find_reduce_action( ** The following routine is called if the stack overflows. */ static void fts5yyStackOverflow(fts5yyParser *fts5yypParser){ - sqlite3Fts5ParserARG_FETCH; - fts5yypParser->fts5yyidx--; + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH #ifndef NDEBUG if( fts5yyTraceFILE ){ fprintf(fts5yyTraceFILE,"%sStack Overflow!\n",fts5yyTracePrompt); } #endif - while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser); + while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser); /* Here code is inserted which will execute if the parser ** stack every overflows */ /******** Begin %stack_overflow code ******************************************/ sqlite3Fts5ParseError(pParse, "fts5: parser stack overflow"); /******** End %stack_overflow code ********************************************/ - sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument var */ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument var */ + sqlite3Fts5ParserCTX_STORE } /* ** Print tracing information for a SHIFT action */ #ifndef NDEBUG -static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState){ +static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState, const char *zTag){ if( fts5yyTraceFILE ){ if( fts5yyNewStatefts5yystack[fts5yypParser->fts5yyidx].major], + fprintf(fts5yyTraceFILE,"%s%s '%s', go to state %d\n", + fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major], fts5yyNewState); }else{ - fprintf(fts5yyTraceFILE,"%sShift '%s'\n", - fts5yyTracePrompt,fts5yyTokenName[fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major]); + fprintf(fts5yyTraceFILE,"%s%s '%s', pending reduce %d\n", + fts5yyTracePrompt, zTag, fts5yyTokenName[fts5yypParser->fts5yytos->major], + fts5yyNewState - fts5YY_MIN_REDUCE); } } } #else -# define fts5yyTraceShift(X,Y) +# define fts5yyTraceShift(X,Y,Z) #endif /* @@ -178288,69 +235454,101 @@ static void fts5yyTraceShift(fts5yyParser *fts5yypParser, int fts5yyNewState){ */ static void fts5yy_shift( fts5yyParser *fts5yypParser, /* The parser to be shifted */ - int fts5yyNewState, /* The new state to shift in */ - int fts5yyMajor, /* The major token to shift in */ + fts5YYACTIONTYPE fts5yyNewState, /* The new state to shift in */ + fts5YYCODETYPE fts5yyMajor, /* The major token to shift in */ sqlite3Fts5ParserFTS5TOKENTYPE fts5yyMinor /* The minor token to shift in */ ){ fts5yyStackEntry *fts5yytos; - fts5yypParser->fts5yyidx++; + fts5yypParser->fts5yytos++; #ifdef fts5YYTRACKMAXSTACKDEPTH - if( fts5yypParser->fts5yyidx>fts5yypParser->fts5yyidxMax ){ - fts5yypParser->fts5yyidxMax = fts5yypParser->fts5yyidx; + if( (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){ + fts5yypParser->fts5yyhwm++; + assert( fts5yypParser->fts5yyhwm == (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack) ); } #endif -#if fts5YYSTACKDEPTH>0 - if( fts5yypParser->fts5yyidx>=fts5YYSTACKDEPTH ){ - fts5yyStackOverflow(fts5yypParser); - return; - } -#else - if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz ){ - fts5yyGrowStack(fts5yypParser); - if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz ){ + fts5yytos = fts5yypParser->fts5yytos; + if( fts5yytos>fts5yypParser->fts5yystackEnd ){ + if( fts5yyGrowStack(fts5yypParser) ){ + fts5yypParser->fts5yytos--; fts5yyStackOverflow(fts5yypParser); return; } + fts5yytos = fts5yypParser->fts5yytos; + assert( fts5yytos <= fts5yypParser->fts5yystackEnd ); } -#endif - fts5yytos = &fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx]; - fts5yytos->stateno = (fts5YYACTIONTYPE)fts5yyNewState; - fts5yytos->major = (fts5YYCODETYPE)fts5yyMajor; + if( fts5yyNewState > fts5YY_MAX_SHIFT ){ + fts5yyNewState += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; + } + fts5yytos->stateno = fts5yyNewState; + fts5yytos->major = fts5yyMajor; fts5yytos->minor.fts5yy0 = fts5yyMinor; - fts5yyTraceShift(fts5yypParser, fts5yyNewState); + fts5yyTraceShift(fts5yypParser, fts5yyNewState, "Shift"); } -/* The following table contains information about every rule that -** is used during the reduce. -*/ -static const struct { - fts5YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ - unsigned char nrhs; /* Number of right-hand side symbols in the rule */ -} fts5yyRuleInfo[] = { - { 15, 1 }, - { 16, 3 }, - { 16, 3 }, - { 16, 3 }, - { 16, 3 }, - { 16, 1 }, - { 18, 1 }, - { 18, 2 }, - { 17, 1 }, - { 17, 3 }, - { 20, 3 }, - { 20, 1 }, - { 21, 2 }, - { 21, 1 }, - { 19, 1 }, - { 19, 5 }, - { 22, 1 }, - { 22, 2 }, - { 24, 0 }, - { 24, 2 }, - { 23, 4 }, - { 23, 2 }, - { 25, 1 }, - { 25, 0 }, +/* For rule J, fts5yyRuleInfoLhs[J] contains the symbol on the left-hand side +** of that rule */ +static const fts5YYCODETYPE fts5yyRuleInfoLhs[] = { + 16, /* (0) input ::= expr */ + 20, /* (1) colset ::= MINUS LCP colsetlist RCP */ + 20, /* (2) colset ::= LCP colsetlist RCP */ + 20, /* (3) colset ::= STRING */ + 20, /* (4) colset ::= MINUS STRING */ + 21, /* (5) colsetlist ::= colsetlist STRING */ + 21, /* (6) colsetlist ::= STRING */ + 17, /* (7) expr ::= expr AND expr */ + 17, /* (8) expr ::= expr OR expr */ + 17, /* (9) expr ::= expr NOT expr */ + 17, /* (10) expr ::= colset COLON LP expr RP */ + 17, /* (11) expr ::= LP expr RP */ + 17, /* (12) expr ::= exprlist */ + 19, /* (13) exprlist ::= cnearset */ + 19, /* (14) exprlist ::= exprlist cnearset */ + 18, /* (15) cnearset ::= nearset */ + 18, /* (16) cnearset ::= colset COLON nearset */ + 22, /* (17) nearset ::= phrase */ + 22, /* (18) nearset ::= CARET phrase */ + 22, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */ + 23, /* (20) nearphrases ::= phrase */ + 23, /* (21) nearphrases ::= nearphrases phrase */ + 25, /* (22) neardist_opt ::= */ + 25, /* (23) neardist_opt ::= COMMA STRING */ + 24, /* (24) phrase ::= phrase PLUS STRING star_opt */ + 24, /* (25) phrase ::= STRING star_opt */ + 26, /* (26) star_opt ::= STAR */ + 26, /* (27) star_opt ::= */ +}; + +/* For rule J, fts5yyRuleInfoNRhs[J] contains the negative of the number +** of symbols on the right-hand side of that rule. */ +static const signed char fts5yyRuleInfoNRhs[] = { + -1, /* (0) input ::= expr */ + -4, /* (1) colset ::= MINUS LCP colsetlist RCP */ + -3, /* (2) colset ::= LCP colsetlist RCP */ + -1, /* (3) colset ::= STRING */ + -2, /* (4) colset ::= MINUS STRING */ + -2, /* (5) colsetlist ::= colsetlist STRING */ + -1, /* (6) colsetlist ::= STRING */ + -3, /* (7) expr ::= expr AND expr */ + -3, /* (8) expr ::= expr OR expr */ + -3, /* (9) expr ::= expr NOT expr */ + -5, /* (10) expr ::= colset COLON LP expr RP */ + -3, /* (11) expr ::= LP expr RP */ + -1, /* (12) expr ::= exprlist */ + -1, /* (13) exprlist ::= cnearset */ + -2, /* (14) exprlist ::= exprlist cnearset */ + -1, /* (15) cnearset ::= nearset */ + -3, /* (16) cnearset ::= colset COLON nearset */ + -1, /* (17) nearset ::= phrase */ + -2, /* (18) nearset ::= CARET phrase */ + -5, /* (19) nearset ::= STRING LP nearphrases neardist_opt RP */ + -1, /* (20) nearphrases ::= phrase */ + -2, /* (21) nearphrases ::= nearphrases phrase */ + 0, /* (22) neardist_opt ::= */ + -2, /* (23) neardist_opt ::= COMMA STRING */ + -4, /* (24) phrase ::= phrase PLUS STRING star_opt */ + -2, /* (25) phrase ::= STRING star_opt */ + -1, /* (26) star_opt ::= STAR */ + 0, /* (27) star_opt ::= */ }; static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */ @@ -178358,49 +235556,28 @@ static void fts5yy_accept(fts5yyParser*); /* Forward Declaration */ /* ** Perform a reduce action and the shift that must immediately ** follow the reduce. +** +** The fts5yyLookahead and fts5yyLookaheadToken parameters provide reduce actions +** access to the lookahead token (if any). The fts5yyLookahead will be fts5YYNOCODE +** if the lookahead token has already been consumed. As this procedure is +** only called from one place, optimizing compilers will in-line it, which +** means that the extra parameters have no performance impact. */ -static void fts5yy_reduce( +static fts5YYACTIONTYPE fts5yy_reduce( fts5yyParser *fts5yypParser, /* The parser */ - unsigned int fts5yyruleno /* Number of the rule by which to reduce */ + unsigned int fts5yyruleno, /* Number of the rule by which to reduce */ + int fts5yyLookahead, /* Lookahead token, or fts5YYNOCODE if none */ + sqlite3Fts5ParserFTS5TOKENTYPE fts5yyLookaheadToken /* Value of the lookahead token */ + sqlite3Fts5ParserCTX_PDECL /* %extra_context */ ){ int fts5yygoto; /* The next state */ - int fts5yyact; /* The next action */ + fts5YYACTIONTYPE fts5yyact; /* The next action */ fts5yyStackEntry *fts5yymsp; /* The top of the parser's stack */ int fts5yysize; /* Amount to pop the stack */ - sqlite3Fts5ParserARG_FETCH; - fts5yymsp = &fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx]; -#ifndef NDEBUG - if( fts5yyTraceFILE && fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) ){ - fts5yysize = fts5yyRuleInfo[fts5yyruleno].nrhs; - fprintf(fts5yyTraceFILE, "%sReduce [%s], go to state %d.\n", fts5yyTracePrompt, - fts5yyRuleName[fts5yyruleno], fts5yymsp[-fts5yysize].stateno); - } -#endif /* NDEBUG */ - - /* Check that the stack is large enough to grow by a single entry - ** if the RHS of the rule is empty. This ensures that there is room - ** enough on the stack to push the LHS value */ - if( fts5yyRuleInfo[fts5yyruleno].nrhs==0 ){ -#ifdef fts5YYTRACKMAXSTACKDEPTH - if( fts5yypParser->fts5yyidx>fts5yypParser->fts5yyidxMax ){ - fts5yypParser->fts5yyidxMax = fts5yypParser->fts5yyidx; - } -#endif -#if fts5YYSTACKDEPTH>0 - if( fts5yypParser->fts5yyidx>=fts5YYSTACKDEPTH-1 ){ - fts5yyStackOverflow(fts5yypParser); - return; - } -#else - if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz-1 ){ - fts5yyGrowStack(fts5yypParser); - if( fts5yypParser->fts5yyidx>=fts5yypParser->fts5yystksz-1 ){ - fts5yyStackOverflow(fts5yypParser); - return; - } - } -#endif - } + sqlite3Fts5ParserARG_FETCH + (void)fts5yyLookahead; + (void)fts5yyLookaheadToken; + fts5yymsp = fts5yypParser->fts5yytos; switch( fts5yyruleno ){ /* Beginning here are the reduction cases. A typical example @@ -178414,141 +235591,167 @@ static void fts5yy_reduce( /********** Begin reduce actions **********************************************/ fts5YYMINORTYPE fts5yylhsminor; case 0: /* input ::= expr */ -{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy18); } +{ sqlite3Fts5ParseFinished(pParse, fts5yymsp[0].minor.fts5yy24); } break; - case 1: /* expr ::= expr AND expr */ + case 1: /* colset ::= MINUS LCP colsetlist RCP */ { - fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0); + fts5yymsp[-3].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11); } - fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; break; - case 2: /* expr ::= expr OR expr */ + case 2: /* colset ::= LCP colsetlist RCP */ +{ fts5yymsp[-2].minor.fts5yy11 = fts5yymsp[-1].minor.fts5yy11; } + break; + case 3: /* colset ::= STRING */ { - fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_OR, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0); + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); } - fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; + fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11; break; - case 3: /* expr ::= expr NOT expr */ + case 4: /* colset ::= MINUS STRING */ { - fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_NOT, fts5yymsp[-2].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18, 0); + fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); + fts5yymsp[-1].minor.fts5yy11 = sqlite3Fts5ParseColsetInvert(pParse, fts5yymsp[-1].minor.fts5yy11); } - fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; break; - case 4: /* expr ::= LP expr RP */ -{fts5yymsp[-2].minor.fts5yy18 = fts5yymsp[-1].minor.fts5yy18;} - break; - case 5: /* expr ::= exprlist */ - case 6: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==6); -{fts5yylhsminor.fts5yy18 = fts5yymsp[0].minor.fts5yy18;} - fts5yymsp[0].minor.fts5yy18 = fts5yylhsminor.fts5yy18; - break; - case 7: /* exprlist ::= exprlist cnearset */ + case 5: /* colsetlist ::= colsetlist STRING */ { - fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy18, fts5yymsp[0].minor.fts5yy18); -} - fts5yymsp[-1].minor.fts5yy18 = fts5yylhsminor.fts5yy18; - break; - case 8: /* cnearset ::= nearset */ -{ - fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy26); -} - fts5yymsp[0].minor.fts5yy18 = fts5yylhsminor.fts5yy18; - break; - case 9: /* cnearset ::= colset COLON nearset */ -{ - sqlite3Fts5ParseSetColset(pParse, fts5yymsp[0].minor.fts5yy26, fts5yymsp[-2].minor.fts5yy3); - fts5yylhsminor.fts5yy18 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy26); -} - fts5yymsp[-2].minor.fts5yy18 = fts5yylhsminor.fts5yy18; - break; - case 10: /* colset ::= LCP colsetlist RCP */ -{ fts5yymsp[-2].minor.fts5yy3 = fts5yymsp[-1].minor.fts5yy3; } - break; - case 11: /* colset ::= STRING */ -{ - fts5yylhsminor.fts5yy3 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); -} - fts5yymsp[0].minor.fts5yy3 = fts5yylhsminor.fts5yy3; - break; - case 12: /* colsetlist ::= colsetlist STRING */ -{ - fts5yylhsminor.fts5yy3 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy3, &fts5yymsp[0].minor.fts5yy0); } - fts5yymsp[-1].minor.fts5yy3 = fts5yylhsminor.fts5yy3; - break; - case 13: /* colsetlist ::= STRING */ -{ - fts5yylhsminor.fts5yy3 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); -} - fts5yymsp[0].minor.fts5yy3 = fts5yylhsminor.fts5yy3; - break; - case 14: /* nearset ::= phrase */ -{ fts5yylhsminor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy11); } - fts5yymsp[0].minor.fts5yy26 = fts5yylhsminor.fts5yy26; - break; - case 15: /* nearset ::= STRING LP nearphrases neardist_opt RP */ -{ - sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0); - sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy26, &fts5yymsp[-1].minor.fts5yy0); - fts5yylhsminor.fts5yy26 = fts5yymsp[-2].minor.fts5yy26; -} - fts5yymsp[-4].minor.fts5yy26 = fts5yylhsminor.fts5yy26; - break; - case 16: /* nearphrases ::= phrase */ -{ - fts5yylhsminor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy11); -} - fts5yymsp[0].minor.fts5yy26 = fts5yylhsminor.fts5yy26; - break; - case 17: /* nearphrases ::= nearphrases phrase */ -{ - fts5yylhsminor.fts5yy26 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy26, fts5yymsp[0].minor.fts5yy11); -} - fts5yymsp[-1].minor.fts5yy26 = fts5yylhsminor.fts5yy26; - break; - case 18: /* neardist_opt ::= */ -{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; } - break; - case 19: /* neardist_opt ::= COMMA STRING */ -{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; } - break; - case 20: /* phrase ::= phrase PLUS STRING star_opt */ -{ - fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy11, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy20); -} - fts5yymsp[-3].minor.fts5yy11 = fts5yylhsminor.fts5yy11; - break; - case 21: /* phrase ::= STRING star_opt */ -{ - fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy20); -} + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, fts5yymsp[-1].minor.fts5yy11, &fts5yymsp[0].minor.fts5yy0); } fts5yymsp[-1].minor.fts5yy11 = fts5yylhsminor.fts5yy11; break; - case 22: /* star_opt ::= STAR */ -{ fts5yymsp[0].minor.fts5yy20 = 1; } + case 6: /* colsetlist ::= STRING */ +{ + fts5yylhsminor.fts5yy11 = sqlite3Fts5ParseColset(pParse, 0, &fts5yymsp[0].minor.fts5yy0); +} + fts5yymsp[0].minor.fts5yy11 = fts5yylhsminor.fts5yy11; break; - case 23: /* star_opt ::= */ -{ fts5yymsp[1].minor.fts5yy20 = 0; } + case 7: /* expr ::= expr AND expr */ +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_AND, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); +} + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 8: /* expr ::= expr OR expr */ +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_OR, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); +} + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 9: /* expr ::= expr NOT expr */ +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_NOT, fts5yymsp[-2].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24, 0); +} + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 10: /* expr ::= colset COLON LP expr RP */ +{ + sqlite3Fts5ParseSetColset(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[-4].minor.fts5yy11); + fts5yylhsminor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24; +} + fts5yymsp[-4].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 11: /* expr ::= LP expr RP */ +{fts5yymsp[-2].minor.fts5yy24 = fts5yymsp[-1].minor.fts5yy24;} + break; + case 12: /* expr ::= exprlist */ + case 13: /* exprlist ::= cnearset */ fts5yytestcase(fts5yyruleno==13); +{fts5yylhsminor.fts5yy24 = fts5yymsp[0].minor.fts5yy24;} + fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 14: /* exprlist ::= exprlist cnearset */ +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseImplicitAnd(pParse, fts5yymsp[-1].minor.fts5yy24, fts5yymsp[0].minor.fts5yy24); +} + fts5yymsp[-1].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 15: /* cnearset ::= nearset */ +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy46); +} + fts5yymsp[0].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 16: /* cnearset ::= colset COLON nearset */ +{ + fts5yylhsminor.fts5yy24 = sqlite3Fts5ParseNode(pParse, FTS5_STRING, 0, 0, fts5yymsp[0].minor.fts5yy46); + sqlite3Fts5ParseSetColset(pParse, fts5yylhsminor.fts5yy24, fts5yymsp[-2].minor.fts5yy11); +} + fts5yymsp[-2].minor.fts5yy24 = fts5yylhsminor.fts5yy24; + break; + case 17: /* nearset ::= phrase */ +{ fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); } + fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 18: /* nearset ::= CARET phrase */ +{ + sqlite3Fts5ParseSetCaret(fts5yymsp[0].minor.fts5yy53); + fts5yymsp[-1].minor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); +} + break; + case 19: /* nearset ::= STRING LP nearphrases neardist_opt RP */ +{ + sqlite3Fts5ParseNear(pParse, &fts5yymsp[-4].minor.fts5yy0); + sqlite3Fts5ParseSetDistance(pParse, fts5yymsp[-2].minor.fts5yy46, &fts5yymsp[-1].minor.fts5yy0); + fts5yylhsminor.fts5yy46 = fts5yymsp[-2].minor.fts5yy46; +} + fts5yymsp[-4].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 20: /* nearphrases ::= phrase */ +{ + fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, 0, fts5yymsp[0].minor.fts5yy53); +} + fts5yymsp[0].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 21: /* nearphrases ::= nearphrases phrase */ +{ + fts5yylhsminor.fts5yy46 = sqlite3Fts5ParseNearset(pParse, fts5yymsp[-1].minor.fts5yy46, fts5yymsp[0].minor.fts5yy53); +} + fts5yymsp[-1].minor.fts5yy46 = fts5yylhsminor.fts5yy46; + break; + case 22: /* neardist_opt ::= */ +{ fts5yymsp[1].minor.fts5yy0.p = 0; fts5yymsp[1].minor.fts5yy0.n = 0; } + break; + case 23: /* neardist_opt ::= COMMA STRING */ +{ fts5yymsp[-1].minor.fts5yy0 = fts5yymsp[0].minor.fts5yy0; } + break; + case 24: /* phrase ::= phrase PLUS STRING star_opt */ +{ + fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, fts5yymsp[-3].minor.fts5yy53, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4); +} + fts5yymsp[-3].minor.fts5yy53 = fts5yylhsminor.fts5yy53; + break; + case 25: /* phrase ::= STRING star_opt */ +{ + fts5yylhsminor.fts5yy53 = sqlite3Fts5ParseTerm(pParse, 0, &fts5yymsp[-1].minor.fts5yy0, fts5yymsp[0].minor.fts5yy4); +} + fts5yymsp[-1].minor.fts5yy53 = fts5yylhsminor.fts5yy53; + break; + case 26: /* star_opt ::= STAR */ +{ fts5yymsp[0].minor.fts5yy4 = 1; } + break; + case 27: /* star_opt ::= */ +{ fts5yymsp[1].minor.fts5yy4 = 0; } break; default: break; /********** End reduce actions ************************************************/ }; - assert( fts5yyrulenofts5YY_MAX_SHIFT ) fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; - fts5yypParser->fts5yyidx -= fts5yysize - 1; - fts5yymsp -= fts5yysize-1; - fts5yymsp->stateno = (fts5YYACTIONTYPE)fts5yyact; - fts5yymsp->major = (fts5YYCODETYPE)fts5yygoto; - fts5yyTraceShift(fts5yypParser, fts5yyact); - }else{ - assert( fts5yyact == fts5YY_ACCEPT_ACTION ); - fts5yypParser->fts5yyidx -= fts5yysize; - fts5yy_accept(fts5yypParser); - } + assert( fts5yyrulenofts5YY_MAX_SHIFT && fts5yyact<=fts5YY_MAX_SHIFTREDUCE) ); + + /* It is not possible for a REDUCE to be followed by an error */ + assert( fts5yyact!=fts5YY_ERROR_ACTION ); + + fts5yymsp += fts5yysize+1; + fts5yypParser->fts5yytos = fts5yymsp; + fts5yymsp->stateno = (fts5YYACTIONTYPE)fts5yyact; + fts5yymsp->major = (fts5YYCODETYPE)fts5yygoto; + fts5yyTraceShift(fts5yypParser, fts5yyact, "... then shift"); + return fts5yyact; } /* @@ -178558,18 +235761,20 @@ static void fts5yy_reduce( static void fts5yy_parse_failed( fts5yyParser *fts5yypParser /* The parser */ ){ - sqlite3Fts5ParserARG_FETCH; + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH #ifndef NDEBUG if( fts5yyTraceFILE ){ fprintf(fts5yyTraceFILE,"%sFail!\n",fts5yyTracePrompt); } #endif - while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser); + while( fts5yypParser->fts5yytos>fts5yypParser->fts5yystack ) fts5yy_pop_parser_stack(fts5yypParser); /* Here code is inserted which will be executed whenever the ** parser fails */ /************ Begin %parse_failure code ***************************************/ /************ End %parse_failure code *****************************************/ - sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserCTX_STORE } #endif /* fts5YYNOERRORRECOVERY */ @@ -178581,7 +235786,8 @@ static void fts5yy_syntax_error( int fts5yymajor, /* The major type of the error token */ sqlite3Fts5ParserFTS5TOKENTYPE fts5yyminor /* The minor type of the error token */ ){ - sqlite3Fts5ParserARG_FETCH; + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH #define FTS5TOKEN fts5yyminor /************ Begin %syntax_error code ****************************************/ @@ -178590,7 +235796,8 @@ static void fts5yy_syntax_error( pParse, "fts5: syntax error near \"%.*s\"",FTS5TOKEN.n,FTS5TOKEN.p ); /************ End %syntax_error code ******************************************/ - sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserCTX_STORE } /* @@ -178599,18 +235806,23 @@ static void fts5yy_syntax_error( static void fts5yy_accept( fts5yyParser *fts5yypParser /* The parser */ ){ - sqlite3Fts5ParserARG_FETCH; + sqlite3Fts5ParserARG_FETCH + sqlite3Fts5ParserCTX_FETCH #ifndef NDEBUG if( fts5yyTraceFILE ){ fprintf(fts5yyTraceFILE,"%sAccept!\n",fts5yyTracePrompt); } #endif - while( fts5yypParser->fts5yyidx>=0 ) fts5yy_pop_parser_stack(fts5yypParser); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif + assert( fts5yypParser->fts5yytos==fts5yypParser->fts5yystack ); /* Here code is inserted which will be executed whenever the ** parser accepts */ /*********** Begin %parse_accept code *****************************************/ /*********** End %parse_accept code *******************************************/ - sqlite3Fts5ParserARG_STORE; /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserARG_STORE /* Suppress warning about unused %extra_argument variable */ + sqlite3Fts5ParserCTX_STORE } /* The main parser program. @@ -178639,59 +235851,88 @@ static void sqlite3Fts5Parser( sqlite3Fts5ParserARG_PDECL /* Optional %extra_argument parameter */ ){ fts5YYMINORTYPE fts5yyminorunion; - unsigned int fts5yyact; /* The parser action. */ + fts5YYACTIONTYPE fts5yyact; /* The parser action. */ #if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) int fts5yyendofinput; /* True if we are at the end of input */ #endif #ifdef fts5YYERRORSYMBOL int fts5yyerrorhit = 0; /* True if fts5yymajor has invoked an error */ #endif - fts5yyParser *fts5yypParser; /* The parser */ + fts5yyParser *fts5yypParser = (fts5yyParser*)fts5yyp; /* The parser */ + sqlite3Fts5ParserCTX_FETCH + sqlite3Fts5ParserARG_STORE - /* (re)initialize the parser, if necessary */ - fts5yypParser = (fts5yyParser*)fts5yyp; - if( fts5yypParser->fts5yyidx<0 ){ -#if fts5YYSTACKDEPTH<=0 - if( fts5yypParser->fts5yystksz <=0 ){ - fts5yyStackOverflow(fts5yypParser); - return; - } -#endif - fts5yypParser->fts5yyidx = 0; -#ifndef fts5YYNOERRORRECOVERY - fts5yypParser->fts5yyerrcnt = -1; -#endif - fts5yypParser->fts5yystack[0].stateno = 0; - fts5yypParser->fts5yystack[0].major = 0; -#ifndef NDEBUG - if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sInitialize. Empty stack. State 0\n", - fts5yyTracePrompt); - } -#endif - } + assert( fts5yypParser->fts5yytos!=0 ); #if !defined(fts5YYERRORSYMBOL) && !defined(fts5YYNOERRORRECOVERY) fts5yyendofinput = (fts5yymajor==0); #endif - sqlite3Fts5ParserARG_STORE; + fts5yyact = fts5yypParser->fts5yytos->stateno; #ifndef NDEBUG if( fts5yyTraceFILE ){ - fprintf(fts5yyTraceFILE,"%sInput '%s'\n",fts5yyTracePrompt,fts5yyTokenName[fts5yymajor]); + if( fts5yyact < fts5YY_MIN_REDUCE ){ + fprintf(fts5yyTraceFILE,"%sInput '%s' in state %d\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact); + }else{ + fprintf(fts5yyTraceFILE,"%sInput '%s' with pending reduce %d\n", + fts5yyTracePrompt,fts5yyTokenName[fts5yymajor],fts5yyact-fts5YY_MIN_REDUCE); + } } #endif - do{ - fts5yyact = fts5yy_find_shift_action(fts5yypParser,(fts5YYCODETYPE)fts5yymajor); - if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ - if( fts5yyact > fts5YY_MAX_SHIFT ) fts5yyact += fts5YY_MIN_REDUCE - fts5YY_MIN_SHIFTREDUCE; - fts5yy_shift(fts5yypParser,fts5yyact,fts5yymajor,fts5yyminor); + while(1){ /* Exit by "break" */ + assert( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystack ); + assert( fts5yyact==fts5yypParser->fts5yytos->stateno ); + fts5yyact = fts5yy_find_shift_action((fts5YYCODETYPE)fts5yymajor,fts5yyact); + if( fts5yyact >= fts5YY_MIN_REDUCE ){ + unsigned int fts5yyruleno = fts5yyact - fts5YY_MIN_REDUCE; /* Reduce by this rule */ +#ifndef NDEBUG + assert( fts5yyruleno<(int)(sizeof(fts5yyRuleName)/sizeof(fts5yyRuleName[0])) ); + if( fts5yyTraceFILE ){ + int fts5yysize = fts5yyRuleInfoNRhs[fts5yyruleno]; + if( fts5yysize ){ + fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s, pop back to state %d.\n", + fts5yyTracePrompt, + fts5yyruleno, fts5yyRuleName[fts5yyruleno], + fts5yyrulenofts5yytos[fts5yysize].stateno); + }else{ + fprintf(fts5yyTraceFILE, "%sReduce %d [%s]%s.\n", + fts5yyTracePrompt, fts5yyruleno, fts5yyRuleName[fts5yyruleno], + fts5yyrulenofts5yytos - fts5yypParser->fts5yystack)>fts5yypParser->fts5yyhwm ){ + fts5yypParser->fts5yyhwm++; + assert( fts5yypParser->fts5yyhwm == + (int)(fts5yypParser->fts5yytos - fts5yypParser->fts5yystack)); + } +#endif + if( fts5yypParser->fts5yytos>=fts5yypParser->fts5yystackEnd ){ + if( fts5yyGrowStack(fts5yypParser) ){ + fts5yyStackOverflow(fts5yypParser); + break; + } + } + } + fts5yyact = fts5yy_reduce(fts5yypParser,fts5yyruleno,fts5yymajor,fts5yyminor sqlite3Fts5ParserCTX_PARAM); + }else if( fts5yyact <= fts5YY_MAX_SHIFTREDUCE ){ + fts5yy_shift(fts5yypParser,fts5yyact,(fts5YYCODETYPE)fts5yymajor,fts5yyminor); #ifndef fts5YYNOERRORRECOVERY fts5yypParser->fts5yyerrcnt--; #endif - fts5yymajor = fts5YYNOCODE; - }else if( fts5yyact <= fts5YY_MAX_REDUCE ){ - fts5yy_reduce(fts5yypParser,fts5yyact-fts5YY_MIN_REDUCE); + break; + }else if( fts5yyact==fts5YY_ACCEPT_ACTION ){ + fts5yypParser->fts5yytos--; + fts5yy_accept(fts5yypParser); + return; }else{ assert( fts5yyact == fts5YY_ERROR_ACTION ); fts5yyminorunion.fts5yy0 = fts5yyminor; @@ -178706,7 +235947,7 @@ static void sqlite3Fts5Parser( #ifdef fts5YYERRORSYMBOL /* A syntax error has occurred. ** The response to an error depends upon whether or not the - ** grammar defines an error token "ERROR". + ** grammar defines an error token "ERROR". ** ** This is what we do if the grammar does define ERROR: ** @@ -178726,7 +235967,7 @@ static void sqlite3Fts5Parser( if( fts5yypParser->fts5yyerrcnt<0 ){ fts5yy_syntax_error(fts5yypParser,fts5yymajor,fts5yyminor); } - fts5yymx = fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].major; + fts5yymx = fts5yypParser->fts5yytos->major; if( fts5yymx==fts5YYERRORSYMBOL || fts5yyerrorhit ){ #ifndef NDEBUG if( fts5yyTraceFILE ){ @@ -178737,18 +235978,18 @@ static void sqlite3Fts5Parser( fts5yy_destructor(fts5yypParser, (fts5YYCODETYPE)fts5yymajor, &fts5yyminorunion); fts5yymajor = fts5YYNOCODE; }else{ - while( - fts5yypParser->fts5yyidx >= 0 && - fts5yymx != fts5YYERRORSYMBOL && - (fts5yyact = fts5yy_find_reduce_action( - fts5yypParser->fts5yystack[fts5yypParser->fts5yyidx].stateno, - fts5YYERRORSYMBOL)) >= fts5YY_MIN_REDUCE - ){ + while( fts5yypParser->fts5yytos > fts5yypParser->fts5yystack ){ + fts5yyact = fts5yy_find_reduce_action(fts5yypParser->fts5yytos->stateno, + fts5YYERRORSYMBOL); + if( fts5yyact<=fts5YY_MAX_SHIFTREDUCE ) break; fts5yy_pop_parser_stack(fts5yypParser); } - if( fts5yypParser->fts5yyidx < 0 || fts5yymajor==0 ){ + if( fts5yypParser->fts5yytos <= fts5yypParser->fts5yystack || fts5yymajor==0 ){ fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); fts5yy_parse_failed(fts5yypParser); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif fts5yymajor = fts5YYNOCODE; }else if( fts5yymx!=fts5YYERRORSYMBOL ){ fts5yy_shift(fts5yypParser,fts5yyact,fts5YYERRORSYMBOL,fts5yyminor); @@ -178756,6 +235997,8 @@ static void sqlite3Fts5Parser( } fts5yypParser->fts5yyerrcnt = 3; fts5yyerrorhit = 1; + if( fts5yymajor==fts5YYNOCODE ) break; + fts5yyact = fts5yypParser->fts5yytos->stateno; #elif defined(fts5YYNOERRORRECOVERY) /* If the fts5YYNOERRORRECOVERY macro is defined, then do not attempt to ** do any kind of error recovery. Instead, simply invoke the syntax @@ -178766,8 +236009,7 @@ static void sqlite3Fts5Parser( */ fts5yy_syntax_error(fts5yypParser,fts5yymajor, fts5yyminor); fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); - fts5yymajor = fts5YYNOCODE; - + break; #else /* fts5YYERRORSYMBOL is not defined */ /* This is what we do if the grammar does not define ERROR: ** @@ -178785,24 +236027,43 @@ static void sqlite3Fts5Parser( fts5yy_destructor(fts5yypParser,(fts5YYCODETYPE)fts5yymajor,&fts5yyminorunion); if( fts5yyendofinput ){ fts5yy_parse_failed(fts5yypParser); +#ifndef fts5YYNOERRORRECOVERY + fts5yypParser->fts5yyerrcnt = -1; +#endif } - fts5yymajor = fts5YYNOCODE; + break; #endif } - }while( fts5yymajor!=fts5YYNOCODE && fts5yypParser->fts5yyidx>=0 ); + } #ifndef NDEBUG if( fts5yyTraceFILE ){ - int i; + fts5yyStackEntry *i; + char cDiv = '['; fprintf(fts5yyTraceFILE,"%sReturn. Stack=",fts5yyTracePrompt); - for(i=1; i<=fts5yypParser->fts5yyidx; i++) - fprintf(fts5yyTraceFILE,"%c%s", i==1 ? '[' : ' ', - fts5yyTokenName[fts5yypParser->fts5yystack[i].major]); + for(i=&fts5yypParser->fts5yystack[1]; i<=fts5yypParser->fts5yytos; i++){ + fprintf(fts5yyTraceFILE,"%c%s", cDiv, fts5yyTokenName[i->major]); + cDiv = ' '; + } fprintf(fts5yyTraceFILE,"]\n"); } #endif return; } +/* +** Return the fallback token corresponding to canonical token iToken, or +** 0 if iToken has no fallback. +*/ +static int sqlite3Fts5ParserFallback(int iToken){ +#ifdef fts5YYFALLBACK + assert( iToken<(int)(sizeof(fts5yyFallback)/sizeof(fts5yyFallback[0])) ); + return fts5yyFallback[iToken]; +#else + (void)iToken; + return 0; +#endif +} + /* ** 2014 May 31 ** @@ -178821,7 +236082,7 @@ static void sqlite3Fts5Parser( #include /* amalgamator: keep */ /* -** Object used to iterate through all "coalesced phrase instances" in +** Object used to iterate through all "coalesced phrase instances" in ** a single column of the current row. If the phrase instances in the ** column being considered do not overlap, this object simply iterates ** through them. Or, if they do overlap (share one or more tokens in @@ -178884,7 +236145,7 @@ static int fts5CInstIterNext(CInstIter *pIter){ } /* -** Initialize the iterator object indicated by the final parameter to +** Initialize the iterator object indicated by the final parameter to ** iterate through coalesced phrase instances in column iCol. */ static int fts5CInstIterInit( @@ -178915,33 +236176,37 @@ static int fts5CInstIterInit( */ typedef struct HighlightContext HighlightContext; struct HighlightContext { - CInstIter iter; /* Coalesced Instance Iterator */ - int iPos; /* Current token offset in zIn[] */ + /* Constant parameters to fts5HighlightCb() */ int iRangeStart; /* First token to include */ int iRangeEnd; /* If non-zero, last token to include */ const char *zOpen; /* Opening highlight */ const char *zClose; /* Closing highlight */ const char *zIn; /* Input text */ int nIn; /* Size of input text in bytes */ - int iOff; /* Current offset within zIn[] */ + + /* Variables modified by fts5HighlightCb() */ + CInstIter iter; /* Coalesced Instance Iterator */ + int iPos; /* Current token offset in zIn[] */ + int iOff; /* Have copied up to this offset in zIn[] */ + int bOpen; /* True if highlight is open */ char *zOut; /* Output value */ }; /* ** Append text to the HighlightContext output string - p->zOut. Argument -** z points to a buffer containing n bytes of text to append. If n is +** z points to a buffer containing n bytes of text to append. If n is ** negative, everything up until the first '\0' is appended to the output. ** -** If *pRc is set to any value other than SQLITE_OK when this function is -** called, it is a no-op. If an error (i.e. an OOM condition) is encountered, -** *pRc is set to an error code before returning. +** If *pRc is set to any value other than SQLITE_OK when this function is +** called, it is a no-op. If an error (i.e. an OOM condition) is encountered, +** *pRc is set to an error code before returning. */ static void fts5HighlightAppend( - int *pRc, - HighlightContext *p, + int *pRc, + HighlightContext *p, const char *z, int n ){ - if( *pRc==SQLITE_OK ){ + if( *pRc==SQLITE_OK && z ){ if( n<0 ) n = (int)strlen(z); p->zOut = sqlite3_mprintf("%z%.*s", p->zOut, n, z); if( p->zOut==0 ) *pRc = SQLITE_NOMEM; @@ -178956,8 +236221,8 @@ static int fts5HighlightCb( int tflags, /* Mask of FTS5_TOKEN_* flags */ const char *pToken, /* Buffer containing token */ int nToken, /* Size of token in bytes */ - int iStartOff, /* Start offset of token */ - int iEndOff /* End offset of token */ + int iStartOff, /* Start byte offset of token */ + int iEndOff /* End byte offset of token */ ){ HighlightContext *p = (HighlightContext*)pContext; int rc = SQLITE_OK; @@ -178968,40 +236233,66 @@ static int fts5HighlightCb( if( tflags & FTS5_TOKEN_COLOCATED ) return SQLITE_OK; iPos = p->iPos++; - if( p->iRangeEnd>0 ){ + if( p->iRangeEnd>=0 ){ if( iPosiRangeStart || iPos>p->iRangeEnd ) return SQLITE_OK; if( p->iRangeStart && iPos==p->iRangeStart ) p->iOff = iStartOff; } - if( iPos==p->iter.iStart ){ + /* If the parenthesis is open, and this token is not part of the current + ** phrase, and the starting byte offset of this token is past the point + ** that has currently been copied into the output buffer, close the + ** parenthesis. */ + if( p->bOpen + && (iPos<=p->iter.iStart || p->iter.iStart<0) + && iStartOff>p->iOff + ){ + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } + + /* If this is the start of a new phrase, and the highlight is not open: + ** + ** * copy text from the input up to the start of the phrase, and + ** * open the highlight. + */ + if( iPos==p->iter.iStart && p->bOpen==0 ){ fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iStartOff - p->iOff); fts5HighlightAppend(&rc, p, p->zOpen, -1); p->iOff = iStartOff; + p->bOpen = 1; } if( iPos==p->iter.iEnd ){ - if( p->iRangeEnd && p->iter.iStartiRangeStart ){ + if( p->bOpen==0 ){ + assert( p->iRangeEnd>=0 ); fts5HighlightAppend(&rc, p, p->zOpen, -1); + p->bOpen = 1; } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); - fts5HighlightAppend(&rc, p, p->zClose, -1); p->iOff = iEndOff; + if( rc==SQLITE_OK ){ rc = fts5CInstIterNext(&p->iter); } } - if( p->iRangeEnd>0 && iPos==p->iRangeEnd ){ + if( iPos==p->iRangeEnd ){ + if( p->bOpen ){ + if( p->iter.iStart>=0 && iPos>=p->iter.iStart ){ + fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); + p->iOff = iEndOff; + } + fts5HighlightAppend(&rc, p, p->zClose, -1); + p->bOpen = 0; + } fts5HighlightAppend(&rc, p, &p->zIn[p->iOff], iEndOff - p->iOff); p->iOff = iEndOff; - if( iPositer.iEnd ){ - fts5HighlightAppend(&rc, p, p->zClose, -1); - } } return rc; } + /* ** Implementation of highlight() function. */ @@ -179026,15 +236317,28 @@ static void fts5HighlightFunction( memset(&ctx, 0, sizeof(HighlightContext)); ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); + ctx.iRangeEnd = -1; rc = pApi->xColumnText(pFts, iCol, &ctx.zIn, &ctx.nIn); - - if( ctx.zIn ){ + if( rc==SQLITE_RANGE ){ + sqlite3_result_text(pCtx, "", -1, SQLITE_STATIC); + rc = SQLITE_OK; + }else if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iCol */ + int nLoc = 0; /* Size of pLoc in bytes */ if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iCol, &ctx.iter); } if( rc==SQLITE_OK ){ - rc = pApi->xTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx, fts5HighlightCb + ); + } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); } fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); @@ -179051,6 +236355,129 @@ static void fts5HighlightFunction( ** End of highlight() implementation. **************************************************************************/ +/* +** Context object passed to the fts5SentenceFinderCb() function. +*/ +typedef struct Fts5SFinder Fts5SFinder; +struct Fts5SFinder { + int iPos; /* Current token position */ + int nFirstAlloc; /* Allocated size of aFirst[] */ + int nFirst; /* Number of entries in aFirst[] */ + int *aFirst; /* Array of first token in each sentence */ + const char *zDoc; /* Document being tokenized */ +}; + +/* +** Add an entry to the Fts5SFinder.aFirst[] array. Grow the array if +** necessary. Return SQLITE_OK if successful, or SQLITE_NOMEM if an +** error occurs. +*/ +static int fts5SentenceFinderAdd(Fts5SFinder *p, int iAdd){ + if( p->nFirstAlloc==p->nFirst ){ + int nNew = p->nFirstAlloc ? p->nFirstAlloc*2 : 64; + int *aNew; + + aNew = (int*)sqlite3_realloc64(p->aFirst, nNew*sizeof(int)); + if( aNew==0 ) return SQLITE_NOMEM; + p->aFirst = aNew; + p->nFirstAlloc = nNew; + } + p->aFirst[p->nFirst++] = iAdd; + return SQLITE_OK; +} + +/* +** This function is an xTokenize() callback used by the auxiliary snippet() +** function. Its job is to identify tokens that are the first in a sentence. +** For each such token, an entry is added to the SFinder.aFirst[] array. +*/ +static int fts5SentenceFinderCb( + void *pContext, /* Pointer to HighlightContext object */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStartOff, /* Start offset of token */ + int iEndOff /* End offset of token */ +){ + int rc = SQLITE_OK; + + UNUSED_PARAM2(pToken, nToken); + UNUSED_PARAM(iEndOff); + + if( (tflags & FTS5_TOKEN_COLOCATED)==0 ){ + Fts5SFinder *p = (Fts5SFinder*)pContext; + if( p->iPos>0 ){ + int i; + char c = 0; + for(i=iStartOff-1; i>=0; i--){ + c = p->zDoc[i]; + if( c!=' ' && c!='\t' && c!='\n' && c!='\r' ) break; + } + if( i!=iStartOff-1 && (c=='.' || c==':') ){ + rc = fts5SentenceFinderAdd(p, p->iPos); + } + }else{ + rc = fts5SentenceFinderAdd(p, 0); + } + p->iPos++; + } + return rc; +} + +static int fts5SnippetScore( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + int nDocsize, /* Size of column in tokens */ + unsigned char *aSeen, /* Array with one element per query phrase */ + int iCol, /* Column to score */ + int iPos, /* Starting offset to score */ + int nToken, /* Max tokens per snippet */ + int *pnScore, /* OUT: Score */ + int *piPos /* OUT: Adjusted offset */ +){ + int rc; + int i; + int ip = 0; + int ic = 0; + int iOff = 0; + int iFirst = -1; + int nInst; + int nScore = 0; + int iLast = 0; + sqlite3_int64 iEnd = (sqlite3_int64)iPos + nToken; + + rc = pApi->xInstCount(pFts, &nInst); + for(i=0; ixInst(pFts, i, &ip, &ic, &iOff); + if( rc==SQLITE_OK && ic==iCol && iOff>=iPos && iOffxPhraseSize(pFts, ip); + } + } + + *pnScore = nScore; + if( piPos ){ + sqlite3_int64 iAdj = iFirst - (nToken - (iLast-iFirst)) / 2; + if( (iAdj+nToken)>nDocsize ) iAdj = nDocsize - nToken; + if( iAdj<0 ) iAdj = 0; + *piPos = (int)iAdj; + } + + return rc; +} + +/* +** Return the value in pVal interpreted as utf-8 text. Except, if pVal +** contains a NULL value, return a pointer to a static string zero +** bytes in length instead of a NULL pointer. +*/ +static const char *fts5ValueToText(sqlite3_value *pVal){ + const char *zRet = (const char*)sqlite3_value_text(pVal); + return zRet ? zRet : ""; +} + /* ** Implementation of snippet() function. */ @@ -179072,9 +236499,10 @@ static void fts5SnippetFunction( unsigned char *aSeen; /* Array of "seen instance" flags */ int iBestCol; /* Column containing best snippet */ int iBestStart = 0; /* First token of best snippet */ - int iBestLast; /* Last token of best snippet */ int nBestScore = 0; /* Score of best snippet */ int nColSize = 0; /* Total size of iBestCol in tokens */ + Fts5SFinder sFinder; /* Used to find the beginnings of sentences */ + int nCol; if( nVal!=5 ){ const char *zErr = "wrong number of arguments to function snippet()"; @@ -179082,13 +236510,14 @@ static void fts5SnippetFunction( return; } + nCol = pApi->xColumnCount(pFts); memset(&ctx, 0, sizeof(HighlightContext)); iCol = sqlite3_value_int(apVal[0]); - ctx.zOpen = (const char*)sqlite3_value_text(apVal[1]); - ctx.zClose = (const char*)sqlite3_value_text(apVal[2]); - zEllips = (const char*)sqlite3_value_text(apVal[3]); + ctx.zOpen = fts5ValueToText(apVal[1]); + ctx.zClose = fts5ValueToText(apVal[2]); + ctx.iRangeEnd = -1; + zEllips = fts5ValueToText(apVal[3]); nToken = sqlite3_value_int(apVal[4]); - iBestLast = nToken-1; iBestCol = (iCol>=0 ? iCol : 0); nPhrase = pApi->xPhraseCount(pFts); @@ -179096,82 +236525,128 @@ static void fts5SnippetFunction( if( aSeen==0 ){ rc = SQLITE_NOMEM; } - if( rc==SQLITE_OK ){ rc = pApi->xInstCount(pFts, &nInst); } - for(i=0; rc==SQLITE_OK && ixInst(pFts, i, &ip, &iSnippetCol, &iStart); - if( rc==SQLITE_OK && (iCol<0 || iSnippetCol==iCol) ){ - int nScore = 1000; - int iLast = iStart - 1 + pApi->xPhraseSize(pFts, ip); - int j; - aSeen[ip] = 1; - for(j=i+1; rc==SQLITE_OK && jxInst(pFts, j, &ip, &ic, &io); - iFinal = io + pApi->xPhraseSize(pFts, ip) - 1; - if( rc==SQLITE_OK && ic==iSnippetCol && iLastiLast ) iLast = iFinal; + memset(&sFinder, 0, sizeof(Fts5SFinder)); + for(i=0; ixColumnText(pFts, i, &sFinder.zDoc, &nDoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnLocale(pFts, i, &pLoc, &nLoc); + if( rc!=SQLITE_OK ) break; + rc = pApi->xTokenize_v2(pFts, + sFinder.zDoc, nDoc, pLoc, nLoc, (void*)&sFinder, fts5SentenceFinderCb + ); + if( rc!=SQLITE_OK ) break; + rc = pApi->xColumnSize(pFts, i, &nDocsize); + if( rc!=SQLITE_OK ) break; + + for(ii=0; rc==SQLITE_OK && iixInst(pFts, ii, &ip, &ic, &io); + if( ic!=i ) continue; + if( io>nDocsize ) rc = FTS5_CORRUPT; + if( rc!=SQLITE_OK ) continue; + memset(aSeen, 0, nPhrase); + rc = fts5SnippetScore(pApi, pFts, nDocsize, aSeen, i, + io, nToken, &nScore, &iAdj + ); + if( rc==SQLITE_OK && nScore>nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = iAdj; + nColSize = nDocsize; } - } - if( rc==SQLITE_OK && nScore>nBestScore ){ - iBestCol = iSnippetCol; - iBestStart = iStart; - iBestLast = iLast; - nBestScore = nScore; + if( rc==SQLITE_OK && sFinder.nFirst && nDocsize>nToken ){ + for(jj=0; jj<(sFinder.nFirst-1); jj++){ + if( sFinder.aFirst[jj+1]>io ) break; + } + + if( sFinder.aFirst[jj]nBestScore ){ + nBestScore = nScore; + iBestCol = i; + iBestStart = sFinder.aFirst[jj]; + nColSize = nDocsize; + } + } + } } } } - if( rc==SQLITE_OK ){ - rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); - } if( rc==SQLITE_OK ){ rc = pApi->xColumnText(pFts, iBestCol, &ctx.zIn, &ctx.nIn); } + if( rc==SQLITE_OK && nColSize==0 ){ + rc = pApi->xColumnSize(pFts, iBestCol, &nColSize); + } if( ctx.zIn ){ + const char *pLoc = 0; /* Locale of column iBestCol */ + int nLoc = 0; /* Bytes in pLoc */ + if( rc==SQLITE_OK ){ rc = fts5CInstIterInit(pApi, pFts, iBestCol, &ctx.iter); } - if( (iBestStart+nToken-1)>iBestLast ){ - iBestStart -= (iBestStart+nToken-1-iBestLast) / 2; - } - if( iBestStart+nToken>nColSize ){ - iBestStart = nColSize - nToken; - } - if( iBestStart<0 ) iBestStart = 0; - ctx.iRangeStart = iBestStart; ctx.iRangeEnd = iBestStart + nToken - 1; if( iBestStart>0 ){ fts5HighlightAppend(&rc, &ctx, zEllips, -1); } + + /* Advance iterator ctx.iter so that it points to the first coalesced + ** phrase instance at or following position iBestStart. */ + while( ctx.iter.iStart>=0 && ctx.iter.iStartxTokenize(pFts, ctx.zIn, ctx.nIn, (void*)&ctx,fts5HighlightCb); + rc = pApi->xColumnLocale(pFts, iBestCol, &pLoc, &nLoc); + } + if( rc==SQLITE_OK ){ + rc = pApi->xTokenize_v2( + pFts, ctx.zIn, ctx.nIn, pLoc, nLoc, (void*)&ctx,fts5HighlightCb + ); + } + if( ctx.bOpen ){ + fts5HighlightAppend(&rc, &ctx, ctx.zClose, -1); } if( ctx.iRangeEnd>=(nColSize-1) ){ fts5HighlightAppend(&rc, &ctx, &ctx.zIn[ctx.iOff], ctx.nIn - ctx.iOff); }else{ fts5HighlightAppend(&rc, &ctx, zEllips, -1); } - - if( rc==SQLITE_OK ){ - sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); - }else{ - sqlite3_result_error_code(pCtx, rc); - } - sqlite3_free(ctx.zOut); } + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, (const char*)ctx.zOut, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_error_code(pCtx, rc); + } + sqlite3_free(ctx.zOut); sqlite3_free(aSeen); + sqlite3_free(sFinder.aFirst); } /************************************************************************/ @@ -179193,7 +236668,7 @@ struct Fts5Bm25Data { ** table matched by each individual phrase within the query. */ static int fts5CountCb( - const Fts5ExtensionApi *pApi, + const Fts5ExtensionApi *pApi, Fts5Context *pFts, void *pUserData /* Pointer to sqlite3_int64 variable */ ){ @@ -179204,34 +236679,34 @@ static int fts5CountCb( } /* -** Set *ppData to point to the Fts5Bm25Data object for the current query. +** Set *ppData to point to the Fts5Bm25Data object for the current query. ** If the object has not already been allocated, allocate and populate it ** now. */ static int fts5Bm25GetData( - const Fts5ExtensionApi *pApi, + const Fts5ExtensionApi *pApi, Fts5Context *pFts, Fts5Bm25Data **ppData /* OUT: bm25-data object for this query */ ){ int rc = SQLITE_OK; /* Return code */ Fts5Bm25Data *p; /* Object to return */ - p = pApi->xGetAuxdata(pFts, 0); + p = (Fts5Bm25Data*)pApi->xGetAuxdata(pFts, 0); if( p==0 ){ int nPhrase; /* Number of phrases in query */ sqlite3_int64 nRow = 0; /* Number of rows in table */ sqlite3_int64 nToken = 0; /* Number of tokens in table */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ int i; /* Allocate the Fts5Bm25Data object */ nPhrase = pApi->xPhraseCount(pFts); nByte = sizeof(Fts5Bm25Data) + nPhrase*2*sizeof(double); - p = (Fts5Bm25Data*)sqlite3_malloc(nByte); + p = (Fts5Bm25Data*)sqlite3_malloc64(nByte); if( p==0 ){ rc = SQLITE_NOMEM; }else{ - memset(p, 0, nByte); + memset(p, 0, (size_t)nByte); p->nPhrase = nPhrase; p->aIDF = (double*)&p[1]; p->aFreq = &p->aIDF[nPhrase]; @@ -179239,6 +236714,7 @@ static int fts5Bm25GetData( /* Calculate the average document length for this FTS5 table */ if( rc==SQLITE_OK ) rc = pApi->xRowCount(pFts, &nRow); + assert( rc!=SQLITE_OK || nRow>0 ); if( rc==SQLITE_OK ) rc = pApi->xColumnTotalSize(pFts, -1, &nToken); if( rc==SQLITE_OK ) p->avgdl = (double)nToken / (double)nRow; @@ -179256,7 +236732,7 @@ static int fts5Bm25GetData( ** is the number that contain at least one instance of the phrase ** under consideration. ** - ** The problem with this is that if (N < 2*nHit), the IDF is + ** The problem with this is that if (N < 2*nHit), the IDF is ** negative. Which is undesirable. So the mimimum allowable IDF is ** (1e-6) - roughly the same as a term that appears in just over ** half of set of 5,000,000 documents. */ @@ -179289,7 +236765,7 @@ static void fts5Bm25Function( ){ const double k1 = 1.2; /* Constant "k1" from BM25 formula */ const double b = 0.75; /* Constant "b" from BM25 formula */ - int rc = SQLITE_OK; /* Error code */ + int rc; /* Error code */ double score = 0.0; /* SQL function return value */ Fts5Bm25Data *pData; /* Values allocated/calculated once only */ int i; /* Iterator variable */ @@ -179321,23 +236797,68 @@ static void fts5Bm25Function( D = (double)nTok; } - /* Determine the BM25 score for the current row. */ - for(i=0; rc==SQLITE_OK && inPhrase; i++){ - score += pData->aIDF[i] * ( - ( aFreq[i] * (k1 + 1.0) ) / - ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) ) - ); - } - - /* If no error has occurred, return the calculated score. Otherwise, - ** throw an SQL exception. */ + /* Determine and return the BM25 score for the current row. Or, if an + ** error has occurred, throw an exception. */ if( rc==SQLITE_OK ){ + for(i=0; inPhrase; i++){ + score += pData->aIDF[i] * ( + ( aFreq[i] * (k1 + 1.0) ) / + ( aFreq[i] + k1 * (1 - b + b * D / pData->avgdl) ) + ); + } sqlite3_result_double(pCtx, -1.0 * score); }else{ sqlite3_result_error_code(pCtx, rc); } } +/* +** Implementation of fts5_get_locale() function. +*/ +static void fts5GetLocaleFunction( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +){ + int iCol = 0; + int eType = 0; + int rc = SQLITE_OK; + const char *zLocale = 0; + int nLocale = 0; + + /* xColumnLocale() must be available */ + assert( pApi->iVersion>=4 ); + + if( nVal!=1 ){ + const char *z = "wrong number of arguments to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + eType = sqlite3_value_numeric_type(apVal[0]); + if( eType!=SQLITE_INTEGER ){ + const char *z = "non-integer argument passed to function fts5_get_locale()"; + sqlite3_result_error(pCtx, z, -1); + return; + } + + iCol = sqlite3_value_int(apVal[0]); + if( iCol<0 || iCol>=pApi->xColumnCount(pFts) ){ + sqlite3_result_error_code(pCtx, SQLITE_RANGE); + return; + } + + rc = pApi->xColumnLocale(pFts, iCol, &zLocale, &nLocale); + if( rc!=SQLITE_OK ){ + sqlite3_result_error_code(pCtx, rc); + return; + } + + sqlite3_result_text(pCtx, zLocale, nLocale, SQLITE_TRANSIENT); +} + static int sqlite3Fts5AuxInit(fts5_api *pApi){ struct Builtin { const char *zFunc; /* Function name (nul-terminated) */ @@ -179345,9 +236866,10 @@ static int sqlite3Fts5AuxInit(fts5_api *pApi){ fts5_extension_function xFunc;/* Callback function */ void (*xDestroy)(void*); /* Destructor function */ } aBuiltin [] = { - { "snippet", 0, fts5SnippetFunction, 0 }, - { "highlight", 0, fts5HighlightFunction, 0 }, - { "bm25", 0, fts5Bm25Function, 0 }, + { "snippet", 0, fts5SnippetFunction, 0 }, + { "highlight", 0, fts5HighlightFunction, 0 }, + { "bm25", 0, fts5Bm25Function, 0 }, + { "fts5_get_locale", 0, fts5GetLocaleFunction, 0 }, }; int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ @@ -179364,8 +236886,6 @@ static int sqlite3Fts5AuxInit(fts5_api *pApi){ return rc; } - - /* ** 2014 May 31 ** @@ -179385,17 +236905,17 @@ static int sqlite3Fts5AuxInit(fts5_api *pApi){ static int sqlite3Fts5BufferSize(int *pRc, Fts5Buffer *pBuf, u32 nByte){ if( (u32)pBuf->nSpacenSpace ? pBuf->nSpace : 64; + u64 nNew = pBuf->nSpace ? pBuf->nSpace : 64; u8 *pNew; while( nNewp, nNew); + pNew = sqlite3_realloc64(pBuf->p, nNew); if( pNew==0 ){ *pRc = SQLITE_NOMEM; return 1; }else{ - pBuf->nSpace = nNew; + pBuf->nSpace = (int)nNew; pBuf->p = pNew; } } @@ -179420,34 +236940,36 @@ static void sqlite3Fts5Put32(u8 *aBuf, int iVal){ } static int sqlite3Fts5Get32(const u8 *aBuf){ - return (aBuf[0] << 24) + (aBuf[1] << 16) + (aBuf[2] << 8) + aBuf[3]; + return (int)((((u32)aBuf[0])<<24) + (aBuf[1]<<16) + (aBuf[2]<<8) + aBuf[3]); } /* -** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set +** Append buffer nData/pData to buffer pBuf. If an OOM error occurs, set ** the error code in p. If an error has already occurred when this function ** is called, it is a no-op. */ static void sqlite3Fts5BufferAppendBlob( int *pRc, - Fts5Buffer *pBuf, - u32 nData, + Fts5Buffer *pBuf, + u32 nData, const u8 *pData ){ - assert_nc( *pRc || nData>=0 ); - if( fts5BufferGrow(pRc, pBuf, nData) ) return; - memcpy(&pBuf->p[pBuf->n], pData, nData); - pBuf->n += nData; + if( nData ){ + if( fts5BufferGrow(pRc, pBuf, nData) ) return; + assert( pBuf->p!=0 ); + memcpy(&pBuf->p[pBuf->n], pData, nData); + pBuf->n += nData; + } } /* ** Append the nul-terminated string zStr to the buffer pBuf. This function -** ensures that the byte following the buffer data is set to 0x00, even +** ensures that the byte following the buffer data is set to 0x00, even ** though this byte is not included in the pBuf->n count. */ static void sqlite3Fts5BufferAppendString( int *pRc, - Fts5Buffer *pBuf, + Fts5Buffer *pBuf, const char *zStr ){ int nStr = (int)strlen(zStr); @@ -179459,13 +236981,13 @@ static void sqlite3Fts5BufferAppendString( ** Argument zFmt is a printf() style format string. This function performs ** the printf() style processing, then appends the results to buffer pBuf. ** -** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte +** Like sqlite3Fts5BufferAppendString(), this function ensures that the byte ** following the buffer data is set to 0x00, even though this byte is not ** included in the pBuf->n count. -*/ +*/ static void sqlite3Fts5BufferAppendPrintf( int *pRc, - Fts5Buffer *pBuf, + Fts5Buffer *pBuf, char *zFmt, ... ){ if( *pRc==SQLITE_OK ){ @@ -179492,12 +237014,12 @@ static char *sqlite3Fts5Mprintf(int *pRc, const char *zFmt, ...){ zRet = sqlite3_vmprintf(zFmt, ap); va_end(ap); if( zRet==0 ){ - *pRc = SQLITE_NOMEM; + *pRc = SQLITE_NOMEM; } } return zRet; } - + /* ** Free any buffer allocated by pBuf. Zero the structure before returning. @@ -179508,7 +237030,7 @@ static void sqlite3Fts5BufferFree(Fts5Buffer *pBuf){ } /* -** Zero the contents of the buffer object. But do not free the associated +** Zero the contents of the buffer object. But do not free the associated ** memory allocation. */ static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ @@ -179522,8 +237044,8 @@ static void sqlite3Fts5BufferZero(Fts5Buffer *pBuf){ */ static void sqlite3Fts5BufferSet( int *pRc, - Fts5Buffer *pBuf, - int nData, + Fts5Buffer *pBuf, + int nData, const u8 *pData ){ pBuf->n = 0; @@ -179536,21 +237058,36 @@ static int sqlite3Fts5PoslistNext64( i64 *piOff /* IN/OUT: Current offset */ ){ int i = *pi; + assert( a!=0 || i==0 ); if( i>=n ){ /* EOF */ *piOff = -1; - return 1; + return 1; }else{ i64 iOff = *piOff; - int iVal; + u32 iVal; + assert( a!=0 ); fts5FastGetVarint32(a, i, iVal); - if( iVal==1 ){ + if( iVal<=1 ){ + if( iVal==0 ){ + *pi = i; + return 0; + } fts5FastGetVarint32(a, i, iVal); iOff = ((i64)iVal) << 32; + assert( iOff>=0 ); fts5FastGetVarint32(a, i, iVal); + if( iVal<2 ){ + /* This is a corrupt record. So stop parsing it here. */ + *piOff = -1; + return 1; + } + *piOff = iOff + ((iVal-2) & 0x7FFFFFFF); + }else{ + *piOff = (iOff & (i64)0x7FFFFFFF<<32)+((iOff + (iVal-2)) & 0x7FFFFFFF); } - *piOff = iOff + (iVal-2); *pi = i; + assert_nc( *piOff>=iOff ); return 0; } } @@ -179585,22 +237122,24 @@ static int sqlite3Fts5PoslistReaderInit( ** to iPos before returning. */ static void sqlite3Fts5PoslistSafeAppend( - Fts5Buffer *pBuf, - i64 *piPrev, + Fts5Buffer *pBuf, + i64 *piPrev, i64 iPos ){ - static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; - if( (iPos & colmask) != (*piPrev & colmask) ){ - pBuf->p[pBuf->n++] = 1; - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); - *piPrev = (iPos & colmask); + if( iPos>=*piPrev ){ + static const i64 colmask = ((i64)(0x7FFFFFFF)) << 32; + if( (iPos & colmask) != (*piPrev & colmask) ){ + pBuf->p[pBuf->n++] = 1; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos>>32)); + *piPrev = (iPos & colmask); + } + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); + *piPrev = iPos; } - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], (iPos-*piPrev)+2); - *piPrev = iPos; } static int sqlite3Fts5PoslistWriterAppend( - Fts5Buffer *pBuf, + Fts5Buffer *pBuf, Fts5PoslistWriter *pWriter, i64 iPos ){ @@ -179610,14 +237149,14 @@ static int sqlite3Fts5PoslistWriterAppend( return SQLITE_OK; } -static void *sqlite3Fts5MallocZero(int *pRc, int nByte){ +static void *sqlite3Fts5MallocZero(int *pRc, sqlite3_int64 nByte){ void *pRet = 0; if( *pRc==SQLITE_OK ){ - pRet = sqlite3_malloc(nByte); - if( pRet==0 && nByte>0 ){ - *pRc = SQLITE_NOMEM; + pRet = sqlite3_malloc64(nByte); + if( pRet==0 ){ + if( nByte>0 ) *pRc = SQLITE_NOMEM; }else{ - memset(pRet, 0, nByte); + memset(pRet, 0, (size_t)nByte); } } return pRet; @@ -179629,7 +237168,7 @@ static void *sqlite3Fts5MallocZero(int *pRc, int nByte){ ** the length of the string is determined using strlen(). ** ** It is the responsibility of the caller to eventually free the returned -** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. +** buffer using sqlite3_free(). If an OOM error occurs, NULL is returned. */ static char *sqlite3Fts5Strndup(int *pRc, const char *pIn, int nIn){ char *zRet = 0; @@ -179696,9 +237235,9 @@ static int sqlite3Fts5TermsetNew(Fts5Termset **pp){ } static int sqlite3Fts5TermsetAdd( - Fts5Termset *p, + Fts5Termset *p, int iIdx, - const char *pTerm, int nTerm, + const char *pTerm, int nTerm, int *pbPresent ){ int rc = SQLITE_OK; @@ -179719,9 +237258,9 @@ static int sqlite3Fts5TermsetAdd( hash = hash % ArraySize(p->apHash); for(pEntry=p->apHash[hash]; pEntry; pEntry=pEntry->pNext){ - if( pEntry->iIdx==iIdx - && pEntry->nTerm==nTerm - && memcmp(pEntry->pTerm, pTerm, nTerm)==0 + if( pEntry->iIdx==iIdx + && pEntry->nTerm==nTerm + && memcmp(pEntry->pTerm, pTerm, nTerm)==0 ){ *pbPresent = 1; break; @@ -179783,8 +237322,10 @@ static void sqlite3Fts5TermsetFree(Fts5Termset *p){ #define FTS5_DEFAULT_CRISISMERGE 16 #define FTS5_DEFAULT_HASHSIZE (1024*1024) +#define FTS5_DEFAULT_DELETE_AUTOMERGE 10 /* default 10% */ + /* Maximum allowed page size */ -#define FTS5_MAX_PAGE_SIZE (128*1024) +#define FTS5_MAX_PAGE_SIZE (64*1024) static int fts5_iswhitespace(char x){ return (x==' '); @@ -179795,8 +237336,8 @@ static int fts5_isopenquote(char x){ } /* -** Argument pIn points to a character that is part of a nul-terminated -** string. Return a pointer to the first character following *pIn in +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in ** the string that is not a white-space character. */ static const char *fts5ConfigSkipWhitespace(const char *pIn){ @@ -179808,8 +237349,8 @@ static const char *fts5ConfigSkipWhitespace(const char *pIn){ } /* -** Argument pIn points to a character that is part of a nul-terminated -** string. Return a pointer to the first character following *pIn in +** Argument pIn points to a character that is part of a nul-terminated +** string. Return a pointer to the first character following *pIn in ** the string that is not a "bareword" character. */ static const char *fts5ConfigSkipBareword(const char *pIn){ @@ -179840,9 +237381,9 @@ static const char *fts5ConfigSkipLiteral(const char *pIn){ p++; if( *p=='\'' ){ p++; - while( (*p>='a' && *p<='f') - || (*p>='A' && *p<='F') - || (*p>='0' && *p<='9') + while( (*p>='a' && *p<='f') + || (*p>='A' && *p<='F') + || (*p>='0' && *p<='9') ){ p++; } @@ -179873,7 +237414,7 @@ static const char *fts5ConfigSkipLiteral(const char *pIn){ if( *p=='+' || *p=='-' ) p++; while( fts5_isdigit(*p) ) p++; - /* At this point, if the literal was an integer, the parse is + /* At this point, if the literal was an integer, the parse is ** finished. Or, if it is a floating point value, it may continue ** with either a decimal point or an 'E' character. */ if( *p=='.' && fts5_isdigit(p[1]) ){ @@ -179897,8 +237438,8 @@ static const char *fts5ConfigSkipLiteral(const char *pIn){ ** nul-terminator byte. ** ** If the close-quote is found, the value returned is the byte offset of -** the character immediately following it. Or, if the close-quote is not -** found, -1 is returned. If -1 is returned, the buffer is left in an +** the character immediately following it. Or, if the close-quote is not +** found, -1 is returned. If -1 is returned, the buffer is left in an ** undefined state. */ static int fts5Dequote(char *z){ @@ -179909,9 +237450,9 @@ static int fts5Dequote(char *z){ /* Set stack variable q to the close-quote character */ assert( q=='[' || q=='\'' || q=='"' || q=='`' ); - if( q=='[' ) q = ']'; + if( q=='[' ) q = ']'; - while( ALWAYS(z[iIn]) ){ + while( z[iIn] ){ if( z[iIn]==q ){ if( z[iIn+1]!=q ){ /* Character iIn was the close quote. */ @@ -179919,7 +237460,7 @@ static int fts5Dequote(char *z){ break; }else{ /* Character iIn and iIn+1 form an escaped quote character. Skip - ** the input cursor past both and copy a single quote character + ** the input cursor past both and copy a single quote character ** to the output buffer. */ iIn += 2; z[iOut++] = q; @@ -179964,8 +237505,8 @@ struct Fts5Enum { typedef struct Fts5Enum Fts5Enum; static int fts5ConfigSetEnum( - const Fts5Enum *aEnum, - const char *zEnum, + const Fts5Enum *aEnum, + const char *zEnum, int *peVal ){ int nEnum = (int)strlen(zEnum); @@ -179993,7 +237534,6 @@ static int fts5ConfigSetEnum( ** eventually free any such error message using sqlite3_free(). */ static int fts5ConfigParseSpecial( - Fts5Global *pGlobal, Fts5Config *pConfig, /* Configuration object to update */ const char *zCmd, /* Special command to parse */ const char *zArg, /* Argument to parse */ @@ -180001,6 +237541,7 @@ static int fts5ConfigParseSpecial( ){ int rc = SQLITE_OK; int nCmd = (int)strlen(zCmd); + if( sqlite3_strnicmp("prefix", zCmd, nCmd)==0 ){ const int nByte = sizeof(int) * FTS5_MAX_PREFIX_INDEXES; const char *p; @@ -180056,13 +237597,12 @@ static int fts5ConfigParseSpecial( if( sqlite3_strnicmp("tokenize", zCmd, nCmd)==0 ){ const char *p = (const char*)zArg; - int nArg = (int)strlen(zArg) + 1; - char **azArg = sqlite3Fts5MallocZero(&rc, sizeof(char*) * nArg); - char *pDel = sqlite3Fts5MallocZero(&rc, nArg * 2); - char *pSpace = pDel; + sqlite3_int64 nArg = strlen(zArg) + 1; + char **azArg = sqlite3Fts5MallocZero(&rc, (sizeof(char*) + 2) * nArg); - if( azArg && pSpace ){ - if( pConfig->pTok ){ + if( azArg ){ + char *pSpace = (char*)&azArg[nArg]; + if( pConfig->t.azArg ){ *pzErr = sqlite3_mprintf("multiple tokenize=... directives"); rc = SQLITE_ERROR; }else{ @@ -180085,16 +237625,14 @@ static int fts5ConfigParseSpecial( *pzErr = sqlite3_mprintf("parse error in tokenize directive"); rc = SQLITE_ERROR; }else{ - rc = sqlite3Fts5GetTokenizer(pGlobal, - (const char**)azArg, nArg, &pConfig->pTok, &pConfig->pTokApi, - pzErr - ); + pConfig->t.azArg = (const char**)azArg; + pConfig->t.nArg = nArg; + azArg = 0; } } } - sqlite3_free(azArg); - sqlite3_free(pDel); + return rc; } @@ -180113,6 +237651,26 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("contentless_delete", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessDelete = (zArg[0]=='1'); + } + return rc; + } + + if( sqlite3_strnicmp("contentless_unindexed", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed contentless_delete=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bContentlessUnindexed = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("content_rowid", zCmd, nCmd)==0 ){ if( pConfig->zContentRowid ){ *pzErr = sqlite3_mprintf("multiple content_rowid=... directives"); @@ -180133,6 +237691,16 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("locale", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed locale=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bLocale = (zArg[0]=='1'); + } + return rc; + } + if( sqlite3_strnicmp("detail", zCmd, nCmd)==0 ){ const Fts5Enum aDetail[] = { { "none", FTS5_DETAIL_NONE }, @@ -180147,22 +237715,20 @@ static int fts5ConfigParseSpecial( return rc; } + if( sqlite3_strnicmp("tokendata", zCmd, nCmd)==0 ){ + if( (zArg[0]!='0' && zArg[0]!='1') || zArg[1]!='\0' ){ + *pzErr = sqlite3_mprintf("malformed tokendata=... directive"); + rc = SQLITE_ERROR; + }else{ + pConfig->bTokendata = (zArg[0]=='1'); + } + return rc; + } + *pzErr = sqlite3_mprintf("unrecognized option: \"%.*s\"", nCmd, zCmd); return SQLITE_ERROR; } -/* -** Allocate an instance of the default tokenizer ("simple") at -** Fts5Config.pTokenizer. Return SQLITE_OK if successful, or an SQLite error -** code if an error occurs. -*/ -static int fts5ConfigDefaultTokenizer(Fts5Global *pGlobal, Fts5Config *pConfig){ - assert( pConfig->pTok==0 && pConfig->pTokApi==0 ); - return sqlite3Fts5GetTokenizer( - pGlobal, 0, 0, &pConfig->pTok, &pConfig->pTokApi, 0 - ); -} - /* ** Gobble up the first bareword or quoted word from the input buffer zIn. ** Return a pointer to the character immediately following the last in @@ -180186,8 +237752,8 @@ static const char *fts5ConfigGobbleWord( ){ const char *zRet = 0; - int nIn = (int)strlen(zIn); - char *zOut = sqlite3_malloc(nIn+1); + sqlite3_int64 nIn = strlen(zIn); + char *zOut = sqlite3_malloc64(nIn+1); assert( *pRc==SQLITE_OK ); *pbQuoted = 0; @@ -180196,7 +237762,7 @@ static const char *fts5ConfigGobbleWord( if( zOut==0 ){ *pRc = SQLITE_NOMEM; }else{ - memcpy(zOut, zIn, nIn+1); + memcpy(zOut, zIn, (size_t)(nIn+1)); if( fts5_isopenquote(zOut[0]) ){ int ii = fts5Dequote(zOut); zRet = &zIn[ii]; @@ -180219,20 +237785,22 @@ static const char *fts5ConfigGobbleWord( } static int fts5ConfigParseColumn( - Fts5Config *p, - char *zCol, - char *zArg, - char **pzErr + Fts5Config *p, + char *zCol, + char *zArg, + char **pzErr, + int *pbUnindexed ){ int rc = SQLITE_OK; - if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) - || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) + if( 0==sqlite3_stricmp(zCol, FTS5_RANK_NAME) + || 0==sqlite3_stricmp(zCol, FTS5_ROWID_NAME) ){ *pzErr = sqlite3_mprintf("reserved fts5 column name: %s", zCol); rc = SQLITE_ERROR; }else if( zArg ){ if( 0==sqlite3_stricmp(zArg, "unindexed") ){ p->abUnindexed[p->nCol] = 1; + *pbUnindexed = 1; }else{ *pzErr = sqlite3_mprintf("unrecognized column option: %s", zArg); rc = SQLITE_ERROR; @@ -180253,11 +237821,26 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, "T.%Q", p->zContentRowid); if( p->eContent!=FTS5_CONTENT_NONE ){ + assert( p->eContent==FTS5_CONTENT_EXTERNAL + || p->eContent==FTS5_CONTENT_NORMAL + || p->eContent==FTS5_CONTENT_UNINDEXED + ); for(i=0; inCol; i++){ if( p->eContent==FTS5_CONTENT_EXTERNAL ){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.%Q", p->azCol[i]); - }else{ + }else if( p->eContent==FTS5_CONTENT_NORMAL || p->abUnindexed[i] ){ sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.c%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); + } + } + } + if( p->eContent==FTS5_CONTENT_NORMAL && p->bLocale ){ + for(i=0; inCol; i++){ + if( p->abUnindexed[i]==0 ){ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", T.l%d", i); + }else{ + sqlite3Fts5BufferAppendPrintf(&rc, &buf, ", NULL"); } } } @@ -180269,14 +237852,14 @@ static int fts5ConfigMakeExprlist(Fts5Config *p){ /* ** Arguments nArg/azArg contain the string arguments passed to the xCreate -** or xConnect method of the virtual table. This function attempts to +** or xConnect method of the virtual table. This function attempts to ** allocate an instance of Fts5Config containing the results of parsing ** those arguments. ** ** If successful, SQLITE_OK is returned and *ppOut is set to point to the -** new Fts5Config object. If an error occurs, an SQLite error code is +** new Fts5Config object. If an error occurs, an SQLite error code is ** returned, *ppOut is set to NULL and an error message may be left in -** *pzErr. It is the responsibility of the caller to eventually free any +** *pzErr. It is the responsibility of the caller to eventually free any ** such error message using sqlite3_free(). */ static int sqlite3Fts5ConfigParse( @@ -180290,17 +237873,19 @@ static int sqlite3Fts5ConfigParse( int rc = SQLITE_OK; /* Return code */ Fts5Config *pRet; /* New object to return */ int i; - int nByte; + sqlite3_int64 nByte; + int bUnindexed = 0; /* True if there are one or more UNINDEXED */ *ppOut = pRet = (Fts5Config*)sqlite3_malloc(sizeof(Fts5Config)); if( pRet==0 ) return SQLITE_NOMEM; memset(pRet, 0, sizeof(Fts5Config)); + pRet->pGlobal = pGlobal; pRet->db = db; pRet->iCookie = -1; nByte = nArg * (sizeof(char*) + sizeof(u8)); pRet->azCol = (char**)sqlite3Fts5MallocZero(&rc, nByte); - pRet->abUnindexed = (u8*)&pRet->azCol[nArg]; + pRet->abUnindexed = pRet->azCol ? (u8*)&pRet->azCol[nArg] : 0; pRet->zDb = sqlite3Fts5Strndup(&rc, azArg[1], -1); pRet->zName = sqlite3Fts5Strndup(&rc, azArg[2], -1); pRet->bColumnsize = 1; @@ -180313,6 +237898,7 @@ static int sqlite3Fts5ConfigParse( rc = SQLITE_ERROR; } + assert( (pRet->abUnindexed && pRet->azCol) || rc!=SQLITE_OK ); for(i=3; rc==SQLITE_OK && ipTok==0 ){ - rc = fts5ConfigDefaultTokenizer(pGlobal, pRet); + /* We only allow contentless_delete=1 if the table is indeed contentless. */ + if( rc==SQLITE_OK + && pRet->bContentlessDelete + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 requires a contentless table" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_delete=1 if columnsize=0 is not present. + ** + ** This restriction may be removed at some point. + */ + if( rc==SQLITE_OK && pRet->bContentlessDelete && pRet->bColumnsize==0 ){ + *pzErr = sqlite3_mprintf( + "contentless_delete=1 is incompatible with columnsize=0" + ); + rc = SQLITE_ERROR; + } + + /* We only allow contentless_unindexed=1 if the table is actually a + ** contentless one. + */ + if( rc==SQLITE_OK + && pRet->bContentlessUnindexed + && pRet->eContent!=FTS5_CONTENT_NONE + ){ + *pzErr = sqlite3_mprintf( + "contentless_unindexed=1 requires a contentless table" + ); + rc = SQLITE_ERROR; } /* If no zContent option was specified, fill in the default values. */ if( rc==SQLITE_OK && pRet->zContent==0 ){ const char *zTail = 0; - assert( pRet->eContent==FTS5_CONTENT_NORMAL - || pRet->eContent==FTS5_CONTENT_NONE + assert( pRet->eContent==FTS5_CONTENT_NORMAL + || pRet->eContent==FTS5_CONTENT_NONE ); if( pRet->eContent==FTS5_CONTENT_NORMAL ){ zTail = "content"; + }else if( bUnindexed && pRet->bContentlessUnindexed ){ + pRet->eContent = FTS5_CONTENT_UNINDEXED; + zTail = "content"; }else if( pRet->bColumnsize ){ zTail = "docsize"; } @@ -180401,9 +238023,14 @@ static int sqlite3Fts5ConfigParse( static void sqlite3Fts5ConfigFree(Fts5Config *pConfig){ if( pConfig ){ int i; - if( pConfig->pTok ){ - pConfig->pTokApi->xDelete(pConfig->pTok); + if( pConfig->t.pTok ){ + if( pConfig->t.pApi1 ){ + pConfig->t.pApi1->xDelete(pConfig->t.pTok); + }else{ + pConfig->t.pApi2->xDelete(pConfig->t.pTok); + } } + sqlite3_free((char*)pConfig->t.azArg); sqlite3_free(pConfig->zDb); sqlite3_free(pConfig->zName); for(i=0; inCol; i++){ @@ -180435,7 +238062,7 @@ static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ const char *zSep = (i==0?"":", "); zSql = sqlite3Fts5Mprintf(&rc, "%z%s%Q", zSql, zSep, pConfig->azCol[i]); } - zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)", + zSql = sqlite3Fts5Mprintf(&rc, "%z, %Q HIDDEN, %s HIDDEN)", zSql, pConfig->zName, FTS5_RANK_NAME ); @@ -180444,7 +238071,7 @@ static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ rc = sqlite3_declare_vtab(pConfig->db, zSql); sqlite3_free(zSql); } - + return rc; } @@ -180462,7 +238089,7 @@ static int sqlite3Fts5ConfigDeclareVtab(Fts5Config *pConfig){ ** int iPos // Position of token in input (first token is 0) ** ** If the callback returns a non-zero value the tokenization is abandoned -** and no further callbacks are issued. +** and no further callbacks are issued. ** ** This function returns SQLITE_OK if successful or an SQLite error code ** if an error occurs. If the tokenization was abandoned early because @@ -180478,10 +238105,24 @@ static int sqlite3Fts5Tokenize( void *pCtx, /* Context passed to xToken() */ int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ ){ - if( pText==0 ) return SQLITE_OK; - return pConfig->pTokApi->xTokenize( - pConfig->pTok, pCtx, flags, pText, nText, xToken - ); + int rc = SQLITE_OK; + if( pText ){ + if( pConfig->t.pTok==0 ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } + if( rc==SQLITE_OK ){ + if( pConfig->t.pApi1 ){ + rc = pConfig->t.pApi1->xTokenize( + pConfig->t.pTok, pCtx, flags, pText, nText, xToken + ); + }else{ + rc = pConfig->t.pApi2->xTokenize(pConfig->t.pTok, pCtx, flags, + pText, nText, pConfig->t.pLocale, pConfig->t.nLocale, xToken + ); + } + } + } + return rc; } /* @@ -180492,7 +238133,7 @@ static int sqlite3Fts5Tokenize( */ static const char *fts5ConfigSkipArgs(const char *pIn){ const char *p = pIn; - + while( 1 ){ p = fts5ConfigSkipWhitespace(p); p = fts5ConfigSkipLiteral(p); @@ -180509,7 +238150,7 @@ static const char *fts5ConfigSkipArgs(const char *pIn){ } /* -** Parameter zIn contains a rank() function specification. The format of +** Parameter zIn contains a rank() function specification. The format of ** this is: ** ** + Bareword (function name) @@ -180551,7 +238192,7 @@ static int sqlite3Fts5ConfigParseRank( p++; } if( rc==SQLITE_OK ){ - const char *pArgs; + const char *pArgs; p = fts5ConfigSkipWhitespace(p); pArgs = p; if( *p!=')' ){ @@ -180577,8 +238218,8 @@ static int sqlite3Fts5ConfigParseRank( } static int sqlite3Fts5ConfigSetValue( - Fts5Config *pConfig, - const char *zKey, + Fts5Config *pConfig, + const char *zKey, sqlite3_value *pVal, int *pbBadkey ){ @@ -180589,7 +238230,7 @@ static int sqlite3Fts5ConfigSetValue( if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ pgsz = sqlite3_value_int(pVal); } - if( pgsz<=0 || pgsz>FTS5_MAX_PAGE_SIZE ){ + if( pgsz<32 || pgsz>FTS5_MAX_PAGE_SIZE ){ *pbBadkey = 1; }else{ pConfig->pgsz = pgsz; @@ -180642,10 +238283,23 @@ static int sqlite3Fts5ConfigSetValue( *pbBadkey = 1; }else{ if( nCrisisMerge<=1 ) nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; + if( nCrisisMerge>=FTS5_MAX_SEGMENT ) nCrisisMerge = FTS5_MAX_SEGMENT-1; pConfig->nCrisisMerge = nCrisisMerge; } } + else if( 0==sqlite3_stricmp(zKey, "deletemerge") ){ + int nVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + nVal = sqlite3_value_int(pVal); + }else{ + *pbBadkey = 1; + } + if( nVal<0 ) nVal = FTS5_DEFAULT_DELETE_AUTOMERGE; + if( nVal>100 ) nVal = 0; + pConfig->nDeleteMerge = nVal; + } + else if( 0==sqlite3_stricmp(zKey, "rank") ){ const char *zIn = (const char*)sqlite3_value_text(pVal); char *zRank; @@ -180660,6 +238314,18 @@ static int sqlite3Fts5ConfigSetValue( rc = SQLITE_OK; *pbBadkey = 1; } + } + + else if( 0==sqlite3_stricmp(zKey, "secure-delete") ){ + int bVal = -1; + if( SQLITE_INTEGER==sqlite3_value_numeric_type(pVal) ){ + bVal = sqlite3_value_int(pVal); + } + if( bVal<0 ){ + *pbBadkey = 1; + }else{ + pConfig->bSecureDelete = (bVal ? 1 : 0); + } }else{ *pbBadkey = 1; } @@ -180682,6 +238348,7 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ pConfig->nUsermerge = FTS5_DEFAULT_USERMERGE; pConfig->nCrisisMerge = FTS5_DEFAULT_CRISISMERGE; pConfig->nHashSize = FTS5_DEFAULT_HASHSIZE; + pConfig->nDeleteMerge = FTS5_DEFAULT_DELETE_AUTOMERGE; zSql = sqlite3Fts5Mprintf(&rc, zSelect, pConfig->zDb, pConfig->zName); if( zSql ){ @@ -180703,16 +238370,18 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ } rc = sqlite3_finalize(p); } - - if( rc==SQLITE_OK && iVersion!=FTS5_CURRENT_VERSION ){ + + if( rc==SQLITE_OK + && iVersion!=FTS5_CURRENT_VERSION + && iVersion!=FTS5_CURRENT_VERSION_SECUREDELETE + ){ rc = SQLITE_ERROR; - if( pConfig->pzErrmsg ){ - assert( 0==*pConfig->pzErrmsg ); - *pConfig->pzErrmsg = sqlite3_mprintf( - "invalid fts5 file format (found %d, expected %d) - run 'rebuild'", - iVersion, FTS5_CURRENT_VERSION - ); - } + sqlite3Fts5ConfigErrmsg(pConfig, "invalid fts5 file format " + "(found %d, expected %d or %d) - run 'rebuild'", + iVersion, FTS5_CURRENT_VERSION, FTS5_CURRENT_VERSION_SECUREDELETE + ); + }else{ + pConfig->iVersion = iVersion; } if( rc==SQLITE_OK ){ @@ -180721,6 +238390,29 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ return rc; } +/* +** Set (*pConfig->pzErrmsg) to point to an sqlite3_malloc()ed buffer +** containing the error message created using printf() style formatting +** string zFmt and its trailing arguments. +*/ +static void sqlite3Fts5ConfigErrmsg(Fts5Config *pConfig, const char *zFmt, ...){ + va_list ap; /* ... printf arguments */ + char *zMsg = 0; + + va_start(ap, zFmt); + zMsg = sqlite3_vmprintf(zFmt, ap); + if( pConfig->pzErrmsg ){ + assert( *pConfig->pzErrmsg==0 ); + *pConfig->pzErrmsg = zMsg; + }else{ + sqlite3_free(zMsg); + } + + va_end(ap); +} + + + /* ** 2014 May 31 ** @@ -180740,6 +238432,10 @@ static int sqlite3Fts5ConfigLoad(Fts5Config *pConfig, int iCookie){ /* #include "fts5Int.h" */ /* #include "fts5parse.h" */ +#ifndef SQLITE_FTS5_MAX_EXPR_DEPTH +# define SQLITE_FTS5_MAX_EXPR_DEPTH 256 +#endif + /* ** All token types in the generated fts5parse.h file are greater than 0. */ @@ -180759,6 +238455,7 @@ static void sqlite3Fts5Parser(void*, int, Fts5Token, Fts5Parse*); /* #include */ static void sqlite3Fts5ParserTrace(FILE*, char*); #endif +static int sqlite3Fts5ParserFallback(int); struct Fts5Expr { @@ -180772,18 +238469,28 @@ struct Fts5Expr { /* ** eType: -** Expression node type. Always one of: +** Expression node type. Usually one of: ** ** FTS5_AND (nChild, apChild valid) ** FTS5_OR (nChild, apChild valid) ** FTS5_NOT (nChild, apChild valid) ** FTS5_STRING (pNear valid) ** FTS5_TERM (pNear valid) +** +** An expression node with eType==0 may also exist. It always matches zero +** rows. This is created when a phrase containing no tokens is parsed. +** e.g. "". +** +** iHeight: +** Distance from this node to furthest leaf. This is always 0 for nodes +** of type FTS5_STRING and FTS5_TERM. For all other nodes it is one +** greater than the largest child value. */ struct Fts5ExprNode { int eType; /* Node type */ int bEof; /* True at EOF */ int bNomatch; /* True if entry is not a match */ + int iHeight; /* Distance to tree leaf nodes */ /* Next method for this node. */ int (*xNext)(Fts5Expr*, Fts5ExprNode*, int, i64); @@ -180791,7 +238498,7 @@ struct Fts5ExprNode { i64 iRowid; /* Current rowid */ Fts5ExprNearset *pNear; /* For FTS5_STRING - cluster of phrases */ - /* Child nodes. For a NOT node, this array always contains 2 entries. For + /* Child nodes. For a NOT node, this array always contains 2 entries. For ** AND or OR nodes, it contains 2 or more entries. */ int nChild; /* Number of child nodes */ Fts5ExprNode *apChild[1]; /* Array of child nodes */ @@ -180810,8 +238517,11 @@ struct Fts5ExprNode { ** or term prefix. */ struct Fts5ExprTerm { - int bPrefix; /* True for a prefix term */ - char *zTerm; /* nul-terminated term */ + u8 bPrefix; /* True for a prefix term */ + u8 bFirst; /* True if token must be first in column */ + char *pTerm; /* Term data */ + int nQueryTerm; /* Effective size of term in bytes */ + int nFullTerm; /* Size of term in bytes incl. tokendata */ Fts5IndexIter *pIter; /* Iterator for this term */ Fts5ExprTerm *pSynonym; /* Pointer to first in list of synonyms */ }; @@ -180849,12 +238559,39 @@ struct Fts5Parse { int nPhrase; /* Size of apPhrase array */ Fts5ExprPhrase **apPhrase; /* Array of all phrases */ Fts5ExprNode *pExpr; /* Result of a successful parse */ + int bPhraseToAnd; /* Convert "a+b" to "a AND b" */ }; +/* +** Check that the Fts5ExprNode.iHeight variables are set correctly in +** the expression tree passed as the only argument. +*/ +#ifndef NDEBUG +static void assert_expr_depth_ok(int rc, Fts5ExprNode *p){ + if( rc==SQLITE_OK ){ + if( p->eType==FTS5_TERM || p->eType==FTS5_STRING || p->eType==0 ){ + assert( p->iHeight==0 ); + }else{ + int ii; + int iMaxChild = 0; + for(ii=0; iinChild; ii++){ + Fts5ExprNode *pChild = p->apChild[ii]; + iMaxChild = MAX(iMaxChild, pChild->iHeight); + assert_expr_depth_ok(SQLITE_OK, pChild); + } + assert( p->iHeight==iMaxChild+1 ); + } + } +} +#else +# define assert_expr_depth_ok(rc, p) +#endif + static void sqlite3Fts5ParseError(Fts5Parse *pParse, const char *zFmt, ...){ va_list ap; va_start(ap, zFmt); if( pParse->rc==SQLITE_OK ){ + assert( pParse->zErr==0 ); pParse->zErr = sqlite3_vmprintf(zFmt, ap); pParse->rc = SQLITE_ERROR; } @@ -180869,7 +238606,7 @@ static int fts5ExprIsspace(char t){ ** Read the first token from the nul-terminated string at *pz. */ static int fts5ExprGetToken( - Fts5Parse *pParse, + Fts5Parse *pParse, const char **pz, /* IN/OUT: Pointer into buffer */ Fts5Token *pToken ){ @@ -180890,6 +238627,8 @@ static int fts5ExprGetToken( case ',': tok = FTS5_COMMA; break; case '+': tok = FTS5_PLUS; break; case '*': tok = FTS5_STAR; break; + case '-': tok = FTS5_MINUS; break; + case '^': tok = FTS5_CARET; break; case '\0': tok = FTS5_EOF; break; case '"': { @@ -180930,13 +238669,15 @@ static int fts5ExprGetToken( return tok; } -static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc((int)t); } +static void *fts5ParseAlloc(u64 t){ return sqlite3_malloc64((sqlite3_int64)t);} static void fts5ParseFree(void *p){ sqlite3_free(p); } static int sqlite3Fts5ExprNew( Fts5Config *pConfig, /* FTS5 Configuration */ + int bPhraseToAnd, + int iCol, const char *zExpr, /* Expression text */ - Fts5Expr **ppNew, + Fts5Expr **ppNew, char **pzErr ){ Fts5Parse sParse; @@ -180949,6 +238690,7 @@ static int sqlite3Fts5ExprNew( *ppNew = 0; *pzErr = 0; memset(&sParse, 0, sizeof(sParse)); + sParse.bPhraseToAnd = bPhraseToAnd; pEngine = sqlite3Fts5ParserAlloc(fts5ParseAlloc); if( pEngine==0 ){ return SQLITE_NOMEM; } sParse.pConfig = pConfig; @@ -180959,6 +238701,21 @@ static int sqlite3Fts5ExprNew( }while( sParse.rc==SQLITE_OK && t!=FTS5_EOF ); sqlite3Fts5ParserFree(pEngine, fts5ParseFree); + assert( sParse.pExpr || sParse.rc!=SQLITE_OK ); + assert_expr_depth_ok(sParse.rc, sParse.pExpr); + + /* If the LHS of the MATCH expression was a user column, apply the + ** implicit column-filter. */ + if( sParse.rc==SQLITE_OK && iColnCol ){ + int n = sizeof(Fts5Colset); + Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&sParse.rc, n); + if( pColset ){ + pColset->nCol = 1; + pColset->aiCol[0] = iCol; + sqlite3Fts5ParseSetColset(&sParse, sParse.pExpr, pColset); + } + } + assert( sParse.rc!=SQLITE_OK || sParse.zErr==0 ); if( sParse.rc==SQLITE_OK ){ *ppNew = pNew = sqlite3_malloc(sizeof(Fts5Expr)); @@ -180966,19 +238723,12 @@ static int sqlite3Fts5ExprNew( sParse.rc = SQLITE_NOMEM; sqlite3Fts5ParseNodeFree(sParse.pExpr); }else{ - if( !sParse.pExpr ){ - const int nByte = sizeof(Fts5ExprNode); - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&sParse.rc, nByte); - if( pNew->pRoot ){ - pNew->pRoot->bEof = 1; - } - }else{ - pNew->pRoot = sParse.pExpr; - } + pNew->pRoot = sParse.pExpr; pNew->pIndex = 0; pNew->pConfig = pConfig; pNew->apExprPhrase = sParse.apPhrase; pNew->nPhrase = sParse.nPhrase; + pNew->bDesc = 0; sParse.apPhrase = 0; } }else{ @@ -180986,10 +238736,103 @@ static int sqlite3Fts5ExprNew( } sqlite3_free(sParse.apPhrase); - *pzErr = sParse.zErr; + if( 0==*pzErr ){ + *pzErr = sParse.zErr; + }else{ + sqlite3_free(sParse.zErr); + } return sParse.rc; } +/* +** Assuming that buffer z is at least nByte bytes in size and contains a +** valid utf-8 string, return the number of characters in the string. +*/ +static int fts5ExprCountChar(const char *z, int nByte){ + int nRet = 0; + int ii; + for(ii=0; ii=3 ){ + int jj; + zExpr[iOut++] = '"'; + for(jj=iFirst; jj0 ){ + int bAnd = 0; + if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ + bAnd = 1; + if( pConfig->eDetail==FTS5_DETAIL_NONE ){ + iCol = pConfig->nCol; + } + } + zExpr[iOut] = '\0'; + rc = sqlite3Fts5ExprNew(pConfig, bAnd, iCol, zExpr, pp,pConfig->pzErrmsg); + }else{ + *pp = 0; + } + sqlite3_free(zExpr); + } + + return rc; +} + /* ** Free the expression node object passed as the only argument. */ @@ -181015,6 +238858,42 @@ static void sqlite3Fts5ExprFree(Fts5Expr *p){ } } +static int sqlite3Fts5ExprAnd(Fts5Expr **pp1, Fts5Expr *p2){ + Fts5Parse sParse; + memset(&sParse, 0, sizeof(sParse)); + + if( *pp1 && p2 ){ + Fts5Expr *p1 = *pp1; + int nPhrase = p1->nPhrase + p2->nPhrase; + + p1->pRoot = sqlite3Fts5ParseNode(&sParse, FTS5_AND, p1->pRoot, p2->pRoot,0); + p2->pRoot = 0; + + if( sParse.rc==SQLITE_OK ){ + Fts5ExprPhrase **ap = (Fts5ExprPhrase**)sqlite3_realloc( + p1->apExprPhrase, nPhrase * sizeof(Fts5ExprPhrase*) + ); + if( ap==0 ){ + sParse.rc = SQLITE_NOMEM; + }else{ + int i; + memmove(&ap[p2->nPhrase], ap, p1->nPhrase*sizeof(Fts5ExprPhrase*)); + for(i=0; inPhrase; i++){ + ap[i] = p2->apExprPhrase[i]; + } + p1->nPhrase = nPhrase; + p1->apExprPhrase = ap; + } + } + sqlite3_free(p2->apExprPhrase); + sqlite3_free(p2); + }else if( p2 ){ + *pp1 = p2; + } + + return sParse.rc; +} + /* ** Argument pTerm must be a synonym iterator. Return the current rowid ** that it points to. @@ -181024,6 +238903,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ int bRetValid = 0; Fts5ExprTerm *p; + assert( pTerm ); assert( pTerm->pSynonym ); assert( bDesc==0 || bDesc==1 ); for(p=pTerm; p; p=p->pSynonym){ @@ -181044,7 +238924,7 @@ static i64 fts5ExprSynonymRowid(Fts5ExprTerm *pTerm, int bDesc, int *pbEof){ ** Argument pTerm must be a synonym iterator. */ static int fts5ExprSynonymList( - Fts5ExprTerm *pTerm, + Fts5ExprTerm *pTerm, i64 iRowid, Fts5Buffer *pBuf, /* Use this buffer for space if required */ u8 **pa, int *pn @@ -181062,8 +238942,8 @@ static int fts5ExprSynonymList( if( sqlite3Fts5IterEof(pIter)==0 && pIter->iRowid==iRowid ){ if( pIter->nData==0 ) continue; if( nIter==nAlloc ){ - int nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; - Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc(nByte); + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nAlloc * 2; + Fts5PoslistReader *aNew = (Fts5PoslistReader*)sqlite3_malloc64(nByte); if( aNew==0 ){ rc = SQLITE_NOMEM; goto synonym_poslist_out; @@ -181117,13 +238997,13 @@ static int fts5ExprSynonymList( /* ** All individual term iterators in pPhrase are guaranteed to be valid and -** pointing to the same rowid when this function is called. This function +** pointing to the same rowid when this function is called. This function ** checks if the current rowid really is a match, and if so populates ** the pPhrase->poslist buffer accordingly. Output parameter *pbMatch ** is set to true if this is really a match, or false otherwise. ** -** SQLITE_OK is returned if an error occurs, or an SQLite error code -** otherwise. It is not considered an error code if the current rowid is +** SQLITE_OK is returned if an error occurs, or an SQLite error code +** otherwise. It is not considered an error code if the current rowid is ** not a match. */ static int fts5ExprPhraseIsMatch( @@ -181136,14 +239016,15 @@ static int fts5ExprPhraseIsMatch( Fts5PoslistReader *aIter = aStatic; int i; int rc = SQLITE_OK; - + int bFirst = pPhrase->aTerm[0].bFirst; + fts5BufferZero(&pPhrase->poslist); /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pPhrase->nTerm>ArraySize(aStatic) ){ - int nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; - aIter = (Fts5PoslistReader*)sqlite3_malloc(nByte); + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * pPhrase->nTerm; + aIter = (Fts5PoslistReader*)sqlite3_malloc64(nByte); if( !aIter ) return SQLITE_NOMEM; } memset(aIter, 0, sizeof(Fts5PoslistReader) * pPhrase->nTerm); @@ -181190,8 +239071,10 @@ static int fts5ExprPhraseIsMatch( }while( bMatch==0 ); /* Append position iPos to the output */ - rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); - if( rc!=SQLITE_OK ) goto ismatch_out; + if( bFirst==0 || FTS5_POS2OFFSET(iPos)==0 ){ + rc = sqlite3Fts5PoslistWriterAppend(&pPhrase->poslist, &writer, iPos); + if( rc!=SQLITE_OK ) goto ismatch_out; + } for(i=0; inTerm; i++){ if( sqlite3Fts5PoslistReaderNext(&aIter[i]) ) goto ismatch_out; @@ -181256,7 +239139,7 @@ struct Fts5NearTrimmer { ** function is called, it is a no-op. Or, if an error (e.g. SQLITE_NOMEM) ** occurs within this function (*pRc) is set accordingly before returning. ** The return value is undefined in both these cases. -** +** ** If no error occurs and non-zero (a match) is returned, the position-list ** of each phrase object is edited to contain only those entries that ** meet the constraint before returning. @@ -181275,7 +239158,7 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ /* If the aStatic[] array is not large enough, allocate a large array ** using sqlite3_malloc(). This approach could be improved upon. */ if( pNear->nPhrase>ArraySize(aStatic) ){ - int nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; + sqlite3_int64 nByte = sizeof(Fts5NearTrimmer) * pNear->nPhrase; a = (Fts5NearTrimmer*)sqlite3Fts5MallocZero(&rc, nByte); }else{ memset(aStatic, 0, sizeof(aStatic)); @@ -181288,7 +239171,7 @@ static int fts5ExprNearIsMatch(int *pRc, Fts5ExprNearset *pNear){ /* Initialize a lookahead iterator for each phrase. After passing the ** buffer and buffer size to the lookaside-reader init function, zero ** the phrase poslist buffer. The new poslist for the phrase (containing - ** the same entries as the original with some entries removed on account + ** the same entries as the original with some entries removed on account ** of the NEAR constraint) is written over the original even as it is ** being read. This is safe as the entries for the new poslist are a ** subset of the old, so it is not possible for data yet to be read to @@ -181445,7 +239328,9 @@ static int fts5ExprNearTest( ** phrase is not a match, break out of the loop early. */ for(i=0; rc==SQLITE_OK && inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym || pNear->pColset ){ + if( pPhrase->nTerm>1 || pPhrase->aTerm[0].pSynonym + || pNear->pColset || pPhrase->aTerm[0].bFirst + ){ int bMatch = 0; rc = fts5ExprPhraseIsMatch(pNode, pPhrase, &bMatch); if( bMatch==0 ) break; @@ -181468,49 +239353,61 @@ static int fts5ExprNearTest( ** Initialize all term iterators in the pNear object. If any term is found ** to match no documents at all, return immediately without initializing any ** further iterators. +** +** If an error occurs, return an SQLite error code. Otherwise, return +** SQLITE_OK. It is not considered an error if some term matches zero +** documents. */ static int fts5ExprNearInitAll( Fts5Expr *pExpr, Fts5ExprNode *pNode ){ Fts5ExprNearset *pNear = pNode->pNear; - int i, j; - int rc = SQLITE_OK; + int i; assert( pNode->bNomatch==0 ); - for(i=0; rc==SQLITE_OK && inPhrase; i++){ + for(i=0; inPhrase; i++){ Fts5ExprPhrase *pPhrase = pNear->apPhrase[i]; - for(j=0; jnTerm; j++){ - Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; - Fts5ExprTerm *p; - int bEof = 1; + if( pPhrase->nTerm==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + }else{ + int j; + for(j=0; jnTerm; j++){ + Fts5ExprTerm *pTerm = &pPhrase->aTerm[j]; + Fts5ExprTerm *p; + int bHit = 0; - for(p=pTerm; p && rc==SQLITE_OK; p=p->pSynonym){ - if( p->pIter ){ - sqlite3Fts5IterClose(p->pIter); - p->pIter = 0; + for(p=pTerm; p; p=p->pSynonym){ + int rc; + if( p->pIter ){ + sqlite3Fts5IterClose(p->pIter); + p->pIter = 0; + } + rc = sqlite3Fts5IndexQuery( + pExpr->pIndex, p->pTerm, p->nQueryTerm, + (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | + (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), + pNear->pColset, + &p->pIter + ); + assert( (rc==SQLITE_OK)==(p->pIter!=0) ); + if( rc!=SQLITE_OK ) return rc; + if( 0==sqlite3Fts5IterEof(p->pIter) ){ + bHit = 1; + } } - rc = sqlite3Fts5IndexQuery( - pExpr->pIndex, p->zTerm, (int)strlen(p->zTerm), - (pTerm->bPrefix ? FTS5INDEX_QUERY_PREFIX : 0) | - (pExpr->bDesc ? FTS5INDEX_QUERY_DESC : 0), - pNear->pColset, - &p->pIter - ); - assert( rc==SQLITE_OK || p->pIter==0 ); - if( p->pIter && 0==sqlite3Fts5IterEof(p->pIter) ){ - bEof = 0; - } - } - if( bEof ){ - pNode->bEof = 1; - return rc; + if( bHit==0 ){ + pNode->bEof = 1; + return SQLITE_OK; + } } } } - return rc; + pNode->bEof = 0; + return SQLITE_OK; } /* @@ -181579,7 +239476,7 @@ static void fts5ExprNodeZeroPoslist(Fts5ExprNode *pNode){ */ static int fts5NodeCompare( Fts5Expr *pExpr, - Fts5ExprNode *p1, + Fts5ExprNode *p1, Fts5ExprNode *p2 ){ if( p2->bEof ) return -1; @@ -181594,7 +239491,7 @@ static int fts5NodeCompare( ** If an EOF is reached before this happens, *pbEof is set to true before ** returning. ** -** SQLITE_OK is returned if an error occurs, or an SQLite error code +** SQLITE_OK is returned if an error occurs, or an SQLite error code ** otherwise. It is not considered an error code if an iterator reaches ** EOF. */ @@ -181611,9 +239508,10 @@ static int fts5ExprNodeTest_STRING( const int bDesc = pExpr->bDesc; /* Check that this node should not be FTS5_TERM */ - assert( pNear->nPhrase>1 - || pNear->apPhrase[0]->nTerm>1 + assert( pNear->nPhrase>1 + || pNear->apPhrase[0]->nTerm>1 || pNear->apPhrase[0]->aTerm[0].pSynonym + || pNear->apPhrase[0]->aTerm[0].bFirst ); /* Initialize iLast, the "lastest" rowid any iterator points to. If the @@ -181671,7 +239569,7 @@ static int fts5ExprNodeNext_STRING( Fts5Expr *pExpr, /* Expression pPhrase belongs to */ Fts5ExprNode *pNode, /* FTS5_STRING or FTS5_TERM node */ int bFromValid, - i64 iFrom + i64 iFrom ){ Fts5ExprTerm *pTerm = &pNode->pNear->apPhrase[0]->aTerm[0]; int rc = SQLITE_OK; @@ -181689,8 +239587,8 @@ static int fts5ExprNodeNext_STRING( for(p=pTerm; p; p=p->pSynonym){ if( sqlite3Fts5IterEof(p->pIter)==0 ){ i64 ii = p->pIter->iRowid; - if( ii==iRowid - || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) + if( ii==iRowid + || (bFromValid && ii!=iFrom && (ii>iFrom)==pExpr->bDesc) ){ if( bFromValid ){ rc = sqlite3Fts5IterNextFrom(p->pIter, iFrom); @@ -181736,9 +239634,9 @@ static int fts5ExprNodeTest_TERM( Fts5Expr *pExpr, /* Expression that pNear is a part of */ Fts5ExprNode *pNode /* The "NEAR" node (FTS5_TERM) */ ){ - /* As this "NEAR" object is actually a single phrase that consists + /* As this "NEAR" object is actually a single phrase that consists ** of a single term only, grab pointers into the poslist managed by the - ** fts5_index.c iterator object. This is much faster than synthesizing + ** fts5_index.c iterator object. This is much faster than synthesizing ** a new poslist the way we have to for more complicated phrase or NEAR ** expressions. */ Fts5ExprPhrase *pPhrase = pNode->pNear->apPhrase[0]; @@ -181761,7 +239659,7 @@ static int fts5ExprNodeTest_TERM( ** xNext() method for a node of type FTS5_TERM. */ static int fts5ExprNodeNext_TERM( - Fts5Expr *pExpr, + Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom @@ -181804,7 +239702,7 @@ static void fts5ExprNodeTest_OR( } static int fts5ExprNodeNext_OR( - Fts5Expr *pExpr, + Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom @@ -181816,11 +239714,14 @@ static int fts5ExprNodeNext_OR( Fts5ExprNode *p1 = pNode->apChild[i]; assert( p1->bEof || fts5RowidCmp(pExpr, p1->iRowid, iLast)>=0 ); if( p1->bEof==0 ){ - if( (p1->iRowid==iLast) + if( (p1->iRowid==iLast) || (bFromValid && fts5RowidCmp(pExpr, p1->iRowid, iFrom)<0) ){ int rc = fts5ExprNodeNext(pExpr, p1, bFromValid, iFrom); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + return rc; + } } } } @@ -181851,7 +239752,10 @@ static int fts5ExprNodeTest_AND( if( cmp>0 ){ /* Advance pChild until it points to iLast or laster */ rc = fts5ExprNodeNext(pExpr, pChild, 1, iLast); - if( rc!=SQLITE_OK ) return rc; + if( rc!=SQLITE_OK ){ + pAnd->bNomatch = 0; + return rc; + } } /* If the child node is now at EOF, so is the parent AND node. Otherwise, @@ -181882,7 +239786,7 @@ static int fts5ExprNodeTest_AND( } static int fts5ExprNodeNext_AND( - Fts5Expr *pExpr, + Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom @@ -181890,6 +239794,8 @@ static int fts5ExprNodeNext_AND( int rc = fts5ExprNodeNext(pExpr, pNode->apChild[0], bFromValid, iFrom); if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_AND(pExpr, pNode); + }else{ + pNode->bNomatch = 0; } return rc; } @@ -181923,7 +239829,7 @@ static int fts5ExprNodeTest_NOT( } static int fts5ExprNodeNext_NOT( - Fts5Expr *pExpr, + Fts5Expr *pExpr, Fts5ExprNode *pNode, int bFromValid, i64 iFrom @@ -181932,6 +239838,9 @@ static int fts5ExprNodeNext_NOT( if( rc==SQLITE_OK ){ rc = fts5ExprNodeTest_NOT(pExpr, pNode); } + if( rc!=SQLITE_OK ){ + pNode->bNomatch = 0; + } return rc; } @@ -181977,7 +239886,7 @@ static int fts5ExprNodeTest( return rc; } - + /* ** Set node pNode, which is part of expression pExpr, to point to the first ** match. If there are no matches, set the Node.bEof flag to indicate EOF. @@ -182031,8 +239940,8 @@ static int fts5ExprNodeFirst(Fts5Expr *pExpr, Fts5ExprNode *pNode){ /* ** Begin iterating through the set of documents in index pIdx matched by -** the MATCH expression passed as the first argument. If the "bDesc" -** parameter is passed a non-zero value, iteration is in descending rowid +** the MATCH expression passed as the first argument. If the "bDesc" +** parameter is passed a non-zero value, iteration is in descending rowid ** order. Or, if it is zero, in ascending order. ** ** If iterating in ascending rowid order (bDesc==0), the first document @@ -182054,20 +239963,23 @@ static int sqlite3Fts5ExprFirst(Fts5Expr *p, Fts5Index *pIdx, i64 iFirst, int bD /* If not at EOF but the current rowid occurs earlier than iFirst in ** the iteration order, move to document iFirst or later. */ - if( pRoot->bEof==0 && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 ){ + if( rc==SQLITE_OK + && 0==pRoot->bEof + && fts5RowidCmp(p, pRoot->iRowid, iFirst)<0 + ){ rc = fts5ExprNodeNext(p, pRoot, 1, iFirst); } /* If the iterator is not at a real match, skip forward until it is. */ - while( pRoot->bNomatch ){ - assert( pRoot->bEof==0 && rc==SQLITE_OK ); + while( pRoot->bNomatch && rc==SQLITE_OK ){ + assert( pRoot->bEof==0 ); rc = fts5ExprNodeNext(p, pRoot, 0, 0); } return rc; } /* -** Move to the next document +** Move to the next document ** ** Return SQLITE_OK if successful, or an SQLite error code otherwise. It ** is not considered an error if the query does not match any documents. @@ -182110,7 +240022,7 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ Fts5ExprTerm *pSyn; Fts5ExprTerm *pNext; Fts5ExprTerm *pTerm = &pPhrase->aTerm[i]; - sqlite3_free(pTerm->zTerm); + sqlite3_free(pTerm->pTerm); sqlite3Fts5IterClose(pTerm->pIter); for(pSyn=pTerm->pSynonym; pSyn; pSyn=pNext){ pNext = pSyn->pSynonym; @@ -182124,6 +240036,16 @@ static void fts5ExprPhraseFree(Fts5ExprPhrase *pPhrase){ } } +/* +** Set the "bFirst" flag on the first token of the phrase passed as the +** only argument. +*/ +static void sqlite3Fts5ParseSetCaret(Fts5ExprPhrase *pPhrase){ + if( pPhrase && pPhrase->nTerm ){ + pPhrase->aTerm[0].bFirst = 1; + } +} + /* ** If argument pNear is NULL, then a new Fts5ExprNearset object is allocated ** and populated with pPhrase. Or, if pNear is not NULL, phrase pPhrase is @@ -182141,22 +240063,21 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( Fts5ExprNearset *pRet = 0; if( pParse->rc==SQLITE_OK ){ - if( pPhrase==0 ){ - return pNear; - } if( pNear==0 ){ - int nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); - pRet = sqlite3_malloc(nByte); + sqlite3_int64 nByte; + nByte = sizeof(Fts5ExprNearset) + SZALLOC * sizeof(Fts5ExprPhrase*); + pRet = sqlite3_malloc64(nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; }else{ - memset(pRet, 0, nByte); + memset(pRet, 0, (size_t)nByte); } }else if( (pNear->nPhrase % SZALLOC)==0 ){ int nNew = pNear->nPhrase + SZALLOC; - int nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + sqlite3_int64 nByte; - pRet = (Fts5ExprNearset*)sqlite3_realloc(pNear, nByte); + nByte = sizeof(Fts5ExprNearset) + nNew * sizeof(Fts5ExprPhrase*); + pRet = (Fts5ExprNearset*)sqlite3_realloc64(pNear, nByte); if( pRet==0 ){ pParse->rc = SQLITE_NOMEM; } @@ -182172,6 +240093,9 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( }else{ if( pRet->nPhrase>0 ){ Fts5ExprPhrase *pLast = pRet->apPhrase[pRet->nPhrase-1]; + assert( pParse!=0 ); + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>=2 ); assert( pLast==pParse->apPhrase[pParse->nPhrase-2] ); if( pPhrase->nTerm==0 ){ fts5ExprPhraseFree(pPhrase); @@ -182193,6 +240117,7 @@ static Fts5ExprNearset *sqlite3Fts5ParseNearset( typedef struct TokenCtx TokenCtx; struct TokenCtx { Fts5ExprPhrase *pPhrase; + Fts5Config *pConfig; int rc; }; @@ -182220,14 +240145,18 @@ static int fts5ParseTokenize( if( pPhrase && pPhrase->nTerm>0 && (tflags & FTS5_TOKEN_COLOCATED) ){ Fts5ExprTerm *pSyn; - int nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; - pSyn = (Fts5ExprTerm*)sqlite3_malloc(nByte); + sqlite3_int64 nByte = sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer) + nToken+1; + pSyn = (Fts5ExprTerm*)sqlite3_malloc64(nByte); if( pSyn==0 ){ rc = SQLITE_NOMEM; }else{ - memset(pSyn, 0, nByte); - pSyn->zTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); - memcpy(pSyn->zTerm, pToken, nToken); + memset(pSyn, 0, (size_t)nByte); + pSyn->pTerm = ((char*)pSyn) + sizeof(Fts5ExprTerm) + sizeof(Fts5Buffer); + pSyn->nFullTerm = pSyn->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata ){ + pSyn->nQueryTerm = (int)strlen(pSyn->pTerm); + } + memcpy(pSyn->pTerm, pToken, nToken); pSyn->pSynonym = pPhrase->aTerm[pPhrase->nTerm-1].pSynonym; pPhrase->aTerm[pPhrase->nTerm-1].pSynonym = pSyn; } @@ -182237,7 +240166,7 @@ static int fts5ParseTokenize( Fts5ExprPhrase *pNew; int nNew = SZALLOC + (pPhrase ? pPhrase->nTerm : 0); - pNew = (Fts5ExprPhrase*)sqlite3_realloc(pPhrase, + pNew = (Fts5ExprPhrase*)sqlite3_realloc64(pPhrase, sizeof(Fts5ExprPhrase) + sizeof(Fts5ExprTerm) * nNew ); if( pNew==0 ){ @@ -182252,7 +240181,11 @@ static int fts5ParseTokenize( if( rc==SQLITE_OK ){ pTerm = &pPhrase->aTerm[pPhrase->nTerm++]; memset(pTerm, 0, sizeof(Fts5ExprTerm)); - pTerm->zTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->pTerm = sqlite3Fts5Strndup(&rc, pToken, nToken); + pTerm->nFullTerm = pTerm->nQueryTerm = nToken; + if( pCtx->pConfig->bTokendata && rc==SQLITE_OK ){ + pTerm->nQueryTerm = (int)strlen(pTerm->pTerm); + } } } @@ -182287,6 +240220,20 @@ static void sqlite3Fts5ParseFinished(Fts5Parse *pParse, Fts5ExprNode *p){ pParse->pExpr = p; } +static int parseGrowPhraseArray(Fts5Parse *pParse){ + if( (pParse->nPhrase % 8)==0 ){ + sqlite3_int64 nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); + Fts5ExprPhrase **apNew; + apNew = (Fts5ExprPhrase**)sqlite3_realloc64(pParse->apPhrase, nByte); + if( apNew==0 ){ + pParse->rc = SQLITE_NOMEM; + return SQLITE_NOMEM; + } + pParse->apPhrase = apNew; + } + return SQLITE_OK; +} + /* ** This function is called by the parser to process a string token. The ** string may or may not be quoted. In any case it is tokenized and a @@ -182305,10 +240252,11 @@ static Fts5ExprPhrase *sqlite3Fts5ParseTerm( memset(&sCtx, 0, sizeof(TokenCtx)); sCtx.pPhrase = pAppend; + sCtx.pConfig = pConfig; rc = fts5ParseStringFromToken(pToken, &z); if( rc==SQLITE_OK ){ - int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_QUERY : 0); + int flags = FTS5_TOKENIZE_QUERY | (bPrefix ? FTS5_TOKENIZE_PREFIX : 0); int n; sqlite3Fts5Dequote(z); n = (int)strlen(z); @@ -182322,16 +240270,9 @@ static Fts5ExprPhrase *sqlite3Fts5ParseTerm( }else{ if( pAppend==0 ){ - if( (pParse->nPhrase % 8)==0 ){ - int nByte = sizeof(Fts5ExprPhrase*) * (pParse->nPhrase + 8); - Fts5ExprPhrase **apNew; - apNew = (Fts5ExprPhrase**)sqlite3_realloc(pParse->apPhrase, nByte); - if( apNew==0 ){ - pParse->rc = SQLITE_NOMEM; - fts5ExprPhraseFree(sCtx.pPhrase); - return 0; - } - pParse->apPhrase = apNew; + if( parseGrowPhraseArray(pParse) ){ + fts5ExprPhraseFree(sCtx.pPhrase); + return 0; } pParse->nPhrase++; } @@ -182341,8 +240282,9 @@ static Fts5ExprPhrase *sqlite3Fts5ParseTerm( ** no token characters at all. (e.g ... MATCH '""'). */ sCtx.pPhrase = sqlite3Fts5MallocZero(&pParse->rc, sizeof(Fts5ExprPhrase)); }else if( sCtx.pPhrase->nTerm ){ - sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = bPrefix; + sCtx.pPhrase->aTerm[sCtx.pPhrase->nTerm-1].bPrefix = (u8)bPrefix; } + assert( pParse->apPhrase!=0 ); pParse->apPhrase[pParse->nPhrase-1] = sCtx.pPhrase; } @@ -182354,57 +240296,70 @@ static Fts5ExprPhrase *sqlite3Fts5ParseTerm( ** expression passed as the second argument. */ static int sqlite3Fts5ExprClonePhrase( - Fts5Expr *pExpr, - int iPhrase, + Fts5Expr *pExpr, + int iPhrase, Fts5Expr **ppNew ){ int rc = SQLITE_OK; /* Return code */ - Fts5ExprPhrase *pOrig; /* The phrase extracted from pExpr */ - int i; /* Used to iterate through phrase terms */ + Fts5ExprPhrase *pOrig = 0; /* The phrase extracted from pExpr */ Fts5Expr *pNew = 0; /* Expression to return via *ppNew */ - TokenCtx sCtx = {0,0}; /* Context object for fts5ParseTokenize */ - - pOrig = pExpr->apExprPhrase[iPhrase]; - pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + TokenCtx sCtx = {0,0,0}; /* Context object for fts5ParseTokenize */ + if( !pExpr || iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + rc = SQLITE_RANGE; + }else{ + pOrig = pExpr->apExprPhrase[iPhrase]; + pNew = (Fts5Expr*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Expr)); + } if( rc==SQLITE_OK ){ - pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, + pNew->apExprPhrase = (Fts5ExprPhrase**)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase*)); } if( rc==SQLITE_OK ){ - pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, + pNew->pRoot = (Fts5ExprNode*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNode)); } if( rc==SQLITE_OK ){ - pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, + pNew->pRoot->pNear = (Fts5ExprNearset*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprNearset) + sizeof(Fts5ExprPhrase*)); } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(pOrig!=0) ){ Fts5Colset *pColsetOrig = pOrig->pNode->pNear->pColset; if( pColsetOrig ){ - int nByte = sizeof(Fts5Colset) + pColsetOrig->nCol * sizeof(int); - Fts5Colset *pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); - if( pColset ){ - memcpy(pColset, pColsetOrig, nByte); + sqlite3_int64 nByte; + Fts5Colset *pColset; + nByte = sizeof(Fts5Colset) + (pColsetOrig->nCol-1) * sizeof(int); + pColset = (Fts5Colset*)sqlite3Fts5MallocZero(&rc, nByte); + if( pColset ){ + memcpy(pColset, pColsetOrig, (size_t)nByte); } pNew->pRoot->pNear->pColset = pColset; } } - for(i=0; rc==SQLITE_OK && inTerm; i++){ - int tflags = 0; - Fts5ExprTerm *p; - for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ - const char *zTerm = p->zTerm; - rc = fts5ParseTokenize((void*)&sCtx, tflags, zTerm, (int)strlen(zTerm), - 0, 0); - tflags = FTS5_TOKEN_COLOCATED; - } - if( rc==SQLITE_OK ){ - sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + if( rc==SQLITE_OK ){ + if( pOrig->nTerm ){ + int i; /* Used to iterate through phrase terms */ + sCtx.pConfig = pExpr->pConfig; + for(i=0; rc==SQLITE_OK && inTerm; i++){ + int tflags = 0; + Fts5ExprTerm *p; + for(p=&pOrig->aTerm[i]; p && rc==SQLITE_OK; p=p->pSynonym){ + rc = fts5ParseTokenize((void*)&sCtx,tflags,p->pTerm,p->nFullTerm,0,0); + tflags = FTS5_TOKEN_COLOCATED; + } + if( rc==SQLITE_OK ){ + sCtx.pPhrase->aTerm[i].bPrefix = pOrig->aTerm[i].bPrefix; + sCtx.pPhrase->aTerm[i].bFirst = pOrig->aTerm[i].bFirst; + } + } + }else{ + /* This happens when parsing a token or quoted phrase that contains + ** no token characters at all. (e.g ... MATCH '""'). */ + sCtx.pPhrase = sqlite3Fts5MallocZero(&rc, sizeof(Fts5ExprPhrase)); } } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && ALWAYS(sCtx.pPhrase) ){ /* All the allocations succeeded. Put the expression object together. */ pNew->pIndex = pExpr->pIndex; pNew->pConfig = pExpr->pConfig; @@ -182414,7 +240369,10 @@ static int sqlite3Fts5ExprClonePhrase( pNew->pRoot->pNear->nPhrase = 1; sCtx.pPhrase->pNode = pNew->pRoot; - if( pOrig->nTerm==1 && pOrig->aTerm[0].pSynonym==0 ){ + if( pOrig->nTerm==1 + && pOrig->aTerm[0].pSynonym==0 + && pOrig->aTerm[0].bFirst==0 + ){ pNew->pRoot->eType = FTS5_TERM; pNew->pRoot->xNext = fts5ExprNodeNext_TERM; }else{ @@ -182446,7 +240404,7 @@ static void sqlite3Fts5ParseNear(Fts5Parse *pParse, Fts5Token *pTok){ } static void sqlite3Fts5ParseSetDistance( - Fts5Parse *pParse, + Fts5Parse *pParse, Fts5ExprNearset *pNear, Fts5Token *p ){ @@ -182475,7 +240433,7 @@ static void sqlite3Fts5ParseSetDistance( ** The second argument passed to this function may be NULL, or it may be ** an existing Fts5Colset object. This function returns a pointer to ** a new colset object containing the contents of (p) with new value column -** number iCol appended. +** number iCol appended. ** ** If an OOM error occurs, store an error code in pParse and return NULL. ** The old colset object (if any) is not freed in this case. @@ -182491,7 +240449,7 @@ static Fts5Colset *fts5ParseColset( assert( pParse->rc==SQLITE_OK ); assert( iCol>=0 && iColpConfig->nCol ); - pNew = sqlite3_realloc(p, sizeof(Fts5Colset) + sizeof(int)*nCol); + pNew = sqlite3_realloc64(p, sizeof(Fts5Colset) + sizeof(int)*nCol); if( pNew==0 ){ pParse->rc = SQLITE_NOMEM; }else{ @@ -182516,6 +240474,34 @@ static Fts5Colset *fts5ParseColset( return pNew; } +/* +** Allocate and return an Fts5Colset object specifying the inverse of +** the colset passed as the second argument. Free the colset passed +** as the second argument before returning. +*/ +static Fts5Colset *sqlite3Fts5ParseColsetInvert(Fts5Parse *pParse, Fts5Colset *p){ + Fts5Colset *pRet; + int nCol = pParse->pConfig->nCol; + + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(&pParse->rc, + sizeof(Fts5Colset) + sizeof(int)*nCol + ); + if( pRet ){ + int i; + int iOld = 0; + for(i=0; i=p->nCol || p->aiCol[iOld]!=i ){ + pRet->aiCol[pRet->nCol++] = i; + }else{ + iOld++; + } + } + } + + sqlite3_free(p); + return pRet; +} + static Fts5Colset *sqlite3Fts5ParseColset( Fts5Parse *pParse, /* Store SQLITE_NOMEM here if required */ Fts5Colset *pColset, /* Existing colset object */ @@ -182548,33 +240534,118 @@ static Fts5Colset *sqlite3Fts5ParseColset( return pRet; } -static void sqlite3Fts5ParseSetColset( - Fts5Parse *pParse, - Fts5ExprNearset *pNear, - Fts5Colset *pColset -){ - if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ - pParse->rc = SQLITE_ERROR; - pParse->zErr = sqlite3_mprintf( - "fts5: column queries are not supported (detail=none)" - ); - sqlite3_free(pColset); - return; - } - - if( pNear ){ - pNear->pColset = pColset; +/* +** If argument pOrig is NULL, or if (*pRc) is set to anything other than +** SQLITE_OK when this function is called, NULL is returned. +** +** Otherwise, a copy of (*pOrig) is made into memory obtained from +** sqlite3Fts5MallocZero() and a pointer to it returned. If the allocation +** fails, (*pRc) is set to SQLITE_NOMEM and NULL is returned. +*/ +static Fts5Colset *fts5CloneColset(int *pRc, Fts5Colset *pOrig){ + Fts5Colset *pRet; + if( pOrig ){ + sqlite3_int64 nByte = sizeof(Fts5Colset) + (pOrig->nCol-1) * sizeof(int); + pRet = (Fts5Colset*)sqlite3Fts5MallocZero(pRc, nByte); + if( pRet ){ + memcpy(pRet, pOrig, (size_t)nByte); + } }else{ - sqlite3_free(pColset); + pRet = 0; } + return pRet; +} + +/* +** Remove from colset pColset any columns that are not also in colset pMerge. +*/ +static void fts5MergeColset(Fts5Colset *pColset, Fts5Colset *pMerge){ + int iIn = 0; /* Next input in pColset */ + int iMerge = 0; /* Next input in pMerge */ + int iOut = 0; /* Next output slot in pColset */ + + while( iInnCol && iMergenCol ){ + int iDiff = pColset->aiCol[iIn] - pMerge->aiCol[iMerge]; + if( iDiff==0 ){ + pColset->aiCol[iOut++] = pMerge->aiCol[iMerge]; + iMerge++; + iIn++; + }else if( iDiff>0 ){ + iMerge++; + }else{ + iIn++; + } + } + pColset->nCol = iOut; +} + +/* +** Recursively apply colset pColset to expression node pNode and all of +** its decendents. If (*ppFree) is not NULL, it contains a spare copy +** of pColset. This function may use the spare copy and set (*ppFree) to +** zero, or it may create copies of pColset using fts5CloneColset(). +*/ +static void fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pNode, + Fts5Colset *pColset, + Fts5Colset **ppFree +){ + if( pParse->rc==SQLITE_OK ){ + assert( pNode->eType==FTS5_TERM || pNode->eType==FTS5_STRING + || pNode->eType==FTS5_AND || pNode->eType==FTS5_OR + || pNode->eType==FTS5_NOT || pNode->eType==FTS5_EOF + ); + if( pNode->eType==FTS5_STRING || pNode->eType==FTS5_TERM ){ + Fts5ExprNearset *pNear = pNode->pNear; + if( pNear->pColset ){ + fts5MergeColset(pNear->pColset, pColset); + if( pNear->pColset->nCol==0 ){ + pNode->eType = FTS5_EOF; + pNode->xNext = 0; + } + }else if( *ppFree ){ + pNear->pColset = pColset; + *ppFree = 0; + }else{ + pNear->pColset = fts5CloneColset(&pParse->rc, pColset); + } + }else{ + int i; + assert( pNode->eType!=FTS5_EOF || pNode->nChild==0 ); + for(i=0; inChild; i++){ + fts5ParseSetColset(pParse, pNode->apChild[i], pColset, ppFree); + } + } + } +} + +/* +** Apply colset pColset to expression node pExpr and all of its descendents. +*/ +static void sqlite3Fts5ParseSetColset( + Fts5Parse *pParse, + Fts5ExprNode *pExpr, + Fts5Colset *pColset +){ + Fts5Colset *pFree = pColset; + if( pParse->pConfig->eDetail==FTS5_DETAIL_NONE ){ + sqlite3Fts5ParseError(pParse, + "fts5: column queries are not supported (detail=none)" + ); + }else{ + fts5ParseSetColset(pParse, pExpr, pColset, &pFree); + } + sqlite3_free(pFree); } static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ switch( pNode->eType ){ case FTS5_STRING: { Fts5ExprNearset *pNear = pNode->pNear; - if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 + if( pNear->nPhrase==1 && pNear->apPhrase[0]->nTerm==1 && pNear->apPhrase[0]->aTerm[0].pSynonym==0 + && pNear->apPhrase[0]->aTerm[0].bFirst==0 ){ pNode->eType = FTS5_TERM; pNode->xNext = fts5ExprNodeNext_TERM; @@ -182601,7 +240672,11 @@ static void fts5ExprAssignXNext(Fts5ExprNode *pNode){ } } +/* +** Add pSub as a child of p. +*/ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ + int ii = p->nChild; if( p->eType!=FTS5_NOT && pSub->eType==p->eType ){ int nByte = sizeof(Fts5ExprNode*) * pSub->nChild; memcpy(&p->apChild[p->nChild], pSub->apChild, nByte); @@ -182610,6 +240685,73 @@ static void fts5ExprAddChildren(Fts5ExprNode *p, Fts5ExprNode *pSub){ }else{ p->apChild[p->nChild++] = pSub; } + for( ; iinChild; ii++){ + p->iHeight = MAX(p->iHeight, p->apChild[ii]->iHeight + 1); + } +} + +/* +** This function is used when parsing LIKE or GLOB patterns against +** trigram indexes that specify either detail=column or detail=none. +** It converts a phrase: +** +** abc + def + ghi +** +** into an AND tree: +** +** abc AND def AND ghi +*/ +static Fts5ExprNode *fts5ParsePhraseToAnd( + Fts5Parse *pParse, + Fts5ExprNearset *pNear +){ + int nTerm = pNear->apPhrase[0]->nTerm; + int ii; + int nByte; + Fts5ExprNode *pRet; + + assert( pNear->nPhrase==1 ); + assert( pParse->bPhraseToAnd ); + + nByte = sizeof(Fts5ExprNode) + nTerm*sizeof(Fts5ExprNode*); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + if( pRet ){ + pRet->eType = FTS5_AND; + pRet->nChild = nTerm; + pRet->iHeight = 1; + fts5ExprAssignXNext(pRet); + pParse->nPhrase--; + for(ii=0; iirc, sizeof(Fts5ExprPhrase) + ); + if( pPhrase ){ + if( parseGrowPhraseArray(pParse) ){ + fts5ExprPhraseFree(pPhrase); + }else{ + Fts5ExprTerm *p = &pNear->apPhrase[0]->aTerm[ii]; + Fts5ExprTerm *pTo = &pPhrase->aTerm[0]; + pParse->apPhrase[pParse->nPhrase++] = pPhrase; + pPhrase->nTerm = 1; + pTo->pTerm = sqlite3Fts5Strndup(&pParse->rc, p->pTerm, p->nFullTerm); + pTo->nQueryTerm = p->nQueryTerm; + pTo->nFullTerm = p->nFullTerm; + pRet->apChild[ii] = sqlite3Fts5ParseNode(pParse, FTS5_STRING, + 0, 0, sqlite3Fts5ParseNearset(pParse, 0, pPhrase) + ); + } + } + } + + if( pParse->rc ){ + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; + }else{ + sqlite3Fts5ParseNearsetFree(pNear); + } + } + + return pRet; } /* @@ -182627,8 +240769,8 @@ static Fts5ExprNode *sqlite3Fts5ParseNode( if( pParse->rc==SQLITE_OK ){ int nChild = 0; /* Number of children of returned node */ - int nByte; /* Bytes of space to allocate for this node */ - + sqlite3_int64 nByte; /* Bytes of space to allocate for this node */ + assert( (eType!=FTS5_STRING && !pNear) || (eType==FTS5_STRING && !pLeft && !pRight) ); @@ -182636,48 +240778,67 @@ static Fts5ExprNode *sqlite3Fts5ParseNode( if( eType!=FTS5_STRING && pLeft==0 ) return pRight; if( eType!=FTS5_STRING && pRight==0 ) return pLeft; - if( eType==FTS5_NOT ){ - nChild = 2; - }else if( eType==FTS5_AND || eType==FTS5_OR ){ - nChild = 2; - if( pLeft->eType==eType ) nChild += pLeft->nChild-1; - if( pRight->eType==eType ) nChild += pRight->nChild-1; - } + if( eType==FTS5_STRING + && pParse->bPhraseToAnd + && pNear->apPhrase[0]->nTerm>1 + ){ + pRet = fts5ParsePhraseToAnd(pParse, pNear); + }else{ + if( eType==FTS5_NOT ){ + nChild = 2; + }else if( eType==FTS5_AND || eType==FTS5_OR ){ + nChild = 2; + if( pLeft->eType==eType ) nChild += pLeft->nChild-1; + if( pRight->eType==eType ) nChild += pRight->nChild-1; + } - nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); - pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); + nByte = sizeof(Fts5ExprNode) + sizeof(Fts5ExprNode*)*(nChild-1); + pRet = (Fts5ExprNode*)sqlite3Fts5MallocZero(&pParse->rc, nByte); - if( pRet ){ - pRet->eType = eType; - pRet->pNear = pNear; - fts5ExprAssignXNext(pRet); - if( eType==FTS5_STRING ){ - int iPhrase; - for(iPhrase=0; iPhrasenPhrase; iPhrase++){ - pNear->apPhrase[iPhrase]->pNode = pRet; - if( pNear->apPhrase[iPhrase]->nTerm==0 ){ - pRet->xNext = 0; - pRet->eType = FTS5_EOF; + if( pRet ){ + pRet->eType = eType; + pRet->pNear = pNear; + fts5ExprAssignXNext(pRet); + if( eType==FTS5_STRING ){ + int iPhrase; + for(iPhrase=0; iPhrasenPhrase; iPhrase++){ + pNear->apPhrase[iPhrase]->pNode = pRet; + if( pNear->apPhrase[iPhrase]->nTerm==0 ){ + pRet->xNext = 0; + pRet->eType = FTS5_EOF; + } + } + + if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL ){ + Fts5ExprPhrase *pPhrase = pNear->apPhrase[0]; + if( pNear->nPhrase!=1 + || pPhrase->nTerm>1 + || (pPhrase->nTerm>0 && pPhrase->aTerm[0].bFirst) + ){ + sqlite3Fts5ParseError(pParse, + "fts5: %s queries are not supported (detail!=full)", + pNear->nPhrase==1 ? "phrase": "NEAR" + ); + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; + pNear = 0; + assert( pLeft==0 && pRight==0 ); + } + } + }else{ + assert( pNear==0 ); + fts5ExprAddChildren(pRet, pLeft); + fts5ExprAddChildren(pRet, pRight); + pLeft = pRight = 0; + if( pRet->iHeight>SQLITE_FTS5_MAX_EXPR_DEPTH ){ + sqlite3Fts5ParseError(pParse, + "fts5 expression tree is too large (maximum depth %d)", + SQLITE_FTS5_MAX_EXPR_DEPTH + ); + sqlite3Fts5ParseNodeFree(pRet); + pRet = 0; } } - - if( pParse->pConfig->eDetail!=FTS5_DETAIL_FULL - && (pNear->nPhrase!=1 || pNear->apPhrase[0]->nTerm>1) - ){ - assert( pParse->rc==SQLITE_OK ); - pParse->rc = SQLITE_ERROR; - assert( pParse->zErr==0 ); - pParse->zErr = sqlite3_mprintf( - "fts5: %s queries are not supported (detail!=full)", - pNear->nPhrase==1 ? "phrase": "NEAR" - ); - sqlite3_free(pRet); - pRet = 0; - } - - }else{ - fts5ExprAddChildren(pRet, pLeft); - fts5ExprAddChildren(pRet, pRight); } } } @@ -182704,14 +240865,15 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( sqlite3Fts5ParseNodeFree(pRight); }else{ - assert( pLeft->eType==FTS5_STRING + assert( pLeft->eType==FTS5_STRING || pLeft->eType==FTS5_TERM || pLeft->eType==FTS5_EOF || pLeft->eType==FTS5_AND ); - assert( pRight->eType==FTS5_STRING - || pRight->eType==FTS5_TERM - || pRight->eType==FTS5_EOF + assert( pRight->eType==FTS5_STRING + || pRight->eType==FTS5_TERM + || pRight->eType==FTS5_EOF + || (pRight->eType==FTS5_AND && pParse->bPhraseToAnd) ); if( pLeft->eType==FTS5_AND ){ @@ -182719,12 +240881,14 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( }else{ pPrev = pLeft; } - assert( pPrev->eType==FTS5_STRING - || pPrev->eType==FTS5_TERM - || pPrev->eType==FTS5_EOF + assert( pPrev->eType==FTS5_STRING + || pPrev->eType==FTS5_TERM + || pPrev->eType==FTS5_EOF ); if( pRight->eType==FTS5_EOF ){ + assert( pParse->apPhrase!=0 ); + assert( pParse->nPhrase>0 ); assert( pParse->apPhrase[pParse->nPhrase-1]==pRight->pNear->apPhrase[0] ); sqlite3Fts5ParseNodeFree(pRight); pRet = pLeft; @@ -182755,23 +240919,25 @@ static Fts5ExprNode *sqlite3Fts5ParseImplicitAnd( return pRet; } +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static char *fts5ExprTermPrint(Fts5ExprTerm *pTerm){ - int nByte = 0; + sqlite3_int64 nByte = 0; Fts5ExprTerm *p; char *zQuoted; /* Determine the maximum amount of space required. */ for(p=pTerm; p; p=p->pSynonym){ - nByte += (int)strlen(pTerm->zTerm) * 2 + 3 + 2; + nByte += pTerm->nQueryTerm * 2 + 3 + 2; } - zQuoted = sqlite3_malloc(nByte); + zQuoted = sqlite3_malloc64(nByte); if( zQuoted ){ int i = 0; for(p=pTerm; p; p=p->pSynonym){ - char *zIn = p->zTerm; + char *zIn = p->pTerm; + char *zEnd = &zIn[p->nQueryTerm]; zQuoted[i++] = '"'; - while( *zIn ){ + while( zIneType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ Fts5ExprNearset *pNear = pExpr->pNear; - int i; + int i; int iTerm; zRet = fts5PrintfAppend(zRet, "%s ", zNearsetCmd); @@ -182849,8 +241015,10 @@ static char *fts5ExprPrintTcl( zRet = fts5PrintfAppend(zRet, " {"); for(iTerm=0; zRet && iTermnTerm; iTerm++){ - char *zTerm = pPhrase->aTerm[iTerm].zTerm; - zRet = fts5PrintfAppend(zRet, "%s%s", iTerm==0?"":" ", zTerm); + Fts5ExprTerm *p = &pPhrase->aTerm[iTerm]; + zRet = fts5PrintfAppend(zRet, "%s%.*s", iTerm==0?"":" ", + p->nQueryTerm, p->pTerm + ); if( pPhrase->aTerm[iTerm].bPrefix ){ zRet = fts5PrintfAppend(zRet, "*"); } @@ -182860,15 +241028,17 @@ static char *fts5ExprPrintTcl( if( zRet==0 ) return 0; } + }else if( pExpr->eType==0 ){ + zRet = sqlite3_mprintf("{}"); }else{ char const *zOp = 0; int i; switch( pExpr->eType ){ case FTS5_AND: zOp = "AND"; break; case FTS5_NOT: zOp = "NOT"; break; - default: + default: assert( pExpr->eType==FTS5_OR ); - zOp = "OR"; + zOp = "OR"; break; } @@ -182894,12 +241064,21 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ }else if( pExpr->eType==FTS5_STRING || pExpr->eType==FTS5_TERM ){ Fts5ExprNearset *pNear = pExpr->pNear; - int i; + int i; int iTerm; if( pNear->pColset ){ - int iCol = pNear->pColset->aiCol[0]; - zRet = fts5PrintfAppend(zRet, "%s : ", pConfig->azCol[iCol]); + int ii; + Fts5Colset *pColset = pNear->pColset; + if( pColset->nCol>1 ) zRet = fts5PrintfAppend(zRet, "{"); + for(ii=0; iinCol; ii++){ + zRet = fts5PrintfAppend(zRet, "%s%s", + pConfig->azCol[pColset->aiCol[ii]], ii==pColset->nCol-1 ? "" : " " + ); + } + if( zRet ){ + zRet = fts5PrintfAppend(zRet, "%s : ", pColset->nCol>1 ? "}" : ""); + } if( zRet==0 ) return 0; } @@ -182939,9 +241118,9 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ switch( pExpr->eType ){ case FTS5_AND: zOp = " AND "; break; case FTS5_NOT: zOp = " NOT "; break; - default: + default: assert( pExpr->eType==FTS5_OR ); - zOp = " OR "; + zOp = " OR "; break; } @@ -182953,7 +241132,7 @@ static char *fts5ExprPrint(Fts5Config *pConfig, Fts5ExprNode *pExpr){ }else{ int e = pExpr->apChild[i]->eType; int b = (e!=FTS5_STRING && e!=FTS5_TERM && e!=FTS5_EOF); - zRet = fts5PrintfAppend(zRet, "%s%s%z%s", + zRet = fts5PrintfAppend(zRet, "%s%s%z%s", (i==0 ? "" : zOp), (b?"(":""), z, (b?")":"") ); @@ -183004,7 +241183,7 @@ static void fts5ExprFunction( } nConfig = 3 + (nArg-iArg); - azConfig = (const char**)sqlite3_malloc(sizeof(char*) * nConfig); + azConfig = (const char**)sqlite3_malloc64(sizeof(char*) * nConfig); if( azConfig==0 ){ sqlite3_result_error_nomem(pCtx); return; @@ -183013,14 +241192,16 @@ static void fts5ExprFunction( azConfig[1] = "main"; azConfig[2] = "tbl"; for(i=3; iArgnCol, zExpr, &pExpr, &zErr); } if( rc==SQLITE_OK ){ char *zText; @@ -183069,7 +241250,7 @@ static void fts5ExprFunctionTcl( /* ** The implementation of an SQLite user-defined-function that accepts a -** single integer as an argument. If the integer is an alpha-numeric +** single integer as an argument. If the integer is an alpha-numeric ** unicode code point, 1 is returned. Otherwise 0. */ static void fts5ExprIsAlnum( @@ -183078,14 +241259,19 @@ static void fts5ExprIsAlnum( sqlite3_value **apVal /* Function arguments */ ){ int iCode; + u8 aArr[32]; if( nArg!=1 ){ - sqlite3_result_error(pCtx, + sqlite3_result_error(pCtx, "wrong number of arguments to function fts5_isalnum", -1 ); return; } + memset(aArr, 0, sizeof(aArr)); + sqlite3Fts5UnicodeCatParse("L*", aArr); + sqlite3Fts5UnicodeCatParse("N*", aArr); + sqlite3Fts5UnicodeCatParse("Co", aArr); iCode = sqlite3_value_int(apVal[0]); - sqlite3_result_int(pCtx, sqlite3Fts5UnicodeIsalnum(iCode)); + sqlite3_result_int(pCtx, aArr[sqlite3Fts5UnicodeCategory((u32)iCode)]); } static void fts5ExprFold( @@ -183094,7 +241280,7 @@ static void fts5ExprFold( sqlite3_value **apVal /* Function arguments */ ){ if( nArg!=1 && nArg!=2 ){ - sqlite3_result_error(pCtx, + sqlite3_result_error(pCtx, "wrong number of arguments to function fts5_fold", -1 ); }else{ @@ -183105,12 +241291,14 @@ static void fts5ExprFold( sqlite3_result_int(pCtx, sqlite3Fts5UnicodeFold(iCode, bRemoveDiacritics)); } } +#endif /* if SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called during initialization to register the fts5_expr() scalar ** UDF with the SQLite handle passed as the only argument. */ static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) struct Fts5ExprFunc { const char *z; void (*x)(sqlite3_context*,int,sqlite3_value**); @@ -183128,11 +241316,17 @@ static int sqlite3Fts5ExprInit(Fts5Global *pGlobal, sqlite3 *db){ struct Fts5ExprFunc *p = &aFunc[i]; rc = sqlite3_create_function(db, p->z, -1, SQLITE_UTF8, pCtx, p->x, 0, 0); } +#else + int rc = SQLITE_OK; + UNUSED_PARAM2(pGlobal,db); +#endif - /* Avoid a warning indicating that sqlite3Fts5ParserTrace() is unused */ + /* Avoid warnings indicating that sqlite3Fts5ParserTrace() and + ** sqlite3Fts5ParserFallback() are unused */ #ifndef NDEBUG (void)sqlite3Fts5ParserTrace; #endif + (void)sqlite3Fts5ParserFallback; return rc; } @@ -183176,17 +241370,26 @@ struct Fts5PoslistPopulator { int bMiss; }; +/* +** Clear the position lists associated with all phrases in the expression +** passed as the first argument. Argument bLive is true if the expression +** might be pointing to a real entry, otherwise it has just been reset. +** +** At present this function is only used for detail=col and detail=none +** fts5 tables. This implies that all phrases must be at most 1 token +** in size, as phrase matches are not supported without detail=full. +*/ static Fts5PoslistPopulator *sqlite3Fts5ExprClearPoslists(Fts5Expr *pExpr, int bLive){ Fts5PoslistPopulator *pRet; - pRet = sqlite3_malloc(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); + pRet = sqlite3_malloc64(sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); if( pRet ){ int i; memset(pRet, 0, sizeof(Fts5PoslistPopulator)*pExpr->nPhrase); for(i=0; inPhrase; i++){ Fts5Buffer *pBuf = &pExpr->apExprPhrase[i]->poslist; Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; - assert( pExpr->apExprPhrase[i]->nTerm==1 ); - if( bLive && + assert( pExpr->apExprPhrase[i]->nTerm<=1 ); + if( bLive && (pBuf->n==0 || pNode->iRowid!=pExpr->pRoot->iRowid || pNode->bEof) ){ pRet[i].bMiss = 1; @@ -183216,6 +241419,17 @@ static int fts5ExprColsetTest(Fts5Colset *pColset, int iCol){ return 0; } +/* +** pToken is a buffer nToken bytes in size that may or may not contain +** an embedded 0x00 byte. If it does, return the number of bytes in +** the buffer before the 0x00. If it does not, return nToken. +*/ +static int fts5QueryTerm(const char *pToken, int nToken){ + int ii; + for(ii=0; iipExpr; int i; + int nQuery = nToken; + i64 iRowid = pExpr->pRoot->iRowid; UNUSED_PARAM2(iUnused1, iUnused2); - if( nToken>FTS5_MAX_TOKEN_SIZE ) nToken = FTS5_MAX_TOKEN_SIZE; + if( nQuery>FTS5_MAX_TOKEN_SIZE ) nQuery = FTS5_MAX_TOKEN_SIZE; + if( pExpr->pConfig->bTokendata ){ + nQuery = fts5QueryTerm(pToken, nQuery); + } if( (tflags & FTS5_TOKEN_COLOCATED)==0 ) p->iOff++; for(i=0; inPhrase; i++){ - Fts5ExprTerm *pTerm; + Fts5ExprTerm *pT; if( p->aPopulator[i].bOk==0 ) continue; - for(pTerm=&pExpr->apExprPhrase[i]->aTerm[0]; pTerm; pTerm=pTerm->pSynonym){ - int nTerm = (int)strlen(pTerm->zTerm); - if( (nTerm==nToken || (nTermbPrefix)) - && memcmp(pTerm->zTerm, pToken, nTerm)==0 + for(pT=&pExpr->apExprPhrase[i]->aTerm[0]; pT; pT=pT->pSynonym){ + if( (pT->nQueryTerm==nQuery || (pT->nQueryTermbPrefix)) + && memcmp(pT->pTerm, pToken, pT->nQueryTerm)==0 ){ int rc = sqlite3Fts5PoslistWriterAppend( &pExpr->apExprPhrase[i]->poslist, &p->aPopulator[i].writer, p->iOff ); + if( rc==SQLITE_OK && pExpr->pConfig->bTokendata && !pT->bPrefix ){ + int iCol = p->iOff>>32; + int iTokOff = p->iOff & 0x7FFFFFFF; + rc = sqlite3Fts5IndexIterWriteTokendata( + pT->pIter, pToken, nToken, iRowid, iCol, iTokOff + ); + } if( rc ) return rc; break; } @@ -183253,9 +241478,9 @@ static int fts5ExprPopulatePoslistsCb( static int sqlite3Fts5ExprPopulatePoslists( Fts5Config *pConfig, - Fts5Expr *pExpr, + Fts5Expr *pExpr, Fts5PoslistPopulator *aPopulator, - int iCol, + int iCol, const char *z, int n ){ int i; @@ -183267,7 +241492,7 @@ static int sqlite3Fts5ExprPopulatePoslists( for(i=0; inPhrase; i++){ Fts5ExprNode *pNode = pExpr->apExprPhrase[i]->pNode; Fts5Colset *pColset = pNode->pNear->pColset; - if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) + if( (pColset && 0==fts5ExprColsetTest(pColset, iCol)) || aPopulator[i].bMiss ){ aPopulator[i].bOk = 0; @@ -183276,7 +241501,7 @@ static int sqlite3Fts5ExprPopulatePoslists( } } - return sqlite3Fts5Tokenize(pConfig, + return sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, z, n, (void*)&sCtx, fts5ExprPopulatePoslistsCb ); } @@ -183296,6 +241521,7 @@ static int fts5ExprCheckPoslists(Fts5ExprNode *pNode, i64 iRowid){ pNode->iRowid = iRowid; pNode->bEof = 0; switch( pNode->eType ){ + case 0: case FTS5_TERM: case FTS5_STRING: return (pNode->pNear->apPhrase[0]->poslist.n>0); @@ -183340,24 +241566,13 @@ static void sqlite3Fts5ExprCheckPoslists(Fts5Expr *pExpr, i64 iRowid){ fts5ExprCheckPoslists(pExpr->pRoot, iRowid); } -static void fts5ExprClearEof(Fts5ExprNode *pNode){ - int i; - for(i=0; inChild; i++){ - fts5ExprClearEof(pNode->apChild[i]); - } - pNode->bEof = 0; -} -static void sqlite3Fts5ExprClearEof(Fts5Expr *pExpr){ - fts5ExprClearEof(pExpr->pRoot); -} - /* -** This function is only called for detail=columns tables. +** This function is only called for detail=columns tables. */ static int sqlite3Fts5ExprPhraseCollist( - Fts5Expr *pExpr, - int iPhrase, - const u8 **ppCollist, + Fts5Expr *pExpr, + int iPhrase, + const u8 **ppCollist, int *pnCollist ){ Fts5ExprPhrase *pPhrase = pExpr->apExprPhrase[iPhrase]; @@ -183367,8 +241582,8 @@ static int sqlite3Fts5ExprPhraseCollist( assert( iPhrase>=0 && iPhrasenPhrase ); assert( pExpr->pConfig->eDetail==FTS5_DETAIL_COLUMNS ); - if( pNode->bEof==0 - && pNode->iRowid==pExpr->pRoot->iRowid + if( pNode->bEof==0 + && pNode->iRowid==pExpr->pRoot->iRowid && pPhrase->poslist.n>0 ){ Fts5ExprTerm *pTerm = &pPhrase->aTerm[0]; @@ -183389,6 +241604,82 @@ static int sqlite3Fts5ExprPhraseCollist( return rc; } +/* +** Does the work of the fts5_api.xQueryToken() API method. +*/ +static int sqlite3Fts5ExprQueryToken( + Fts5Expr *pExpr, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + + *ppOut = pPhrase->aTerm[iToken].pTerm; + *pnOut = pPhrase->aTerm[iToken].nFullTerm; + return SQLITE_OK; +} + +/* +** Does the work of the fts5_api.xInstToken() API method. +*/ +static int sqlite3Fts5ExprInstToken( + Fts5Expr *pExpr, + i64 iRowid, + int iPhrase, + int iCol, + int iOff, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5ExprPhrase *pPhrase = 0; + Fts5ExprTerm *pTerm = 0; + int rc = SQLITE_OK; + + if( iPhrase<0 || iPhrase>=pExpr->nPhrase ){ + return SQLITE_RANGE; + } + pPhrase = pExpr->apExprPhrase[iPhrase]; + if( iToken<0 || iToken>=pPhrase->nTerm ){ + return SQLITE_RANGE; + } + pTerm = &pPhrase->aTerm[iToken]; + if( pTerm->bPrefix==0 ){ + if( pExpr->pConfig->bTokendata ){ + rc = sqlite3Fts5IterToken( + pTerm->pIter, iRowid, iCol, iOff+iToken, ppOut, pnOut + ); + }else{ + *ppOut = pTerm->pTerm; + *pnOut = pTerm->nFullTerm; + } + } + return rc; +} + +/* +** Clear the token mappings for all Fts5IndexIter objects mannaged by +** the expression passed as the only argument. +*/ +static void sqlite3Fts5ExprClearTokens(Fts5Expr *pExpr){ + int ii; + for(ii=0; iinPhrase; ii++){ + Fts5ExprTerm *pT; + for(pT=&pExpr->apExprPhrase[ii]->aTerm[0]; pT; pT=pT->pSynonym){ + sqlite3Fts5IndexIterClearTokendata(pT->pIter); + } + } +} /* ** 2014 August 11 @@ -183427,10 +241718,16 @@ struct Fts5Hash { }; /* -** Each entry in the hash table is represented by an object of the -** following type. Each object, its key (zKey[]) and its current data -** are stored in a single memory allocation. The position list data -** immediately follows the key data in memory. +** Each entry in the hash table is represented by an object of the +** following type. Each object, its key, and its current data are stored +** in a single memory allocation. The key immediately follows the object +** in memory. The position list data immediately follows the key data +** in memory. +** +** The key is Fts5HashEntry.nKey bytes in size. It consists of a single +** byte identifying the index (either the main term index or a prefix-index), +** followed by the term data. For example: "0token". There is no +** nul-terminator - in this case nKey=6. ** ** The data that follows the key is in a similar, but not identical format ** to the doclist data stored in the database. It is: @@ -183450,24 +241747,24 @@ struct Fts5Hash { struct Fts5HashEntry { Fts5HashEntry *pHashNext; /* Next hash entry with same hash-key */ Fts5HashEntry *pScanNext; /* Next entry in sorted order */ - + int nAlloc; /* Total size of allocation */ int iSzPoslist; /* Offset of space for 4-byte poslist size */ int nData; /* Total bytes of data (incl. structure) */ - int nKey; /* Length of zKey[] in bytes */ + int nKey; /* Length of key in bytes */ u8 bDel; /* Set delete-flag @ iSzPoslist */ u8 bContent; /* Set content-flag (detail=none mode) */ i16 iCol; /* Column of last value written */ int iPos; /* Position of last value written */ i64 iRowid; /* Rowid of last value written */ - char zKey[8]; /* Nul-terminated entry key */ }; /* -** Size of Fts5HashEntry without the zKey[] array. +** Eqivalent to: +** +** char *fts5EntryKey(Fts5HashEntry *pEntry){ return zKey; } */ -#define FTS5_HASHENTRYSIZE (sizeof(Fts5HashEntry)-8) - +#define fts5EntryKey(p) ( ((char *)(&(p)[1])) ) /* @@ -183481,20 +241778,20 @@ static int sqlite3Fts5HashNew(Fts5Config *pConfig, Fts5Hash **ppNew, int *pnByte if( pNew==0 ){ rc = SQLITE_NOMEM; }else{ - int nByte; + sqlite3_int64 nByte; memset(pNew, 0, sizeof(Fts5Hash)); pNew->pnByte = pnByte; pNew->eDetail = pConfig->eDetail; pNew->nSlot = 1024; nByte = sizeof(Fts5HashEntry*) * pNew->nSlot; - pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc(nByte); + pNew->aSlot = (Fts5HashEntry**)sqlite3_malloc64(nByte); if( pNew->aSlot==0 ){ sqlite3_free(pNew); *ppNew = 0; rc = SQLITE_NOMEM; }else{ - memset(pNew->aSlot, 0, nByte); + memset(pNew->aSlot, 0, (size_t)nByte); } } return rc; @@ -183556,16 +241853,16 @@ static int fts5HashResize(Fts5Hash *pHash){ Fts5HashEntry **apNew; Fts5HashEntry **apOld = pHash->aSlot; - apNew = (Fts5HashEntry**)sqlite3_malloc(nNew*sizeof(Fts5HashEntry*)); + apNew = (Fts5HashEntry**)sqlite3_malloc64(nNew*sizeof(Fts5HashEntry*)); if( !apNew ) return SQLITE_NOMEM; memset(apNew, 0, nNew*sizeof(Fts5HashEntry*)); for(i=0; inSlot; i++){ while( apOld[i] ){ - int iHash; + unsigned int iHash; Fts5HashEntry *p = apOld[i]; apOld[i] = p->pHashNext; - iHash = fts5HashKey(nNew, (u8*)p->zKey, (int)strlen(p->zKey)); + iHash = fts5HashKey(nNew, (u8*)fts5EntryKey(p), p->nKey); p->pHashNext = apNew[iHash]; apNew[iHash] = p; } @@ -183577,19 +241874,25 @@ static int fts5HashResize(Fts5Hash *pHash){ return SQLITE_OK; } -static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ +static int fts5HashAddPoslistSize( + Fts5Hash *pHash, + Fts5HashEntry *p, + Fts5HashEntry *p2 +){ + int nRet = 0; if( p->iSzPoslist ){ - u8 *pPtr = (u8*)p; + u8 *pPtr = p2 ? (u8*)p2 : (u8*)p; + int nData = p->nData; if( pHash->eDetail==FTS5_DETAIL_NONE ){ - assert( p->nData==p->iSzPoslist ); + assert( nData==p->iSzPoslist ); if( p->bDel ){ - pPtr[p->nData++] = 0x00; + pPtr[nData++] = 0x00; if( p->bContent ){ - pPtr[p->nData++] = 0x00; + pPtr[nData++] = 0x00; } } }else{ - int nSz = (p->nData - p->iSzPoslist - 1); /* Size in bytes */ + int nSz = (nData - p->iSzPoslist - 1); /* Size in bytes */ int nPos = nSz*2 + p->bDel; /* Value of nPos field */ assert( p->bDel==0 || p->bDel==1 ); @@ -183599,14 +241902,19 @@ static void fts5HashAddPoslistSize(Fts5Hash *pHash, Fts5HashEntry *p){ int nByte = sqlite3Fts5GetVarintLen((u32)nPos); memmove(&pPtr[p->iSzPoslist + nByte], &pPtr[p->iSzPoslist + 1], nSz); sqlite3Fts5PutVarint(&pPtr[p->iSzPoslist], nPos); - p->nData += (nByte-1); + nData += (nByte-1); } } - p->iSzPoslist = 0; - p->bDel = 0; - p->bContent = 0; + nRet = nData - p->nData; + if( p2==0 ){ + p->iSzPoslist = 0; + p->bDel = 0; + p->bContent = 0; + p->nData = nData; + } } + return nRet; } /* @@ -183630,15 +241938,16 @@ static int sqlite3Fts5HashWrite( u8 *pPtr; int nIncr = 0; /* Amount to increment (*pHash->pnByte) by */ int bNew; /* If non-delete entry should be written */ - + bNew = (pHash->eDetail==FTS5_DETAIL_FULL); /* Attempt to locate an existing hash entry */ iHash = fts5HashKey2(pHash->nSlot, (u8)bByte, (const u8*)pToken, nToken); for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ - if( p->zKey[0]==bByte - && p->nKey==nToken - && memcmp(&p->zKey[1], pToken, nToken)==0 + char *zKey = fts5EntryKey(p); + if( zKey[0]==bByte + && p->nKey==nToken+1 + && memcmp(&zKey[1], pToken, nToken)==0 ){ break; } @@ -183647,7 +241956,8 @@ static int sqlite3Fts5HashWrite( /* If an existing hash entry cannot be found, create a new one. */ if( p==0 ){ /* Figure out how much space to allocate */ - int nByte = FTS5_HASHENTRYSIZE + (nToken+1) + 1 + 64; + char *zKey; + sqlite3_int64 nByte = sizeof(Fts5HashEntry) + (nToken+1) + 1 + 64; if( nByte<128 ) nByte = 128; /* Grow the Fts5Hash.aSlot[] array if necessary. */ @@ -183658,16 +241968,17 @@ static int sqlite3Fts5HashWrite( } /* Allocate new Fts5HashEntry and add it to the hash table. */ - p = (Fts5HashEntry*)sqlite3_malloc(nByte); + p = (Fts5HashEntry*)sqlite3_malloc64(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, FTS5_HASHENTRYSIZE); - p->nAlloc = nByte; - p->zKey[0] = bByte; - memcpy(&p->zKey[1], pToken, nToken); - assert( iHash==fts5HashKey(pHash->nSlot, (u8*)p->zKey, nToken+1) ); - p->nKey = nToken; - p->zKey[nToken+1] = '\0'; - p->nData = nToken+1 + 1 + FTS5_HASHENTRYSIZE; + memset(p, 0, sizeof(Fts5HashEntry)); + p->nAlloc = (int)nByte; + zKey = fts5EntryKey(p); + zKey[0] = bByte; + memcpy(&zKey[1], pToken, nToken); + assert( iHash==fts5HashKey(pHash->nSlot, (u8*)zKey, nToken+1) ); + p->nKey = nToken+1; + zKey[nToken+1] = '\0'; + p->nData = nToken+1 + sizeof(Fts5HashEntry); p->pHashNext = pHash->aSlot[iHash]; pHash->aSlot[iHash] = p; pHash->nEntry++; @@ -183682,11 +241993,10 @@ static int sqlite3Fts5HashWrite( p->iCol = (pHash->eDetail==FTS5_DETAIL_FULL ? 0 : -1); } - nIncr += p->nData; }else{ - /* Appending to an existing hash-entry. Check that there is enough - ** space to append the largest possible new entry. Worst case scenario + /* Appending to an existing hash-entry. Check that there is enough + ** space to append the largest possible new entry. Worst case scenario ** is: ** ** + 9 bytes for a new rowid, @@ -183696,12 +242006,12 @@ static int sqlite3Fts5HashWrite( ** + 5 bytes for the new position offset (32-bit max). */ if( (p->nAlloc - p->nData) < (9 + 4 + 1 + 3 + 5) ){ - int nNew = p->nAlloc * 2; + sqlite3_int64 nNew = p->nAlloc * 2; Fts5HashEntry *pNew; Fts5HashEntry **pp; - pNew = (Fts5HashEntry*)sqlite3_realloc(p, nNew); + pNew = (Fts5HashEntry*)sqlite3_realloc64(p, nNew); if( pNew==0 ) return SQLITE_NOMEM; - pNew->nAlloc = nNew; + pNew->nAlloc = (int)nNew; for(pp=&pHash->aSlot[iHash]; *pp!=p; pp=&(*pp)->pHashNext); *pp = pNew; p = pNew; @@ -183715,8 +242025,9 @@ static int sqlite3Fts5HashWrite( /* If this is a new rowid, append the 4-byte size field for the previous ** entry, and the new rowid for this entry. */ if( iRowid!=p->iRowid ){ - fts5HashAddPoslistSize(pHash, p); - p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iRowid - p->iRowid); + u64 iDiff = (u64)iRowid - (u64)p->iRowid; + fts5HashAddPoslistSize(pHash, p, 0); + p->nData += sqlite3Fts5PutVarint(&pPtr[p->nData], iDiff); p->iRowid = iRowid; bNew = 1; p->iSzPoslist = p->nData; @@ -183732,7 +242043,7 @@ static int sqlite3Fts5HashWrite( p->bContent = 1; }else{ /* Append a new column value, if necessary */ - assert( iCol>=p->iCol ); + assert_nc( iCol>=p->iCol ); if( iCol!=p->iCol ){ if( pHash->eDetail==FTS5_DETAIL_FULL ){ pPtr[p->nData++] = 0x01; @@ -183784,10 +242095,17 @@ static Fts5HashEntry *fts5HashEntryMerge( *ppOut = p1; p1 = 0; }else{ - int i = 0; - while( p1->zKey[i]==p2->zKey[i] ) i++; + char *zKey1 = fts5EntryKey(p1); + char *zKey2 = fts5EntryKey(p2); + int nMin = MIN(p1->nKey, p2->nKey); - if( ((u8)p1->zKey[i])>((u8)p2->zKey[i]) ){ + int cmp = memcmp(zKey1, zKey2, nMin); + if( cmp==0 ){ + cmp = p1->nKey - p2->nKey; + } + assert( cmp!=0 ); + + if( cmp>0 ){ /* p2 is smaller */ *ppOut = p2; ppOut = &p2->pScanNext; @@ -183806,13 +242124,11 @@ static Fts5HashEntry *fts5HashEntryMerge( } /* -** Extract all tokens from hash table iHash and link them into a list -** in sorted order. The hash table is cleared before returning. It is -** the responsibility of the caller to free the elements of the returned -** list. +** Link all tokens from hash table iHash into a list in sorted order. The +** tokens are not removed from the hash table. */ static int fts5HashEntrySort( - Fts5Hash *pHash, + Fts5Hash *pHash, const char *pTerm, int nTerm, /* Query prefix, if any */ Fts5HashEntry **ppSorted ){ @@ -183823,14 +242139,16 @@ static int fts5HashEntrySort( int i; *ppSorted = 0; - ap = sqlite3_malloc(sizeof(Fts5HashEntry*) * nMergeSlot); + ap = sqlite3_malloc64(sizeof(Fts5HashEntry*) * nMergeSlot); if( !ap ) return SQLITE_NOMEM; memset(ap, 0, sizeof(Fts5HashEntry*) * nMergeSlot); for(iSlot=0; iSlotnSlot; iSlot++){ Fts5HashEntry *pIter; for(pIter=pHash->aSlot[iSlot]; pIter; pIter=pIter->pHashNext){ - if( pTerm==0 || 0==memcmp(pIter->zKey, pTerm, nTerm) ){ + if( pTerm==0 + || (pIter->nKey>=nTerm && 0==memcmp(fts5EntryKey(pIter), pTerm, nTerm)) + ){ Fts5HashEntry *pEntry = pIter; pEntry->pScanNext = 0; for(i=0; ap[i]; i++){ @@ -183847,7 +242165,6 @@ static int fts5HashEntrySort( pList = fts5HashEntryMerge(pList, ap[i]); } - pHash->nEntry = 0; sqlite3_free(ap); *ppSorted = pList; return SQLITE_OK; @@ -183858,23 +242175,35 @@ static int fts5HashEntrySort( */ static int sqlite3Fts5HashQuery( Fts5Hash *pHash, /* Hash table to query */ + int nPre, const char *pTerm, int nTerm, /* Query term */ - const u8 **ppDoclist, /* OUT: Pointer to doclist for pTerm */ + void **ppOut, /* OUT: Pointer to new object */ int *pnDoclist /* OUT: Size of doclist in bytes */ ){ unsigned int iHash = fts5HashKey(pHash->nSlot, (const u8*)pTerm, nTerm); + char *zKey = 0; Fts5HashEntry *p; for(p=pHash->aSlot[iHash]; p; p=p->pHashNext){ - if( memcmp(p->zKey, pTerm, nTerm)==0 && p->zKey[nTerm]==0 ) break; + zKey = fts5EntryKey(p); + if( nTerm==p->nKey && memcmp(zKey, pTerm, nTerm)==0 ) break; } if( p ){ - fts5HashAddPoslistSize(pHash, p); - *ppDoclist = (const u8*)&p->zKey[nTerm+1]; - *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); + int nHashPre = sizeof(Fts5HashEntry) + nTerm; + int nList = p->nData - nHashPre; + u8 *pRet = (u8*)(*ppOut = sqlite3_malloc64(nPre + nList + 10)); + if( pRet ){ + Fts5HashEntry *pFaux = (Fts5HashEntry*)&pRet[nPre-nHashPre]; + memcpy(&pRet[nPre], &((u8*)p)[nHashPre], nList); + nList += fts5HashAddPoslistSize(pHash, p, pFaux); + *pnDoclist = nList; + }else{ + *pnDoclist = 0; + return SQLITE_NOMEM; + } }else{ - *ppDoclist = 0; + *ppOut = 0; *pnDoclist = 0; } @@ -183888,6 +242217,28 @@ static int sqlite3Fts5HashScanInit( return fts5HashEntrySort(p, pTerm, nTerm, &p->pScan); } +#ifdef SQLITE_DEBUG +static int fts5HashCount(Fts5Hash *pHash){ + int nEntry = 0; + int ii; + for(ii=0; iinSlot; ii++){ + Fts5HashEntry *p = 0; + for(p=pHash->aSlot[ii]; p; p=p->pHashNext){ + nEntry++; + } + } + return nEntry; +} +#endif + +/* +** Return true if the hash table is empty, false otherwise. +*/ +static int sqlite3Fts5HashIsEmpty(Fts5Hash *pHash){ + assert( pHash->nEntry==fts5HashCount(pHash) ); + return pHash->nEntry==0; +} + static void sqlite3Fts5HashScanNext(Fts5Hash *p){ assert( !sqlite3Fts5HashScanEof(p) ); p->pScan = p->pScan->pScanNext; @@ -183900,24 +242251,27 @@ static int sqlite3Fts5HashScanEof(Fts5Hash *p){ static void sqlite3Fts5HashScanEntry( Fts5Hash *pHash, const char **pzTerm, /* OUT: term (nul-terminated) */ + int *pnTerm, /* OUT: Size of term in bytes */ const u8 **ppDoclist, /* OUT: pointer to doclist */ int *pnDoclist /* OUT: size of doclist in bytes */ ){ Fts5HashEntry *p; if( (p = pHash->pScan) ){ - int nTerm = (int)strlen(p->zKey); - fts5HashAddPoslistSize(pHash, p); - *pzTerm = p->zKey; - *ppDoclist = (const u8*)&p->zKey[nTerm+1]; - *pnDoclist = p->nData - (FTS5_HASHENTRYSIZE + nTerm + 1); + char *zKey = fts5EntryKey(p); + int nTerm = p->nKey; + fts5HashAddPoslistSize(pHash, p, 0); + *pzTerm = zKey; + *pnTerm = nTerm; + *ppDoclist = (const u8*)&zKey[nTerm]; + *pnDoclist = p->nData - (sizeof(Fts5HashEntry) + nTerm); }else{ *pzTerm = 0; + *pnTerm = 0; *ppDoclist = 0; *pnDoclist = 0; } } - /* ** 2014 May 31 ** @@ -183930,7 +242284,7 @@ static void sqlite3Fts5HashScanEntry( ** ****************************************************************************** ** -** Low level access to the FTS index stored in the database file. The +** Low level access to the FTS index stored in the database file. The ** routines in this file file implement all read and write access to the ** %_data table. Other parts of the system access this functionality via ** the interface defined in fts5Int.h. @@ -183946,10 +242300,10 @@ static void sqlite3Fts5HashScanEntry( ** As well as the main term index, there may be up to 31 prefix indexes. ** The format is similar to FTS3/4, except that: ** -** * all segment b-tree leaf data is stored in fixed size page records -** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is -** taken to ensure it is possible to iterate in either direction through -** the entries in a doclist, or to seek to a specific entry within a +** * all segment b-tree leaf data is stored in fixed size page records +** (e.g. 1000 bytes). A single doclist may span multiple pages. Care is +** taken to ensure it is possible to iterate in either direction through +** the entries in a doclist, or to seek to a specific entry within a ** doclist, without loading it into memory. ** ** * large doclists that span many pages have associated "doclist index" @@ -183974,6 +242328,26 @@ static void sqlite3Fts5HashScanEntry( # error "FTS5_MAX_PREFIX_INDEXES is too large" #endif +#define FTS5_MAX_LEVEL 64 + +/* +** There are two versions of the format used for the structure record: +** +** 1. the legacy format, that may be read by all fts5 versions, and +** +** 2. the V2 format, which is used by contentless_delete=1 databases. +** +** Both begin with a 4-byte "configuration cookie" value. Then, a legacy +** format structure record contains a varint - the number of levels in +** the structure. Whereas a V2 structure record contains the constant +** 4 bytes [0xff 0x00 0x00 0x01]. This is unambiguous as the value of a +** varint has to be at least 16256 to begin with "0xFF". And the default +** maximum number of levels is 64. +** +** See below for more on structure record formats. +*/ +#define FTS5_STRUCTURE_V2 "\xFF\x00\x00\x01" + /* ** Details: ** @@ -183981,21 +242355,21 @@ static void sqlite3Fts5HashScanEntry( ** ** CREATE TABLE %_data(id INTEGER PRIMARY KEY, block BLOB); ** -** , contains the following 5 types of records. See the comments surrounding -** the FTS5_*_ROWID macros below for a description of how %_data rowids are +** , contains the following 6 types of records. See the comments surrounding +** the FTS5_*_ROWID macros below for a description of how %_data rowids are ** assigned to each fo them. ** ** 1. Structure Records: ** ** The set of segments that make up an index - the index structure - are ** recorded in a single record within the %_data table. The record consists -** of a single 32-bit configuration cookie value followed by a list of -** SQLite varints. If the FTS table features more than one index (because -** there are one or more prefix indexes), it is guaranteed that all share -** the same cookie value. +** of a single 32-bit configuration cookie value followed by a list of +** SQLite varints. ** -** Immediately following the configuration cookie, the record begins with -** three varints: +** If the structure record is a V2 record, the configuration cookie is +** followed by the following 4 bytes: [0xFF 0x00 0x00 0x01]. +** +** Next, the record continues with three varints: ** ** + number of levels, ** + total number of segments on all levels, @@ -184010,6 +242384,12 @@ static void sqlite3Fts5HashScanEntry( ** + first leaf page number (often 1, always greater than 0) ** + final leaf page number ** +** Then, for V2 structures only: +** +** + lower origin counter value, +** + upper origin counter value, +** + the number of tombstone hash pages. +** ** 2. The Averages Record: ** ** A single record within the %_data table. The data is a list of varints. @@ -184021,7 +242401,7 @@ static void sqlite3Fts5HashScanEntry( ** ** TERM/DOCLIST FORMAT: ** -** Most of each segment leaf is taken up by term/doclist data. The +** Most of each segment leaf is taken up by term/doclist data. The ** general format of term/doclist, starting with the first term ** on the leaf page, is: ** @@ -184064,7 +242444,7 @@ static void sqlite3Fts5HashScanEntry( ** ** PAGE FORMAT ** -** Each leaf page begins with a 4-byte header containing 2 16-bit +** Each leaf page begins with a 4-byte header containing 2 16-bit ** unsigned integer fields in big-endian format. They are: ** ** * The byte offset of the first rowid on the page, if it exists @@ -184099,7 +242479,7 @@ static void sqlite3Fts5HashScanEntry( ** 5. Segment doclist indexes: ** ** Doclist indexes are themselves b-trees, however they usually consist of -** a single leaf record only. The format of each doclist index leaf page +** a single leaf record only. The format of each doclist index leaf page ** is: ** ** * Flags byte. Bits are: @@ -184109,8 +242489,8 @@ static void sqlite3Fts5HashScanEntry( ** ** * First rowid on page indicated by previous field. As a varint. ** -** * A list of varints, one for each subsequent termless page. A -** positive delta if the termless page contains at least one rowid, +** * A list of varints, one for each subsequent termless page. A +** positive delta if the termless page contains at least one rowid, ** or an 0x00 byte otherwise. ** ** Internal doclist index nodes are: @@ -184123,8 +242503,40 @@ static void sqlite3Fts5HashScanEntry( ** * Copy of first rowid on page indicated by previous field. As a varint. ** ** * A list of delta-encoded varints - the first rowid on each subsequent -** child page. +** child page. ** +** 6. Tombstone Hash Page +** +** These records are only ever present in contentless_delete=1 tables. +** There are zero or more of these associated with each segment. They +** are used to store the tombstone rowids for rows contained in the +** associated segments. +** +** The set of nHashPg tombstone hash pages associated with a single +** segment together form a single hash table containing tombstone rowids. +** To find the page of the hash on which a key might be stored: +** +** iPg = (rowid % nHashPg) +** +** Then, within page iPg, which has nSlot slots: +** +** iSlot = (rowid / nHashPg) % nSlot +** +** Each tombstone hash page begins with an 8 byte header: +** +** 1-byte: Key-size (the size in bytes of each slot). Either 4 or 8. +** 1-byte: rowid-0-tombstone flag. This flag is only valid on the +** first tombstone hash page for each segment (iPg=0). If set, +** the hash table contains rowid 0. If clear, it does not. +** Rowid 0 is handled specially. +** 2-bytes: unused. +** 4-bytes: Big-endian integer containing number of entries on page. +** +** Following this are nSlot 4 or 8 byte slots (depending on the key-size +** in the first byte of the page header). The number of slots may be +** determined based on the size of the page record and the key-size: +** +** nSlot = (nByte - 8) / key-size */ /* @@ -184140,7 +242552,7 @@ static void sqlite3Fts5HashScanEntry( ** ** Each segment has a unique non-zero 16-bit id. ** -** The rowid for each segment leaf is found by passing the segment id and +** The rowid for each segment leaf is found by passing the segment id and ** the leaf page number to the FTS5_SEGMENT_ROWID macro. Leaves are numbered ** sequentially starting from 1. */ @@ -184158,11 +242570,7 @@ static void sqlite3Fts5HashScanEntry( #define FTS5_SEGMENT_ROWID(segid, pgno) fts5_dri(segid, 0, 0, pgno) #define FTS5_DLIDX_ROWID(segid, height, pgno) fts5_dri(segid, 1, height, pgno) - -/* -** Maximum segments permitted in a single index -*/ -#define FTS5_MAX_SEGMENT 2000 +#define FTS5_TOMBSTONE_ROWID(segid,ipg) fts5_dri(segid+(1<<16), 0, 0, ipg) #ifdef SQLITE_DEBUG static int sqlite3Fts5Corrupt() { return SQLITE_CORRUPT_VTAB; } @@ -184189,6 +242597,9 @@ typedef struct Fts5SegWriter Fts5SegWriter; typedef struct Fts5Structure Fts5Structure; typedef struct Fts5StructureLevel Fts5StructureLevel; typedef struct Fts5StructureSegment Fts5StructureSegment; +typedef struct Fts5TokenDataIter Fts5TokenDataIter; +typedef struct Fts5TokenDataMap Fts5TokenDataMap; +typedef struct Fts5TombstoneArray Fts5TombstoneArray; struct Fts5Data { u8 *p; /* Pointer to buffer containing record */ @@ -184198,6 +242609,12 @@ struct Fts5Data { /* ** One object per %_data table. +** +** nContentlessDelete: +** The number of contentless delete operations since the most recent +** call to fts5IndexFlush() or fts5IndexDiscardData(). This is tracked +** so that extra auto-merge work can be done by fts5IndexFlush() to +** account for the delete operations. */ struct Fts5Index { Fts5Config *pConfig; /* Virtual table configuration */ @@ -184212,19 +242629,25 @@ struct Fts5Index { int nPendingData; /* Current bytes of pending data */ i64 iWriteRowid; /* Rowid for current doc being written */ int bDelete; /* Current write is a delete */ + int nContentlessDelete; /* Number of contentless delete ops */ + int nPendingRow; /* Number of INSERT in hash table */ /* Error state. */ int rc; /* Current error code */ + int flushRc; /* State used by the fts5DataXXX() functions. */ sqlite3_blob *pReader; /* RO incr-blob open on %_data table */ sqlite3_stmt *pWriter; /* "INSERT ... %_data VALUES(?,?)" */ sqlite3_stmt *pDeleter; /* "DELETE FROM %_data ... id>=? AND id<=?" */ sqlite3_stmt *pIdxWriter; /* "INSERT ... %_idx VALUES(?,?,?,?)" */ - sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=? */ + sqlite3_stmt *pIdxDeleter; /* "DELETE FROM %_idx WHERE segid=?" */ sqlite3_stmt *pIdxSelect; + sqlite3_stmt *pIdxNextSelect; int nRead; /* Total number of blocks read */ + sqlite3_stmt *pDeleteFromIdx; + sqlite3_stmt *pDataVersion; i64 iStructVersion; /* data_version when pStruct read */ Fts5Structure *pStruct; /* Current db structure (or NULL) */ @@ -184242,13 +242665,25 @@ struct Fts5DoclistIter { /* ** The contents of the "structure" record for each index are represented -** using an Fts5Structure record in memory. Which uses instances of the +** using an Fts5Structure record in memory. Which uses instances of the ** other Fts5StructureXXX types as components. +** +** nOriginCntr: +** This value is set to non-zero for structure records created for +** contentlessdelete=1 tables only. In that case it represents the +** origin value to apply to the next top-level segment created. */ struct Fts5StructureSegment { int iSegid; /* Segment id */ int pgnoFirst; /* First leaf page number in segment */ int pgnoLast; /* Last leaf page number in segment */ + + /* contentlessdelete=1 tables only: */ + u64 iOrigin1; + u64 iOrigin2; + int nPgTombstone; /* Number of tombstone hash table pages */ + u64 nEntryTombstone; /* Number of tombstone entries that "count" */ + u64 nEntry; /* Number of rows in this segment */ }; struct Fts5StructureLevel { int nMerge; /* Number of segments in incr-merge */ @@ -184258,6 +242693,7 @@ struct Fts5StructureLevel { struct Fts5Structure { int nRef; /* Object reference count */ u64 nWriteCounter; /* Total leaves written to level 0 */ + u64 nOriginCntr; /* Origin value for next top-level segment */ int nSegment; /* Total segments in this structure */ int nLevel; /* Number of levels in this index */ Fts5StructureLevel aLevel[1]; /* Array of nLevel level objects */ @@ -184315,11 +242751,8 @@ struct Fts5CResult { ** Current leaf page number within segment. ** ** iLeafOffset: -** Byte offset within the current leaf that is the first byte of the +** Byte offset within the current leaf that is the first byte of the ** position list data (one byte passed the position-list size field). -** rowid field of the current entry. Usually this is the size field of the -** position list data. The exception is if the rowid for the current entry -** is the last thing on the leaf page. ** ** pLeaf: ** Buffer containing current leaf page data. Set to NULL at EOF. @@ -184332,7 +242765,7 @@ struct Fts5CResult { ** Mask of FTS5_SEGITER_XXX values. Interpreted as follows: ** ** FTS5_SEGITER_ONETERM: -** If set, set the iterator to point to EOF after the current doclist +** If set, set the iterator to point to EOF after the current doclist ** has been exhausted. Do not proceed to the next term in the segment. ** ** FTS5_SEGITER_REVERSE: @@ -184349,6 +242782,13 @@ struct Fts5CResult { ** ** iTermIdx: ** Index of current term on iTermLeafPgno. +** +** apTombstone/nTombstone: +** These are used for contentless_delete=1 tables only. When the cursor +** is first allocated, the apTombstone[] array is allocated so that it +** is large enough for all tombstones hash pages associated with the +** segment. The pages themselves are loaded lazily from the database as +** they are required. */ struct Fts5SegIter { Fts5StructureSegment *pSeg; /* Segment to iterate through */ @@ -184356,12 +242796,13 @@ struct Fts5SegIter { int iLeafPgno; /* Current leaf page number */ Fts5Data *pLeaf; /* Current leaf data */ Fts5Data *pNextLeaf; /* Leaf page (iLeafPgno+1) */ - int iLeafOffset; /* Byte offset within current leaf */ + i64 iLeafOffset; /* Byte offset within current leaf */ + Fts5TombstoneArray *pTombArray; /* Array of tombstone pages */ /* Next method */ void (*xNext)(Fts5Index*, Fts5SegIter*, int*); - /* The page and offset from which the current term was read. The offset + /* The page and offset from which the current term was read. The offset ** is the offset of the first rowid in the current doclist. */ int iTermLeafPgno; int iTermLeafOffset; @@ -184384,7 +242825,16 @@ struct Fts5SegIter { }; /* -** Argument is a pointer to an Fts5Data structure that contains a +** Array of tombstone pages. Reference counted. +*/ +struct Fts5TombstoneArray { + int nRef; /* Number of pointers to this object */ + int nTombstone; + Fts5Data *apTombstone[1]; /* Array of tombstone pages */ +}; + +/* +** Argument is a pointer to an Fts5Data structure that contains a ** leaf page. */ #define ASSERT_SZLEAF_OK(x) assert( \ @@ -184394,7 +242844,7 @@ struct Fts5SegIter { #define FTS5_SEGITER_ONETERM 0x01 #define FTS5_SEGITER_REVERSE 0x02 -/* +/* ** Argument is a pointer to an Fts5Data structure that contains a leaf ** page. This macro evaluates to true if the leaf contains no terms, or ** false if it contains at least one term. @@ -184416,23 +242866,29 @@ struct Fts5SegIter { ** on empty segments. ** ** The results of comparing segments aSeg[N] and aSeg[N+1], where N is an -** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the +** even number, is stored in aFirst[(nSeg+N)/2]. The "result" of the ** comparison in this context is the index of the iterator that currently ** points to the smaller term/rowid combination. Iterators at EOF are ** considered to be greater than all other iterators. ** ** aFirst[1] contains the index in aSeg[] of the iterator that points to -** the smallest key overall. aFirst[0] is unused. +** the smallest key overall. aFirst[0] is unused. ** ** poslist: ** Used by sqlite3Fts5IterPoslist() when the poslist needs to be buffered. ** There is no way to tell if this is populated or not. +** +** pColset: +** If not NULL, points to an object containing a set of column indices. +** Only matches that occur in one of these columns will be returned. +** The Fts5Iter does not own the Fts5Colset object, and so it is not +** freed when the iterator is closed - it is owned by the upper layer. */ struct Fts5Iter { Fts5IndexIter base; /* Base class containing output vars */ + Fts5TokenDataIter *pTokenDataIter; Fts5Index *pIndex; /* Index that owns this iterator */ - Fts5Structure *pStruct; /* Database structure for this iterator */ Fts5Buffer poslist; /* Buffer containing current poslist */ Fts5Colset *pColset; /* Restrict matches to these columns */ @@ -184448,7 +242904,6 @@ struct Fts5Iter { Fts5SegIter aSeg[1]; /* Array of segment iterators */ }; - /* ** An instance of the following type is used to iterate through the contents ** of a doclist-index record. @@ -184485,7 +242940,61 @@ static void fts5PutU16(u8 *aOut, u16 iVal){ static u16 fts5GetU16(const u8 *aIn){ return ((u16)aIn[0] << 8) + aIn[1]; -} +} + +/* +** The only argument points to a buffer at least 8 bytes in size. This +** function interprets the first 8 bytes of the buffer as a 64-bit big-endian +** unsigned integer and returns the result. +*/ +static u64 fts5GetU64(u8 *a){ + return ((u64)a[0] << 56) + + ((u64)a[1] << 48) + + ((u64)a[2] << 40) + + ((u64)a[3] << 32) + + ((u64)a[4] << 24) + + ((u64)a[5] << 16) + + ((u64)a[6] << 8) + + ((u64)a[7] << 0); +} + +/* +** The only argument points to a buffer at least 4 bytes in size. This +** function interprets the first 4 bytes of the buffer as a 32-bit big-endian +** unsigned integer and returns the result. +*/ +static u32 fts5GetU32(const u8 *a){ + return ((u32)a[0] << 24) + + ((u32)a[1] << 16) + + ((u32)a[2] << 8) + + ((u32)a[3] << 0); +} + +/* +** Write iVal, formated as a 64-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU64(u8 *a, u64 iVal){ + a[0] = ((iVal >> 56) & 0xFF); + a[1] = ((iVal >> 48) & 0xFF); + a[2] = ((iVal >> 40) & 0xFF); + a[3] = ((iVal >> 32) & 0xFF); + a[4] = ((iVal >> 24) & 0xFF); + a[5] = ((iVal >> 16) & 0xFF); + a[6] = ((iVal >> 8) & 0xFF); + a[7] = ((iVal >> 0) & 0xFF); +} + +/* +** Write iVal, formated as a 32-bit big-endian unsigned integer, to the +** buffer indicated by the first argument. +*/ +static void fts5PutU32(u8 *a, u32 iVal){ + a[0] = ((iVal >> 24) & 0xFF); + a[1] = ((iVal >> 16) & 0xFF); + a[2] = ((iVal >> 8) & 0xFF); + a[3] = ((iVal >> 0) & 0xFF); +} /* ** Allocate and return a buffer at least nByte bytes in size. @@ -184493,7 +243002,7 @@ static u16 fts5GetU16(const u8 *aIn){ ** If an OOM error is encountered, return NULL and set the error code in ** the Fts5Index handle passed as the first argument. */ -static void *fts5IdxMalloc(Fts5Index *p, int nByte){ +static void *fts5IdxMalloc(Fts5Index *p, sqlite3_int64 nByte){ return sqlite3Fts5MallocZero(&p->rc, nByte); } @@ -184526,8 +243035,11 @@ static int fts5BufferCompareBlob( ** res = *pLeft - *pRight */ static int fts5BufferCompare(Fts5Buffer *pLeft, Fts5Buffer *pRight){ - int nCmp = MIN(pLeft->n, pRight->n); - int res = memcmp(pLeft->p, pRight->p, nCmp); + int nCmp, res; + nCmp = MIN(pLeft->n, pRight->n); + assert( nCmp<=0 || pLeft->p!=0 ); + assert( nCmp<=0 || pRight->p!=0 ); + res = fts5Memcmp(pLeft->p, pRight->p, nCmp); return (res==0 ? (pLeft->n - pRight->n) : res); } @@ -184540,7 +243052,7 @@ static int fts5LeafFirstTermOff(Fts5Data *pLeaf){ /* ** Close the read-only blob handle, if it is open. */ -static void fts5CloseReader(Fts5Index *p){ +static void sqlite3Fts5IndexCloseReader(Fts5Index *p){ if( p->pReader ){ sqlite3_blob *pReader = p->pReader; p->pReader = 0; @@ -184548,11 +243060,10 @@ static void fts5CloseReader(Fts5Index *p){ } } - /* ** Retrieve a record from the %_data table. ** -** If an error occurs, NULL is returned and an error left in the +** If an error occurs, NULL is returned and an error left in the ** Fts5Index object. */ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ @@ -184570,16 +243081,16 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ assert( p->pReader==0 ); p->pReader = pBlob; if( rc!=SQLITE_OK ){ - fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); } if( rc==SQLITE_ABORT ) rc = SQLITE_OK; } - /* If the blob handle is not open at this point, open it and seek + /* If the blob handle is not open at this point, open it and seek ** to the requested entry. */ if( p->pReader==0 && rc==SQLITE_OK ){ Fts5Config *pConfig = p->pConfig; - rc = sqlite3_blob_open(pConfig->db, + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, "block", iRowid, 0, &p->pReader ); } @@ -184587,18 +243098,19 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ /* If either of the sqlite3_blob_open() or sqlite3_blob_reopen() calls ** above returned SQLITE_ERROR, return SQLITE_CORRUPT_VTAB instead. ** All the reasons those functions might return SQLITE_ERROR - missing - ** table, missing row, non-blob/text in block column - indicate + ** table, missing row, non-blob/text in block column - indicate ** backing store corruption. */ if( rc==SQLITE_ERROR ) rc = FTS5_CORRUPT; if( rc==SQLITE_OK ){ u8 *aOut = 0; /* Read blob data into this buffer */ int nByte = sqlite3_blob_bytes(p->pReader); - int nAlloc = sizeof(Fts5Data) + nByte + FTS5_DATA_PADDING; - pRet = (Fts5Data*)sqlite3_malloc(nAlloc); + int szData = (sizeof(Fts5Data) + 7) & ~7; + sqlite3_int64 nAlloc = szData + nByte + FTS5_DATA_PADDING; + pRet = (Fts5Data*)sqlite3_malloc64(nAlloc); if( pRet ){ pRet->nn = nByte; - aOut = pRet->p = (u8*)&pRet[1]; + aOut = pRet->p = (u8*)pRet + szData; }else{ rc = SQLITE_NOMEM; } @@ -184611,6 +243123,8 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ pRet = 0; }else{ /* TODO1: Fix this */ + pRet->p[nByte] = 0x00; + pRet->p[nByte+1] = 0x00; pRet->szLeaf = fts5GetU16(&pRet->p[2]); } } @@ -184619,6 +243133,7 @@ static Fts5Data *fts5DataRead(Fts5Index *p, i64 iRowid){ } assert( (pRet==0)==(p->rc!=SQLITE_OK) ); + assert( pRet==0 || EIGHT_BYTE_ALIGNMENT( pRet->p ) ); return pRet; } @@ -184631,6 +243146,18 @@ static void fts5DataRelease(Fts5Data *pData){ sqlite3_free(pData); } +static Fts5Data *fts5LeafRead(Fts5Index *p, i64 iRowid){ + Fts5Data *pRet = fts5DataRead(p, iRowid); + if( pRet ){ + if( pRet->nn<4 || pRet->szLeaf>pRet->nn ){ + p->rc = FTS5_CORRUPT; + fts5DataRelease(pRet); + pRet = 0; + } + } + return pRet; +} + static int fts5IndexPrepareStmt( Fts5Index *p, sqlite3_stmt **ppStmt, @@ -184638,7 +243165,9 @@ static int fts5IndexPrepareStmt( ){ if( p->rc==SQLITE_OK ){ if( zSql ){ - p->rc = sqlite3_prepare_v2(p->pConfig->db, zSql, -1, ppStmt, 0); + p->rc = sqlite3_prepare_v3(p->pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_NO_VTAB, + ppStmt, 0); }else{ p->rc = SQLITE_NOMEM; } @@ -184657,7 +243186,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ if( p->pWriter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pWriter, sqlite3_mprintf( - "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", + "REPLACE INTO '%q'.'%q_data'(id, block) VALUES(?,?)", pConfig->zDb, pConfig->zName )); if( p->rc ) return; @@ -184667,6 +243196,7 @@ static void fts5DataWrite(Fts5Index *p, i64 iRowid, const u8 *pData, int nData){ sqlite3_bind_blob(p->pWriter, 2, pData, nData, SQLITE_STATIC); sqlite3_step(p->pWriter); p->rc = sqlite3_reset(p->pWriter); + sqlite3_bind_null(p->pWriter, 2); } /* @@ -184678,22 +243208,12 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ if( p->rc!=SQLITE_OK ) return; if( p->pDeleter==0 ){ - int rc; Fts5Config *pConfig = p->pConfig; char *zSql = sqlite3_mprintf( - "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", + "DELETE FROM '%q'.'%q_data' WHERE id>=? AND id<=?", pConfig->zDb, pConfig->zName ); - if( zSql==0 ){ - rc = SQLITE_NOMEM; - }else{ - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &p->pDeleter, 0); - sqlite3_free(zSql); - } - if( rc!=SQLITE_OK ){ - p->rc = rc; - return; - } + if( fts5IndexPrepareStmt(p, &p->pDeleter, zSql) ) return; } sqlite3_bind_int64(p->pDeleter, 1, iFirst); @@ -184705,10 +243225,17 @@ static void fts5DataDelete(Fts5Index *p, i64 iFirst, i64 iLast){ /* ** Remove all records associated with segment iSegid. */ -static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ +static void fts5DataRemoveSegment(Fts5Index *p, Fts5StructureSegment *pSeg){ + int iSegid = pSeg->iSegid; i64 iFirst = FTS5_SEGMENT_ROWID(iSegid, 0); i64 iLast = FTS5_SEGMENT_ROWID(iSegid+1, 0)-1; fts5DataDelete(p, iFirst, iLast); + + if( pSeg->nPgTombstone ){ + i64 iTomb1 = FTS5_TOMBSTONE_ROWID(iSegid, 0); + i64 iTomb2 = FTS5_TOMBSTONE_ROWID(iSegid, pSeg->nPgTombstone-1); + fts5DataDelete(p, iTomb1, iTomb2); + } if( p->pIdxDeleter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxDeleter, sqlite3_mprintf( @@ -184724,7 +243251,7 @@ static void fts5DataRemoveSegment(Fts5Index *p, int iSegid){ } /* -** Release a reference to an Fts5Structure object returned by an earlier +** Release a reference to an Fts5Structure object returned by an earlier ** call to fts5StructureRead() or fts5StructureDecode(). */ static void fts5StructureRelease(Fts5Structure *pStruct){ @@ -184742,6 +243269,58 @@ static void fts5StructureRef(Fts5Structure *pStruct){ pStruct->nRef++; } +static void *sqlite3Fts5StructureRef(Fts5Index *p){ + fts5StructureRef(p->pStruct); + return (void*)p->pStruct; +} +static void sqlite3Fts5StructureRelease(void *p){ + if( p ){ + fts5StructureRelease((Fts5Structure*)p); + } +} +static int sqlite3Fts5StructureTest(Fts5Index *p, void *pStruct){ + if( p->pStruct!=(Fts5Structure*)pStruct ){ + return SQLITE_ABORT; + } + return SQLITE_OK; +} + +/* +** Ensure that structure object (*pp) is writable. +** +** This function is a no-op if (*pRc) is not SQLITE_OK when it is called. If +** an error occurs, (*pRc) is set to an SQLite error code before returning. +*/ +static void fts5StructureMakeWritable(int *pRc, Fts5Structure **pp){ + Fts5Structure *p = *pp; + if( *pRc==SQLITE_OK && p->nRef>1 ){ + i64 nByte = sizeof(Fts5Structure)+(p->nLevel-1)*sizeof(Fts5StructureLevel); + Fts5Structure *pNew; + pNew = (Fts5Structure*)sqlite3Fts5MallocZero(pRc, nByte); + if( pNew ){ + int i; + memcpy(pNew, p, nByte); + for(i=0; inLevel; i++) pNew->aLevel[i].aSeg = 0; + for(i=0; inLevel; i++){ + Fts5StructureLevel *pLvl = &pNew->aLevel[i]; + nByte = sizeof(Fts5StructureSegment) * pNew->aLevel[i].nSeg; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(pRc, nByte); + if( pLvl->aSeg==0 ){ + for(i=0; inLevel; i++){ + sqlite3_free(pNew->aLevel[i].aSeg); + } + sqlite3_free(pNew); + return; + } + memcpy(pLvl->aSeg, p->aLevel[i].aSeg, nByte); + } + p->nRef--; + pNew->nRef = 1; + } + *pp = pNew; + } +} + /* ** Deserialize and return the structure record currently stored in serialized ** form within buffer pData/nData. @@ -184765,17 +243344,30 @@ static int fts5StructureDecode( int iLvl; int nLevel = 0; int nSegment = 0; - int nByte; /* Bytes of space to allocate at pRet */ + sqlite3_int64 nByte; /* Bytes of space to allocate at pRet */ Fts5Structure *pRet = 0; /* Structure object to return */ + int bStructureV2 = 0; /* True for FTS5_STRUCTURE_V2 */ + u64 nOriginCntr = 0; /* Largest origin value seen so far */ /* Grab the cookie value */ if( piCookie ) *piCookie = sqlite3Fts5Get32(pData); i = 4; + /* Check if this is a V2 structure record. Set bStructureV2 if it is. */ + if( 0==memcmp(&pData[i], FTS5_STRUCTURE_V2, 4) ){ + i += 4; + bStructureV2 = 1; + } + /* Read the total number of levels and segments from the start of the ** structure record. */ i += fts5GetVarint32(&pData[i], nLevel); i += fts5GetVarint32(&pData[i], nSegment); + if( nLevel>FTS5_MAX_SEGMENT || nLevel<0 + || nSegment>FTS5_MAX_SEGMENT || nSegment<0 + ){ + return FTS5_CORRUPT; + } nByte = ( sizeof(Fts5Structure) + /* Main structure */ sizeof(Fts5StructureLevel) * (nLevel-1) /* aLevel[] array */ @@ -184798,25 +243390,47 @@ static int fts5StructureDecode( }else{ i += fts5GetVarint32(&pData[i], pLvl->nMerge); i += fts5GetVarint32(&pData[i], nTotal); - assert( nTotal>=pLvl->nMerge ); - pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, + if( nTotalnMerge ) rc = FTS5_CORRUPT; + pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&rc, nTotal * sizeof(Fts5StructureSegment) ); + nSegment -= nTotal; } if( rc==SQLITE_OK ){ pLvl->nSeg = nTotal; for(iSeg=0; iSegaSeg[iSeg]; if( i>=nData ){ rc = FTS5_CORRUPT; break; } - i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].iSegid); - i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoFirst); - i += fts5GetVarint32(&pData[i], pLvl->aSeg[iSeg].pgnoLast); + assert( pSeg!=0 ); + i += fts5GetVarint32(&pData[i], pSeg->iSegid); + i += fts5GetVarint32(&pData[i], pSeg->pgnoFirst); + i += fts5GetVarint32(&pData[i], pSeg->pgnoLast); + if( bStructureV2 ){ + i += fts5GetVarint(&pData[i], &pSeg->iOrigin1); + i += fts5GetVarint(&pData[i], &pSeg->iOrigin2); + i += fts5GetVarint32(&pData[i], pSeg->nPgTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntryTombstone); + i += fts5GetVarint(&pData[i], &pSeg->nEntry); + nOriginCntr = MAX(nOriginCntr, pSeg->iOrigin2); + } + if( pSeg->pgnoLastpgnoFirst ){ + rc = FTS5_CORRUPT; + break; + } } + if( iLvl>0 && pLvl[-1].nMerge && nTotal==0 ) rc = FTS5_CORRUPT; + if( iLvl==nLevel-1 && pLvl->nMerge ) rc = FTS5_CORRUPT; } } + if( nSegment!=0 && rc==SQLITE_OK ) rc = FTS5_CORRUPT; + if( bStructureV2 ){ + pRet->nOriginCntr = nOriginCntr+1; + } + if( rc!=SQLITE_OK ){ fts5StructureRelease(pRet); pRet = 0; @@ -184828,18 +243442,21 @@ static int fts5StructureDecode( } /* -** +** Add a level to the Fts5Structure.aLevel[] array of structure object +** (*ppStruct). */ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ + fts5StructureMakeWritable(pRc, ppStruct); + assert( (ppStruct!=0 && (*ppStruct)!=0) || (*pRc)!=SQLITE_OK ); if( *pRc==SQLITE_OK ){ Fts5Structure *pStruct = *ppStruct; int nLevel = pStruct->nLevel; - int nByte = ( + sqlite3_int64 nByte = ( sizeof(Fts5Structure) + /* Main structure */ sizeof(Fts5StructureLevel) * (nLevel+1) /* aLevel[] array */ ); - pStruct = sqlite3_realloc(pStruct, nByte); + pStruct = sqlite3_realloc64(pStruct, nByte); if( pStruct ){ memset(&pStruct->aLevel[nLevel], 0, sizeof(Fts5StructureLevel)); pStruct->nLevel++; @@ -184855,19 +243472,19 @@ static void fts5StructureAddLevel(int *pRc, Fts5Structure **ppStruct){ ** segments. */ static void fts5StructureExtendLevel( - int *pRc, - Fts5Structure *pStruct, - int iLvl, - int nExtra, + int *pRc, + Fts5Structure *pStruct, + int iLvl, + int nExtra, int bInsert ){ if( *pRc==SQLITE_OK ){ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; Fts5StructureSegment *aNew; - int nByte; + sqlite3_int64 nByte; nByte = (pLvl->nSeg + nExtra) * sizeof(Fts5StructureSegment); - aNew = sqlite3_realloc(pLvl->aSeg, nByte); + aNew = sqlite3_realloc64(pLvl->aSeg, nByte); if( aNew ){ if( bInsert==0 ){ memset(&aNew[pLvl->nSeg], 0, sizeof(Fts5StructureSegment) * nExtra); @@ -184894,7 +243511,7 @@ static Fts5Structure *fts5StructureReadUncached(Fts5Index *p){ /* TODO: Do we need this if the leaf-index is appended? Probably... */ memset(&pData->p[pData->nn], 0, FTS5_DATA_PADDING); p->rc = fts5StructureDecode(pData->p, pData->nn, &iCookie, &pRet); - if( p->rc==SQLITE_OK && pConfig->iCookie!=iCookie ){ + if( p->rc==SQLITE_OK && (pConfig->pgsz==0 || pConfig->iCookie!=iCookie) ){ p->rc = sqlite3Fts5ConfigLoad(pConfig, iCookie); } fts5DataRelease(pData); @@ -184912,7 +243529,7 @@ static i64 fts5IndexDataVersion(Fts5Index *p){ if( p->rc==SQLITE_OK ){ if( p->pDataVersion==0 ){ - p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, + p->rc = fts5IndexPrepareStmt(p, &p->pDataVersion, sqlite3_mprintf("PRAGMA %Q.data_version", p->pConfig->zDb) ); if( p->rc ) return 0; @@ -184931,7 +243548,7 @@ static i64 fts5IndexDataVersion(Fts5Index *p){ ** Read, deserialize and return the structure record. ** ** The Fts5Structure.aLevel[] and each Fts5StructureLevel.aSeg[] array -** are over-allocated as described for function fts5StructureDecode() +** are over-allocated as described for function fts5StructureDecode() ** above. ** ** If an error occurs, NULL is returned and an error code left in the @@ -185025,6 +243642,7 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ Fts5Buffer buf; /* Buffer to serialize record into */ int iLvl; /* Used to iterate through levels */ int iCookie; /* Cookie value to store */ + int nHdr = (pStruct->nOriginCntr>0 ? (4+4+9+9+9) : (4+9+9)); assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); memset(&buf, 0, sizeof(Fts5Buffer)); @@ -185033,9 +243651,12 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ iCookie = p->pConfig->iCookie; if( iCookie<0 ) iCookie = 0; - if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, 4+9+9+9) ){ + if( 0==sqlite3Fts5BufferSize(&p->rc, &buf, nHdr) ){ sqlite3Fts5Put32(buf.p, iCookie); buf.n = 4; + if( pStruct->nOriginCntr>0 ){ + fts5BufferSafeAppendBlob(&buf, FTS5_STRUCTURE_V2, 4); + } fts5BufferSafeAppendVarint(&buf, pStruct->nLevel); fts5BufferSafeAppendVarint(&buf, pStruct->nSegment); fts5BufferSafeAppendVarint(&buf, (i64)pStruct->nWriteCounter); @@ -185049,9 +243670,17 @@ static void fts5StructureWrite(Fts5Index *p, Fts5Structure *pStruct){ assert( pLvl->nMerge<=pLvl->nSeg ); for(iSeg=0; iSegnSeg; iSeg++){ - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].iSegid); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoFirst); - fts5BufferAppendVarint(&p->rc, &buf, pLvl->aSeg[iSeg].pgnoLast); + Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iSegid); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoFirst); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->pgnoLast); + if( pStruct->nOriginCntr>0 ){ + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin1); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->iOrigin2); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nPgTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntryTombstone); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->nEntry); + } } } @@ -185080,8 +243709,8 @@ static int fts5SegmentSize(Fts5StructureSegment *pSeg){ } /* -** Return a copy of index structure pStruct. Except, promote as many -** segments as possible to level iPromote. If an OOM occurs, NULL is +** Return a copy of index structure pStruct. Except, promote as many +** segments as possible to level iPromote. If an OOM occurs, NULL is ** returned. */ static void fts5StructurePromoteTo( @@ -185121,8 +243750,8 @@ static void fts5StructurePromoteTo( ** ** b) If the segment just written is larger than the newest segment on ** the next populated level, then that segment, and any other adjacent -** segments that are also smaller than the one just written, are -** promoted. +** segments that are also smaller than the one just written, are +** promoted. ** ** If one or more segments are promoted, the structure object is updated ** to reflect this. @@ -185156,7 +243785,7 @@ static void fts5StructurePromote( if( sz>szMax ) szMax = sz; } if( szMax>=szSeg ){ - /* Condition (a) is true. Promote the newest segment on level + /* Condition (a) is true. Promote the newest segment on level ** iLvl to level iTst. */ iPromote = iTst; szPromote = szMax; @@ -185175,7 +243804,7 @@ static void fts5StructurePromote( /* -** Advance the iterator passed as the only argument. If the end of the +** Advance the iterator passed as the only argument. If the end of the ** doclist-index page is reached, return non-zero. */ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ @@ -185190,13 +243819,13 @@ static int fts5DlidxLvlNext(Fts5DlidxLvl *pLvl){ }else{ int iOff; for(iOff=pLvl->iOff; iOffnn; iOff++){ - if( pData->p[iOff] ) break; + if( pData->p[iOff] ) break; } if( iOffnn ){ - i64 iVal; + u64 iVal; pLvl->iLeafPgno += (iOff - pLvl->iOff) + 1; - iOff += fts5GetVarint(&pData->p[iOff], (u64*)&iVal); + iOff += fts5GetVarint(&pData->p[iOff], &iVal); pLvl->iRowid += iVal; pLvl->iOff = iOff; }else{ @@ -185220,7 +243849,7 @@ static int fts5DlidxIterNextR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ if( pLvl[1].bEof==0 ){ fts5DataRelease(pLvl->pData); memset(pLvl, 0, sizeof(Fts5DlidxLvl)); - pLvl->pData = fts5DataRead(p, + pLvl->pData = fts5DataRead(p, FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) ); if( pLvl->pData ) fts5DlidxLvlNext(pLvl); @@ -185240,7 +243869,7 @@ static int fts5DlidxIterNext(Fts5Index *p, Fts5DlidxIter *pIter){ ** points to the first rowid in the doclist-index. ** ** pData: -** pointer to doclist-index record, +** pointer to doclist-index record, ** ** When this function is called pIter->iLeafPgno is the page number the ** doclist is associated with (the one featuring the term). @@ -185271,7 +243900,7 @@ static void fts5DlidxIterLast(Fts5Index *p, Fts5DlidxIter *pIter){ Fts5DlidxLvl *pChild = &pLvl[-1]; fts5DataRelease(pChild->pData); memset(pChild, 0, sizeof(Fts5DlidxLvl)); - pChild->pData = fts5DataRead(p, + pChild->pData = fts5DataRead(p, FTS5_DLIDX_ROWID(pIter->iSegid, i-1, pLvl->iLeafPgno) ); } @@ -185289,42 +243918,25 @@ static int fts5DlidxLvlPrev(Fts5DlidxLvl *pLvl){ pLvl->bEof = 1; }else{ u8 *a = pLvl->pData->p; - i64 iVal; - int iLimit; - int ii; - int nZero = 0; - /* Currently iOff points to the first byte of a varint. This block - ** decrements iOff until it points to the first byte of the previous - ** varint. Taking care not to read any memory locations that occur - ** before the buffer in memory. */ - iLimit = (iOff>9 ? iOff-9 : 0); - for(iOff--; iOff>iLimit; iOff--){ - if( (a[iOff-1] & 0x80)==0 ) break; - } + pLvl->iOff = 0; + fts5DlidxLvlNext(pLvl); + while( 1 ){ + int nZero = 0; + int ii = pLvl->iOff; + u64 delta = 0; - fts5GetVarint(&a[iOff], (u64*)&iVal); - pLvl->iRowid -= iVal; - pLvl->iLeafPgno--; - - /* Skip backwards past any 0x00 varints. */ - for(ii=iOff-1; ii>=pLvl->iFirstOff && a[ii]==0x00; ii--){ - nZero++; - } - if( ii>=pLvl->iFirstOff && (a[ii] & 0x80) ){ - /* The byte immediately before the last 0x00 byte has the 0x80 bit - ** set. So the last 0x00 is only a varint 0 if there are 8 more 0x80 - ** bytes before a[ii]. */ - int bZero = 0; /* True if last 0x00 counts */ - if( (ii-8)>=pLvl->iFirstOff ){ - int j; - for(j=1; j<=8 && (a[ii-j] & 0x80); j++); - bZero = (j>8); + while( a[ii]==0 ){ + nZero++; + ii++; } - if( bZero==0 ) nZero--; + ii += sqlite3Fts5GetVarint(&a[ii], &delta); + + if( ii>=iOff ) break; + pLvl->iLeafPgno += nZero+1; + pLvl->iRowid += delta; + pLvl->iOff = ii; } - pLvl->iLeafPgno -= nZero; - pLvl->iOff = iOff - nZero; } return pLvl->bEof; @@ -185340,7 +243952,7 @@ static int fts5DlidxIterPrevR(Fts5Index *p, Fts5DlidxIter *pIter, int iLvl){ if( pLvl[1].bEof==0 ){ fts5DataRelease(pLvl->pData); memset(pLvl, 0, sizeof(Fts5DlidxLvl)); - pLvl->pData = fts5DataRead(p, + pLvl->pData = fts5DataRead(p, FTS5_DLIDX_ROWID(pIter->iSegid, iLvl, pLvl[1].iLeafPgno) ); if( pLvl->pData ){ @@ -185381,10 +243993,10 @@ static Fts5DlidxIter *fts5DlidxIterInit( int bDone = 0; for(i=0; p->rc==SQLITE_OK && bDone==0; i++){ - int nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); + sqlite3_int64 nByte = sizeof(Fts5DlidxIter) + i * sizeof(Fts5DlidxLvl); Fts5DlidxIter *pNew; - pNew = (Fts5DlidxIter*)sqlite3_realloc(pIter, nByte); + pNew = (Fts5DlidxIter*)sqlite3_realloc64(pIter, nByte); if( pNew==0 ){ p->rc = SQLITE_NOMEM; }else{ @@ -185439,7 +244051,7 @@ static void fts5SegIterNextPage( pIter->pLeaf = pIter->pNextLeaf; pIter->pNextLeaf = 0; }else if( pIter->iLeafPgno<=pSeg->pgnoLast ){ - pIter->pLeaf = fts5DataRead(p, + pIter->pLeaf = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->iSegid, pIter->iLeafPgno) ); }else{ @@ -185483,7 +244095,7 @@ static int fts5GetPoslistSize(const u8 *p, int *pnSz, int *pbDel){ ** Fts5SegIter.nPos ** Fts5SegIter.bDel ** -** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the +** Leave Fts5SegIter.iLeafOffset pointing to the first byte of the ** position list content (if any). */ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ @@ -185517,10 +244129,10 @@ static void fts5SegIterLoadNPos(Fts5Index *p, Fts5SegIter *pIter){ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ - int iOff = pIter->iLeafOffset; + i64 iOff = pIter->iLeafOffset; ASSERT_SZLEAF_OK(pIter->pLeaf); - if( iOff>=pIter->pLeaf->szLeaf ){ + while( iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( pIter->pLeaf==0 ){ if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; @@ -185534,7 +244146,7 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ } /* -** Fts5SegIter.iLeafOffset currently points to the first byte of the +** Fts5SegIter.iLeafOffset currently points to the first byte of the ** "nSuffix" field of a term. Function parameter nKeep contains the value ** of the "nPrefix" field (if there was one - it is passed 0 if this is ** the first term in the segment). @@ -185545,21 +244157,22 @@ static void fts5SegIterLoadRowid(Fts5Index *p, Fts5SegIter *pIter){ ** Fts5SegIter.rowid ** ** accordingly and leaves (Fts5SegIter.iLeafOffset) set to the content of -** the first position list. The position list belonging to document +** the first position list. The position list belonging to document ** (Fts5SegIter.iRowid). */ static void fts5SegIterLoadTerm(Fts5Index *p, Fts5SegIter *pIter, int nKeep){ u8 *a = pIter->pLeaf->p; /* Buffer to read data from */ - int iOff = pIter->iLeafOffset; /* Offset to read at */ + i64 iOff = pIter->iLeafOffset; /* Offset to read at */ int nNew; /* Bytes of new data */ iOff += fts5GetVarint32(&a[iOff], nNew); - if( iOff+nNew>pIter->pLeaf->nn ){ + if( iOff+nNew>pIter->pLeaf->szLeaf || nKeep>pIter->term.n || nNew==0 ){ p->rc = FTS5_CORRUPT; return; } pIter->term.n = nKeep; fts5BufferAppendBlob(&p->rc, &pIter->term, nNew, &a[iOff]); + assert( pIter->term.n<=pIter->term.nSpace ); iOff += nNew; pIter->iTermLeafOffset = iOff; pIter->iTermLeafPgno = pIter->iLeafPgno; @@ -185590,12 +244203,31 @@ static void fts5SegIterSetNext(Fts5Index *p, Fts5SegIter *pIter){ } } +/* +** Allocate a tombstone hash page array object (pIter->pTombArray) for +** the iterator passed as the second argument. If an OOM error occurs, +** leave an error in the Fts5Index object. +*/ +static void fts5SegIterAllocTombstone(Fts5Index *p, Fts5SegIter *pIter){ + const int nTomb = pIter->pSeg->nPgTombstone; + if( nTomb>0 ){ + int nByte = nTomb * sizeof(Fts5Data*) + sizeof(Fts5TombstoneArray); + Fts5TombstoneArray *pNew; + pNew = (Fts5TombstoneArray*)sqlite3Fts5MallocZero(&p->rc, nByte); + if( pNew ){ + pNew->nTombstone = nTomb; + pNew->nRef = 1; + pIter->pTombArray = pNew; + } + } +} + /* ** Initialize the iterator object pIter to iterate through the entries in -** segment pSeg. The iterator is left pointing to the first entry when +** segment pSeg. The iterator is left pointing to the first entry when ** this function returns. ** -** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterInit( @@ -185618,16 +244250,20 @@ static void fts5SegIterInit( fts5SegIterSetNext(p, pIter); pIter->pSeg = pSeg; pIter->iLeafPgno = pSeg->pgnoFirst-1; - fts5SegIterNextPage(p, pIter); + do { + fts5SegIterNextPage(p, pIter); + }while( p->rc==SQLITE_OK && pIter->pLeaf && pIter->pLeaf->nn==4 ); } - if( p->rc==SQLITE_OK ){ + if( p->rc==SQLITE_OK && pIter->pLeaf ){ pIter->iLeafOffset = 4; + assert( pIter->pLeaf!=0 ); assert_nc( pIter->pLeaf->nn>4 ); - assert( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); + assert_nc( fts5LeafFirstTermOff(pIter->pLeaf)==4 ); pIter->iPgidxOff = pIter->pLeaf->szLeaf+1; fts5SegIterLoadTerm(p, pIter, 0); fts5SegIterLoadNPos(p, pIter); + fts5SegIterAllocTombstone(p, pIter); } } @@ -185640,8 +244276,8 @@ static void fts5SegIterInit( ** the position-list size field for the first relevant rowid on the page. ** Fts5SegIter.rowid is set, but nPos and bDel are not. ** -** This function advances the iterator so that it points to the last -** relevant rowid on the page and, if necessary, initializes the +** This function advances the iterator so that it points to the last +** relevant rowid on the page and, if necessary, initializes the ** aRowidOffset[] and iRowidOffset variables. At this point the iterator ** is in its regular state - Fts5SegIter.iLeafOffset points to the first ** byte of the position list content associated with said rowid. @@ -185659,7 +244295,7 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ ASSERT_SZLEAF_OK(pIter->pLeaf); while( 1 ){ - i64 iDelta = 0; + u64 iDelta = 0; if( eDetail==FTS5_DETAIL_NONE ){ /* todo */ @@ -185674,13 +244310,13 @@ static void fts5SegIterReverseInitPage(Fts5Index *p, Fts5SegIter *pIter){ i += nPos; } if( i>=n ) break; - i += fts5GetVarint(&a[i], (u64*)&iDelta); + i += fts5GetVarint(&a[i], &iDelta); pIter->iRowid += iDelta; /* If necessary, grow the pIter->aRowidOffset[] array. */ if( iRowidOffset>=pIter->nRowidOffset ){ int nNew = pIter->nRowidOffset + 8; - int *aNew = (int*)sqlite3_realloc(pIter->aRowidOffset, nNew*sizeof(int)); + int *aNew = (int*)sqlite3_realloc64(pIter->aRowidOffset,nNew*sizeof(int)); if( aNew==0 ){ p->rc = SQLITE_NOMEM; break; @@ -185725,8 +244361,12 @@ static void fts5SegIterReverseNewPage(Fts5Index *p, Fts5SegIter *pIter){ int iRowidOff; iRowidOff = fts5LeafFirstRowidOff(pNew); if( iRowidOff ){ - pIter->pLeaf = pNew; - pIter->iLeafOffset = iRowidOff; + if( iRowidOff>=pNew->szLeaf ){ + p->rc = FTS5_CORRUPT; + }else{ + pIter->pLeaf = pNew; + pIter->iLeafOffset = iRowidOff; + } } } @@ -185773,7 +244413,7 @@ static void fts5SegIterNext_Reverse( if( pIter->iRowidOffset>0 ){ u8 *a = pIter->pLeaf->p; int iOff; - i64 iDelta; + u64 iDelta; pIter->iRowidOffset--; pIter->iLeafOffset = pIter->aRowidOffset[pIter->iRowidOffset]; @@ -185782,7 +244422,7 @@ static void fts5SegIterNext_Reverse( if( p->pConfig->eDetail!=FTS5_DETAIL_NONE ){ iOff += pIter->nPos; } - fts5GetVarint(&a[iOff], (u64*)&iDelta); + fts5GetVarint(&a[iOff], &iDelta); pIter->iRowid -= iDelta; }else{ fts5SegIterReverseNewPage(p, pIter); @@ -185810,7 +244450,7 @@ static void fts5SegIterNext_None( iOff = pIter->iLeafOffset; /* Next entry is on the next page */ - if( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ + while( pIter->pSeg && iOff>=pIter->pLeaf->szLeaf ){ fts5SegIterNextPage(p, pIter); if( p->rc || pIter->pLeaf==0 ) return; pIter->iRowid = 0; @@ -185819,7 +244459,7 @@ static void fts5SegIterNext_None( if( iOffiEndofDoclist ){ /* Next entry is on the current page */ - i64 iDelta; + u64 iDelta; iOff += sqlite3Fts5GetVarint(&pIter->pLeaf->p[iOff], (u64*)&iDelta); pIter->iLeafOffset = iOff; pIter->iRowid += iDelta; @@ -185834,15 +244474,16 @@ static void fts5SegIterNext_None( }else{ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList; sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); if( pList==0 ) goto next_none_eof; pIter->pLeaf->p = (u8*)pList; pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList; - sqlite3Fts5BufferSet(&p->rc,&pIter->term, (int)strlen(zTerm), (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc,&pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); } @@ -185861,10 +244502,10 @@ static void fts5SegIterNext_None( /* -** Advance iterator pIter to the next entry. +** Advance iterator pIter to the next entry. ** -** If an error occurs, Fts5Index.rc is set to an appropriate error code. It -** is not considered an error if the iterator reaches EOF. If an error has +** If an error occurs, Fts5Index.rc is set to an appropriate error code. It +** is not considered an error if the iterator reaches EOF. If an error has ** already occurred when this function is called, it is a no-op. */ static void fts5SegIterNext( @@ -185908,11 +244549,12 @@ static void fts5SegIterNext( }else if( pIter->pSeg==0 ){ const u8 *pList = 0; const char *zTerm = 0; + int nTerm = 0; int nList = 0; assert( (pIter->flags & FTS5_SEGITER_ONETERM) || pbNewTerm ); if( 0==(pIter->flags & FTS5_SEGITER_ONETERM) ){ sqlite3Fts5HashScanNext(p->pHash); - sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &pList, &nList); + sqlite3Fts5HashScanEntry(p->pHash, &zTerm, &nTerm, &pList, &nList); } if( pList==0 ){ fts5DataRelease(pIter->pLeaf); @@ -185922,8 +244564,7 @@ static void fts5SegIterNext( pIter->pLeaf->nn = nList; pIter->pLeaf->szLeaf = nList; pIter->iEndofDoclist = nList+1; - sqlite3Fts5BufferSet(&p->rc, &pIter->term, (int)strlen(zTerm), - (u8*)zTerm); + sqlite3Fts5BufferSet(&p->rc, &pIter->term, nTerm, (u8*)zTerm); pIter->iLeafOffset = fts5GetVarint(pList, (u64*)&pIter->iRowid); *pbNewTerm = 1; } @@ -185942,14 +244583,13 @@ static void fts5SegIterNext( if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], pIter->iEndofDoclist - ); + ); } - } else if( pLeaf->nn>pLeaf->szLeaf ){ pIter->iPgidxOff = pLeaf->szLeaf + fts5GetVarint32( &pLeaf->p[pLeaf->szLeaf], iOff - ); + ); pIter->iLeafOffset = iOff; pIter->iEndofDoclist = iOff; bNewTerm = 1; @@ -185976,13 +244616,9 @@ static void fts5SegIterNext( }else{ /* The following could be done by calling fts5SegIterLoadNPos(). But ** this block is particularly performance critical, so equivalent - ** code is inlined. - ** - ** Later: Switched back to fts5SegIterLoadNPos() because it supports - ** detail=none mode. Not ideal. - */ + ** code is inlined. */ int nSz; - assert( p->rc==SQLITE_OK ); + assert_nc( pIter->iLeafOffset<=pIter->pLeaf->nn ); fts5FastGetVarint32(pIter->pLeaf->p, pIter->iLeafOffset, nSz); pIter->bDel = (nSz & 0x0001); pIter->nPos = nSz>>1; @@ -186008,10 +244644,10 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ Fts5Data *pLast = 0; int pgnoLast = 0; - if( pDlidx ){ + if( pDlidx && p->pConfig->iVersion==FTS5_CURRENT_VERSION ){ int iSegid = pIter->pSeg->iSegid; pgnoLast = fts5DlidxIterPgno(pDlidx); - pLast = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); + pLast = fts5LeafRead(p, FTS5_SEGMENT_ROWID(iSegid, pgnoLast)); }else{ Fts5Data *pLeaf = pIter->pLeaf; /* Current leaf data */ @@ -186038,7 +244674,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ ** forward to find the page containing the last rowid. */ for(pgno=pIter->iLeafPgno+1; !p->rc && pgno<=pSeg->pgnoLast; pgno++){ i64 iAbs = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); - Fts5Data *pNew = fts5DataRead(p, iAbs); + Fts5Data *pNew = fts5LeafRead(p, iAbs); if( pNew ){ int iRowid, bTermless; iRowid = fts5LeafFirstRowidOff(pNew); @@ -186055,7 +244691,7 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ } /* If pLast is NULL at this point, then the last rowid for this doclist - ** lies on the page currently indicated by the iterator. In this case + ** lies on the page currently indicated by the iterator. In this case ** pIter->iLeafOffset is already set to point to the position-list size ** field associated with the first relevant rowid on the page. ** @@ -186069,6 +244705,10 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ pIter->pLeaf = pLast; pIter->iLeafPgno = pgnoLast; iOff = fts5LeafFirstRowidOff(pLast); + if( iOff>pLast->szLeaf ){ + p->rc = FTS5_CORRUPT; + return; + } iOff += fts5GetVarint(&pLast->p[iOff], (u64*)&pIter->iRowid); pIter->iLeafOffset = iOff; @@ -186077,7 +244717,6 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ }else{ pIter->iEndofDoclist = fts5LeafFirstTermOff(pLast); } - } fts5SegIterReverseInitPage(p, pIter); @@ -186085,8 +244724,8 @@ static void fts5SegIterReverse(Fts5Index *p, Fts5SegIter *pIter){ /* ** Iterator pIter currently points to the first rowid of a doclist. -** There is a doclist-index associated with the final term on the current -** page. If the current term is the last term on the page, load the +** There is a doclist-index associated with the final term on the current +** page. If the current term is the last term on the page, load the ** doclist-index from disk and initialize an iterator at (pIter->pDlidx). */ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ @@ -186100,8 +244739,8 @@ static void fts5SegIterLoadDlidx(Fts5Index *p, Fts5SegIter *pIter){ /* Check if the current doclist ends on this page. If it does, return ** early without loading the doclist-index (as it belongs to a different ** term. */ - if( pIter->iTermLeafPgno==pIter->iLeafPgno - && pIter->iEndofDoclistszLeaf + if( pIter->iTermLeafPgno==pIter->iLeafPgno + && pIter->iEndofDoclistszLeaf ){ return; } @@ -186129,21 +244768,20 @@ static void fts5LeafSeek( Fts5SegIter *pIter, /* Iterator to seek */ const u8 *pTerm, int nTerm /* Term to search for */ ){ - int iOff; + u32 iOff; const u8 *a = pIter->pLeaf->p; - int szLeaf = pIter->pLeaf->szLeaf; - int n = pIter->pLeaf->nn; + u32 n = (u32)pIter->pLeaf->nn; - int nMatch = 0; - int nKeep = 0; - int nNew = 0; - int iTermOff; - int iPgidx; /* Current offset in pgidx */ + u32 nMatch = 0; + u32 nKeep = 0; + u32 nNew = 0; + u32 iTermOff; + u32 iPgidx; /* Current offset in pgidx */ int bEndOfPage = 0; assert( p->rc==SQLITE_OK ); - iPgidx = szLeaf; + iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&a[iPgidx], iTermOff); iOff = iTermOff; if( iOff>n ){ @@ -186161,15 +244799,15 @@ static void fts5LeafSeek( assert( nKeep>=nMatch ); if( nKeep==nMatch ){ - int nCmp; - int i; - nCmp = MIN(nNew, nTerm-nMatch); + u32 nCmp; + u32 i; + nCmp = (u32)MIN(nNew, nTerm-nMatch); for(i=0; i=n ){ + p->rc = FTS5_CORRUPT; + return; + } + /* Read the nKeep field of the next term. */ fts5FastGetVarint32(a, iOff, nKeep); } @@ -186204,14 +244847,15 @@ static void fts5LeafSeek( if( pIter->pLeaf==0 ) return; a = pIter->pLeaf->p; if( fts5LeafIsTermless(pIter->pLeaf)==0 ){ - iPgidx = pIter->pLeaf->szLeaf; + iPgidx = (u32)pIter->pLeaf->szLeaf; iPgidx += fts5GetVarint32(&pIter->pLeaf->p[iPgidx], iOff); - if( iOff<4 || iOff>=pIter->pLeaf->szLeaf ){ + if( iOff<4 || (i64)iOff>=pIter->pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; + return; }else{ nKeep = 0; iTermOff = iOff; - n = pIter->pLeaf->nn; + n = (u32)pIter->pLeaf->nn; iOff += fts5GetVarint32(&a[iOff], nNew); break; } @@ -186220,7 +244864,10 @@ static void fts5LeafSeek( } search_success: - + if( (i64)iOff+nNew>n || nNew<1 ){ + p->rc = FTS5_CORRUPT; + return; + } pIter->iLeafOffset = iOff + nNew; pIter->iTermLeafOffset = pIter->iLeafOffset; pIter->iTermLeafPgno = pIter->iLeafPgno; @@ -186257,7 +244904,7 @@ static sqlite3_stmt *fts5IdxSelectStmt(Fts5Index *p){ ** Initialize the object pIter to point to term pTerm/nTerm within segment ** pSeg. If there is no such term in the index, the iterator is set to EOF. ** -** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterSeekInit( @@ -186289,6 +244936,7 @@ static void fts5SegIterSeekInit( bDlidx = (val & 0x0001); } p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); if( iPgpgnoFirst ){ iPg = pSeg->pgnoFirst; @@ -186302,7 +244950,7 @@ static void fts5SegIterSeekInit( fts5LeafSeek(p, bGe, pIter, pTerm, nTerm); } - if( p->rc==SQLITE_OK && bGe==0 ){ + if( p->rc==SQLITE_OK && (bGe==0 || (flags & FTS5INDEX_QUERY_SCANONETERM)) ){ pIter->flags |= FTS5_SEGITER_ONETERM; if( pIter->pLeaf ){ if( flags & FTS5INDEX_QUERY_DESC ){ @@ -186318,6 +244966,9 @@ static void fts5SegIterSeekInit( } fts5SegIterSetNext(p, pIter); + if( 0==(flags & FTS5INDEX_QUERY_SCANONETERM) ){ + fts5SegIterAllocTombstone(p, pIter); + } /* Either: ** @@ -186327,19 +244978,92 @@ static void fts5SegIterSeekInit( ** 4) the FTS5INDEX_QUERY_SCAN flag was set and the iterator points ** to an entry with a term greater than or equal to (pTerm/nTerm). */ - assert( p->rc!=SQLITE_OK /* 1 */ + assert_nc( p->rc!=SQLITE_OK /* 1 */ || pIter->pLeaf==0 /* 2 */ || fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)==0 /* 3 */ || (bGe && fts5BufferCompareBlob(&pIter->term, pTerm, nTerm)>0) /* 4 */ ); } + +/* +** SQL used by fts5SegIterNextInit() to find the page to open. +*/ +static sqlite3_stmt *fts5IdxNextStmt(Fts5Index *p){ + if( p->pIdxNextSelect==0 ){ + Fts5Config *pConfig = p->pConfig; + fts5IndexPrepareStmt(p, &p->pIdxNextSelect, sqlite3_mprintf( + "SELECT pgno FROM '%q'.'%q_idx' WHERE " + "segid=? AND term>? ORDER BY term ASC LIMIT 1", + pConfig->zDb, pConfig->zName + )); + + } + return p->pIdxNextSelect; +} + +/* +** This is similar to fts5SegIterSeekInit(), except that it initializes +** the segment iterator to point to the first term following the page +** with pToken/nToken on it. +*/ +static void fts5SegIterNextInit( + Fts5Index *p, + const char *pTerm, int nTerm, + Fts5StructureSegment *pSeg, /* Description of segment */ + Fts5SegIter *pIter /* Object to populate */ +){ + int iPg = -1; /* Page of segment to open */ + int bDlidx = 0; + sqlite3_stmt *pSel = 0; /* SELECT to find iPg */ + + pSel = fts5IdxNextStmt(p); + if( pSel ){ + assert( p->rc==SQLITE_OK ); + sqlite3_bind_int(pSel, 1, pSeg->iSegid); + sqlite3_bind_blob(pSel, 2, pTerm, nTerm, SQLITE_STATIC); + + if( sqlite3_step(pSel)==SQLITE_ROW ){ + i64 val = sqlite3_column_int64(pSel, 0); + iPg = (int)(val>>1); + bDlidx = (val & 0x0001); + } + p->rc = sqlite3_reset(pSel); + sqlite3_bind_null(pSel, 2); + if( p->rc ) return; + } + + memset(pIter, 0, sizeof(*pIter)); + pIter->pSeg = pSeg; + pIter->flags |= FTS5_SEGITER_ONETERM; + if( iPg>=0 ){ + pIter->iLeafPgno = iPg - 1; + fts5SegIterNextPage(p, pIter); + fts5SegIterSetNext(p, pIter); + } + if( pIter->pLeaf ){ + const u8 *a = pIter->pLeaf->p; + int iTermOff = 0; + + pIter->iPgidxOff = pIter->pLeaf->szLeaf; + pIter->iPgidxOff += fts5GetVarint32(&a[pIter->iPgidxOff], iTermOff); + pIter->iLeafOffset = iTermOff; + fts5SegIterLoadTerm(p, pIter, 0); + fts5SegIterLoadNPos(p, pIter); + if( bDlidx ) fts5SegIterLoadDlidx(p, pIter); + + assert( p->rc!=SQLITE_OK || + fts5BufferCompareBlob(&pIter->term, (const u8*)pTerm, nTerm)>0 + ); + } +} + /* ** Initialize the object pIter to point to term pTerm/nTerm within the -** in-memory hash table. If there is no such term in the hash-table, the +** in-memory hash table. If there is no such term in the hash-table, the ** iterator is set to EOF. ** -** If an error occurs, Fts5Index.rc is set to an appropriate error code. If +** If an error occurs, Fts5Index.rc is set to an appropriate error code. If ** an error has already occurred when this function is called, it is a no-op. */ static void fts5SegIterHashInit( @@ -186348,31 +245072,47 @@ static void fts5SegIterHashInit( int flags, /* Mask of FTS5INDEX_XXX flags */ Fts5SegIter *pIter /* Object to populate */ ){ - const u8 *pList = 0; int nList = 0; const u8 *z = 0; int n = 0; + Fts5Data *pLeaf = 0; assert( p->pHash ); assert( p->rc==SQLITE_OK ); if( pTerm==0 || (flags & FTS5INDEX_QUERY_SCAN) ){ + const u8 *pList = 0; + p->rc = sqlite3Fts5HashScanInit(p->pHash, (const char*)pTerm, nTerm); - sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &pList, &nList); - n = (z ? (int)strlen((const char*)z) : 0); + sqlite3Fts5HashScanEntry(p->pHash, (const char**)&z, &n, &pList, &nList); + if( pList ){ + pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); + if( pLeaf ){ + pLeaf->p = (u8*)pList; + } + } + + /* The call to sqlite3Fts5HashScanInit() causes the hash table to + ** fill the size field of all existing position lists. This means they + ** can no longer be appended to. Since the only scenario in which they + ** can be appended to is if the previous operation on this table was + ** a DELETE, by clearing the Fts5Index.bDelete flag we can avoid this + ** possibility altogether. */ + p->bDelete = 0; }else{ - pIter->flags |= FTS5_SEGITER_ONETERM; - sqlite3Fts5HashQuery(p->pHash, (const char*)pTerm, nTerm, &pList, &nList); + p->rc = sqlite3Fts5HashQuery(p->pHash, sizeof(Fts5Data), + (const char*)pTerm, nTerm, (void**)&pLeaf, &nList + ); + if( pLeaf ){ + pLeaf->p = (u8*)&pLeaf[1]; + } z = pTerm; n = nTerm; + pIter->flags |= FTS5_SEGITER_ONETERM; } - if( pList ){ - Fts5Data *pLeaf; + if( pLeaf ){ sqlite3Fts5BufferSet(&p->rc, &pIter->term, n, z); - pLeaf = fts5IdxMalloc(p, sizeof(Fts5Data)); - if( pLeaf==0 ) return; - pLeaf->p = (u8*)pList; pLeaf->nn = pLeaf->szLeaf = nList; pIter->pLeaf = pLeaf; pIter->iLeafOffset = fts5GetVarint(pLeaf->p, (u64*)&pIter->iRowid); @@ -186389,6 +245129,37 @@ static void fts5SegIterHashInit( fts5SegIterSetNext(p, pIter); } +/* +** Array ap[] contains n elements. Release each of these elements using +** fts5DataRelease(). Then free the array itself using sqlite3_free(). +*/ +static void fts5IndexFreeArray(Fts5Data **ap, int n){ + if( ap ){ + int ii; + for(ii=0; iinRef--; + if( p->nRef<=0 ){ + int ii; + for(ii=0; iinTombstone; ii++){ + fts5DataRelease(p->apTombstone[ii]); + } + sqlite3_free(p); + } + } +} + /* ** Zero the iterator passed as the only argument. */ @@ -186396,6 +245167,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){ fts5BufferFree(&pIter->term); fts5DataRelease(pIter->pLeaf); fts5DataRelease(pIter->pNextLeaf); + fts5TombstoneArrayDelete(pIter->pTombArray); fts5DlidxIterFree(pIter->pDlidx); sqlite3_free(pIter->aRowidOffset); memset(pIter, 0, sizeof(Fts5SegIter)); @@ -186410,7 +245182,7 @@ static void fts5SegIterClear(Fts5SegIter *pIter){ ** two iterators. */ static void fts5AssertComparisonResult( - Fts5Iter *pIter, + Fts5Iter *pIter, Fts5SegIter *p1, Fts5SegIter *p2, Fts5CResult *pRes @@ -186425,7 +245197,7 @@ static void fts5AssertComparisonResult( assert( pRes->iFirst==i1 ); }else{ int nMin = MIN(p1->term.n, p2->term.n); - int res = memcmp(p1->term.p, p2->term.p, nMin); + int res = fts5Memcmp(p1->term.p, p2->term.p, nMin); if( res==0 ) res = p1->term.n - p2->term.n; if( res==0 ){ @@ -186447,7 +245219,7 @@ static void fts5AssertComparisonResult( /* ** This function is a no-op unless SQLITE_DEBUG is defined when this module -** is compiled. In that case, this function is essentially an assert() +** is compiled. In that case, this function is essentially an assert() ** statement used to verify that the contents of the pIter->aFirst[] array ** are correct. */ @@ -186461,9 +245233,9 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ /* Check that pIter->iSwitchRowid is set correctly. */ for(i=0; inSeg; i++){ Fts5SegIter *p1 = &pIter->aSeg[i]; - assert( p1==pFirst - || p1->pLeaf==0 - || fts5BufferCompare(&pFirst->term, &p1->term) + assert( p1==pFirst + || p1->pLeaf==0 + || fts5BufferCompare(&pFirst->term, &p1->term) || p1->iRowid==pIter->iSwitchRowid || (p1->iRowidiSwitchRowid)==pIter->bRev ); @@ -186493,7 +245265,7 @@ static void fts5AssertMultiIterSetup(Fts5Index *p, Fts5Iter *pIter){ ** ** If the returned value is non-zero, then it is the index of an entry ** in the pIter->aSeg[] array that is (a) not at EOF, and (b) pointing -** to a key that is a duplicate of another, higher priority, +** to a key that is a duplicate of another, higher priority, ** segment-iterator in the pSeg->aSeg[] array. */ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ @@ -186525,11 +245297,10 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ }else{ int res = fts5BufferCompare(&p1->term, &p2->term); if( res==0 ){ - assert( i2>i1 ); - assert( i2!=0 ); + assert_nc( i2>i1 ); + assert_nc( i2!=0 ); pRes->bTermEq = 1; if( p1->iRowid==p2->iRowid ){ - p1->bDel = p2->bDel; return i2; } res = ((p1->iRowid > p2->iRowid)==pIter->bRev) ? -1 : +1; @@ -186548,7 +245319,8 @@ static int fts5MultiIterDoCompare(Fts5Iter *pIter, int iOut){ /* ** Move the seg-iter so that it points to the first rowid on page iLeafPgno. -** It is an error if leaf iLeafPgno does not exist or contains no rowids. +** It is an error if leaf iLeafPgno does not exist. Unless the db is +** a 'secure-delete' db, if it contains no rowids then this is also an error. */ static void fts5SegIterGotoPage( Fts5Index *p, /* FTS5 backend object */ @@ -186563,28 +245335,30 @@ static void fts5SegIterGotoPage( fts5DataRelease(pIter->pNextLeaf); pIter->pNextLeaf = 0; pIter->iLeafPgno = iLeafPgno-1; - fts5SegIterNextPage(p, pIter); - assert( p->rc!=SQLITE_OK || pIter->iLeafPgno==iLeafPgno ); - if( p->rc==SQLITE_OK ){ + while( p->rc==SQLITE_OK ){ int iOff; - u8 *a = pIter->pLeaf->p; - int n = pIter->pLeaf->szLeaf; - + fts5SegIterNextPage(p, pIter); + if( pIter->pLeaf==0 ) break; iOff = fts5LeafFirstRowidOff(pIter->pLeaf); - if( iOff<4 || iOff>=n ){ - p->rc = FTS5_CORRUPT; - }else{ - iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); - pIter->iLeafOffset = iOff; - fts5SegIterLoadNPos(p, pIter); + if( iOff>0 ){ + u8 *a = pIter->pLeaf->p; + int n = pIter->pLeaf->szLeaf; + if( iOff<4 || iOff>=n ){ + p->rc = FTS5_CORRUPT; + }else{ + iOff += fts5GetVarint(&a[iOff], (u64*)&pIter->iRowid); + pIter->iLeafOffset = iOff; + fts5SegIterLoadNPos(p, pIter); + } + break; } } } } /* -** Advance the iterator passed as the second argument until it is at or +** Advance the iterator passed as the second argument until it is at or ** past rowid iFrom. Regardless of the value of iFrom, the iterator is ** always advanced at least once. */ @@ -186638,7 +245412,6 @@ static void fts5SegIterNextFrom( }while( p->rc==SQLITE_OK ); } - /* ** Free the iterator object passed as the second argument. */ @@ -186648,7 +245421,6 @@ static void fts5MultiIterFree(Fts5Iter *pIter){ for(i=0; inSeg; i++){ fts5SegIterClear(&pIter->aSeg[i]); } - fts5StructureRelease(pIter->pStruct); fts5BufferFree(&pIter->poslist); sqlite3_free(pIter); } @@ -186681,7 +245453,7 @@ static void fts5MultiIterAdvanced( ** If non-zero is returned, the caller should call fts5MultiIterAdvanced() ** on the iterator instead. That function does the same as this one, except ** that it deals with more complicated cases as well. -*/ +*/ static int fts5MultiIterAdvanceRowid( Fts5Iter *pIter, /* Iterator to update aFirst[] array for */ int iChanged, /* Index of sub-iterator just advanced */ @@ -186732,19 +245504,99 @@ static void fts5MultiIterSetEof(Fts5Iter *pIter){ } /* -** Move the iterator to the next entry. +** The argument to this macro must be an Fts5Data structure containing a +** tombstone hash page. This macro returns the key-size of the hash-page. +*/ +#define TOMBSTONE_KEYSIZE(pPg) (pPg->p[0]==4 ? 4 : 8) + +#define TOMBSTONE_NSLOT(pPg) \ + ((pPg->nn > 16) ? ((pPg->nn-8) / TOMBSTONE_KEYSIZE(pPg)) : 1) + +/* +** Query a single tombstone hash table for rowid iRowid. Return true if +** it is found or false otherwise. The tombstone hash table is one of +** nHashTable tables. +*/ +static int fts5IndexTombstoneQuery( + Fts5Data *pHash, /* Hash table page to query */ + int nHashTable, /* Number of pages attached to segment */ + u64 iRowid /* Rowid to query hash for */ +){ + const int szKey = TOMBSTONE_KEYSIZE(pHash); + const int nSlot = TOMBSTONE_NSLOT(pHash); + int iSlot = (iRowid / nHashTable) % nSlot; + int nCollide = nSlot; + + if( iRowid==0 ){ + return pHash->p[1]; + }else if( szKey==4 ){ + u32 *aSlot = (u32*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU32((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + }else{ + u64 *aSlot = (u64*)&pHash->p[8]; + while( aSlot[iSlot] ){ + if( fts5GetU64((u8*)&aSlot[iSlot])==iRowid ) return 1; + if( nCollide--==0 ) break; + iSlot = (iSlot+1)%nSlot; + } + } + + return 0; +} + +/* +** Return true if the iterator passed as the only argument points +** to an segment entry for which there is a tombstone. Return false +** if there is no tombstone or if the iterator is already at EOF. +*/ +static int fts5MultiIterIsDeleted(Fts5Iter *pIter){ + int iFirst = pIter->aFirst[1].iFirst; + Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; + Fts5TombstoneArray *pArray = pSeg->pTombArray; + + if( pSeg->pLeaf && pArray ){ + /* Figure out which page the rowid might be present on. */ + int iPg = ((u64)pSeg->iRowid) % pArray->nTombstone; + assert( iPg>=0 ); + + /* If tombstone hash page iPg has not yet been loaded from the + ** database, load it now. */ + if( pArray->apTombstone[iPg]==0 ){ + pArray->apTombstone[iPg] = fts5DataRead(pIter->pIndex, + FTS5_TOMBSTONE_ROWID(pSeg->pSeg->iSegid, iPg) + ); + if( pArray->apTombstone[iPg]==0 ) return 0; + } + + return fts5IndexTombstoneQuery( + pArray->apTombstone[iPg], + pArray->nTombstone, + pSeg->iRowid + ); + } + + return 0; +} + +/* +** Move the iterator to the next entry. ** -** If an error occurs, an error code is left in Fts5Index.rc. It is not -** considered an error if the iterator reaches EOF, or if it is already at +** If an error occurs, an error code is left in Fts5Index.rc. It is not +** considered an error if the iterator reaches EOF, or if it is already at ** EOF when this function is called. */ static void fts5MultiIterNext( - Fts5Index *p, + Fts5Index *p, Fts5Iter *pIter, int bFrom, /* True if argument iFrom is valid */ i64 iFrom /* Advance at least as far as this */ ){ int bUseFrom = bFrom; + assert( pIter->base.bEof==0 ); while( p->rc==SQLITE_OK ){ int iFirst = pIter->aFirst[1].iFirst; int bNewTerm = 0; @@ -186756,7 +245608,7 @@ static void fts5MultiIterNext( pSeg->xNext(p, pSeg, &bNewTerm); } - if( pSeg->pLeaf==0 || bNewTerm + if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); @@ -186767,7 +245619,9 @@ static void fts5MultiIterNext( fts5AssertMultiIterSetup(p, pIter); assert( pSeg==&pIter->aSeg[pIter->aFirst[1].iFirst] && pSeg->pLeaf ); - if( pIter->bSkipEmpty==0 || pSeg->nPos ){ + if( (pIter->bSkipEmpty==0 || pSeg->nPos) + && 0==fts5MultiIterIsDeleted(pIter) + ){ pIter->xSetOutputs(pIter, pSeg); return; } @@ -186776,31 +245630,32 @@ static void fts5MultiIterNext( } static void fts5MultiIterNext2( - Fts5Index *p, + Fts5Index *p, Fts5Iter *pIter, int *pbNewTerm /* OUT: True if *might* be new term */ ){ assert( pIter->bSkipEmpty ); if( p->rc==SQLITE_OK ){ - do { + *pbNewTerm = 0; + do{ int iFirst = pIter->aFirst[1].iFirst; Fts5SegIter *pSeg = &pIter->aSeg[iFirst]; int bNewTerm = 0; assert( p->rc==SQLITE_OK ); pSeg->xNext(p, pSeg, &bNewTerm); - if( pSeg->pLeaf==0 || bNewTerm + if( pSeg->pLeaf==0 || bNewTerm || fts5MultiIterAdvanceRowid(pIter, iFirst, &pSeg) ){ fts5MultiIterAdvanced(p, pIter, iFirst, 1); fts5MultiIterSetEof(pIter); *pbNewTerm = 1; - }else{ - *pbNewTerm = 0; } fts5AssertMultiIterSetup(p, pIter); - }while( fts5MultiIterIsEmpty(p, pIter) ); + }while( (fts5MultiIterIsEmpty(p, pIter) || fts5MultiIterIsDeleted(pIter)) + && (p->rc==SQLITE_OK) + ); } } @@ -186813,10 +245668,10 @@ static Fts5Iter *fts5MultiIterAlloc( int nSeg ){ Fts5Iter *pNew; - int nSlot; /* Power of two >= nSeg */ + i64 nSlot; /* Power of two >= nSeg */ for(nSlot=2; nSlotaSeg[] */ sizeof(Fts5CResult) * nSlot /* pNew->aFirst[] */ @@ -186831,8 +245686,8 @@ static Fts5Iter *fts5MultiIterAlloc( } static void fts5PoslistCallback( - Fts5Index *pUnused, - void *pContext, + Fts5Index *pUnused, + void *pContext, const u8 *pChunk, int nChunk ){ UNUSED_PARAM(pUnused); @@ -186869,8 +245724,8 @@ static int fts5IndexColsetTest(Fts5Colset *pColset, int iCol){ } static void fts5PoslistOffsetsCallback( - Fts5Index *pUnused, - void *pContext, + Fts5Index *pUnused, + void *pContext, const u8 *pChunk, int nChunk ){ PoslistOffsetsCtx *pCtx = (PoslistOffsetsCtx*)pContext; @@ -186893,7 +245748,7 @@ static void fts5PoslistOffsetsCallback( static void fts5PoslistFilterCallback( Fts5Index *pUnused, - void *pContext, + void *pContext, const u8 *pChunk, int nChunk ){ PoslistCallbackCtx *pCtx = (PoslistCallbackCtx*)pContext; @@ -186956,7 +245811,7 @@ static void fts5ChunkIterate( int pgno = pSeg->iLeafPgno; int pgnoSave = 0; - /* This function does notmwork with detail=none databases. */ + /* This function does not work with detail=none databases. */ assert( p->pConfig->eDetail!=FTS5_DETAIL_NONE ); if( (pSeg->flags & FTS5_SEGITER_REVERSE)==0 ){ @@ -186969,9 +245824,12 @@ static void fts5ChunkIterate( fts5DataRelease(pData); if( nRem<=0 ){ break; + }else if( pSeg->pSeg==0 ){ + p->rc = FTS5_CORRUPT; + return; }else{ pgno++; - pData = fts5DataRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); + pData = fts5LeafRead(p, FTS5_SEGMENT_ROWID(pSeg->pSeg->iSegid, pgno)); if( pData==0 ) break; pChunk = &pData->p[4]; nChunk = MIN(nRem, pData->szLeaf - 4); @@ -186996,7 +245854,12 @@ static void fts5SegiterPoslist( Fts5Colset *pColset, Fts5Buffer *pBuf ){ - if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos) ){ + assert( pBuf!=0 ); + assert( pSeg!=0 ); + if( 0==fts5BufferGrow(&p->rc, pBuf, pSeg->nPos+FTS5_DATA_ZERO_PADDING) ){ + assert( pBuf->p!=0 ); + assert( pBuf->nSpace >= pBuf->n+pSeg->nPos+FTS5_DATA_ZERO_PADDING ); + memset(&pBuf->p[pBuf->n+pSeg->nPos], 0, FTS5_DATA_ZERO_PADDING); if( pColset==0 ){ fts5ChunkIterate(p, pSeg, (void*)pBuf, fts5PoslistCallback); }else{ @@ -187019,66 +245882,72 @@ static void fts5SegiterPoslist( } /* -** IN/OUT parameter (*pa) points to a position list n bytes in size. If -** the position list contains entries for column iCol, then (*pa) is set -** to point to the sub-position-list for that column and the number of -** bytes in it returned. Or, if the argument position list does not -** contain any entries for column iCol, return 0. +** Parameter pPos points to a buffer containing a position list, size nPos. +** This function filters it according to pColset (which must be non-NULL) +** and sets pIter->base.pData/nData to point to the new position list. +** If memory is required for the new position list, use buffer pIter->poslist. +** Or, if the new position list is a contiguous subset of the input, set +** pIter->base.pData/nData to point directly to it. +** +** This function is a no-op if *pRc is other than SQLITE_OK when it is +** called. If an OOM error is encountered, *pRc is set to SQLITE_NOMEM +** before returning. */ -static int fts5IndexExtractCol( - const u8 **pa, /* IN/OUT: Pointer to poslist */ - int n, /* IN: Size of poslist in bytes */ - int iCol /* Column to extract from poslist */ -){ - int iCurrent = 0; /* Anything before the first 0x01 is col 0 */ - const u8 *p = *pa; - const u8 *pEnd = &p[n]; /* One byte past end of position list */ - - while( iCol>iCurrent ){ - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint. Note that it is not possible for a negative - ** or extremely large varint to occur within an uncorrupted position - ** list. So the last byte of each varint may be assumed to have a clear - ** 0x80 bit. */ - while( *p!=0x01 ){ - while( *p++ & 0x80 ); - if( p>=pEnd ) return 0; - } - *pa = p++; - iCurrent = *p++; - if( iCurrent & 0x80 ){ - p--; - p += fts5GetVarint32(p, iCurrent); - } - } - if( iCol!=iCurrent ) return 0; - - /* Advance pointer p until it points to pEnd or an 0x01 byte that is - ** not part of a varint */ - while( pnCol; i++){ - const u8 *pSub = pPos; - int nSub = fts5IndexExtractCol(&pSub, nPos, pColset->aiCol[i]); - if( nSub ){ - fts5BufferAppendBlob(&rc, pBuf, nSub, pSub); + if( pColset->nCol>1 && sqlite3Fts5BufferSize(pRc, &pIter->poslist, nPos) ){ + return; + } + + while( 1 ){ + while( pColset->aiCol[i]nCol ){ + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + return; + } + } + + /* Advance pointer p until it points to pEnd or an 0x01 byte that is + ** not part of a varint */ + while( paiCol[i]==iCurrent ){ + if( pColset->nCol==1 ){ + pIter->base.pData = aCopy; + pIter->base.nData = p-aCopy; + return; + } + fts5BufferSafeAppendBlob(&pIter->poslist, aCopy, p-aCopy); + } + if( p>=pEnd ){ + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + return; + } + aCopy = p++; + iCurrent = *p++; + if( iCurrent & 0x80 ){ + p--; + p += fts5GetVarint32(p, iCurrent); + } } } - return rc; + } /* @@ -187102,7 +245971,7 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ assert( pIter->pColset==0 ); if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output + /* All data is stored on the current page. Populate the output ** variables to point into the body of the page object. */ pIter->base.pData = &pSeg->pLeaf->p[pSeg->iLeafOffset]; }else{ @@ -187115,6 +245984,15 @@ static void fts5IterSetOutputs_Nocolset(Fts5Iter *pIter, Fts5SegIter *pSeg){ } } +/* +** xSetOutputs callback used when the Fts5Colset object has nCol==0 (match +** against no columns at all). +*/ +static void fts5IterSetOutputs_ZeroColset(Fts5Iter *pIter, Fts5SegIter *pSeg){ + UNUSED_PARAM(pSeg); + pIter->base.nData = 0; +} + /* ** xSetOutputs callback used by detail=col when there is a column filter ** and there are 100 or more columns. Also called as a fallback from @@ -187129,13 +246007,13 @@ static void fts5IterSetOutputs_Col(Fts5Iter *pIter, Fts5SegIter *pSeg){ } /* -** xSetOutputs callback used when: +** xSetOutputs callback used when: ** ** * detail=col, ** * there is a column filter, and -** * the table contains 100 or fewer columns. +** * the table contains 100 or fewer columns. ** -** The last point is to ensure all column numbers are stored as +** The last point is to ensure all column numbers are stored as ** single-byte varints. */ static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ @@ -187147,7 +246025,7 @@ static void fts5IterSetOutputs_Col100(Fts5Iter *pIter, Fts5SegIter *pSeg){ fts5IterSetOutputs_Col(pIter, pSeg); }else{ u8 *a = (u8*)&pSeg->pLeaf->p[pSeg->iLeafOffset]; - u8 *pEnd = (u8*)&a[pSeg->nPos]; + u8 *pEnd = (u8*)&a[pSeg->nPos]; int iPrev = 0; int *aiCol = pIter->pColset->aiCol; int *aiColEnd = &aiCol[pIter->pColset->nCol]; @@ -187186,18 +246064,12 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ assert( pColset ); if( pSeg->iLeafOffset+pSeg->nPos<=pSeg->pLeaf->szLeaf ){ - /* All data is stored on the current page. Populate the output + /* All data is stored on the current page. Populate the output ** variables to point into the body of the page object. */ const u8 *a = &pSeg->pLeaf->p[pSeg->iLeafOffset]; - if( pColset->nCol==1 ){ - pIter->base.nData = fts5IndexExtractCol(&a, pSeg->nPos,pColset->aiCol[0]); - pIter->base.pData = a; - }else{ - fts5BufferZero(&pIter->poslist); - fts5IndexExtractColset(pColset, a, pSeg->nPos, &pIter->poslist); - pIter->base.pData = pIter->poslist.p; - pIter->base.nData = pIter->poslist.n; - } + int *pRc = &pIter->pIndex->rc; + fts5BufferZero(&pIter->poslist); + fts5IndexExtractColset(pRc, pColset, a, pSeg->nPos, pIter); }else{ /* The data is distributed over two or more pages. Copy it into the ** Fts5Iter.poslist buffer and then set the output pointer to point @@ -187210,6 +246082,7 @@ static void fts5IterSetOutputs_Full(Fts5Iter *pIter, Fts5SegIter *pSeg){ } static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ + assert( pIter!=0 || (*pRc)!=SQLITE_OK ); if( *pRc==SQLITE_OK ){ Fts5Config *pConfig = pIter->pIndex->pConfig; if( pConfig->eDetail==FTS5_DETAIL_NONE ){ @@ -187220,6 +246093,10 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ pIter->xSetOutputs = fts5IterSetOutputs_Nocolset; } + else if( pIter->pColset->nCol==0 ){ + pIter->xSetOutputs = fts5IterSetOutputs_ZeroColset; + } + else if( pConfig->eDetail==FTS5_DETAIL_FULL ){ pIter->xSetOutputs = fts5IterSetOutputs_Full; } @@ -187236,6 +246113,32 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ } } +/* +** All the component segment-iterators of pIter have been set up. This +** functions finishes setup for iterator pIter itself. +*/ +static void fts5MultiIterFinishSetup(Fts5Index *p, Fts5Iter *pIter){ + int iIter; + for(iIter=pIter->nSeg-1; iIter>0; iIter--){ + int iEq; + if( (iEq = fts5MultiIterDoCompare(pIter, iIter)) ){ + Fts5SegIter *pSeg = &pIter->aSeg[iEq]; + if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); + fts5MultiIterAdvanced(p, pIter, iEq, iIter); + } + } + fts5MultiIterSetEof(pIter); + fts5AssertMultiIterSetup(p, pIter); + + if( (pIter->bSkipEmpty && fts5MultiIterIsEmpty(p, pIter)) + || fts5MultiIterIsDeleted(pIter) + ){ + fts5MultiIterNext(p, pIter, 0, 0); + }else if( pIter->base.bEof==0 ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + pIter->xSetOutputs(pIter, pSeg); + } +} /* ** Allocate a new Fts5Iter object. @@ -187245,7 +246148,7 @@ static void fts5IterSetOutputCb(int *pRc, Fts5Iter *pIter){ ** is zero or greater, data from the first nSegment segments on level iLevel ** is merged. ** -** The iterator initially points to the first term/rowid entry in the +** The iterator initially points to the first term/rowid entry in the ** iterated data. */ static void fts5MultiIterNew( @@ -187271,18 +246174,19 @@ static void fts5MultiIterNew( if( iLevel<0 ){ assert( pStruct->nSegment==fts5StructureCountSegments(pStruct) ); nSeg = pStruct->nSegment; - nSeg += (p->pHash ? 1 : 0); + nSeg += (p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH)); }else{ nSeg = MIN(pStruct->aLevel[iLevel].nSeg, nSegment); } } *ppOut = pNew = fts5MultiIterAlloc(p, nSeg); - if( pNew==0 ) return; + if( pNew==0 ){ + assert( p->rc!=SQLITE_OK ); + goto fts5MultiIterNew_post_check; + } pNew->bRev = (0!=(flags & FTS5INDEX_QUERY_DESC)); pNew->bSkipEmpty = (0!=(flags & FTS5INDEX_QUERY_SKIPEMPTY)); - pNew->pStruct = pStruct; pNew->pColset = pColset; - fts5StructureRef(pStruct); if( (flags & FTS5INDEX_QUERY_NOOUTPUT)==0 ){ fts5IterSetOutputCb(&p->rc, pNew); } @@ -187291,7 +246195,7 @@ static void fts5MultiIterNew( if( p->rc==SQLITE_OK ){ if( iLevel<0 ){ Fts5StructureLevel *pEnd = &pStruct->aLevel[pStruct->nLevel]; - if( p->pHash ){ + if( p->pHash && 0==(flags & FTS5INDEX_QUERY_SKIPHASH) ){ /* Add a segment iterator for the current contents of the hash table. */ Fts5SegIter *pIter = &pNew->aSeg[iIter++]; fts5SegIterHashInit(p, pTerm, nTerm, flags, pIter); @@ -187316,33 +246220,20 @@ static void fts5MultiIterNew( assert( iIter==nSeg ); } - /* If the above was successful, each component iterators now points - ** to the first entry in its segment. In this case initialize the + /* If the above was successful, each component iterator now points + ** to the first entry in its segment. In this case initialize the ** aFirst[] array. Or, if an error has occurred, free the iterator ** object and set the output variable to NULL. */ if( p->rc==SQLITE_OK ){ - for(iIter=pNew->nSeg-1; iIter>0; iIter--){ - int iEq; - if( (iEq = fts5MultiIterDoCompare(pNew, iIter)) ){ - Fts5SegIter *pSeg = &pNew->aSeg[iEq]; - if( p->rc==SQLITE_OK ) pSeg->xNext(p, pSeg, 0); - fts5MultiIterAdvanced(p, pNew, iEq, iIter); - } - } - fts5MultiIterSetEof(pNew); - fts5AssertMultiIterSetup(p, pNew); - - if( pNew->bSkipEmpty && fts5MultiIterIsEmpty(p, pNew) ){ - fts5MultiIterNext(p, pNew, 0, 0); - }else if( pNew->base.bEof==0 ){ - Fts5SegIter *pSeg = &pNew->aSeg[pNew->aFirst[1].iFirst]; - pNew->xSetOutputs(pNew, pSeg); - } - + fts5MultiIterFinishSetup(p, pNew); }else{ fts5MultiIterFree(pNew); *ppOut = 0; } + +fts5MultiIterNew_post_check: + assert( (*ppOut)!=0 || p->rc!=SQLITE_OK ); + return; } /* @@ -187359,7 +246250,6 @@ static void fts5MultiIterNew2( pNew = fts5MultiIterAlloc(p, 2); if( pNew ){ Fts5SegIter *pIter = &pNew->aSeg[1]; - pIter->flags = FTS5_SEGITER_ONETERM; if( pData->szLeaf>0 ){ pIter->pLeaf = pData; @@ -187386,12 +246276,13 @@ static void fts5MultiIterNew2( } /* -** Return true if the iterator is at EOF or if an error has occurred. +** Return true if the iterator is at EOF or if an error has occurred. ** False otherwise. */ static int fts5MultiIterEof(Fts5Index *p, Fts5Iter *pIter){ - assert( p->rc - || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof + assert( pIter!=0 || p->rc!=SQLITE_OK ); + assert( p->rc!=SQLITE_OK + || (pIter->aSeg[ pIter->aFirst[1].iFirst ].pLeaf==0)==pIter->base.bEof ); return (p->rc || pIter->base.bEof); } @@ -187410,8 +246301,8 @@ static i64 fts5MultiIterRowid(Fts5Iter *pIter){ ** Move the iterator to the next entry at or following iMatch. */ static void fts5MultiIterNextFrom( - Fts5Index *p, - Fts5Iter *pIter, + Fts5Index *p, + Fts5Iter *pIter, i64 iMatch ){ while( 1 ){ @@ -187425,7 +246316,7 @@ static void fts5MultiIterNextFrom( } /* -** Return a pointer to a buffer containing the term associated with the +** Return a pointer to a buffer containing the term associated with the ** entry that the iterator currently points to. */ static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ @@ -187436,11 +246327,11 @@ static const u8 *fts5MultiIterTerm(Fts5Iter *pIter, int *pn){ /* ** Allocate a new segment-id for the structure pStruct. The new segment -** id must be between 1 and 65335 inclusive, and must not be used by +** id must be between 1 and 65335 inclusive, and must not be used by ** any currently existing segment. If a free segment id cannot be found, ** SQLITE_FULL is returned. ** -** If an error has already occurred, this function is a no-op. 0 is +** If an error has already occurred, this function is a no-op. 0 is ** returned in this case. */ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ @@ -187460,24 +246351,24 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ for(iLvl=0; iLvlnLevel; iLvl++){ for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ int iId = pStruct->aLevel[iLvl].aSeg[iSeg].iSegid; - if( iId<=FTS5_MAX_SEGMENT ){ - aUsed[(iId-1) / 32] |= 1 << ((iId-1) % 32); + if( iId<=FTS5_MAX_SEGMENT && iId>0 ){ + aUsed[(iId-1) / 32] |= (u32)1 << ((iId-1) % 32); } } } for(i=0; aUsed[i]==0xFFFFFFFF; i++); mask = aUsed[i]; - for(iSegid=0; mask & (1 << iSegid); iSegid++); + for(iSegid=0; mask & ((u32)1 << iSegid); iSegid++); iSegid += 1 + i*32; #ifdef SQLITE_DEBUG for(iLvl=0; iLvlnLevel; iLvl++){ for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - assert( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); + assert_nc( iSegid!=pStruct->aLevel[iLvl].aSeg[iSeg].iSegid ); } } - assert( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); + assert_nc( iSegid>0 && iSegid<=FTS5_MAX_SEGMENT ); { sqlite3_stmt *pIdxSelect = fts5IdxSelectStmt(p); @@ -187485,8 +246376,9 @@ static int fts5AllocateSegid(Fts5Index *p, Fts5Structure *pStruct){ u8 aBlob[2] = {0xff, 0xff}; sqlite3_bind_int(pIdxSelect, 1, iSegid); sqlite3_bind_blob(pIdxSelect, 2, aBlob, 2, SQLITE_STATIC); - assert( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); + assert_nc( sqlite3_step(pIdxSelect)!=SQLITE_ROW ); p->rc = sqlite3_reset(pIdxSelect); + sqlite3_bind_null(pIdxSelect, 2); } } #endif @@ -187504,14 +246396,17 @@ static void fts5IndexDiscardData(Fts5Index *p){ if( p->pHash ){ sqlite3Fts5HashClear(p->pHash); p->nPendingData = 0; + p->nPendingRow = 0; + p->flushRc = SQLITE_OK; } + p->nContentlessDelete = 0; } /* -** Return the size of the prefix, in bytes, that buffer +** Return the size of the prefix, in bytes, that buffer ** (pNew/) shares with buffer (pOld/nOld). ** -** Buffer (pNew/) is guaranteed to be greater +** Buffer (pNew/) is guaranteed to be greater ** than buffer (pOld/nOld). */ static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ @@ -187523,7 +246418,7 @@ static int fts5PrefixCompress(int nOld, const u8 *pOld, const u8 *pNew){ } static void fts5WriteDlidxClear( - Fts5Index *p, + Fts5Index *p, Fts5SegWriter *pWriter, int bFlush /* If true, write dlidx to disk */ ){ @@ -187534,7 +246429,7 @@ static void fts5WriteDlidxClear( if( pDlidx->buf.n==0 ) break; if( bFlush ){ assert( pDlidx->pgno!=0 ); - fts5DataWrite(p, + fts5DataWrite(p, FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), pDlidx->buf.p, pDlidx->buf.n ); @@ -187554,13 +246449,13 @@ static int fts5WriteDlidxGrow( int nLvl ){ if( p->rc==SQLITE_OK && nLvl>=pWriter->nDlidx ){ - Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc( + Fts5DlidxWriter *aDlidx = (Fts5DlidxWriter*)sqlite3_realloc64( pWriter->aDlidx, sizeof(Fts5DlidxWriter) * nLvl ); if( aDlidx==0 ){ p->rc = SQLITE_NOMEM; }else{ - int nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); + size_t nByte = sizeof(Fts5DlidxWriter) * (nLvl - pWriter->nDlidx); memset(&aDlidx[pWriter->nDlidx], 0, nByte); pWriter->aDlidx = aDlidx; pWriter->nDlidx = nLvl; @@ -187588,8 +246483,8 @@ static int fts5WriteFlushDlidx(Fts5Index *p, Fts5SegWriter *pWriter){ } /* -** This function is called whenever processing of the doclist for the -** last term on leaf page (pWriter->iBtPage) is completed. +** This function is called whenever processing of the doclist for the +** last term on leaf page (pWriter->iBtPage) is completed. ** ** The doclist-index for that term is currently stored in-memory within the ** Fts5SegWriter.aDlidx[] array. If it is large enough, this function @@ -187613,6 +246508,7 @@ static void fts5WriteFlushBtree(Fts5Index *p, Fts5SegWriter *pWriter){ sqlite3_bind_int64(p->pIdxWriter, 3, bFlag + ((i64)pWriter->iBtPage<<1)); sqlite3_step(p->pIdxWriter); p->rc = sqlite3_reset(p->pIdxWriter); + sqlite3_bind_null(p->pIdxWriter, 2); } pWriter->iBtPage = 0; } @@ -187632,8 +246528,10 @@ static void fts5WriteBtreeTerm( int nTerm, const u8 *pTerm /* First term on new page */ ){ fts5WriteFlushBtree(p, pWriter); - fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); - pWriter->iBtPage = pWriter->writer.pgno; + if( p->rc==SQLITE_OK ){ + fts5BufferSet(&p->rc, &pWriter->btterm, nTerm, pTerm); + pWriter->iBtPage = pWriter->writer.pgno; + } } /* @@ -187671,8 +246569,8 @@ static i64 fts5DlidxExtractFirstRowid(Fts5Buffer *pBuf){ ** doclist-index. */ static void fts5WriteDlidxAppend( - Fts5Index *p, - Fts5SegWriter *pWriter, + Fts5Index *p, + Fts5SegWriter *pWriter, i64 iRowid ){ int i; @@ -187685,11 +246583,11 @@ static void fts5WriteDlidxAppend( if( pDlidx->buf.n>=p->pConfig->pgsz ){ /* The current doclist-index page is full. Write it to disk and push ** a copy of iRowid (which will become the first rowid on the next - ** doclist-index leaf page) up into the next level of the b-tree + ** doclist-index leaf page) up into the next level of the b-tree ** hierarchy. If the node being flushed is currently the root node, ** also push its first rowid upwards. */ pDlidx->buf.p[0] = 0x01; /* Not the root node */ - fts5DataWrite(p, + fts5DataWrite(p, FTS5_DLIDX_ROWID(pWriter->iSegid, i, pDlidx->pgno), pDlidx->buf.p, pDlidx->buf.n ); @@ -187715,7 +246613,7 @@ static void fts5WriteDlidxAppend( } if( pDlidx->bPrevValid ){ - iVal = iRowid - pDlidx->iPrev; + iVal = (u64)iRowid - (u64)pDlidx->iPrev; }else{ i64 iPgno = (i==0 ? pWriter->writer.pgno : pDlidx[-1].pgno); assert( pDlidx->buf.n==0 ); @@ -187735,9 +246633,6 @@ static void fts5WriteFlushLeaf(Fts5Index *p, Fts5SegWriter *pWriter){ Fts5PageWriter *pPage = &pWriter->writer; i64 iRowid; -static int nCall = 0; -nCall++; - assert( (pPage->pgidx.n==0)==(pWriter->bFirstTermInPage) ); /* Set the szLeaf header field. */ @@ -187776,17 +246671,18 @@ nCall++; ** Append term pTerm/nTerm to the segment being written by the writer passed ** as the second argument. ** -** If an error occurs, set the Fts5Index.rc error code. If an error has +** If an error occurs, set the Fts5Index.rc error code. If an error has ** already occurred, this function is a no-op. */ static void fts5WriteAppendTerm( - Fts5Index *p, + Fts5Index *p, Fts5SegWriter *pWriter, - int nTerm, const u8 *pTerm + int nTerm, const u8 *pTerm ){ int nPrefix; /* Bytes of prefix compression for term */ Fts5PageWriter *pPage = &pWriter->writer; Fts5Buffer *pPgidx = &pWriter->writer.pgidx; + int nMin = MIN(pPage->term.n, nTerm); assert( p->rc==SQLITE_OK ); assert( pPage->buf.n>=4 ); @@ -187796,10 +246692,11 @@ static void fts5WriteAppendTerm( if( (pPage->buf.n + pPgidx->n + nTerm + 2)>=p->pConfig->pgsz ){ if( pPage->buf.n>4 ){ fts5WriteFlushLeaf(p, pWriter); + if( p->rc!=SQLITE_OK ) return; } fts5BufferGrow(&p->rc, &pPage->buf, nTerm+FTS5_DATA_PADDING); } - + /* TODO1: Updating pgidx here. */ pPgidx->n += sqlite3Fts5PutVarint( &pPgidx->p[pPgidx->n], pPage->buf.n - pPage->iPrevPgidx @@ -187815,11 +246712,11 @@ static void fts5WriteAppendTerm( if( pPage->pgno!=1 ){ /* This is the first term on a leaf that is not the leftmost leaf in ** the segment b-tree. In this case it is necessary to add a term to - ** the b-tree hierarchy that is (a) larger than the largest term + ** the b-tree hierarchy that is (a) larger than the largest term ** already written to the segment and (b) smaller than or equal to ** this term. In other words, a prefix of (pTerm/nTerm) that is one ** byte longer than the longest prefix (pTerm/nTerm) shares with the - ** previous term. + ** previous term. ** ** Usually, the previous term is available in pPage->term. The exception ** is if this is the first term written in an incremental-merge step. @@ -187828,13 +246725,14 @@ static void fts5WriteAppendTerm( ** inefficient, but still correct. */ int n = nTerm; if( pPage->term.n ){ - n = 1 + fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); + n = 1 + fts5PrefixCompress(nMin, pPage->term.p, pTerm); } fts5WriteBtreeTerm(p, pWriter, n, pTerm); + if( p->rc!=SQLITE_OK ) return; pPage = &pWriter->writer; } }else{ - nPrefix = fts5PrefixCompress(pPage->term.n, pPage->term.p, pTerm); + nPrefix = fts5PrefixCompress(nMin, pPage->term.p, pTerm); fts5BufferAppendVarint(&p->rc, &pPage->buf, nPrefix); } @@ -187855,10 +246753,10 @@ static void fts5WriteAppendTerm( } /* -** Append a rowid and position-list size field to the writers output. +** Append a rowid and position-list size field to the writers output. */ static void fts5WriteAppendRowid( - Fts5Index *p, + Fts5Index *p, Fts5SegWriter *pWriter, i64 iRowid ){ @@ -187869,7 +246767,7 @@ static void fts5WriteAppendRowid( fts5WriteFlushLeaf(p, pWriter); } - /* If this is to be the first rowid written to the page, set the + /* If this is to be the first rowid written to the page, set the ** rowid-pointer in the page-header. Also append a value to the dlidx ** buffer, in case a doclist-index is required. */ if( pWriter->bFirstRowidInPage ){ @@ -187881,8 +246779,10 @@ static void fts5WriteAppendRowid( if( pWriter->bFirstRowidInDoclist || pWriter->bFirstRowidInPage ){ fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid); }else{ - assert( p->rc || iRowid>pWriter->iPrevRowid ); - fts5BufferAppendVarint(&p->rc, &pPage->buf, iRowid - pWriter->iPrevRowid); + assert_nc( p->rc || iRowid>pWriter->iPrevRowid ); + fts5BufferAppendVarint(&p->rc, &pPage->buf, + (u64)iRowid - (u64)pWriter->iPrevRowid + ); } pWriter->iPrevRowid = iRowid; pWriter->bFirstRowidInDoclist = 0; @@ -187891,18 +246791,18 @@ static void fts5WriteAppendRowid( } static void fts5WriteAppendPoslistData( - Fts5Index *p, - Fts5SegWriter *pWriter, - const u8 *aData, + Fts5Index *p, + Fts5SegWriter *pWriter, + const u8 *aData, int nData ){ Fts5PageWriter *pPage = &pWriter->writer; const u8 *a = aData; int n = nData; - - assert( p->pConfig->pgsz>0 ); - while( p->rc==SQLITE_OK - && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz + + assert( p->pConfig->pgsz>0 || p->rc!=SQLITE_OK ); + while( p->rc==SQLITE_OK + && (pPage->buf.n + pPage->pgidx.n + n)>=p->pConfig->pgsz ){ int nReq = p->pConfig->pgsz - pPage->buf.n - pPage->pgidx.n; int nCopy = 0; @@ -187925,7 +246825,7 @@ static void fts5WriteAppendPoslistData( ** allocations associated with the writer. */ static void fts5WriteFinish( - Fts5Index *p, + Fts5Index *p, Fts5SegWriter *pWriter, /* Writer object */ int *pnLeaf /* OUT: Number of leaf pages in b-tree */ ){ @@ -187953,8 +246853,8 @@ static void fts5WriteFinish( } static void fts5WriteInit( - Fts5Index *p, - Fts5SegWriter *pWriter, + Fts5Index *p, + Fts5SegWriter *pWriter, int iSegid ){ const int nBuffer = p->pConfig->pgsz + FTS5_DATA_PADDING; @@ -187977,7 +246877,7 @@ static void fts5WriteInit( if( p->pIdxWriter==0 ){ Fts5Config *pConfig = p->pConfig; fts5IndexPrepareStmt(p, &p->pIdxWriter, sqlite3_mprintf( - "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", + "INSERT INTO '%q'.'%q_idx'(segid,term,pgno) VALUES(?,?,?)", pConfig->zDb, pConfig->zName )); } @@ -188003,7 +246903,7 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ int i; Fts5Buffer buf; memset(&buf, 0, sizeof(Fts5Buffer)); - for(i=0; inSeg; i++){ + for(i=0; inSeg && p->rc==SQLITE_OK; i++){ Fts5SegIter *pSeg = &pIter->aSeg[i]; if( pSeg->pSeg==0 ){ /* no-op */ @@ -188021,35 +246921,44 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ u8 aHdr[4] = {0x00, 0x00, 0x00, 0x00}; iLeafRowid = FTS5_SEGMENT_ROWID(iId, pSeg->iTermLeafPgno); - pData = fts5DataRead(p, iLeafRowid); + pData = fts5LeafRead(p, iLeafRowid); if( pData ){ - fts5BufferZero(&buf); - fts5BufferGrow(&p->rc, &buf, pData->nn); - fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); - fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); - fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); - fts5BufferAppendBlob(&p->rc, &buf, pData->szLeaf-iOff, &pData->p[iOff]); - if( p->rc==SQLITE_OK ){ - /* Set the szLeaf field */ - fts5PutU16(&buf.p[2], (u16)buf.n); - } + if( iOff>pData->szLeaf ){ + /* This can occur if the pages that the segments occupy overlap - if + ** a single page has been assigned to more than one segment. In + ** this case a prior iteration of this loop may have corrupted the + ** segment currently being trimmed. */ + p->rc = FTS5_CORRUPT; + }else{ + fts5BufferZero(&buf); + fts5BufferGrow(&p->rc, &buf, pData->nn); + fts5BufferAppendBlob(&p->rc, &buf, sizeof(aHdr), aHdr); + fts5BufferAppendVarint(&p->rc, &buf, pSeg->term.n); + fts5BufferAppendBlob(&p->rc, &buf, pSeg->term.n, pSeg->term.p); + fts5BufferAppendBlob(&p->rc, &buf,pData->szLeaf-iOff,&pData->p[iOff]); + if( p->rc==SQLITE_OK ){ + /* Set the szLeaf field */ + fts5PutU16(&buf.p[2], (u16)buf.n); + } - /* Set up the new page-index array */ - fts5BufferAppendVarint(&p->rc, &buf, 4); - if( pSeg->iLeafPgno==pSeg->iTermLeafPgno - && pSeg->iEndofDoclistszLeaf - ){ - int nDiff = pData->szLeaf - pSeg->iEndofDoclist; - fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); - fts5BufferAppendBlob(&p->rc, &buf, - pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] - ); - } + /* Set up the new page-index array */ + fts5BufferAppendVarint(&p->rc, &buf, 4); + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno + && pSeg->iEndofDoclistszLeaf + && pSeg->iPgidxOff<=pData->nn + ){ + int nDiff = pData->szLeaf - pSeg->iEndofDoclist; + fts5BufferAppendVarint(&p->rc, &buf, buf.n - 1 - nDiff - 4); + fts5BufferAppendBlob(&p->rc, &buf, + pData->nn - pSeg->iPgidxOff, &pData->p[pSeg->iPgidxOff] + ); + } + pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; + fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); + fts5DataWrite(p, iLeafRowid, buf.p, buf.n); + } fts5DataRelease(pData); - pSeg->pSeg->pgnoFirst = pSeg->iTermLeafPgno; - fts5DataDelete(p, FTS5_SEGMENT_ROWID(iId, 1), iLeafRowid); - fts5DataWrite(p, iLeafRowid, buf.p, buf.n); } } } @@ -188057,8 +246966,8 @@ static void fts5TrimSegments(Fts5Index *p, Fts5Iter *pIter){ } static void fts5MergeChunkCallback( - Fts5Index *p, - void *pCtx, + Fts5Index *p, + void *pCtx, const u8 *pChunk, int nChunk ){ Fts5SegWriter *pWriter = (Fts5SegWriter*)pCtx; @@ -188086,6 +246995,7 @@ static void fts5IndexMergeLevel( int bOldest; /* True if the output segment is the oldest */ int eDetail = p->pConfig->eDetail; const int flags = FTS5INDEX_QUERY_NOOUTPUT; + int bTermWritten = 0; /* True if current term already output */ assert( iLvlnLevel ); assert( pLvl->nMerge<=pLvl->nSeg ); @@ -188126,6 +247036,12 @@ static void fts5IndexMergeLevel( /* Read input from all segments in the input level */ nInput = pLvl->nSeg; + + /* Set the range of origins that will go into the output segment. */ + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pLvl->aSeg[0].iOrigin1; + pSeg->iOrigin2 = pLvl->aSeg[pLvl->nSeg-1].iOrigin2; + } } bOldest = (pLvlOut->nSeg==1 && pStruct->nLevel==iLvl+2); @@ -188139,18 +247055,22 @@ static void fts5IndexMergeLevel( int nTerm; const u8 *pTerm; - /* Check for key annihilation. */ - if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; - pTerm = fts5MultiIterTerm(pIter, &nTerm); - if( nTerm!=term.n || memcmp(pTerm, term.p, nTerm) ){ + if( nTerm!=term.n || fts5Memcmp(pTerm, term.p, nTerm) ){ if( pnRem && writer.nLeafWritten>nRem ){ break; } + fts5BufferSet(&p->rc, &term, nTerm, pTerm); + bTermWritten =0; + } + /* Check for key annihilation. */ + if( pSegIter->nPos==0 && (bOldest || pSegIter->bDel==0) ) continue; + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ /* This is a new term. Append a term to the output segment. */ fts5WriteAppendTerm(p, &writer, nTerm, pTerm); - fts5BufferSet(&p->rc, &term, nTerm, pTerm); + bTermWritten = 1; } /* Append the rowid to the output */ @@ -188176,12 +247096,16 @@ static void fts5IndexMergeLevel( ** and last leaf page number at the same time. */ fts5WriteFinish(p, &writer, &pSeg->pgnoLast); + assert( pIter!=0 || p->rc!=SQLITE_OK ); if( fts5MultiIterEof(p, pIter) ){ int i; /* Remove the redundant segments from the %_data table */ + assert( pSeg->nEntry==0 ); for(i=0; iaSeg[i].iSegid); + Fts5StructureSegment *pOld = &pLvl->aSeg[i]; + pSeg->nEntry += (pOld->nEntry - pOld->nEntryTombstone); + fts5DataRemoveSegment(p, pOld); } /* Remove the redundant segments from the input level */ @@ -188207,6 +247131,48 @@ static void fts5IndexMergeLevel( if( pnRem ) *pnRem -= writer.nLeafWritten; } +/* +** If this is not a contentless_delete=1 table, or if the 'deletemerge' +** configuration option is set to 0, then this function always returns -1. +** Otherwise, it searches the structure object passed as the second argument +** for a level suitable for merging due to having a large number of +** tombstones in the tombstone hash. If one is found, its index is returned. +** Otherwise, if there is no suitable level, -1. +*/ +static int fts5IndexFindDeleteMerge(Fts5Index *p, Fts5Structure *pStruct){ + Fts5Config *pConfig = p->pConfig; + int iRet = -1; + if( pConfig->bContentlessDelete && pConfig->nDeleteMerge>0 ){ + int ii; + int nBest = 0; + + for(ii=0; iinLevel; ii++){ + Fts5StructureLevel *pLvl = &pStruct->aLevel[ii]; + i64 nEntry = 0; + i64 nTomb = 0; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + nEntry += pLvl->aSeg[iSeg].nEntry; + nTomb += pLvl->aSeg[iSeg].nEntryTombstone; + } + assert_nc( nEntry>0 || pLvl->nSeg==0 ); + if( nEntry>0 ){ + int nPercent = (nTomb * 100) / nEntry; + if( nPercent>=pConfig->nDeleteMerge && nPercent>nBest ){ + iRet = ii; + nBest = nPercent; + } + } + + /* If pLvl is already the input level to an ongoing merge, look no + ** further for a merge candidate. The caller should be allowed to + ** continue merging from pLvl first. */ + if( pLvl->nMerge ) break; + } + } + return iRet; +} + /* ** Do up to nPg pages of automerge work on the index. ** @@ -188226,14 +247192,15 @@ static int fts5IndexMerge( int iBestLvl = 0; /* Level offering the most input segments */ int nBest = 0; /* Number of input segments on best level */ - /* Set iBestLvl to the level to read input segments from. */ + /* Set iBestLvl to the level to read input segments from. Or to -1 if + ** there is no level suitable to merge segments from. */ assert( pStruct->nLevel>0 ); for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &pStruct->aLevel[iLvl]; if( pLvl->nMerge ){ if( pLvl->nMerge>nBest ){ iBestLvl = iLvl; - nBest = pLvl->nMerge; + nBest = nMin; } break; } @@ -188242,22 +247209,18 @@ static int fts5IndexMerge( iBestLvl = iLvl; } } - - /* If nBest is still 0, then the index must be empty. */ -#ifdef SQLITE_DEBUG - for(iLvl=0; nBest==0 && iLvlnLevel; iLvl++){ - assert( pStruct->aLevel[iLvl].nSeg==0 ); + if( nBestaLevel[iBestLvl].nMerge==0 ){ - break; - } + if( iBestLvl<0 ) break; bRet = 1; fts5IndexMergeLevel(p, &pStruct, iBestLvl, &nRem); if( p->rc==SQLITE_OK && pStruct->aLevel[iBestLvl].nMerge==0 ){ fts5StructurePromote(p, iBestLvl+1, pStruct); } + + if( nMin==1 ) nMin = 2; } *ppStruct = pStruct; return bRet; @@ -188268,7 +247231,7 @@ static int fts5IndexMerge( ** segment. This function updates the write-counter accordingly and, if ** necessary, performs incremental merge work. ** -** If an error occurs, set the Fts5Index.rc error code. If an error has +** If an error occurs, set the Fts5Index.rc error code. If an error has ** already occurred, this function is a no-op. */ static void fts5IndexAutomerge( @@ -188276,7 +247239,7 @@ static void fts5IndexAutomerge( Fts5Structure **ppStruct, /* IN/OUT: Current structure of index */ int nLeaf /* Number of output leaves just written */ ){ - if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 ){ + if( p->rc==SQLITE_OK && p->pConfig->nAutomerge>0 && ALWAYS((*ppStruct)!=0) ){ Fts5Structure *pStruct = *ppStruct; u64 nWrite; /* Initial value of write-counter */ int nWork; /* Number of work-quanta to perform */ @@ -188298,16 +247261,16 @@ static void fts5IndexCrisismerge( ){ const int nCrisis = p->pConfig->nCrisisMerge; Fts5Structure *pStruct = *ppStruct; - int iLvl = 0; - - assert( p->rc!=SQLITE_OK || pStruct->nLevel>0 ); - while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ - fts5IndexMergeLevel(p, &pStruct, iLvl, 0); - assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); - fts5StructurePromote(p, iLvl+1, pStruct); - iLvl++; + if( pStruct && pStruct->nLevel>0 ){ + int iLvl = 0; + while( p->rc==SQLITE_OK && pStruct->aLevel[iLvl].nSeg>=nCrisis ){ + fts5IndexMergeLevel(p, &pStruct, iLvl, 0); + assert( p->rc!=SQLITE_OK || pStruct->nLevel>(iLvl+1) ); + fts5StructurePromote(p, iLvl+1, pStruct); + iLvl++; + } + *ppStruct = pStruct; } - *ppStruct = pStruct; } static int fts5IndexReturn(Fts5Index *p){ @@ -188319,12 +247282,12 @@ static int fts5IndexReturn(Fts5Index *p){ typedef struct Fts5FlushCtx Fts5FlushCtx; struct Fts5FlushCtx { Fts5Index *pIdx; - Fts5SegWriter writer; + Fts5SegWriter writer; }; /* ** Buffer aBuf[] contains a list of varints, all small enough to fit -** in a 32-bit integer. Return the size of the largest prefix of this +** in a 32-bit integer. Return the size of the largest prefix of this ** list nMax bytes or less in size. */ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ @@ -188342,10 +247305,473 @@ static int fts5PoslistPrefix(const u8 *aBuf, int nMax){ } /* -** Flush the contents of in-memory hash table iHash to a new level-0 +** Execute the SQL statement: +** +** DELETE FROM %_idx WHERE (segid, (pgno/2)) = ($iSegid, $iPgno); +** +** This is used when a secure-delete operation removes the last term +** from a segment leaf page. In that case the %_idx entry is removed +** too. This is done to ensure that if all instances of a token are +** removed from an fts5 database in secure-delete mode, no trace of +** the token itself remains in the database. +*/ +static void fts5SecureDeleteIdxEntry( + Fts5Index *p, /* FTS5 backend object */ + int iSegid, /* Id of segment to delete entry for */ + int iPgno /* Page number within segment */ +){ + if( iPgno!=1 ){ + assert( p->pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE ); + if( p->pDeleteFromIdx==0 ){ + fts5IndexPrepareStmt(p, &p->pDeleteFromIdx, sqlite3_mprintf( + "DELETE FROM '%q'.'%q_idx' WHERE (segid, (pgno/2)) = (?1, ?2)", + p->pConfig->zDb, p->pConfig->zName + )); + } + if( p->rc==SQLITE_OK ){ + sqlite3_bind_int(p->pDeleteFromIdx, 1, iSegid); + sqlite3_bind_int(p->pDeleteFromIdx, 2, iPgno); + sqlite3_step(p->pDeleteFromIdx); + p->rc = sqlite3_reset(p->pDeleteFromIdx); + } + } +} + +/* +** This is called when a secure-delete operation removes a position-list +** that overflows onto segment page iPgno of segment pSeg. This function +** rewrites node iPgno, and possibly one or more of its right-hand peers, +** to remove this portion of the position list. +** +** Output variable (*pbLastInDoclist) is set to true if the position-list +** removed is followed by a new term or the end-of-segment, or false if +** it is followed by another rowid/position list. +*/ +static void fts5SecureDeleteOverflow( + Fts5Index *p, + Fts5StructureSegment *pSeg, + int iPgno, + int *pbLastInDoclist +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int pgno; + Fts5Data *pLeaf = 0; + assert( iPgno!=1 ); + + *pbLastInDoclist = 1; + for(pgno=iPgno; p->rc==SQLITE_OK && pgno<=pSeg->pgnoLast; pgno++){ + i64 iRowid = FTS5_SEGMENT_ROWID(pSeg->iSegid, pgno); + int iNext = 0; + u8 *aPg = 0; + + pLeaf = fts5DataRead(p, iRowid); + if( pLeaf==0 ) break; + aPg = pLeaf->p; + + iNext = fts5GetU16(&aPg[0]); + if( iNext!=0 ){ + *pbLastInDoclist = 0; + } + if( iNext==0 && pLeaf->szLeaf!=pLeaf->nn ){ + fts5GetVarint32(&aPg[pLeaf->szLeaf], iNext); + } + + if( iNext==0 ){ + /* The page contains no terms or rowids. Replace it with an empty + ** page and move on to the right-hand peer. */ + const u8 aEmpty[] = {0x00, 0x00, 0x00, 0x04}; + assert_nc( bDetailNone==0 || pLeaf->nn==4 ); + if( bDetailNone==0 ) fts5DataWrite(p, iRowid, aEmpty, sizeof(aEmpty)); + fts5DataRelease(pLeaf); + pLeaf = 0; + }else if( bDetailNone ){ + break; + }else if( iNext>=pLeaf->szLeaf || pLeaf->nnszLeaf || iNext<4 ){ + p->rc = FTS5_CORRUPT; + break; + }else{ + int nShift = iNext - 4; + int nPg; + + int nIdx = 0; + u8 *aIdx = 0; + + /* Unless the current page footer is 0 bytes in size (in which case + ** the new page footer will be as well), allocate and populate a + ** buffer containing the new page footer. Set stack variables aIdx + ** and nIdx accordingly. */ + if( pLeaf->nn>pLeaf->szLeaf ){ + int iFirst = 0; + int i1 = pLeaf->szLeaf; + int i2 = 0; + + i1 += fts5GetVarint32(&aPg[i1], iFirst); + if( iFirstrc = FTS5_CORRUPT; + break; + } + aIdx = sqlite3Fts5MallocZero(&p->rc, (pLeaf->nn-pLeaf->szLeaf)+2); + if( aIdx==0 ) break; + i2 = sqlite3Fts5PutVarint(aIdx, iFirst-nShift); + if( i1nn ){ + memcpy(&aIdx[i2], &aPg[i1], pLeaf->nn-i1); + i2 += (pLeaf->nn-i1); + } + nIdx = i2; + } + + /* Modify the contents of buffer aPg[]. Set nPg to the new size + ** in bytes. The new page is always smaller than the old. */ + nPg = pLeaf->szLeaf - nShift; + memmove(&aPg[4], &aPg[4+nShift], nPg-4); + fts5PutU16(&aPg[2], nPg); + if( fts5GetU16(&aPg[0]) ) fts5PutU16(&aPg[0], 4); + if( nIdx>0 ){ + memcpy(&aPg[nPg], aIdx, nIdx); + nPg += nIdx; + } + sqlite3_free(aIdx); + + /* Write the new page to disk and exit the loop */ + assert( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, iRowid, aPg, nPg); + break; + } + } + fts5DataRelease(pLeaf); +} + +/* +** Completely remove the entry that pSeg currently points to from +** the database. +*/ +static void fts5DoSecureDelete( + Fts5Index *p, + Fts5SegIter *pSeg +){ + const int bDetailNone = (p->pConfig->eDetail==FTS5_DETAIL_NONE); + int iSegid = pSeg->pSeg->iSegid; + u8 *aPg = pSeg->pLeaf->p; + int nPg = pSeg->pLeaf->nn; + int iPgIdx = pSeg->pLeaf->szLeaf; + + u64 iDelta = 0; + int iNextOff = 0; + int iOff = 0; + int nIdx = 0; + u8 *aIdx = 0; + int bLastInDoclist = 0; + int iIdx = 0; + int iStart = 0; + int iDelKeyOff = 0; /* Offset of deleted key, if any */ + + nIdx = nPg-iPgIdx; + aIdx = sqlite3Fts5MallocZero(&p->rc, nIdx+16); + if( p->rc ) return; + memcpy(aIdx, &aPg[iPgIdx], nIdx); + + /* At this point segment iterator pSeg points to the entry + ** this function should remove from the b-tree segment. + ** + ** In detail=full or detail=column mode, pSeg->iLeafOffset is the + ** offset of the first byte in the position-list for the entry to + ** remove. Immediately before this comes two varints that will also + ** need to be removed: + ** + ** + the rowid or delta rowid value for the entry, and + ** + the size of the position list in bytes. + ** + ** Or, in detail=none mode, there is a single varint prior to + ** pSeg->iLeafOffset - the rowid or delta rowid value. + ** + ** This block sets the following variables: + ** + ** iStart: + ** The offset of the first byte of the rowid or delta-rowid + ** value for the doclist entry being removed. + ** + ** iDelta: + ** The value of the rowid or delta-rowid value for the doclist + ** entry being removed. + ** + ** iNextOff: + ** The offset of the next entry following the position list + ** for the one being removed. If the position list for this + ** entry overflows onto the next leaf page, this value will be + ** greater than pLeaf->szLeaf. + */ + { + int iSOP; /* Start-Of-Position-list */ + if( pSeg->iLeafPgno==pSeg->iTermLeafPgno ){ + iStart = pSeg->iTermLeafOffset; + }else{ + iStart = fts5GetU16(&aPg[0]); + } + + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + assert_nc( iSOP<=pSeg->iLeafOffset ); + + if( bDetailNone ){ + while( iSOPiLeafOffset ){ + if( aPg[iSOP]==0x00 ) iSOP++; + if( aPg[iSOP]==0x00 ) iSOP++; + iStart = iSOP; + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + } + + iNextOff = iSOP; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + if( iNextOffiEndofDoclist && aPg[iNextOff]==0x00 ) iNextOff++; + + }else{ + int nPos = 0; + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + while( iSOPiLeafOffset ){ + iStart = iSOP + (nPos/2); + iSOP = iStart + fts5GetVarint(&aPg[iStart], &iDelta); + iSOP += fts5GetVarint32(&aPg[iSOP], nPos); + } + assert_nc( iSOP==pSeg->iLeafOffset ); + iNextOff = pSeg->iLeafOffset + pSeg->nPos; + } + } + + iOff = iStart; + + /* If the position-list for the entry being removed flows over past + ** the end of this page, delete the portion of the position-list on the + ** next page and beyond. + ** + ** Set variable bLastInDoclist to true if this entry happens + ** to be the last rowid in the doclist for its term. */ + if( iNextOff>=iPgIdx ){ + int pgno = pSeg->iLeafPgno+1; + fts5SecureDeleteOverflow(p, pSeg->pSeg, pgno, &bLastInDoclist); + iNextOff = iPgIdx; + } + + if( pSeg->bDel==0 ){ + if( iNextOff!=iPgIdx ){ + /* Loop through the page-footer. If iNextOff (offset of the + ** entry following the one we are removing) is equal to the + ** offset of a key on this page, then the entry is the last + ** in its doclist. */ + int iKeyOff = 0; + for(iIdx=0; iIdxbDel ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta); + aPg[iOff++] = 0x01; + }else if( bLastInDoclist==0 ){ + if( iNextOff!=iPgIdx ){ + u64 iNextDelta = 0; + iNextOff += fts5GetVarint(&aPg[iNextOff], &iNextDelta); + iOff += sqlite3Fts5PutVarint(&aPg[iOff], iDelta + iNextDelta); + } + }else if( + pSeg->iLeafPgno==pSeg->iTermLeafPgno + && iStart==pSeg->iTermLeafOffset + ){ + /* The entry being removed was the only position list in its + ** doclist. Therefore the term needs to be removed as well. */ + int iKey = 0; + int iKeyOff = 0; + + /* Set iKeyOff to the offset of the term that will be removed - the + ** last offset in the footer that is not greater than iStart. */ + for(iIdx=0; iIdx(u32)iStart ) break; + iKeyOff += iVal; + } + assert_nc( iKey>=1 ); + + /* Set iDelKeyOff to the value of the footer entry to remove from + ** the page. */ + iDelKeyOff = iOff = iKeyOff; + + if( iNextOff!=iPgIdx ){ + /* This is the only position-list associated with the term, and there + ** is another term following it on this page. So the subsequent term + ** needs to be moved to replace the term associated with the entry + ** being removed. */ + int nPrefix = 0; + int nSuffix = 0; + int nPrefix2 = 0; + int nSuffix2 = 0; + + iDelKeyOff = iNextOff; + iNextOff += fts5GetVarint32(&aPg[iNextOff], nPrefix2); + iNextOff += fts5GetVarint32(&aPg[iNextOff], nSuffix2); + + if( iKey!=1 ){ + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nPrefix); + } + iKeyOff += fts5GetVarint32(&aPg[iKeyOff], nSuffix); + + nPrefix = MIN(nPrefix, nPrefix2); + nSuffix = (nPrefix2 + nSuffix2) - nPrefix; + + if( (iKeyOff+nSuffix)>iPgIdx || (iNextOff+nSuffix2)>iPgIdx ){ + p->rc = FTS5_CORRUPT; + }else{ + if( iKey!=1 ){ + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nPrefix); + } + iOff += sqlite3Fts5PutVarint(&aPg[iOff], nSuffix); + if( nPrefix2>pSeg->term.n ){ + p->rc = FTS5_CORRUPT; + }else if( nPrefix2>nPrefix ){ + memcpy(&aPg[iOff], &pSeg->term.p[nPrefix], nPrefix2-nPrefix); + iOff += (nPrefix2-nPrefix); + } + memmove(&aPg[iOff], &aPg[iNextOff], nSuffix2); + iOff += nSuffix2; + iNextOff += nSuffix2; + } + } + }else if( iStart==4 ){ + int iPgno; + + assert_nc( pSeg->iLeafPgno>pSeg->iTermLeafPgno ); + /* The entry being removed may be the only position list in + ** its doclist. */ + for(iPgno=pSeg->iLeafPgno-1; iPgno>pSeg->iTermLeafPgno; iPgno-- ){ + Fts5Data *pPg = fts5DataRead(p, FTS5_SEGMENT_ROWID(iSegid, iPgno)); + int bEmpty = (pPg && pPg->nn==4); + fts5DataRelease(pPg); + if( bEmpty==0 ) break; + } + + if( iPgno==pSeg->iTermLeafPgno ){ + i64 iId = FTS5_SEGMENT_ROWID(iSegid, pSeg->iTermLeafPgno); + Fts5Data *pTerm = fts5DataRead(p, iId); + if( pTerm && pTerm->szLeaf==pSeg->iTermLeafOffset ){ + u8 *aTermIdx = &pTerm->p[pTerm->szLeaf]; + int nTermIdx = pTerm->nn - pTerm->szLeaf; + int iTermIdx = 0; + int iTermOff = 0; + + while( 1 ){ + u32 iVal = 0; + int nByte = fts5GetVarint32(&aTermIdx[iTermIdx], iVal); + iTermOff += iVal; + if( (iTermIdx+nByte)>=nTermIdx ) break; + iTermIdx += nByte; + } + nTermIdx = iTermIdx; + + memmove(&pTerm->p[iTermOff], &pTerm->p[pTerm->szLeaf], nTermIdx); + fts5PutU16(&pTerm->p[2], iTermOff); + + fts5DataWrite(p, iId, pTerm->p, iTermOff+nTermIdx); + if( nTermIdx==0 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iTermLeafPgno); + } + } + fts5DataRelease(pTerm); + } + } + + /* Assuming no error has occurred, this block does final edits to the + ** leaf page before writing it back to disk. Input variables are: + ** + ** nPg: Total initial size of leaf page. + ** iPgIdx: Initial offset of page footer. + ** + ** iOff: Offset to move data to + ** iNextOff: Offset to move data from + */ + if( p->rc==SQLITE_OK ){ + const int nMove = nPg - iNextOff; /* Number of bytes to move */ + int nShift = iNextOff - iOff; /* Distance to move them */ + + int iPrevKeyOut = 0; + int iKeyIn = 0; + + memmove(&aPg[iOff], &aPg[iNextOff], nMove); + iPgIdx -= nShift; + nPg = iPgIdx; + fts5PutU16(&aPg[2], iPgIdx); + + for(iIdx=0; iIdxiOff ? nShift : 0)); + nPg += sqlite3Fts5PutVarint(&aPg[nPg], iKeyOut - iPrevKeyOut); + iPrevKeyOut = iKeyOut; + } + } + + if( iPgIdx==nPg && nIdx>0 && pSeg->iLeafPgno!=1 ){ + fts5SecureDeleteIdxEntry(p, iSegid, pSeg->iLeafPgno); + } + + assert_nc( nPg>4 || fts5GetU16(aPg)==0 ); + fts5DataWrite(p, FTS5_SEGMENT_ROWID(iSegid,pSeg->iLeafPgno), aPg, nPg); + } + sqlite3_free(aIdx); +} + +/* +** This is called as part of flushing a delete to disk in 'secure-delete' +** mode. It edits the segments within the database described by argument +** pStruct to remove the entries for term zTerm, rowid iRowid. +*/ +static void fts5FlushSecureDelete( + Fts5Index *p, + Fts5Structure *pStruct, + const char *zTerm, + int nTerm, + i64 iRowid +){ + const int f = FTS5INDEX_QUERY_SKIPHASH; + Fts5Iter *pIter = 0; /* Used to find term instance */ + + fts5MultiIterNew(p, pStruct, f, 0, (const u8*)zTerm, nTerm, -1, 0, &pIter); + if( fts5MultiIterEof(p, pIter)==0 ){ + i64 iThis = fts5MultiIterRowid(pIter); + if( iThisrc==SQLITE_OK + && fts5MultiIterEof(p, pIter)==0 + && iRowid==fts5MultiIterRowid(pIter) + ){ + Fts5SegIter *pSeg = &pIter->aSeg[pIter->aFirst[1].iFirst]; + fts5DoSecureDelete(p, pSeg); + } + } + + fts5MultiIterFree(pIter); +} + + +/* +** Flush the contents of in-memory hash table iHash to a new level-0 ** segment on disk. Also update the corresponding structure record. ** -** If an error occurs, set the Fts5Index.rc error code. If an error has +** If an error occurs, set the Fts5Index.rc error code. If an error has ** already occurred, this function is a no-op. */ static void fts5FlushOneHash(Fts5Index *p){ @@ -188357,141 +247783,197 @@ static void fts5FlushOneHash(Fts5Index *p){ /* Obtain a reference to the index structure and allocate a new segment-id ** for the new level-0 segment. */ pStruct = fts5StructureRead(p); - iSegid = fts5AllocateSegid(p, pStruct); fts5StructureInvalidate(p); - if( iSegid ){ - const int pgsz = p->pConfig->pgsz; - int eDetail = p->pConfig->eDetail; - Fts5StructureSegment *pSeg; /* New segment within pStruct */ - Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ - Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ + if( sqlite3Fts5HashIsEmpty(pHash)==0 ){ + iSegid = fts5AllocateSegid(p, pStruct); + if( iSegid ){ + const int pgsz = p->pConfig->pgsz; + int eDetail = p->pConfig->eDetail; + int bSecureDelete = p->pConfig->bSecureDelete; + Fts5StructureSegment *pSeg; /* New segment within pStruct */ + Fts5Buffer *pBuf; /* Buffer in which to assemble leaf page */ + Fts5Buffer *pPgidx; /* Buffer in which to assemble pgidx */ - Fts5SegWriter writer; - fts5WriteInit(p, &writer, iSegid); + Fts5SegWriter writer; + fts5WriteInit(p, &writer, iSegid); - pBuf = &writer.writer.buf; - pPgidx = &writer.writer.pgidx; + pBuf = &writer.writer.buf; + pPgidx = &writer.writer.pgidx; - /* fts5WriteInit() should have initialized the buffers to (most likely) - ** the maximum space required. */ - assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + /* fts5WriteInit() should have initialized the buffers to (most likely) + ** the maximum space required. */ + assert( p->rc || pBuf->nSpace>=(pgsz + FTS5_DATA_PADDING) ); + assert( p->rc || pPgidx->nSpace>=(pgsz + FTS5_DATA_PADDING) ); - /* Begin scanning through hash table entries. This loop runs once for each - ** term/doclist currently stored within the hash table. */ - if( p->rc==SQLITE_OK ){ - p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); - } - while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ - const char *zTerm; /* Buffer containing term */ - const u8 *pDoclist; /* Pointer to doclist for this term */ - int nDoclist; /* Size of doclist in bytes */ + /* Begin scanning through hash table entries. This loop runs once for each + ** term/doclist currently stored within the hash table. */ + if( p->rc==SQLITE_OK ){ + p->rc = sqlite3Fts5HashScanInit(pHash, 0, 0); + } + while( p->rc==SQLITE_OK && 0==sqlite3Fts5HashScanEof(pHash) ){ + const char *zTerm; /* Buffer containing term */ + int nTerm; /* Size of zTerm in bytes */ + const u8 *pDoclist; /* Pointer to doclist for this term */ + int nDoclist; /* Size of doclist in bytes */ - /* Write the term for this entry to disk. */ - sqlite3Fts5HashScanEntry(pHash, &zTerm, &pDoclist, &nDoclist); - fts5WriteAppendTerm(p, &writer, (int)strlen(zTerm), (const u8*)zTerm); + /* Get the term and doclist for this entry. */ + sqlite3Fts5HashScanEntry(pHash, &zTerm, &nTerm, &pDoclist, &nDoclist); + if( bSecureDelete==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + if( p->rc!=SQLITE_OK ) break; + assert( writer.bFirstRowidInPage==0 ); + } - assert( writer.bFirstRowidInPage==0 ); - if( pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ - /* The entire doclist will fit on the current leaf. */ - fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); - }else{ - i64 iRowid = 0; - i64 iDelta = 0; - int iOff = 0; + if( !bSecureDelete && pgsz>=(pBuf->n + pPgidx->n + nDoclist + 1) ){ + /* The entire doclist will fit on the current leaf. */ + fts5BufferSafeAppendBlob(pBuf, pDoclist, nDoclist); + }else{ + int bTermWritten = !bSecureDelete; + i64 iRowid = 0; + i64 iPrev = 0; + int iOff = 0; - /* The entire doclist will not fit on this leaf. The following - ** loop iterates through the poslists that make up the current - ** doclist. */ - while( p->rc==SQLITE_OK && iOffp[0], (u16)pBuf->n); /* first rowid on page */ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); - writer.bFirstRowidInPage = 0; - fts5WriteDlidxAppend(p, &writer, iRowid); - }else{ - pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iDelta); - } - assert( pBuf->n<=pBuf->nSpace ); + /* The entire doclist will not fit on this leaf. The following + ** loop iterates through the poslists that make up the current + ** doclist. */ + while( p->rc==SQLITE_OK && iOffp[pBuf->n++] = 0; - iOff++; + /* If in secure delete mode, and if this entry in the poslist is + ** in fact a delete, then edit the existing segments directly + ** using fts5FlushSecureDelete(). */ + if( bSecureDelete ){ + if( eDetail==FTS5_DETAIL_NONE ){ + if( iOffrc!=SQLITE_OK || pDoclist[iOff]==0x01 ){ + iOff++; + continue; + } + } + } + + if( p->rc==SQLITE_OK && bTermWritten==0 ){ + fts5WriteAppendTerm(p, &writer, nTerm, (const u8*)zTerm); + bTermWritten = 1; + assert( p->rc!=SQLITE_OK || writer.bFirstRowidInPage==0 ); + } + + if( writer.bFirstRowidInPage ){ + fts5PutU16(&pBuf->p[0], (u16)pBuf->n); /* first rowid on page */ + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowid); + writer.bFirstRowidInPage = 0; + fts5WriteDlidxAppend(p, &writer, iRowid); + }else{ + u64 iRowidDelta = (u64)iRowid - (u64)iPrev; + pBuf->n += sqlite3Fts5PutVarint(&pBuf->p[pBuf->n], iRowidDelta); + } + if( p->rc!=SQLITE_OK ) break; + assert( pBuf->n<=pBuf->nSpace ); + iPrev = iRowid; + + if( eDetail==FTS5_DETAIL_NONE ){ if( iOffp[pBuf->n++] = 0; iOff++; + if( iOffp[pBuf->n++] = 0; + iOff++; + } + } + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); } - } - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - }else{ - int bDummy; - int nPos; - int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDummy); - nCopy += nPos; - if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ - /* The entire poslist will fit on the current leaf. So copy - ** it in one go. */ - fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); }else{ - /* The entire poslist will not fit on this leaf. So it needs - ** to be broken into sections. The only qualification being - ** that each varint must be stored contiguously. */ - const u8 *pPoslist = &pDoclist[iOff]; - int iPos = 0; - while( p->rc==SQLITE_OK ){ - int nSpace = pgsz - pBuf->n - pPgidx->n; - int n = 0; - if( (nCopy - iPos)<=nSpace ){ - n = nCopy - iPos; - }else{ - n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); - } - assert( n>0 ); - fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); - iPos += n; - if( (pBuf->n + pPgidx->n)>=pgsz ){ - fts5WriteFlushLeaf(p, &writer); - } - if( iPos>=nCopy ) break; + int bDel = 0; + int nPos = 0; + int nCopy = fts5GetPoslistSize(&pDoclist[iOff], &nPos, &bDel); + if( bDel && bSecureDelete ){ + fts5BufferAppendVarint(&p->rc, pBuf, nPos*2); + iOff += nCopy; + nCopy = nPos; + }else{ + nCopy += nPos; } + if( (pBuf->n + pPgidx->n + nCopy) <= pgsz ){ + /* The entire poslist will fit on the current leaf. So copy + ** it in one go. */ + fts5BufferSafeAppendBlob(pBuf, &pDoclist[iOff], nCopy); + }else{ + /* The entire poslist will not fit on this leaf. So it needs + ** to be broken into sections. The only qualification being + ** that each varint must be stored contiguously. */ + const u8 *pPoslist = &pDoclist[iOff]; + int iPos = 0; + while( p->rc==SQLITE_OK ){ + int nSpace = pgsz - pBuf->n - pPgidx->n; + int n = 0; + if( (nCopy - iPos)<=nSpace ){ + n = nCopy - iPos; + }else{ + n = fts5PoslistPrefix(&pPoslist[iPos], nSpace); + } + assert( n>0 ); + fts5BufferSafeAppendBlob(pBuf, &pPoslist[iPos], n); + iPos += n; + if( (pBuf->n + pPgidx->n)>=pgsz ){ + fts5WriteFlushLeaf(p, &writer); + } + if( iPos>=nCopy ) break; + } + } + iOff += nCopy; } - iOff += nCopy; } } + + /* TODO2: Doclist terminator written here. */ + /* pBuf->p[pBuf->n++] = '\0'; */ + assert( pBuf->n<=pBuf->nSpace ); + if( p->rc==SQLITE_OK ) sqlite3Fts5HashScanNext(pHash); } + fts5WriteFinish(p, &writer, &pgnoLast); - /* TODO2: Doclist terminator written here. */ - /* pBuf->p[pBuf->n++] = '\0'; */ - assert( pBuf->n<=pBuf->nSpace ); - sqlite3Fts5HashScanNext(pHash); + assert( p->rc!=SQLITE_OK || bSecureDelete || pgnoLast>0 ); + if( pgnoLast>0 ){ + /* Update the Fts5Structure. It is written back to the database by the + ** fts5StructureRelease() call below. */ + if( pStruct->nLevel==0 ){ + fts5StructureAddLevel(&p->rc, &pStruct); + } + fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); + if( p->rc==SQLITE_OK ){ + pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; + pSeg->iSegid = iSegid; + pSeg->pgnoFirst = 1; + pSeg->pgnoLast = pgnoLast; + if( pStruct->nOriginCntr>0 ){ + pSeg->iOrigin1 = pStruct->nOriginCntr; + pSeg->iOrigin2 = pStruct->nOriginCntr; + pSeg->nEntry = p->nPendingRow; + pStruct->nOriginCntr++; + } + pStruct->nSegment++; + } + fts5StructurePromote(p, 0, pStruct); + } } - sqlite3Fts5HashClear(pHash); - fts5WriteFinish(p, &writer, &pgnoLast); - - /* Update the Fts5Structure. It is written back to the database by the - ** fts5StructureRelease() call below. */ - if( pStruct->nLevel==0 ){ - fts5StructureAddLevel(&p->rc, &pStruct); - } - fts5StructureExtendLevel(&p->rc, pStruct, 0, 1, 0); - if( p->rc==SQLITE_OK ){ - pSeg = &pStruct->aLevel[0].aSeg[ pStruct->aLevel[0].nSeg++ ]; - pSeg->iSegid = iSegid; - pSeg->pgnoFirst = 1; - pSeg->pgnoLast = pgnoLast; - pStruct->nSegment++; - } - fts5StructurePromote(p, 0, pStruct); } - fts5IndexAutomerge(p, &pStruct, pgnoLast); + fts5IndexAutomerge(p, &pStruct, pgnoLast + p->nContentlessDelete); fts5IndexCrisismerge(p, &pStruct); fts5StructureWrite(p, pStruct); fts5StructureRelease(pStruct); @@ -188502,36 +247984,52 @@ static void fts5FlushOneHash(Fts5Index *p){ */ static void fts5IndexFlush(Fts5Index *p){ /* Unless it is empty, flush the hash table to disk */ - if( p->nPendingData ){ + if( p->flushRc ){ + p->rc = p->flushRc; + return; + } + if( p->nPendingData || p->nContentlessDelete ){ assert( p->pHash ); - p->nPendingData = 0; fts5FlushOneHash(p); + if( p->rc==SQLITE_OK ){ + sqlite3Fts5HashClear(p->pHash); + p->nPendingData = 0; + p->nPendingRow = 0; + p->nContentlessDelete = 0; + }else if( p->nPendingData || p->nContentlessDelete ){ + p->flushRc = p->rc; + } } } static Fts5Structure *fts5IndexOptimizeStruct( - Fts5Index *p, + Fts5Index *p, Fts5Structure *pStruct ){ Fts5Structure *pNew = 0; - int nByte = sizeof(Fts5Structure); + sqlite3_int64 nByte = sizeof(Fts5Structure); int nSeg = pStruct->nSegment; int i; /* Figure out if this structure requires optimization. A structure does ** not require optimization if either: ** - ** + it consists of fewer than two segments, or - ** + all segments are on the same level, or - ** + all segments except one are currently inputs to a merge operation. + ** 1. it consists of fewer than two segments, or + ** 2. all segments are on the same level, or + ** 3. all segments except one are currently inputs to a merge operation. ** - ** In the first case, return NULL. In the second, increment the ref-count - ** on *pStruct and return a copy of the pointer to it. + ** In the first case, if there are no tombstone hash pages, return NULL. In + ** the second, increment the ref-count on *pStruct and return a copy of the + ** pointer to it. */ - if( nSeg<2 ) return 0; + if( nSeg==0 ) return 0; for(i=0; inLevel; i++){ int nThis = pStruct->aLevel[i].nSeg; - if( nThis==nSeg || (nThis==nSeg-1 && pStruct->aLevel[i].nMerge==nThis) ){ + int nMerge = pStruct->aLevel[i].nMerge; + if( nThis>0 && (nThis==nSeg || (nThis==nSeg-1 && nMerge==nThis)) ){ + if( nSeg==1 && nThis==1 && pStruct->aLevel[i].aSeg[0].nPgTombstone==0 ){ + return 0; + } fts5StructureRef(pStruct); return pStruct; } @@ -188544,10 +248042,11 @@ static Fts5Structure *fts5IndexOptimizeStruct( if( pNew ){ Fts5StructureLevel *pLvl; nByte = nSeg * sizeof(Fts5StructureSegment); - pNew->nLevel = pStruct->nLevel+1; + pNew->nLevel = MIN(pStruct->nLevel+1, FTS5_MAX_LEVEL); pNew->nRef = 1; pNew->nWriteCounter = pStruct->nWriteCounter; - pLvl = &pNew->aLevel[pStruct->nLevel]; + pNew->nOriginCntr = pStruct->nOriginCntr; + pLvl = &pNew->aLevel[pNew->nLevel-1]; pLvl->aSeg = (Fts5StructureSegment*)sqlite3Fts5MallocZero(&p->rc, nByte); if( pLvl->aSeg ){ int iLvl, iSeg; @@ -188577,7 +248076,9 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); + assert( p->rc!=SQLITE_OK || p->nContentlessDelete==0 ); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || pStruct!=0 ); fts5StructureInvalidate(p); if( pStruct ){ @@ -188598,7 +248099,7 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ fts5StructureRelease(pNew); } - return fts5IndexReturn(p); + return fts5IndexReturn(p); } /* @@ -188606,7 +248107,10 @@ static int sqlite3Fts5IndexOptimize(Fts5Index *p){ ** INSERT command. */ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ - Fts5Structure *pStruct = fts5StructureRead(p); + Fts5Structure *pStruct = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); if( pStruct ){ int nMin = p->pConfig->nUsermerge; fts5StructureInvalidate(p); @@ -188614,7 +248118,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ Fts5Structure *pNew = fts5IndexOptimizeStruct(p, pStruct); fts5StructureRelease(pStruct); pStruct = pNew; - nMin = 2; + nMin = 1; nMerge = nMerge*-1; } if( pStruct && pStruct->nLevel ){ @@ -188629,7 +248133,7 @@ static int sqlite3Fts5IndexMerge(Fts5Index *p, int nMerge){ static void fts5AppendRowid( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pUnused, Fts5Buffer *pBuf ){ @@ -188639,16 +248143,18 @@ static void fts5AppendRowid( static void fts5AppendPoslist( Fts5Index *p, - i64 iDelta, + u64 iDelta, Fts5Iter *pMulti, Fts5Buffer *pBuf ){ int nData = pMulti->base.nData; + int nByte = nData + 9 + 9 + FTS5_DATA_ZERO_PADDING; assert( nData>0 ); - if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nData+9+9) ){ + if( p->rc==SQLITE_OK && 0==fts5BufferGrow(&p->rc, pBuf, nByte) ){ fts5BufferSafeAppendVarint(pBuf, iDelta); fts5BufferSafeAppendVarint(pBuf, nData*2); fts5BufferSafeAppendBlob(pBuf, pMulti->base.pData, nData); + memset(&pBuf->p[pBuf->n], 0, FTS5_DATA_ZERO_PADDING); } } @@ -188656,7 +248162,7 @@ static void fts5AppendPoslist( static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ u8 *p = pIter->aPoslist + pIter->nSize + pIter->nPoslist; - assert( pIter->aPoslist ); + assert( pIter->aPoslist || (p==0 && pIter->aPoslist==0) ); if( p>=pIter->aEof ){ pIter->aPoslist = 0; }else{ @@ -188676,17 +248182,22 @@ static void fts5DoclistIterNext(Fts5DoclistIter *pIter){ } pIter->aPoslist = p; + if( &pIter->aPoslist[pIter->nPoslist]>pIter->aEof ){ + pIter->aPoslist = 0; + } } } static void fts5DoclistIterInit( - Fts5Buffer *pBuf, + Fts5Buffer *pBuf, Fts5DoclistIter *pIter ){ memset(pIter, 0, sizeof(*pIter)); - pIter->aPoslist = pBuf->p; - pIter->aEof = &pBuf->p[pBuf->n]; - fts5DoclistIterNext(pIter); + if( pBuf->n>0 ){ + pIter->aPoslist = pBuf->p; + pIter->aEof = &pBuf->p[pBuf->n]; + fts5DoclistIterNext(pIter); + } } #if 0 @@ -188707,10 +248218,10 @@ static void fts5MergeAppendDocid( } #endif -#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ - assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ - fts5BufferSafeAppendVarint((pBuf), (iRowid) - (iLastRowid)); \ - (iLastRowid) = (iRowid); \ +#define fts5MergeAppendDocid(pBuf, iLastRowid, iRowid) { \ + assert( (pBuf)->n!=0 || (iLastRowid)==0 ); \ + fts5BufferSafeAppendVarint((pBuf), (u64)(iRowid) - (u64)(iLastRowid)); \ + (iLastRowid) = (iRowid); \ } /* @@ -188740,16 +248251,20 @@ static void fts5NextRowid(Fts5Buffer *pBuf, int *piOff, i64 *piRowid){ static void fts5MergeRowidLists( Fts5Index *p, /* FTS5 backend object */ Fts5Buffer *p1, /* First list to merge */ - Fts5Buffer *p2 /* Second list to merge */ + int nBuf, /* Number of entries in apBuf[] */ + Fts5Buffer *aBuf /* Array of other lists to merge into p1 */ ){ int i1 = 0; int i2 = 0; i64 iRowid1 = 0; i64 iRowid2 = 0; i64 iOut = 0; - + Fts5Buffer *p2 = &aBuf[0]; Fts5Buffer out; + + (void)nBuf; memset(&out, 0, sizeof(out)); + assert( nBuf==1 ); sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n); if( p->rc ) return; @@ -188776,155 +248291,225 @@ static void fts5MergeRowidLists( fts5BufferFree(&out); } +typedef struct PrefixMerger PrefixMerger; +struct PrefixMerger { + Fts5DoclistIter iter; /* Doclist iterator */ + i64 iPos; /* For iterating through a position list */ + int iOff; + u8 *aPos; + PrefixMerger *pNext; /* Next in docid/poslist order */ +}; + +static void fts5PrefixMergerInsertByRowid( + PrefixMerger **ppHead, + PrefixMerger *p +){ + if( p->iter.aPoslist ){ + PrefixMerger **pp = ppHead; + while( *pp && p->iter.iRowid>(*pp)->iter.iRowid ){ + pp = &(*pp)->pNext; + } + p->pNext = *pp; + *pp = p; + } +} + +static void fts5PrefixMergerInsertByPosition( + PrefixMerger **ppHead, + PrefixMerger *p +){ + if( p->iPos>=0 ){ + PrefixMerger **pp = ppHead; + while( *pp && p->iPos>(*pp)->iPos ){ + pp = &(*pp)->pNext; + } + p->pNext = *pp; + *pp = p; + } +} + + /* -** Buffers p1 and p2 contain doclists. This function merges the content -** of the two doclists together and sets buffer p1 to the result before -** returning. -** -** If an error occurs, an error code is left in p->rc. If an error has -** already occurred, this function is a no-op. +** Array aBuf[] contains nBuf doclists. These are all merged in with the +** doclist in buffer p1. */ static void fts5MergePrefixLists( Fts5Index *p, /* FTS5 backend object */ Fts5Buffer *p1, /* First list to merge */ - Fts5Buffer *p2 /* Second list to merge */ + int nBuf, /* Number of buffers in array aBuf[] */ + Fts5Buffer *aBuf /* Other lists to merge in */ ){ - if( p2->n ){ - i64 iLastRowid = 0; - Fts5DoclistIter i1; - Fts5DoclistIter i2; - Fts5Buffer out = {0, 0, 0}; - Fts5Buffer tmp = {0, 0, 0}; +#define fts5PrefixMergerNextPosition(p) \ + sqlite3Fts5PoslistNext64((p)->aPos,(p)->iter.nPoslist,&(p)->iOff,&(p)->iPos) +#define FTS5_MERGE_NLIST 16 + PrefixMerger aMerger[FTS5_MERGE_NLIST]; + PrefixMerger *pHead = 0; + int i; + int nOut = 0; + Fts5Buffer out = {0, 0, 0}; + Fts5Buffer tmp = {0, 0, 0}; + i64 iLastRowid = 0; - if( sqlite3Fts5BufferSize(&p->rc, &out, p1->n + p2->n) ) return; - fts5DoclistIterInit(p1, &i1); - fts5DoclistIterInit(p2, &i2); - - while( 1 ){ - if( i1.iRowidrc, &tmp, i1.nPoslist + i2.nPoslist); - if( p->rc ) break; - - sqlite3Fts5PoslistNext64(a1, i1.nPoslist, &iOff1, &iPos1); - sqlite3Fts5PoslistNext64(a2, i2.nPoslist, &iOff2, &iPos2); - assert( iPos1>=0 && iPos2>=0 ); - - if( iPos1=0 && iPos2>=0 ){ - while( 1 ){ - if( iPos1=0 ){ - if( iPos1!=iPrev ){ - sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos1); - } - fts5BufferSafeAppendBlob(&tmp, &a1[iOff1], i1.nPoslist-iOff1); - }else{ - assert( iPos2>=0 && iPos2!=iPrev ); - sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, iPos2); - fts5BufferSafeAppendBlob(&tmp, &a2[iOff2], i2.nPoslist-iOff2); - } - - /* WRITEPOSLISTSIZE */ - fts5BufferSafeAppendVarint(&out, tmp.n * 2); - fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); - fts5DoclistIterNext(&i1); - fts5DoclistIterNext(&i2); - if( i1.aPoslist==0 || i2.aPoslist==0 ) break; - } - } - - if( i1.aPoslist ){ - fts5MergeAppendDocid(&out, iLastRowid, i1.iRowid); - fts5BufferSafeAppendBlob(&out, i1.aPoslist, i1.aEof - i1.aPoslist); - } - else if( i2.aPoslist ){ - fts5MergeAppendDocid(&out, iLastRowid, i2.iRowid); - fts5BufferSafeAppendBlob(&out, i2.aPoslist, i2.aEof - i2.aPoslist); - } - - fts5BufferSet(&p->rc, p1, out.n, out.p); - fts5BufferFree(&tmp); - fts5BufferFree(&out); + /* Initialize a doclist-iterator for each input buffer. Arrange them in + ** a linked-list starting at pHead in ascending order of rowid. Avoid + ** linking any iterators already at EOF into the linked list at all. */ + assert( nBuf+1<=(int)(sizeof(aMerger)/sizeof(aMerger[0])) ); + memset(aMerger, 0, sizeof(PrefixMerger)*(nBuf+1)); + pHead = &aMerger[nBuf]; + fts5DoclistIterInit(p1, &pHead->iter); + for(i=0; in + 9 + 10*nBuf; + + /* The maximum size of the output is equal to the sum of the + ** input sizes + 1 varint (9 bytes). The extra varint is because if the + ** first rowid in one input is a large negative number, and the first in + ** the other a non-negative number, the delta for the non-negative + ** number will be larger on disk than the literal integer value + ** was. + ** + ** Or, if the input position-lists are corrupt, then the output might + ** include up to (nBuf+1) extra 10-byte positions created by interpreting -1 + ** (the value PoslistNext64() uses for EOF) as a position and appending + ** it to the output. This can happen at most once for each input + ** position-list, hence (nBuf+1) 10 byte paddings. */ + if( sqlite3Fts5BufferSize(&p->rc, &out, nOut) ) return; + + while( pHead ){ + fts5MergeAppendDocid(&out, iLastRowid, pHead->iter.iRowid); + + if( pHead->pNext && iLastRowid==pHead->pNext->iter.iRowid ){ + /* Merge data from two or more poslists */ + i64 iPrev = 0; + int nTmp = FTS5_DATA_ZERO_PADDING; + int nMerge = 0; + PrefixMerger *pSave = pHead; + PrefixMerger *pThis = 0; + int nTail = 0; + + pHead = 0; + while( pSave && pSave->iter.iRowid==iLastRowid ){ + PrefixMerger *pNext = pSave->pNext; + pSave->iOff = 0; + pSave->iPos = 0; + pSave->aPos = &pSave->iter.aPoslist[pSave->iter.nSize]; + fts5PrefixMergerNextPosition(pSave); + nTmp += pSave->iter.nPoslist + 10; + nMerge++; + fts5PrefixMergerInsertByPosition(&pHead, pSave); + pSave = pNext; + } + + if( pHead==0 || pHead->pNext==0 ){ + p->rc = FTS5_CORRUPT; + break; + } + + /* See the earlier comment in this function for an explanation of why + ** corrupt input position lists might cause the output to consume + ** at most nMerge*10 bytes of unexpected space. */ + if( sqlite3Fts5BufferSize(&p->rc, &tmp, nTmp+nMerge*10) ){ + break; + } + fts5BufferZero(&tmp); + + pThis = pHead; + pHead = pThis->pNext; + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos); + fts5PrefixMergerNextPosition(pThis); + fts5PrefixMergerInsertByPosition(&pHead, pThis); + + while( pHead->pNext ){ + pThis = pHead; + if( pThis->iPos!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pThis->iPos); + } + fts5PrefixMergerNextPosition(pThis); + pHead = pThis->pNext; + fts5PrefixMergerInsertByPosition(&pHead, pThis); + } + + if( pHead->iPos!=iPrev ){ + sqlite3Fts5PoslistSafeAppend(&tmp, &iPrev, pHead->iPos); + } + nTail = pHead->iter.nPoslist - pHead->iOff; + + /* WRITEPOSLISTSIZE */ + assert_nc( tmp.n+nTail<=nTmp ); + assert( tmp.n+nTail<=nTmp+nMerge*10 ); + if( tmp.n+nTail>nTmp-FTS5_DATA_ZERO_PADDING ){ + if( p->rc==SQLITE_OK ) p->rc = FTS5_CORRUPT; + break; + } + fts5BufferSafeAppendVarint(&out, (tmp.n+nTail) * 2); + fts5BufferSafeAppendBlob(&out, tmp.p, tmp.n); + if( nTail>0 ){ + fts5BufferSafeAppendBlob(&out, &pHead->aPos[pHead->iOff], nTail); + } + + pHead = pSave; + for(i=0; iiter.aPoslist && pX->iter.iRowid==iLastRowid ){ + fts5DoclistIterNext(&pX->iter); + fts5PrefixMergerInsertByRowid(&pHead, pX); + } + } + + }else{ + /* Copy poslist from pHead to output */ + PrefixMerger *pThis = pHead; + Fts5DoclistIter *pI = &pThis->iter; + fts5BufferSafeAppendBlob(&out, pI->aPoslist, pI->nPoslist+pI->nSize); + fts5DoclistIterNext(pI); + pHead = pThis->pNext; + fts5PrefixMergerInsertByRowid(&pHead, pThis); + } + } + + fts5BufferFree(p1); + fts5BufferFree(&tmp); + memset(&out.p[out.n], 0, FTS5_DATA_ZERO_PADDING); + *p1 = out; } static void fts5SetupPrefixIter( Fts5Index *p, /* Index to read from */ int bDesc, /* True for "ORDER BY rowid DESC" */ - const u8 *pToken, /* Buffer containing prefix to match */ + int iIdx, /* Index to scan for data */ + u8 *pToken, /* Buffer containing prefix to match */ int nToken, /* Size of buffer pToken in bytes */ Fts5Colset *pColset, /* Restrict matches to these columns */ - Fts5Iter **ppIter /* OUT: New iterator */ + Fts5Iter **ppIter /* OUT: New iterator */ ){ Fts5Structure *pStruct; Fts5Buffer *aBuf; - const int nBuf = 32; + int nBuf = 32; + int nMerge = 1; - void (*xMerge)(Fts5Index*, Fts5Buffer*, Fts5Buffer*); - void (*xAppend)(Fts5Index*, i64, Fts5Iter*, Fts5Buffer*); + void (*xMerge)(Fts5Index*, Fts5Buffer*, int, Fts5Buffer*); + void (*xAppend)(Fts5Index*, u64, Fts5Iter*, Fts5Buffer*); if( p->pConfig->eDetail==FTS5_DETAIL_NONE ){ xMerge = fts5MergeRowidLists; xAppend = fts5AppendRowid; }else{ + nMerge = FTS5_MERGE_NLIST-1; + nBuf = nMerge*8; /* Sufficient to merge (16^8)==(2^32) lists */ xMerge = fts5MergePrefixLists; xAppend = fts5AppendPoslist; } aBuf = (Fts5Buffer*)fts5IdxMalloc(p, sizeof(Fts5Buffer)*nBuf); pStruct = fts5StructureRead(p); + assert( p->rc!=SQLITE_OK || (aBuf && pStruct) ); - if( aBuf && pStruct ){ - const int flags = FTS5INDEX_QUERY_SCAN - | FTS5INDEX_QUERY_SKIPEMPTY + if( p->rc==SQLITE_OK ){ + const int flags = FTS5INDEX_QUERY_SCAN + | FTS5INDEX_QUERY_SKIPEMPTY | FTS5INDEX_QUERY_NOOUTPUT; int i; i64 iLastRowid = 0; @@ -188934,8 +248519,36 @@ static void fts5SetupPrefixIter( int bNewTerm = 1; memset(&doclist, 0, sizeof(doclist)); + + /* If iIdx is non-zero, then it is the number of a prefix-index for + ** prefixes 1 character longer than the prefix being queried for. That + ** index contains all the doclists required, except for the one + ** corresponding to the prefix itself. That one is extracted from the + ** main term index here. */ + if( iIdx!=0 ){ + int dummy = 0; + const int f2 = FTS5INDEX_QUERY_SKIPEMPTY|FTS5INDEX_QUERY_NOOUTPUT; + pToken[0] = FTS5_MAIN_PREFIX; + fts5MultiIterNew(p, pStruct, f2, pColset, pToken, nToken, -1, 0, &p1); + fts5IterSetOutputCb(&p->rc, p1); + for(; + fts5MultiIterEof(p, p1)==0; + fts5MultiIterNext2(p, p1, &dummy) + ){ + Fts5SegIter *pSeg = &p1->aSeg[ p1->aFirst[1].iFirst ]; + p1->xSetOutputs(p1, pSeg); + if( p1->base.nData ){ + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); + iLastRowid = p1->base.iRowid; + } + } + fts5MultiIterFree(p1); + } + + pToken[0] = FTS5_MAIN_PREFIX + iIdx; fts5MultiIterNew(p, pStruct, flags, pColset, pToken, nToken, -1, 0, &p1); fts5IterSetOutputCb(&p->rc, p1); + for( /* no-op */ ; fts5MultiIterEof(p, p1)==0; fts5MultiIterNext2(p, p1, &bNewTerm) @@ -188951,38 +248564,49 @@ static void fts5SetupPrefixIter( } if( p1->base.nData==0 ) continue; - if( p1->base.iRowid<=iLastRowid && doclist.n>0 ){ for(i=0; p->rc==SQLITE_OK && doclist.n; i++){ - assert( ibase.iRowid-iLastRowid, p1, &doclist); + xAppend(p, (u64)p1->base.iRowid-(u64)iLastRowid, p1, &doclist); iLastRowid = p1->base.iRowid; } - for(i=0; irc==SQLITE_OK ){ - xMerge(p, &doclist, &aBuf[i]); + xMerge(p, &doclist, nMerge, &aBuf[i]); + } + for(iFree=i; iFreep = (u8*)&pData[1]; pData->nn = pData->szLeaf = doclist.n; - memcpy(pData->p, doclist.p, doclist.n); + if( doclist.n ) memcpy(pData->p, doclist.p, doclist.n); fts5MultiIterNew2(p, pData, bDesc, ppIter); } fts5BufferFree(&doclist); @@ -189006,36 +248630,39 @@ static int sqlite3Fts5IndexBeginWrite(Fts5Index *p, int bDelete, i64 iRowid){ } /* Flush the hash table to disk if required */ - if( iRowidiWriteRowid + if( iRowidiWriteRowid || (iRowid==p->iWriteRowid && p->bDelete==0) - || (p->nPendingData > p->pConfig->nHashSize) + || (p->nPendingData > p->pConfig->nHashSize) ){ fts5IndexFlush(p); } p->iWriteRowid = iRowid; p->bDelete = bDelete; + if( bDelete==0 ){ + p->nPendingRow++; + } return fts5IndexReturn(p); } /* ** Commit data to disk. */ -static int sqlite3Fts5IndexSync(Fts5Index *p, int bCommit){ +static int sqlite3Fts5IndexSync(Fts5Index *p){ assert( p->rc==SQLITE_OK ); fts5IndexFlush(p); - if( bCommit ) fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); return fts5IndexReturn(p); } /* ** Discard any data stored in the in-memory hash tables. Do not write it ** to the database. Additionally, assume that the contents of the %_data -** table may have changed on disk. So any in-memory caches of %_data +** table may have changed on disk. So any in-memory caches of %_data ** records must be invalidated. */ static int sqlite3Fts5IndexRollback(Fts5Index *p){ - fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); fts5IndexDiscardData(p); fts5StructureInvalidate(p); /* assert( p->rc==SQLITE_OK ); */ @@ -189050,7 +248677,11 @@ static int sqlite3Fts5IndexRollback(Fts5Index *p){ static int sqlite3Fts5IndexReinit(Fts5Index *p){ Fts5Structure s; fts5StructureInvalidate(p); + fts5IndexDiscardData(p); memset(&s, 0, sizeof(Fts5Structure)); + if( p->pConfig->bContentlessDelete ){ + s.nOriginCntr = 1; + } fts5DataWrite(p, FTS5_AVERAGES_ROWID, (const u8*)"", 0); fts5StructureWrite(p, &s); return fts5IndexReturn(p); @@ -189064,8 +248695,8 @@ static int sqlite3Fts5IndexReinit(Fts5Index *p){ ** Otherwise, set *pp to NULL and return an SQLite error code. */ static int sqlite3Fts5IndexOpen( - Fts5Config *pConfig, - int bCreate, + Fts5Config *pConfig, + int bCreate, Fts5Index **pp, char **pzErr ){ @@ -189082,8 +248713,8 @@ static int sqlite3Fts5IndexOpen( pConfig, "data", "id INTEGER PRIMARY KEY, block BLOB", 0, pzErr ); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5CreateTable(pConfig, "idx", - "segid, term, pgno, PRIMARY KEY(segid, term)", + rc = sqlite3Fts5CreateTable(pConfig, "idx", + "segid, term, pgno, PRIMARY KEY(segid, term)", 1, pzErr ); } @@ -189114,7 +248745,9 @@ static int sqlite3Fts5IndexClose(Fts5Index *p){ sqlite3_finalize(p->pIdxWriter); sqlite3_finalize(p->pIdxDeleter); sqlite3_finalize(p->pIdxSelect); + sqlite3_finalize(p->pIdxNextSelect); sqlite3_finalize(p->pDataVersion); + sqlite3_finalize(p->pDeleteFromIdx); sqlite3Fts5HashFree(p->pHash); sqlite3_free(p->zDataTbl); sqlite3_free(p); @@ -189123,13 +248756,13 @@ static int sqlite3Fts5IndexClose(Fts5Index *p){ } /* -** Argument p points to a buffer containing utf-8 text that is n bytes in +** Argument p points to a buffer containing utf-8 text that is n bytes in ** size. Return the number of bytes in the nChar character prefix of the ** buffer, or 0 if there are less than nChar characters in total. */ static int sqlite3Fts5IndexCharlenToBytelen( - const char *p, - int nByte, + const char *p, + int nByte, int nChar ){ int n = 0; @@ -189137,7 +248770,14 @@ static int sqlite3Fts5IndexCharlenToBytelen( for(i=0; i=nByte ) return 0; /* Input contains fewer than nChar chars */ if( (unsigned char)p[n++]>=0xc0 ){ - while( (p[n] & 0xc0)==0x80 ) n++; + if( n>=nByte ) return 0; + while( (p[n] & 0xc0)==0x80 ){ + n++; + if( n>=nByte ){ + if( i+1==nChar ) break; + return 0; + } + } } } return n; @@ -189148,7 +248788,7 @@ static int sqlite3Fts5IndexCharlenToBytelen( ** unicode characters in the string. */ static int fts5IndexCharlen(const char *pIn, int nIn){ - int nChar = 0; + int nChar = 0; int i = 0; while( i=0xc0 ){ @@ -189160,7 +248800,7 @@ static int fts5IndexCharlen(const char *pIn, int nIn){ } /* -** Insert or remove data to or from the index. Each time a document is +** Insert or remove data to or from the index. Each time a document is ** added to or removed from the index, this function is called one or more ** times. ** @@ -189191,7 +248831,7 @@ static int sqlite3Fts5IndexWrite( const int nChar = pConfig->aPrefix[i]; int nByte = sqlite3Fts5IndexCharlenToBytelen(pToken, nToken, nChar); if( nByte ){ - rc = sqlite3Fts5HashWrite(p->pHash, + rc = sqlite3Fts5HashWrite(p->pHash, p->iWriteRowid, iCol, iPos, (char)(FTS5_MAIN_PREFIX+i+1), pToken, nByte ); @@ -189202,7 +248842,458 @@ static int sqlite3Fts5IndexWrite( } /* -** Open a new iterator to iterate though all rowid that match the +** pToken points to a buffer of size nToken bytes containing a search +** term, including the index number at the start, used on a tokendata=1 +** table. This function returns true if the term in buffer pBuf matches +** token pToken/nToken. +*/ +static int fts5IsTokendataPrefix( + Fts5Buffer *pBuf, + const u8 *pToken, + int nToken +){ + return ( + pBuf->n>=nToken + && 0==memcmp(pBuf->p, pToken, nToken) + && (pBuf->n==nToken || pBuf->p[nToken]==0x00) + ); +} + +/* +** Ensure the segment-iterator passed as the only argument points to EOF. +*/ +static void fts5SegIterSetEOF(Fts5SegIter *pSeg){ + fts5DataRelease(pSeg->pLeaf); + pSeg->pLeaf = 0; +} + +/* +** Usually, a tokendata=1 iterator (struct Fts5TokenDataIter) accumulates an +** array of these for each row it visits. Or, for an iterator used by an +** "ORDER BY rank" query, it accumulates an array of these for the entire +** query. +** +** Each instance in the array indicates the iterator (and therefore term) +** associated with position iPos of rowid iRowid. This is used by the +** xInstToken() API. +*/ +struct Fts5TokenDataMap { + i64 iRowid; /* Row this token is located in */ + i64 iPos; /* Position of token */ + int iIter; /* Iterator token was read from */ +}; + +/* +** An object used to supplement Fts5Iter for tokendata=1 iterators. +*/ +struct Fts5TokenDataIter { + int nIter; + int nIterAlloc; + + int nMap; + int nMapAlloc; + Fts5TokenDataMap *aMap; + + Fts5PoslistReader *aPoslistReader; + int *aPoslistToIter; + Fts5Iter *apIter[1]; +}; + +/* +** This function appends iterator pAppend to Fts5TokenDataIter pIn and +** returns the result. +*/ +static Fts5TokenDataIter *fts5AppendTokendataIter( + Fts5Index *p, /* Index object (for error code) */ + Fts5TokenDataIter *pIn, /* Current Fts5TokenDataIter struct */ + Fts5Iter *pAppend /* Append this iterator */ +){ + Fts5TokenDataIter *pRet = pIn; + + if( p->rc==SQLITE_OK ){ + if( pIn==0 || pIn->nIter==pIn->nIterAlloc ){ + int nAlloc = pIn ? pIn->nIterAlloc*2 : 16; + int nByte = nAlloc * sizeof(Fts5Iter*) + sizeof(Fts5TokenDataIter); + Fts5TokenDataIter *pNew = (Fts5TokenDataIter*)sqlite3_realloc(pIn, nByte); + + if( pNew==0 ){ + p->rc = SQLITE_NOMEM; + }else{ + if( pIn==0 ) memset(pNew, 0, nByte); + pRet = pNew; + pNew->nIterAlloc = nAlloc; + } + } + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pAppend); + }else{ + pRet->apIter[pRet->nIter++] = pAppend; + } + assert( pRet==0 || pRet->nIter<=pRet->nIterAlloc ); + + return pRet; +} + +/* +** Delete an Fts5TokenDataIter structure and its contents. +*/ +static void fts5TokendataIterDelete(Fts5TokenDataIter *pSet){ + if( pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + fts5MultiIterFree(pSet->apIter[ii]); + } + sqlite3_free(pSet->aPoslistReader); + sqlite3_free(pSet->aMap); + sqlite3_free(pSet); + } +} + +/* +** Append a mapping to the token-map belonging to object pT. +*/ +static void fts5TokendataIterAppendMap( + Fts5Index *p, + Fts5TokenDataIter *pT, + int iIter, + i64 iRowid, + i64 iPos +){ + if( p->rc==SQLITE_OK ){ + if( pT->nMap==pT->nMapAlloc ){ + int nNew = pT->nMapAlloc ? pT->nMapAlloc*2 : 64; + int nByte = nNew * sizeof(Fts5TokenDataMap); + Fts5TokenDataMap *aNew; + + aNew = (Fts5TokenDataMap*)sqlite3_realloc(pT->aMap, nByte); + if( aNew==0 ){ + p->rc = SQLITE_NOMEM; + return; + } + + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pT->aMap[pT->nMap].iRowid = iRowid; + pT->aMap[pT->nMap].iPos = iPos; + pT->aMap[pT->nMap].iIter = iIter; + pT->nMap++; + } +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function sets the iterator output +** variables (pIter->base.*) according to the contents of the current +** row. +*/ +static void fts5IterSetOutputsTokendata(Fts5Iter *pIter){ + int ii; + int nHit = 0; + i64 iRowid = SMALLEST_INT64; + int iMin = 0; + + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + + pIter->base.nData = 0; + pIter->base.pData = 0; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 ){ + if( nHit==0 || p->base.iRowidbase.iRowid; + nHit = 1; + pIter->base.pData = p->base.pData; + pIter->base.nData = p->base.nData; + iMin = ii; + }else if( p->base.iRowid==iRowid ){ + nHit++; + } + } + } + + if( nHit==0 ){ + pIter->base.bEof = 1; + }else{ + int eDetail = pIter->pIndex->pConfig->eDetail; + pIter->base.bEof = 0; + pIter->base.iRowid = iRowid; + + if( nHit==1 && eDetail==FTS5_DETAIL_FULL ){ + fts5TokendataIterAppendMap(pIter->pIndex, pT, iMin, iRowid, -1); + }else + if( nHit>1 && eDetail!=FTS5_DETAIL_NONE ){ + int nReader = 0; + int nByte = 0; + i64 iPrev = 0; + + /* Allocate array of iterators if they are not already allocated. */ + if( pT->aPoslistReader==0 ){ + pT->aPoslistReader = (Fts5PoslistReader*)sqlite3Fts5MallocZero( + &pIter->pIndex->rc, + pT->nIter * (sizeof(Fts5PoslistReader) + sizeof(int)) + ); + if( pT->aPoslistReader==0 ) return; + pT->aPoslistToIter = (int*)&pT->aPoslistReader[pT->nIter]; + } + + /* Populate an iterator for each poslist that will be merged */ + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( iRowid==p->base.iRowid ){ + pT->aPoslistToIter[nReader] = ii; + sqlite3Fts5PoslistReaderInit( + p->base.pData, p->base.nData, &pT->aPoslistReader[nReader++] + ); + nByte += p->base.nData; + } + } + + /* Ensure the output buffer is large enough */ + if( fts5BufferGrow(&pIter->pIndex->rc, &pIter->poslist, nByte+nHit*10) ){ + return; + } + + /* Ensure the token-mapping is large enough */ + if( eDetail==FTS5_DETAIL_FULL && pT->nMapAlloc<(pT->nMap + nByte) ){ + int nNew = (pT->nMapAlloc + nByte) * 2; + Fts5TokenDataMap *aNew = (Fts5TokenDataMap*)sqlite3_realloc( + pT->aMap, nNew*sizeof(Fts5TokenDataMap) + ); + if( aNew==0 ){ + pIter->pIndex->rc = SQLITE_NOMEM; + return; + } + pT->aMap = aNew; + pT->nMapAlloc = nNew; + } + + pIter->poslist.n = 0; + + while( 1 ){ + i64 iMinPos = LARGEST_INT64; + + /* Find smallest position */ + iMin = 0; + for(ii=0; iiaPoslistReader[ii]; + if( pReader->bEof==0 ){ + if( pReader->iPosiPos; + iMin = ii; + } + } + } + + /* If all readers were at EOF, break out of the loop. */ + if( iMinPos==LARGEST_INT64 ) break; + + sqlite3Fts5PoslistSafeAppend(&pIter->poslist, &iPrev, iMinPos); + sqlite3Fts5PoslistReaderNext(&pT->aPoslistReader[iMin]); + + if( eDetail==FTS5_DETAIL_FULL ){ + pT->aMap[pT->nMap].iPos = iMinPos; + pT->aMap[pT->nMap].iIter = pT->aPoslistToIter[iMin]; + pT->aMap[pT->nMap].iRowid = iRowid; + pT->nMap++; + } + } + + pIter->base.pData = pIter->poslist.p; + pIter->base.nData = pIter->poslist.n; + } + } +} + +/* +** The iterator passed as the only argument must be a tokendata=1 iterator +** (pIter->pTokenDataIter!=0). This function advances the iterator. If +** argument bFrom is false, then the iterator is advanced to the next +** entry. Or, if bFrom is true, it is advanced to the first entry with +** a rowid of iFrom or greater. +*/ +static void fts5TokendataIterNext(Fts5Iter *pIter, int bFrom, i64 iFrom){ + int ii; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *pIndex = pIter->pIndex; + + for(ii=0; iinIter; ii++){ + Fts5Iter *p = pT->apIter[ii]; + if( p->base.bEof==0 + && (p->base.iRowid==pIter->base.iRowid || (bFrom && p->base.iRowidbase.bEof==0 + && p->base.iRowidrc==SQLITE_OK + ){ + fts5MultiIterNext(pIndex, p, 0, 0); + } + } + } + + if( pIndex->rc==SQLITE_OK ){ + fts5IterSetOutputsTokendata(pIter); + } +} + +/* +** If the segment-iterator passed as the first argument is at EOF, then +** set pIter->term to a copy of buffer pTerm. +*/ +static void fts5TokendataSetTermIfEof(Fts5Iter *pIter, Fts5Buffer *pTerm){ + if( pIter && pIter->aSeg[0].pLeaf==0 ){ + fts5BufferSet(&pIter->pIndex->rc, &pIter->aSeg[0].term, pTerm->n, pTerm->p); + } +} + +/* +** This function sets up an iterator to use for a non-prefix query on a +** tokendata=1 table. +*/ +static Fts5Iter *fts5SetupTokendataIter( + Fts5Index *p, /* FTS index to query */ + const u8 *pToken, /* Buffer containing query term */ + int nToken, /* Size of buffer pToken in bytes */ + Fts5Colset *pColset /* Colset to filter on */ +){ + Fts5Iter *pRet = 0; + Fts5TokenDataIter *pSet = 0; + Fts5Structure *pStruct = 0; + const int flags = FTS5INDEX_QUERY_SCANONETERM | FTS5INDEX_QUERY_SCAN; + + Fts5Buffer bSeek = {0, 0, 0}; + Fts5Buffer *pSmall = 0; + + fts5IndexFlush(p); + pStruct = fts5StructureRead(p); + + while( p->rc==SQLITE_OK ){ + Fts5Iter *pPrev = pSet ? pSet->apIter[pSet->nIter-1] : 0; + Fts5Iter *pNew = 0; + Fts5SegIter *pNewIter = 0; + Fts5SegIter *pPrevIter = 0; + + int iLvl, iSeg, ii; + + pNew = fts5MultiIterAlloc(p, pStruct->nSegment); + if( pSmall ){ + fts5BufferSet(&p->rc, &bSeek, pSmall->n, pSmall->p); + fts5BufferAppendBlob(&p->rc, &bSeek, 1, (const u8*)"\0"); + }else{ + fts5BufferSet(&p->rc, &bSeek, nToken, pToken); + } + if( p->rc ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + pNewIter = &pNew->aSeg[0]; + pPrevIter = (pPrev ? &pPrev->aSeg[0] : 0); + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + int bDone = 0; + + if( pPrevIter ){ + if( fts5BufferCompare(pSmall, &pPrevIter->term) ){ + memcpy(pNewIter, pPrevIter, sizeof(Fts5SegIter)); + memset(pPrevIter, 0, sizeof(Fts5SegIter)); + bDone = 1; + }else if( pPrevIter->iEndofDoclist>pPrevIter->pLeaf->szLeaf ){ + fts5SegIterNextInit(p,(const char*)bSeek.p,bSeek.n-1,pSeg,pNewIter); + bDone = 1; + } + } + + if( bDone==0 ){ + fts5SegIterSeekInit(p, bSeek.p, bSeek.n, flags, pSeg, pNewIter); + } + + if( pPrevIter ){ + if( pPrevIter->pTombArray ){ + pNewIter->pTombArray = pPrevIter->pTombArray; + pNewIter->pTombArray->nRef++; + } + }else{ + fts5SegIterAllocTombstone(p, pNewIter); + } + + pNewIter++; + if( pPrevIter ) pPrevIter++; + if( p->rc ) break; + } + } + fts5TokendataSetTermIfEof(pPrev, pSmall); + + pNew->bSkipEmpty = 1; + pNew->pColset = pColset; + fts5IterSetOutputCb(&p->rc, pNew); + + /* Loop through all segments in the new iterator. Find the smallest + ** term that any segment-iterator points to. Iterator pNew will be + ** used for this term. Also, set any iterator that points to a term that + ** does not match pToken/nToken to point to EOF */ + pSmall = 0; + for(ii=0; iinSeg; ii++){ + Fts5SegIter *pII = &pNew->aSeg[ii]; + if( 0==fts5IsTokendataPrefix(&pII->term, pToken, nToken) ){ + fts5SegIterSetEOF(pII); + } + if( pII->pLeaf && (!pSmall || fts5BufferCompare(pSmall, &pII->term)>0) ){ + pSmall = &pII->term; + } + } + + /* If pSmall is still NULL at this point, then the new iterator does + ** not point to any terms that match the query. So delete it and break + ** out of the loop - all required iterators have been collected. */ + if( pSmall==0 ){ + sqlite3Fts5IterClose((Fts5IndexIter*)pNew); + break; + } + + /* Append this iterator to the set and continue. */ + pSet = fts5AppendTokendataIter(p, pSet, pNew); + } + + if( p->rc==SQLITE_OK && pSet ){ + int ii; + for(ii=0; iinIter; ii++){ + Fts5Iter *pIter = pSet->apIter[ii]; + int iSeg; + for(iSeg=0; iSegnSeg; iSeg++){ + pIter->aSeg[iSeg].flags |= FTS5_SEGITER_ONETERM; + } + fts5MultiIterFinishSetup(p, pIter); + } + } + + if( p->rc==SQLITE_OK ){ + pRet = fts5MultiIterAlloc(p, 0); + } + if( pRet ){ + pRet->pTokenDataIter = pSet; + if( pSet ){ + fts5IterSetOutputsTokendata(pRet); + }else{ + pRet->base.bEof = 1; + } + }else{ + fts5TokendataIterDelete(pSet); + } + + fts5StructureRelease(pStruct); + fts5BufferFree(&bSeek); + return pRet; +} + + +/* +** Open a new iterator to iterate though all rowid that match the ** specified token or token prefix. */ static int sqlite3Fts5IndexQuery( @@ -189221,7 +249312,13 @@ static int sqlite3Fts5IndexQuery( if( sqlite3Fts5BufferSize(&p->rc, &buf, nToken+1)==0 ){ int iIdx = 0; /* Index to search */ - memcpy(&buf.p[1], pToken, nToken); + int iPrefixIdx = 0; /* +1 prefix index */ + int bTokendata = pConfig->bTokendata; + if( nToken>0 ) memcpy(&buf.p[1], pToken, nToken); + + if( flags & (FTS5INDEX_QUERY_NOTOKENDATA|FTS5INDEX_QUERY_SCAN) ){ + bTokendata = 0; + } /* Figure out which index to search and set iIdx accordingly. If this ** is a prefix query for which there is no prefix index, set iIdx to @@ -189229,9 +249326,9 @@ static int sqlite3Fts5IndexQuery( ** satisfied by scanning multiple terms in the main index. ** ** If the QUERY_TEST_NOIDX flag was specified, then this must be a - ** prefix-query. Instead of using a prefix-index (if one exists), + ** prefix-query. Instead of using a prefix-index (if one exists), ** evaluate the prefix query using the main FTS index. This is used - ** for internal sanity checking by the integrity-check in debug + ** for internal sanity checking by the integrity-check in debug ** mode only. */ #ifdef SQLITE_DEBUG if( pConfig->bPrefixIndex==0 || (flags & FTS5INDEX_QUERY_TEST_NOIDX) ){ @@ -189242,16 +249339,21 @@ static int sqlite3Fts5IndexQuery( if( flags & FTS5INDEX_QUERY_PREFIX ){ int nChar = fts5IndexCharlen(pToken, nToken); for(iIdx=1; iIdx<=pConfig->nPrefix; iIdx++){ - if( pConfig->aPrefix[iIdx-1]==nChar ) break; + int nIdxChar = pConfig->aPrefix[iIdx-1]; + if( nIdxChar==nChar ) break; + if( nIdxChar==nChar+1 ) iPrefixIdx = iIdx; } } - if( iIdx<=pConfig->nPrefix ){ + if( bTokendata && iIdx==0 ){ + buf.p[0] = '0'; + pRet = fts5SetupTokendataIter(p, buf.p, nToken+1, pColset); + }else if( iIdx<=pConfig->nPrefix ){ /* Straight index lookup */ Fts5Structure *pStruct = fts5StructureRead(p); buf.p[0] = (u8)(FTS5_MAIN_PREFIX + iIdx); if( pStruct ){ - fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, + fts5MultiIterNew(p, pStruct, flags | FTS5INDEX_QUERY_SKIPEMPTY, pColset, buf.p, nToken+1, -1, 0, &pRet ); fts5StructureRelease(pStruct); @@ -189259,23 +249361,26 @@ static int sqlite3Fts5IndexQuery( }else{ /* Scan multiple terms in the main index */ int bDesc = (flags & FTS5INDEX_QUERY_DESC)!=0; - buf.p[0] = FTS5_MAIN_PREFIX; - fts5SetupPrefixIter(p, bDesc, buf.p, nToken+1, pColset, &pRet); - assert( p->rc!=SQLITE_OK || pRet->pColset==0 ); - fts5IterSetOutputCb(&p->rc, pRet); - if( p->rc==SQLITE_OK ){ - Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; - if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + fts5SetupPrefixIter(p, bDesc, iPrefixIdx, buf.p, nToken+1, pColset,&pRet); + if( pRet==0 ){ + assert( p->rc!=SQLITE_OK ); + }else{ + assert( pRet->pColset==0 ); + fts5IterSetOutputCb(&p->rc, pRet); + if( p->rc==SQLITE_OK ){ + Fts5SegIter *pSeg = &pRet->aSeg[pRet->aFirst[1].iFirst]; + if( pSeg->pLeaf ) pRet->xSetOutputs(pRet, pSeg); + } } } if( p->rc ){ - sqlite3Fts5IterClose(&pRet->base); + sqlite3Fts5IterClose((Fts5IndexIter*)pRet); pRet = 0; - fts5CloseReader(p); + sqlite3Fts5IndexCloseReader(p); } - *ppIter = &pRet->base; + *ppIter = (Fts5IndexIter*)pRet; sqlite3Fts5BufferFree(&buf); } return fts5IndexReturn(p); @@ -189285,12 +249390,16 @@ static int sqlite3Fts5IndexQuery( ** Return true if the iterator passed as the only argument is at EOF. */ /* -** Move to the next matching rowid. +** Move to the next matching rowid. */ static int sqlite3Fts5IterNext(Fts5IndexIter *pIndexIter){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; assert( pIter->pIndex->rc==SQLITE_OK ); - fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + if( pIter->pTokenDataIter ){ + fts5TokendataIterNext(pIter, 0, 0); + }else{ + fts5MultiIterNext(pIter->pIndex, pIter, 0, 0); + } return fts5IndexReturn(pIter->pIndex); } @@ -189323,7 +249432,11 @@ static int sqlite3Fts5IterNextScan(Fts5IndexIter *pIndexIter){ */ static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; - fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + if( pIter->pTokenDataIter ){ + fts5TokendataIterNext(pIter, 1, iMatch); + }else{ + fts5MultiIterNextFrom(pIter->pIndex, pIter, iMatch); + } return fts5IndexReturn(pIter->pIndex); } @@ -189333,8 +249446,102 @@ static int sqlite3Fts5IterNextFrom(Fts5IndexIter *pIndexIter, i64 iMatch){ static const char *sqlite3Fts5IterTerm(Fts5IndexIter *pIndexIter, int *pn){ int n; const char *z = (const char*)fts5MultiIterTerm((Fts5Iter*)pIndexIter, &n); + assert_nc( z || n<=1 ); *pn = n-1; - return &z[1]; + return (z ? &z[1] : 0); +} + +/* +** This is used by xInstToken() to access the token at offset iOff, column +** iCol of row iRowid. The token is returned via output variables *ppOut +** and *pnOut. The iterator passed as the first argument must be a tokendata=1 +** iterator (pIter->pTokenDataIter!=0). +*/ +static int sqlite3Fts5IterToken( + Fts5IndexIter *pIndexIter, + i64 iRowid, + int iCol, + int iOff, + const char **ppOut, int *pnOut +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5TokenDataMap *aMap = pT->aMap; + i64 iPos = (((i64)iCol)<<32) + iOff; + + int i1 = 0; + int i2 = pT->nMap; + int iTest = 0; + + while( i2>i1 ){ + iTest = (i1 + i2) / 2; + + if( aMap[iTest].iRowidiRowid ){ + i2 = iTest; + }else{ + if( aMap[iTest].iPosiPos ){ + i2 = iTest; + }else{ + break; + } + } + } + + if( i2>i1 ){ + Fts5Iter *pMap = pT->apIter[aMap[iTest].iIter]; + *ppOut = (const char*)pMap->aSeg[0].term.p+1; + *pnOut = pMap->aSeg[0].term.n-1; + } + + return SQLITE_OK; +} + +/* +** Clear any existing entries from the token-map associated with the +** iterator passed as the only argument. +*/ +static void sqlite3Fts5IndexIterClearTokendata(Fts5IndexIter *pIndexIter){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + if( pIter && pIter->pTokenDataIter ){ + pIter->pTokenDataIter->nMap = 0; + } +} + +/* +** Set a token-mapping for the iterator passed as the first argument. This +** is used in detail=column or detail=none mode when a token is requested +** using the xInstToken() API. In this case the caller tokenizers the +** current row and configures the token-mapping via multiple calls to this +** function. +*/ +static int sqlite3Fts5IndexIterWriteTokendata( + Fts5IndexIter *pIndexIter, + const char *pToken, int nToken, + i64 iRowid, int iCol, int iOff +){ + Fts5Iter *pIter = (Fts5Iter*)pIndexIter; + Fts5TokenDataIter *pT = pIter->pTokenDataIter; + Fts5Index *p = pIter->pIndex; + int ii; + + assert( p->pConfig->eDetail!=FTS5_DETAIL_FULL ); + assert( pIter->pTokenDataIter ); + + for(ii=0; iinIter; ii++){ + Fts5Buffer *pTerm = &pT->apIter[ii]->aSeg[0].term; + if( nToken==pTerm->n-1 && memcmp(pToken, pTerm->p+1, nToken)==0 ) break; + } + if( iinIter ){ + fts5TokendataIterAppendMap(p, pT, ii, iRowid, (((i64)iCol)<<32) + iOff); + } + return fts5IndexReturn(p); } /* @@ -189344,13 +249551,14 @@ static void sqlite3Fts5IterClose(Fts5IndexIter *pIndexIter){ if( pIndexIter ){ Fts5Iter *pIter = (Fts5Iter*)pIndexIter; Fts5Index *pIndex = pIter->pIndex; + fts5TokendataIterDelete(pIter->pTokenDataIter); fts5MultiIterFree(pIter); - fts5CloseReader(pIndex); + sqlite3Fts5IndexCloseReader(pIndex); } } /* -** Read and decode the "averages" record from the database. +** Read and decode the "averages" record from the database. ** ** Parameter anSize must point to an array of size nCol, where nCol is ** the number of user defined columns in the FTS table. @@ -189376,7 +249584,7 @@ static int sqlite3Fts5IndexGetAverages(Fts5Index *p, i64 *pnRow, i64 *anSize){ } /* -** Replace the current "averages" record with the contents of the buffer +** Replace the current "averages" record with the contents of the buffer ** supplied as the second argument. */ static int sqlite3Fts5IndexSetAverages(Fts5Index *p, const u8 *pData, int nData){ @@ -189394,7 +249602,7 @@ static int sqlite3Fts5IndexReads(Fts5Index *p){ } /* -** Set the 32-bit cookie value stored at the start of all structure +** Set the 32-bit cookie value stored at the start of all structure ** records to the value passed as the second argument. ** ** Return SQLITE_OK if successful, or an SQLite error code if an error @@ -189409,7 +249617,7 @@ static int sqlite3Fts5IndexSetCookie(Fts5Index *p, int iNew){ assert( p->rc==SQLITE_OK ); sqlite3Fts5Put32(aCookie, iNew); - rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, + rc = sqlite3_blob_open(pConfig->db, pConfig->zDb, p->zDataTbl, "block", FTS5_STRUCTURE_ROWID, 1, &pBlob ); if( rc==SQLITE_OK ){ @@ -189427,10 +249635,351 @@ static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ return fts5IndexReturn(p); } +/* +** Retrieve the origin value that will be used for the segment currently +** being accumulated in the in-memory hash table when it is flushed to +** disk. If successful, SQLITE_OK is returned and (*piOrigin) set to +** the queried value. Or, if an error occurs, an error code is returned +** and the final value of (*piOrigin) is undefined. +*/ +static int sqlite3Fts5IndexGetOrigin(Fts5Index *p, i64 *piOrigin){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + *piOrigin = pStruct->nOriginCntr; + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} + +/* +** Buffer pPg contains a page of a tombstone hash table - one of nPg pages +** associated with the same segment. This function adds rowid iRowid to +** the hash table. The caller is required to guarantee that there is at +** least one free slot on the page. +** +** If parameter bForce is false and the hash table is deemed to be full +** (more than half of the slots are occupied), then non-zero is returned +** and iRowid not inserted. Or, if bForce is true or if the hash table page +** is not full, iRowid is inserted and zero returned. +*/ +static int fts5IndexTombstoneAddToPage( + Fts5Data *pPg, + int bForce, + int nPg, + u64 iRowid +){ + const int szKey = TOMBSTONE_KEYSIZE(pPg); + const int nSlot = TOMBSTONE_NSLOT(pPg); + const int nElem = fts5GetU32(&pPg->p[4]); + int iSlot = (iRowid / nPg) % nSlot; + int nCollide = nSlot; + + if( szKey==4 && iRowid>0xFFFFFFFF ) return 2; + if( iRowid==0 ){ + pPg->p[1] = 0x01; + return 0; + } + + if( bForce==0 && nElem>=(nSlot/2) ){ + return 1; + } + + fts5PutU32(&pPg->p[4], nElem+1); + if( szKey==4 ){ + u32 *aSlot = (u32*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU32((u8*)&aSlot[iSlot], (u32)iRowid); + }else{ + u64 *aSlot = (u64*)&pPg->p[8]; + while( aSlot[iSlot] ){ + iSlot = (iSlot + 1) % nSlot; + if( nCollide--==0 ) return 0; + } + fts5PutU64((u8*)&aSlot[iSlot], iRowid); + } + + return 0; +} + +/* +** This function attempts to build a new hash containing all the keys +** currently in the tombstone hash table for segment pSeg. The new +** hash will be stored in the nOut buffers passed in array apOut[]. +** All pages of the new hash use key-size szKey (4 or 8). +** +** Return 0 if the hash is successfully rebuilt into the nOut pages. +** Or non-zero if it is not (because one page became overfull). In this +** case the caller should retry with a larger nOut parameter. +** +** Parameter pData1 is page iPg1 of the hash table being rebuilt. +*/ +static int fts5IndexTombstoneRehash( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int nOut, /* Number of output pages */ + Fts5Data **apOut /* Array of output hash pages */ +){ + int ii; + int res = 0; + + /* Initialize the headers of all the output pages */ + for(ii=0; iip[0] = szKey; + fts5PutU32(&apOut[ii]->p[4], 0); + } + + /* Loop through the current pages of the hash table. */ + for(ii=0; res==0 && iinPgTombstone; ii++){ + Fts5Data *pData = 0; /* Page ii of the current hash table */ + Fts5Data *pFree = 0; /* Free this at the end of the loop */ + + if( iPg1==ii ){ + pData = pData1; + }else{ + pFree = pData = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid, ii)); + } + + if( pData ){ + int szKeyIn = TOMBSTONE_KEYSIZE(pData); + int nSlotIn = (pData->nn - 8) / szKeyIn; + int iIn; + for(iIn=0; iInp[8]; + if( aSlot[iIn] ) iVal = fts5GetU32((u8*)&aSlot[iIn]); + }else{ + u64 *aSlot = (u64*)&pData->p[8]; + if( aSlot[iIn] ) iVal = fts5GetU64((u8*)&aSlot[iIn]); + } + + /* If iVal is not 0 at this point, insert it into the new hash table */ + if( iVal ){ + Fts5Data *pPg = apOut[(iVal % nOut)]; + res = fts5IndexTombstoneAddToPage(pPg, 0, nOut, iVal); + if( res ) break; + } + } + + /* If this is page 0 of the old hash, copy the rowid-0-flag from the + ** old hash to the new. */ + if( ii==0 ){ + apOut[0]->p[1] = pData->p[1]; + } + } + fts5DataRelease(pFree); + } + + return res; +} + +/* +** This is called to rebuild the hash table belonging to segment pSeg. +** If parameter pData1 is not NULL, then one page of the existing hash table +** has already been loaded - pData1, which is page iPg1. The key-size for +** the new hash table is szKey (4 or 8). +** +** If successful, the new hash table is not written to disk. Instead, +** output parameter (*pnOut) is set to the number of pages in the new +** hash table, and (*papOut) to point to an array of buffers containing +** the new page data. +** +** If an error occurs, an error code is left in the Fts5Index object and +** both output parameters set to 0 before returning. +*/ +static void fts5IndexTombstoneRebuild( + Fts5Index *p, + Fts5StructureSegment *pSeg, /* Segment to rebuild hash of */ + Fts5Data *pData1, /* One page of current hash - or NULL */ + int iPg1, /* Which page of the current hash is pData1 */ + int szKey, /* 4 or 8, the keysize */ + int *pnOut, /* OUT: Number of output pages */ + Fts5Data ***papOut /* OUT: Output hash pages */ +){ + const int MINSLOT = 32; + int nSlotPerPage = MAX(MINSLOT, (p->pConfig->pgsz - 8) / szKey); + int nSlot = 0; /* Number of slots in each output page */ + int nOut = 0; + + /* Figure out how many output pages (nOut) and how many slots per + ** page (nSlot). There are three possibilities: + ** + ** 1. The hash table does not yet exist. In this case the new hash + ** table will consist of a single page with MINSLOT slots. + ** + ** 2. The hash table exists but is currently a single page. In this + ** case an attempt is made to grow the page to accommodate the new + ** entry. The page is allowed to grow up to nSlotPerPage (see above) + ** slots. + ** + ** 3. The hash table already consists of more than one page, or of + ** a single page already so large that it cannot be grown. In this + ** case the new hash consists of (nPg*2+1) pages of nSlotPerPage + ** slots each, where nPg is the current number of pages in the + ** hash table. + */ + if( pSeg->nPgTombstone==0 ){ + /* Case 1. */ + nOut = 1; + nSlot = MINSLOT; + }else if( pSeg->nPgTombstone==1 ){ + /* Case 2. */ + int nElem = (int)fts5GetU32(&pData1->p[4]); + assert( pData1 && iPg1==0 ); + nOut = 1; + nSlot = MAX(nElem*4, MINSLOT); + if( nSlot>nSlotPerPage ) nOut = 0; + } + if( nOut==0 ){ + /* Case 3. */ + nOut = (pSeg->nPgTombstone * 2 + 1); + nSlot = nSlotPerPage; + } + + /* Allocate the required array and output pages */ + while( 1 ){ + int res = 0; + int ii = 0; + int szPage = 0; + Fts5Data **apOut = 0; + + /* Allocate space for the new hash table */ + assert( nSlot>=MINSLOT ); + apOut = (Fts5Data**)sqlite3Fts5MallocZero(&p->rc, sizeof(Fts5Data*) * nOut); + szPage = 8 + nSlot*szKey; + for(ii=0; iirc, + sizeof(Fts5Data)+szPage + ); + if( pNew ){ + pNew->nn = szPage; + pNew->p = (u8*)&pNew[1]; + apOut[ii] = pNew; + } + } + + /* Rebuild the hash table. */ + if( p->rc==SQLITE_OK ){ + res = fts5IndexTombstoneRehash(p, pSeg, pData1, iPg1, szKey, nOut, apOut); + } + if( res==0 ){ + if( p->rc ){ + fts5IndexFreeArray(apOut, nOut); + apOut = 0; + nOut = 0; + } + *pnOut = nOut; + *papOut = apOut; + break; + } + + /* If control flows to here, it was not possible to rebuild the hash + ** table. Free all buffers and then try again with more pages. */ + assert( p->rc==SQLITE_OK ); + fts5IndexFreeArray(apOut, nOut); + nSlot = nSlotPerPage; + nOut = nOut*2 + 1; + } +} + + +/* +** Add a tombstone for rowid iRowid to segment pSeg. +*/ +static void fts5IndexTombstoneAdd( + Fts5Index *p, + Fts5StructureSegment *pSeg, + u64 iRowid +){ + Fts5Data *pPg = 0; + int iPg = -1; + int szKey = 0; + int nHash = 0; + Fts5Data **apHash = 0; + + p->nContentlessDelete++; + + if( pSeg->nPgTombstone>0 ){ + iPg = iRowid % pSeg->nPgTombstone; + pPg = fts5DataRead(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg)); + if( pPg==0 ){ + assert( p->rc!=SQLITE_OK ); + return; + } + + if( 0==fts5IndexTombstoneAddToPage(pPg, 0, pSeg->nPgTombstone, iRowid) ){ + fts5DataWrite(p, FTS5_TOMBSTONE_ROWID(pSeg->iSegid,iPg), pPg->p, pPg->nn); + fts5DataRelease(pPg); + return; + } + } + + /* Have to rebuild the hash table. First figure out the key-size (4 or 8). */ + szKey = pPg ? TOMBSTONE_KEYSIZE(pPg) : 4; + if( iRowid>0xFFFFFFFF ) szKey = 8; + + /* Rebuild the hash table */ + fts5IndexTombstoneRebuild(p, pSeg, pPg, iPg, szKey, &nHash, &apHash); + assert( p->rc==SQLITE_OK || (nHash==0 && apHash==0) ); + + /* If all has succeeded, write the new rowid into one of the new hash + ** table pages, then write them all out to disk. */ + if( nHash ){ + int ii = 0; + fts5IndexTombstoneAddToPage(apHash[iRowid % nHash], 1, nHash, iRowid); + for(ii=0; iiiSegid, ii); + fts5DataWrite(p, iTombstoneRowid, apHash[ii]->p, apHash[ii]->nn); + } + pSeg->nPgTombstone = nHash; + fts5StructureWrite(p, p->pStruct); + } + + fts5DataRelease(pPg); + fts5IndexFreeArray(apHash, nHash); +} + +/* +** Add iRowid to the tombstone list of the segment or segments that contain +** rows from origin iOrigin. Return SQLITE_OK if successful, or an SQLite +** error code otherwise. +*/ +static int sqlite3Fts5IndexContentlessDelete(Fts5Index *p, i64 iOrigin, i64 iRowid){ + Fts5Structure *pStruct; + pStruct = fts5StructureRead(p); + if( pStruct ){ + int bFound = 0; /* True after pSeg->nEntryTombstone incr. */ + int iLvl; + for(iLvl=pStruct->nLevel-1; iLvl>=0; iLvl--){ + int iSeg; + for(iSeg=pStruct->aLevel[iLvl].nSeg-1; iSeg>=0; iSeg--){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + if( pSeg->iOrigin1<=(u64)iOrigin && pSeg->iOrigin2>=(u64)iOrigin ){ + if( bFound==0 ){ + pSeg->nEntryTombstone++; + bFound = 1; + } + fts5IndexTombstoneAdd(p, pSeg, iRowid); + } + } + } + fts5StructureRelease(pStruct); + } + return fts5IndexReturn(p); +} /************************************************************************* ************************************************************************** -** Below this point is the implementation of the integrity-check +** Below this point is the implementation of the integrity-check ** functionality. */ @@ -189438,9 +249987,9 @@ static int sqlite3Fts5IndexLoadConfig(Fts5Index *p){ ** Return a simple checksum value based on the arguments. */ static u64 sqlite3Fts5IndexEntryCksum( - i64 iRowid, - int iCol, - int iPos, + i64 iRowid, + int iCol, + int iPos, int iIdx, const char *pTerm, int nTerm @@ -189456,15 +250005,15 @@ static u64 sqlite3Fts5IndexEntryCksum( #ifdef SQLITE_DEBUG /* -** This function is purely an internal test. It does not contribute to +** This function is purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. ** -** Instead, it tests that the same set of pgno/rowid combinations are +** Instead, it tests that the same set of pgno/rowid combinations are ** visited regardless of whether the doclist-index identified by parameters ** iSegid/iLeaf is iterated in forwards or reverse order. */ static void fts5TestDlidxReverse( - Fts5Index *p, + Fts5Index *p, int iSegid, /* Segment id to load from */ int iLeaf /* Load doclist-index for this leaf */ ){ @@ -189510,9 +250059,11 @@ static int fts5QueryCksum( int eDetail = p->pConfig->eDetail; u64 cksum = *pCksum; Fts5IndexIter *pIter = 0; - int rc = sqlite3Fts5IndexQuery(p, z, n, flags, 0, &pIter); + int rc = sqlite3Fts5IndexQuery( + p, z, n, (flags | FTS5INDEX_QUERY_NOTOKENDATA), 0, &pIter + ); - while( rc==SQLITE_OK && 0==sqlite3Fts5IterEof(pIter) ){ + while( rc==SQLITE_OK && ALWAYS(pIter!=0) && 0==sqlite3Fts5IterEof(pIter) ){ i64 rowid = pIter->iRowid; if( eDetail==FTS5_DETAIL_NONE ){ @@ -189538,13 +250089,44 @@ static int fts5QueryCksum( return rc; } +/* +** Check if buffer z[], size n bytes, contains as series of valid utf-8 +** encoded codepoints. If so, return 0. Otherwise, if the buffer does not +** contain valid utf-8, return non-zero. +*/ +static int fts5TestUtf8(const char *z, int n){ + int i = 0; + assert_nc( n>0 ); + while( i=n || (z[i+1] & 0xC0)!=0x80 ) return 1; + i += 2; + }else + if( (z[i] & 0xF0)==0xE0 ){ + if( i+2>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; + i += 3; + }else + if( (z[i] & 0xF8)==0xF0 ){ + if( i+3>=n || (z[i+1] & 0xC0)!=0x80 || (z[i+2] & 0xC0)!=0x80 ) return 1; + if( (z[i+2] & 0xC0)!=0x80 ) return 1; + i += 3; + }else{ + return 1; + } + } + + return 0; +} /* -** This function is also purely an internal test. It does not contribute to +** This function is also purely an internal test. It does not contribute to ** FTS functionality, or even the integrity-check, in any way. */ static void fts5TestTerm( - Fts5Index *p, + Fts5Index *p, Fts5Buffer *pPrev, /* Previous term */ const char *z, int n, /* Possibly new term to test */ u64 expected, @@ -189573,13 +250155,19 @@ static void fts5TestTerm( if( rc==SQLITE_OK && ck1!=ck2 ) rc = FTS5_CORRUPT; /* If this is a prefix query, check that the results returned if the - ** the index is disabled are the same. In both ASC and DESC order. + ** the index is disabled are the same. In both ASC and DESC order. ** ** This check may only be performed if the hash table is empty. This ** is because the hash table only supports a single scan query at ** a time, and the multi-iter loop from which this function is called - ** is already performing such a scan. */ - if( p->nPendingData==0 ){ + ** is already performing such a scan. + ** + ** Also only do this if buffer zTerm contains nTerm bytes of valid + ** utf-8. Otherwise, the last part of the buffer contents might contain + ** a non-utf-8 sequence that happens to be a prefix of a valid utf-8 + ** character stored in the main fts index, which will cause the + ** test to fail. */ + if( p->nPendingData==0 && 0==fts5TestUtf8(zTerm, nTerm) ){ if( iIdx>0 && rc==SQLITE_OK ){ int f = flags|FTS5INDEX_QUERY_TEST_NOIDX; ck2 = 0; @@ -189604,7 +250192,7 @@ static void fts5TestTerm( } p->rc = rc; } - + #else # define fts5TestDlidxReverse(x,y,z) # define fts5TestTerm(u,v,w,x,y,z) @@ -189640,7 +250228,7 @@ static void fts5IndexIntegrityCheckEmpty( } static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ - int iTermOff = 0; + i64 iTermOff = 0; int ii; Fts5Buffer buf1 = {0,0,0}; @@ -189649,7 +250237,7 @@ static void fts5IntegrityCheckPgidx(Fts5Index *p, Fts5Data *pLeaf){ ii = pLeaf->szLeaf; while( iinn && p->rc==SQLITE_OK ){ int res; - int iOff; + i64 iOff; int nIncr; ii += fts5GetVarint32(&pLeaf->p[ii], nIncr); @@ -189694,6 +250282,7 @@ static void fts5IndexIntegrityCheckSegment( Fts5StructureSegment *pSeg /* Segment to check internal consistency */ ){ Fts5Config *pConfig = p->pConfig; + int bSecureDelete = (pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE); sqlite3_stmt *pStmt = 0; int rc2; int iIdxPrevLeaf = pSeg->pgnoFirst-1; @@ -189702,7 +250291,8 @@ static void fts5IndexIntegrityCheckSegment( if( pSeg->pgnoFirst==0 ) return; fts5IndexPrepareStmt(p, &pStmt, sqlite3_mprintf( - "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d", + "SELECT segid, term, (pgno>>1), (pgno&1) FROM %Q.'%q_idx' WHERE segid=%d " + "ORDER BY 1, 2", pConfig->zDb, pConfig->zName, pSeg->iSegid )); @@ -189711,16 +250301,16 @@ static void fts5IndexIntegrityCheckSegment( i64 iRow; /* Rowid for this leaf */ Fts5Data *pLeaf; /* Data for this leaf */ + const char *zIdxTerm = (const char*)sqlite3_column_blob(pStmt, 1); int nIdxTerm = sqlite3_column_bytes(pStmt, 1); - const char *zIdxTerm = (const char*)sqlite3_column_text(pStmt, 1); int iIdxLeaf = sqlite3_column_int(pStmt, 2); int bIdxDlidx = sqlite3_column_int(pStmt, 3); - /* If the leaf in question has already been trimmed from the segment, + /* If the leaf in question has already been trimmed from the segment, ** ignore this b-tree entry. Otherwise, load it into memory. */ if( iIdxLeafpgnoFirst ) continue; iRow = FTS5_SEGMENT_ROWID(pSeg->iSegid, iIdxLeaf); - pLeaf = fts5DataRead(p, iRow); + pLeaf = fts5LeafRead(p, iRow); if( pLeaf==0 ) break; /* Check that the leaf contains at least one term, and that it is equal @@ -189728,7 +250318,19 @@ static void fts5IndexIntegrityCheckSegment( ** is also a rowid pointer within the leaf page header, it points to a ** location before the term. */ if( pLeaf->nn<=pLeaf->szLeaf ){ - p->rc = FTS5_CORRUPT; + + if( nIdxTerm==0 + && pConfig->iVersion==FTS5_CURRENT_VERSION_SECUREDELETE + && pLeaf->nn==pLeaf->szLeaf + && pLeaf->nn==4 + ){ + /* special case - the very first page in a segment keeps its %_idx + ** entry even if all the terms are removed from it by secure-delete + ** operations. */ + }else{ + p->rc = FTS5_CORRUPT; + } + }else{ int iOff; /* Offset of first term on leaf */ int iRowidOff; /* Offset of first rowid on leaf */ @@ -189737,11 +250339,11 @@ static void fts5IndexIntegrityCheckSegment( iOff = fts5LeafFirstTermOff(pLeaf); iRowidOff = fts5LeafFirstRowidOff(pLeaf); - if( iRowidOff>=iOff ){ + if( iRowidOff>=iOff || iOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; }else{ iOff += fts5GetVarint32(&pLeaf->p[iOff], nTerm); - res = memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); + res = fts5Memcmp(&pLeaf->p[iOff], zIdxTerm, MIN(nTerm, nIdxTerm)); if( res==0 ) res = nTerm - nIdxTerm; if( res<0 ) p->rc = FTS5_CORRUPT; } @@ -189792,9 +250394,12 @@ static void fts5IndexIntegrityCheckSegment( ASSERT_SZLEAF_OK(pLeaf); if( iRowidOff>=pLeaf->szLeaf ){ p->rc = FTS5_CORRUPT; - }else{ + }else if( bSecureDelete==0 || iRowidOff>0 ){ + i64 iDlRowid = fts5DlidxIterRowid(pDlidx); fts5GetVarint(&pLeaf->p[iRowidOff], (u64*)&iRowid); - if( iRowid!=fts5DlidxIterRowid(pDlidx) ) p->rc = FTS5_CORRUPT; + if( iRowidrc = FTS5_CORRUPT; + } } fts5DataRelease(pLeaf); } @@ -189824,7 +250429,7 @@ static void fts5IndexIntegrityCheckSegment( /* -** Run internal checks to ensure that the FTS index (a) is internally +** Run internal checks to ensure that the FTS index (a) is internally ** consistent and (b) contains entries for which the XOR of the checksums ** as calculated by sqlite3Fts5IndexEntryCksum() is cksum. ** @@ -189833,12 +250438,13 @@ static void fts5IndexIntegrityCheckSegment( ** error, or some other SQLite error code if another error (e.g. OOM) ** occurs. */ -static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ +static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum, int bUseCksum){ int eDetail = p->pConfig->eDetail; u64 cksum2 = 0; /* Checksum based on contents of indexes */ Fts5Buffer poslist = {0,0,0}; /* Buffer used to hold a poslist */ Fts5Iter *pIter; /* Used to iterate through entire index */ Fts5Structure *pStruct; /* Index structure */ + int iLvl, iSeg; #ifdef SQLITE_DEBUG /* Used by extra internal tests only run if NDEBUG is not defined */ @@ -189846,18 +250452,19 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ Fts5Buffer term = {0,0,0}; /* Buffer used to hold most recent term */ #endif const int flags = FTS5INDEX_QUERY_NOOUTPUT; - + /* Load the FTS index structure */ pStruct = fts5StructureRead(p); + if( pStruct==0 ){ + assert( p->rc!=SQLITE_OK ); + return fts5IndexReturn(p); + } /* Check that the internal nodes of each segment match the leaves */ - if( pStruct ){ - int iLvl, iSeg; - for(iLvl=0; iLvlnLevel; iLvl++){ - for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ - Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; - fts5IndexIntegrityCheckSegment(p, pSeg); - } + for(iLvl=0; iLvlnLevel; iLvl++){ + for(iSeg=0; iSegaLevel[iLvl].nSeg; iSeg++){ + Fts5StructureSegment *pSeg = &pStruct->aLevel[iLvl].aSeg[iSeg]; + fts5IndexIntegrityCheckSegment(p, pSeg); } } @@ -189868,7 +250475,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ ** ** Two versions of the same checksum are calculated. The first (stack ** variable cksum2) based on entries extracted from the full-text index - ** while doing a linear scan of each individual index in turn. + ** while doing a linear scan of each individual index in turn. ** ** As each term visited by the linear scans, a separate query for the ** same term is performed. cksum3 is calculated based on the entries @@ -189886,6 +250493,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ /* If this is a new term, query for it. Update cksum3 with the results. */ fts5TestTerm(p, &term, z, n, cksum2, &cksum3); + if( p->rc ) break; if( eDetail==FTS5_DETAIL_NONE ){ if( 0==fts5MultiIterIsEmpty(p, pIter) ){ @@ -189894,6 +250502,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ }else{ poslist.n = 0; fts5SegiterPoslist(p, &pIter->aSeg[pIter->aFirst[1].iFirst], 0, &poslist); + fts5BufferAppendBlob(&p->rc, &poslist, 4, (const u8*)"\0\0\0\0"); while( 0==sqlite3Fts5PoslistNext64(poslist.p, poslist.n, &iOff, &iPos) ){ int iCol = FTS5_POS2COLUMN(iPos); int iTokOff = FTS5_POS2OFFSET(iPos); @@ -189904,7 +250513,7 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ fts5TestTerm(p, &term, 0, 0, cksum2, &cksum3); fts5MultiIterFree(pIter); - if( p->rc==SQLITE_OK && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; + if( p->rc==SQLITE_OK && bUseCksum && cksum!=cksum2 ) p->rc = FTS5_CORRUPT; fts5StructureRelease(pStruct); #ifdef SQLITE_DEBUG @@ -189920,12 +250529,14 @@ static int sqlite3Fts5IndexIntegrityCheck(Fts5Index *p, u64 cksum){ ** function only. */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Decode a segment-data rowid from the %_data table. This function is ** the opposite of macro FTS5_SEGMENT_ROWID(). */ static void fts5DecodeRowid( i64 iRowid, /* Rowid from %_data table */ + int *pbTombstone, /* OUT: Tombstone hash flag */ int *piSegid, /* OUT: Segment id */ int *pbDlidx, /* OUT: Dlidx flag */ int *piHeight, /* OUT: Height */ @@ -189941,11 +250552,16 @@ static void fts5DecodeRowid( iRowid >>= FTS5_DATA_DLI_B; *piSegid = (int)(iRowid & (((i64)1 << FTS5_DATA_ID_B) - 1)); -} + iRowid >>= FTS5_DATA_ID_B; + *pbTombstone = (int)(iRowid & 0x0001); +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ - int iSegid, iHeight, iPgno, bDlidx; /* Rowid compenents */ - fts5DecodeRowid(iKey, &iSegid, &bDlidx, &iHeight, &iPgno); + int iSegid, iHeight, iPgno, bDlidx, bTomb; /* Rowid compenents */ + fts5DecodeRowid(iKey, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); if( iSegid==0 ){ if( iKey==FTS5_AVERAGES_ROWID ){ @@ -189955,12 +250571,16 @@ static void fts5DebugRowid(int *pRc, Fts5Buffer *pBuf, i64 iKey){ } } else{ - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%ssegid=%d h=%d pgno=%d}", - bDlidx ? "dlidx " : "", iSegid, iHeight, iPgno + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "{%s%ssegid=%d h=%d pgno=%d}", + bDlidx ? "dlidx " : "", + bTomb ? "tombstone " : "", + iSegid, iHeight, iPgno ); } } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) static void fts5DebugStructure( int *pRc, /* IN/OUT: error code */ Fts5Buffer *pBuf, @@ -189970,25 +250590,33 @@ static void fts5DebugStructure( for(iLvl=0; iLvlnLevel; iLvl++){ Fts5StructureLevel *pLvl = &p->aLevel[iLvl]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {lvl=%d nMerge=%d nSeg=%d", iLvl, pLvl->nMerge, pLvl->nSeg ); for(iSeg=0; iSegnSeg; iSeg++){ Fts5StructureSegment *pSeg = &pLvl->aSeg[iSeg]; - sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d}", + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " {id=%d leaves=%d..%d", pSeg->iSegid, pSeg->pgnoFirst, pSeg->pgnoLast ); + if( pSeg->iOrigin1>0 ){ + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " origin=%lld..%lld", + pSeg->iOrigin1, pSeg->iOrigin2 + ); + } + sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } sqlite3Fts5BufferAppendPrintf(pRc, pBuf, "}"); } } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** ** Arguments pBlob/nBlob contain a serialized Fts5Structure object. This ** function appends a human-readable representation of the same object -** to the buffer passed as the second argument. +** to the buffer passed as the second argument. */ static void fts5DecodeStructure( int *pRc, /* IN/OUT: error code */ @@ -190007,13 +250635,15 @@ static void fts5DecodeStructure( fts5DebugStructure(pRc, pBuf, p); fts5StructureRelease(p); } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** This is part of the fts5_decode() debugging aid. ** -** Arguments pBlob/nBlob contain an "averages" record. This function -** appends a human-readable representation of record to the buffer passed -** as the second argument. +** Arguments pBlob/nBlob contain an "averages" record. This function +** appends a human-readable representation of record to the buffer passed +** as the second argument. */ static void fts5DecodeAverages( int *pRc, /* IN/OUT: error code */ @@ -190030,7 +250660,9 @@ static void fts5DecodeAverages( zSpace = " "; } } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** Buffer (a/n) is assumed to contain a list of serialized varints. Read ** each varint and append its string representation to buffer pBuf. Return @@ -190047,12 +250679,14 @@ static int fts5DecodePoslist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ } return iOff; } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The start of buffer (a/n) contains the start of a doclist. The doclist ** may or may not finish within the buffer. This function appends a text ** representation of the part of the doclist that is present to buffer -** pBuf. +** pBuf. ** ** The return value is the number of bytes read from the input buffer. */ @@ -190080,9 +250714,11 @@ static int fts5DecodeDoclist(int *pRc, Fts5Buffer *pBuf, const u8 *a, int n){ return iOff; } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* -** This function is part of the fts5_decode() debugging function. It is +** This function is part of the fts5_decode() debugging function. It is ** only ever used with detail=none tables. ** ** Buffer (pData/nData) contains a doclist in the format used by detail=none @@ -190121,7 +250757,27 @@ static void fts5DecodeRowidList( sqlite3Fts5BufferAppendPrintf(pRc, pBuf, " %lld%s", iRowid, zApp); } } +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) +static void fts5BufferAppendTerm(int *pRc, Fts5Buffer *pBuf, Fts5Buffer *pTerm){ + int ii; + fts5BufferGrow(pRc, pBuf, pTerm->n*2 + 1); + if( *pRc==SQLITE_OK ){ + for(ii=0; iin; ii++){ + if( pTerm->p[ii]==0x00 ){ + pBuf->p[pBuf->n++] = '\\'; + pBuf->p[pBuf->n++] = '0'; + }else{ + pBuf->p[pBuf->n++] = pTerm->p[ii]; + } + } + pBuf->p[pBuf->n] = 0x00; + } +} +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ + +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) /* ** The implementation of user-defined scalar function fts5_decode(). */ @@ -190132,11 +250788,12 @@ static void fts5DecodeFunction( ){ i64 iRowid; /* Rowid for record being decoded */ int iSegid,iHeight,iPgno,bDlidx;/* Rowid components */ + int bTomb; const u8 *aBlob; int n; /* Record to decode */ u8 *a = 0; Fts5Buffer s; /* Build up text to return here */ int rc = SQLITE_OK; /* Return code */ - int nSpace = 0; + sqlite3_int64 nSpace = 0; int eDetailNone = (sqlite3_user_data(pCtx)!=0); assert( nArg==2 ); @@ -190152,10 +250809,9 @@ static void fts5DecodeFunction( nSpace = n + FTS5_DATA_ZERO_PADDING; a = (u8*)sqlite3Fts5MallocZero(&rc, nSpace); if( a==0 ) goto decode_out; - memcpy(a, aBlob, n); + if( n>0 ) memcpy(a, aBlob, n); - - fts5DecodeRowid(iRowid, &iSegid, &bDlidx, &iHeight, &iPgno); + fts5DecodeRowid(iRowid, &bTomb, &iSegid, &bDlidx, &iHeight, &iPgno); fts5DebugRowid(&rc, &s, iRowid); if( bDlidx ){ @@ -190170,10 +250826,32 @@ static void fts5DecodeFunction( lvl.iLeafPgno = iPgno; for(fts5DlidxLvlNext(&lvl); lvl.bEof==0; fts5DlidxLvlNext(&lvl)){ - sqlite3Fts5BufferAppendPrintf(&rc, &s, + sqlite3Fts5BufferAppendPrintf(&rc, &s, " %d(%lld)", lvl.iLeafPgno, lvl.iRowid ); } + }else if( bTomb ){ + u32 nElem = fts5GetU32(&a[4]); + int szKey = (aBlob[0]==4 || aBlob[0]==8) ? aBlob[0] : 8; + int nSlot = (n - 8) / szKey; + int ii; + sqlite3Fts5BufferAppendPrintf(&rc, &s, " nElem=%d", (int)nElem); + if( aBlob[1] ){ + sqlite3Fts5BufferAppendPrintf(&rc, &s, " 0"); + } + for(ii=0; iiestimatedCost = (double)100; + pIdxInfo->estimatedRows = 100; + pIdxInfo->idxNum = 0; + for(i=0, p=pIdxInfo->aConstraint; inConstraint; i++, p++){ + if( p->usable==0 ) continue; + if( p->op==SQLITE_INDEX_CONSTRAINT_EQ && p->iColumn==11 ){ + rc = SQLITE_OK; + pIdxInfo->aConstraintUsage[i].omit = 1; + pIdxInfo->aConstraintUsage[i].argvIndex = 1; + break; + } + } + return rc; +} + +/* +** This method is the destructor for bytecodevtab objects. +*/ +static int fts5structDisconnectMethod(sqlite3_vtab *pVtab){ + Fts5StructVtab *p = (Fts5StructVtab*)pVtab; + sqlite3_free(p); + return SQLITE_OK; +} + +/* +** Constructor for a new bytecodevtab_cursor object. +*/ +static int fts5structOpenMethod(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCsr){ + int rc = SQLITE_OK; + Fts5StructVcsr *pNew = 0; + + pNew = sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + *ppCsr = (sqlite3_vtab_cursor*)pNew; + + return SQLITE_OK; +} + +/* +** Destructor for a bytecodevtab_cursor. +*/ +static int fts5structCloseMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + fts5StructureRelease(pCsr->pStruct); + sqlite3_free(pCsr); + return SQLITE_OK; +} + + +/* +** Advance a bytecodevtab_cursor to its next row of output. +*/ +static int fts5structNextMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + + assert( pCsr->pStruct ); + pCsr->iSeg++; + pCsr->iRowid++; + while( pCsr->iLevelnLevel && pCsr->iSeg>=p->aLevel[pCsr->iLevel].nSeg ){ + pCsr->iLevel++; + pCsr->iSeg = 0; + } + if( pCsr->iLevel>=p->nLevel ){ + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + } + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int fts5structEofMethod(sqlite3_vtab_cursor *cur){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + return pCsr->pStruct==0; +} + +static int fts5structRowidMethod( + sqlite3_vtab_cursor *cur, + sqlite_int64 *piRowid +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + *piRowid = pCsr->iRowid; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the bytecodevtab_cursor +** is currently pointing. +*/ +static int fts5structColumnMethod( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr*)cur; + Fts5Structure *p = pCsr->pStruct; + Fts5StructureSegment *pSeg = &p->aLevel[pCsr->iLevel].aSeg[pCsr->iSeg]; + + switch( i ){ + case 0: /* level */ + sqlite3_result_int(ctx, pCsr->iLevel); + break; + case 1: /* segment */ + sqlite3_result_int(ctx, pCsr->iSeg); + break; + case 2: /* merge */ + sqlite3_result_int(ctx, pCsr->iSeg < p->aLevel[pCsr->iLevel].nMerge); + break; + case 3: /* segid */ + sqlite3_result_int(ctx, pSeg->iSegid); + break; + case 4: /* leaf1 */ + sqlite3_result_int(ctx, pSeg->pgnoFirst); + break; + case 5: /* leaf2 */ + sqlite3_result_int(ctx, pSeg->pgnoLast); + break; + case 6: /* origin1 */ + sqlite3_result_int64(ctx, pSeg->iOrigin1); + break; + case 7: /* origin2 */ + sqlite3_result_int64(ctx, pSeg->iOrigin2); + break; + case 8: /* npgtombstone */ + sqlite3_result_int(ctx, pSeg->nPgTombstone); + break; + case 9: /* nentrytombstone */ + sqlite3_result_int64(ctx, pSeg->nEntryTombstone); + break; + case 10: /* nentry */ + sqlite3_result_int64(ctx, pSeg->nEntry); + break; + } + return SQLITE_OK; +} + +/* +** Initialize a cursor. +** +** idxNum==0 means show all subprograms +** idxNum==1 means show only the main bytecode and omit subprograms. +*/ +static int fts5structFilterMethod( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + Fts5StructVcsr *pCsr = (Fts5StructVcsr *)pVtabCursor; + int rc = SQLITE_OK; + + const u8 *aBlob = 0; + int nBlob = 0; + + assert( argc==1 ); + fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; + + nBlob = sqlite3_value_bytes(argv[0]); + aBlob = (const u8*)sqlite3_value_blob(argv[0]); + rc = fts5StructureDecode(aBlob, nBlob, 0, &pCsr->pStruct); + if( rc==SQLITE_OK ){ + pCsr->iLevel = 0; + pCsr->iRowid = 0; + pCsr->iSeg = -1; + rc = fts5structNextMethod(pVtabCursor); + } + + return rc; +} + +#endif /* SQLITE_TEST || SQLITE_FTS5_DEBUG */ /* ** This is called as part of registering the FTS5 module with database @@ -190352,13 +251285,14 @@ static void fts5RowidFunction( ** SQLite error code is returned instead. */ static int sqlite3Fts5IndexInit(sqlite3 *db){ +#if defined(SQLITE_TEST) || defined(SQLITE_FTS5_DEBUG) int rc = sqlite3_create_function( db, "fts5_decode", 2, SQLITE_UTF8, 0, fts5DecodeFunction, 0, 0 ); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( - db, "fts5_decode_none", 2, + db, "fts5_decode_none", 2, SQLITE_UTF8, (void*)db, fts5DecodeFunction, 0, 0 ); } @@ -190368,7 +251302,42 @@ static int sqlite3Fts5IndexInit(sqlite3 *db){ db, "fts5_rowid", -1, SQLITE_UTF8, 0, fts5RowidFunction, 0, 0 ); } + + if( rc==SQLITE_OK ){ + static const sqlite3_module fts5structure_module = { + 0, /* iVersion */ + 0, /* xCreate */ + fts5structConnectMethod, /* xConnect */ + fts5structBestIndexMethod, /* xBestIndex */ + fts5structDisconnectMethod, /* xDisconnect */ + 0, /* xDestroy */ + fts5structOpenMethod, /* xOpen */ + fts5structCloseMethod, /* xClose */ + fts5structFilterMethod, /* xFilter */ + fts5structNextMethod, /* xNext */ + fts5structEofMethod, /* xEof */ + fts5structColumnMethod, /* xColumn */ + fts5structRowidMethod, /* xRowid */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindFunction */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ + }; + rc = sqlite3_create_module(db, "fts5_structure", &fts5structure_module, 0); + } return rc; +#else + return SQLITE_OK; + UNUSED_PARAM(db); +#endif } @@ -190404,20 +251373,22 @@ static int sqlite3Fts5IndexReset(Fts5Index *p){ ** assert() conditions in the fts5 code are activated - conditions that are ** only true if it is guaranteed that the fts5 database is not corrupt. */ +#ifdef SQLITE_DEBUG SQLITE_API int sqlite3_fts5_may_be_corrupt = 1; +#endif typedef struct Fts5Auxdata Fts5Auxdata; typedef struct Fts5Auxiliary Fts5Auxiliary; typedef struct Fts5Cursor Fts5Cursor; +typedef struct Fts5FullTable Fts5FullTable; typedef struct Fts5Sorter Fts5Sorter; -typedef struct Fts5Table Fts5Table; typedef struct Fts5TokenizerModule Fts5TokenizerModule; /* -** NOTES ON TRANSACTIONS: +** NOTES ON TRANSACTIONS: ** -** SQLite invokes the following virtual table methods as transactions are +** SQLite invokes the following virtual table methods as transactions are ** opened and closed by the user: ** ** xBegin(): Start of a new transaction. @@ -190426,7 +251397,7 @@ typedef struct Fts5TokenizerModule Fts5TokenizerModule; ** xRollback(): Rollback the transaction. ** ** Anything that is required as part of a commit that may fail is performed -** in the xSync() callback. Current versions of SQLite ignore any errors +** in the xSync() callback. Current versions of SQLite ignore any errors ** returned by xCommit(). ** ** And as sub-transactions are opened/closed: @@ -190435,9 +251406,9 @@ typedef struct Fts5TokenizerModule Fts5TokenizerModule; ** xRelease(int S): Commit and close savepoint S. ** xRollbackTo(int S): Rollback to start of savepoint S. ** -** During a write-transaction the fts5_index.c module may cache some data +** During a write-transaction the fts5_index.c module may cache some data ** in-memory. It is flushed to disk whenever xSync(), xRelease() or -** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo() +** xSavepoint() is called. And discarded whenever xRollback() or xRollbackTo() ** is called. ** ** Additionally, if SQLITE_DEBUG is defined, an instance of the following @@ -190451,20 +251422,29 @@ struct Fts5TransactionState { }; /* -** A single object of this type is allocated when the FTS5 module is +** A single object of this type is allocated when the FTS5 module is ** registered with a database handle. It is used to store pointers to ** all registered FTS5 extensions - tokenizers and auxiliary functions. */ struct Fts5Global { fts5_api api; /* User visible part of object (see fts5.h) */ - sqlite3 *db; /* Associated database connection */ + sqlite3 *db; /* Associated database connection */ i64 iNextId; /* Used to allocate unique cursor ids */ Fts5Auxiliary *pAux; /* First in list of all aux. functions */ Fts5TokenizerModule *pTok; /* First in list of all tokenizer modules */ Fts5TokenizerModule *pDfltTok; /* Default tokenizer module */ Fts5Cursor *pCsr; /* First in list of all open cursors */ + u32 aLocaleHdr[4]; }; +/* +** Size of header on fts5_locale() values. And macro to access a buffer +** containing a copy of the header from an Fts5Config pointer. +*/ +#define FTS5_LOCALE_HDR_SIZE ((int)sizeof( ((Fts5Global*)0)->aLocaleHdr )) +#define FTS5_LOCALE_HDR(pConfig) ((const u8*)(pConfig->pGlobal->aLocaleHdr)) + + /* ** Each auxiliary function registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part @@ -190483,25 +251463,39 @@ struct Fts5Auxiliary { ** Each tokenizer module registered with the FTS5 module is represented ** by an object of the following type. All such objects are stored as part ** of the Fts5Global.pTok list. +** +** bV2Native: +** True if the tokenizer was registered using xCreateTokenizer_v2(), false +** for xCreateTokenizer(). If this variable is true, then x2 is populated +** with the routines as supplied by the caller and x1 contains synthesized +** wrapper routines. In this case the user-data pointer passed to +** x1.xCreate should be a pointer to the Fts5TokenizerModule structure, +** not a copy of pUserData. +** +** Of course, if bV2Native is false, then x1 contains the real routines and +** x2 the synthesized ones. In this case a pointer to the Fts5TokenizerModule +** object should be passed to x2.xCreate. +** +** The synthesized wrapper routines are necessary for xFindTokenizer(_v2) +** calls. */ struct Fts5TokenizerModule { char *zName; /* Name of tokenizer */ void *pUserData; /* User pointer passed to xCreate() */ - fts5_tokenizer x; /* Tokenizer functions */ + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ void (*xDestroy)(void*); /* Destructor function */ Fts5TokenizerModule *pNext; /* Next registered tokenizer module */ }; -/* -** Virtual-table object. -*/ -struct Fts5Table { - sqlite3_vtab base; /* Base class used by SQLite core */ - Fts5Config *pConfig; /* Virtual table configuration */ - Fts5Index *pIndex; /* Full-text index */ +struct Fts5FullTable { + Fts5Table p; /* Public class members from fts5Int.h */ Fts5Storage *pStorage; /* Document store */ Fts5Global *pGlobal; /* Global (connection wide) data */ Fts5Cursor *pSortCsr; /* Sort data from this cursor */ + int iSavepoint; /* Successful xSavepoint()+1 */ + #ifdef SQLITE_DEBUG struct Fts5TransactionState ts; #endif @@ -190518,7 +251512,7 @@ struct Fts5MatchPhrase { ** ** aIdx[]: ** There is one entry in the aIdx[] array for each phrase in the query, -** the value of which is the offset within aPoslist[] following the last +** the value of which is the offset within aPoslist[] following the last ** byte of the position list for the corresponding phrase. */ struct Fts5Sorter { @@ -190534,8 +251528,8 @@ struct Fts5Sorter { ** Virtual-table cursor object. ** ** iSpecial: -** If this is a 'special' query (refer to function fts5SpecialMatch()), -** then this variable contains the result of the query. +** If this is a 'special' query (refer to function fts5SpecialMatch()), +** then this variable contains the result of the query. ** ** iFirstRowid, iLastRowid: ** These variables are only used for FTS5_PLAN_MATCH cursors. Assuming the @@ -190578,7 +251572,7 @@ struct Fts5Cursor { Fts5Auxiliary *pAux; /* Currently executing extension function */ Fts5Auxdata *pAuxdata; /* First in linked list of saved aux-data */ - /* Cache used by auxiliary functions xInst() and xInstCount() */ + /* Cache used by auxiliary API functions xInst() and xInstCount() */ Fts5PoslistReader *aInstIter; /* One for each phrase */ int nInstAlloc; /* Size of aInst[] array (entries / 3) */ int nInstCount; /* Number of phrase instances */ @@ -190586,7 +251580,7 @@ struct Fts5Cursor { }; /* -** Bits that make up the "idxNum" parameter passed indirectly by +** Bits that make up the "idxNum" parameter passed indirectly by ** xBestIndex() to xFilter(). */ #define FTS5_BI_MATCH 0x0001 /* MATCH ? */ @@ -190636,7 +251630,7 @@ struct Fts5Auxdata { #define FTS5_SAVEPOINT 5 #define FTS5_RELEASE 6 #define FTS5_ROLLBACKTO 7 -static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ +static void fts5CheckTransactionState(Fts5FullTable *p, int op, int iSavepoint){ switch( op ){ case FTS5_BEGIN: assert( p->ts.eState==0 ); @@ -190645,7 +251639,7 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ break; case FTS5_SYNC: - assert( p->ts.eState==1 ); + assert( p->ts.eState==1 || p->ts.eState==2 ); p->ts.eState = 2; break; @@ -190660,23 +251654,26 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ break; case FTS5_SAVEPOINT: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=0 ); - assert( iSavepoint>p->ts.iSavepoint ); + assert( iSavepoint>=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint; break; - + case FTS5_RELEASE: - assert( p->ts.eState==1 ); + assert( p->ts.eState>=1 ); assert( iSavepoint>=0 ); assert( iSavepoint<=p->ts.iSavepoint ); p->ts.iSavepoint = iSavepoint-1; break; case FTS5_ROLLBACKTO: - assert( p->ts.eState==1 ); - assert( iSavepoint>=0 ); - assert( iSavepoint<=p->ts.iSavepoint ); + assert( p->ts.eState>=1 ); + assert( iSavepoint>=-1 ); + /* The following assert() can fail if another vtab strikes an error + ** within an xSavepoint() call then SQLite calls xRollbackTo() - without + ** having called xSavepoint() on this vtab. */ + /* assert( iSavepoint<=p->ts.iSavepoint ); */ p->ts.iSavepoint = iSavepoint; break; } @@ -190686,20 +251683,26 @@ static void fts5CheckTransactionState(Fts5Table *p, int op, int iSavepoint){ #endif /* -** Return true if pTab is a contentless table. +** Return true if pTab is a contentless table. If parameter bIncludeUnindexed +** is true, this includes contentless tables that store UNINDEXED columns +** only. */ -static int fts5IsContentless(Fts5Table *pTab){ - return pTab->pConfig->eContent==FTS5_CONTENT_NONE; +static int fts5IsContentless(Fts5FullTable *pTab, int bIncludeUnindexed){ + int eContent = pTab->p.pConfig->eContent; + return ( + eContent==FTS5_CONTENT_NONE + || (bIncludeUnindexed && eContent==FTS5_CONTENT_UNINDEXED) + ); } /* -** Delete a virtual table handle allocated by fts5InitVtab(). +** Delete a virtual table handle allocated by fts5InitVtab(). */ -static void fts5FreeVtab(Fts5Table *pTab){ +static void fts5FreeVtab(Fts5FullTable *pTab){ if( pTab ){ - sqlite3Fts5IndexClose(pTab->pIndex); + sqlite3Fts5IndexClose(pTab->p.pIndex); sqlite3Fts5StorageClose(pTab->pStorage); - sqlite3Fts5ConfigFree(pTab->pConfig); + sqlite3Fts5ConfigFree(pTab->p.pConfig); sqlite3_free(pTab); } } @@ -190708,7 +251711,7 @@ static void fts5FreeVtab(Fts5Table *pTab){ ** The xDisconnect() virtual table method. */ static int fts5DisconnectMethod(sqlite3_vtab *pVtab){ - fts5FreeVtab((Fts5Table*)pVtab); + fts5FreeVtab((Fts5FullTable*)pVtab); return SQLITE_OK; } @@ -190719,7 +251722,7 @@ static int fts5DestroyMethod(sqlite3_vtab *pVtab){ Fts5Table *pTab = (Fts5Table*)pVtab; int rc = sqlite3Fts5DropAll(pTab->pConfig); if( rc==SQLITE_OK ){ - fts5FreeVtab((Fts5Table*)pVtab); + fts5FreeVtab((Fts5FullTable*)pVtab); } return rc; } @@ -190748,28 +251751,32 @@ static int fts5InitVtab( const char **azConfig = (const char**)argv; int rc = SQLITE_OK; /* Return code */ Fts5Config *pConfig = 0; /* Results of parsing argc/argv */ - Fts5Table *pTab = 0; /* New virtual table object */ + Fts5FullTable *pTab = 0; /* New virtual table object */ /* Allocate the new vtab object and parse the configuration */ - pTab = (Fts5Table*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5Table)); + pTab = (Fts5FullTable*)sqlite3Fts5MallocZero(&rc, sizeof(Fts5FullTable)); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ConfigParse(pGlobal, db, argc, azConfig, &pConfig, pzErr); assert( (rc==SQLITE_OK && *pzErr==0) || pConfig==0 ); } if( rc==SQLITE_OK ){ - pTab->pConfig = pConfig; + pConfig->pzErrmsg = pzErr; + pTab->p.pConfig = pConfig; pTab->pGlobal = pGlobal; + if( bCreate || sqlite3Fts5TokenizerPreload(&pConfig->t) ){ + rc = sqlite3Fts5LoadTokenizer(pConfig); + } } /* Open the index sub-system */ if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->pIndex, pzErr); + rc = sqlite3Fts5IndexOpen(pConfig, bCreate, &pTab->p.pIndex, pzErr); } /* Open the storage sub-system */ if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageOpen( - pConfig, pTab->pIndex, bCreate, &pTab->pStorage, pzErr + pConfig, pTab->p.pIndex, bCreate, &pTab->pStorage, pzErr ); } @@ -190780,13 +251787,17 @@ static int fts5InitVtab( /* Load the initial configuration */ if( rc==SQLITE_OK ){ - assert( pConfig->pzErrmsg==0 ); - pConfig->pzErrmsg = pzErr; - rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); - sqlite3Fts5IndexRollback(pTab->pIndex); - pConfig->pzErrmsg = 0; + rc = sqlite3Fts5ConfigLoad(pTab->p.pConfig, pTab->p.pConfig->iCookie-1); } + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_CONSTRAINT_SUPPORT, (int)1); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_vtab_config(db, SQLITE_VTAB_INNOCUOUS); + } + + if( pConfig ) pConfig->pzErrmsg = 0; if( rc!=SQLITE_OK ){ fts5FreeVtab(pTab); pTab = 0; @@ -190848,25 +251859,66 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ #endif } +static int fts5UsePatternMatch( + Fts5Config *pConfig, + struct sqlite3_index_constraint *p +){ + assert( FTS5_PATTERN_GLOB==SQLITE_INDEX_CONSTRAINT_GLOB ); + assert( FTS5_PATTERN_LIKE==SQLITE_INDEX_CONSTRAINT_LIKE ); + if( pConfig->t.ePattern==FTS5_PATTERN_GLOB && p->op==FTS5_PATTERN_GLOB ){ + return 1; + } + if( pConfig->t.ePattern==FTS5_PATTERN_LIKE + && (p->op==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB) + ){ + return 1; + } + return 0; +} + /* -** Implementation of the xBestIndex method for FTS5 tables. Within the +** Implementation of the xBestIndex method for FTS5 tables. Within the ** WHERE constraint, it searches for the following: ** -** 1. A MATCH constraint against the special column. +** 1. A MATCH constraint against the table column. ** 2. A MATCH constraint against the "rank" column. -** 3. An == constraint against the rowid column. -** 4. A < or <= constraint against the rowid column. -** 5. A > or >= constraint against the rowid column. +** 3. A MATCH constraint against some other column. +** 4. An == constraint against the rowid column. +** 5. A < or <= constraint against the rowid column. +** 6. A > or >= constraint against the rowid column. ** -** Within the ORDER BY, either: +** Within the ORDER BY, the following are supported: ** ** 5. ORDER BY rank [ASC|DESC] ** 6. ORDER BY rowid [ASC|DESC] ** -** Costs are assigned as follows: +** Information for the xFilter call is passed via both the idxNum and +** idxStr variables. Specifically, idxNum is a bitmask of the following +** flags used to encode the ORDER BY clause: ** -** a) If an unusable MATCH operator is present in the WHERE clause, the -** cost is unconditionally set to 1e50 (a really big number). +** FTS5_BI_ORDER_RANK +** FTS5_BI_ORDER_ROWID +** FTS5_BI_ORDER_DESC +** +** idxStr is used to encode data from the WHERE clause. For each argument +** passed to the xFilter method, the following is appended to idxStr: +** +** Match against table column: "m" +** Match against rank column: "r" +** Match against other column: "M" +** LIKE against other column: "L" +** GLOB against other column: "G" +** Equality constraint against the rowid: "=" +** A < or <= against the rowid: "<" +** A > or >= against the rowid: ">" +** +** This function ensures that there is at most one "r" or "=". And that if +** there exists an "=" then there is no "<" or ">". +** +** If an unusable MATCH operator is present in the WHERE clause, then +** SQLITE_CONSTRAINT is returned. +** +** Costs are assigned as follows: ** ** a) If a MATCH operator is present, the cost depends on the other ** constraints also present. As follows: @@ -190888,60 +251940,113 @@ static void fts5SetUniqueFlag(sqlite3_index_info *pIdxInfo){ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ Fts5Table *pTab = (Fts5Table*)pVTab; Fts5Config *pConfig = pTab->pConfig; + const int nCol = pConfig->nCol; int idxFlags = 0; /* Parameter passed through to xFilter() */ - int bHasMatch; - int iNext; int i; - struct Constraint { - int op; /* Mask against sqlite3_index_constraint.op */ - int fts5op; /* FTS5 mask for idxFlags */ - int iCol; /* 0==rowid, 1==tbl, 2==rank */ - int omit; /* True to omit this if found */ - int iConsIndex; /* Index in pInfo->aConstraint[] */ - } aConstraint[] = { - {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, - FTS5_BI_MATCH, 1, 1, -1}, - {SQLITE_INDEX_CONSTRAINT_MATCH|SQLITE_INDEX_CONSTRAINT_EQ, - FTS5_BI_RANK, 2, 1, -1}, - {SQLITE_INDEX_CONSTRAINT_EQ, FTS5_BI_ROWID_EQ, 0, 0, -1}, - {SQLITE_INDEX_CONSTRAINT_LT|SQLITE_INDEX_CONSTRAINT_LE, - FTS5_BI_ROWID_LE, 0, 0, -1}, - {SQLITE_INDEX_CONSTRAINT_GT|SQLITE_INDEX_CONSTRAINT_GE, - FTS5_BI_ROWID_GE, 0, 0, -1}, - }; + char *idxStr; + int iIdxStr = 0; + int iCons = 0; - int aColMap[3]; - aColMap[0] = -1; - aColMap[1] = pConfig->nCol; - aColMap[2] = pConfig->nCol+1; + int bSeenEq = 0; + int bSeenGt = 0; + int bSeenLt = 0; + int nSeenMatch = 0; + int bSeenRank = 0; + + + assert( SQLITE_INDEX_CONSTRAINT_EQbLock ){ + pTab->base.zErrMsg = sqlite3_mprintf( + "recursively defined fts5 content table" + ); + return SQLITE_ERROR; + } + + idxStr = (char*)sqlite3_malloc(pInfo->nConstraint * 8 + 1); + if( idxStr==0 ) return SQLITE_NOMEM; + pInfo->idxStr = idxStr; + pInfo->needToFreeIdxStr = 1; - /* Set idxFlags flags for all WHERE clause terms that will be used. */ for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; - int j; - for(j=0; jiColumn==aColMap[pC->iCol] && p->op & pC->op ){ - if( p->usable ){ - pC->iConsIndex = i; - idxFlags |= pC->fts5op; - }else if( j==0 ){ - /* As there exists an unusable MATCH constraint this is an - ** unusable plan. Set a prohibitively high cost. */ - pInfo->estimatedCost = 1e50; - return SQLITE_OK; + int iCol = p->iColumn; + if( p->op==SQLITE_INDEX_CONSTRAINT_MATCH + || (p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol>=nCol) + ){ + /* A MATCH operator or equivalent */ + if( p->usable==0 || iCol<0 ){ + /* As there exists an unusable MATCH constraint this is an + ** unusable plan. Return SQLITE_CONSTRAINT. */ + return SQLITE_CONSTRAINT; + }else{ + if( iCol==nCol+1 ){ + if( bSeenRank ) continue; + idxStr[iIdxStr++] = 'r'; + bSeenRank = 1; + }else{ + nSeenMatch++; + idxStr[iIdxStr++] = 'M'; + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); + idxStr += strlen(&idxStr[iIdxStr]); + assert( idxStr[iIdxStr]=='\0' ); } + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + pInfo->aConstraintUsage[i].omit = 1; + } + }else if( p->usable ){ + if( iCol>=0 && iColop==FTS5_PATTERN_LIKE || p->op==FTS5_PATTERN_GLOB ); + idxStr[iIdxStr++] = p->op==FTS5_PATTERN_LIKE ? 'L' : 'G'; + sqlite3_snprintf(6, &idxStr[iIdxStr], "%d", iCol); + idxStr += strlen(&idxStr[iIdxStr]); + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + assert( idxStr[iIdxStr]=='\0' ); + nSeenMatch++; + }else if( bSeenEq==0 && p->op==SQLITE_INDEX_CONSTRAINT_EQ && iCol<0 ){ + idxStr[iIdxStr++] = '='; + bSeenEq = 1; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; } } } - /* Set idxFlags flags for the ORDER BY clause */ + if( bSeenEq==0 ){ + for(i=0; inConstraint; i++){ + struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; + if( p->iColumn<0 && p->usable ){ + int op = p->op; + if( op==SQLITE_INDEX_CONSTRAINT_LT || op==SQLITE_INDEX_CONSTRAINT_LE ){ + if( bSeenLt ) continue; + idxStr[iIdxStr++] = '<'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenLt = 1; + }else + if( op==SQLITE_INDEX_CONSTRAINT_GT || op==SQLITE_INDEX_CONSTRAINT_GE ){ + if( bSeenGt ) continue; + idxStr[iIdxStr++] = '>'; + pInfo->aConstraintUsage[i].argvIndex = ++iCons; + bSeenGt = 1; + } + } + } + } + idxStr[iIdxStr] = '\0'; + + /* Set idxFlags flags for the ORDER BY clause + ** + ** Note that tokendata=1 tables cannot currently handle "ORDER BY rowid DESC". + */ if( pInfo->nOrderBy==1 ){ int iSort = pInfo->aOrderBy[0].iColumn; - if( iSort==(pConfig->nCol+1) && BitFlagTest(idxFlags, FTS5_BI_MATCH) ){ + if( iSort==(pConfig->nCol+1) && nSeenMatch>0 ){ idxFlags |= FTS5_BI_ORDER_RANK; - }else if( iSort==-1 ){ + }else if( iSort==-1 && (!pInfo->aOrderBy[0].desc || !pConfig->bTokendata) ){ idxFlags |= FTS5_BI_ORDER_ROWID; } if( BitFlagTest(idxFlags, FTS5_BI_ORDER_RANK|FTS5_BI_ORDER_ROWID) ){ @@ -190953,33 +252058,25 @@ static int fts5BestIndexMethod(sqlite3_vtab *pVTab, sqlite3_index_info *pInfo){ } /* Calculate the estimated cost based on the flags set in idxFlags. */ - bHasMatch = BitFlagTest(idxFlags, FTS5_BI_MATCH); - if( BitFlagTest(idxFlags, FTS5_BI_ROWID_EQ) ){ - pInfo->estimatedCost = bHasMatch ? 100.0 : 10.0; - if( bHasMatch==0 ) fts5SetUniqueFlag(pInfo); - }else if( BitFlagAllTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ - pInfo->estimatedCost = bHasMatch ? 500.0 : 250000.0; - }else if( BitFlagTest(idxFlags, FTS5_BI_ROWID_LE|FTS5_BI_ROWID_GE) ){ - pInfo->estimatedCost = bHasMatch ? 750.0 : 750000.0; + if( bSeenEq ){ + pInfo->estimatedCost = nSeenMatch ? 1000.0 : 10.0; + if( nSeenMatch==0 ) fts5SetUniqueFlag(pInfo); + }else if( bSeenLt && bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 5000.0 : 250000.0; + }else if( bSeenLt || bSeenGt ){ + pInfo->estimatedCost = nSeenMatch ? 7500.0 : 750000.0; }else{ - pInfo->estimatedCost = bHasMatch ? 1000.0 : 1000000.0; + pInfo->estimatedCost = nSeenMatch ? 10000.0 : 1000000.0; } - - /* Assign argvIndex values to each constraint in use. */ - iNext = 1; - for(i=0; iiConsIndex>=0 ){ - pInfo->aConstraintUsage[pC->iConsIndex].argvIndex = iNext++; - pInfo->aConstraintUsage[pC->iConsIndex].omit = (unsigned char)pC->omit; - } + for(i=1; iestimatedCost *= 0.4; } pInfo->idxNum = idxFlags; return SQLITE_OK; } -static int fts5NewTransaction(Fts5Table *pTab){ +static int fts5NewTransaction(Fts5FullTable *pTab){ Fts5Cursor *pCsr; for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ if( pCsr->base.pVtab==(sqlite3_vtab*)pTab ) return SQLITE_OK; @@ -190991,19 +252088,19 @@ static int fts5NewTransaction(Fts5Table *pTab){ ** Implementation of xOpen method. */ static int fts5OpenMethod(sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr){ - Fts5Table *pTab = (Fts5Table*)pVTab; - Fts5Config *pConfig = pTab->pConfig; + Fts5FullTable *pTab = (Fts5FullTable*)pVTab; + Fts5Config *pConfig = pTab->p.pConfig; Fts5Cursor *pCsr = 0; /* New cursor object */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ int rc; /* Return code */ rc = fts5NewTransaction(pTab); if( rc==SQLITE_OK ){ nByte = sizeof(Fts5Cursor) + pConfig->nCol * sizeof(int); - pCsr = (Fts5Cursor*)sqlite3_malloc(nByte); + pCsr = (Fts5Cursor*)sqlite3_malloc64(nByte); if( pCsr ){ Fts5Global *pGlobal = pTab->pGlobal; - memset(pCsr, 0, nByte); + memset(pCsr, 0, (size_t)nByte); pCsr->aColumnSize = (int*)&pCsr[1]; pCsr->pNext = pGlobal->pCsr; pGlobal->pCsr = pCsr; @@ -191025,20 +252122,20 @@ static int fts5StmtType(Fts5Cursor *pCsr){ /* ** This function is called after the cursor passed as the only argument -** is moved to point at a different row. It clears all cached data +** is moved to point at a different row. It clears all cached data ** specific to the previous row stored by the cursor object. */ static void fts5CsrNewrow(Fts5Cursor *pCsr){ - CsrFlagSet(pCsr, - FTS5CSR_REQUIRE_CONTENT - | FTS5CSR_REQUIRE_DOCSIZE - | FTS5CSR_REQUIRE_INST - | FTS5CSR_REQUIRE_POSLIST + CsrFlagSet(pCsr, + FTS5CSR_REQUIRE_CONTENT + | FTS5CSR_REQUIRE_DOCSIZE + | FTS5CSR_REQUIRE_INST + | FTS5CSR_REQUIRE_POSLIST ); } static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); Fts5Auxdata *pData; Fts5Auxdata *pNext; @@ -191072,6 +252169,7 @@ static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ sqlite3_free(pCsr->zRankArgs); } + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan - (u8*)pCsr)); } @@ -191082,7 +252180,7 @@ static void fts5FreeCursorComponents(Fts5Cursor *pCsr){ */ static int fts5CloseMethod(sqlite3_vtab_cursor *pCursor){ if( pCursor ){ - Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; Fts5Cursor **pp; @@ -191103,7 +252201,7 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ rc = sqlite3_step(pSorter->pStmt); if( rc==SQLITE_DONE ){ rc = SQLITE_OK; - CsrFlagSet(pCsr, FTS5CSR_EOF); + CsrFlagSet(pCsr, FTS5CSR_EOF|FTS5CSR_REQUIRE_CONTENT); }else if( rc==SQLITE_ROW ){ const u8 *a; const u8 *aBlob; @@ -191136,14 +252234,14 @@ static int fts5SorterNext(Fts5Cursor *pCsr){ /* -** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors +** Set the FTS5CSR_REQUIRE_RESEEK flag on all FTS5_PLAN_MATCH cursors ** open on table pTab. */ -static void fts5TripCursors(Fts5Table *pTab){ +static void fts5TripCursors(Fts5FullTable *pTab){ Fts5Cursor *pCsr; for(pCsr=pTab->pGlobal->pCsr; pCsr; pCsr=pCsr->pNext){ if( pCsr->ePlan==FTS5_PLAN_MATCH - && pCsr->base.pVtab==(sqlite3_vtab*)pTab + && pCsr->base.pVtab==(sqlite3_vtab*)pTab ){ CsrFlagSet(pCsr, FTS5CSR_REQUIRE_RESEEK); } @@ -191152,25 +252250,25 @@ static void fts5TripCursors(Fts5Table *pTab){ /* ** If the REQUIRE_RESEEK flag is set on the cursor passed as the first -** argument, close and reopen all Fts5IndexIter iterators that the cursor +** argument, close and reopen all Fts5IndexIter iterators that the cursor ** is using. Then attempt to move the cursor to a rowid equal to or laster -** (in the cursors sort order - ASC or DESC) than the current rowid. +** (in the cursors sort order - ASC or DESC) than the current rowid. ** ** If the new rowid is not equal to the old, set output parameter *pbSkip ** to 1 before returning. Otherwise, leave it unchanged. ** -** Return SQLITE_OK if successful or if no reseek was required, or an +** Return SQLITE_OK if successful or if no reseek was required, or an ** error code if an error occurred. */ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ int rc = SQLITE_OK; assert( *pbSkip==0 ); if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_RESEEK) ){ - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); int bDesc = pCsr->bDesc; i64 iRowid = sqlite3Fts5ExprRowid(pCsr->pExpr); - rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->pIndex, iRowid, bDesc); + rc = sqlite3Fts5ExprFirst(pCsr->pExpr, pTab->p.pIndex, iRowid, bDesc); if( rc==SQLITE_OK && iRowid!=sqlite3Fts5ExprRowid(pCsr->pExpr) ){ *pbSkip = 1; } @@ -191187,7 +252285,7 @@ static int fts5CursorReseek(Fts5Cursor *pCsr, int *pbSkip){ /* -** Advance the cursor to the next row in the table that matches the +** Advance the cursor to the next row in the table that matches the ** search criteria. ** ** Return SQLITE_OK if nothing goes wrong. SQLITE_OK is returned @@ -191199,10 +252297,20 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ int rc; assert( (pCsr->ePlan<3)== - (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) + (pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SOURCE) ); assert( !CsrFlagTest(pCsr, FTS5CSR_EOF) ); + /* If this cursor uses FTS5_PLAN_MATCH and this is a tokendata=1 table, + ** clear any token mappings accumulated at the fts5_index.c level. In + ** other cases, specifically FTS5_PLAN_SOURCE and FTS5_PLAN_SORTED_MATCH, + ** we need to retain the mappings for the entire query. */ + if( pCsr->ePlan==FTS5_PLAN_MATCH + && ((Fts5Table*)pCursor->pVtab)->pConfig->bTokendata + ){ + sqlite3Fts5ExprClearTokens(pCsr->pExpr); + } + if( pCsr->ePlan<3 ){ int bSkip = 0; if( (rc = fts5CursorReseek(pCsr, &bSkip)) || bSkip ) return rc; @@ -191216,31 +252324,41 @@ static int fts5NextMethod(sqlite3_vtab_cursor *pCursor){ rc = SQLITE_OK; break; } - + case FTS5_PLAN_SORTED_MATCH: { rc = fts5SorterNext(pCsr); break; } - - default: + + default: { + Fts5Config *pConfig = ((Fts5Table*)pCursor->pVtab)->pConfig; + pConfig->bLock++; rc = sqlite3_step(pCsr->pStmt); + pConfig->bLock--; if( rc!=SQLITE_ROW ){ CsrFlagSet(pCsr, FTS5CSR_EOF); rc = sqlite3_reset(pCsr->pStmt); + if( rc!=SQLITE_OK ){ + pCursor->pVtab->zErrMsg = sqlite3_mprintf( + "%s", sqlite3_errmsg(pConfig->db) + ); + } }else{ rc = SQLITE_OK; + CsrFlagSet(pCsr, FTS5CSR_REQUIRE_DOCSIZE); } break; + } } } - + return rc; } static int fts5PrepareStatement( sqlite3_stmt **ppStmt, - Fts5Config *pConfig, + Fts5Config *pConfig, const char *zFmt, ... ){ @@ -191252,11 +252370,12 @@ static int fts5PrepareStatement( va_start(ap, zFmt); zSql = sqlite3_vmprintf(zFmt, ap); if( zSql==0 ){ - rc = SQLITE_NOMEM; + rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pRet, 0); + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT, &pRet, 0); if( rc!=SQLITE_OK ){ - *pConfig->pzErrmsg = sqlite3_mprintf("%s", sqlite3_errmsg(pConfig->db)); + sqlite3Fts5ConfigErrmsg(pConfig, "%s", sqlite3_errmsg(pConfig->db)); } sqlite3_free(zSql); } @@ -191264,33 +252383,37 @@ static int fts5PrepareStatement( va_end(ap); *ppStmt = pRet; return rc; -} +} -static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ - Fts5Config *pConfig = pTab->pConfig; +static int fts5CursorFirstSorted( + Fts5FullTable *pTab, + Fts5Cursor *pCsr, + int bDesc +){ + Fts5Config *pConfig = pTab->p.pConfig; Fts5Sorter *pSorter; int nPhrase; - int nByte; + sqlite3_int64 nByte; int rc; const char *zRank = pCsr->zRank; const char *zRankArgs = pCsr->zRankArgs; - + nPhrase = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); nByte = sizeof(Fts5Sorter) + sizeof(int) * (nPhrase-1); - pSorter = (Fts5Sorter*)sqlite3_malloc(nByte); + pSorter = (Fts5Sorter*)sqlite3_malloc64(nByte); if( pSorter==0 ) return SQLITE_NOMEM; - memset(pSorter, 0, nByte); + memset(pSorter, 0, (size_t)nByte); pSorter->nIdx = nPhrase; /* TODO: It would be better to have some system for reusing statement ** handles here, rather than preparing a new one for each query. But that ** is not possible as SQLite reference counts the virtual table objects. - ** And since the statement required here reads from this very virtual + ** And since the statement required here reads from this very virtual ** table, saving it creates a circular reference. ** ** If SQLite a built-in statement cache, this wouldn't be a problem. */ rc = fts5PrepareStatement(&pSorter->pStmt, pConfig, - "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(%s%s%s) %s", + "SELECT rowid, rank FROM %Q.%Q ORDER BY %s(\"%w\"%s%s) %s", pConfig->zDb, pConfig->zName, zRank, pConfig->zName, (zRankArgs ? ", " : ""), (zRankArgs ? zRankArgs : ""), @@ -191314,10 +252437,10 @@ static int fts5CursorFirstSorted(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ return rc; } -static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ +static int fts5CursorFirst(Fts5FullTable *pTab, Fts5Cursor *pCsr, int bDesc){ int rc; Fts5Expr *pExpr = pCsr->pExpr; - rc = sqlite3Fts5ExprFirst(pExpr, pTab->pIndex, pCsr->iFirstRowid, bDesc); + rc = sqlite3Fts5ExprFirst(pExpr, pTab->p.pIndex, pCsr->iFirstRowid, bDesc); if( sqlite3Fts5ExprEof(pExpr) ){ CsrFlagSet(pCsr, FTS5CSR_EOF); } @@ -191332,8 +252455,8 @@ static int fts5CursorFirst(Fts5Table *pTab, Fts5Cursor *pCsr, int bDesc){ ** parameters. */ static int fts5SpecialMatch( - Fts5Table *pTab, - Fts5Cursor *pCsr, + Fts5FullTable *pTab, + Fts5Cursor *pCsr, const char *zQuery ){ int rc = SQLITE_OK; /* Return code */ @@ -191343,18 +252466,18 @@ static int fts5SpecialMatch( while( z[0]==' ' ) z++; for(n=0; z[n] && z[n]!=' '; n++); - assert( pTab->base.zErrMsg==0 ); + assert( pTab->p.base.zErrMsg==0 ); pCsr->ePlan = FTS5_PLAN_SPECIAL; - if( 0==sqlite3_strnicmp("reads", z, n) ){ - pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->pIndex); + if( n==5 && 0==sqlite3_strnicmp("reads", z, n) ){ + pCsr->iSpecial = sqlite3Fts5IndexReads(pTab->p.pIndex); } - else if( 0==sqlite3_strnicmp("id", z, n) ){ + else if( n==2 && 0==sqlite3_strnicmp("id", z, n) ){ pCsr->iSpecial = pCsr->iCsrId; } else{ /* An unrecognized directive. Return an error message. */ - pTab->base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); + pTab->p.base.zErrMsg = sqlite3_mprintf("unknown special query: %.*s", n, z); rc = SQLITE_ERROR; } @@ -191366,7 +252489,7 @@ static int fts5SpecialMatch( ** pTab. If one is found, return a pointer to the corresponding Fts5Auxiliary ** structure. Otherwise, if no such function exists, return NULL. */ -static Fts5Auxiliary *fts5FindAuxiliary(Fts5Table *pTab, const char *zName){ +static Fts5Auxiliary *fts5FindAuxiliary(Fts5FullTable *pTab, const char *zName){ Fts5Auxiliary *pAux; for(pAux=pTab->pGlobal->pAux; pAux; pAux=pAux->pNext){ @@ -191379,8 +252502,8 @@ static Fts5Auxiliary *fts5FindAuxiliary(Fts5Table *pTab, const char *zName){ static int fts5FindRankFunction(Fts5Cursor *pCsr){ - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - Fts5Config *pConfig = pTab->pConfig; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->p.pConfig; int rc = SQLITE_OK; Fts5Auxiliary *pAux = 0; const char *zRank = pCsr->zRank; @@ -191390,12 +252513,13 @@ static int fts5FindRankFunction(Fts5Cursor *pCsr){ char *zSql = sqlite3Fts5Mprintf(&rc, "SELECT %s", zRankArgs); if( zSql ){ sqlite3_stmt *pStmt = 0; - rc = sqlite3_prepare_v2(pConfig->db, zSql, -1, &pStmt, 0); + rc = sqlite3_prepare_v3(pConfig->db, zSql, -1, + SQLITE_PREPARE_PERSISTENT, &pStmt, 0); sqlite3_free(zSql); assert( rc==SQLITE_OK || pCsr->pRankArgStmt==0 ); if( rc==SQLITE_OK ){ if( SQLITE_ROW==sqlite3_step(pStmt) ){ - int nByte; + sqlite3_int64 nByte; pCsr->nRankArg = sqlite3_column_count(pStmt); nByte = sizeof(sqlite3_value*)*pCsr->nRankArg; pCsr->apRankArg = (sqlite3_value**)sqlite3Fts5MallocZero(&rc, nByte); @@ -191417,8 +252541,8 @@ static int fts5FindRankFunction(Fts5Cursor *pCsr){ if( rc==SQLITE_OK ){ pAux = fts5FindAuxiliary(pTab, zRank); if( pAux==0 ){ - assert( pTab->base.zErrMsg==0 ); - pTab->base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank); + assert( pTab->p.base.zErrMsg==0 ); + pTab->p.base.zErrMsg = sqlite3_mprintf("no such function: %s", zRank); rc = SQLITE_ERROR; } } @@ -191430,7 +252554,7 @@ static int fts5FindRankFunction(Fts5Cursor *pCsr){ static int fts5CursorParseRank( Fts5Config *pConfig, - Fts5Cursor *pCsr, + Fts5Cursor *pCsr, sqlite3_value *pRank ){ int rc = SQLITE_OK; @@ -191475,11 +252599,150 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ return iDefault; } +/* +** Set the error message on the virtual table passed as the first argument. +*/ +static void fts5SetVtabError(Fts5FullTable *p, const char *zFormat, ...){ + va_list ap; /* ... printf arguments */ + va_start(ap, zFormat); + sqlite3_free(p->p.base.zErrMsg); + p->p.base.zErrMsg = sqlite3_vmprintf(zFormat, ap); + va_end(ap); +} + +/* +** Arrange for subsequent calls to sqlite3Fts5Tokenize() to use the locale +** specified by pLocale/nLocale. The buffer indicated by pLocale must remain +** valid until after the final call to sqlite3Fts5Tokenize() that will use +** the locale. +*/ +static void sqlite3Fts5SetLocale( + Fts5Config *pConfig, + const char *zLocale, + int nLocale +){ + Fts5TokenizerConfig *pT = &pConfig->t; + pT->pLocale = zLocale; + pT->nLocale = nLocale; +} + +/* +** Clear any locale configured by an earlier call to sqlite3Fts5SetLocale(). +*/ +static void sqlite3Fts5ClearLocale(Fts5Config *pConfig){ + sqlite3Fts5SetLocale(pConfig, 0, 0); +} + +/* +** Return true if the value passed as the only argument is an +** fts5_locale() value. +*/ +static int sqlite3Fts5IsLocaleValue(Fts5Config *pConfig, sqlite3_value *pVal){ + int ret = 0; + if( sqlite3_value_type(pVal)==SQLITE_BLOB ){ + /* Call sqlite3_value_bytes() after sqlite3_value_blob() in this case. + ** If the blob was created using zeroblob(), then sqlite3_value_blob() + ** may call malloc(). If this malloc() fails, then the values returned + ** by both value_blob() and value_bytes() will be 0. If value_bytes() were + ** called first, then the NULL pointer returned by value_blob() might + ** be dereferenced. */ + const u8 *pBlob = sqlite3_value_blob(pVal); + int nBlob = sqlite3_value_bytes(pVal); + if( nBlob>FTS5_LOCALE_HDR_SIZE + && 0==memcmp(pBlob, FTS5_LOCALE_HDR(pConfig), FTS5_LOCALE_HDR_SIZE) + ){ + ret = 1; + } + } + return ret; +} + +/* +** Value pVal is guaranteed to be an fts5_locale() value, according to +** sqlite3Fts5IsLocaleValue(). This function extracts the text and locale +** from the value and returns them separately. +** +** If successful, SQLITE_OK is returned and (*ppText) and (*ppLoc) set +** to point to buffers containing the text and locale, as utf-8, +** respectively. In this case output parameters (*pnText) and (*pnLoc) are +** set to the sizes in bytes of these two buffers. +** +** Or, if an error occurs, then an SQLite error code is returned. The final +** value of the four output parameters is undefined in this case. +*/ +static int sqlite3Fts5DecodeLocaleValue( + sqlite3_value *pVal, + const char **ppText, + int *pnText, + const char **ppLoc, + int *pnLoc +){ + const char *p = sqlite3_value_blob(pVal); + int n = sqlite3_value_bytes(pVal); + int nLoc = 0; + + assert( sqlite3_value_type(pVal)==SQLITE_BLOB ); + assert( n>FTS5_LOCALE_HDR_SIZE ); + + for(nLoc=FTS5_LOCALE_HDR_SIZE; p[nLoc]; nLoc++){ + if( nLoc==(n-1) ){ + return SQLITE_MISMATCH; + } + } + *ppLoc = &p[FTS5_LOCALE_HDR_SIZE]; + *pnLoc = nLoc - FTS5_LOCALE_HDR_SIZE; + + *ppText = &p[nLoc+1]; + *pnText = n - nLoc - 1; + return SQLITE_OK; +} + +/* +** Argument pVal is the text of a full-text search expression. It may or +** may not have been wrapped by fts5_locale(). This function extracts +** the text of the expression, and sets output variable (*pzText) to +** point to a nul-terminated buffer containing the expression. +** +** If pVal was an fts5_locale() value, then sqlite3Fts5SetLocale() is called +** to set the tokenizer to use the specified locale. +** +** If output variable (*pbFreeAndReset) is set to true, then the caller +** is required to (a) call sqlite3Fts5ClearLocale() to reset the tokenizer +** locale, and (b) call sqlite3_free() to free (*pzText). +*/ +static int fts5ExtractExprText( + Fts5Config *pConfig, /* Fts5 configuration */ + sqlite3_value *pVal, /* Value to extract expression text from */ + char **pzText, /* OUT: nul-terminated buffer of text */ + int *pbFreeAndReset /* OUT: Free (*pzText) and clear locale */ +){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + *pzText = sqlite3Fts5Mprintf(&rc, "%.*s", nText, pText); + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + } + *pbFreeAndReset = 1; + }else{ + *pzText = (char*)sqlite3_value_text(pVal); + *pbFreeAndReset = 0; + } + + return rc; +} + + /* ** This is the xFilter interface for the virtual table. See ** the virtual table xFilter method documentation for additional ** information. -** +** ** There are three possible query strategies: ** ** 1. Full-text search using a MATCH operator. @@ -191489,27 +252752,27 @@ static i64 fts5GetRowidLimit(sqlite3_value *pVal, i64 iDefault){ static int fts5FilterMethod( sqlite3_vtab_cursor *pCursor, /* The cursor used for this query */ int idxNum, /* Strategy index */ - const char *zUnused, /* Unused */ + const char *idxStr, /* Unused */ int nVal, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ - Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); - Fts5Config *pConfig = pTab->pConfig; + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->p.pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; /* Error code */ - int iVal = 0; /* Counter for apVal[] */ int bDesc; /* True if ORDER BY [rank|rowid] DESC */ int bOrderByRank; /* True if ORDER BY rank */ - sqlite3_value *pMatch = 0; /* MATCH ? expression (or NULL) */ sqlite3_value *pRank = 0; /* rank MATCH ? expression (or NULL) */ sqlite3_value *pRowidEq = 0; /* rowid = ? expression (or NULL) */ sqlite3_value *pRowidLe = 0; /* rowid <= ? expression (or NULL) */ sqlite3_value *pRowidGe = 0; /* rowid >= ? expression (or NULL) */ + int iCol; /* Column on LHS of MATCH operator */ char **pzErrmsg = pConfig->pzErrmsg; + int i; + int iIdxStr = 0; + Fts5Expr *pExpr = 0; - UNUSED_PARAM(zUnused); - UNUSED_PARAM(nVal); - + assert( pConfig->bLock==0 ); if( pCsr->ePlan ){ fts5FreeCursorComponents(pCsr); memset(&pCsr->ePlan, 0, sizeof(Fts5Cursor) - ((u8*)&pCsr->ePlan-(u8*)pCsr)); @@ -191521,25 +252784,90 @@ static int fts5FilterMethod( assert( pCsr->pRank==0 ); assert( pCsr->zRank==0 ); assert( pCsr->zRankArgs==0 ); + assert( pTab->pSortCsr==0 || nVal==0 ); - assert( pzErrmsg==0 || pzErrmsg==&pTab->base.zErrMsg ); - pConfig->pzErrmsg = &pTab->base.zErrMsg; + assert( pzErrmsg==0 || pzErrmsg==&pTab->p.base.zErrMsg ); + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; - /* Decode the arguments passed through to this function. - ** - ** Note: The following set of if(...) statements must be in the same - ** order as the corresponding entries in the struct at the top of - ** fts5BestIndexMethod(). */ - if( BitFlagTest(idxNum, FTS5_BI_MATCH) ) pMatch = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_RANK) ) pRank = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_ROWID_EQ) ) pRowidEq = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_ROWID_LE) ) pRowidLe = apVal[iVal++]; - if( BitFlagTest(idxNum, FTS5_BI_ROWID_GE) ) pRowidGe = apVal[iVal++]; - assert( iVal==nVal ); + /* Decode the arguments passed through to this function. */ + for(i=0; i='0' && idxStr[iIdxStr]<='9' ); + + if( zText[0]=='*' ){ + /* The user has issued a query of the form "MATCH '*...'". This + ** indicates that the MATCH expression is not a full text query, + ** but a request for an internal parameter. */ + rc = fts5SpecialMatch(pTab, pCsr, &zText[1]); + bInternal = 1; + }else{ + char **pzErr = &pTab->p.base.zErrMsg; + rc = sqlite3Fts5ExprNew(pConfig, 0, iCol, zText, &pExpr, pzErr); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + } + + if( bFreeAndReset ){ + sqlite3_free(zText); + sqlite3Fts5ClearLocale(pConfig); + } + + if( bInternal || rc!=SQLITE_OK ) goto filter_out; + + break; + } + case 'L': + case 'G': { + int bGlob = (idxStr[iIdxStr-1]=='G'); + const char *zText = (const char*)sqlite3_value_text(apVal[i]); + iCol = 0; + do{ + iCol = iCol*10 + (idxStr[iIdxStr]-'0'); + iIdxStr++; + }while( idxStr[iIdxStr]>='0' && idxStr[iIdxStr]<='9' ); + if( zText ){ + rc = sqlite3Fts5ExprPattern(pConfig, bGlob, iCol, zText, &pExpr); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ExprAnd(&pCsr->pExpr, pExpr); + pExpr = 0; + } + if( rc!=SQLITE_OK ) goto filter_out; + break; + } + case '=': + pRowidEq = apVal[i]; + break; + case '<': + pRowidLe = apVal[i]; + break; + default: assert( idxStr[iIdxStr-1]=='>' ); + pRowidGe = apVal[i]; + break; + } + } bOrderByRank = ((idxNum & FTS5_BI_ORDER_RANK) ? 1 : 0); pCsr->bDesc = bDesc = ((idxNum & FTS5_BI_ORDER_DESC) ? 1 : 0); - /* Set the cursor upper and lower rowid limits. Only some strategies + /* Set the cursor upper and lower rowid limits. Only some strategies ** actually use them. This is ok, as the xBestIndex() method leaves the ** sqlite3_index_constraint.omit flag clear for range constraints ** on the rowid field. */ @@ -191554,61 +252882,56 @@ static int fts5FilterMethod( pCsr->iFirstRowid = fts5GetRowidLimit(pRowidGe, SMALLEST_INT64); } + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) goto filter_out; + if( pTab->pSortCsr ){ - /* If pSortCsr is non-NULL, then this call is being made as part of + /* If pSortCsr is non-NULL, then this call is being made as part of ** processing for a "... MATCH ORDER BY rank" query (ePlan is ** set to FTS5_PLAN_SORTED_MATCH). pSortCsr is the cursor that will - ** return results to the user for this query. The current cursor - ** (pCursor) is used to execute the query issued by function + ** return results to the user for this query. The current cursor + ** (pCursor) is used to execute the query issued by function ** fts5CursorFirstSorted() above. */ assert( pRowidEq==0 && pRowidLe==0 && pRowidGe==0 && pRank==0 ); - assert( nVal==0 && pMatch==0 && bOrderByRank==0 && bDesc==0 ); + assert( nVal==0 && bOrderByRank==0 && bDesc==0 ); assert( pCsr->iLastRowid==LARGEST_INT64 ); assert( pCsr->iFirstRowid==SMALLEST_INT64 ); + if( pTab->pSortCsr->bDesc ){ + pCsr->iLastRowid = pTab->pSortCsr->iFirstRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iLastRowid; + }else{ + pCsr->iLastRowid = pTab->pSortCsr->iLastRowid; + pCsr->iFirstRowid = pTab->pSortCsr->iFirstRowid; + } pCsr->ePlan = FTS5_PLAN_SOURCE; pCsr->pExpr = pTab->pSortCsr->pExpr; rc = fts5CursorFirst(pTab, pCsr, bDesc); - sqlite3Fts5ExprClearEof(pCsr->pExpr); - }else if( pMatch ){ - const char *zExpr = (const char*)sqlite3_value_text(apVal[0]); - if( zExpr==0 ) zExpr = ""; - + }else if( pCsr->pExpr ){ + assert( rc==SQLITE_OK ); rc = fts5CursorParseRank(pConfig, pCsr, pRank); if( rc==SQLITE_OK ){ - if( zExpr[0]=='*' ){ - /* The user has issued a query of the form "MATCH '*...'". This - ** indicates that the MATCH expression is not a full text query, - ** but a request for an internal parameter. */ - rc = fts5SpecialMatch(pTab, pCsr, &zExpr[1]); + if( bOrderByRank ){ + pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; + rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); }else{ - char **pzErr = &pTab->base.zErrMsg; - rc = sqlite3Fts5ExprNew(pConfig, zExpr, &pCsr->pExpr, pzErr); - if( rc==SQLITE_OK ){ - if( bOrderByRank ){ - pCsr->ePlan = FTS5_PLAN_SORTED_MATCH; - rc = fts5CursorFirstSorted(pTab, pCsr, bDesc); - }else{ - pCsr->ePlan = FTS5_PLAN_MATCH; - rc = fts5CursorFirst(pTab, pCsr, bDesc); - } - } + pCsr->ePlan = FTS5_PLAN_MATCH; + rc = fts5CursorFirst(pTab, pCsr, bDesc); } } }else if( pConfig->zContent==0 ){ - *pConfig->pzErrmsg = sqlite3_mprintf( - "%s: table does not support scanning", pConfig->zName - ); + fts5SetVtabError(pTab,"%s: table does not support scanning",pConfig->zName); rc = SQLITE_ERROR; }else{ /* This is either a full-table scan (ePlan==FTS5_PLAN_SCAN) or a lookup ** by rowid (ePlan==FTS5_PLAN_ROWID). */ pCsr->ePlan = (pRowidEq ? FTS5_PLAN_ROWID : FTS5_PLAN_SCAN); rc = sqlite3Fts5StorageStmt( - pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->base.zErrMsg + pTab->pStorage, fts5StmtType(pCsr), &pCsr->pStmt, &pTab->p.base.zErrMsg ); if( rc==SQLITE_OK ){ - if( pCsr->ePlan==FTS5_PLAN_ROWID ){ - sqlite3_bind_value(pCsr->pStmt, 1, apVal[0]); + if( pRowidEq!=0 ){ + assert( pCsr->ePlan==FTS5_PLAN_ROWID ); + sqlite3_bind_value(pCsr->pStmt, 1, pRowidEq); }else{ sqlite3_bind_int64(pCsr->pStmt, 1, pCsr->iFirstRowid); sqlite3_bind_int64(pCsr->pStmt, 2, pCsr->iLastRowid); @@ -191617,12 +252940,14 @@ static int fts5FilterMethod( } } + filter_out: + sqlite3Fts5ExprFree(pExpr); pConfig->pzErrmsg = pzErrmsg; return rc; } -/* -** This is the xEof method of the virtual table. SQLite calls this +/* +** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ @@ -191634,18 +252959,22 @@ static int fts5EofMethod(sqlite3_vtab_cursor *pCursor){ ** Return the rowid that the cursor currently points to. */ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ - assert( pCsr->ePlan==FTS5_PLAN_MATCH - || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH - || pCsr->ePlan==FTS5_PLAN_SOURCE + assert( pCsr->ePlan==FTS5_PLAN_MATCH + || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH + || pCsr->ePlan==FTS5_PLAN_SOURCE + || pCsr->ePlan==FTS5_PLAN_SCAN + || pCsr->ePlan==FTS5_PLAN_ROWID ); if( pCsr->pSorter ){ return pCsr->pSorter->iRowid; + }else if( pCsr->ePlan>=FTS5_PLAN_SCAN ){ + return sqlite3_column_int64(pCsr->pStmt, 0); }else{ return sqlite3Fts5ExprRowid(pCsr->pExpr); } } -/* +/* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. fts5 ** exposes %_content.rowid as the rowid for the virtual table. The @@ -191654,27 +252983,18 @@ static i64 fts5CursorRowid(Fts5Cursor *pCsr){ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int ePlan = pCsr->ePlan; - + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); - switch( ePlan ){ - case FTS5_PLAN_SPECIAL: - *pRowid = 0; - break; - - case FTS5_PLAN_SOURCE: - case FTS5_PLAN_MATCH: - case FTS5_PLAN_SORTED_MATCH: - *pRowid = fts5CursorRowid(pCsr); - break; - - default: - *pRowid = sqlite3_column_int64(pCsr->pStmt, 0); - break; + if( ePlan==FTS5_PLAN_SPECIAL ){ + *pRowid = 0; + }else{ + *pRowid = fts5CursorRowid(pCsr); } return SQLITE_OK; } + /* ** If the cursor requires seeking (bSeekRequired flag is set), seek it. ** Return SQLITE_OK if no error occurs, or an SQLite error code otherwise. @@ -191685,22 +253005,25 @@ static int fts5RowidMethod(sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid){ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ int rc = SQLITE_OK; - /* If the cursor does not yet have a statement handle, obtain one now. */ + /* If the cursor does not yet have a statement handle, obtain one now. */ if( pCsr->pStmt==0 ){ - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); int eStmt = fts5StmtType(pCsr); rc = sqlite3Fts5StorageStmt( - pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->base.zErrMsg:0) + pTab->pStorage, eStmt, &pCsr->pStmt, (bErrormsg?&pTab->p.base.zErrMsg:0) ); - assert( rc!=SQLITE_OK || pTab->base.zErrMsg==0 ); + assert( rc!=SQLITE_OK || pTab->p.base.zErrMsg==0 ); assert( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ); } if( rc==SQLITE_OK && CsrFlagTest(pCsr, FTS5CSR_REQUIRE_CONTENT) ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); assert( pCsr->pExpr ); sqlite3_reset(pCsr->pStmt); sqlite3_bind_int64(pCsr->pStmt, 1, fts5CursorRowid(pCsr)); + pTab->pConfig->bLock++; rc = sqlite3_step(pCsr->pStmt); + pTab->pConfig->bLock--; if( rc==SQLITE_ROW ){ rc = SQLITE_OK; CsrFlagClear(pCsr, FTS5CSR_REQUIRE_CONTENT); @@ -191708,20 +253031,21 @@ static int fts5SeekCursor(Fts5Cursor *pCsr, int bErrormsg){ rc = sqlite3_reset(pCsr->pStmt); if( rc==SQLITE_OK ){ rc = FTS5_CORRUPT; + fts5SetVtabError((Fts5FullTable*)pTab, + "fts5: missing row %lld from content table %s", + fts5CursorRowid(pCsr), + pTab->pConfig->zContent + ); + }else if( pTab->pConfig->pzErrmsg ){ + fts5SetVtabError((Fts5FullTable*)pTab, + "%s", sqlite3_errmsg(pTab->pConfig->db) + ); } } } return rc; } -static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ - va_list ap; /* ... printf arguments */ - va_start(ap, zFormat); - assert( p->base.zErrMsg==0 ); - p->base.zErrMsg = sqlite3_vmprintf(zFormat, ap); - va_end(ap); -} - /* ** This function is called to handle an FTS INSERT command. In other words, ** an INSERT statement of the form: @@ -191729,7 +253053,7 @@ static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ ** INSERT INTO fts(fts) VALUES($pCmd) ** INSERT INTO fts(fts, rank) VALUES($pCmd, $pVal) ** -** Argument pVal is the value assigned to column "fts" by the INSERT +** Argument pVal is the value assigned to column "fts" by the INSERT ** statement. This function returns SQLITE_OK if successful, or an SQLite ** error code if an error occurs. ** @@ -191738,17 +253062,18 @@ static void fts5SetVtabError(Fts5Table *p, const char *zFormat, ...){ ** more commands are added to this function. */ static int fts5SpecialInsert( - Fts5Table *pTab, /* Fts5 table object */ + Fts5FullTable *pTab, /* Fts5 table object */ const char *zCmd, /* Text inserted into table-name column */ sqlite3_value *pVal /* Value inserted into rank column */ ){ - Fts5Config *pConfig = pTab->pConfig; + Fts5Config *pConfig = pTab->p.pConfig; int rc = SQLITE_OK; int bError = 0; + int bLoadConfig = 0; if( 0==sqlite3_stricmp("delete-all", zCmd) ){ if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ - fts5SetVtabError(pTab, + fts5SetVtabError(pTab, "'delete-all' may only be used with a " "contentless or external content fts5 table" ); @@ -191756,30 +253081,38 @@ static int fts5SpecialInsert( }else{ rc = sqlite3Fts5StorageDeleteAll(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("rebuild", zCmd) ){ - if( pConfig->eContent==FTS5_CONTENT_NONE ){ - fts5SetVtabError(pTab, + if( fts5IsContentless(pTab, 1) ){ + fts5SetVtabError(pTab, "'rebuild' may not be used with a contentless fts5 table" ); rc = SQLITE_ERROR; }else{ rc = sqlite3Fts5StorageRebuild(pTab->pStorage); } + bLoadConfig = 1; }else if( 0==sqlite3_stricmp("optimize", zCmd) ){ rc = sqlite3Fts5StorageOptimize(pTab->pStorage); }else if( 0==sqlite3_stricmp("merge", zCmd) ){ int nMerge = sqlite3_value_int(pVal); rc = sqlite3Fts5StorageMerge(pTab->pStorage, nMerge); }else if( 0==sqlite3_stricmp("integrity-check", zCmd) ){ - rc = sqlite3Fts5StorageIntegrity(pTab->pStorage); + int iArg = sqlite3_value_int(pVal); + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, iArg); #ifdef SQLITE_DEBUG }else if( 0==sqlite3_stricmp("prefix-index", zCmd) ){ pConfig->bPrefixIndex = sqlite3_value_int(pVal); #endif + }else if( 0==sqlite3_stricmp("flush", zCmd) ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); }else{ - rc = sqlite3Fts5IndexLoadConfig(pTab->pIndex); + rc = sqlite3Fts5FlushToDisk(&pTab->p); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5ConfigSetValue(pTab->pConfig, zCmd, pVal, &bError); + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5ConfigSetValue(pTab->p.pConfig, zCmd, pVal, &bError); } if( rc==SQLITE_OK ){ if( bError ){ @@ -191789,31 +253122,37 @@ static int fts5SpecialInsert( } } } + + if( rc==SQLITE_OK && bLoadConfig ){ + pTab->p.pConfig->iCookie--; + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + } + return rc; } static int fts5SpecialDelete( - Fts5Table *pTab, + Fts5FullTable *pTab, sqlite3_value **apVal ){ int rc = SQLITE_OK; int eType1 = sqlite3_value_type(apVal[1]); if( eType1==SQLITE_INTEGER ){ sqlite3_int64 iDel = sqlite3_value_int64(apVal[1]); - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2]); + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, &apVal[2], 0); } return rc; } static void fts5StorageInsert( - int *pRc, - Fts5Table *pTab, - sqlite3_value **apVal, + int *pRc, + Fts5FullTable *pTab, + sqlite3_value **apVal, i64 *piRowid ){ int rc = *pRc; if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, piRowid); + rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, 0, apVal, piRowid); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *piRowid); @@ -191821,13 +253160,74 @@ static void fts5StorageInsert( *pRc = rc; } -/* -** This function is the implementation of the xUpdate callback used by +/* +** +** This function is called when the user attempts an UPDATE on a contentless +** table. Parameter bRowidModified is true if the UPDATE statement modifies +** the rowid value. Parameter apVal[] contains the new values for each user +** defined column of the fts5 table. pConfig is the configuration object of the +** table being updated (guaranteed to be contentless). The contentless_delete=1 +** and contentless_unindexed=1 options may or may not be set. +** +** This function returns SQLITE_OK if the UPDATE can go ahead, or an SQLite +** error code if it cannot. In this case an error message is also loaded into +** pConfig. Output parameter (*pbContent) is set to true if the caller should +** update the %_content table only - not the FTS index or any other shadow +** table. This occurs when an UPDATE modifies only UNINDEXED columns of the +** table. +** +** An UPDATE may proceed if: +** +** * The only columns modified are UNINDEXED columns, or +** +** * The contentless_delete=1 option was specified and all of the indexed +** columns (not a subset) have been modified. +*/ +static int fts5ContentlessUpdate( + Fts5Config *pConfig, + sqlite3_value **apVal, + int bRowidModified, + int *pbContent +){ + int ii; + int bSeenIndex = 0; /* Have seen modified indexed column */ + int bSeenIndexNC = 0; /* Have seen unmodified indexed column */ + int rc = SQLITE_OK; + + for(ii=0; iinCol; ii++){ + if( pConfig->abUnindexed[ii]==0 ){ + if( sqlite3_value_nochange(apVal[ii]) ){ + bSeenIndexNC++; + }else{ + bSeenIndex++; + } + } + } + + if( bSeenIndex==0 && bRowidModified==0 ){ + *pbContent = 1; + }else{ + if( bSeenIndexNC || pConfig->bContentlessDelete==0 ){ + rc = SQLITE_ERROR; + sqlite3Fts5ConfigErrmsg(pConfig, + (pConfig->bContentlessDelete ? + "%s a subset of columns on fts5 contentless-delete table: %s" : + "%s contentless fts5 table: %s") + , "cannot UPDATE", pConfig->zName + ); + } + } + + return rc; +} + +/* +** This function is the implementation of the xUpdate callback used by ** FTS3 virtual tables. It is invoked by SQLite each time a row is to be ** inserted, updated or deleted. ** ** A delete specifies a single argument - the rowid of the row to remove. -** +** ** Update and insert operations pass: ** ** 1. The "old" rowid, or NULL. @@ -191841,42 +253241,55 @@ static int fts5UpdateMethod( sqlite3_value **apVal, /* Array of arguments */ sqlite_int64 *pRowid /* OUT: The affected (or effected) rowid */ ){ - Fts5Table *pTab = (Fts5Table*)pVtab; - Fts5Config *pConfig = pTab->pConfig; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + Fts5Config *pConfig = pTab->p.pConfig; int eType0; /* value_type() of apVal[0] */ int rc = SQLITE_OK; /* Return code */ + int bUpdateOrDelete = 0; /* A transaction must be open when this is called. */ - assert( pTab->ts.eState==1 ); + assert( pTab->ts.eState==1 || pTab->ts.eState==2 ); assert( pVtab->zErrMsg==0 ); assert( nArg==1 || nArg==(2+pConfig->nCol+2) ); - assert( nArg==1 - || sqlite3_value_type(apVal[1])==SQLITE_INTEGER - || sqlite3_value_type(apVal[1])==SQLITE_NULL + assert( sqlite3_value_type(apVal[0])==SQLITE_INTEGER + || sqlite3_value_type(apVal[0])==SQLITE_NULL ); - assert( pTab->pConfig->pzErrmsg==0 ); - pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; + assert( pTab->p.pConfig->pzErrmsg==0 ); + if( pConfig->pgsz==0 ){ + rc = sqlite3Fts5IndexLoadConfig(pTab->p.pIndex); + if( rc!=SQLITE_OK ) return rc; + } + + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; /* Put any active cursors into REQUIRE_SEEK state. */ fts5TripCursors(pTab); eType0 = sqlite3_value_type(apVal[0]); - if( eType0==SQLITE_NULL - && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL + if( eType0==SQLITE_NULL + && sqlite3_value_type(apVal[2+pConfig->nCol])!=SQLITE_NULL ){ /* A "special" INSERT op. These are handled separately. */ const char *z = (const char*)sqlite3_value_text(apVal[2+pConfig->nCol]); - if( pConfig->eContent!=FTS5_CONTENT_NORMAL - && 0==sqlite3_stricmp("delete", z) + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && 0==sqlite3_stricmp("delete", z) ){ - rc = fts5SpecialDelete(pTab, apVal); + if( pConfig->bContentlessDelete ){ + fts5SetVtabError(pTab, + "'delete' may not be used with a contentless_delete=1 table" + ); + rc = SQLITE_ERROR; + }else{ + rc = fts5SpecialDelete(pTab, apVal); + bUpdateOrDelete = 1; + } }else{ rc = fts5SpecialInsert(pTab, z, apVal[2 + pConfig->nCol + 1]); } }else{ /* A regular INSERT, UPDATE or DELETE statement. The trick here is that - ** any conflict on the rowid value must be detected before any + ** any conflict on the rowid value must be detected before any ** modifications are made to the database file. There are 4 cases: ** ** 1) DELETE @@ -191887,93 +253300,154 @@ static int fts5UpdateMethod( ** Cases 3 and 4 may violate the rowid constraint. */ int eConflict = SQLITE_ABORT; - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL || pConfig->bContentlessDelete ){ eConflict = sqlite3_vtab_on_conflict(pConfig->db); } assert( eType0==SQLITE_INTEGER || eType0==SQLITE_NULL ); assert( nArg!=1 || eType0==SQLITE_INTEGER ); - /* Filter out attempts to run UPDATE or DELETE on contentless tables. - ** This is not suported. */ - if( eType0==SQLITE_INTEGER && fts5IsContentless(pTab) ){ - pTab->base.zErrMsg = sqlite3_mprintf( - "cannot %s contentless fts5 table: %s", - (nArg>1 ? "UPDATE" : "DELETE from"), pConfig->zName - ); - rc = SQLITE_ERROR; - } - /* DELETE */ - else if( nArg==1 ){ - i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0); - } - - /* INSERT */ - else if( eType0!=SQLITE_INTEGER ){ - /* If this is a REPLACE, first remove the current entry (if any) */ - if( eConflict==SQLITE_REPLACE - && sqlite3_value_type(apVal[1])==SQLITE_INTEGER - ){ - i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); + if( nArg==1 ){ + /* It is only possible to DELETE from a contentless table if the + ** contentless_delete=1 flag is set. */ + if( fts5IsContentless(pTab, 1) && pConfig->bContentlessDelete==0 ){ + fts5SetVtabError(pTab, + "cannot DELETE from contentless fts5 table: %s", pConfig->zName + ); + rc = SQLITE_ERROR; + }else{ + i64 iDel = sqlite3_value_int64(apVal[0]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iDel, 0, 0); + bUpdateOrDelete = 1; } - fts5StorageInsert(&rc, pTab, apVal, pRowid); } - /* UPDATE */ + /* INSERT or UPDATE */ else{ - i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ - i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ - if( iOld!=iNew ){ - if( eConflict==SQLITE_REPLACE ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0); - } - fts5StorageInsert(&rc, pTab, apVal, pRowid); - }else{ - rc = sqlite3Fts5StorageContentInsert(pTab->pStorage, apVal, pRowid); - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); - } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5StorageIndexInsert(pTab->pStorage, apVal, *pRowid); + int eType1 = sqlite3_value_numeric_type(apVal[1]); + + /* It is an error to write an fts5_locale() value to a table without + ** the locale=1 option. */ + if( pConfig->bLocale==0 ){ + int ii; + for(ii=0; iinCol; ii++){ + sqlite3_value *pVal = apVal[ii+2]; + if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + fts5SetVtabError(pTab, "fts5_locale() requires locale=1"); + rc = SQLITE_MISMATCH; + goto update_out; } } - }else{ - rc = sqlite3Fts5StorageDelete(pTab->pStorage, iOld, 0); + } + + if( eType0!=SQLITE_INTEGER ){ + /* An INSERT statement. If the conflict-mode is REPLACE, first remove + ** the current entry (if any). */ + if( eConflict==SQLITE_REPLACE && eType1==SQLITE_INTEGER ){ + i64 iNew = sqlite3_value_int64(apVal[1]); /* Rowid to delete */ + rc = sqlite3Fts5StorageDelete(pTab->pStorage, iNew, 0, 0); + bUpdateOrDelete = 1; + } fts5StorageInsert(&rc, pTab, apVal, pRowid); } + + /* UPDATE */ + else{ + Fts5Storage *pStorage = pTab->pStorage; + i64 iOld = sqlite3_value_int64(apVal[0]); /* Old rowid */ + i64 iNew = sqlite3_value_int64(apVal[1]); /* New rowid */ + int bContent = 0; /* Content only update */ + + /* If this is a contentless table (including contentless_unindexed=1 + ** tables), check if the UPDATE may proceed. */ + if( fts5IsContentless(pTab, 1) ){ + rc = fts5ContentlessUpdate(pConfig, &apVal[2], iOld!=iNew, &bContent); + if( rc!=SQLITE_OK ) goto update_out; + } + + if( eType1!=SQLITE_INTEGER ){ + rc = SQLITE_MISMATCH; + }else if( iOld!=iNew ){ + assert( bContent==0 ); + if( eConflict==SQLITE_REPLACE ){ + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pStorage, iNew, 0, 0); + } + fts5StorageInsert(&rc, pTab, apVal, pRowid); + }else{ + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pStorage, 0, apVal, pRowid); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 0); + } + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageIndexInsert(pStorage, apVal, *pRowid); + } + } + }else if( bContent ){ + /* This occurs when an UPDATE on a contentless table affects *only* + ** UNINDEXED columns. This is a no-op for contentless_unindexed=0 + ** tables, or a write to the %_content table only for =1 tables. */ + assert( fts5IsContentless(pTab, 1) ); + rc = sqlite3Fts5StorageFindDeleteRow(pStorage, iOld); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5StorageContentInsert(pStorage, 1, apVal, pRowid); + } + }else{ + rc = sqlite3Fts5StorageDelete(pStorage, iOld, 0, 1); + fts5StorageInsert(&rc, pTab, apVal, pRowid); + } + bUpdateOrDelete = 1; + sqlite3Fts5StorageReleaseDeleteRow(pStorage); + } + } } - pTab->pConfig->pzErrmsg = 0; + if( rc==SQLITE_OK + && bUpdateOrDelete + && pConfig->bSecureDelete + && pConfig->iVersion==FTS5_CURRENT_VERSION + ){ + rc = sqlite3Fts5StorageConfigValue( + pTab->pStorage, "version", 0, FTS5_CURRENT_VERSION_SECUREDELETE + ); + if( rc==SQLITE_OK ){ + pConfig->iVersion = FTS5_CURRENT_VERSION_SECUREDELETE; + } + } + + update_out: + pTab->p.pConfig->pzErrmsg = 0; return rc; } /* -** Implementation of xSync() method. +** Implementation of xSync() method. */ static int fts5SyncMethod(sqlite3_vtab *pVtab){ int rc; - Fts5Table *pTab = (Fts5Table*)pVtab; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_SYNC, 0); - pTab->pConfig->pzErrmsg = &pTab->base.zErrMsg; - fts5TripCursors(pTab); - rc = sqlite3Fts5StorageSync(pTab->pStorage, 1); - pTab->pConfig->pzErrmsg = 0; + pTab->p.pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = sqlite3Fts5FlushToDisk(&pTab->p); + pTab->p.pConfig->pzErrmsg = 0; return rc; } /* -** Implementation of xBegin() method. +** Implementation of xBegin() method. */ static int fts5BeginMethod(sqlite3_vtab *pVtab){ - fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_BEGIN, 0); - fts5NewTransaction((Fts5Table*)pVtab); - return SQLITE_OK; + int rc = fts5NewTransaction((Fts5FullTable*)pVtab); + if( rc==SQLITE_OK ){ + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_BEGIN, 0); + } + return rc; } /* @@ -191983,7 +253457,7 @@ static int fts5BeginMethod(sqlite3_vtab *pVtab){ */ static int fts5CommitMethod(sqlite3_vtab *pVtab){ UNUSED_PARAM(pVtab); /* Call below is a no-op for NDEBUG builds */ - fts5CheckTransactionState((Fts5Table*)pVtab, FTS5_COMMIT, 0); + fts5CheckTransactionState((Fts5FullTable*)pVtab, FTS5_COMMIT, 0); return SQLITE_OK; } @@ -191993,7 +253467,7 @@ static int fts5CommitMethod(sqlite3_vtab *pVtab){ */ static int fts5RollbackMethod(sqlite3_vtab *pVtab){ int rc; - Fts5Table *pTab = (Fts5Table*)pVtab; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; fts5CheckTransactionState(pTab, FTS5_ROLLBACK, 0); rc = sqlite3Fts5StorageRollback(pTab->pStorage); return rc; @@ -192012,32 +253486,55 @@ static int fts5ApiColumnCount(Fts5Context *pCtx){ } static int fts5ApiColumnTotalSize( - Fts5Context *pCtx, - int iCol, + Fts5Context *pCtx, + int iCol, sqlite3_int64 *pnToken ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); return sqlite3Fts5StorageSize(pTab->pStorage, iCol, pnToken); } static int fts5ApiRowCount(Fts5Context *pCtx, i64 *pnRow){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); return sqlite3Fts5StorageRowCount(pTab->pStorage, pnRow); } -static int fts5ApiTokenize( - Fts5Context *pCtx, - const char *pText, int nText, +/* +** Implementation of xTokenize_v2() API. +*/ +static int fts5ApiTokenize_v2( + Fts5Context *pCtx, + const char *pText, int nText, + const char *pLoc, int nLoc, void *pUserData, int (*xToken)(void*, int, const char*, int, int, int) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - return sqlite3Fts5Tokenize( - pTab->pConfig, FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken + int rc = SQLITE_OK; + + sqlite3Fts5SetLocale(pTab->pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pTab->pConfig, + FTS5_TOKENIZE_AUX, pText, nText, pUserData, xToken ); + sqlite3Fts5SetLocale(pTab->pConfig, 0, 0); + + return rc; +} + +/* +** Implementation of xTokenize() API. This is just xTokenize_v2() with NULL/0 +** passed as the locale. +*/ +static int fts5ApiTokenize( + Fts5Context *pCtx, + const char *pText, int nText, + void *pUserData, + int (*xToken)(void*, int, const char*, int, int, int) +){ + return fts5ApiTokenize_v2(pCtx, pText, nText, 0, 0, pUserData, xToken); } static int fts5ApiPhraseCount(Fts5Context *pCtx){ @@ -192050,52 +253547,120 @@ static int fts5ApiPhraseSize(Fts5Context *pCtx, int iPhrase){ return sqlite3Fts5ExprPhraseSize(pCsr->pExpr, iPhrase); } +/* +** Argument pStmt is an SQL statement of the type used by Fts5Cursor. This +** function extracts the text value of column iCol of the current row. +** Additionally, if there is an associated locale, it invokes +** sqlite3Fts5SetLocale() to configure the tokenizer. In all cases the caller +** should invoke sqlite3Fts5ClearLocale() to clear the locale at some point +** after this function returns. +** +** If successful, (*ppText) is set to point to a buffer containing the text +** value as utf-8 and SQLITE_OK returned. (*pnText) is set to the size of that +** buffer in bytes. It is not guaranteed to be nul-terminated. If an error +** occurs, an SQLite error code is returned. The final values of the two +** output parameters are undefined in this case. +*/ +static int fts5TextFromStmt( + Fts5Config *pConfig, + sqlite3_stmt *pStmt, + int iCol, + const char **ppText, + int *pnText +){ + sqlite3_value *pVal = sqlite3_column_value(pStmt, iCol+1); + const char *pLoc = 0; + int nLoc = 0; + int rc = SQLITE_OK; + + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, ppText, pnText, &pLoc, &nLoc); + }else{ + *ppText = (const char*)sqlite3_value_text(pVal); + *pnText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + pLoc = (const char*)sqlite3_column_text(pStmt, iCol+1+pConfig->nCol); + nLoc = sqlite3_column_bytes(pStmt, iCol+1+pConfig->nCol); + } + } + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + return rc; +} + static int fts5ApiColumnText( - Fts5Context *pCtx, - int iCol, - const char **pz, + Fts5Context *pCtx, + int iCol, + const char **pz, int *pn ){ int rc = SQLITE_OK; Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - if( fts5IsContentless((Fts5Table*)(pCsr->base.pVtab)) ){ + Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pTab->pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( fts5IsContentless((Fts5FullTable*)(pCsr->base.pVtab), 0) ){ *pz = 0; *pn = 0; }else{ rc = fts5SeekCursor(pCsr, 0); if( rc==SQLITE_OK ){ - *pz = (const char*)sqlite3_column_text(pCsr->pStmt, iCol+1); - *pn = sqlite3_column_bytes(pCsr->pStmt, iCol+1); + rc = fts5TextFromStmt(pTab->pConfig, pCsr->pStmt, iCol, pz, pn); + sqlite3Fts5ClearLocale(pTab->pConfig); } } return rc; } +/* +** This is called by various API functions - xInst, xPhraseFirst, +** xPhraseFirstColumn etc. - to obtain the position list for phrase iPhrase +** of the current row. This function works for both detail=full tables (in +** which case the position-list was read from the fts index) or for other +** detail= modes if the row content is available. +*/ static int fts5CsrPoslist( - Fts5Cursor *pCsr, - int iPhrase, - const u8 **pa, - int *pn + Fts5Cursor *pCsr, /* Fts5 cursor object */ + int iPhrase, /* Phrase to find position list for */ + const u8 **pa, /* OUT: Pointer to position list buffer */ + int *pn /* OUT: Size of (*pa) in bytes */ ){ Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; int rc = SQLITE_OK; int bLive = (pCsr->pSorter==0); - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ - + if( iPhrase<0 || iPhrase>=sqlite3Fts5ExprPhraseCount(pCsr->pExpr) ){ + rc = SQLITE_RANGE; + }else if( pConfig->eDetail!=FTS5_DETAIL_FULL + && fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + ){ + *pa = 0; + *pn = 0; + return SQLITE_OK; + }else if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_POSLIST) ){ if( pConfig->eDetail!=FTS5_DETAIL_FULL ){ Fts5PoslistPopulator *aPopulator; int i; + aPopulator = sqlite3Fts5ExprClearPoslists(pCsr->pExpr, bLive); if( aPopulator==0 ) rc = SQLITE_NOMEM; + if( rc==SQLITE_OK ){ + rc = fts5SeekCursor(pCsr, 0); + } for(i=0; inCol && rc==SQLITE_OK; i++){ - int n; const char *z; - rc = fts5ApiColumnText((Fts5Context*)pCsr, i, &z, &n); + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ rc = sqlite3Fts5ExprPopulatePoslists( pConfig, pCsr->pExpr, aPopulator, i, z, n ); } + sqlite3Fts5ClearLocale(pConfig); } sqlite3_free(aPopulator); @@ -192106,13 +253671,18 @@ static int fts5CsrPoslist( CsrFlagClear(pCsr, FTS5CSR_REQUIRE_POSLIST); } - if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ - Fts5Sorter *pSorter = pCsr->pSorter; - int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); - *pn = pSorter->aIdx[iPhrase] - i1; - *pa = &pSorter->aPoslist[i1]; + if( rc==SQLITE_OK ){ + if( pCsr->pSorter && pConfig->eDetail==FTS5_DETAIL_FULL ){ + Fts5Sorter *pSorter = pCsr->pSorter; + int i1 = (iPhrase==0 ? 0 : pSorter->aIdx[iPhrase-1]); + *pn = pSorter->aIdx[iPhrase] - i1; + *pa = &pSorter->aPoslist[i1]; + }else{ + *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + } }else{ - *pn = sqlite3Fts5ExprPoslist(pCsr->pExpr, iPhrase, pa); + *pa = 0; + *pn = 0; } return rc; @@ -192127,10 +253697,11 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ int rc = SQLITE_OK; Fts5PoslistReader *aIter; /* One iterator for each phrase */ int nIter; /* Number of iterators/phrases */ - + int nCol = ((Fts5Table*)pCsr->base.pVtab)->pConfig->nCol; + nIter = sqlite3Fts5ExprPhraseCount(pCsr->pExpr); if( pCsr->aInstIter==0 ){ - int nByte = sizeof(Fts5PoslistReader) * nIter; + sqlite3_int64 nByte = sizeof(Fts5PoslistReader) * nIter; pCsr->aInstIter = (Fts5PoslistReader*)sqlite3Fts5MallocZero(&rc, nByte); } aIter = pCsr->aInstIter; @@ -192142,7 +253713,7 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ /* Initialize all iterators */ for(i=0; i=pCsr->nInstAlloc ){ - pCsr->nInstAlloc = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; - aInst = (int*)sqlite3_realloc( - pCsr->aInst, pCsr->nInstAlloc*sizeof(int)*3 + int nNewSize = pCsr->nInstAlloc ? pCsr->nInstAlloc*2 : 32; + aInst = (int*)sqlite3_realloc64( + pCsr->aInst, nNewSize*sizeof(int)*3 ); if( aInst ){ pCsr->aInst = aInst; + pCsr->nInstAlloc = nNewSize; }else{ + nInst--; rc = SQLITE_NOMEM; break; } @@ -192180,6 +253753,11 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ aInst[0] = iBest; aInst[1] = FTS5_POS2COLUMN(aIter[iBest].iPos); aInst[2] = FTS5_POS2OFFSET(aIter[iBest].iPos); + assert( aInst[1]>=0 ); + if( aInst[1]>=nCol ){ + rc = FTS5_CORRUPT; + break; + } sqlite3Fts5PoslistReaderNext(&aIter[iBest]); } } @@ -192193,7 +253771,7 @@ static int fts5CacheInstArray(Fts5Cursor *pCsr){ static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int rc = SQLITE_OK; - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ *pnInst = pCsr->nInstCount; } @@ -192201,25 +253779,19 @@ static int fts5ApiInstCount(Fts5Context *pCtx, int *pnInst){ } static int fts5ApiInst( - Fts5Context *pCtx, - int iIdx, - int *piPhrase, - int *piCol, + Fts5Context *pCtx, + int iIdx, + int *piPhrase, + int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int rc = SQLITE_OK; - if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 - || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) ){ if( iIdx<0 || iIdx>=pCsr->nInstCount ){ rc = SQLITE_RANGE; -#if 0 - }else if( fts5IsOffsetless((Fts5Table*)pCsr->base.pVtab) ){ - *piPhrase = pCsr->aInst[iIdx*3]; - *piCol = pCsr->aInst[iIdx*3 + 2]; - *piOff = -1; -#endif }else{ *piPhrase = pCsr->aInst[iIdx*3]; *piCol = pCsr->aInst[iIdx*3 + 1]; @@ -192252,15 +253824,15 @@ static int fts5ColumnSizeCb( static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); - Fts5Config *pConfig = pTab->pConfig; + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); + Fts5Config *pConfig = pTab->p.pConfig; int rc = SQLITE_OK; if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_DOCSIZE) ){ if( pConfig->bColumnsize ){ i64 iRowid = fts5CursorRowid(pCsr); rc = sqlite3Fts5StorageDocsize(pTab->pStorage, iRowid, pCsr->aColumnSize); - }else if( pConfig->zContent==0 ){ + }else if( !pConfig->zContent || pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ int i; for(i=0; inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ @@ -192269,17 +253841,19 @@ static int fts5ApiColumnSize(Fts5Context *pCtx, int iCol, int *pnToken){ } }else{ int i; + rc = fts5SeekCursor(pCsr, 0); for(i=0; rc==SQLITE_OK && inCol; i++){ if( pConfig->abUnindexed[i]==0 ){ - const char *z; int n; - void *p = (void*)(&pCsr->aColumnSize[i]); + const char *z = 0; + int n = 0; pCsr->aColumnSize[i] = 0; - rc = fts5ApiColumnText(pCtx, i, &z, &n); + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, i, &z, &n); if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize( - pConfig, FTS5_TOKENIZE_AUX, z, n, p, fts5ColumnSizeCb + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_AUX, + z, n, (void*)&pCsr->aColumnSize[i], fts5ColumnSizeCb ); } + sqlite3Fts5ClearLocale(pConfig); } } } @@ -192359,11 +253933,10 @@ static void *fts5ApiGetAuxdata(Fts5Context *pCtx, int bClear){ } static void fts5ApiPhraseNext( - Fts5Context *pUnused, - Fts5PhraseIter *pIter, + Fts5Context *pCtx, + Fts5PhraseIter *pIter, int *piCol, int *piOff ){ - UNUSED_PARAM(pUnused); if( pIter->a>=pIter->b ){ *piCol = -1; *piOff = -1; @@ -192371,8 +253944,12 @@ static void fts5ApiPhraseNext( int iVal; pIter->a += fts5GetVarint32(pIter->a, iVal); if( iVal==1 ){ + /* Avoid returning a (*piCol) value that is too large for the table, + ** even if the position-list is corrupt. The caller might not be + ** expecting it. */ + int nCol = ((Fts5Table*)(((Fts5Cursor*)pCtx)->base.pVtab))->pConfig->nCol; pIter->a += fts5GetVarint32(pIter->a, iVal); - *piCol = iVal; + *piCol = (iVal>=nCol ? nCol-1 : iVal); *piOff = 0; pIter->a += fts5GetVarint32(pIter->a, iVal); } @@ -192381,16 +253958,17 @@ static void fts5ApiPhraseNext( } static int fts5ApiPhraseFirst( - Fts5Context *pCtx, - int iPhrase, - Fts5PhraseIter *pIter, + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, int *piCol, int *piOff ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; int n; int rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ - pIter->b = &pIter->a[n]; + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); *piCol = 0; *piOff = 0; fts5ApiPhraseNext(pCtx, pIter, piCol, piOff); @@ -192399,8 +253977,8 @@ static int fts5ApiPhraseFirst( } static void fts5ApiPhraseNextColumn( - Fts5Context *pCtx, - Fts5PhraseIter *pIter, + Fts5Context *pCtx, + Fts5PhraseIter *pIter, int *piCol ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; @@ -192429,9 +254007,9 @@ static void fts5ApiPhraseNextColumn( } static int fts5ApiPhraseFirstColumn( - Fts5Context *pCtx, - int iPhrase, - Fts5PhraseIter *pIter, + Fts5Context *pCtx, + int iPhrase, + Fts5PhraseIter *pIter, int *piCol ){ int rc = SQLITE_OK; @@ -192449,7 +254027,8 @@ static int fts5ApiPhraseFirstColumn( rc = sqlite3Fts5ExprPhraseCollist(pCsr->pExpr, iPhrase, &pIter->a, &n); } if( rc==SQLITE_OK ){ - pIter->b = &pIter->a[n]; + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); *piCol = 0; fts5ApiPhraseNextColumn(pCtx, pIter, piCol); } @@ -192457,7 +254036,8 @@ static int fts5ApiPhraseFirstColumn( int n; rc = fts5CsrPoslist(pCsr, iPhrase, &pIter->a, &n); if( rc==SQLITE_OK ){ - pIter->b = &pIter->a[n]; + assert( pIter->a || n==0 ); + pIter->b = (pIter->a ? &pIter->a[n] : 0); if( n<=0 ){ *piCol = -1; }else if( pIter->a[0]==0x01 ){ @@ -192471,13 +254051,96 @@ static int fts5ApiPhraseFirstColumn( return rc; } +/* +** xQueryToken() API implemenetation. +*/ +static int fts5ApiQueryToken( + Fts5Context* pCtx, + int iPhrase, + int iToken, + const char **ppOut, + int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + return sqlite3Fts5ExprQueryToken(pCsr->pExpr, iPhrase, iToken, ppOut, pnOut); +} -static int fts5ApiQueryPhrase(Fts5Context*, int, void*, +/* +** xInstToken() API implemenetation. +*/ +static int fts5ApiInstToken( + Fts5Context *pCtx, + int iIdx, + int iToken, + const char **ppOut, int *pnOut +){ + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + int rc = SQLITE_OK; + if( CsrFlagTest(pCsr, FTS5CSR_REQUIRE_INST)==0 + || SQLITE_OK==(rc = fts5CacheInstArray(pCsr)) + ){ + if( iIdx<0 || iIdx>=pCsr->nInstCount ){ + rc = SQLITE_RANGE; + }else{ + int iPhrase = pCsr->aInst[iIdx*3]; + int iCol = pCsr->aInst[iIdx*3 + 1]; + int iOff = pCsr->aInst[iIdx*3 + 2]; + i64 iRowid = fts5CursorRowid(pCsr); + rc = sqlite3Fts5ExprInstToken( + pCsr->pExpr, iRowid, iPhrase, iCol, iOff, iToken, ppOut, pnOut + ); + } + } + return rc; +} + + +static int fts5ApiQueryPhrase(Fts5Context*, int, void*, int(*)(const Fts5ExtensionApi*, Fts5Context*, void*) ); +/* +** The xColumnLocale() API. +*/ +static int fts5ApiColumnLocale( + Fts5Context *pCtx, + int iCol, + const char **pzLocale, + int *pnLocale +){ + int rc = SQLITE_OK; + Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; + Fts5Config *pConfig = ((Fts5Table*)(pCsr->base.pVtab))->pConfig; + + *pzLocale = 0; + *pnLocale = 0; + + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); + if( iCol<0 || iCol>=pConfig->nCol ){ + rc = SQLITE_RANGE; + }else if( + pConfig->abUnindexed[iCol]==0 + && 0==fts5IsContentless((Fts5FullTable*)pCsr->base.pVtab, 1) + && pConfig->bLocale + ){ + rc = fts5SeekCursor(pCsr, 0); + if( rc==SQLITE_OK ){ + const char *zDummy = 0; + int nDummy = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &zDummy, &nDummy); + if( rc==SQLITE_OK ){ + *pzLocale = pConfig->t.pLocale; + *pnLocale = pConfig->t.nLocale; + } + sqlite3Fts5ClearLocale(pConfig); + } + } + + return rc; +} + static const Fts5ExtensionApi sFts5Api = { - 2, /* iVersion */ + 4, /* iVersion */ fts5ApiUserData, fts5ApiColumnCount, fts5ApiRowCount, @@ -192497,19 +254160,23 @@ static const Fts5ExtensionApi sFts5Api = { fts5ApiPhraseNext, fts5ApiPhraseFirstColumn, fts5ApiPhraseNextColumn, + fts5ApiQueryToken, + fts5ApiInstToken, + fts5ApiColumnLocale, + fts5ApiTokenize_v2 }; /* ** Implementation of API function xQueryPhrase(). */ static int fts5ApiQueryPhrase( - Fts5Context *pCtx, - int iPhrase, + Fts5Context *pCtx, + int iPhrase, void *pUserData, int(*xCallback)(const Fts5ExtensionApi*, Fts5Context*, void*) ){ Fts5Cursor *pCsr = (Fts5Cursor*)pCtx; - Fts5Table *pTab = (Fts5Table*)(pCsr->base.pVtab); + Fts5FullTable *pTab = (Fts5FullTable*)(pCsr->base.pVtab); int rc; Fts5Cursor *pNew = 0; @@ -192547,6 +254214,7 @@ static void fts5ApiInvoke( sqlite3_value **argv ){ assert( pCsr->pAux==0 ); + assert( pCsr->ePlan!=FTS5_PLAN_SPECIAL ); pCsr->pAux = pAux; pAux->xFunc(&sFts5Api, (Fts5Context*)pCsr, context, argc, argv); pCsr->pAux = 0; @@ -192560,6 +254228,21 @@ static Fts5Cursor *fts5CursorFromCsrid(Fts5Global *pGlobal, i64 iCsrId){ return pCsr; } +/* +** Parameter zFmt is a printf() style formatting string. This function +** formats it using the trailing arguments and returns the result as +** an error message to the context passed as the first argument. +*/ +static void fts5ResultError(sqlite3_context *pCtx, const char *zFmt, ...){ + char *zErr = 0; + va_list ap; + va_start(ap, zFmt); + zErr = sqlite3_vmprintf(zFmt, ap); + sqlite3_result_error(pCtx, zErr, -1); + sqlite3_free(zErr); + va_end(ap); +} + static void fts5ApiCallback( sqlite3_context *context, int argc, @@ -192575,36 +254258,31 @@ static void fts5ApiCallback( iCsrId = sqlite3_value_int64(argv[0]); pCsr = fts5CursorFromCsrid(pAux->pGlobal, iCsrId); - if( pCsr==0 ){ - char *zErr = sqlite3_mprintf("no such cursor: %lld", iCsrId); - sqlite3_result_error(context, zErr, -1); - sqlite3_free(zErr); + if( pCsr==0 || (pCsr->ePlan==0 || pCsr->ePlan==FTS5_PLAN_SPECIAL) ){ + fts5ResultError(context, "no such cursor: %lld", iCsrId); }else{ + sqlite3_vtab *pTab = pCsr->base.pVtab; fts5ApiInvoke(pAux, pCsr, context, argc-1, &argv[1]); + sqlite3_free(pTab->zErrMsg); + pTab->zErrMsg = 0; } } /* -** Given cursor id iId, return a pointer to the corresponding Fts5Index +** Given cursor id iId, return a pointer to the corresponding Fts5Table ** object. Or NULL If the cursor id does not exist. -** -** If successful, set *ppConfig to point to the associated config object -** before returning. */ -static Fts5Index *sqlite3Fts5IndexFromCsrid( +static Fts5Table *sqlite3Fts5TableFromCsrid( Fts5Global *pGlobal, /* FTS5 global context for db handle */ - i64 iCsrId, /* Id of cursor to find */ - Fts5Config **ppConfig /* OUT: Configuration object */ + i64 iCsrId /* Id of cursor to find */ ){ Fts5Cursor *pCsr; - Fts5Table *pTab; - pCsr = fts5CursorFromCsrid(pGlobal, iCsrId); - pTab = (Fts5Table*)pCsr->base.pVtab; - *ppConfig = pTab->pConfig; - - return pTab->pIndex; + if( pCsr ){ + return (Fts5Table*)pCsr->base.pVtab; + } + return 0; } /* @@ -192675,7 +254353,7 @@ static int fts5PoslistBlob(sqlite3_context *pCtx, Fts5Cursor *pCsr){ return rc; } -/* +/* ** This is the xColumn method, called by SQLite to request a value from ** the row that the supplied cursor currently points to. */ @@ -192684,11 +254362,11 @@ static int fts5ColumnMethod( sqlite3_context *pCtx, /* Context for sqlite3_result_xxx() calls */ int iCol /* Index of column to read value from */ ){ - Fts5Table *pTab = (Fts5Table*)(pCursor->pVtab); - Fts5Config *pConfig = pTab->pConfig; + Fts5FullTable *pTab = (Fts5FullTable*)(pCursor->pVtab); + Fts5Config *pConfig = pTab->p.pConfig; Fts5Cursor *pCsr = (Fts5Cursor*)pCursor; int rc = SQLITE_OK; - + assert( CsrFlagTest(pCsr, FTS5CSR_EOF)==0 ); if( pCsr->ePlan==FTS5_PLAN_SPECIAL ){ @@ -192704,11 +254382,11 @@ static int fts5ColumnMethod( ** auxiliary function. */ sqlite3_result_int64(pCtx, pCsr->iCsrId); }else if( iCol==pConfig->nCol+1 ){ - /* The value of the "rank" column. */ + if( pCsr->ePlan==FTS5_PLAN_SOURCE ){ fts5PoslistBlob(pCtx, pCsr); - }else if( + }else if( pCsr->ePlan==FTS5_PLAN_MATCH || pCsr->ePlan==FTS5_PLAN_SORTED_MATCH ){ @@ -192716,12 +254394,32 @@ static int fts5ColumnMethod( fts5ApiInvoke(pCsr->pRank, pCsr, pCtx, pCsr->nRankArg, pCsr->apRankArg); } } - }else if( !fts5IsContentless(pTab) ){ - rc = fts5SeekCursor(pCsr, 1); - if( rc==SQLITE_OK ){ - sqlite3_result_value(pCtx, sqlite3_column_value(pCsr->pStmt, iCol+1)); + }else{ + if( !sqlite3_vtab_nochange(pCtx) && pConfig->eContent!=FTS5_CONTENT_NONE ){ + pConfig->pzErrmsg = &pTab->p.base.zErrMsg; + rc = fts5SeekCursor(pCsr, 1); + if( rc==SQLITE_OK ){ + sqlite3_value *pVal = sqlite3_column_value(pCsr->pStmt, iCol+1); + if( pConfig->bLocale + && pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + const char *z = 0; + int n = 0; + rc = fts5TextFromStmt(pConfig, pCsr->pStmt, iCol, &z, &n); + if( rc==SQLITE_OK ){ + sqlite3_result_text(pCtx, z, n, SQLITE_TRANSIENT); + } + sqlite3Fts5ClearLocale(pConfig); + }else{ + sqlite3_result_value(pCtx, pVal); + } + } + + pConfig->pzErrmsg = 0; } } + return rc; } @@ -192737,7 +254435,7 @@ static int fts5FindFunctionMethod( void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), /* OUT: Result */ void **ppArg /* OUT: User data for *pxFunc */ ){ - Fts5Table *pTab = (Fts5Table*)pVtab; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; Fts5Auxiliary *pAux; UNUSED_PARAM(nUnused); @@ -192759,8 +254457,15 @@ static int fts5RenameMethod( sqlite3_vtab *pVtab, /* Virtual table handle */ const char *zName /* New name of table */ ){ - Fts5Table *pTab = (Fts5Table*)pVtab; - return sqlite3Fts5StorageRename(pTab->pStorage, zName); + int rc; + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + rc = sqlite3Fts5StorageRename(pTab->pStorage, zName); + return rc; +} + +static int sqlite3Fts5FlushToDisk(Fts5Table *pTab){ + fts5TripCursors((Fts5FullTable*)pTab); + return sqlite3Fts5StorageSync(((Fts5FullTable*)pTab)->pStorage); } /* @@ -192769,11 +254474,15 @@ static int fts5RenameMethod( ** Flush the contents of the pending-terms table to disk. */ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts5Table *pTab = (Fts5Table*)pVtab; - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; + fts5CheckTransactionState(pTab, FTS5_SAVEPOINT, iSavepoint); - fts5TripCursors(pTab); - return sqlite3Fts5StorageSync(pTab->pStorage, 0); + rc = sqlite3Fts5FlushToDisk((Fts5Table*)pVtab); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint+1; + } + return rc; } /* @@ -192782,11 +254491,16 @@ static int fts5SavepointMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** This is a no-op. */ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts5Table *pTab = (Fts5Table*)pVtab; - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; fts5CheckTransactionState(pTab, FTS5_RELEASE, iSavepoint); - fts5TripCursors(pTab); - return sqlite3Fts5StorageSync(pTab->pStorage, 0); + if( (iSavepoint+1)iSavepoint ){ + rc = sqlite3Fts5FlushToDisk(&pTab->p); + if( rc==SQLITE_OK ){ + pTab->iSavepoint = iSavepoint; + } + } + return rc; } /* @@ -192795,11 +254509,15 @@ static int fts5ReleaseMethod(sqlite3_vtab *pVtab, int iSavepoint){ ** Discard the contents of the pending terms table. */ static int fts5RollbackToMethod(sqlite3_vtab *pVtab, int iSavepoint){ - Fts5Table *pTab = (Fts5Table*)pVtab; - UNUSED_PARAM(iSavepoint); /* Call below is a no-op for NDEBUG builds */ + Fts5FullTable *pTab = (Fts5FullTable*)pVtab; + int rc = SQLITE_OK; fts5CheckTransactionState(pTab, FTS5_ROLLBACKTO, iSavepoint); fts5TripCursors(pTab); - return sqlite3Fts5StorageRollback(pTab->pStorage); + if( (iSavepoint+1)<=pTab->iSavepoint ){ + pTab->p.pConfig->pgsz = 0; + rc = sqlite3Fts5StorageRollback(pTab->pStorage); + } + return rc; } /* @@ -192816,14 +254534,14 @@ static int fts5CreateAux( int rc = sqlite3_overload_function(pGlobal->db, zName, -1); if( rc==SQLITE_OK ){ Fts5Auxiliary *pAux; - int nName; /* Size of zName in bytes, including \0 */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nName; /* Size of zName in bytes, including \0 */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ - nName = (int)strlen(zName) + 1; + nName = strlen(zName) + 1; nByte = sizeof(Fts5Auxiliary) + nName; - pAux = (Fts5Auxiliary*)sqlite3_malloc(nByte); + pAux = (Fts5Auxiliary*)sqlite3_malloc64(nByte); if( pAux ){ - memset(pAux, 0, nByte); + memset(pAux, 0, (size_t)nByte); pAux->zFunc = (char*)&pAux[1]; memcpy(pAux->zFunc, zName, nName); pAux->pGlobal = pGlobal; @@ -192841,8 +254559,179 @@ static int fts5CreateAux( } /* -** Register a new tokenizer. This is the implementation of the -** fts5_api.xCreateTokenizer() method. +** This function is used by xCreateTokenizer_v2() and xCreateTokenizer(). +** It allocates and partially populates a new Fts5TokenizerModule object. +** The new object is already linked into the Fts5Global context before +** returning. +** +** If successful, SQLITE_OK is returned and a pointer to the new +** Fts5TokenizerModule object returned via output parameter (*ppNew). All +** that is required is for the caller to fill in the methods in +** Fts5TokenizerModule.x1 and x2, and to set Fts5TokenizerModule.bV2Native +** as appropriate. +** +** If an error occurs, an SQLite error code is returned and the final value +** of (*ppNew) undefined. +*/ +static int fts5NewTokenizerModule( + Fts5Global *pGlobal, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + void(*xDestroy)(void*), /* Destructor for pUserData */ + Fts5TokenizerModule **ppNew +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pNew; + sqlite3_int64 nName; /* Size of zName and its \0 terminator */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ + + nName = strlen(zName) + 1; + nByte = sizeof(Fts5TokenizerModule) + nName; + *ppNew = pNew = (Fts5TokenizerModule*)sqlite3Fts5MallocZero(&rc, nByte); + if( pNew ){ + pNew->zName = (char*)&pNew[1]; + memcpy(pNew->zName, zName, nName); + pNew->pUserData = pUserData; + pNew->xDestroy = xDestroy; + pNew->pNext = pGlobal->pTok; + pGlobal->pTok = pNew; + if( pNew->pNext==0 ){ + pGlobal->pDfltTok = pNew; + } + } + + return rc; +} + +/* +** An instance of this type is used as the Fts5Tokenizer object for +** wrapper tokenizers - those that provide access to a v1 tokenizer via +** the fts5_tokenizer_v2 API, and those that provide access to a v2 tokenizer +** via the fts5_tokenizer API. +*/ +typedef struct Fts5VtoVTokenizer Fts5VtoVTokenizer; +struct Fts5VtoVTokenizer { + int bV2Native; /* True if v2 native tokenizer */ + fts5_tokenizer x1; /* Tokenizer functions */ + fts5_tokenizer_v2 x2; /* V2 tokenizer functions */ + Fts5Tokenizer *pReal; +}; + +/* +** Create a wrapper tokenizer. The context argument pCtx points to the +** Fts5TokenizerModule object. +*/ +static int fts5VtoVCreate( + void *pCtx, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + Fts5TokenizerModule *pMod = (Fts5TokenizerModule*)pCtx; + Fts5VtoVTokenizer *pNew = 0; + int rc = SQLITE_OK; + + pNew = (Fts5VtoVTokenizer*)sqlite3Fts5MallocZero(&rc, sizeof(*pNew)); + if( rc==SQLITE_OK ){ + pNew->x1 = pMod->x1; + pNew->x2 = pMod->x2; + pNew->bV2Native = pMod->bV2Native; + if( pMod->bV2Native ){ + rc = pMod->x2.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + }else{ + rc = pMod->x1.xCreate(pMod->pUserData, azArg, nArg, &pNew->pReal); + } + if( rc!=SQLITE_OK ){ + sqlite3_free(pNew); + pNew = 0; + } + } + + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Delete an Fts5VtoVTokenizer wrapper tokenizer. +*/ +static void fts5VtoVDelete(Fts5Tokenizer *pTok){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + if( p ){ + if( p->bV2Native ){ + p->x2.xDelete(p->pReal); + }else{ + p->x1.xDelete(p->pReal); + } + sqlite3_free(p); + } +} + + +/* +** xTokenizer method for a wrapper tokenizer that offers the v1 interface +** (no support for locales). +*/ +static int fts5V1toV2Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native ); + return p->x2.xTokenize(p->pReal, pCtx, flags, pText, nText, 0, 0, xToken); +} + +/* +** xTokenizer method for a wrapper tokenizer that offers the v2 interface +** (with locale support). +*/ +static int fts5V2toV1Tokenize( + Fts5Tokenizer *pTok, + void *pCtx, int flags, + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)(void*, int, const char*, int, int, int) +){ + Fts5VtoVTokenizer *p = (Fts5VtoVTokenizer*)pTok; + assert( p->bV2Native==0 ); + UNUSED_PARAM2(pLocale,nLocale); + return p->x1.xTokenize(p->pReal, pCtx, flags, pText, nText, xToken); +} + +/* +** Register a new tokenizer. This is the implementation of the +** fts5_api.xCreateTokenizer_v2() method. +*/ +static int fts5CreateTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of new function */ + void *pUserData, /* User data for aux. function */ + fts5_tokenizer_v2 *pTokenizer, /* Tokenizer implementation */ + void(*xDestroy)(void*) /* Destructor for pUserData */ +){ + Fts5Global *pGlobal = (Fts5Global*)pApi; + int rc = SQLITE_OK; + + if( pTokenizer->iVersion>2 ){ + rc = SQLITE_ERROR; + }else{ + Fts5TokenizerModule *pNew = 0; + rc = fts5NewTokenizerModule(pGlobal, zName, pUserData, xDestroy, &pNew); + if( pNew ){ + pNew->x2 = *pTokenizer; + pNew->bV2Native = 1; + pNew->x1.xCreate = fts5VtoVCreate; + pNew->x1.xTokenize = fts5V1toV2Tokenize; + pNew->x1.xDelete = fts5VtoVDelete; + } + } + + return rc; +} + +/* +** The fts5_api.xCreateTokenizer() method. */ static int fts5CreateTokenizer( fts5_api *pApi, /* Global context (one per db handle) */ @@ -192851,37 +254740,29 @@ static int fts5CreateTokenizer( fts5_tokenizer *pTokenizer, /* Tokenizer implementation */ void(*xDestroy)(void*) /* Destructor for pUserData */ ){ - Fts5Global *pGlobal = (Fts5Global*)pApi; - Fts5TokenizerModule *pNew; - int nName; /* Size of zName and its \0 terminator */ - int nByte; /* Bytes of space to allocate */ + Fts5TokenizerModule *pNew = 0; int rc = SQLITE_OK; - nName = (int)strlen(zName) + 1; - nByte = sizeof(Fts5TokenizerModule) + nName; - pNew = (Fts5TokenizerModule*)sqlite3_malloc(nByte); + rc = fts5NewTokenizerModule( + (Fts5Global*)pApi, zName, pUserData, xDestroy, &pNew + ); if( pNew ){ - memset(pNew, 0, nByte); - pNew->zName = (char*)&pNew[1]; - memcpy(pNew->zName, zName, nName); - pNew->pUserData = pUserData; - pNew->x = *pTokenizer; - pNew->xDestroy = xDestroy; - pNew->pNext = pGlobal->pTok; - pGlobal->pTok = pNew; - if( pNew->pNext==0 ){ - pGlobal->pDfltTok = pNew; - } - }else{ - rc = SQLITE_NOMEM; + pNew->x1 = *pTokenizer; + pNew->x2.xCreate = fts5VtoVCreate; + pNew->x2.xTokenize = fts5V2toV1Tokenize; + pNew->x2.xDelete = fts5VtoVDelete; } - return rc; } +/* +** Search the global context passed as the first argument for a tokenizer +** module named zName. If found, return a pointer to the Fts5TokenizerModule +** object. Otherwise, return NULL. +*/ static Fts5TokenizerModule *fts5LocateTokenizer( - Fts5Global *pGlobal, - const char *zName + Fts5Global *pGlobal, /* Global (one per db handle) object */ + const char *zName /* Name of tokenizer module to find */ ){ Fts5TokenizerModule *pMod = 0; @@ -192897,7 +254778,37 @@ static Fts5TokenizerModule *fts5LocateTokenizer( } /* -** Find a tokenizer. This is the implementation of the +** Find a tokenizer. This is the implementation of the +** fts5_api.xFindTokenizer_v2() method. +*/ +static int fts5FindTokenizer_v2( + fts5_api *pApi, /* Global context (one per db handle) */ + const char *zName, /* Name of tokenizer */ + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer /* Populate this object */ +){ + int rc = SQLITE_OK; + Fts5TokenizerModule *pMod; + + pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); + if( pMod ){ + if( pMod->bV2Native ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *ppTokenizer = &pMod->x2; + }else{ + *ppTokenizer = 0; + *ppUserData = 0; + rc = SQLITE_ERROR; + } + + return rc; +} + +/* +** Find a tokenizer. This is the implementation of the ** fts5_api.xFindTokenizer() method. */ static int fts5FindTokenizer( @@ -192911,48 +254822,75 @@ static int fts5FindTokenizer( pMod = fts5LocateTokenizer((Fts5Global*)pApi, zName); if( pMod ){ - *pTokenizer = pMod->x; - *ppUserData = pMod->pUserData; + if( pMod->bV2Native==0 ){ + *ppUserData = pMod->pUserData; + }else{ + *ppUserData = (void*)pMod; + } + *pTokenizer = pMod->x1; }else{ - memset(pTokenizer, 0, sizeof(fts5_tokenizer)); + memset(pTokenizer, 0, sizeof(*pTokenizer)); + *ppUserData = 0; rc = SQLITE_ERROR; } return rc; } -static int sqlite3Fts5GetTokenizer( - Fts5Global *pGlobal, - const char **azArg, - int nArg, - Fts5Tokenizer **ppTok, - fts5_tokenizer **ppTokApi, - char **pzErr -){ - Fts5TokenizerModule *pMod; +/* +** Attempt to instantiate the tokenizer. +*/ +static int sqlite3Fts5LoadTokenizer(Fts5Config *pConfig){ + const char **azArg = pConfig->t.azArg; + const int nArg = pConfig->t.nArg; + Fts5TokenizerModule *pMod = 0; int rc = SQLITE_OK; - pMod = fts5LocateTokenizer(pGlobal, nArg==0 ? 0 : azArg[0]); + pMod = fts5LocateTokenizer(pConfig->pGlobal, nArg==0 ? 0 : azArg[0]); if( pMod==0 ){ assert( nArg>0 ); rc = SQLITE_ERROR; - *pzErr = sqlite3_mprintf("no such tokenizer: %s", azArg[0]); + sqlite3Fts5ConfigErrmsg(pConfig, "no such tokenizer: %s", azArg[0]); }else{ - rc = pMod->x.xCreate(pMod->pUserData, &azArg[1], (nArg?nArg-1:0), ppTok); - *ppTokApi = &pMod->x; - if( rc!=SQLITE_OK && pzErr ){ - *pzErr = sqlite3_mprintf("error in tokenizer constructor"); + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**) = 0; + if( pMod->bV2Native ){ + xCreate = pMod->x2.xCreate; + pConfig->t.pApi2 = &pMod->x2; + }else{ + pConfig->t.pApi1 = &pMod->x1; + xCreate = pMod->x1.xCreate; + } + + rc = xCreate(pMod->pUserData, + (azArg?&azArg[1]:0), (nArg?nArg-1:0), &pConfig->t.pTok + ); + + if( rc!=SQLITE_OK ){ + if( rc!=SQLITE_NOMEM ){ + sqlite3Fts5ConfigErrmsg(pConfig, "error in tokenizer constructor"); + } + }else if( pMod->bV2Native==0 ){ + pConfig->t.ePattern = sqlite3Fts5TokenizerPattern( + pMod->x1.xCreate, pConfig->t.pTok + ); } } if( rc!=SQLITE_OK ){ - *ppTokApi = 0; - *ppTok = 0; + pConfig->t.pApi1 = 0; + pConfig->t.pApi2 = 0; + pConfig->t.pTok = 0; } return rc; } + +/* +** xDestroy callback passed to sqlite3_create_module(). This is invoked +** when the db handle is being closed. Free memory associated with +** tokenizers and aux functions registered with this db handle. +*/ static void fts5ModuleDestroy(void *pCtx){ Fts5TokenizerModule *pTok, *pNextTok; Fts5Auxiliary *pAux, *pNextAux; @@ -192973,18 +254911,21 @@ static void fts5ModuleDestroy(void *pCtx){ sqlite3_free(pGlobal); } +/* +** Implementation of the fts5() function used by clients to obtain the +** API pointer. +*/ static void fts5Fts5Func( sqlite3_context *pCtx, /* Function call context */ int nArg, /* Number of args */ - sqlite3_value **apUnused /* Function arguments */ + sqlite3_value **apArg /* Function arguments */ ){ Fts5Global *pGlobal = (Fts5Global*)sqlite3_user_data(pCtx); - char buf[8]; - UNUSED_PARAM2(nArg, apUnused); - assert( nArg==0 ); - assert( sizeof(buf)>=sizeof(pGlobal) ); - memcpy(buf, (void*)&pGlobal, sizeof(pGlobal)); - sqlite3_result_blob(pCtx, buf, sizeof(pGlobal), SQLITE_TRANSIENT); + fts5_api **ppApi; + UNUSED_PARAM(nArg); + assert( nArg==1 ); + ppApi = (fts5_api**)sqlite3_value_pointer(apArg[0], "fts5_api_ptr"); + if( ppApi ) *ppApi = &pGlobal->api; } /* @@ -192997,12 +254938,126 @@ static void fts5SourceIdFunc( ){ assert( nArg==0 ); UNUSED_PARAM2(nArg, apUnused); - sqlite3_result_text(pCtx, "fts5: 2016-05-18 10:57:30 fc49f556e48970561d7ab6a2f24fdd7d9eb81ff2", -1, SQLITE_TRANSIENT); + sqlite3_result_text(pCtx, "fts5: 2024-11-25 12:07:48 b95d11e958643b969c47a8e5857f3793b9e69700b8f1469371386369a26e577e", -1, SQLITE_TRANSIENT); +} + +/* +** Implementation of fts5_locale(LOCALE, TEXT) function. +** +** If parameter LOCALE is NULL, or a zero-length string, then a copy of +** TEXT is returned. Otherwise, both LOCALE and TEXT are interpreted as +** text, and the value returned is a blob consisting of: +** +** * The 4 bytes 0x00, 0xE0, 0xB2, 0xEb (FTS5_LOCALE_HEADER). +** * The LOCALE, as utf-8 text, followed by +** * 0x00, followed by +** * The TEXT, as utf-8 text. +** +** There is no final nul-terminator following the TEXT value. +*/ +static void fts5LocaleFunc( + sqlite3_context *pCtx, /* Function call context */ + int nArg, /* Number of args */ + sqlite3_value **apArg /* Function arguments */ +){ + const char *zLocale = 0; + int nLocale = 0; + const char *zText = 0; + int nText = 0; + + assert( nArg==2 ); + UNUSED_PARAM(nArg); + + zLocale = (const char*)sqlite3_value_text(apArg[0]); + nLocale = sqlite3_value_bytes(apArg[0]); + + zText = (const char*)sqlite3_value_text(apArg[1]); + nText = sqlite3_value_bytes(apArg[1]); + + if( zLocale==0 || zLocale[0]=='\0' ){ + sqlite3_result_text(pCtx, zText, nText, SQLITE_TRANSIENT); + }else{ + Fts5Global *p = (Fts5Global*)sqlite3_user_data(pCtx); + u8 *pBlob = 0; + u8 *pCsr = 0; + int nBlob = 0; + + nBlob = FTS5_LOCALE_HDR_SIZE + nLocale + 1 + nText; + pBlob = (u8*)sqlite3_malloc(nBlob); + if( pBlob==0 ){ + sqlite3_result_error_nomem(pCtx); + return; + } + + pCsr = pBlob; + memcpy(pCsr, (const u8*)p->aLocaleHdr, FTS5_LOCALE_HDR_SIZE); + pCsr += FTS5_LOCALE_HDR_SIZE; + memcpy(pCsr, zLocale, nLocale); + pCsr += nLocale; + (*pCsr++) = 0x00; + if( zText ) memcpy(pCsr, zText, nText); + assert( &pCsr[nText]==&pBlob[nBlob] ); + + sqlite3_result_blob(pCtx, pBlob, nBlob, sqlite3_free); + } +} + +/* +** Return true if zName is the extension on one of the shadow tables used +** by this module. +*/ +static int fts5ShadowName(const char *zName){ + static const char *azName[] = { + "config", "content", "data", "docsize", "idx" + }; + unsigned int i; + for(i=0; ip.pConfig->pzErrmsg==0 ); + pTab->p.pConfig->pzErrmsg = pzErr; + rc = sqlite3Fts5StorageIntegrity(pTab->pStorage, 0); + if( *pzErr==0 && rc!=SQLITE_OK ){ + if( (rc&0xff)==SQLITE_CORRUPT ){ + *pzErr = sqlite3_mprintf("malformed inverted index for FTS5 table %s.%s", + zSchema, zTabname); + rc = (*pzErr) ? SQLITE_OK : SQLITE_NOMEM; + }else{ + *pzErr = sqlite3_mprintf("unable to validate the inverted index for" + " FTS5 table %s.%s: %s", + zSchema, zTabname, sqlite3_errstr(rc)); + } + } + + sqlite3Fts5IndexCloseReader(pTab->p.pIndex); + pTab->p.pConfig->pzErrmsg = 0; + + return rc; } static int fts5Init(sqlite3 *db){ static const sqlite3_module fts5Mod = { - /* iVersion */ 2, + /* iVersion */ 4, /* xCreate */ fts5CreateMethod, /* xConnect */ fts5ConnectMethod, /* xBestIndex */ fts5BestIndexMethod, @@ -193025,6 +255080,8 @@ static int fts5Init(sqlite3 *db){ /* xSavepoint */ fts5SavepointMethod, /* xRelease */ fts5ReleaseMethod, /* xRollbackTo */ fts5RollbackToMethod, + /* xShadowName */ fts5ShadowName, + /* xIntegrity */ fts5IntegrityMethod }; int rc; @@ -193037,10 +255094,22 @@ static int fts5Init(sqlite3 *db){ void *p = (void*)pGlobal; memset(pGlobal, 0, sizeof(Fts5Global)); pGlobal->db = db; - pGlobal->api.iVersion = 2; + pGlobal->api.iVersion = 3; pGlobal->api.xCreateFunction = fts5CreateAux; pGlobal->api.xCreateTokenizer = fts5CreateTokenizer; pGlobal->api.xFindTokenizer = fts5FindTokenizer; + pGlobal->api.xCreateTokenizer_v2 = fts5CreateTokenizer_v2; + pGlobal->api.xFindTokenizer_v2 = fts5FindTokenizer_v2; + + /* Initialize pGlobal->aLocaleHdr[] to a 128-bit pseudo-random vector. + ** The constants below were generated randomly. */ + sqlite3_randomness(sizeof(pGlobal->aLocaleHdr), pGlobal->aLocaleHdr); + pGlobal->aLocaleHdr[0] ^= 0xF924976D; + pGlobal->aLocaleHdr[1] ^= 0x16596E13; + pGlobal->aLocaleHdr[2] ^= 0x7C80BEAA; + pGlobal->aLocaleHdr[3] ^= 0x9B03A67F; + assert( sizeof(pGlobal->aLocaleHdr)==16 ); + rc = sqlite3_create_module_v2(db, "fts5", &fts5Mod, p, fts5ModuleDestroy); if( rc==SQLITE_OK ) rc = sqlite3Fts5IndexInit(db); if( rc==SQLITE_OK ) rc = sqlite3Fts5ExprInit(pGlobal, db); @@ -193049,12 +255118,21 @@ static int fts5Init(sqlite3 *db){ if( rc==SQLITE_OK ) rc = sqlite3Fts5VocabInit(pGlobal, db); if( rc==SQLITE_OK ){ rc = sqlite3_create_function( - db, "fts5", 0, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 + db, "fts5", 1, SQLITE_UTF8, p, fts5Fts5Func, 0, 0 ); } if( rc==SQLITE_OK ){ rc = sqlite3_create_function( - db, "fts5_source_id", 0, SQLITE_UTF8, p, fts5SourceIdFunc, 0, 0 + db, "fts5_source_id", 0, + SQLITE_UTF8|SQLITE_DETERMINISTIC|SQLITE_INNOCUOUS, + p, fts5SourceIdFunc, 0, 0 + ); + } + if( rc==SQLITE_OK ){ + rc = sqlite3_create_function( + db, "fts5_locale", 2, + SQLITE_UTF8|SQLITE_INNOCUOUS|SQLITE_RESULT_SUBTYPE, + p, fts5LocaleFunc, 0, 0 ); } } @@ -193077,7 +255155,7 @@ static int fts5Init(sqlite3 *db){ ** this module is being built as part of the SQLite core (SQLITE_CORE is ** defined), then sqlite3_open() will call sqlite3Fts5Init() directly. ** -** Or, if this module is being built as a loadable extension, +** Or, if this module is being built as a loadable extension, ** sqlite3Fts5Init() is omitted and the two standard entry points ** sqlite3_fts_init() and sqlite3_fts5_init() defined instead. */ @@ -193085,7 +255163,7 @@ static int fts5Init(sqlite3 *db){ #ifdef _WIN32 __declspec(dllexport) #endif -SQLITE_API int SQLITE_STDCALL sqlite3_fts_init( +SQLITE_API int sqlite3_fts_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -193098,7 +255176,7 @@ SQLITE_API int SQLITE_STDCALL sqlite3_fts_init( #ifdef _WIN32 __declspec(dllexport) #endif -SQLITE_API int SQLITE_STDCALL sqlite3_fts5_init( +SQLITE_API int sqlite3_fts5_init( sqlite3 *db, char **pzErrMsg, const sqlite3_api_routines *pApi @@ -193131,34 +255209,62 @@ SQLITE_PRIVATE int sqlite3Fts5Init(sqlite3 *db){ /* #include "fts5Int.h" */ +/* +** pSavedRow: +** SQL statement FTS5_STMT_LOOKUP2 is a copy of FTS5_STMT_LOOKUP, it +** does a by-rowid lookup to retrieve a single row from the %_content +** table or equivalent external-content table/view. +** +** However, FTS5_STMT_LOOKUP2 is only used when retrieving the original +** values for a row being UPDATEd. In that case, the SQL statement is +** not reset and pSavedRow is set to point at it. This is so that the +** insert operation that follows the delete may access the original +** row values for any new values for which sqlite3_value_nochange() returns +** true. i.e. if the user executes: +** +** CREATE VIRTUAL TABLE ft USING fts5(a, b, c, locale=1); +** ... +** UPDATE fts SET a=?, b=? WHERE rowid=?; +** +** then the value passed to the xUpdate() method of this table as the +** new.c value is an sqlite3_value_nochange() value. So in this case it +** must be read from the saved row stored in Fts5Storage.pSavedRow. +** +** This is necessary - using sqlite3_value_nochange() instead of just having +** SQLite pass the original value back via xUpdate() - so as not to discard +** any locale information associated with such values. +** +*/ struct Fts5Storage { Fts5Config *pConfig; Fts5Index *pIndex; int bTotalsValid; /* True if nTotalRow/aTotalSize[] are valid */ i64 nTotalRow; /* Total number of rows in FTS table */ - i64 *aTotalSize; /* Total sizes of each column */ - sqlite3_stmt *aStmt[11]; + i64 *aTotalSize; /* Total sizes of each column */ + sqlite3_stmt *pSavedRow; + sqlite3_stmt *aStmt[12]; }; -#if FTS5_STMT_SCAN_ASC!=0 -# error "FTS5_STMT_SCAN_ASC mismatch" +#if FTS5_STMT_SCAN_ASC!=0 +# error "FTS5_STMT_SCAN_ASC mismatch" #endif -#if FTS5_STMT_SCAN_DESC!=1 -# error "FTS5_STMT_SCAN_DESC mismatch" +#if FTS5_STMT_SCAN_DESC!=1 +# error "FTS5_STMT_SCAN_DESC mismatch" #endif #if FTS5_STMT_LOOKUP!=2 -# error "FTS5_STMT_LOOKUP mismatch" +# error "FTS5_STMT_LOOKUP mismatch" #endif -#define FTS5_STMT_INSERT_CONTENT 3 -#define FTS5_STMT_REPLACE_CONTENT 4 -#define FTS5_STMT_DELETE_CONTENT 5 -#define FTS5_STMT_REPLACE_DOCSIZE 6 -#define FTS5_STMT_DELETE_DOCSIZE 7 -#define FTS5_STMT_LOOKUP_DOCSIZE 8 -#define FTS5_STMT_REPLACE_CONFIG 9 -#define FTS5_STMT_SCAN 10 +#define FTS5_STMT_LOOKUP2 3 +#define FTS5_STMT_INSERT_CONTENT 4 +#define FTS5_STMT_REPLACE_CONTENT 5 +#define FTS5_STMT_DELETE_CONTENT 6 +#define FTS5_STMT_REPLACE_DOCSIZE 7 +#define FTS5_STMT_DELETE_DOCSIZE 8 +#define FTS5_STMT_LOOKUP_DOCSIZE 9 +#define FTS5_STMT_REPLACE_CONFIG 10 +#define FTS5_STMT_SCAN 11 /* ** Prepare the two insert statements - Fts5Storage.pInsertContent and @@ -193174,12 +255280,12 @@ static int fts5StorageGetStmt( ){ int rc = SQLITE_OK; - /* If there is no %_docsize table, there should be no requests for + /* If there is no %_docsize table, there should be no requests for ** statements to operate on it. */ assert( p->pConfig->bColumnsize || ( - eStmt!=FTS5_STMT_REPLACE_DOCSIZE - && eStmt!=FTS5_STMT_DELETE_DOCSIZE - && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE + eStmt!=FTS5_STMT_REPLACE_DOCSIZE + && eStmt!=FTS5_STMT_DELETE_DOCSIZE + && eStmt!=FTS5_STMT_LOOKUP_DOCSIZE )); assert( eStmt>=0 && eStmtaStmt) ); @@ -193188,14 +255294,15 @@ static int fts5StorageGetStmt( "SELECT %s FROM %s T WHERE T.%Q >= ? AND T.%Q <= ? ORDER BY T.%Q ASC", "SELECT %s FROM %s T WHERE T.%Q <= ? AND T.%Q >= ? ORDER BY T.%Q DESC", "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP */ + "SELECT %s FROM %s T WHERE T.%Q=?", /* LOOKUP2 */ "INSERT INTO %Q.'%q_content' VALUES(%s)", /* INSERT_CONTENT */ "REPLACE INTO %Q.'%q_content' VALUES(%s)", /* REPLACE_CONTENT */ "DELETE FROM %Q.'%q_content' WHERE id=?", /* DELETE_CONTENT */ - "REPLACE INTO %Q.'%q_docsize' VALUES(?,?)", /* REPLACE_DOCSIZE */ + "REPLACE INTO %Q.'%q_docsize' VALUES(?,?%s)", /* REPLACE_DOCSIZE */ "DELETE FROM %Q.'%q_docsize' WHERE id=?", /* DELETE_DOCSIZE */ - "SELECT sz FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ + "SELECT sz%s FROM %Q.'%q_docsize' WHERE id=?", /* LOOKUP_DOCSIZE */ "REPLACE INTO %Q.'%q_config' VALUES(?,?)", /* REPLACE_CONFIG */ "SELECT %s FROM %s AS T", /* SCAN */ @@ -193203,46 +255310,77 @@ static int fts5StorageGetStmt( Fts5Config *pC = p->pConfig; char *zSql = 0; + assert( ArraySize(azStmt)==ArraySize(p->aStmt) ); + switch( eStmt ){ case FTS5_STMT_SCAN: - zSql = sqlite3_mprintf(azStmt[eStmt], + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent ); break; case FTS5_STMT_SCAN_ASC: case FTS5_STMT_SCAN_DESC: - zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent, pC->zContentRowid, pC->zContentRowid, pC->zContentRowid ); break; case FTS5_STMT_LOOKUP: - zSql = sqlite3_mprintf(azStmt[eStmt], + case FTS5_STMT_LOOKUP2: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zContentExprlist, pC->zContent, pC->zContentRowid ); break; - case FTS5_STMT_INSERT_CONTENT: + case FTS5_STMT_INSERT_CONTENT: case FTS5_STMT_REPLACE_CONTENT: { - int nCol = pC->nCol + 1; - char *zBind; + char *zBind = 0; int i; - zBind = sqlite3_malloc(1 + nCol*2); - if( zBind ){ - for(i=0; ieContent==FTS5_CONTENT_NORMAL + || pC->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Add bindings for the "c*" columns - those that store the actual + ** table content. If eContent==NORMAL, then there is one binding + ** for each column. Or, if eContent==UNINDEXED, then there are only + ** bindings for the UNINDEXED columns. */ + for(i=0; rc==SQLITE_OK && i<(pC->nCol+1); i++){ + if( !i || pC->eContent==FTS5_CONTENT_NORMAL || pC->abUnindexed[i-1] ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z%s?%d", zBind, zBind?",":"",i+1); } - zBind[i*2-1] = '\0'; - zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, zBind); - sqlite3_free(zBind); } + + /* Add bindings for any "l*" columns. Only non-UNINDEXED columns + ** require these. */ + if( pC->bLocale && pC->eContent==FTS5_CONTENT_NORMAL ){ + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pC->abUnindexed[i]==0 ){ + zBind = sqlite3Fts5Mprintf(&rc, "%z,?%d", zBind, pC->nCol+i+2); + } + } + } + + zSql = sqlite3Fts5Mprintf(&rc, azStmt[eStmt], pC->zDb, pC->zName,zBind); + sqlite3_free(zBind); break; } + case FTS5_STMT_REPLACE_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName, + (pC->bContentlessDelete ? ",?" : "") + ); + break; + + case FTS5_STMT_LOOKUP_DOCSIZE: + zSql = sqlite3_mprintf(azStmt[eStmt], + (pC->bContentlessDelete ? ",origin" : ""), + pC->zDb, pC->zName + ); + break; + default: zSql = sqlite3_mprintf(azStmt[eStmt], pC->zDb, pC->zName); break; @@ -193251,7 +255389,11 @@ static int fts5StorageGetStmt( if( zSql==0 ){ rc = SQLITE_NOMEM; }else{ - rc = sqlite3_prepare_v2(pC->db, zSql, -1, &p->aStmt[eStmt], 0); + int f = SQLITE_PREPARE_PERSISTENT; + if( eStmt>FTS5_STMT_LOOKUP2 ) f |= SQLITE_PREPARE_NO_VTAB; + p->pConfig->bLock++; + rc = sqlite3_prepare_v3(pC->db, zSql, -1, f, &p->aStmt[eStmt], 0); + p->pConfig->bLock--; sqlite3_free(zSql); if( rc!=SQLITE_OK && pzErrMsg ){ *pzErrMsg = sqlite3_mprintf("%s", sqlite3_errmsg(pC->db)); @@ -193294,7 +255436,7 @@ static int fts5ExecPrintf( ** code otherwise. */ static int sqlite3Fts5DropAll(Fts5Config *pConfig){ - int rc = fts5ExecPrintf(pConfig->db, 0, + int rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_data';" "DROP TABLE IF EXISTS %Q.'%q_idx';" "DROP TABLE IF EXISTS %Q.'%q_config';", @@ -193303,13 +255445,13 @@ static int sqlite3Fts5DropAll(Fts5Config *pConfig){ pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ - rc = fts5ExecPrintf(pConfig->db, 0, + rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_docsize';", pConfig->zDb, pConfig->zName ); } if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ - rc = fts5ExecPrintf(pConfig->db, 0, + rc = fts5ExecPrintf(pConfig->db, 0, "DROP TABLE IF EXISTS %Q.'%q_content';", pConfig->zDb, pConfig->zName ); @@ -193324,7 +255466,7 @@ static void fts5StorageRenameOne( const char *zName /* New name of FTS5 table */ ){ if( *pRc==SQLITE_OK ){ - *pRc = fts5ExecPrintf(pConfig->db, 0, + *pRc = fts5ExecPrintf(pConfig->db, 0, "ALTER TABLE %Q.'%q_%s' RENAME TO '%q_%s';", pConfig->zDb, pConfig->zName, zTail, zName, zTail ); @@ -193333,7 +255475,7 @@ static void fts5StorageRenameOne( static int sqlite3Fts5StorageRename(Fts5Storage *pStorage, const char *zName){ Fts5Config *pConfig = pStorage->pConfig; - int rc = sqlite3Fts5StorageSync(pStorage, 1); + int rc = sqlite3Fts5StorageSync(pStorage); fts5StorageRenameOne(pConfig, &rc, "data", zName); fts5StorageRenameOne(pConfig, &rc, "idx", zName); @@ -193362,11 +255504,15 @@ static int sqlite3Fts5CreateTable( char *zErr = 0; rc = fts5ExecPrintf(pConfig->db, &zErr, "CREATE TABLE %Q.'%q_%q'(%s)%s", - pConfig->zDb, pConfig->zName, zPost, zDefn, bWithout?" WITHOUT ROWID":"" + pConfig->zDb, pConfig->zName, zPost, zDefn, +#ifndef SQLITE_FTS5_NO_WITHOUT_ROWID + bWithout?" WITHOUT ROWID": +#endif + "" ); if( zErr ){ *pzErr = sqlite3_mprintf( - "fts5: error creating shadow table %q_%s: %s", + "fts5: error creating shadow table %q_%s: %s", pConfig->zName, zPost, zErr ); sqlite3_free(zErr); @@ -193377,36 +255523,38 @@ static int sqlite3Fts5CreateTable( /* ** Open a new Fts5Index handle. If the bCreate argument is true, create -** and initialize the underlying tables +** and initialize the underlying tables ** ** If successful, set *pp to point to the new object and return SQLITE_OK. ** Otherwise, set *pp to NULL and return an SQLite error code. */ static int sqlite3Fts5StorageOpen( - Fts5Config *pConfig, - Fts5Index *pIndex, - int bCreate, + Fts5Config *pConfig, + Fts5Index *pIndex, + int bCreate, Fts5Storage **pp, char **pzErr /* OUT: Error message */ ){ int rc = SQLITE_OK; Fts5Storage *p; /* New object */ - int nByte; /* Bytes of space to allocate */ + sqlite3_int64 nByte; /* Bytes of space to allocate */ nByte = sizeof(Fts5Storage) /* Fts5Storage object */ + pConfig->nCol * sizeof(i64); /* Fts5Storage.aTotalSize[] */ - *pp = p = (Fts5Storage*)sqlite3_malloc(nByte); + *pp = p = (Fts5Storage*)sqlite3_malloc64(nByte); if( !p ) return SQLITE_NOMEM; - memset(p, 0, nByte); + memset(p, 0, (size_t)nByte); p->aTotalSize = (i64*)&p[1]; p->pConfig = pConfig; p->pIndex = pIndex; if( bCreate ){ - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ int nDefn = 32 + pConfig->nCol*10; - char *zDefn = sqlite3_malloc(32 + pConfig->nCol * 10); + char *zDefn = sqlite3_malloc64(32 + (sqlite3_int64)pConfig->nCol * 20); if( zDefn==0 ){ rc = SQLITE_NOMEM; }else{ @@ -193415,8 +255563,20 @@ static int sqlite3Fts5StorageOpen( sqlite3_snprintf(nDefn, zDefn, "id INTEGER PRIMARY KEY"); iOff = (int)strlen(zDefn); for(i=0; inCol; i++){ - sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); - iOff += (int)strlen(&zDefn[iOff]); + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->abUnindexed[i] + ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", c%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } + if( pConfig->bLocale ){ + for(i=0; inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + sqlite3_snprintf(nDefn-iOff, &zDefn[iOff], ", l%d", i); + iOff += (int)strlen(&zDefn[iOff]); + } + } } rc = sqlite3Fts5CreateTable(pConfig, "content", zDefn, 0, pzErr); } @@ -193424,9 +255584,11 @@ static int sqlite3Fts5StorageOpen( } if( rc==SQLITE_OK && pConfig->bColumnsize ){ - rc = sqlite3Fts5CreateTable( - pConfig, "docsize", "id INTEGER PRIMARY KEY, sz BLOB", 0, pzErr - ); + const char *zCols = "id INTEGER PRIMARY KEY, sz BLOB"; + if( pConfig->bContentlessDelete ){ + zCols = "id INTEGER PRIMARY KEY, sz BLOB, origin INTEGER"; + } + rc = sqlite3Fts5CreateTable(pConfig, "docsize", zCols, 0, pzErr); } if( rc==SQLITE_OK ){ rc = sqlite3Fts5CreateTable( @@ -193491,60 +255653,180 @@ static int fts5StorageInsertCallback( return sqlite3Fts5IndexWrite(pIdx, pCtx->iCol, pCtx->szCol-1, pToken, nToken); } +/* +** This function is used as part of an UPDATE statement that modifies the +** rowid of a row. In that case, this function is called first to set +** Fts5Storage.pSavedRow to point to a statement that may be used to +** access the original values of the row being deleted - iDel. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** It is not considered an error if row iDel does not exist. In this case +** pSavedRow is not set and SQLITE_OK returned. +*/ +static int sqlite3Fts5StorageFindDeleteRow(Fts5Storage *p, i64 iDel){ + int rc = SQLITE_OK; + sqlite3_stmt *pSeek = 0; + + assert( p->pSavedRow==0 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+1, &pSeek, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + rc = sqlite3_reset(pSeek); + }else{ + p->pSavedRow = pSeek; + } + } + + return rc; +} + /* ** If a row with rowid iDel is present in the %_content table, add the ** delete-markers to the FTS index necessary to delete it. Do not actually ** remove the %_content row at this time though. +** +** If parameter bSaveRow is true, then Fts5Storage.pSavedRow is left +** pointing to a statement (FTS5_STMT_LOOKUP2) that may be used to access +** the original values of the row being deleted. This is used by UPDATE +** statements. */ static int fts5StorageDeleteFromIndex( - Fts5Storage *p, - i64 iDel, - sqlite3_value **apVal + Fts5Storage *p, + i64 iDel, + sqlite3_value **apVal, + int bSaveRow /* True to set pSavedRow */ ){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pSeek = 0; /* SELECT to read row iDel from %_data */ - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int rc2; /* sqlite3_reset() return code */ int iCol; Fts5InsertCtx ctx; + assert( bSaveRow==0 || apVal==0 ); + assert( bSaveRow==0 || bSaveRow==1 ); + assert( FTS5_STMT_LOOKUP2==FTS5_STMT_LOOKUP+1 ); + if( apVal==0 ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP, &pSeek, 0); - if( rc!=SQLITE_OK ) return rc; - sqlite3_bind_int64(pSeek, 1, iDel); - if( sqlite3_step(pSeek)!=SQLITE_ROW ){ - return sqlite3_reset(pSeek); + if( p->pSavedRow && bSaveRow ){ + pSeek = p->pSavedRow; + p->pSavedRow = 0; + }else{ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP+bSaveRow, &pSeek, 0); + if( rc!=SQLITE_OK ) return rc; + sqlite3_bind_int64(pSeek, 1, iDel); + if( sqlite3_step(pSeek)!=SQLITE_ROW ){ + return sqlite3_reset(pSeek); + } } } ctx.pStorage = p; ctx.iCol = -1; - rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); for(iCol=1; rc==SQLITE_OK && iCol<=pConfig->nCol; iCol++){ if( pConfig->abUnindexed[iCol-1]==0 ){ - const char *zText; - int nText; + sqlite3_value *pVal = 0; + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + + assert( pSeek==0 || apVal==0 ); + assert( pSeek!=0 || apVal!=0 ); if( pSeek ){ - zText = (const char*)sqlite3_column_text(pSeek, iCol); - nText = sqlite3_column_bytes(pSeek, iCol); + pVal = sqlite3_column_value(pSeek, iCol); }else{ - zText = (const char*)sqlite3_value_text(apVal[iCol-1]); - nText = sqlite3_value_bytes(apVal[iCol-1]); + pVal = apVal[iCol-1]; + } + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale && pSeek ){ + pLoc = (const char*)sqlite3_column_text(pSeek, iCol + pConfig->nCol); + nLoc = sqlite3_column_bytes(pSeek, iCol + pConfig->nCol); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + ctx.szCol = 0; + rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, + pText, nText, (void*)&ctx, fts5StorageInsertCallback + ); + p->aTotalSize[iCol-1] -= (i64)ctx.szCol; + if( rc==SQLITE_OK && p->aTotalSize[iCol-1]<0 ){ + rc = FTS5_CORRUPT; + } + sqlite3Fts5ClearLocale(pConfig); } - ctx.szCol = 0; - rc = sqlite3Fts5Tokenize(pConfig, FTS5_TOKENIZE_DOCUMENT, - zText, nText, (void*)&ctx, fts5StorageInsertCallback - ); - p->aTotalSize[iCol-1] -= (i64)ctx.szCol; } } - p->nTotalRow--; + if( rc==SQLITE_OK && p->nTotalRow<1 ){ + rc = FTS5_CORRUPT; + }else{ + p->nTotalRow--; + } - rc2 = sqlite3_reset(pSeek); - if( rc==SQLITE_OK ) rc = rc2; + if( rc==SQLITE_OK && bSaveRow ){ + assert( p->pSavedRow==0 ); + p->pSavedRow = pSeek; + }else{ + rc2 = sqlite3_reset(pSeek); + if( rc==SQLITE_OK ) rc = rc2; + } return rc; } +/* +** Reset any saved statement pSavedRow. Zero pSavedRow as well. This +** should be called by the xUpdate() method of the fts5 table before +** returning from any operation that may have set Fts5Storage.pSavedRow. +*/ +static void sqlite3Fts5StorageReleaseDeleteRow(Fts5Storage *pStorage){ + assert( pStorage->pSavedRow==0 + || pStorage->pSavedRow==pStorage->aStmt[FTS5_STMT_LOOKUP2] + ); + sqlite3_reset(pStorage->pSavedRow); + pStorage->pSavedRow = 0; +} + +/* +** This function is called to process a DELETE on a contentless_delete=1 +** table. It adds the tombstone required to delete the entry with rowid +** iDel. If successful, SQLITE_OK is returned. Or, if an error occurs, +** an SQLite error code. +*/ +static int fts5StorageContentlessDelete(Fts5Storage *p, i64 iDel){ + i64 iOrigin = 0; + sqlite3_stmt *pLookup = 0; + int rc = SQLITE_OK; + + assert( p->pConfig->bContentlessDelete ); + assert( p->pConfig->eContent==FTS5_CONTENT_NONE + || p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ); + + /* Look up the origin of the document in the %_docsize table. Store + ** this in stack variable iOrigin. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); + if( rc==SQLITE_OK ){ + sqlite3_bind_int64(pLookup, 1, iDel); + if( SQLITE_ROW==sqlite3_step(pLookup) ){ + iOrigin = sqlite3_column_int64(pLookup, 1); + } + rc = sqlite3_reset(pLookup); + } + + if( rc==SQLITE_OK && iOrigin!=0 ){ + rc = sqlite3Fts5IndexContentlessDelete(p->pIndex, iOrigin, iDel); + } + + return rc; +} /* ** Insert a record into the %_docsize table. Specifically, do: @@ -193565,16 +255847,24 @@ static int fts5StorageInsertDocsize( rc = fts5StorageGetStmt(p, FTS5_STMT_REPLACE_DOCSIZE, &pReplace, 0); if( rc==SQLITE_OK ){ sqlite3_bind_int64(pReplace, 1, iRowid); + if( p->pConfig->bContentlessDelete ){ + i64 iOrigin = 0; + rc = sqlite3Fts5IndexGetOrigin(p->pIndex, &iOrigin); + sqlite3_bind_int64(pReplace, 3, iOrigin); + } + } + if( rc==SQLITE_OK ){ sqlite3_bind_blob(pReplace, 2, pBuf->p, pBuf->n, SQLITE_STATIC); sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 2); } } return rc; } /* -** Load the contents of the "averages" record from disk into the +** Load the contents of the "averages" record from disk into the ** p->nTotalRow and p->aTotalSize[] variables. If successful, and if ** argument bCache is true, set the p->bTotalsValid flag to indicate ** that the contents of aTotalSize[] and nTotalRow are valid until @@ -193593,7 +255883,7 @@ static int fts5StorageLoadTotals(Fts5Storage *p, int bCache){ } /* -** Store the current contents of the p->nTotalRow and p->aTotalSize[] +** Store the current contents of the p->nTotalRow and p->aTotalSize[] ** variables in the "averages" record on disk. ** ** Return SQLITE_OK if successful, or an SQLite error code if an error @@ -193621,7 +255911,12 @@ static int fts5StorageSaveTotals(Fts5Storage *p){ /* ** Remove a row from the FTS table. */ -static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **apVal){ +static int sqlite3Fts5StorageDelete( + Fts5Storage *p, /* Storage object */ + i64 iDel, /* Rowid to delete from table */ + sqlite3_value **apVal, /* Optional - values to remove from index */ + int bSaveRow /* If true, set pSavedRow for deleted row */ +){ Fts5Config *pConfig = p->pConfig; int rc; sqlite3_stmt *pDel = 0; @@ -193631,7 +255926,21 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap /* Delete the index records */ if( rc==SQLITE_OK ){ - rc = fts5StorageDeleteFromIndex(p, iDel, apVal); + rc = sqlite3Fts5IndexBeginWrite(p->pIndex, 1, iDel); + } + + if( rc==SQLITE_OK ){ + if( p->pConfig->bContentlessDelete ){ + rc = fts5StorageContentlessDelete(p, iDel); + if( rc==SQLITE_OK + && bSaveRow + && p->pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ + rc = sqlite3Fts5StorageFindDeleteRow(p, iDel); + } + }else{ + rc = fts5StorageDeleteFromIndex(p, iDel, apVal, bSaveRow); + } } /* Delete the %_docsize record */ @@ -193645,7 +255954,9 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap } /* Delete the %_content record */ - if( pConfig->eContent==FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent==FTS5_CONTENT_NORMAL + || pConfig->eContent==FTS5_CONTENT_UNINDEXED + ){ if( rc==SQLITE_OK ){ rc = fts5StorageGetStmt(p, FTS5_STMT_DELETE_CONTENT, &pDel, 0); } @@ -193656,11 +255967,6 @@ static int sqlite3Fts5StorageDelete(Fts5Storage *p, i64 iDel, sqlite3_value **ap } } - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -193671,17 +255977,24 @@ static int sqlite3Fts5StorageDeleteAll(Fts5Storage *p){ Fts5Config *pConfig = p->pConfig; int rc; + p->bTotalsValid = 0; + /* Delete the contents of the %_data and %_docsize tables. */ rc = fts5ExecPrintf(pConfig->db, 0, - "DELETE FROM %Q.'%q_data';" + "DELETE FROM %Q.'%q_data';" "DELETE FROM %Q.'%q_idx';", pConfig->zDb, pConfig->zName, pConfig->zDb, pConfig->zName ); if( rc==SQLITE_OK && pConfig->bColumnsize ){ rc = fts5ExecPrintf(pConfig->db, 0, - "DELETE FROM %Q.'%q_docsize';", - pConfig->zDb, pConfig->zName + "DELETE FROM %Q.'%q_docsize';", pConfig->zDb, pConfig->zName + ); + } + + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_UNINDEXED ){ + rc = fts5ExecPrintf(pConfig->db, 0, + "DELETE FROM %Q.'%q_content';", pConfig->zDb, pConfig->zName ); } @@ -193701,7 +256014,7 @@ static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ Fts5Config *pConfig = p->pConfig; sqlite3_stmt *pScan = 0; Fts5InsertCtx ctx; - int rc; + int rc, rc2; memset(&ctx, 0, sizeof(Fts5InsertCtx)); ctx.pStorage = p; @@ -193711,7 +256024,7 @@ static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ } if( rc==SQLITE_OK ){ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, pConfig->pzErrmsg); } while( rc==SQLITE_OK && SQLITE_ROW==sqlite3_step(pScan) ){ @@ -193722,13 +256035,36 @@ static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_column_text(pScan, ctx.iCol+1), - sqlite3_column_bytes(pScan, ctx.iCol+1), - (void*)&ctx, - fts5StorageInsertCallback - ); + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pLoc in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = sqlite3_column_value(pScan, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + if( pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -193740,6 +256076,8 @@ static int sqlite3Fts5StorageRebuild(Fts5Storage *p){ } } sqlite3_free(buf.p); + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; /* Write the averages record */ if( rc==SQLITE_OK ){ @@ -193791,15 +256129,18 @@ static int fts5StorageNewRowid(Fts5Storage *p, i64 *piRowid){ ** Insert a new row into the FTS content table. */ static int sqlite3Fts5StorageContentInsert( - Fts5Storage *p, - sqlite3_value **apVal, + Fts5Storage *p, + int bReplace, /* True to use REPLACE instead of INSERT */ + sqlite3_value **apVal, i64 *piRowid ){ Fts5Config *pConfig = p->pConfig; int rc = SQLITE_OK; /* Insert the new row into the %_content table. */ - if( pConfig->eContent!=FTS5_CONTENT_NORMAL ){ + if( pConfig->eContent!=FTS5_CONTENT_NORMAL + && pConfig->eContent!=FTS5_CONTENT_UNINDEXED + ){ if( sqlite3_value_type(apVal[1])==SQLITE_INTEGER ){ *piRowid = sqlite3_value_int64(apVal[1]); }else{ @@ -193808,9 +256149,52 @@ static int sqlite3Fts5StorageContentInsert( }else{ sqlite3_stmt *pInsert = 0; /* Statement to write %_content table */ int i; /* Counter variable */ - rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT, &pInsert, 0); - for(i=1; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ - rc = sqlite3_bind_value(pInsert, i, apVal[i]); + + assert( FTS5_STMT_INSERT_CONTENT+1==FTS5_STMT_REPLACE_CONTENT ); + assert( bReplace==0 || bReplace==1 ); + rc = fts5StorageGetStmt(p, FTS5_STMT_INSERT_CONTENT+bReplace, &pInsert, 0); + if( pInsert ) sqlite3_clear_bindings(pInsert); + + /* Bind the rowid value */ + sqlite3_bind_value(pInsert, 1, apVal[1]); + + /* Loop through values for user-defined columns. i=2 is the leftmost + ** user-defined column. As is column 1 of pSavedRow. */ + for(i=2; rc==SQLITE_OK && i<=pConfig->nCol+1; i++){ + int bUnindexed = pConfig->abUnindexed[i-2]; + if( pConfig->eContent==FTS5_CONTENT_NORMAL || bUnindexed ){ + sqlite3_value *pVal = apVal[i]; + + if( sqlite3_value_nochange(pVal) && p->pSavedRow ){ + /* This is an UPDATE statement, and user-defined column (i-2) was not + ** modified. Retrieve the value from Fts5Storage.pSavedRow. */ + pVal = sqlite3_column_value(p->pSavedRow, i-1); + if( pConfig->bLocale && bUnindexed==0 ){ + sqlite3_bind_value(pInsert, pConfig->nCol + i, + sqlite3_column_value(p->pSavedRow, pConfig->nCol + i - 1) + ); + } + }else if( sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + const char *pText = 0; + const char *pLoc = 0; + int nText = 0; + int nLoc = 0; + assert( pConfig->bLocale ); + + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + if( rc==SQLITE_OK ){ + sqlite3_bind_text(pInsert, i, pText, nText, SQLITE_TRANSIENT); + if( bUnindexed==0 ){ + int iLoc = pConfig->nCol + i; + sqlite3_bind_text(pInsert, iLoc, pLoc, nLoc, SQLITE_TRANSIENT); + } + } + + continue; + } + + rc = sqlite3_bind_value(pInsert, i, pVal); + } } if( rc==SQLITE_OK ){ sqlite3_step(pInsert); @@ -193826,8 +256210,8 @@ static int sqlite3Fts5StorageContentInsert( ** Insert new entries into the FTS index and %_docsize table. */ static int sqlite3Fts5StorageIndexInsert( - Fts5Storage *p, - sqlite3_value **apVal, + Fts5Storage *p, + sqlite3_value **apVal, i64 iRowid ){ Fts5Config *pConfig = p->pConfig; @@ -193845,13 +256229,38 @@ static int sqlite3Fts5StorageIndexInsert( for(ctx.iCol=0; rc==SQLITE_OK && ctx.iColnCol; ctx.iCol++){ ctx.szCol = 0; if( pConfig->abUnindexed[ctx.iCol]==0 ){ - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_value_text(apVal[ctx.iCol+2]), - sqlite3_value_bytes(apVal[ctx.iCol+2]), - (void*)&ctx, - fts5StorageInsertCallback - ); + int nText = 0; /* Size of pText in bytes */ + const char *pText = 0; /* Pointer to buffer containing text value */ + int nLoc = 0; /* Size of pText in bytes */ + const char *pLoc = 0; /* Pointer to buffer containing text value */ + + sqlite3_value *pVal = apVal[ctx.iCol+2]; + if( p->pSavedRow && sqlite3_value_nochange(pVal) ){ + pVal = sqlite3_column_value(p->pSavedRow, ctx.iCol+1); + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = ctx.iCol + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(p->pSavedRow, iCol); + nLoc = sqlite3_column_bytes(p->pSavedRow, iCol); + } + }else{ + pVal = apVal[ctx.iCol+2]; + } + + if( pConfig->bLocale && sqlite3Fts5IsLocaleValue(pConfig, pVal) ){ + rc = sqlite3Fts5DecodeLocaleValue(pVal, &pText, &nText, &pLoc, &nLoc); + }else{ + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, pText, nText, (void*)&ctx, + fts5StorageInsertCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } } sqlite3Fts5BufferAppendVarint(&rc, &buf, ctx.szCol); p->aTotalSize[ctx.iCol] += (i64)ctx.szCol; @@ -193864,11 +256273,6 @@ static int sqlite3Fts5StorageIndexInsert( } sqlite3_free(buf.p); - /* Write the averages record */ - if( rc==SQLITE_OK ){ - rc = fts5StorageSaveTotals(p); - } - return rc; } @@ -193877,7 +256281,7 @@ static int fts5StorageCount(Fts5Storage *p, const char *zSuffix, i64 *pnRow){ char *zSql; int rc; - zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", + zSql = sqlite3_mprintf("SELECT count(*) FROM %Q.'%q_%s'", pConfig->zDb, pConfig->zName, zSuffix ); if( zSql==0 ){ @@ -193984,97 +256388,136 @@ static int fts5StorageIntegrityCallback( ** some other SQLite error code if an error occurs while attempting to ** determine this. */ -static int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ +static int sqlite3Fts5StorageIntegrity(Fts5Storage *p, int iArg){ Fts5Config *pConfig = p->pConfig; - int rc; /* Return code */ + int rc = SQLITE_OK; /* Return code */ int *aColSize; /* Array of size pConfig->nCol */ i64 *aTotalSize; /* Array of size pConfig->nCol */ Fts5IntegrityCtx ctx; sqlite3_stmt *pScan; + int bUseCksum; memset(&ctx, 0, sizeof(Fts5IntegrityCtx)); ctx.pConfig = p->pConfig; - aTotalSize = (i64*)sqlite3_malloc(pConfig->nCol * (sizeof(int)+sizeof(i64))); + aTotalSize = (i64*)sqlite3_malloc64(pConfig->nCol*(sizeof(int)+sizeof(i64))); if( !aTotalSize ) return SQLITE_NOMEM; aColSize = (int*)&aTotalSize[pConfig->nCol]; memset(aTotalSize, 0, sizeof(i64) * pConfig->nCol); - /* Generate the expected index checksum based on the contents of the - ** %_content table. This block stores the checksum in ctx.cksum. */ - rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); - if( rc==SQLITE_OK ){ - int rc2; - while( SQLITE_ROW==sqlite3_step(pScan) ){ - int i; - ctx.iRowid = sqlite3_column_int64(pScan, 0); - ctx.szCol = 0; - if( pConfig->bColumnsize ){ - rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); - } - if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ - rc = sqlite3Fts5TermsetNew(&ctx.pTermset); - } - for(i=0; rc==SQLITE_OK && inCol; i++){ - if( pConfig->abUnindexed[i] ) continue; - ctx.iCol = i; + bUseCksum = (pConfig->eContent==FTS5_CONTENT_NORMAL + || (pConfig->eContent==FTS5_CONTENT_EXTERNAL && iArg) + ); + if( bUseCksum ){ + /* Generate the expected index checksum based on the contents of the + ** %_content table. This block stores the checksum in ctx.cksum. */ + rc = fts5StorageGetStmt(p, FTS5_STMT_SCAN, &pScan, 0); + if( rc==SQLITE_OK ){ + int rc2; + while( SQLITE_ROW==sqlite3_step(pScan) ){ + int i; + ctx.iRowid = sqlite3_column_int64(pScan, 0); ctx.szCol = 0; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + if( pConfig->bColumnsize ){ + rc = sqlite3Fts5StorageDocsize(p, ctx.iRowid, aColSize); + } + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_NONE ){ rc = sqlite3Fts5TermsetNew(&ctx.pTermset); } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5Tokenize(pConfig, - FTS5_TOKENIZE_DOCUMENT, - (const char*)sqlite3_column_text(pScan, i+1), - sqlite3_column_bytes(pScan, i+1), - (void*)&ctx, - fts5StorageIntegrityCallback - ); - } - if( rc==SQLITE_OK && pConfig->bColumnsize && ctx.szCol!=aColSize[i] ){ - rc = FTS5_CORRUPT; - } - aTotalSize[i] += ctx.szCol; - if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( pConfig->abUnindexed[i]==0 ){ + const char *pText = 0; + int nText = 0; + const char *pLoc = 0; + int nLoc = 0; + sqlite3_value *pVal = sqlite3_column_value(pScan, i+1); + + if( pConfig->eContent==FTS5_CONTENT_EXTERNAL + && sqlite3Fts5IsLocaleValue(pConfig, pVal) + ){ + rc = sqlite3Fts5DecodeLocaleValue( + pVal, &pText, &nText, &pLoc, &nLoc + ); + }else{ + if( pConfig->eContent==FTS5_CONTENT_NORMAL && pConfig->bLocale ){ + int iCol = i + 1 + pConfig->nCol; + pLoc = (const char*)sqlite3_column_text(pScan, iCol); + nLoc = sqlite3_column_bytes(pScan, iCol); + } + pText = (const char*)sqlite3_value_text(pVal); + nText = sqlite3_value_bytes(pVal); + } + + ctx.iCol = i; + ctx.szCol = 0; + + if( rc==SQLITE_OK && pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + rc = sqlite3Fts5TermsetNew(&ctx.pTermset); + } + + if( rc==SQLITE_OK ){ + sqlite3Fts5SetLocale(pConfig, pLoc, nLoc); + rc = sqlite3Fts5Tokenize(pConfig, + FTS5_TOKENIZE_DOCUMENT, + pText, nText, + (void*)&ctx, + fts5StorageIntegrityCallback + ); + sqlite3Fts5ClearLocale(pConfig); + } + + /* If this is not a columnsize=0 database, check that the number + ** of tokens in the value matches the aColSize[] value read from + ** the %_docsize table. */ + if( rc==SQLITE_OK + && pConfig->bColumnsize + && ctx.szCol!=aColSize[i] + ){ + rc = FTS5_CORRUPT; + } + aTotalSize[i] += ctx.szCol; + if( pConfig->eDetail==FTS5_DETAIL_COLUMNS ){ + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + } + } } + sqlite3Fts5TermsetFree(ctx.pTermset); + ctx.pTermset = 0; + + if( rc!=SQLITE_OK ) break; } - sqlite3Fts5TermsetFree(ctx.pTermset); - ctx.pTermset = 0; - - if( rc!=SQLITE_OK ) break; + rc2 = sqlite3_reset(pScan); + if( rc==SQLITE_OK ) rc = rc2; } - rc2 = sqlite3_reset(pScan); - if( rc==SQLITE_OK ) rc = rc2; - } - /* Test that the "totals" (sometimes called "averages") record looks Ok */ - if( rc==SQLITE_OK ){ - int i; - rc = fts5StorageLoadTotals(p, 0); - for(i=0; rc==SQLITE_OK && inCol; i++){ - if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT; + /* Test that the "totals" (sometimes called "averages") record looks Ok */ + if( rc==SQLITE_OK ){ + int i; + rc = fts5StorageLoadTotals(p, 0); + for(i=0; rc==SQLITE_OK && inCol; i++){ + if( p->aTotalSize[i]!=aTotalSize[i] ) rc = FTS5_CORRUPT; + } } - } - /* Check that the %_docsize and %_content tables contain the expected - ** number of rows. */ - if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ - i64 nRow = 0; - rc = fts5StorageCount(p, "content", &nRow); - if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; - } - if( rc==SQLITE_OK && pConfig->bColumnsize ){ - i64 nRow = 0; - rc = fts5StorageCount(p, "docsize", &nRow); - if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + /* Check that the %_docsize and %_content tables contain the expected + ** number of rows. */ + if( rc==SQLITE_OK && pConfig->eContent==FTS5_CONTENT_NORMAL ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "content", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } + if( rc==SQLITE_OK && pConfig->bColumnsize ){ + i64 nRow = 0; + rc = fts5StorageCount(p, "docsize", &nRow); + if( rc==SQLITE_OK && nRow!=p->nTotalRow ) rc = FTS5_CORRUPT; + } } /* Pass the expected checksum down to the FTS index module. It will ** verify, amongst other things, that it matches the checksum generated by ** inspecting the index itself. */ if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum); + rc = sqlite3Fts5IndexIntegrityCheck(p->pIndex, ctx.cksum, bUseCksum); } sqlite3_free(aTotalSize); @@ -194086,13 +256529,13 @@ static int sqlite3Fts5StorageIntegrity(Fts5Storage *p){ ** %_content table. */ static int sqlite3Fts5StorageStmt( - Fts5Storage *p, - int eStmt, - sqlite3_stmt **pp, + Fts5Storage *p, + int eStmt, + sqlite3_stmt **pp, char **pzErrMsg ){ int rc; - assert( eStmt==FTS5_STMT_SCAN_ASC + assert( eStmt==FTS5_STMT_SCAN_ASC || eStmt==FTS5_STMT_SCAN_DESC || eStmt==FTS5_STMT_LOOKUP ); @@ -194110,8 +256553,8 @@ static int sqlite3Fts5StorageStmt( ** must match that passed to the sqlite3Fts5StorageStmt() call. */ static void sqlite3Fts5StorageStmtRelease( - Fts5Storage *p, - int eStmt, + Fts5Storage *p, + int eStmt, sqlite3_stmt *pStmt ){ assert( eStmt==FTS5_STMT_SCAN_ASC @@ -194154,8 +256597,9 @@ static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ assert( p->pConfig->bColumnsize ); rc = fts5StorageGetStmt(p, FTS5_STMT_LOOKUP_DOCSIZE, &pLookup, 0); - if( rc==SQLITE_OK ){ + if( pLookup ){ int bCorrupt = 1; + assert( rc==SQLITE_OK ); sqlite3_bind_int64(pLookup, 1, iRowid); if( SQLITE_ROW==sqlite3_step(pLookup) ){ const u8 *aBlob = sqlite3_column_blob(pLookup, 0); @@ -194168,6 +256612,8 @@ static int sqlite3Fts5StorageDocsize(Fts5Storage *p, i64 iRowid, int *aCol){ if( bCorrupt && rc==SQLITE_OK ){ rc = FTS5_CORRUPT; } + }else{ + assert( rc!=SQLITE_OK ); } return rc; @@ -194194,7 +256640,13 @@ static int sqlite3Fts5StorageSize(Fts5Storage *p, int iCol, i64 *pnToken){ static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ int rc = fts5StorageLoadTotals(p, 0); if( rc==SQLITE_OK ){ + /* nTotalRow being zero does not necessarily indicate a corrupt + ** database - it might be that the FTS5 table really does contain zero + ** rows. However this function is only called from the xRowCount() API, + ** and there is no way for that API to be invoked if the table contains + ** no rows. Hence the FTS5_CORRUPT return. */ *pnRow = p->nTotalRow; + if( p->nTotalRow<=0 ) rc = FTS5_CORRUPT; } return rc; } @@ -194202,13 +256654,20 @@ static int sqlite3Fts5StorageRowCount(Fts5Storage *p, i64 *pnRow){ /* ** Flush any data currently held in-memory to disk. */ -static int sqlite3Fts5StorageSync(Fts5Storage *p, int bCommit){ - if( bCommit && p->bTotalsValid ){ - int rc = fts5StorageSaveTotals(p); - p->bTotalsValid = 0; - if( rc!=SQLITE_OK ) return rc; +static int sqlite3Fts5StorageSync(Fts5Storage *p){ + int rc = SQLITE_OK; + i64 iLastRowid = sqlite3_last_insert_rowid(p->pConfig->db); + if( p->bTotalsValid ){ + rc = fts5StorageSaveTotals(p); + if( rc==SQLITE_OK ){ + p->bTotalsValid = 0; + } } - return sqlite3Fts5IndexSync(p->pIndex, bCommit); + if( rc==SQLITE_OK ){ + rc = sqlite3Fts5IndexSync(p->pIndex); + } + sqlite3_set_last_insert_rowid(p->pConfig->db, iLastRowid); + return rc; } static int sqlite3Fts5StorageRollback(Fts5Storage *p){ @@ -194217,7 +256676,7 @@ static int sqlite3Fts5StorageRollback(Fts5Storage *p){ } static int sqlite3Fts5StorageConfigValue( - Fts5Storage *p, + Fts5Storage *p, const char *z, sqlite3_value *pVal, int iVal @@ -194233,6 +256692,7 @@ static int sqlite3Fts5StorageConfigValue( } sqlite3_step(pReplace); rc = sqlite3_reset(pReplace); + sqlite3_bind_null(pReplace, 1); } if( rc==SQLITE_OK && pVal ){ int iNew = p->pConfig->iCookie + 1; @@ -194266,7 +256726,7 @@ static int sqlite3Fts5StorageConfigValue( /* ** For tokenizers with no "unicode" modifier, the set of token characters -** is the same as the set of ASCII range alphanumeric characters. +** is the same as the set of ASCII range alphanumeric characters. */ static unsigned char aAsciiTokenChar[128] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, /* 0x00..0x0F */ @@ -194285,8 +256745,8 @@ struct AsciiTokenizer { }; static void fts5AsciiAddExceptions( - AsciiTokenizer *p, - const char *zArg, + AsciiTokenizer *p, + const char *zArg, int bTokenChars ){ int i; @@ -194308,7 +256768,7 @@ static void fts5AsciiDelete(Fts5Tokenizer *p){ ** Create an "ascii" tokenizer. */ static int fts5AsciiCreate( - void *pUnused, + void *pUnused, const char **azArg, int nArg, Fts5Tokenizer **ppOut ){ @@ -194398,7 +256858,7 @@ static int fts5AsciiTokenize( nByte = ie-is; if( nByte>nFold ){ if( pFold!=aFold ) sqlite3_free(pFold); - pFold = sqlite3_malloc(nByte*2); + pFold = sqlite3_malloc64((sqlite3_int64)nByte*2); if( pFold==0 ){ rc = SQLITE_NOMEM; break; @@ -194411,7 +256871,7 @@ static int fts5AsciiTokenize( rc = xToken(pCtx, 0, pFold, nByte, is, ie); is = ie+1; } - + if( pFold!=aFold ) sqlite3_free(pFold); if( rc==SQLITE_DONE ) rc = SQLITE_OK; return rc; @@ -194444,7 +256904,7 @@ static const unsigned char sqlite3Utf8Trans1[] = { c = *(zIn++); \ if( c>=0xc0 ){ \ c = sqlite3Utf8Trans1[c-0xc0]; \ - while( zIn!=zTerm && (*zIn & 0xc0)==0x80 ){ \ + while( zIn=0xc0 ){ \ + while( (((unsigned char)*zIn) & 0xc0)==0x80 ){ zIn++; } \ + } \ +} + typedef struct Unicode61Tokenizer Unicode61Tokenizer; struct Unicode61Tokenizer { unsigned char aTokenChar[128]; /* ASCII range token characters */ char *aFold; /* Buffer to fold text into */ int nFold; /* Size of aFold[] in bytes */ - int bRemoveDiacritic; /* True if remove_diacritics=1 is set */ + int eRemoveDiacritic; /* True if remove_diacritics=1 is set */ int nException; int *aiException; + + unsigned char aCategory[32]; /* True for token char categories */ }; +/* Values for eRemoveDiacritic (must match internals of fts5_unicode2.c) */ +#define FTS5_REMOVE_DIACRITICS_NONE 0 +#define FTS5_REMOVE_DIACRITICS_SIMPLE 1 +#define FTS5_REMOVE_DIACRITICS_COMPLEX 2 + static int fts5UnicodeAddExceptions( Unicode61Tokenizer *p, /* Tokenizer object */ const char *z, /* Characters to treat as exceptions */ @@ -194495,25 +256968,26 @@ static int fts5UnicodeAddExceptions( int *aNew; if( n>0 ){ - aNew = (int*)sqlite3_realloc(p->aiException, (n+p->nException)*sizeof(int)); + aNew = (int*)sqlite3_realloc64(p->aiException, + (n+p->nException)*sizeof(int)); if( aNew ){ int nNew = p->nException; const unsigned char *zCsr = (const unsigned char*)z; const unsigned char *zTerm = (const unsigned char*)&z[n]; while( zCsraTokenChar[iCode] = (unsigned char)bTokenChars; }else{ - bToken = sqlite3Fts5UnicodeIsalnum(iCode); - assert( (bToken==0 || bToken==1) ); + bToken = p->aCategory[sqlite3Fts5UnicodeCategory(iCode)]; + assert( (bToken==0 || bToken==1) ); assert( (bTokenChars==0 || bTokenChars==1) ); if( bToken!=bTokenChars && sqlite3Fts5UnicodeIsdiacritic(iCode)==0 ){ int i; for(i=0; iiCode ) break; + if( (u32)aNew[i]>iCode ) break; } memmove(&aNew[i+1], &aNew[i], (nNew-i)*sizeof(int)); aNew[i] = iCode; @@ -194568,16 +257042,31 @@ static void fts5UnicodeDelete(Fts5Tokenizer *pTok){ return; } +static int unicodeSetCategories(Unicode61Tokenizer *p, const char *zCat){ + const char *z = zCat; + + while( *z ){ + while( *z==' ' || *z=='\t' ) z++; + if( *z && sqlite3Fts5UnicodeCatParse(z, p->aCategory) ){ + return SQLITE_ERROR; + } + while( *z!=' ' && *z!='\t' && *z!='\0' ) z++; + } + + sqlite3Fts5UnicodeAscii(p->aCategory, p->aTokenChar); + return SQLITE_OK; +} + /* ** Create a "unicode61" tokenizer. */ static int fts5UnicodeCreate( - void *pUnused, + void *pUnused, const char **azArg, int nArg, Fts5Tokenizer **ppOut ){ int rc = SQLITE_OK; /* Return code */ - Unicode61Tokenizer *p = 0; /* New tokenizer object */ + Unicode61Tokenizer *p = 0; /* New tokenizer object */ UNUSED_PARAM(pUnused); @@ -194586,28 +257075,48 @@ static int fts5UnicodeCreate( }else{ p = (Unicode61Tokenizer*)sqlite3_malloc(sizeof(Unicode61Tokenizer)); if( p ){ + const char *zCat = "L* N* Co"; int i; memset(p, 0, sizeof(Unicode61Tokenizer)); - memcpy(p->aTokenChar, aAsciiTokenChar, sizeof(aAsciiTokenChar)); - p->bRemoveDiacritic = 1; + + p->eRemoveDiacritic = FTS5_REMOVE_DIACRITICS_SIMPLE; p->nFold = 64; - p->aFold = sqlite3_malloc(p->nFold * sizeof(char)); + p->aFold = sqlite3_malloc64(p->nFold * sizeof(char)); if( p->aFold==0 ){ rc = SQLITE_NOMEM; } + + /* Search for a "categories" argument */ + for(i=0; rc==SQLITE_OK && ieRemoveDiacritic = (zArg[0] - '0'); + assert( p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_NONE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_SIMPLE + || p->eRemoveDiacritic==FTS5_REMOVE_DIACRITICS_COMPLEX + ); } - p->bRemoveDiacritic = (zArg[0]=='1'); }else if( 0==sqlite3_stricmp(azArg[i], "tokenchars") ){ rc = fts5UnicodeAddExceptions(p, zArg, 1); }else if( 0==sqlite3_stricmp(azArg[i], "separators") ){ rc = fts5UnicodeAddExceptions(p, zArg, 0); + }else + if( 0==sqlite3_stricmp(azArg[i], "categories") ){ + /* no-op */ }else{ rc = SQLITE_ERROR; } @@ -194626,12 +257135,14 @@ static int fts5UnicodeCreate( /* ** Return true if, for the purposes of tokenizing with the tokenizer -** passed as the first argument, codepoint iCode is considered a token +** passed as the first argument, codepoint iCode is considered a token ** character (not a separator). */ static int fts5UnicodeIsAlnum(Unicode61Tokenizer *p, int iCode){ - assert( (sqlite3Fts5UnicodeIsalnum(iCode) & 0xFFFFFFFE)==0 ); - return sqlite3Fts5UnicodeIsalnum(iCode) ^ fts5UnicodeIsException(p, iCode); + return ( + p->aCategory[sqlite3Fts5UnicodeCategory((u32)iCode)] + ^ fts5UnicodeIsException(p, iCode) + ); } static int fts5UnicodeTokenize( @@ -194658,7 +257169,7 @@ static int fts5UnicodeTokenize( /* Each iteration of this loop gobbles up a contiguous run of separators, ** then the next token. */ while( rc==SQLITE_OK ){ - int iCode; /* non-ASCII codepoint read from input */ + u32 iCode; /* non-ASCII codepoint read from input */ char *zOut = aFold; int is; int ie; @@ -194690,7 +257201,7 @@ static int fts5UnicodeTokenize( /* Grow the output buffer so that there is sufficient space to fit the ** largest possible utf-8 character. */ if( zOut>pEnd ){ - aFold = sqlite3_malloc(nFold*2); + aFold = sqlite3_malloc64((sqlite3_int64)nFold*2); if( aFold==0 ){ rc = SQLITE_NOMEM; goto tokenize_done; @@ -194709,14 +257220,14 @@ static int fts5UnicodeTokenize( READ_UTF8(zCsr, zTerm, iCode); if( fts5UnicodeIsAlnum(p,iCode)||sqlite3Fts5UnicodeIsdiacritic(iCode) ){ non_ascii_tokenchar: - iCode = sqlite3Fts5UnicodeFold(iCode, p->bRemoveDiacritic); + iCode = sqlite3Fts5UnicodeFold(iCode, p->eRemoveDiacritic); if( iCode ) WRITE_UTF8(zOut, iCode); }else{ break; } }else if( a[*zCsr]==0 ){ /* An ascii-range separator character. End of token. */ - break; + break; }else{ ascii_tokenchar: if( *zCsr>='A' && *zCsr<='Z' ){ @@ -194730,9 +257241,9 @@ static int fts5UnicodeTokenize( } /* Invoke the token callback */ - rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); + rc = xToken(pCtx, 0, aFold, zOut-aFold, is, ie); } - + tokenize_done: if( rc==SQLITE_DONE ) rc = SQLITE_OK; return rc; @@ -194748,7 +257259,7 @@ static int fts5UnicodeTokenize( typedef struct PorterTokenizer PorterTokenizer; struct PorterTokenizer { - fts5_tokenizer tokenizer; /* Parent tokenizer module */ + fts5_tokenizer_v2 tokenizer_v2; /* Parent tokenizer module */ Fts5Tokenizer *pTokenizer; /* Parent tokenizer instance */ char aBuf[FTS5_PORTER_MAX_TOKEN + 64]; }; @@ -194760,7 +257271,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){ if( pTok ){ PorterTokenizer *p = (PorterTokenizer*)pTok; if( p->pTokenizer ){ - p->tokenizer.xDelete(p->pTokenizer); + p->tokenizer_v2.xDelete(p->pTokenizer); } sqlite3_free(p); } @@ -194770,7 +257281,7 @@ static void fts5PorterDelete(Fts5Tokenizer *pTok){ ** Create a "porter" tokenizer. */ static int fts5PorterCreate( - void *pCtx, + void *pCtx, const char **azArg, int nArg, Fts5Tokenizer **ppOut ){ @@ -194779,6 +257290,7 @@ static int fts5PorterCreate( PorterTokenizer *pRet; void *pUserdata = 0; const char *zBase = "unicode61"; + fts5_tokenizer_v2 *pV2 = 0; if( nArg>0 ){ zBase = azArg[0]; @@ -194787,14 +257299,15 @@ static int fts5PorterCreate( pRet = (PorterTokenizer*)sqlite3_malloc(sizeof(PorterTokenizer)); if( pRet ){ memset(pRet, 0, sizeof(PorterTokenizer)); - rc = pApi->xFindTokenizer(pApi, zBase, &pUserdata, &pRet->tokenizer); + rc = pApi->xFindTokenizer_v2(pApi, zBase, &pUserdata, &pV2); }else{ rc = SQLITE_NOMEM; } if( rc==SQLITE_OK ){ int nArg2 = (nArg>0 ? nArg-1 : 0); - const char **azArg2 = (nArg2 ? &azArg[1] : 0); - rc = pRet->tokenizer.xCreate(pUserdata, azArg2, nArg2, &pRet->pTokenizer); + const char **az2 = (nArg2 ? &azArg[1] : 0); + memcpy(&pRet->tokenizer_v2, pV2, sizeof(fts5_tokenizer_v2)); + rc = pRet->tokenizer_v2.xCreate(pUserdata, az2, nArg2, &pRet->pTokenizer); } if( rc!=SQLITE_OK ){ @@ -194914,7 +257427,7 @@ static int fts5Porter_Ostar(char *zStem, int nStem){ /* porter rule condition: (m > 1 and (*S or *T)) */ static int fts5Porter_MGt1_and_S_or_T(char *zStem, int nStem){ assert( nStem>0 ); - return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t') + return (zStem[nStem-1]=='s' || zStem[nStem-1]=='t') && fts5Porter_MGt1(zStem, nStem); } @@ -194939,16 +257452,16 @@ static int fts5PorterStep4(char *aBuf, int *pnBuf){ int ret = 0; int nBuf = *pnBuf; switch( aBuf[nBuf-2] ){ - - case 'a': + + case 'a': if( nBuf>2 && 0==memcmp("al", &aBuf[nBuf-2], 2) ){ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ *pnBuf = nBuf - 2; } } break; - - case 'c': + + case 'c': if( nBuf>4 && 0==memcmp("ance", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ *pnBuf = nBuf - 4; @@ -194959,24 +257472,24 @@ static int fts5PorterStep4(char *aBuf, int *pnBuf){ } } break; - - case 'e': + + case 'e': if( nBuf>2 && 0==memcmp("er", &aBuf[nBuf-2], 2) ){ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ *pnBuf = nBuf - 2; } } break; - - case 'i': + + case 'i': if( nBuf>2 && 0==memcmp("ic", &aBuf[nBuf-2], 2) ){ if( fts5Porter_MGt1(aBuf, nBuf-2) ){ *pnBuf = nBuf - 2; } } break; - - case 'l': + + case 'l': if( nBuf>4 && 0==memcmp("able", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt1(aBuf, nBuf-4) ){ *pnBuf = nBuf - 4; @@ -194987,8 +257500,8 @@ static int fts5PorterStep4(char *aBuf, int *pnBuf){ } } break; - - case 'n': + + case 'n': if( nBuf>3 && 0==memcmp("ant", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; @@ -195007,8 +257520,8 @@ static int fts5PorterStep4(char *aBuf, int *pnBuf){ } } break; - - case 'o': + + case 'o': if( nBuf>3 && 0==memcmp("ion", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1_and_S_or_T(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; @@ -195019,16 +257532,16 @@ static int fts5PorterStep4(char *aBuf, int *pnBuf){ } } break; - - case 's': + + case 's': if( nBuf>3 && 0==memcmp("ism", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; } } break; - - case 't': + + case 't': if( nBuf>3 && 0==memcmp("ate", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; @@ -195039,76 +257552,76 @@ static int fts5PorterStep4(char *aBuf, int *pnBuf){ } } break; - - case 'u': + + case 'u': if( nBuf>3 && 0==memcmp("ous", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; } } break; - - case 'v': + + case 'v': if( nBuf>3 && 0==memcmp("ive", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; } } break; - - case 'z': + + case 'z': if( nBuf>3 && 0==memcmp("ize", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt1(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; } } break; - + } return ret; } - + static int fts5PorterStep1B2(char *aBuf, int *pnBuf){ int ret = 0; int nBuf = *pnBuf; switch( aBuf[nBuf-2] ){ - - case 'a': + + case 'a': if( nBuf>2 && 0==memcmp("at", &aBuf[nBuf-2], 2) ){ memcpy(&aBuf[nBuf-2], "ate", 3); *pnBuf = nBuf - 2 + 3; ret = 1; } break; - - case 'b': + + case 'b': if( nBuf>2 && 0==memcmp("bl", &aBuf[nBuf-2], 2) ){ memcpy(&aBuf[nBuf-2], "ble", 3); *pnBuf = nBuf - 2 + 3; ret = 1; } break; - - case 'i': + + case 'i': if( nBuf>2 && 0==memcmp("iz", &aBuf[nBuf-2], 2) ){ memcpy(&aBuf[nBuf-2], "ize", 3); *pnBuf = nBuf - 2 + 3; ret = 1; } break; - + } return ret; } - + static int fts5PorterStep2(char *aBuf, int *pnBuf){ int ret = 0; int nBuf = *pnBuf; switch( aBuf[nBuf-2] ){ - - case 'a': + + case 'a': if( nBuf>7 && 0==memcmp("ational", &aBuf[nBuf-7], 7) ){ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ memcpy(&aBuf[nBuf-7], "ate", 3); @@ -195121,8 +257634,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 'c': + + case 'c': if( nBuf>4 && 0==memcmp("enci", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ memcpy(&aBuf[nBuf-4], "ence", 4); @@ -195135,8 +257648,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 'e': + + case 'e': if( nBuf>4 && 0==memcmp("izer", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ memcpy(&aBuf[nBuf-4], "ize", 3); @@ -195144,8 +257657,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 'g': + + case 'g': if( nBuf>4 && 0==memcmp("logi", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ memcpy(&aBuf[nBuf-4], "log", 3); @@ -195153,8 +257666,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 'l': + + case 'l': if( nBuf>3 && 0==memcmp("bli", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ memcpy(&aBuf[nBuf-3], "ble", 3); @@ -195182,8 +257695,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 'o': + + case 'o': if( nBuf>7 && 0==memcmp("ization", &aBuf[nBuf-7], 7) ){ if( fts5Porter_MGt0(aBuf, nBuf-7) ){ memcpy(&aBuf[nBuf-7], "ize", 3); @@ -195201,8 +257714,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 's': + + case 's': if( nBuf>5 && 0==memcmp("alism", &aBuf[nBuf-5], 5) ){ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ memcpy(&aBuf[nBuf-5], "al", 2); @@ -195225,8 +257738,8 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - - case 't': + + case 't': if( nBuf>5 && 0==memcmp("aliti", &aBuf[nBuf-5], 5) ){ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ memcpy(&aBuf[nBuf-5], "al", 2); @@ -195244,18 +257757,18 @@ static int fts5PorterStep2(char *aBuf, int *pnBuf){ } } break; - + } return ret; } - + static int fts5PorterStep3(char *aBuf, int *pnBuf){ int ret = 0; int nBuf = *pnBuf; switch( aBuf[nBuf-2] ){ - - case 'a': + + case 'a': if( nBuf>4 && 0==memcmp("ical", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ memcpy(&aBuf[nBuf-4], "ic", 2); @@ -195263,16 +257776,16 @@ static int fts5PorterStep3(char *aBuf, int *pnBuf){ } } break; - - case 's': + + case 's': if( nBuf>4 && 0==memcmp("ness", &aBuf[nBuf-4], 4) ){ if( fts5Porter_MGt0(aBuf, nBuf-4) ){ *pnBuf = nBuf - 4; } } break; - - case 't': + + case 't': if( nBuf>5 && 0==memcmp("icate", &aBuf[nBuf-5], 5) ){ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ memcpy(&aBuf[nBuf-5], "ic", 2); @@ -195285,24 +257798,24 @@ static int fts5PorterStep3(char *aBuf, int *pnBuf){ } } break; - - case 'u': + + case 'u': if( nBuf>3 && 0==memcmp("ful", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; } } break; - - case 'v': + + case 'v': if( nBuf>5 && 0==memcmp("ative", &aBuf[nBuf-5], 5) ){ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ *pnBuf = nBuf - 5; } } break; - - case 'z': + + case 'z': if( nBuf>5 && 0==memcmp("alize", &aBuf[nBuf-5], 5) ){ if( fts5Porter_MGt0(aBuf, nBuf-5) ){ memcpy(&aBuf[nBuf-5], "al", 2); @@ -195310,18 +257823,18 @@ static int fts5PorterStep3(char *aBuf, int *pnBuf){ } } break; - + } return ret; } - + static int fts5PorterStep1B(char *aBuf, int *pnBuf){ int ret = 0; int nBuf = *pnBuf; switch( aBuf[nBuf-2] ){ - - case 'e': + + case 'e': if( nBuf>3 && 0==memcmp("eed", &aBuf[nBuf-3], 3) ){ if( fts5Porter_MGt0(aBuf, nBuf-3) ){ memcpy(&aBuf[nBuf-3], "ee", 2); @@ -195334,8 +257847,8 @@ static int fts5PorterStep1B(char *aBuf, int *pnBuf){ } } break; - - case 'n': + + case 'n': if( nBuf>3 && 0==memcmp("ing", &aBuf[nBuf-3], 3) ){ if( fts5Porter_Vowel(aBuf, nBuf-3) ){ *pnBuf = nBuf - 3; @@ -195343,12 +257856,12 @@ static int fts5PorterStep1B(char *aBuf, int *pnBuf){ } } break; - + } return ret; } - -/* + +/* ** GENERATED CODE ENDS HERE (mkportersteps.tcl) *************************************************************************** **************************************************************************/ @@ -195357,7 +257870,7 @@ static void fts5PorterStep1A(char *aBuf, int *pnBuf){ int nBuf = *pnBuf; if( aBuf[nBuf-1]=='s' ){ if( aBuf[nBuf-2]=='e' ){ - if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s') + if( (nBuf>4 && aBuf[nBuf-4]=='s' && aBuf[nBuf-3]=='s') || (nBuf>3 && aBuf[nBuf-3]=='i' ) ){ *pnBuf = nBuf-2; @@ -195372,11 +257885,11 @@ static void fts5PorterStep1A(char *aBuf, int *pnBuf){ } static int fts5PorterCb( - void *pCtx, + void *pCtx, int tflags, - const char *pToken, - int nToken, - int iStart, + const char *pToken, + int nToken, + int iStart, int iEnd ){ PorterContext *p = (PorterContext*)pCtx; @@ -195394,8 +257907,8 @@ static int fts5PorterCb( if( fts5PorterStep1B(aBuf, &nBuf) ){ if( fts5PorterStep1B2(aBuf, &nBuf)==0 ){ char c = aBuf[nBuf-1]; - if( fts5PorterIsVowel(c, 0)==0 - && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] + if( fts5PorterIsVowel(c, 0)==0 + && c!='l' && c!='s' && c!='z' && c==aBuf[nBuf-2] ){ nBuf--; }else if( fts5Porter_MEq1(aBuf, nBuf) && fts5Porter_Ostar(aBuf, nBuf) ){ @@ -195417,7 +257930,7 @@ static int fts5PorterCb( /* Step 5a. */ assert( nBuf>0 ); if( aBuf[nBuf-1]=='e' ){ - if( fts5Porter_MGt1(aBuf, nBuf-1) + if( fts5Porter_MGt1(aBuf, nBuf-1) || (fts5Porter_MEq1(aBuf, nBuf-1) && !fts5Porter_Ostar(aBuf, nBuf-1)) ){ nBuf--; @@ -195425,8 +257938,8 @@ static int fts5PorterCb( } /* Step 5b. */ - if( nBuf>1 && aBuf[nBuf-1]=='l' - && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) + if( nBuf>1 && aBuf[nBuf-1]=='l' + && aBuf[nBuf-2]=='l' && fts5Porter_MGt1(aBuf, nBuf-1) ){ nBuf--; } @@ -195445,6 +257958,7 @@ static int fts5PorterTokenize( void *pCtx, int flags, const char *pText, int nText, + const char *pLoc, int nLoc, int (*xToken)(void*, int, const char*, int nToken, int iStart, int iEnd) ){ PorterTokenizer *p = (PorterTokenizer*)pTokenizer; @@ -195452,11 +257966,191 @@ static int fts5PorterTokenize( sCtx.xToken = xToken; sCtx.pCtx = pCtx; sCtx.aBuf = p->aBuf; - return p->tokenizer.xTokenize( - p->pTokenizer, (void*)&sCtx, flags, pText, nText, fts5PorterCb + return p->tokenizer_v2.xTokenize( + p->pTokenizer, (void*)&sCtx, flags, pText, nText, pLoc, nLoc, fts5PorterCb ); } +/************************************************************************** +** Start of trigram implementation. +*/ +typedef struct TrigramTokenizer TrigramTokenizer; +struct TrigramTokenizer { + int bFold; /* True to fold to lower-case */ + int iFoldParam; /* Parameter to pass to Fts5UnicodeFold() */ +}; + +/* +** Free a trigram tokenizer. +*/ +static void fts5TriDelete(Fts5Tokenizer *p){ + sqlite3_free(p); +} + +/* +** Allocate a trigram tokenizer. +*/ +static int fts5TriCreate( + void *pUnused, + const char **azArg, + int nArg, + Fts5Tokenizer **ppOut +){ + int rc = SQLITE_OK; + TrigramTokenizer *pNew = 0; + UNUSED_PARAM(pUnused); + if( nArg%2 ){ + rc = SQLITE_ERROR; + }else{ + int i; + pNew = (TrigramTokenizer*)sqlite3_malloc(sizeof(*pNew)); + if( pNew==0 ){ + rc = SQLITE_NOMEM; + }else{ + pNew->bFold = 1; + pNew->iFoldParam = 0; + + for(i=0; rc==SQLITE_OK && ibFold = (zArg[0]=='0'); + } + }else if( 0==sqlite3_stricmp(azArg[i], "remove_diacritics") ){ + if( (zArg[0]!='0' && zArg[0]!='1' && zArg[0]!='2') || zArg[1] ){ + rc = SQLITE_ERROR; + }else{ + pNew->iFoldParam = (zArg[0]!='0') ? 2 : 0; + } + }else{ + rc = SQLITE_ERROR; + } + } + + if( pNew->iFoldParam!=0 && pNew->bFold==0 ){ + rc = SQLITE_ERROR; + } + + if( rc!=SQLITE_OK ){ + fts5TriDelete((Fts5Tokenizer*)pNew); + pNew = 0; + } + } + } + *ppOut = (Fts5Tokenizer*)pNew; + return rc; +} + +/* +** Trigram tokenizer tokenize routine. +*/ +static int fts5TriTokenize( + Fts5Tokenizer *pTok, + void *pCtx, + int unusedFlags, + const char *pText, int nText, + int (*xToken)(void*, int, const char*, int, int, int) +){ + TrigramTokenizer *p = (TrigramTokenizer*)pTok; + int rc = SQLITE_OK; + char aBuf[32]; + char *zOut = aBuf; + int ii; + const unsigned char *zIn = (const unsigned char*)pText; + const unsigned char *zEof = &zIn[nText]; + u32 iCode; + int aStart[3]; /* Input offset of each character in aBuf[] */ + + UNUSED_PARAM(unusedFlags); + + /* Populate aBuf[] with the characters for the first trigram. */ + for(ii=0; ii<3; ii++){ + do { + aStart[ii] = zIn - (const unsigned char*)pText; + READ_UTF8(zIn, zEof, iCode); + if( iCode==0 ) return SQLITE_OK; + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + WRITE_UTF8(zOut, iCode); + } + + /* At the start of each iteration of this loop: + ** + ** aBuf: Contains 3 characters. The 3 characters of the next trigram. + ** zOut: Points to the byte following the last character in aBuf. + ** aStart[3]: Contains the byte offset in the input text corresponding + ** to the start of each of the three characters in the buffer. + */ + assert( zIn<=zEof ); + while( 1 ){ + int iNext; /* Start of character following current tri */ + const char *z1; + + /* Read characters from the input up until the first non-diacritic */ + do { + iNext = zIn - (const unsigned char*)pText; + READ_UTF8(zIn, zEof, iCode); + if( iCode==0 ) break; + if( p->bFold ) iCode = sqlite3Fts5UnicodeFold(iCode, p->iFoldParam); + }while( iCode==0 ); + + /* Pass the current trigram back to fts5 */ + rc = xToken(pCtx, 0, aBuf, zOut-aBuf, aStart[0], iNext); + if( iCode==0 || rc!=SQLITE_OK ) break; + + /* Remove the first character from buffer aBuf[]. Append the character + ** with codepoint iCode. */ + z1 = aBuf; + FTS5_SKIP_UTF8(z1); + memmove(aBuf, z1, zOut - z1); + zOut -= (z1 - aBuf); + WRITE_UTF8(zOut, iCode); + + /* Update the aStart[] array */ + aStart[0] = aStart[1]; + aStart[1] = aStart[2]; + aStart[2] = iNext; + } + + return rc; +} + +/* +** Argument xCreate is a pointer to a constructor function for a tokenizer. +** pTok is a tokenizer previously created using the same method. This function +** returns one of FTS5_PATTERN_NONE, FTS5_PATTERN_LIKE or FTS5_PATTERN_GLOB +** indicating the style of pattern matching that the tokenizer can support. +** In practice, this is: +** +** "trigram" tokenizer, case_sensitive=1 - FTS5_PATTERN_GLOB +** "trigram" tokenizer, case_sensitive=0 (the default) - FTS5_PATTERN_LIKE +** all other tokenizers - FTS5_PATTERN_NONE +*/ +static int sqlite3Fts5TokenizerPattern( + int (*xCreate)(void*, const char**, int, Fts5Tokenizer**), + Fts5Tokenizer *pTok +){ + if( xCreate==fts5TriCreate ){ + TrigramTokenizer *p = (TrigramTokenizer*)pTok; + if( p->iFoldParam==0 ){ + return p->bFold ? FTS5_PATTERN_LIKE : FTS5_PATTERN_GLOB; + } + } + return FTS5_PATTERN_NONE; +} + +/* +** Return true if the tokenizer described by p->azArg[] is the trigram +** tokenizer. This tokenizer needs to be loaded before xBestIndex is +** called for the first time in order to correctly handle LIKE/GLOB. +*/ +static int sqlite3Fts5TokenizerPreload(Fts5TokenizerConfig *p){ + return (p->nArg>=1 && 0==sqlite3_stricmp(p->azArg[0], "trigram")); +} + + /* ** Register all built-in tokenizers with FTS5. */ @@ -195467,9 +258161,9 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ } aBuiltin[] = { { "unicode61", {fts5UnicodeCreate, fts5UnicodeDelete, fts5UnicodeTokenize}}, { "ascii", {fts5AsciiCreate, fts5AsciiDelete, fts5AsciiTokenize }}, - { "porter", {fts5PorterCreate, fts5PorterDelete, fts5PorterTokenize }}, + { "trigram", {fts5TriCreate, fts5TriDelete, fts5TriTokenize}}, }; - + int rc = SQLITE_OK; /* Return code */ int i; /* To iterate through builtin functions */ @@ -195481,14 +258175,25 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ 0 ); } - + if( rc==SQLITE_OK ){ + fts5_tokenizer_v2 sPorter = { + 2, + fts5PorterCreate, + fts5PorterDelete, + fts5PorterTokenize + }; + rc = pApi->xCreateTokenizer_v2(pApi, + "porter", + (void*)pApi, + &sPorter, + 0 + ); + } return rc; } - - /* -** 2012 May 25 +** 2012-05-25 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -195507,135 +258212,6 @@ static int sqlite3Fts5TokenizerInit(fts5_api *pApi){ /* #include */ -/* -** Return true if the argument corresponds to a unicode codepoint -** classified as either a letter or a number. Otherwise false. -** -** The results are undefined if the value passed to this function -** is less than zero. -*/ -static int sqlite3Fts5UnicodeIsalnum(int c){ - /* Each unsigned integer in the following array corresponds to a contiguous - ** range of unicode codepoints that are not either letters or numbers (i.e. - ** codepoints for which this function should return 0). - ** - ** The most significant 22 bits in each 32-bit value contain the first - ** codepoint in the range. The least significant 10 bits are used to store - ** the size of the range (always at least 1). In other words, the value - ** ((C<<22) + N) represents a range of N codepoints starting with codepoint - ** C. It is not possible to represent a range larger than 1023 codepoints - ** using this format. - */ - static const unsigned int aEntry[] = { - 0x00000030, 0x0000E807, 0x00016C06, 0x0001EC2F, 0x0002AC07, - 0x0002D001, 0x0002D803, 0x0002EC01, 0x0002FC01, 0x00035C01, - 0x0003DC01, 0x000B0804, 0x000B480E, 0x000B9407, 0x000BB401, - 0x000BBC81, 0x000DD401, 0x000DF801, 0x000E1002, 0x000E1C01, - 0x000FD801, 0x00120808, 0x00156806, 0x00162402, 0x00163C01, - 0x00164437, 0x0017CC02, 0x00180005, 0x00181816, 0x00187802, - 0x00192C15, 0x0019A804, 0x0019C001, 0x001B5001, 0x001B580F, - 0x001B9C07, 0x001BF402, 0x001C000E, 0x001C3C01, 0x001C4401, - 0x001CC01B, 0x001E980B, 0x001FAC09, 0x001FD804, 0x00205804, - 0x00206C09, 0x00209403, 0x0020A405, 0x0020C00F, 0x00216403, - 0x00217801, 0x0023901B, 0x00240004, 0x0024E803, 0x0024F812, - 0x00254407, 0x00258804, 0x0025C001, 0x00260403, 0x0026F001, - 0x0026F807, 0x00271C02, 0x00272C03, 0x00275C01, 0x00278802, - 0x0027C802, 0x0027E802, 0x00280403, 0x0028F001, 0x0028F805, - 0x00291C02, 0x00292C03, 0x00294401, 0x0029C002, 0x0029D401, - 0x002A0403, 0x002AF001, 0x002AF808, 0x002B1C03, 0x002B2C03, - 0x002B8802, 0x002BC002, 0x002C0403, 0x002CF001, 0x002CF807, - 0x002D1C02, 0x002D2C03, 0x002D5802, 0x002D8802, 0x002DC001, - 0x002E0801, 0x002EF805, 0x002F1803, 0x002F2804, 0x002F5C01, - 0x002FCC08, 0x00300403, 0x0030F807, 0x00311803, 0x00312804, - 0x00315402, 0x00318802, 0x0031FC01, 0x00320802, 0x0032F001, - 0x0032F807, 0x00331803, 0x00332804, 0x00335402, 0x00338802, - 0x00340802, 0x0034F807, 0x00351803, 0x00352804, 0x00355C01, - 0x00358802, 0x0035E401, 0x00360802, 0x00372801, 0x00373C06, - 0x00375801, 0x00376008, 0x0037C803, 0x0038C401, 0x0038D007, - 0x0038FC01, 0x00391C09, 0x00396802, 0x003AC401, 0x003AD006, - 0x003AEC02, 0x003B2006, 0x003C041F, 0x003CD00C, 0x003DC417, - 0x003E340B, 0x003E6424, 0x003EF80F, 0x003F380D, 0x0040AC14, - 0x00412806, 0x00415804, 0x00417803, 0x00418803, 0x00419C07, - 0x0041C404, 0x0042080C, 0x00423C01, 0x00426806, 0x0043EC01, - 0x004D740C, 0x004E400A, 0x00500001, 0x0059B402, 0x005A0001, - 0x005A6C02, 0x005BAC03, 0x005C4803, 0x005CC805, 0x005D4802, - 0x005DC802, 0x005ED023, 0x005F6004, 0x005F7401, 0x0060000F, - 0x0062A401, 0x0064800C, 0x0064C00C, 0x00650001, 0x00651002, - 0x0066C011, 0x00672002, 0x00677822, 0x00685C05, 0x00687802, - 0x0069540A, 0x0069801D, 0x0069FC01, 0x006A8007, 0x006AA006, - 0x006C0005, 0x006CD011, 0x006D6823, 0x006E0003, 0x006E840D, - 0x006F980E, 0x006FF004, 0x00709014, 0x0070EC05, 0x0071F802, - 0x00730008, 0x00734019, 0x0073B401, 0x0073C803, 0x00770027, - 0x0077F004, 0x007EF401, 0x007EFC03, 0x007F3403, 0x007F7403, - 0x007FB403, 0x007FF402, 0x00800065, 0x0081A806, 0x0081E805, - 0x00822805, 0x0082801A, 0x00834021, 0x00840002, 0x00840C04, - 0x00842002, 0x00845001, 0x00845803, 0x00847806, 0x00849401, - 0x00849C01, 0x0084A401, 0x0084B801, 0x0084E802, 0x00850005, - 0x00852804, 0x00853C01, 0x00864264, 0x00900027, 0x0091000B, - 0x0092704E, 0x00940200, 0x009C0475, 0x009E53B9, 0x00AD400A, - 0x00B39406, 0x00B3BC03, 0x00B3E404, 0x00B3F802, 0x00B5C001, - 0x00B5FC01, 0x00B7804F, 0x00B8C00C, 0x00BA001A, 0x00BA6C59, - 0x00BC00D6, 0x00BFC00C, 0x00C00005, 0x00C02019, 0x00C0A807, - 0x00C0D802, 0x00C0F403, 0x00C26404, 0x00C28001, 0x00C3EC01, - 0x00C64002, 0x00C6580A, 0x00C70024, 0x00C8001F, 0x00C8A81E, - 0x00C94001, 0x00C98020, 0x00CA2827, 0x00CB003F, 0x00CC0100, - 0x01370040, 0x02924037, 0x0293F802, 0x02983403, 0x0299BC10, - 0x029A7C01, 0x029BC008, 0x029C0017, 0x029C8002, 0x029E2402, - 0x02A00801, 0x02A01801, 0x02A02C01, 0x02A08C09, 0x02A0D804, - 0x02A1D004, 0x02A20002, 0x02A2D011, 0x02A33802, 0x02A38012, - 0x02A3E003, 0x02A4980A, 0x02A51C0D, 0x02A57C01, 0x02A60004, - 0x02A6CC1B, 0x02A77802, 0x02A8A40E, 0x02A90C01, 0x02A93002, - 0x02A97004, 0x02A9DC03, 0x02A9EC01, 0x02AAC001, 0x02AAC803, - 0x02AADC02, 0x02AAF802, 0x02AB0401, 0x02AB7802, 0x02ABAC07, - 0x02ABD402, 0x02AF8C0B, 0x03600001, 0x036DFC02, 0x036FFC02, - 0x037FFC01, 0x03EC7801, 0x03ECA401, 0x03EEC810, 0x03F4F802, - 0x03F7F002, 0x03F8001A, 0x03F88007, 0x03F8C023, 0x03F95013, - 0x03F9A004, 0x03FBFC01, 0x03FC040F, 0x03FC6807, 0x03FCEC06, - 0x03FD6C0B, 0x03FF8007, 0x03FFA007, 0x03FFE405, 0x04040003, - 0x0404DC09, 0x0405E411, 0x0406400C, 0x0407402E, 0x040E7C01, - 0x040F4001, 0x04215C01, 0x04247C01, 0x0424FC01, 0x04280403, - 0x04281402, 0x04283004, 0x0428E003, 0x0428FC01, 0x04294009, - 0x0429FC01, 0x042CE407, 0x04400003, 0x0440E016, 0x04420003, - 0x0442C012, 0x04440003, 0x04449C0E, 0x04450004, 0x04460003, - 0x0446CC0E, 0x04471404, 0x045AAC0D, 0x0491C004, 0x05BD442E, - 0x05BE3C04, 0x074000F6, 0x07440027, 0x0744A4B5, 0x07480046, - 0x074C0057, 0x075B0401, 0x075B6C01, 0x075BEC01, 0x075C5401, - 0x075CD401, 0x075D3C01, 0x075DBC01, 0x075E2401, 0x075EA401, - 0x075F0C01, 0x07BBC002, 0x07C0002C, 0x07C0C064, 0x07C2800F, - 0x07C2C40E, 0x07C3040F, 0x07C3440F, 0x07C4401F, 0x07C4C03C, - 0x07C5C02B, 0x07C7981D, 0x07C8402B, 0x07C90009, 0x07C94002, - 0x07CC0021, 0x07CCC006, 0x07CCDC46, 0x07CE0014, 0x07CE8025, - 0x07CF1805, 0x07CF8011, 0x07D0003F, 0x07D10001, 0x07D108B6, - 0x07D3E404, 0x07D4003E, 0x07D50004, 0x07D54018, 0x07D7EC46, - 0x07D9140B, 0x07DA0046, 0x07DC0074, 0x38000401, 0x38008060, - 0x380400F0, - }; - static const unsigned int aAscii[4] = { - 0xFFFFFFFF, 0xFC00FFFF, 0xF8000001, 0xF8000001, - }; - - if( (unsigned int)c<128 ){ - return ( (aAscii[c >> 5] & (1 << (c & 0x001F)))==0 ); - }else if( (unsigned int)c<(1<<22) ){ - unsigned int key = (((unsigned int)c)<<10) | 0x000003FF; - int iRes = 0; - int iHi = sizeof(aEntry)/sizeof(aEntry[0]) - 1; - int iLo = 0; - while( iHi>=iLo ){ - int iTest = (iHi + iLo) / 2; - if( key >= aEntry[iTest] ){ - iRes = iTest; - iLo = iTest+1; - }else{ - iHi = iTest-1; - } - } - assert( aEntry[0]=aEntry[iRes] ); - return (((unsigned int)c) >= ((aEntry[iRes]>>10) + (aEntry[iRes]&0x3FF))); - } - return 1; -} /* @@ -195646,32 +258222,48 @@ static int sqlite3Fts5UnicodeIsalnum(int c){ ** E"). The resuls of passing a codepoint that corresponds to an ** uppercase letter are undefined. */ -static int fts5_remove_diacritic(int c){ +static int fts5_remove_diacritic(int c, int bComplex){ unsigned short aDia[] = { - 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, - 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, - 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, - 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, - 3456, 3696, 3712, 3728, 3744, 3896, 3912, 3928, - 3968, 4008, 4040, 4106, 4138, 4170, 4202, 4234, - 4266, 4296, 4312, 4344, 4408, 4424, 4472, 4504, - 6148, 6198, 6264, 6280, 6360, 6429, 6505, 6529, - 61448, 61468, 61534, 61592, 61642, 61688, 61704, 61726, - 61784, 61800, 61836, 61880, 61914, 61948, 61998, 62122, - 62154, 62200, 62218, 62302, 62364, 62442, 62478, 62536, - 62554, 62584, 62604, 62640, 62648, 62656, 62664, 62730, - 62924, 63050, 63082, 63274, 63390, + 0, 1797, 1848, 1859, 1891, 1928, 1940, 1995, + 2024, 2040, 2060, 2110, 2168, 2206, 2264, 2286, + 2344, 2383, 2472, 2488, 2516, 2596, 2668, 2732, + 2782, 2842, 2894, 2954, 2984, 3000, 3028, 3336, + 3456, 3696, 3712, 3728, 3744, 3766, 3832, 3896, + 3912, 3928, 3944, 3968, 4008, 4040, 4056, 4106, + 4138, 4170, 4202, 4234, 4266, 4296, 4312, 4344, + 4408, 4424, 4442, 4472, 4488, 4504, 6148, 6198, + 6264, 6280, 6360, 6429, 6505, 6529, 61448, 61468, + 61512, 61534, 61592, 61610, 61642, 61672, 61688, 61704, + 61726, 61784, 61800, 61816, 61836, 61880, 61896, 61914, + 61948, 61998, 62062, 62122, 62154, 62184, 62200, 62218, + 62252, 62302, 62364, 62410, 62442, 62478, 62536, 62554, + 62584, 62604, 62640, 62648, 62656, 62664, 62730, 62766, + 62830, 62890, 62924, 62974, 63032, 63050, 63082, 63118, + 63182, 63242, 63274, 63310, 63368, 63390, }; - char aChar[] = { - '\0', 'a', 'c', 'e', 'i', 'n', 'o', 'u', 'y', 'y', 'a', 'c', - 'd', 'e', 'e', 'g', 'h', 'i', 'j', 'k', 'l', 'n', 'o', 'r', - 's', 't', 'u', 'u', 'w', 'y', 'z', 'o', 'u', 'a', 'i', 'o', - 'u', 'g', 'k', 'o', 'j', 'g', 'n', 'a', 'e', 'i', 'o', 'r', - 'u', 's', 't', 'h', 'a', 'e', 'o', 'y', '\0', '\0', '\0', '\0', - '\0', '\0', '\0', '\0', 'a', 'b', 'd', 'd', 'e', 'f', 'g', 'h', - 'h', 'i', 'k', 'l', 'l', 'm', 'n', 'p', 'r', 'r', 's', 't', - 'u', 'v', 'w', 'w', 'x', 'y', 'z', 'h', 't', 'w', 'y', 'a', - 'e', 'i', 'o', 'u', 'y', +#define HIBIT ((unsigned char)0x80) + unsigned char aChar[] = { + '\0', 'a', 'c', 'e', 'i', 'n', + 'o', 'u', 'y', 'y', 'a', 'c', + 'd', 'e', 'e', 'g', 'h', 'i', + 'j', 'k', 'l', 'n', 'o', 'r', + 's', 't', 'u', 'u', 'w', 'y', + 'z', 'o', 'u', 'a', 'i', 'o', + 'u', 'u'|HIBIT, 'a'|HIBIT, 'g', 'k', 'o', + 'o'|HIBIT, 'j', 'g', 'n', 'a'|HIBIT, 'a', + 'e', 'i', 'o', 'r', 'u', 's', + 't', 'h', 'a', 'e', 'o'|HIBIT, 'o', + 'o'|HIBIT, 'y', '\0', '\0', '\0', '\0', + '\0', '\0', '\0', '\0', 'a', 'b', + 'c'|HIBIT, 'd', 'd', 'e'|HIBIT, 'e', 'e'|HIBIT, + 'f', 'g', 'h', 'h', 'i', 'i'|HIBIT, + 'k', 'l', 'l'|HIBIT, 'l', 'm', 'n', + 'o'|HIBIT, 'p', 'r', 'r'|HIBIT, 'r', 's', + 's'|HIBIT, 't', 'u', 'u'|HIBIT, 'v', 'w', + 'w', 'x', 'y', 'z', 'h', 't', + 'w', 'y', 'a', 'a'|HIBIT, 'a'|HIBIT, 'a'|HIBIT, + 'e', 'e'|HIBIT, 'e'|HIBIT, 'i', 'o', 'o'|HIBIT, + 'o'|HIBIT, 'o'|HIBIT, 'u', 'u'|HIBIT, 'u'|HIBIT, 'y', }; unsigned int key = (((unsigned int)c)<<3) | 0x00000007; @@ -195688,7 +258280,8 @@ static int fts5_remove_diacritic(int c){ } } assert( key>=aDia[iRes] ); - return ((c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : (int)aChar[iRes]); + if( bComplex==0 && (aChar[iRes] & 0x80) ) return c; + return (c > (aDia[iRes]>>3) + (aDia[iRes]&0x07)) ? c : ((int)aChar[iRes] & 0x7F); } @@ -195701,8 +258294,8 @@ static int sqlite3Fts5UnicodeIsdiacritic(int c){ unsigned int mask1 = 0x000361F8; if( c<768 || c>817 ) return 0; return (c < 768+32) ? - (mask0 & (1 << (c-768))) : - (mask1 & (1 << (c-768-32))); + (mask0 & ((unsigned int)1 << (c-768))) : + (mask1 & ((unsigned int)1 << (c-768-32))); } @@ -195715,7 +258308,7 @@ static int sqlite3Fts5UnicodeIsdiacritic(int c){ ** The results are undefined if the value passed to this function ** is less than zero. */ -static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ +static int sqlite3Fts5UnicodeFold(int c, int eRemoveDiacritic){ /* Each entry in the following array defines a rule for folding a range ** of codepoints to lower case. The rule applies to a range of nRange ** codepoints starting at codepoint iCode. @@ -195792,19 +258385,19 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ {42802, 1, 62}, {42873, 1, 4}, {42877, 76, 1}, {42878, 1, 10}, {42891, 0, 1}, {42893, 74, 1}, {42896, 1, 4}, {42912, 1, 10}, {42922, 72, 1}, - {65313, 14, 26}, + {65313, 14, 26}, }; static const unsigned short aiOff[] = { - 1, 2, 8, 15, 16, 26, 28, 32, - 37, 38, 40, 48, 63, 64, 69, 71, - 79, 80, 116, 202, 203, 205, 206, 207, - 209, 210, 211, 213, 214, 217, 218, 219, - 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, - 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, - 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, - 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, - 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, - 65514, 65521, 65527, 65528, 65529, + 1, 2, 8, 15, 16, 26, 28, 32, + 37, 38, 40, 48, 63, 64, 69, 71, + 79, 80, 116, 202, 203, 205, 206, 207, + 209, 210, 211, 213, 214, 217, 218, 219, + 775, 7264, 10792, 10795, 23228, 23256, 30204, 54721, + 54753, 54754, 54756, 54787, 54793, 54809, 57153, 57274, + 57921, 58019, 58363, 61722, 65268, 65341, 65373, 65406, + 65408, 65410, 65415, 65424, 65436, 65439, 65450, 65462, + 65472, 65476, 65478, 65480, 65482, 65488, 65506, 65511, + 65514, 65521, 65527, 65528, 65529, }; int ret = c; @@ -195838,9 +258431,11 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ assert( ret>0 ); } - if( bRemoveDiacritic ) ret = fts5_remove_diacritic(ret); + if( eRemoveDiacritic ){ + ret = fts5_remove_diacritic(ret, eRemoveDiacritic==2); + } } - + else if( c>=66560 && c<66600 ){ ret = c + 40; } @@ -195848,6 +258443,537 @@ static int sqlite3Fts5UnicodeFold(int c, int bRemoveDiacritic){ return ret; } + +static int sqlite3Fts5UnicodeCatParse(const char *zCat, u8 *aArray){ + aArray[0] = 1; + switch( zCat[0] ){ + case 'C': + switch( zCat[1] ){ + case 'c': aArray[1] = 1; break; + case 'f': aArray[2] = 1; break; + case 'n': aArray[3] = 1; break; + case 's': aArray[4] = 1; break; + case 'o': aArray[31] = 1; break; + case '*': + aArray[1] = 1; + aArray[2] = 1; + aArray[3] = 1; + aArray[4] = 1; + aArray[31] = 1; + break; + default: return 1; } + break; + + case 'L': + switch( zCat[1] ){ + case 'l': aArray[5] = 1; break; + case 'm': aArray[6] = 1; break; + case 'o': aArray[7] = 1; break; + case 't': aArray[8] = 1; break; + case 'u': aArray[9] = 1; break; + case 'C': aArray[30] = 1; break; + case '*': + aArray[5] = 1; + aArray[6] = 1; + aArray[7] = 1; + aArray[8] = 1; + aArray[9] = 1; + aArray[30] = 1; + break; + default: return 1; } + break; + + case 'M': + switch( zCat[1] ){ + case 'c': aArray[10] = 1; break; + case 'e': aArray[11] = 1; break; + case 'n': aArray[12] = 1; break; + case '*': + aArray[10] = 1; + aArray[11] = 1; + aArray[12] = 1; + break; + default: return 1; } + break; + + case 'N': + switch( zCat[1] ){ + case 'd': aArray[13] = 1; break; + case 'l': aArray[14] = 1; break; + case 'o': aArray[15] = 1; break; + case '*': + aArray[13] = 1; + aArray[14] = 1; + aArray[15] = 1; + break; + default: return 1; } + break; + + case 'P': + switch( zCat[1] ){ + case 'c': aArray[16] = 1; break; + case 'd': aArray[17] = 1; break; + case 'e': aArray[18] = 1; break; + case 'f': aArray[19] = 1; break; + case 'i': aArray[20] = 1; break; + case 'o': aArray[21] = 1; break; + case 's': aArray[22] = 1; break; + case '*': + aArray[16] = 1; + aArray[17] = 1; + aArray[18] = 1; + aArray[19] = 1; + aArray[20] = 1; + aArray[21] = 1; + aArray[22] = 1; + break; + default: return 1; } + break; + + case 'S': + switch( zCat[1] ){ + case 'c': aArray[23] = 1; break; + case 'k': aArray[24] = 1; break; + case 'm': aArray[25] = 1; break; + case 'o': aArray[26] = 1; break; + case '*': + aArray[23] = 1; + aArray[24] = 1; + aArray[25] = 1; + aArray[26] = 1; + break; + default: return 1; } + break; + + case 'Z': + switch( zCat[1] ){ + case 'l': aArray[27] = 1; break; + case 'p': aArray[28] = 1; break; + case 's': aArray[29] = 1; break; + case '*': + aArray[27] = 1; + aArray[28] = 1; + aArray[29] = 1; + break; + default: return 1; } + break; + + + default: + return 1; + } + return 0; +} + +static u16 aFts5UnicodeBlock[] = { + 0, 1471, 1753, 1760, 1760, 1760, 1760, 1760, 1760, 1760, + 1760, 1760, 1760, 1760, 1760, 1763, 1765, + }; +static u16 aFts5UnicodeMap[] = { + 0, 32, 33, 36, 37, 40, 41, 42, 43, 44, + 45, 46, 48, 58, 60, 63, 65, 91, 92, 93, + 94, 95, 96, 97, 123, 124, 125, 126, 127, 160, + 161, 162, 166, 167, 168, 169, 170, 171, 172, 173, + 174, 175, 176, 177, 178, 180, 181, 182, 184, 185, + 186, 187, 188, 191, 192, 215, 216, 223, 247, 248, + 256, 312, 313, 329, 330, 377, 383, 385, 387, 388, + 391, 394, 396, 398, 402, 403, 405, 406, 409, 412, + 414, 415, 417, 418, 423, 427, 428, 431, 434, 436, + 437, 440, 442, 443, 444, 446, 448, 452, 453, 454, + 455, 456, 457, 458, 459, 460, 461, 477, 478, 496, + 497, 498, 499, 500, 503, 505, 506, 564, 570, 572, + 573, 575, 577, 580, 583, 584, 592, 660, 661, 688, + 706, 710, 722, 736, 741, 748, 749, 750, 751, 768, + 880, 884, 885, 886, 890, 891, 894, 900, 902, 903, + 904, 908, 910, 912, 913, 931, 940, 975, 977, 978, + 981, 984, 1008, 1012, 1014, 1015, 1018, 1020, 1021, 1072, + 1120, 1154, 1155, 1160, 1162, 1217, 1231, 1232, 1329, 1369, + 1370, 1377, 1417, 1418, 1423, 1425, 1470, 1471, 1472, 1473, + 1475, 1476, 1478, 1479, 1488, 1520, 1523, 1536, 1542, 1545, + 1547, 1548, 1550, 1552, 1563, 1566, 1568, 1600, 1601, 1611, + 1632, 1642, 1646, 1648, 1649, 1748, 1749, 1750, 1757, 1758, + 1759, 1765, 1767, 1769, 1770, 1774, 1776, 1786, 1789, 1791, + 1792, 1807, 1808, 1809, 1810, 1840, 1869, 1958, 1969, 1984, + 1994, 2027, 2036, 2038, 2039, 2042, 2048, 2070, 2074, 2075, + 2084, 2085, 2088, 2089, 2096, 2112, 2137, 2142, 2208, 2210, + 2276, 2304, 2307, 2308, 2362, 2363, 2364, 2365, 2366, 2369, + 2377, 2381, 2382, 2384, 2385, 2392, 2402, 2404, 2406, 2416, + 2417, 2418, 2425, 2433, 2434, 2437, 2447, 2451, 2474, 2482, + 2486, 2492, 2493, 2494, 2497, 2503, 2507, 2509, 2510, 2519, + 2524, 2527, 2530, 2534, 2544, 2546, 2548, 2554, 2555, 2561, + 2563, 2565, 2575, 2579, 2602, 2610, 2613, 2616, 2620, 2622, + 2625, 2631, 2635, 2641, 2649, 2654, 2662, 2672, 2674, 2677, + 2689, 2691, 2693, 2703, 2707, 2730, 2738, 2741, 2748, 2749, + 2750, 2753, 2759, 2761, 2763, 2765, 2768, 2784, 2786, 2790, + 2800, 2801, 2817, 2818, 2821, 2831, 2835, 2858, 2866, 2869, + 2876, 2877, 2878, 2879, 2880, 2881, 2887, 2891, 2893, 2902, + 2903, 2908, 2911, 2914, 2918, 2928, 2929, 2930, 2946, 2947, + 2949, 2958, 2962, 2969, 2972, 2974, 2979, 2984, 2990, 3006, + 3008, 3009, 3014, 3018, 3021, 3024, 3031, 3046, 3056, 3059, + 3065, 3066, 3073, 3077, 3086, 3090, 3114, 3125, 3133, 3134, + 3137, 3142, 3146, 3157, 3160, 3168, 3170, 3174, 3192, 3199, + 3202, 3205, 3214, 3218, 3242, 3253, 3260, 3261, 3262, 3263, + 3264, 3270, 3271, 3274, 3276, 3285, 3294, 3296, 3298, 3302, + 3313, 3330, 3333, 3342, 3346, 3389, 3390, 3393, 3398, 3402, + 3405, 3406, 3415, 3424, 3426, 3430, 3440, 3449, 3450, 3458, + 3461, 3482, 3507, 3517, 3520, 3530, 3535, 3538, 3542, 3544, + 3570, 3572, 3585, 3633, 3634, 3636, 3647, 3648, 3654, 3655, + 3663, 3664, 3674, 3713, 3716, 3719, 3722, 3725, 3732, 3737, + 3745, 3749, 3751, 3754, 3757, 3761, 3762, 3764, 3771, 3773, + 3776, 3782, 3784, 3792, 3804, 3840, 3841, 3844, 3859, 3860, + 3861, 3864, 3866, 3872, 3882, 3892, 3893, 3894, 3895, 3896, + 3897, 3898, 3899, 3900, 3901, 3902, 3904, 3913, 3953, 3967, + 3968, 3973, 3974, 3976, 3981, 3993, 4030, 4038, 4039, 4046, + 4048, 4053, 4057, 4096, 4139, 4141, 4145, 4146, 4152, 4153, + 4155, 4157, 4159, 4160, 4170, 4176, 4182, 4184, 4186, 4190, + 4193, 4194, 4197, 4199, 4206, 4209, 4213, 4226, 4227, 4229, + 4231, 4237, 4238, 4239, 4240, 4250, 4253, 4254, 4256, 4295, + 4301, 4304, 4347, 4348, 4349, 4682, 4688, 4696, 4698, 4704, + 4746, 4752, 4786, 4792, 4800, 4802, 4808, 4824, 4882, 4888, + 4957, 4960, 4969, 4992, 5008, 5024, 5120, 5121, 5741, 5743, + 5760, 5761, 5787, 5788, 5792, 5867, 5870, 5888, 5902, 5906, + 5920, 5938, 5941, 5952, 5970, 5984, 5998, 6002, 6016, 6068, + 6070, 6071, 6078, 6086, 6087, 6089, 6100, 6103, 6104, 6107, + 6108, 6109, 6112, 6128, 6144, 6150, 6151, 6155, 6158, 6160, + 6176, 6211, 6212, 6272, 6313, 6314, 6320, 6400, 6432, 6435, + 6439, 6441, 6448, 6450, 6451, 6457, 6464, 6468, 6470, 6480, + 6512, 6528, 6576, 6593, 6600, 6608, 6618, 6622, 6656, 6679, + 6681, 6686, 6688, 6741, 6742, 6743, 6744, 6752, 6753, 6754, + 6755, 6757, 6765, 6771, 6783, 6784, 6800, 6816, 6823, 6824, + 6912, 6916, 6917, 6964, 6965, 6966, 6971, 6972, 6973, 6978, + 6979, 6981, 6992, 7002, 7009, 7019, 7028, 7040, 7042, 7043, + 7073, 7074, 7078, 7080, 7082, 7083, 7084, 7086, 7088, 7098, + 7142, 7143, 7144, 7146, 7149, 7150, 7151, 7154, 7164, 7168, + 7204, 7212, 7220, 7222, 7227, 7232, 7245, 7248, 7258, 7288, + 7294, 7360, 7376, 7379, 7380, 7393, 7394, 7401, 7405, 7406, + 7410, 7412, 7413, 7424, 7468, 7531, 7544, 7545, 7579, 7616, + 7676, 7680, 7830, 7838, 7936, 7944, 7952, 7960, 7968, 7976, + 7984, 7992, 8000, 8008, 8016, 8025, 8027, 8029, 8031, 8033, + 8040, 8048, 8064, 8072, 8080, 8088, 8096, 8104, 8112, 8118, + 8120, 8124, 8125, 8126, 8127, 8130, 8134, 8136, 8140, 8141, + 8144, 8150, 8152, 8157, 8160, 8168, 8173, 8178, 8182, 8184, + 8188, 8189, 8192, 8203, 8208, 8214, 8216, 8217, 8218, 8219, + 8221, 8222, 8223, 8224, 8232, 8233, 8234, 8239, 8240, 8249, + 8250, 8251, 8255, 8257, 8260, 8261, 8262, 8263, 8274, 8275, + 8276, 8277, 8287, 8288, 8298, 8304, 8305, 8308, 8314, 8317, + 8318, 8319, 8320, 8330, 8333, 8334, 8336, 8352, 8400, 8413, + 8417, 8418, 8421, 8448, 8450, 8451, 8455, 8456, 8458, 8459, + 8462, 8464, 8467, 8468, 8469, 8470, 8472, 8473, 8478, 8484, + 8485, 8486, 8487, 8488, 8489, 8490, 8494, 8495, 8496, 8500, + 8501, 8505, 8506, 8508, 8510, 8512, 8517, 8519, 8522, 8523, + 8524, 8526, 8527, 8528, 8544, 8579, 8581, 8585, 8592, 8597, + 8602, 8604, 8608, 8609, 8611, 8612, 8614, 8615, 8622, 8623, + 8654, 8656, 8658, 8659, 8660, 8661, 8692, 8960, 8968, 8972, + 8992, 8994, 9001, 9002, 9003, 9084, 9085, 9115, 9140, 9180, + 9186, 9216, 9280, 9312, 9372, 9450, 9472, 9655, 9656, 9665, + 9666, 9720, 9728, 9839, 9840, 9985, 10088, 10089, 10090, 10091, + 10092, 10093, 10094, 10095, 10096, 10097, 10098, 10099, 10100, 10101, + 10102, 10132, 10176, 10181, 10182, 10183, 10214, 10215, 10216, 10217, + 10218, 10219, 10220, 10221, 10222, 10223, 10224, 10240, 10496, 10627, + 10628, 10629, 10630, 10631, 10632, 10633, 10634, 10635, 10636, 10637, + 10638, 10639, 10640, 10641, 10642, 10643, 10644, 10645, 10646, 10647, + 10648, 10649, 10712, 10713, 10714, 10715, 10716, 10748, 10749, 10750, + 11008, 11056, 11077, 11079, 11088, 11264, 11312, 11360, 11363, 11365, + 11367, 11374, 11377, 11378, 11380, 11381, 11383, 11388, 11390, 11393, + 11394, 11492, 11493, 11499, 11503, 11506, 11513, 11517, 11518, 11520, + 11559, 11565, 11568, 11631, 11632, 11647, 11648, 11680, 11688, 11696, + 11704, 11712, 11720, 11728, 11736, 11744, 11776, 11778, 11779, 11780, + 11781, 11782, 11785, 11786, 11787, 11788, 11789, 11790, 11799, 11800, + 11802, 11803, 11804, 11805, 11806, 11808, 11809, 11810, 11811, 11812, + 11813, 11814, 11815, 11816, 11817, 11818, 11823, 11824, 11834, 11904, + 11931, 12032, 12272, 12288, 12289, 12292, 12293, 12294, 12295, 12296, + 12297, 12298, 12299, 12300, 12301, 12302, 12303, 12304, 12305, 12306, + 12308, 12309, 12310, 12311, 12312, 12313, 12314, 12315, 12316, 12317, + 12318, 12320, 12321, 12330, 12334, 12336, 12337, 12342, 12344, 12347, + 12348, 12349, 12350, 12353, 12441, 12443, 12445, 12447, 12448, 12449, + 12539, 12540, 12543, 12549, 12593, 12688, 12690, 12694, 12704, 12736, + 12784, 12800, 12832, 12842, 12872, 12880, 12881, 12896, 12928, 12938, + 12977, 12992, 13056, 13312, 19893, 19904, 19968, 40908, 40960, 40981, + 40982, 42128, 42192, 42232, 42238, 42240, 42508, 42509, 42512, 42528, + 42538, 42560, 42606, 42607, 42608, 42611, 42612, 42622, 42623, 42624, + 42655, 42656, 42726, 42736, 42738, 42752, 42775, 42784, 42786, 42800, + 42802, 42864, 42865, 42873, 42878, 42888, 42889, 42891, 42896, 42912, + 43000, 43002, 43003, 43010, 43011, 43014, 43015, 43019, 43020, 43043, + 43045, 43047, 43048, 43056, 43062, 43064, 43065, 43072, 43124, 43136, + 43138, 43188, 43204, 43214, 43216, 43232, 43250, 43256, 43259, 43264, + 43274, 43302, 43310, 43312, 43335, 43346, 43359, 43360, 43392, 43395, + 43396, 43443, 43444, 43446, 43450, 43452, 43453, 43457, 43471, 43472, + 43486, 43520, 43561, 43567, 43569, 43571, 43573, 43584, 43587, 43588, + 43596, 43597, 43600, 43612, 43616, 43632, 43633, 43639, 43642, 43643, + 43648, 43696, 43697, 43698, 43701, 43703, 43705, 43710, 43712, 43713, + 43714, 43739, 43741, 43742, 43744, 43755, 43756, 43758, 43760, 43762, + 43763, 43765, 43766, 43777, 43785, 43793, 43808, 43816, 43968, 44003, + 44005, 44006, 44008, 44009, 44011, 44012, 44013, 44016, 44032, 55203, + 55216, 55243, 55296, 56191, 56319, 57343, 57344, 63743, 63744, 64112, + 64256, 64275, 64285, 64286, 64287, 64297, 64298, 64312, 64318, 64320, + 64323, 64326, 64434, 64467, 64830, 64831, 64848, 64914, 65008, 65020, + 65021, 65024, 65040, 65047, 65048, 65049, 65056, 65072, 65073, 65075, + 65077, 65078, 65079, 65080, 65081, 65082, 65083, 65084, 65085, 65086, + 65087, 65088, 65089, 65090, 65091, 65092, 65093, 65095, 65096, 65097, + 65101, 65104, 65108, 65112, 65113, 65114, 65115, 65116, 65117, 65118, + 65119, 65122, 65123, 65124, 65128, 65129, 65130, 65136, 65142, 65279, + 65281, 65284, 65285, 65288, 65289, 65290, 65291, 65292, 65293, 65294, + 65296, 65306, 65308, 65311, 65313, 65339, 65340, 65341, 65342, 65343, + 65344, 65345, 65371, 65372, 65373, 65374, 65375, 65376, 65377, 65378, + 65379, 65380, 65382, 65392, 65393, 65438, 65440, 65474, 65482, 65490, + 65498, 65504, 65506, 65507, 65508, 65509, 65512, 65513, 65517, 65529, + 65532, 0, 13, 40, 60, 63, 80, 128, 256, 263, + 311, 320, 373, 377, 394, 400, 464, 509, 640, 672, + 768, 800, 816, 833, 834, 842, 896, 927, 928, 968, + 976, 977, 1024, 1064, 1104, 1184, 2048, 2056, 2058, 2103, + 2108, 2111, 2135, 2136, 2304, 2326, 2335, 2336, 2367, 2432, + 2494, 2560, 2561, 2565, 2572, 2576, 2581, 2585, 2616, 2623, + 2624, 2640, 2656, 2685, 2687, 2816, 2873, 2880, 2904, 2912, + 2936, 3072, 3680, 4096, 4097, 4098, 4099, 4152, 4167, 4178, + 4198, 4224, 4226, 4227, 4272, 4275, 4279, 4281, 4283, 4285, + 4286, 4304, 4336, 4352, 4355, 4391, 4396, 4397, 4406, 4416, + 4480, 4482, 4483, 4531, 4534, 4543, 4545, 4549, 4560, 5760, + 5803, 5804, 5805, 5806, 5808, 5814, 5815, 5824, 8192, 9216, + 9328, 12288, 26624, 28416, 28496, 28497, 28559, 28563, 45056, 53248, + 53504, 53545, 53605, 53607, 53610, 53613, 53619, 53627, 53635, 53637, + 53644, 53674, 53678, 53760, 53826, 53829, 54016, 54112, 54272, 54298, + 54324, 54350, 54358, 54376, 54402, 54428, 54430, 54434, 54437, 54441, + 54446, 54454, 54459, 54461, 54469, 54480, 54506, 54532, 54535, 54541, + 54550, 54558, 54584, 54587, 54592, 54598, 54602, 54610, 54636, 54662, + 54688, 54714, 54740, 54766, 54792, 54818, 54844, 54870, 54896, 54922, + 54952, 54977, 54978, 55003, 55004, 55010, 55035, 55036, 55061, 55062, + 55068, 55093, 55094, 55119, 55120, 55126, 55151, 55152, 55177, 55178, + 55184, 55209, 55210, 55235, 55236, 55242, 55246, 60928, 60933, 60961, + 60964, 60967, 60969, 60980, 60985, 60987, 60994, 60999, 61001, 61003, + 61005, 61009, 61012, 61015, 61017, 61019, 61021, 61023, 61025, 61028, + 61031, 61036, 61044, 61049, 61054, 61056, 61067, 61089, 61093, 61099, + 61168, 61440, 61488, 61600, 61617, 61633, 61649, 61696, 61712, 61744, + 61808, 61926, 61968, 62016, 62032, 62208, 62256, 62263, 62336, 62368, + 62406, 62432, 62464, 62528, 62530, 62713, 62720, 62784, 62800, 62971, + 63045, 63104, 63232, 0, 42710, 42752, 46900, 46912, 47133, 63488, + 1, 32, 256, 0, 65533, + }; +static u16 aFts5UnicodeData[] = { + 1025, 61, 117, 55, 117, 54, 50, 53, 57, 53, + 49, 85, 333, 85, 121, 85, 841, 54, 53, 50, + 56, 48, 56, 837, 54, 57, 50, 57, 1057, 61, + 53, 151, 58, 53, 56, 58, 39, 52, 57, 34, + 58, 56, 58, 57, 79, 56, 37, 85, 56, 47, + 39, 51, 111, 53, 745, 57, 233, 773, 57, 261, + 1822, 37, 542, 37, 1534, 222, 69, 73, 37, 126, + 126, 73, 69, 137, 37, 73, 37, 105, 101, 73, + 37, 73, 37, 190, 158, 37, 126, 126, 73, 37, + 126, 94, 37, 39, 94, 69, 135, 41, 40, 37, + 41, 40, 37, 41, 40, 37, 542, 37, 606, 37, + 41, 40, 37, 126, 73, 37, 1886, 197, 73, 37, + 73, 69, 126, 105, 37, 286, 2181, 39, 869, 582, + 152, 390, 472, 166, 248, 38, 56, 38, 568, 3596, + 158, 38, 56, 94, 38, 101, 53, 88, 41, 53, + 105, 41, 73, 37, 553, 297, 1125, 94, 37, 105, + 101, 798, 133, 94, 57, 126, 94, 37, 1641, 1541, + 1118, 58, 172, 75, 1790, 478, 37, 2846, 1225, 38, + 213, 1253, 53, 49, 55, 1452, 49, 44, 53, 76, + 53, 76, 53, 44, 871, 103, 85, 162, 121, 85, + 55, 85, 90, 364, 53, 85, 1031, 38, 327, 684, + 333, 149, 71, 44, 3175, 53, 39, 236, 34, 58, + 204, 70, 76, 58, 140, 71, 333, 103, 90, 39, + 469, 34, 39, 44, 967, 876, 2855, 364, 39, 333, + 1063, 300, 70, 58, 117, 38, 711, 140, 38, 300, + 38, 108, 38, 172, 501, 807, 108, 53, 39, 359, + 876, 108, 42, 1735, 44, 42, 44, 39, 106, 268, + 138, 44, 74, 39, 236, 327, 76, 85, 333, 53, + 38, 199, 231, 44, 74, 263, 71, 711, 231, 39, + 135, 44, 39, 106, 140, 74, 74, 44, 39, 42, + 71, 103, 76, 333, 71, 87, 207, 58, 55, 76, + 42, 199, 71, 711, 231, 71, 71, 71, 44, 106, + 76, 76, 108, 44, 135, 39, 333, 76, 103, 44, + 76, 42, 295, 103, 711, 231, 71, 167, 44, 39, + 106, 172, 76, 42, 74, 44, 39, 71, 76, 333, + 53, 55, 44, 74, 263, 71, 711, 231, 71, 167, + 44, 39, 42, 44, 42, 140, 74, 74, 44, 44, + 42, 71, 103, 76, 333, 58, 39, 207, 44, 39, + 199, 103, 135, 71, 39, 71, 71, 103, 391, 74, + 44, 74, 106, 106, 44, 39, 42, 333, 111, 218, + 55, 58, 106, 263, 103, 743, 327, 167, 39, 108, + 138, 108, 140, 76, 71, 71, 76, 333, 239, 58, + 74, 263, 103, 743, 327, 167, 44, 39, 42, 44, + 170, 44, 74, 74, 76, 74, 39, 71, 76, 333, + 71, 74, 263, 103, 1319, 39, 106, 140, 106, 106, + 44, 39, 42, 71, 76, 333, 207, 58, 199, 74, + 583, 775, 295, 39, 231, 44, 106, 108, 44, 266, + 74, 53, 1543, 44, 71, 236, 55, 199, 38, 268, + 53, 333, 85, 71, 39, 71, 39, 39, 135, 231, + 103, 39, 39, 71, 135, 44, 71, 204, 76, 39, + 167, 38, 204, 333, 135, 39, 122, 501, 58, 53, + 122, 76, 218, 333, 335, 58, 44, 58, 44, 58, + 44, 54, 50, 54, 50, 74, 263, 1159, 460, 42, + 172, 53, 76, 167, 364, 1164, 282, 44, 218, 90, + 181, 154, 85, 1383, 74, 140, 42, 204, 42, 76, + 74, 76, 39, 333, 213, 199, 74, 76, 135, 108, + 39, 106, 71, 234, 103, 140, 423, 44, 74, 76, + 202, 44, 39, 42, 333, 106, 44, 90, 1225, 41, + 41, 1383, 53, 38, 10631, 135, 231, 39, 135, 1319, + 135, 1063, 135, 231, 39, 135, 487, 1831, 135, 2151, + 108, 309, 655, 519, 346, 2727, 49, 19847, 85, 551, + 61, 839, 54, 50, 2407, 117, 110, 423, 135, 108, + 583, 108, 85, 583, 76, 423, 103, 76, 1671, 76, + 42, 236, 266, 44, 74, 364, 117, 38, 117, 55, + 39, 44, 333, 335, 213, 49, 149, 108, 61, 333, + 1127, 38, 1671, 1319, 44, 39, 2247, 935, 108, 138, + 76, 106, 74, 44, 202, 108, 58, 85, 333, 967, + 167, 1415, 554, 231, 74, 333, 47, 1114, 743, 76, + 106, 85, 1703, 42, 44, 42, 236, 44, 42, 44, + 74, 268, 202, 332, 44, 333, 333, 245, 38, 213, + 140, 42, 1511, 44, 42, 172, 42, 44, 170, 44, + 74, 231, 333, 245, 346, 300, 314, 76, 42, 967, + 42, 140, 74, 76, 42, 44, 74, 71, 333, 1415, + 44, 42, 76, 106, 44, 42, 108, 74, 149, 1159, + 266, 268, 74, 76, 181, 333, 103, 333, 967, 198, + 85, 277, 108, 53, 428, 42, 236, 135, 44, 135, + 74, 44, 71, 1413, 2022, 421, 38, 1093, 1190, 1260, + 140, 4830, 261, 3166, 261, 265, 197, 201, 261, 265, + 261, 265, 197, 201, 261, 41, 41, 41, 94, 229, + 265, 453, 261, 264, 261, 264, 261, 264, 165, 69, + 137, 40, 56, 37, 120, 101, 69, 137, 40, 120, + 133, 69, 137, 120, 261, 169, 120, 101, 69, 137, + 40, 88, 381, 162, 209, 85, 52, 51, 54, 84, + 51, 54, 52, 277, 59, 60, 162, 61, 309, 52, + 51, 149, 80, 117, 57, 54, 50, 373, 57, 53, + 48, 341, 61, 162, 194, 47, 38, 207, 121, 54, + 50, 38, 335, 121, 54, 50, 422, 855, 428, 139, + 44, 107, 396, 90, 41, 154, 41, 90, 37, 105, + 69, 105, 37, 58, 41, 90, 57, 169, 218, 41, + 58, 41, 58, 41, 58, 137, 58, 37, 137, 37, + 135, 37, 90, 69, 73, 185, 94, 101, 58, 57, + 90, 37, 58, 527, 1134, 94, 142, 47, 185, 186, + 89, 154, 57, 90, 57, 90, 57, 250, 57, 1018, + 89, 90, 57, 58, 57, 1018, 8601, 282, 153, 666, + 89, 250, 54, 50, 2618, 57, 986, 825, 1306, 217, + 602, 1274, 378, 1935, 2522, 719, 5882, 57, 314, 57, + 1754, 281, 3578, 57, 4634, 3322, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 975, 1434, 185, 54, 50, 1017, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 537, 8218, 4217, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 54, + 50, 2041, 54, 50, 54, 50, 1049, 54, 50, 8281, + 1562, 697, 90, 217, 346, 1513, 1509, 126, 73, 69, + 254, 105, 37, 94, 37, 94, 165, 70, 105, 37, + 3166, 37, 218, 158, 108, 94, 149, 47, 85, 1221, + 37, 37, 1799, 38, 53, 44, 743, 231, 231, 231, + 231, 231, 231, 231, 231, 1036, 85, 52, 51, 52, + 51, 117, 52, 51, 53, 52, 51, 309, 49, 85, + 49, 53, 52, 51, 85, 52, 51, 54, 50, 54, + 50, 54, 50, 54, 50, 181, 38, 341, 81, 858, + 2874, 6874, 410, 61, 117, 58, 38, 39, 46, 54, + 50, 54, 50, 54, 50, 54, 50, 54, 50, 90, + 54, 50, 54, 50, 54, 50, 54, 50, 49, 54, + 82, 58, 302, 140, 74, 49, 166, 90, 110, 38, + 39, 53, 90, 2759, 76, 88, 70, 39, 49, 2887, + 53, 102, 39, 1319, 3015, 90, 143, 346, 871, 1178, + 519, 1018, 335, 986, 271, 58, 495, 1050, 335, 1274, + 495, 2042, 8218, 39, 39, 2074, 39, 39, 679, 38, + 36583, 1786, 1287, 198, 85, 8583, 38, 117, 519, 333, + 71, 1502, 39, 44, 107, 53, 332, 53, 38, 798, + 44, 2247, 334, 76, 213, 760, 294, 88, 478, 69, + 2014, 38, 261, 190, 350, 38, 88, 158, 158, 382, + 70, 37, 231, 44, 103, 44, 135, 44, 743, 74, + 76, 42, 154, 207, 90, 55, 58, 1671, 149, 74, + 1607, 522, 44, 85, 333, 588, 199, 117, 39, 333, + 903, 268, 85, 743, 364, 74, 53, 935, 108, 42, + 1511, 44, 74, 140, 74, 44, 138, 437, 38, 333, + 85, 1319, 204, 74, 76, 74, 76, 103, 44, 263, + 44, 42, 333, 149, 519, 38, 199, 122, 39, 42, + 1543, 44, 39, 108, 71, 76, 167, 76, 39, 44, + 39, 71, 38, 85, 359, 42, 76, 74, 85, 39, + 70, 42, 44, 199, 199, 199, 231, 231, 1127, 74, + 44, 74, 44, 74, 53, 42, 44, 333, 39, 39, + 743, 1575, 36, 68, 68, 36, 63, 63, 11719, 3399, + 229, 165, 39, 44, 327, 57, 423, 167, 39, 71, + 71, 3463, 536, 11623, 54, 50, 2055, 1735, 391, 55, + 58, 524, 245, 54, 50, 53, 236, 53, 81, 80, + 54, 50, 54, 50, 54, 50, 54, 50, 54, 50, + 54, 50, 54, 50, 54, 50, 85, 54, 50, 149, + 112, 117, 149, 49, 54, 50, 54, 50, 54, 50, + 117, 57, 49, 121, 53, 55, 85, 167, 4327, 34, + 117, 55, 117, 54, 50, 53, 57, 53, 49, 85, + 333, 85, 121, 85, 841, 54, 53, 50, 56, 48, + 56, 837, 54, 57, 50, 57, 54, 50, 53, 54, + 50, 85, 327, 38, 1447, 70, 999, 199, 199, 199, + 103, 87, 57, 56, 58, 87, 58, 153, 90, 98, + 90, 391, 839, 615, 71, 487, 455, 3943, 117, 1455, + 314, 1710, 143, 570, 47, 410, 1466, 44, 935, 1575, + 999, 143, 551, 46, 263, 46, 967, 53, 1159, 263, + 53, 174, 1289, 1285, 2503, 333, 199, 39, 1415, 71, + 39, 743, 53, 271, 711, 207, 53, 839, 53, 1799, + 71, 39, 108, 76, 140, 135, 103, 871, 108, 44, + 271, 309, 935, 79, 53, 1735, 245, 711, 271, 615, + 271, 2343, 1007, 42, 44, 42, 1703, 492, 245, 655, + 333, 76, 42, 1447, 106, 140, 74, 76, 85, 34, + 149, 807, 333, 108, 1159, 172, 42, 268, 333, 149, + 76, 42, 1543, 106, 300, 74, 135, 149, 333, 1383, + 44, 42, 44, 74, 204, 42, 44, 333, 28135, 3182, + 149, 34279, 18215, 2215, 39, 1482, 140, 422, 71, 7898, + 1274, 1946, 74, 108, 122, 202, 258, 268, 90, 236, + 986, 140, 1562, 2138, 108, 58, 2810, 591, 841, 837, + 841, 229, 581, 841, 837, 41, 73, 41, 73, 137, + 265, 133, 37, 229, 357, 841, 837, 73, 137, 265, + 233, 837, 73, 137, 169, 41, 233, 837, 841, 837, + 841, 837, 841, 837, 841, 837, 841, 837, 841, 901, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 809, 57, 805, 57, 197, + 809, 57, 805, 57, 197, 94, 1613, 135, 871, 71, + 39, 39, 327, 135, 39, 39, 39, 39, 39, 39, + 103, 71, 39, 39, 39, 39, 39, 39, 71, 39, + 135, 231, 135, 135, 39, 327, 551, 103, 167, 551, + 89, 1434, 3226, 506, 474, 506, 506, 367, 1018, 1946, + 1402, 954, 1402, 314, 90, 1082, 218, 2266, 666, 1210, + 186, 570, 2042, 58, 5850, 154, 2010, 154, 794, 2266, + 378, 2266, 3738, 39, 39, 39, 39, 39, 39, 17351, + 34, 3074, 7692, 63, 63, + }; + +static int sqlite3Fts5UnicodeCategory(u32 iCode) { + int iRes = -1; + int iHi; + int iLo; + int ret; + u16 iKey; + + if( iCode>=(1<<20) ){ + return 0; + } + iLo = aFts5UnicodeBlock[(iCode>>16)]; + iHi = aFts5UnicodeBlock[1+(iCode>>16)]; + iKey = (iCode & 0xFFFF); + while( iHi>iLo ){ + int iTest = (iHi + iLo) / 2; + assert( iTest>=iLo && iTest=aFts5UnicodeMap[iTest] ){ + iRes = iTest; + iLo = iTest+1; + }else{ + iHi = iTest; + } + } + + if( iRes<0 ) return 0; + if( iKey>=(aFts5UnicodeMap[iRes]+(aFts5UnicodeData[iRes]>>5)) ) return 0; + ret = aFts5UnicodeData[iRes] & 0x1F; + if( ret!=30 ) return ret; + return ((iKey - aFts5UnicodeMap[iRes]) & 0x01) ? 5 : 9; +} + +static void sqlite3Fts5UnicodeAscii(u8 *aArray, u8 *aAscii){ + int i = 0; + int iTbl = 0; + while( i<128 ){ + int bToken = aArray[ aFts5UnicodeData[iTbl] & 0x1F ]; + int n = (aFts5UnicodeData[iTbl] >> 5) + i; + for(; i<128 && i3 && n<=9 ); return n; } @@ -196154,7 +259280,7 @@ static int FTS5_NOINLINE fts5PutVarint64(unsigned char *p, u64 v){ v >>= 7; } return 9; - } + } n = 0; do{ buf[n++] = (u8)((v & 0x7f) | 0x80); @@ -196193,7 +259319,6 @@ static int sqlite3Fts5GetVarintLen(u32 iVal){ return 5; } - /* ** 2015 May 08 ** @@ -196207,7 +259332,7 @@ static int sqlite3Fts5GetVarintLen(u32 iVal){ ****************************************************************************** ** ** This is an SQLite virtual table module implementing direct access to an -** existing FTS5 index. The module may create several different types of +** existing FTS5 index. The module may create several different types of ** tables: ** ** col: @@ -196215,16 +259340,21 @@ static int sqlite3Fts5GetVarintLen(u32 iVal){ ** ** One row for each term/column combination. The value of $doc is set to ** the number of fts5 rows that contain at least one instance of term -** $term within column $col. Field $cnt is set to the total number of -** instances of term $term in column $col (in any row of the fts5 table). +** $term within column $col. Field $cnt is set to the total number of +** instances of term $term in column $col (in any row of the fts5 table). ** ** row: ** CREATE TABLE vocab(term, doc, cnt, PRIMARY KEY(term)); ** ** One row for each term in the database. The value of $doc is set to ** the number of fts5 rows that contain at least one instance of term -** $term. Field $cnt is set to the total number of instances of term +** $term. Field $cnt is set to the total number of instances of term ** $term in the database. +** +** instance: +** CREATE TABLE vocab(term, doc, col, offset, PRIMARY KEY()); +** +** One row for each term instance in the database. */ @@ -196240,47 +259370,57 @@ struct Fts5VocabTable { char *zFts5Db; /* Db containing fts5 table */ sqlite3 *db; /* Database handle */ Fts5Global *pGlobal; /* FTS5 global object for this database */ - int eType; /* FTS5_VOCAB_COL or ROW */ + int eType; /* FTS5_VOCAB_COL, ROW or INSTANCE */ + unsigned bBusy; /* True if busy */ }; struct Fts5VocabCursor { sqlite3_vtab_cursor base; sqlite3_stmt *pStmt; /* Statement holding lock on pIndex */ - Fts5Index *pIndex; /* Associated FTS5 index */ + Fts5Table *pFts5; /* Associated FTS5 table */ int bEof; /* True if this cursor is at EOF */ Fts5IndexIter *pIter; /* Term/rowid iterator object */ + void *pStruct; /* From sqlite3Fts5StructureRef() */ int nLeTerm; /* Size of zLeTerm in bytes */ char *zLeTerm; /* (term <= $zLeTerm) paramater, or NULL */ + int colUsed; /* Copy of sqlite3_index_info.colUsed */ /* These are used by 'col' tables only */ - Fts5Config *pConfig; /* Fts5 table configuration */ int iCol; i64 *aCnt; i64 *aDoc; - /* Output values used by 'row' and 'col' tables */ + /* Output values used by all tables. */ i64 rowid; /* This table's current rowid value */ Fts5Buffer term; /* Current value of 'term' column */ + + /* Output values Used by 'instance' tables only */ + i64 iInstPos; + int iInstOff; }; -#define FTS5_VOCAB_COL 0 -#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_COL 0 +#define FTS5_VOCAB_ROW 1 +#define FTS5_VOCAB_INSTANCE 2 #define FTS5_VOCAB_COL_SCHEMA "term, col, doc, cnt" #define FTS5_VOCAB_ROW_SCHEMA "term, doc, cnt" +#define FTS5_VOCAB_INST_SCHEMA "term, doc, col, offset" /* ** Bits for the mask used as the idxNum value by xBestIndex/xFilter. */ -#define FTS5_VOCAB_TERM_EQ 0x01 -#define FTS5_VOCAB_TERM_GE 0x02 -#define FTS5_VOCAB_TERM_LE 0x04 +#define FTS5_VOCAB_TERM_EQ 0x0100 +#define FTS5_VOCAB_TERM_GE 0x0200 +#define FTS5_VOCAB_TERM_LE 0x0400 + +#define FTS5_VOCAB_COLUSED_MASK 0xFF /* -** Translate a string containing an fts5vocab table type to an +** Translate a string containing an fts5vocab table type to an ** FTS5_VOCAB_XXX constant. If successful, set *peType to the output ** value and return SQLITE_OK. Otherwise, set *pzErr to an error message ** and return SQLITE_ERROR. @@ -196297,6 +259437,9 @@ static int fts5VocabTableType(const char *zType, char **pzErr, int *peType){ if( sqlite3_stricmp(zCopy, "row")==0 ){ *peType = FTS5_VOCAB_ROW; }else + if( sqlite3_stricmp(zCopy, "instance")==0 ){ + *peType = FTS5_VOCAB_INSTANCE; + }else { *pzErr = sqlite3_mprintf("fts5vocab: unknown table type: %Q", zCopy); rc = SQLITE_ERROR; @@ -196355,9 +259498,10 @@ static int fts5VocabInitVtab( sqlite3_vtab **ppVTab, /* Write the resulting vtab structure here */ char **pzErr /* Write any error message here */ ){ - const char *azSchema[] = { - "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", - "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")" + const char *azSchema[] = { + "CREATE TABlE vocab(" FTS5_VOCAB_COL_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_ROW_SCHEMA ")", + "CREATE TABlE vocab(" FTS5_VOCAB_INST_SCHEMA ")" }; Fts5VocabTable *pRet = 0; @@ -196374,10 +259518,10 @@ static int fts5VocabInitVtab( const char *zDb = bDb ? argv[3] : argv[1]; const char *zTab = bDb ? argv[4] : argv[3]; const char *zType = bDb ? argv[5] : argv[4]; - int nDb = (int)strlen(zDb)+1; + int nDb = (int)strlen(zDb)+1; int nTab = (int)strlen(zTab)+1; int eType = 0; - + rc = fts5VocabTableType(zType, pzErr, &eType); if( rc==SQLITE_OK ){ assert( eType>=0 && eType= ? +** +** are interpreted. Less-than and less-than-or-equal are treated +** identically, as are greater-than and greater-than-or-equal. */ static int fts5VocabBestIndexMethod( sqlite3_vtab *pUnused, @@ -196440,11 +259593,13 @@ static int fts5VocabBestIndexMethod( int iTermEq = -1; int iTermGe = -1; int iTermLe = -1; - int idxNum = 0; + int idxNum = (int)pInfo->colUsed; int nArg = 0; UNUSED_PARAM(pUnused); + assert( (pInfo->colUsed & FTS5_VOCAB_COLUSED_MASK)==pInfo->colUsed ); + for(i=0; inConstraint; i++){ struct sqlite3_index_constraint *p = &pInfo->aConstraint[i]; if( p->usable==0 ) continue; @@ -196475,8 +259630,19 @@ static int fts5VocabBestIndexMethod( } } - pInfo->idxNum = idxNum; + /* This virtual table always delivers results in ascending order of + ** the "term" column (column 0). So if the user has requested this + ** specifically - "ORDER BY term" or "ORDER BY term ASC" - set the + ** sqlite3_index_info.orderByConsumed flag to tell the core the results + ** are already in sorted order. */ + if( pInfo->nOrderBy==1 + && pInfo->aOrderBy[0].iColumn==0 + && pInfo->aOrderBy[0].desc==0 + ){ + pInfo->orderByConsumed = 1; + } + pInfo->idxNum = idxNum; return SQLITE_OK; } @@ -196484,17 +259650,22 @@ static int fts5VocabBestIndexMethod( ** Implementation of xOpen method. */ static int fts5VocabOpenMethod( - sqlite3_vtab *pVTab, + sqlite3_vtab *pVTab, sqlite3_vtab_cursor **ppCsr ){ Fts5VocabTable *pTab = (Fts5VocabTable*)pVTab; - Fts5Index *pIndex = 0; - Fts5Config *pConfig = 0; + Fts5Table *pFts5 = 0; Fts5VocabCursor *pCsr = 0; int rc = SQLITE_OK; sqlite3_stmt *pStmt = 0; char *zSql = 0; + if( pTab->bBusy ){ + pVTab->zErrMsg = sqlite3_mprintf( + "recursive definition for %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + return SQLITE_ERROR; + } zSql = sqlite3Fts5Mprintf(&rc, "SELECT t.%Q FROM %Q.%Q AS t WHERE t.%Q MATCH '*id'", pTab->zFts5Tbl, pTab->zFts5Db, pTab->zFts5Tbl, pTab->zFts5Tbl @@ -196506,33 +259677,38 @@ static int fts5VocabOpenMethod( assert( rc==SQLITE_OK || pStmt==0 ); if( rc==SQLITE_ERROR ) rc = SQLITE_OK; + pTab->bBusy = 1; if( pStmt && sqlite3_step(pStmt)==SQLITE_ROW ){ i64 iId = sqlite3_column_int64(pStmt, 0); - pIndex = sqlite3Fts5IndexFromCsrid(pTab->pGlobal, iId, &pConfig); + pFts5 = sqlite3Fts5TableFromCsrid(pTab->pGlobal, iId); } + pTab->bBusy = 0; - if( rc==SQLITE_OK && pIndex==0 ){ - rc = sqlite3_finalize(pStmt); - pStmt = 0; - if( rc==SQLITE_OK ){ - pVTab->zErrMsg = sqlite3_mprintf( - "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl - ); - rc = SQLITE_ERROR; + if( rc==SQLITE_OK ){ + if( pFts5==0 ){ + rc = sqlite3_finalize(pStmt); + pStmt = 0; + if( rc==SQLITE_OK ){ + pVTab->zErrMsg = sqlite3_mprintf( + "no such fts5 table: %s.%s", pTab->zFts5Db, pTab->zFts5Tbl + ); + rc = SQLITE_ERROR; + } + }else{ + rc = sqlite3Fts5FlushToDisk(pFts5); } } if( rc==SQLITE_OK ){ - int nByte = pConfig->nCol * sizeof(i64) * 2 + sizeof(Fts5VocabCursor); + i64 nByte = pFts5->pConfig->nCol * sizeof(i64)*2 + sizeof(Fts5VocabCursor); pCsr = (Fts5VocabCursor*)sqlite3Fts5MallocZero(&rc, nByte); } if( pCsr ){ - pCsr->pIndex = pIndex; + pCsr->pFts5 = pFts5; pCsr->pStmt = pStmt; - pCsr->pConfig = pConfig; pCsr->aCnt = (i64*)&pCsr[1]; - pCsr->aDoc = &pCsr->aCnt[pConfig->nCol]; + pCsr->aDoc = &pCsr->aCnt[pFts5->pConfig->nCol]; }else{ sqlite3_finalize(pStmt); } @@ -196544,10 +259720,13 @@ static int fts5VocabOpenMethod( static void fts5VocabResetCursor(Fts5VocabCursor *pCsr){ pCsr->rowid = 0; sqlite3Fts5IterClose(pCsr->pIter); + sqlite3Fts5StructureRelease(pCsr->pStruct); + pCsr->pStruct = 0; pCsr->pIter = 0; sqlite3_free(pCsr->zLeTerm); pCsr->nLeTerm = -1; pCsr->zLeTerm = 0; + pCsr->bEof = 0; } /* @@ -196563,6 +259742,56 @@ static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ return SQLITE_OK; } +static int fts5VocabInstanceNewTerm(Fts5VocabCursor *pCsr){ + int rc = SQLITE_OK; + + if( sqlite3Fts5IterEof(pCsr->pIter) ){ + pCsr->bEof = 1; + }else{ + const char *zTerm; + int nTerm; + zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + if( pCsr->nLeTerm>=0 ){ + int nCmp = MIN(nTerm, pCsr->nLeTerm); + int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); + if( bCmp<0 || (bCmp==0 && pCsr->nLeTermbEof = 1; + } + } + + sqlite3Fts5BufferSet(&rc, &pCsr->term, nTerm, (const u8*)zTerm); + } + return rc; +} + +static int fts5VocabInstanceNext(Fts5VocabCursor *pCsr){ + int eDetail = pCsr->pFts5->pConfig->eDetail; + int rc = SQLITE_OK; + Fts5IndexIter *pIter = pCsr->pIter; + i64 *pp = &pCsr->iInstPos; + int *po = &pCsr->iInstOff; + + assert( sqlite3Fts5IterEof(pIter)==0 ); + assert( pCsr->bEof==0 ); + while( eDetail==FTS5_DETAIL_NONE + || sqlite3Fts5PoslistNext64(pIter->pData, pIter->nData, po, pp) + ){ + pCsr->iInstPos = 0; + pCsr->iInstOff = 0; + + rc = sqlite3Fts5IterNextScan(pCsr->pIter); + if( rc==SQLITE_OK ){ + rc = fts5VocabInstanceNewTerm(pCsr); + if( pCsr->bEof || eDetail==FTS5_DETAIL_NONE ) break; + } + if( rc ){ + pCsr->bEof = 1; + break; + } + } + + return rc; +} /* ** Advance the cursor to the next row in the table. @@ -196570,18 +259799,24 @@ static int fts5VocabCloseMethod(sqlite3_vtab_cursor *pCursor){ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; - int rc = SQLITE_OK; - int nCol = pCsr->pConfig->nCol; + int nCol = pCsr->pFts5->pConfig->nCol; + int rc; + rc = sqlite3Fts5StructureTest(pCsr->pFts5->pIndex, pCsr->pStruct); + if( rc!=SQLITE_OK ) return rc; pCsr->rowid++; + if( pTab->eType==FTS5_VOCAB_INSTANCE ){ + return fts5VocabInstanceNext(pCsr); + } + if( pTab->eType==FTS5_VOCAB_COL ){ for(pCsr->iCol++; pCsr->iColiCol++){ if( pCsr->aDoc[pCsr->iCol] ) break; } } - if( pTab->eType==FTS5_VOCAB_ROW || pCsr->iCol>=nCol ){ + if( pTab->eType!=FTS5_VOCAB_COL || pCsr->iCol>=nCol ){ if( sqlite3Fts5IterEof(pCsr->pIter) ){ pCsr->bEof = 1; }else{ @@ -196589,6 +259824,7 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ int nTerm; zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); + assert( nTerm>=0 ); if( pCsr->nLeTerm>=0 ){ int nCmp = MIN(nTerm, pCsr->nLeTerm); int bCmp = memcmp(pCsr->zLeTerm, zTerm, nCmp); @@ -196605,26 +259841,39 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ assert( pTab->eType==FTS5_VOCAB_COL || pTab->eType==FTS5_VOCAB_ROW ); while( rc==SQLITE_OK ){ + int eDetail = pCsr->pFts5->pConfig->eDetail; const u8 *pPos; int nPos; /* Position list */ i64 iPos = 0; /* 64-bit position read from poslist */ int iOff = 0; /* Current offset within position list */ pPos = pCsr->pIter->pData; nPos = pCsr->pIter->nData; - switch( pCsr->pConfig->eDetail ){ - case FTS5_DETAIL_FULL: - pPos = pCsr->pIter->pData; - nPos = pCsr->pIter->nData; - if( pTab->eType==FTS5_VOCAB_ROW ){ - while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ - pCsr->aCnt[0]++; + + switch( pTab->eType ){ + case FTS5_VOCAB_ROW: + /* Do not bother counting the number of instances if the "cnt" + ** column is not being read (according to colUsed). */ + if( eDetail==FTS5_DETAIL_FULL && (pCsr->colUsed & 0x04) ){ + while( iPosaCnt[] */ + pCsr->aCnt[0]++; + } } - pCsr->aDoc[0]++; - }else{ + } + pCsr->aDoc[0]++; + break; + + case FTS5_VOCAB_COL: + if( eDetail==FTS5_DETAIL_FULL ){ int iCol = -1; while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff, &iPos) ){ int ii = FTS5_POS2COLUMN(iPos); - pCsr->aCnt[ii]++; if( iCol!=ii ){ if( ii>=nCol ){ rc = FTS5_CORRUPT; @@ -196633,14 +259882,9 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ pCsr->aDoc[ii]++; iCol = ii; } + pCsr->aCnt[ii]++; } - } - break; - - case FTS5_DETAIL_COLUMNS: - if( pTab->eType==FTS5_VOCAB_ROW ){ - pCsr->aDoc[0]++; - }else{ + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ while( 0==sqlite3Fts5PoslistNext64(pPos, nPos, &iOff,&iPos) ){ assert_nc( iPos>=0 && iPos=nCol ){ @@ -196649,22 +259893,27 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ } pCsr->aDoc[iPos]++; } + }else{ + assert( eDetail==FTS5_DETAIL_NONE ); + pCsr->aDoc[0]++; } break; - default: - assert( pCsr->pConfig->eDetail==FTS5_DETAIL_NONE ); - pCsr->aDoc[0]++; + default: + assert( pTab->eType==FTS5_VOCAB_INSTANCE ); break; } if( rc==SQLITE_OK ){ rc = sqlite3Fts5IterNextScan(pCsr->pIter); } + if( pTab->eType==FTS5_VOCAB_INSTANCE ) break; if( rc==SQLITE_OK ){ zTerm = sqlite3Fts5IterTerm(pCsr->pIter, &nTerm); - if( nTerm!=pCsr->term.n || memcmp(zTerm, pCsr->term.p, nTerm) ){ + if( nTerm!=pCsr->term.n + || (nTerm>0 && memcmp(zTerm, pCsr->term.p, nTerm)) + ){ break; } if( sqlite3Fts5IterEof(pCsr->pIter) ) break; @@ -196674,8 +259923,10 @@ static int fts5VocabNextMethod(sqlite3_vtab_cursor *pCursor){ } if( rc==SQLITE_OK && pCsr->bEof==0 && pTab->eType==FTS5_VOCAB_COL ){ - while( pCsr->aDoc[pCsr->iCol]==0 ) pCsr->iCol++; - assert( pCsr->iColpConfig->nCol ); + for(/* noop */; pCsr->iColaDoc[pCsr->iCol]==0; pCsr->iCol++); + if( pCsr->iCol==nCol ){ + rc = FTS5_CORRUPT; + } } return rc; } @@ -196690,7 +259941,9 @@ static int fts5VocabFilterMethod( int nUnused, /* Number of elements in apVal */ sqlite3_value **apVal /* Arguments for the indexing scheme */ ){ + Fts5VocabTable *pTab = (Fts5VocabTable*)pCursor->pVtab; Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; + int eType = pTab->eType; int rc = SQLITE_OK; int iVal = 0; @@ -196708,11 +259961,12 @@ static int fts5VocabFilterMethod( if( idxNum & FTS5_VOCAB_TERM_EQ ) pEq = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_GE ) pGe = apVal[iVal++]; if( idxNum & FTS5_VOCAB_TERM_LE ) pLe = apVal[iVal++]; + pCsr->colUsed = (idxNum & FTS5_VOCAB_COLUSED_MASK); if( pEq ){ zTerm = (const char *)sqlite3_value_text(pEq); nTerm = sqlite3_value_bytes(pEq); - f = 0; + f = FTS5INDEX_QUERY_NOTOKENDATA; }else{ if( pGe ){ zTerm = (const char *)sqlite3_value_text(pGe); @@ -196720,6 +259974,7 @@ static int fts5VocabFilterMethod( } if( pLe ){ const char *zCopy = (const char *)sqlite3_value_text(pLe); + if( zCopy==0 ) zCopy = ""; pCsr->nLeTerm = sqlite3_value_bytes(pLe); pCsr->zLeTerm = sqlite3_malloc(pCsr->nLeTerm+1); if( pCsr->zLeTerm==0 ){ @@ -196730,19 +259985,28 @@ static int fts5VocabFilterMethod( } } - if( rc==SQLITE_OK ){ - rc = sqlite3Fts5IndexQuery(pCsr->pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); + Fts5Index *pIndex = pCsr->pFts5->pIndex; + rc = sqlite3Fts5IndexQuery(pIndex, zTerm, nTerm, f, 0, &pCsr->pIter); + if( rc==SQLITE_OK ){ + pCsr->pStruct = sqlite3Fts5StructureRef(pIndex); + } } - if( rc==SQLITE_OK ){ + if( rc==SQLITE_OK && eType==FTS5_VOCAB_INSTANCE ){ + rc = fts5VocabInstanceNewTerm(pCsr); + } + if( rc==SQLITE_OK && !pCsr->bEof + && (eType!=FTS5_VOCAB_INSTANCE + || pCsr->pFts5->pConfig->eDetail!=FTS5_DETAIL_NONE) + ){ rc = fts5VocabNextMethod(pCursor); } return rc; } -/* -** This is the xEof method of the virtual table. SQLite calls this +/* +** This is the xEof method of the virtual table. SQLite calls this ** routine to find out if it has reached the end of a result set. */ static int fts5VocabEofMethod(sqlite3_vtab_cursor *pCursor){ @@ -196756,7 +260020,7 @@ static int fts5VocabColumnMethod( int iCol /* Index of column to read value from */ ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; - int eDetail = pCsr->pConfig->eDetail; + int eDetail = pCsr->pFts5->pConfig->eDetail; int eType = ((Fts5VocabTable*)(pCursor->pVtab))->eType; i64 iVal = 0; @@ -196768,7 +260032,7 @@ static int fts5VocabColumnMethod( assert( iCol==1 || iCol==2 || iCol==3 ); if( iCol==1 ){ if( eDetail!=FTS5_DETAIL_NONE ){ - const char *z = pCsr->pConfig->azCol[pCsr->iCol]; + const char *z = pCsr->pFts5->pConfig->azCol[pCsr->iCol]; sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); } }else if( iCol==2 ){ @@ -196776,26 +260040,54 @@ static int fts5VocabColumnMethod( }else{ iVal = pCsr->aCnt[pCsr->iCol]; } - }else{ + }else if( eType==FTS5_VOCAB_ROW ){ assert( iCol==1 || iCol==2 ); if( iCol==1 ){ iVal = pCsr->aDoc[0]; }else{ iVal = pCsr->aCnt[0]; } + }else{ + assert( eType==FTS5_VOCAB_INSTANCE ); + switch( iCol ){ + case 1: + sqlite3_result_int64(pCtx, pCsr->pIter->iRowid); + break; + case 2: { + int ii = -1; + if( eDetail==FTS5_DETAIL_FULL ){ + ii = FTS5_POS2COLUMN(pCsr->iInstPos); + }else if( eDetail==FTS5_DETAIL_COLUMNS ){ + ii = (int)pCsr->iInstPos; + } + if( ii>=0 && iipFts5->pConfig->nCol ){ + const char *z = pCsr->pFts5->pConfig->azCol[ii]; + sqlite3_result_text(pCtx, z, -1, SQLITE_STATIC); + } + break; + } + default: { + assert( iCol==3 ); + if( eDetail==FTS5_DETAIL_FULL ){ + int ii = FTS5_POS2OFFSET(pCsr->iInstPos); + sqlite3_result_int(pCtx, ii); + } + break; + } + } } if( iVal>0 ) sqlite3_result_int64(pCtx, iVal); return SQLITE_OK; } -/* +/* ** This is the xRowid method. The SQLite core calls this routine to ** retrieve the rowid for the current row of the result set. The ** rowid should be written to *pRowid. */ static int fts5VocabRowidMethod( - sqlite3_vtab_cursor *pCursor, + sqlite3_vtab_cursor *pCursor, sqlite_int64 *pRowid ){ Fts5VocabCursor *pCsr = (Fts5VocabCursor*)pCursor; @@ -196828,6 +260120,8 @@ static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ /* xSavepoint */ 0, /* xRelease */ 0, /* xRollbackTo */ 0, + /* xShadowName */ 0, + /* xIntegrity */ 0 }; void *p = (void*)pGlobal; @@ -196835,9 +260129,360 @@ static int sqlite3Fts5VocabInit(Fts5Global *pGlobal, sqlite3 *db){ } - - - +/* Here ends the fts5.c composite file. */ #endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_FTS5) */ /************** End of fts5.c ************************************************/ +/************** Begin file stmt.c ********************************************/ +/* +** 2017-05-31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +** +** This file demonstrates an eponymous virtual table that returns information +** about all prepared statements for the database connection. +** +** Usage example: +** +** .load ./stmt +** .mode line +** .header on +** SELECT * FROM stmt; +*/ +#if !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_STMTVTAB) +#if !defined(SQLITEINT_H) +/* #include "sqlite3ext.h" */ +#endif +SQLITE_EXTENSION_INIT1 +/* #include */ +/* #include */ + +#ifndef SQLITE_OMIT_VIRTUALTABLE + + +#define STMT_NUM_INTEGER_COLUMN 10 +typedef struct StmtRow StmtRow; +struct StmtRow { + sqlite3_int64 iRowid; /* Rowid value */ + char *zSql; /* column "sql" */ + int aCol[STMT_NUM_INTEGER_COLUMN+1]; /* all other column values */ + StmtRow *pNext; /* Next row to return */ +}; + +/* stmt_vtab is a subclass of sqlite3_vtab which will +** serve as the underlying representation of a stmt virtual table +*/ +typedef struct stmt_vtab stmt_vtab; +struct stmt_vtab { + sqlite3_vtab base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this stmt vtab */ +}; + +/* stmt_cursor is a subclass of sqlite3_vtab_cursor which will +** serve as the underlying representation of a cursor that scans +** over rows of the result +*/ +typedef struct stmt_cursor stmt_cursor; +struct stmt_cursor { + sqlite3_vtab_cursor base; /* Base class - must be first */ + sqlite3 *db; /* Database connection for this cursor */ + StmtRow *pRow; /* Current row */ +}; + +/* +** The stmtConnect() method is invoked to create a new +** stmt_vtab that describes the stmt virtual table. +** +** Think of this routine as the constructor for stmt_vtab objects. +** +** All this routine needs to do is: +** +** (1) Allocate the stmt_vtab object and initialize all fields. +** +** (2) Tell SQLite (via the sqlite3_declare_vtab() interface) what the +** result set of queries against stmt will look like. +*/ +static int stmtConnect( + sqlite3 *db, + void *pAux, + int argc, const char *const*argv, + sqlite3_vtab **ppVtab, + char **pzErr +){ + stmt_vtab *pNew; + int rc; + +/* Column numbers */ +#define STMT_COLUMN_SQL 0 /* SQL for the statement */ +#define STMT_COLUMN_NCOL 1 /* Number of result columns */ +#define STMT_COLUMN_RO 2 /* True if read-only */ +#define STMT_COLUMN_BUSY 3 /* True if currently busy */ +#define STMT_COLUMN_NSCAN 4 /* SQLITE_STMTSTATUS_FULLSCAN_STEP */ +#define STMT_COLUMN_NSORT 5 /* SQLITE_STMTSTATUS_SORT */ +#define STMT_COLUMN_NAIDX 6 /* SQLITE_STMTSTATUS_AUTOINDEX */ +#define STMT_COLUMN_NSTEP 7 /* SQLITE_STMTSTATUS_VM_STEP */ +#define STMT_COLUMN_REPREP 8 /* SQLITE_STMTSTATUS_REPREPARE */ +#define STMT_COLUMN_RUN 9 /* SQLITE_STMTSTATUS_RUN */ +#define STMT_COLUMN_MEM 10 /* SQLITE_STMTSTATUS_MEMUSED */ + + + (void)pAux; + (void)argc; + (void)argv; + (void)pzErr; + rc = sqlite3_declare_vtab(db, + "CREATE TABLE x(sql,ncol,ro,busy,nscan,nsort,naidx,nstep," + "reprep,run,mem)"); + if( rc==SQLITE_OK ){ + pNew = sqlite3_malloc64( sizeof(*pNew) ); + *ppVtab = (sqlite3_vtab*)pNew; + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(*pNew)); + pNew->db = db; + } + return rc; +} + +/* +** This method is the destructor for stmt_cursor objects. +*/ +static int stmtDisconnect(sqlite3_vtab *pVtab){ + sqlite3_free(pVtab); + return SQLITE_OK; +} + +/* +** Constructor for a new stmt_cursor object. +*/ +static int stmtOpen(sqlite3_vtab *p, sqlite3_vtab_cursor **ppCursor){ + stmt_cursor *pCur; + pCur = sqlite3_malloc64( sizeof(*pCur) ); + if( pCur==0 ) return SQLITE_NOMEM; + memset(pCur, 0, sizeof(*pCur)); + pCur->db = ((stmt_vtab*)p)->db; + *ppCursor = &pCur->base; + return SQLITE_OK; +} + +static void stmtCsrReset(stmt_cursor *pCur){ + StmtRow *pRow = 0; + StmtRow *pNext = 0; + for(pRow=pCur->pRow; pRow; pRow=pNext){ + pNext = pRow->pNext; + sqlite3_free(pRow); + } + pCur->pRow = 0; +} + +/* +** Destructor for a stmt_cursor. +*/ +static int stmtClose(sqlite3_vtab_cursor *cur){ + stmtCsrReset((stmt_cursor*)cur); + sqlite3_free(cur); + return SQLITE_OK; +} + + +/* +** Advance a stmt_cursor to its next row of output. +*/ +static int stmtNext(sqlite3_vtab_cursor *cur){ + stmt_cursor *pCur = (stmt_cursor*)cur; + StmtRow *pNext = pCur->pRow->pNext; + sqlite3_free(pCur->pRow); + pCur->pRow = pNext; + return SQLITE_OK; +} + +/* +** Return values of columns for the row at which the stmt_cursor +** is currently pointing. +*/ +static int stmtColumn( + sqlite3_vtab_cursor *cur, /* The cursor */ + sqlite3_context *ctx, /* First argument to sqlite3_result_...() */ + int i /* Which column to return */ +){ + stmt_cursor *pCur = (stmt_cursor*)cur; + StmtRow *pRow = pCur->pRow; + if( i==STMT_COLUMN_SQL ){ + sqlite3_result_text(ctx, pRow->zSql, -1, SQLITE_TRANSIENT); + }else{ + sqlite3_result_int(ctx, pRow->aCol[i]); + } + return SQLITE_OK; +} + +/* +** Return the rowid for the current row. In this implementation, the +** rowid is the same as the output value. +*/ +static int stmtRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid){ + stmt_cursor *pCur = (stmt_cursor*)cur; + *pRowid = pCur->pRow->iRowid; + return SQLITE_OK; +} + +/* +** Return TRUE if the cursor has been moved off of the last +** row of output. +*/ +static int stmtEof(sqlite3_vtab_cursor *cur){ + stmt_cursor *pCur = (stmt_cursor*)cur; + return pCur->pRow==0; +} + +/* +** This method is called to "rewind" the stmt_cursor object back +** to the first row of output. This method is always called at least +** once prior to any call to stmtColumn() or stmtRowid() or +** stmtEof(). +*/ +static int stmtFilter( + sqlite3_vtab_cursor *pVtabCursor, + int idxNum, const char *idxStr, + int argc, sqlite3_value **argv +){ + stmt_cursor *pCur = (stmt_cursor *)pVtabCursor; + sqlite3_stmt *p = 0; + sqlite3_int64 iRowid = 1; + StmtRow **ppRow = 0; + + (void)idxNum; + (void)idxStr; + (void)argc; + (void)argv; + stmtCsrReset(pCur); + ppRow = &pCur->pRow; + for(p=sqlite3_next_stmt(pCur->db, 0); p; p=sqlite3_next_stmt(pCur->db, p)){ + const char *zSql = sqlite3_sql(p); + sqlite3_int64 nSql = zSql ? strlen(zSql)+1 : 0; + StmtRow *pNew = (StmtRow*)sqlite3_malloc64(sizeof(StmtRow) + nSql); + + if( pNew==0 ) return SQLITE_NOMEM; + memset(pNew, 0, sizeof(StmtRow)); + if( zSql ){ + pNew->zSql = (char*)&pNew[1]; + memcpy(pNew->zSql, zSql, nSql); + } + pNew->aCol[STMT_COLUMN_NCOL] = sqlite3_column_count(p); + pNew->aCol[STMT_COLUMN_RO] = sqlite3_stmt_readonly(p); + pNew->aCol[STMT_COLUMN_BUSY] = sqlite3_stmt_busy(p); + pNew->aCol[STMT_COLUMN_NSCAN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_FULLSCAN_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_NSORT] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_SORT, 0 + ); + pNew->aCol[STMT_COLUMN_NAIDX] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_AUTOINDEX, 0 + ); + pNew->aCol[STMT_COLUMN_NSTEP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_VM_STEP, 0 + ); + pNew->aCol[STMT_COLUMN_REPREP] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_REPREPARE, 0 + ); + pNew->aCol[STMT_COLUMN_RUN] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_RUN, 0 + ); + pNew->aCol[STMT_COLUMN_MEM] = sqlite3_stmt_status( + p, SQLITE_STMTSTATUS_MEMUSED, 0 + ); + pNew->iRowid = iRowid++; + *ppRow = pNew; + ppRow = &pNew->pNext; + } + + return SQLITE_OK; +} + +/* +** SQLite will invoke this method one or more times while planning a query +** that uses the stmt virtual table. This routine needs to create +** a query plan for each invocation and compute an estimated cost for that +** plan. +*/ +static int stmtBestIndex( + sqlite3_vtab *tab, + sqlite3_index_info *pIdxInfo +){ + (void)tab; + pIdxInfo->estimatedCost = (double)500; + pIdxInfo->estimatedRows = 500; + return SQLITE_OK; +} + +/* +** This following structure defines all the methods for the +** stmt virtual table. +*/ +static sqlite3_module stmtModule = { + 0, /* iVersion */ + 0, /* xCreate */ + stmtConnect, /* xConnect */ + stmtBestIndex, /* xBestIndex */ + stmtDisconnect, /* xDisconnect */ + 0, /* xDestroy */ + stmtOpen, /* xOpen - open a cursor */ + stmtClose, /* xClose - close a cursor */ + stmtFilter, /* xFilter - configure scan constraints */ + stmtNext, /* xNext - advance a cursor */ + stmtEof, /* xEof - check for end of scan */ + stmtColumn, /* xColumn - read data */ + stmtRowid, /* xRowid - read data */ + 0, /* xUpdate */ + 0, /* xBegin */ + 0, /* xSync */ + 0, /* xCommit */ + 0, /* xRollback */ + 0, /* xFindMethod */ + 0, /* xRename */ + 0, /* xSavepoint */ + 0, /* xRelease */ + 0, /* xRollbackTo */ + 0, /* xShadowName */ + 0 /* xIntegrity */ +}; + +#endif /* SQLITE_OMIT_VIRTUALTABLE */ + +SQLITE_PRIVATE int sqlite3StmtVtabInit(sqlite3 *db){ + int rc = SQLITE_OK; +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3_create_module(db, "sqlite_stmt", &stmtModule, 0); +#endif + return rc; +} + +#ifndef SQLITE_CORE +#ifdef _WIN32 +__declspec(dllexport) +#endif +SQLITE_API int sqlite3_stmt_init( + sqlite3 *db, + char **pzErrMsg, + const sqlite3_api_routines *pApi +){ + int rc = SQLITE_OK; + SQLITE_EXTENSION_INIT2(pApi); +#ifndef SQLITE_OMIT_VIRTUALTABLE + rc = sqlite3StmtVtabInit(db); +#endif + return rc; +} +#endif /* SQLITE_CORE */ +#endif /* !defined(SQLITE_CORE) || defined(SQLITE_ENABLE_STMTVTAB) */ + +/************** End of stmt.c ************************************************/ +/* Return the source-id for this library */ +SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } +/************************** End of sqlite3.c ******************************/ diff --git a/thirdparty/SQLite/sqlite3.h b/thirdparty/SQLite/sqlite3.h index 21a21b5b..dbecc3fe 100644 --- a/thirdparty/SQLite/sqlite3.h +++ b/thirdparty/SQLite/sqlite3.h @@ -1,5 +1,5 @@ /* -** 2001 September 15 +** 2001-09-15 ** ** The author disclaims copyright to this source code. In place of ** a legal notice, here is a blessing: @@ -17,23 +17,21 @@ ** ** Some of the definitions that are in this file are marked as ** "experimental". Experimental interfaces are normally new -** features recently added to SQLite. We do not anticipate changes -** to experimental interfaces but reserve to make minor changes if -** experience from use "in the wild" suggest such changes are prudent. +** features recently added to SQLite. We do not anticipate changes +** to experimental interfaces but reserve the right to make minor changes +** if experience from use "in the wild" suggest such changes are prudent. ** ** The official C-language API documentation for SQLite is derived ** from comments in this file. This file is the authoritative source -** on how SQLite interfaces are suppose to operate. +** on how SQLite interfaces are supposed to operate. ** ** The name of this file under configuration management is "sqlite.h.in". ** The makefile makes some minor changes to this file (such as inserting ** the version number) and changes its name to "sqlite3.h" as ** part of the build process. -** -** @(#) $Id: sqlite.h.in,v 1.212 2007/06/14 20:57:19 drh Exp $ */ -#ifndef _SQLITE3_H_ -#define _SQLITE3_H_ +#ifndef SQLITE3_H +#define SQLITE3_H #include /* Needed for the definition of va_list */ /* @@ -43,9 +41,73 @@ extern "C" { #endif + /* -** Make sure these symbols where not defined by some previous header -** file. +** Facilitate override of interface linkage and calling conventions. +** Be aware that these macros may not be used within this particular +** translation of the amalgamation and its associated header file. +** +** The SQLITE_EXTERN and SQLITE_API macros are used to instruct the +** compiler that the target identifier should have external linkage. +** +** The SQLITE_CDECL macro is used to set the calling convention for +** public functions that accept a variable number of arguments. +** +** The SQLITE_APICALL macro is used to set the calling convention for +** public functions that accept a fixed number of arguments. +** +** The SQLITE_STDCALL macro is no longer used and is now deprecated. +** +** The SQLITE_CALLBACK macro is used to set the calling convention for +** function pointers. +** +** The SQLITE_SYSAPI macro is used to set the calling convention for +** functions provided by the operating system. +** +** Currently, the SQLITE_CDECL, SQLITE_APICALL, SQLITE_CALLBACK, and +** SQLITE_SYSAPI macros are used only when building for environments +** that require non-default calling conventions. +*/ +#ifndef SQLITE_EXTERN +# define SQLITE_EXTERN extern +#endif +#ifndef SQLITE_API +# define SQLITE_API +#endif +#ifndef SQLITE_CDECL +# define SQLITE_CDECL +#endif +#ifndef SQLITE_APICALL +# define SQLITE_APICALL +#endif +#ifndef SQLITE_STDCALL +# define SQLITE_STDCALL SQLITE_APICALL +#endif +#ifndef SQLITE_CALLBACK +# define SQLITE_CALLBACK +#endif +#ifndef SQLITE_SYSAPI +# define SQLITE_SYSAPI +#endif + +/* +** These no-op macros are used in front of interfaces to mark those +** interfaces as either deprecated or experimental. New applications +** should not use deprecated interfaces - they are supported for backwards +** compatibility only. Application writers should be aware that +** experimental interfaces are subject to change in point releases. +** +** These macros used to resolve to various kinds of compiler magic that +** would generate warning messages when they were used. But that +** compiler magic ended up generating such a flurry of bug reports +** that we have taken it all out and gone back to using simple +** noop macros. +*/ +#define SQLITE_DEPRECATED +#define SQLITE_EXPERIMENTAL + +/* +** Ensure these symbols were not defined by some previous header file. */ #ifdef SQLITE_VERSION # undef SQLITE_VERSION @@ -57,79 +119,181 @@ extern "C" { /* ** CAPI3REF: Compile-Time Library Version Numbers ** -** The version of the SQLite library is contained in the sqlite3.h -** header file in a #define named SQLITE_VERSION. The SQLITE_VERSION -** macro resolves to a string constant. +** ^(The [SQLITE_VERSION] C preprocessor macro in the sqlite3.h header +** evaluates to a string literal that is the SQLite version in the +** format "X.Y.Z" where X is the major version number (always 3 for +** SQLite3) and Y is the minor version number and Z is the release number.)^ +** ^(The [SQLITE_VERSION_NUMBER] C preprocessor macro resolves to an integer +** with the value (X*1000000 + Y*1000 + Z) where X, Y, and Z are the same +** numbers used in [SQLITE_VERSION].)^ +** The SQLITE_VERSION_NUMBER for any given release of SQLite will also +** be larger than the release from which it is derived. Either Y will +** be held constant and Z will be incremented or else Y will be incremented +** and Z will be reset to zero. ** -** The format of the version string is "X.Y.Z", where -** X is the major version number, Y is the minor version number and Z -** is the release number. The X.Y.Z might be followed by "alpha" or "beta". -** For example "3.1.1beta". +** Since [version 3.6.18] ([dateof:3.6.18]), +** SQLite source code has been stored in the +** Fossil configuration management +** system. ^The SQLITE_SOURCE_ID macro evaluates to +** a string which identifies a particular check-in of SQLite +** within its configuration management system. ^The SQLITE_SOURCE_ID +** string contains the date and time of the check-in (UTC) and a SHA1 +** or SHA3-256 hash of the entire source tree. If the source code has +** been edited in any way since it was last checked in, then the last +** four hexadecimal digits of the hash may be modified. ** -** The X value is always 3 in SQLite. The X value only changes when -** backwards compatibility is broken and we intend to never break -** backwards compatibility. The Y value only changes when -** there are major feature enhancements that are forwards compatible -** but not backwards compatible. The Z value is incremented with -** each release but resets back to 0 when Y is incremented. -** -** The SQLITE_VERSION_NUMBER is an integer with the value -** (X*1000000 + Y*1000 + Z). For example, for version "3.1.1beta", -** SQLITE_VERSION_NUMBER is set to 3001001. To detect if they are using -** version 3.1.1 or greater at compile time, programs may use the test -** (SQLITE_VERSION_NUMBER>=3001001). -** -** See also: [sqlite3_libversion()] and [sqlite3_libversion_number()]. +** See also: [sqlite3_libversion()], +** [sqlite3_libversion_number()], [sqlite3_sourceid()], +** [sqlite_version()] and [sqlite_source_id()]. */ -#define SQLITE_VERSION "3.4.0" -#define SQLITE_VERSION_NUMBER 3004000 +#define SQLITE_VERSION "3.47.1" +#define SQLITE_VERSION_NUMBER 3047001 +#define SQLITE_SOURCE_ID "2024-11-25 12:07:48 b95d11e958643b969c47a8e5857f3793b9e69700b8f1469371386369a26e577e" /* ** CAPI3REF: Run-Time Library Version Numbers +** KEYWORDS: sqlite3_version sqlite3_sourceid ** -** These routines return values equivalent to the header constants -** [SQLITE_VERSION] and [SQLITE_VERSION_NUMBER]. The values returned -** by this routines should only be different from the header values -** if you compile your program using an sqlite3.h header from a -** different version of SQLite that the version of the library you -** link against. +** These interfaces provide the same information as the [SQLITE_VERSION], +** [SQLITE_VERSION_NUMBER], and [SQLITE_SOURCE_ID] C preprocessor macros +** but are associated with the library instead of the header file. ^(Cautious +** programmers might include assert() statements in their application to +** verify that values returned by these interfaces match the macros in +** the header, and thus ensure that the application is +** compiled with matching library and header files. ** -** The sqlite3_version[] string constant contains the text of the -** [SQLITE_VERSION] string. The sqlite3_libversion() function returns -** a poiner to the sqlite3_version[] string constant. The function -** is provided for DLL users who can only access functions and not -** constants within the DLL. +** 
                +** assert( sqlite3_libversion_number()==SQLITE_VERSION_NUMBER );
+** assert( strncmp(sqlite3_sourceid(),SQLITE_SOURCE_ID,80)==0 );
+** assert( strcmp(sqlite3_libversion(),SQLITE_VERSION)==0 );
+**
                )^ +** +** ^The sqlite3_version[] string constant contains the text of [SQLITE_VERSION] +** macro. ^The sqlite3_libversion() function returns a pointer to the +** to the sqlite3_version[] string constant. The sqlite3_libversion() +** function is provided for use in DLLs since DLL users usually do not have +** direct access to string constants within the DLL. ^The +** sqlite3_libversion_number() function returns an integer equal to +** [SQLITE_VERSION_NUMBER]. ^(The sqlite3_sourceid() function returns +** a pointer to a string constant whose value is the same as the +** [SQLITE_SOURCE_ID] C preprocessor macro. Except if SQLite is built +** using an edited copy of [the amalgamation], then the last four characters +** of the hash might be different from [SQLITE_SOURCE_ID].)^ +** +** See also: [sqlite_version()] and [sqlite_source_id()]. */ -extern const char sqlite3_version[]; -const char *sqlite3_libversion(void); -int sqlite3_libversion_number(void); +SQLITE_API SQLITE_EXTERN const char sqlite3_version[]; +SQLITE_API const char *sqlite3_libversion(void); +SQLITE_API const char *sqlite3_sourceid(void); +SQLITE_API int sqlite3_libversion_number(void); + +/* +** CAPI3REF: Run-Time Library Compilation Options Diagnostics +** +** ^The sqlite3_compileoption_used() function returns 0 or 1 +** indicating whether the specified option was defined at +** compile time. ^The SQLITE_ prefix may be omitted from the +** option name passed to sqlite3_compileoption_used(). +** +** ^The sqlite3_compileoption_get() function allows iterating +** over the list of options that were defined at compile time by +** returning the N-th compile time option string. ^If N is out of range, +** sqlite3_compileoption_get() returns a NULL pointer. ^The SQLITE_ +** prefix is omitted from any strings returned by +** sqlite3_compileoption_get(). +** +** ^Support for the diagnostic functions sqlite3_compileoption_used() +** and sqlite3_compileoption_get() may be omitted by specifying the +** [SQLITE_OMIT_COMPILEOPTION_DIAGS] option at compile time. +** +** See also: SQL functions [sqlite_compileoption_used()] and +** [sqlite_compileoption_get()] and the [compile_options pragma]. +*/ +#ifndef SQLITE_OMIT_COMPILEOPTION_DIAGS +SQLITE_API int sqlite3_compileoption_used(const char *zOptName); +SQLITE_API const char *sqlite3_compileoption_get(int N); +#else +# define sqlite3_compileoption_used(X) 0 +# define sqlite3_compileoption_get(X) ((void*)0) +#endif + +/* +** CAPI3REF: Test To See If The Library Is Threadsafe +** +** ^The sqlite3_threadsafe() function returns zero if and only if +** SQLite was compiled with mutexing code omitted due to the +** [SQLITE_THREADSAFE] compile-time option being set to 0. +** +** SQLite can be compiled with or without mutexes. When +** the [SQLITE_THREADSAFE] C preprocessor macro is 1 or 2, mutexes +** are enabled and SQLite is threadsafe. When the +** [SQLITE_THREADSAFE] macro is 0, +** the mutexes are omitted. Without the mutexes, it is not safe +** to use SQLite concurrently from more than one thread. +** +** Enabling mutexes incurs a measurable performance penalty. +** So if speed is of utmost importance, it makes sense to disable +** the mutexes. But for maximum safety, mutexes should be enabled. +** ^The default behavior is for mutexes to be enabled. +** +** This interface can be used by an application to make sure that the +** version of SQLite that it is linking against was compiled with +** the desired setting of the [SQLITE_THREADSAFE] macro. +** +** This interface only reports on the compile-time mutex setting +** of the [SQLITE_THREADSAFE] flag. If SQLite is compiled with +** SQLITE_THREADSAFE=1 or =2 then mutexes are enabled by default but +** can be fully or partially disabled using a call to [sqlite3_config()] +** with the verbs [SQLITE_CONFIG_SINGLETHREAD], [SQLITE_CONFIG_MULTITHREAD], +** or [SQLITE_CONFIG_SERIALIZED]. ^(The return value of the +** sqlite3_threadsafe() function shows only the compile-time setting of +** thread safety, not any run-time changes to that setting made by +** sqlite3_config(). In other words, the return value from sqlite3_threadsafe() +** is unchanged by calls to sqlite3_config().)^ +** +** See the [threading mode] documentation for additional information. +*/ +SQLITE_API int sqlite3_threadsafe(void); /* ** CAPI3REF: Database Connection Handle +** KEYWORDS: {database connection} {database connections} ** -** Each open SQLite database is represented by pointer to an instance of the -** opaque structure named "sqlite3". It is useful to think of an sqlite3 -** pointer as an object. The [sqlite3_open] interface is its constructor -** and [sqlite3_close] is its destructor. There are many other interfaces -** (such as [sqlite3_prepare_v2], [sqlite3_create_function], and -** [sqlite3_busy_timeout] to name but three) that are methods on this -** object. +** Each open SQLite database is represented by a pointer to an instance of +** the opaque structure named "sqlite3". It is useful to think of an sqlite3 +** pointer as an object. The [sqlite3_open()], [sqlite3_open16()], and +** [sqlite3_open_v2()] interfaces are its constructors, and [sqlite3_close()] +** and [sqlite3_close_v2()] are its destructors. There are many other +** interfaces (such as +** [sqlite3_prepare_v2()], [sqlite3_create_function()], and +** [sqlite3_busy_timeout()] to name but three) that are methods on an +** sqlite3 object. */ typedef struct sqlite3 sqlite3; - /* ** CAPI3REF: 64-Bit Integer Types +** KEYWORDS: sqlite_int64 sqlite_uint64 ** -** Some compilers do not support the "long long" datatype. So we have -** to do compiler-specific typedefs for 64-bit signed and unsigned integers. +** Because there is no cross-platform way to specify 64-bit integer types +** SQLite includes typedefs for 64-bit signed and unsigned integers. ** -** Many SQLite interface functions require a 64-bit integer arguments. -** Those interfaces are declared using this typedef. +** The sqlite3_int64 and sqlite3_uint64 are the preferred type definitions. +** The sqlite_int64 and sqlite_uint64 types are supported for backwards +** compatibility only. +** +** ^The sqlite3_int64 and sqlite_int64 types can store integer values +** between -9223372036854775808 and +9223372036854775807 inclusive. ^The +** sqlite3_uint64 and sqlite_uint64 types can store integer values +** between 0 and +18446744073709551615 inclusive. */ #ifdef SQLITE_INT64_TYPE typedef SQLITE_INT64_TYPE sqlite_int64; - typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# ifdef SQLITE_UINT64_TYPE + typedef SQLITE_UINT64_TYPE sqlite_uint64; +# else + typedef unsigned SQLITE_INT64_TYPE sqlite_uint64; +# endif #elif defined(_MSC_VER) || defined(__BORLANDC__) typedef __int64 sqlite_int64; typedef unsigned __int64 sqlite_uint64; @@ -137,28 +301,57 @@ typedef struct sqlite3 sqlite3; typedef long long int sqlite_int64; typedef unsigned long long int sqlite_uint64; #endif +typedef sqlite_int64 sqlite3_int64; +typedef sqlite_uint64 sqlite3_uint64; /* ** If compiling for a processor that lacks floating point support, -** substitute integer for floating-point +** substitute integer for floating-point. */ #ifdef SQLITE_OMIT_FLOATING_POINT -# define double sqlite_int64 +# define double sqlite3_int64 #endif /* ** CAPI3REF: Closing A Database Connection +** DESTRUCTOR: sqlite3 ** -** Call this function with a pointer to a structure that was previously -** returned from [sqlite3_open()] and the corresponding database will by -** closed. +** ^The sqlite3_close() and sqlite3_close_v2() routines are destructors +** for the [sqlite3] object. +** ^Calls to sqlite3_close() and sqlite3_close_v2() return [SQLITE_OK] if +** the [sqlite3] object is successfully destroyed and all associated +** resources are deallocated. ** -** All SQL statements prepared using [sqlite3_prepare_v2()] or -** [sqlite3_prepare16_v2()] must be destroyed using [sqlite3_finalize()] -** before this routine is called. Otherwise, SQLITE_BUSY is returned and the -** database connection remains open. +** Ideally, applications should [sqlite3_finalize | finalize] all +** [prepared statements], [sqlite3_blob_close | close] all [BLOB handles], and +** [sqlite3_backup_finish | finish] all [sqlite3_backup] objects associated +** with the [sqlite3] object prior to attempting to close the object. +** ^If the database connection is associated with unfinalized prepared +** statements, BLOB handlers, and/or unfinished sqlite3_backup objects then +** sqlite3_close() will leave the database connection open and return +** [SQLITE_BUSY]. ^If sqlite3_close_v2() is called with unfinalized prepared +** statements, unclosed BLOB handlers, and/or unfinished sqlite3_backups, +** it returns [SQLITE_OK] regardless, but instead of deallocating the database +** connection immediately, it marks the database connection as an unusable +** "zombie" and makes arrangements to automatically deallocate the database +** connection after all prepared statements are finalized, all BLOB handles +** are closed, and all backups have finished. The sqlite3_close_v2() interface +** is intended for use with host languages that are garbage collected, and +** where the order in which destructors are called is arbitrary. +** +** ^If an [sqlite3] object is destroyed while a transaction is open, +** the transaction is automatically rolled back. +** +** The C parameter to [sqlite3_close(C)] and [sqlite3_close_v2(C)] +** must be either a NULL +** pointer or an [sqlite3] object pointer obtained +** from [sqlite3_open()], [sqlite3_open16()], or +** [sqlite3_open_v2()], and not previously closed. +** ^Calling sqlite3_close() or sqlite3_close_v2() with a NULL pointer +** argument is a harmless no-op. */ -int sqlite3_close(sqlite3 *); +SQLITE_API int sqlite3_close(sqlite3*); +SQLITE_API int sqlite3_close_v2(sqlite3*); /* ** The type for a callback function. @@ -169,51 +362,71 @@ typedef int (*sqlite3_callback)(void*,int,char**, char**); /* ** CAPI3REF: One-Step Query Execution Interface +** METHOD: sqlite3 ** -** This interface is used to do a one-time evaluatation of zero -** or more SQL statements. UTF-8 text of the SQL statements to -** be evaluted is passed in as the second parameter. The statements -** are prepared one by one using [sqlite3_prepare()], evaluated -** using [sqlite3_step()], then destroyed using [sqlite3_finalize()]. +** The sqlite3_exec() interface is a convenience wrapper around +** [sqlite3_prepare_v2()], [sqlite3_step()], and [sqlite3_finalize()], +** that allows an application to run multiple statements of SQL +** without having to use a lot of C code. ** -** If one or more of the SQL statements are queries, then -** the callback function specified by the 3rd parameter is -** invoked once for each row of the query result. This callback -** should normally return 0. If the callback returns a non-zero -** value then the query is aborted, all subsequent SQL statements -** are skipped and the sqlite3_exec() function returns the SQLITE_ABORT. +** ^The sqlite3_exec() interface runs zero or more UTF-8 encoded, +** semicolon-separate SQL statements passed into its 2nd argument, +** in the context of the [database connection] passed in as its 1st +** argument. ^If the callback function of the 3rd argument to +** sqlite3_exec() is not NULL, then it is invoked for each result row +** coming out of the evaluated SQL statements. ^The 4th argument to +** sqlite3_exec() is relayed through to the 1st argument of each +** callback invocation. ^If the callback pointer to sqlite3_exec() +** is NULL, then no callback is ever invoked and result rows are +** ignored. ** -** The 4th parameter to this interface is an arbitrary pointer that is -** passed through to the callback function as its first parameter. +** ^If an error occurs while evaluating the SQL statements passed into +** sqlite3_exec(), then execution of the current statement stops and +** subsequent statements are skipped. ^If the 5th parameter to sqlite3_exec() +** is not NULL then any error message is written into memory obtained +** from [sqlite3_malloc()] and passed back through the 5th parameter. +** To avoid memory leaks, the application should invoke [sqlite3_free()] +** on error message strings returned through the 5th parameter of +** sqlite3_exec() after the error message string is no longer needed. +** ^If the 5th parameter to sqlite3_exec() is not NULL and no errors +** occur, then sqlite3_exec() sets the pointer in its 5th parameter to +** NULL before returning. ** -** The 2nd parameter to the callback function is the number of -** columns in the query result. The 3rd parameter to the callback -** is an array of strings holding the values for each column -** as extracted using [sqlite3_column_text()]. -** The 4th parameter to the callback is an array of strings -** obtained using [sqlite3_column_name()] and holding -** the names of each column. +** ^If an sqlite3_exec() callback returns non-zero, the sqlite3_exec() +** routine returns SQLITE_ABORT without invoking the callback again and +** without running any subsequent SQL statements. ** -** The callback function may be NULL, even for queries. A NULL -** callback is not an error. It just means that no callback -** will be invoked. +** ^The 2nd argument to the sqlite3_exec() callback function is the +** number of columns in the result. ^The 3rd argument to the sqlite3_exec() +** callback is an array of pointers to strings obtained as if from +** [sqlite3_column_text()], one for each column. ^If an element of a +** result row is NULL then the corresponding string pointer for the +** sqlite3_exec() callback is a NULL pointer. ^The 4th argument to the +** sqlite3_exec() callback is an array of pointers to strings where each +** entry represents the name of corresponding result column as obtained +** from [sqlite3_column_name()]. ** -** If an error occurs while parsing or evaluating the SQL (but -** not while executing the callback) then an appropriate error -** message is written into memory obtained from [sqlite3_malloc()] and -** *errmsg is made to point to that message. The calling function -** is responsible for freeing the memory that holds the error -** message. Use [sqlite3_free()] for this. If errmsg==NULL, -** then no error message is ever written. +** ^If the 2nd parameter to sqlite3_exec() is a NULL pointer, a pointer +** to an empty string, or a pointer that contains only whitespace and/or +** SQL comments, then no SQL statements are evaluated and the database +** is not changed. ** -** The return value is is SQLITE_OK if there are no errors and -** some other [SQLITE_OK | return code] if there is an error. -** The particular return value depends on the type of error. +** Restrictions: ** +**
                  +**
                • The application must ensure that the 1st parameter to sqlite3_exec() +** is a valid and open [database connection]. +**
                • The application must not close the [database connection] specified by +** the 1st parameter to sqlite3_exec() while sqlite3_exec() is running. +**
                • The application must not modify the SQL statement text passed into +** the 2nd parameter of sqlite3_exec() while sqlite3_exec() is running. +**
                • The application must not dereference the arrays or string pointers +** passed as the 3rd and 4th callback parameters after it returns. +**
                */ -int sqlite3_exec( +SQLITE_API int sqlite3_exec( sqlite3*, /* An open database */ - const char *sql, /* SQL to be evaluted */ + const char *sql, /* SQL to be evaluated */ int (*callback)(void*,int,char**,char**), /* Callback function */ void *, /* 1st argument to callback */ char **errmsg /* Error msg written here */ @@ -221,23 +434,19 @@ int sqlite3_exec( /* ** CAPI3REF: Result Codes -** KEYWORDS: SQLITE_OK +** KEYWORDS: {result code definitions} ** ** Many SQLite functions return an integer result code from the set shown -** above in order to indicates success or failure. +** here in order to indicate success or failure. ** -** The result codes above are the only ones returned by SQLite in its -** default configuration. However, the [sqlite3_extended_result_codes()] -** API can be used to set a database connectoin to return more detailed -** result codes. -** -** See also: [SQLITE_IOERR_READ | extended result codes] +** New error codes may be added in future versions of SQLite. ** +** See also: [extended result code definitions] */ #define SQLITE_OK 0 /* Successful result */ /* beginning-of-error-codes */ -#define SQLITE_ERROR 1 /* SQL error or missing database */ -#define SQLITE_INTERNAL 2 /* NOT USED. Internal logic error in SQLite */ +#define SQLITE_ERROR 1 /* Generic error */ +#define SQLITE_INTERNAL 2 /* Internal logic error in SQLite */ #define SQLITE_PERM 3 /* Access permission denied */ #define SQLITE_ABORT 4 /* Callback routine requested an abort */ #define SQLITE_BUSY 5 /* The database file is locked */ @@ -247,222 +456,2387 @@ int sqlite3_exec( #define SQLITE_INTERRUPT 9 /* Operation terminated by sqlite3_interrupt()*/ #define SQLITE_IOERR 10 /* Some kind of disk I/O error occurred */ #define SQLITE_CORRUPT 11 /* The database disk image is malformed */ -#define SQLITE_NOTFOUND 12 /* NOT USED. Table or record not found */ +#define SQLITE_NOTFOUND 12 /* Unknown opcode in sqlite3_file_control() */ #define SQLITE_FULL 13 /* Insertion failed because database is full */ #define SQLITE_CANTOPEN 14 /* Unable to open the database file */ -#define SQLITE_PROTOCOL 15 /* NOT USED. Database lock protocol error */ -#define SQLITE_EMPTY 16 /* Database is empty */ +#define SQLITE_PROTOCOL 15 /* Database lock protocol error */ +#define SQLITE_EMPTY 16 /* Internal use only */ #define SQLITE_SCHEMA 17 /* The database schema changed */ #define SQLITE_TOOBIG 18 /* String or BLOB exceeds size limit */ -#define SQLITE_CONSTRAINT 19 /* Abort due to contraint violation */ +#define SQLITE_CONSTRAINT 19 /* Abort due to constraint violation */ #define SQLITE_MISMATCH 20 /* Data type mismatch */ #define SQLITE_MISUSE 21 /* Library used incorrectly */ #define SQLITE_NOLFS 22 /* Uses OS features not supported on host */ #define SQLITE_AUTH 23 /* Authorization denied */ -#define SQLITE_FORMAT 24 /* Auxiliary database format error */ +#define SQLITE_FORMAT 24 /* Not used */ #define SQLITE_RANGE 25 /* 2nd parameter to sqlite3_bind out of range */ #define SQLITE_NOTADB 26 /* File opened that is not a database file */ +#define SQLITE_NOTICE 27 /* Notifications from sqlite3_log() */ +#define SQLITE_WARNING 28 /* Warnings from sqlite3_log() */ #define SQLITE_ROW 100 /* sqlite3_step() has another row ready */ #define SQLITE_DONE 101 /* sqlite3_step() has finished executing */ /* end-of-error-codes */ /* ** CAPI3REF: Extended Result Codes +** KEYWORDS: {extended result code definitions} ** -** In its default configuration, SQLite API routines return one of 26 integer -** result codes described at result-codes. However, experience has shown that -** many of these result codes are too course-grained. They do not provide as -** much information about problems as users might like. In an effort to -** address this, newer versions of SQLite (version 3.3.8 and later) include +** In its default configuration, SQLite API routines return one of 30 integer +** [result codes]. However, experience has shown that many of +** these result codes are too coarse-grained. They do not provide as +** much information about problems as programmers might like. In an effort to +** address this, newer versions of SQLite (version 3.3.8 [dateof:3.3.8] +** and later) include ** support for additional result codes that provide more detailed information -** about errors. The extended result codes are enabled (or disabled) for -** each database -** connection using the [sqlite3_extended_result_codes()] API. -** -** Some of the available extended result codes are listed above. -** We expect the number of extended result codes will be expand -** over time. Software that uses extended result codes should expect -** to see new result codes in future releases of SQLite. -** -** The symbolic name for an extended result code always contains a related -** primary result code as a prefix. Primary result codes contain a single -** "_" character. Extended result codes contain two or more "_" characters. -** The numeric value of an extended result code can be converted to its -** corresponding primary result code by masking off the lower 8 bytes. -** -** The SQLITE_OK result code will never be extended. It will always -** be exactly zero. +** about errors. These [extended result codes] are enabled or disabled +** on a per database connection basis using the +** [sqlite3_extended_result_codes()] API. Or, the extended code for +** the most recent error can be obtained using +** [sqlite3_extended_errcode()]. */ -#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) -#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) -#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) -#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) -#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) -#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) -#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) -#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) -#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) -#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) -#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_ERROR_MISSING_COLLSEQ (SQLITE_ERROR | (1<<8)) +#define SQLITE_ERROR_RETRY (SQLITE_ERROR | (2<<8)) +#define SQLITE_ERROR_SNAPSHOT (SQLITE_ERROR | (3<<8)) +#define SQLITE_IOERR_READ (SQLITE_IOERR | (1<<8)) +#define SQLITE_IOERR_SHORT_READ (SQLITE_IOERR | (2<<8)) +#define SQLITE_IOERR_WRITE (SQLITE_IOERR | (3<<8)) +#define SQLITE_IOERR_FSYNC (SQLITE_IOERR | (4<<8)) +#define SQLITE_IOERR_DIR_FSYNC (SQLITE_IOERR | (5<<8)) +#define SQLITE_IOERR_TRUNCATE (SQLITE_IOERR | (6<<8)) +#define SQLITE_IOERR_FSTAT (SQLITE_IOERR | (7<<8)) +#define SQLITE_IOERR_UNLOCK (SQLITE_IOERR | (8<<8)) +#define SQLITE_IOERR_RDLOCK (SQLITE_IOERR | (9<<8)) +#define SQLITE_IOERR_DELETE (SQLITE_IOERR | (10<<8)) +#define SQLITE_IOERR_BLOCKED (SQLITE_IOERR | (11<<8)) +#define SQLITE_IOERR_NOMEM (SQLITE_IOERR | (12<<8)) +#define SQLITE_IOERR_ACCESS (SQLITE_IOERR | (13<<8)) +#define SQLITE_IOERR_CHECKRESERVEDLOCK (SQLITE_IOERR | (14<<8)) +#define SQLITE_IOERR_LOCK (SQLITE_IOERR | (15<<8)) +#define SQLITE_IOERR_CLOSE (SQLITE_IOERR | (16<<8)) +#define SQLITE_IOERR_DIR_CLOSE (SQLITE_IOERR | (17<<8)) +#define SQLITE_IOERR_SHMOPEN (SQLITE_IOERR | (18<<8)) +#define SQLITE_IOERR_SHMSIZE (SQLITE_IOERR | (19<<8)) +#define SQLITE_IOERR_SHMLOCK (SQLITE_IOERR | (20<<8)) +#define SQLITE_IOERR_SHMMAP (SQLITE_IOERR | (21<<8)) +#define SQLITE_IOERR_SEEK (SQLITE_IOERR | (22<<8)) +#define SQLITE_IOERR_DELETE_NOENT (SQLITE_IOERR | (23<<8)) +#define SQLITE_IOERR_MMAP (SQLITE_IOERR | (24<<8)) +#define SQLITE_IOERR_GETTEMPPATH (SQLITE_IOERR | (25<<8)) +#define SQLITE_IOERR_CONVPATH (SQLITE_IOERR | (26<<8)) +#define SQLITE_IOERR_VNODE (SQLITE_IOERR | (27<<8)) +#define SQLITE_IOERR_AUTH (SQLITE_IOERR | (28<<8)) +#define SQLITE_IOERR_BEGIN_ATOMIC (SQLITE_IOERR | (29<<8)) +#define SQLITE_IOERR_COMMIT_ATOMIC (SQLITE_IOERR | (30<<8)) +#define SQLITE_IOERR_ROLLBACK_ATOMIC (SQLITE_IOERR | (31<<8)) +#define SQLITE_IOERR_DATA (SQLITE_IOERR | (32<<8)) +#define SQLITE_IOERR_CORRUPTFS (SQLITE_IOERR | (33<<8)) +#define SQLITE_IOERR_IN_PAGE (SQLITE_IOERR | (34<<8)) +#define SQLITE_LOCKED_SHAREDCACHE (SQLITE_LOCKED | (1<<8)) +#define SQLITE_LOCKED_VTAB (SQLITE_LOCKED | (2<<8)) +#define SQLITE_BUSY_RECOVERY (SQLITE_BUSY | (1<<8)) +#define SQLITE_BUSY_SNAPSHOT (SQLITE_BUSY | (2<<8)) +#define SQLITE_BUSY_TIMEOUT (SQLITE_BUSY | (3<<8)) +#define SQLITE_CANTOPEN_NOTEMPDIR (SQLITE_CANTOPEN | (1<<8)) +#define SQLITE_CANTOPEN_ISDIR (SQLITE_CANTOPEN | (2<<8)) +#define SQLITE_CANTOPEN_FULLPATH (SQLITE_CANTOPEN | (3<<8)) +#define SQLITE_CANTOPEN_CONVPATH (SQLITE_CANTOPEN | (4<<8)) +#define SQLITE_CANTOPEN_DIRTYWAL (SQLITE_CANTOPEN | (5<<8)) /* Not Used */ +#define SQLITE_CANTOPEN_SYMLINK (SQLITE_CANTOPEN | (6<<8)) +#define SQLITE_CORRUPT_VTAB (SQLITE_CORRUPT | (1<<8)) +#define SQLITE_CORRUPT_SEQUENCE (SQLITE_CORRUPT | (2<<8)) +#define SQLITE_CORRUPT_INDEX (SQLITE_CORRUPT | (3<<8)) +#define SQLITE_READONLY_RECOVERY (SQLITE_READONLY | (1<<8)) +#define SQLITE_READONLY_CANTLOCK (SQLITE_READONLY | (2<<8)) +#define SQLITE_READONLY_ROLLBACK (SQLITE_READONLY | (3<<8)) +#define SQLITE_READONLY_DBMOVED (SQLITE_READONLY | (4<<8)) +#define SQLITE_READONLY_CANTINIT (SQLITE_READONLY | (5<<8)) +#define SQLITE_READONLY_DIRECTORY (SQLITE_READONLY | (6<<8)) +#define SQLITE_ABORT_ROLLBACK (SQLITE_ABORT | (2<<8)) +#define SQLITE_CONSTRAINT_CHECK (SQLITE_CONSTRAINT | (1<<8)) +#define SQLITE_CONSTRAINT_COMMITHOOK (SQLITE_CONSTRAINT | (2<<8)) +#define SQLITE_CONSTRAINT_FOREIGNKEY (SQLITE_CONSTRAINT | (3<<8)) +#define SQLITE_CONSTRAINT_FUNCTION (SQLITE_CONSTRAINT | (4<<8)) +#define SQLITE_CONSTRAINT_NOTNULL (SQLITE_CONSTRAINT | (5<<8)) +#define SQLITE_CONSTRAINT_PRIMARYKEY (SQLITE_CONSTRAINT | (6<<8)) +#define SQLITE_CONSTRAINT_TRIGGER (SQLITE_CONSTRAINT | (7<<8)) +#define SQLITE_CONSTRAINT_UNIQUE (SQLITE_CONSTRAINT | (8<<8)) +#define SQLITE_CONSTRAINT_VTAB (SQLITE_CONSTRAINT | (9<<8)) +#define SQLITE_CONSTRAINT_ROWID (SQLITE_CONSTRAINT |(10<<8)) +#define SQLITE_CONSTRAINT_PINNED (SQLITE_CONSTRAINT |(11<<8)) +#define SQLITE_CONSTRAINT_DATATYPE (SQLITE_CONSTRAINT |(12<<8)) +#define SQLITE_NOTICE_RECOVER_WAL (SQLITE_NOTICE | (1<<8)) +#define SQLITE_NOTICE_RECOVER_ROLLBACK (SQLITE_NOTICE | (2<<8)) +#define SQLITE_NOTICE_RBU (SQLITE_NOTICE | (3<<8)) +#define SQLITE_WARNING_AUTOINDEX (SQLITE_WARNING | (1<<8)) +#define SQLITE_AUTH_USER (SQLITE_AUTH | (1<<8)) +#define SQLITE_OK_LOAD_PERMANENTLY (SQLITE_OK | (1<<8)) +#define SQLITE_OK_SYMLINK (SQLITE_OK | (2<<8)) /* internal use only */ + +/* +** CAPI3REF: Flags For File Open Operations +** +** These bit values are intended for use in the +** 3rd parameter to the [sqlite3_open_v2()] interface and +** in the 4th parameter to the [sqlite3_vfs.xOpen] method. +** +** Only those flags marked as "Ok for sqlite3_open_v2()" may be +** used as the third argument to the [sqlite3_open_v2()] interface. +** The other flags have historically been ignored by sqlite3_open_v2(), +** though future versions of SQLite might change so that an error is +** raised if any of the disallowed bits are passed into sqlite3_open_v2(). +** Applications should not depend on the historical behavior. +** +** Note in particular that passing the SQLITE_OPEN_EXCLUSIVE flag into +** [sqlite3_open_v2()] does *not* cause the underlying database file +** to be opened using O_EXCL. Passing SQLITE_OPEN_EXCLUSIVE into +** [sqlite3_open_v2()] has historically be a no-op and might become an +** error in future versions of SQLite. +*/ +#define SQLITE_OPEN_READONLY 0x00000001 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_READWRITE 0x00000002 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_CREATE 0x00000004 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_DELETEONCLOSE 0x00000008 /* VFS only */ +#define SQLITE_OPEN_EXCLUSIVE 0x00000010 /* VFS only */ +#define SQLITE_OPEN_AUTOPROXY 0x00000020 /* VFS only */ +#define SQLITE_OPEN_URI 0x00000040 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MEMORY 0x00000080 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_MAIN_DB 0x00000100 /* VFS only */ +#define SQLITE_OPEN_TEMP_DB 0x00000200 /* VFS only */ +#define SQLITE_OPEN_TRANSIENT_DB 0x00000400 /* VFS only */ +#define SQLITE_OPEN_MAIN_JOURNAL 0x00000800 /* VFS only */ +#define SQLITE_OPEN_TEMP_JOURNAL 0x00001000 /* VFS only */ +#define SQLITE_OPEN_SUBJOURNAL 0x00002000 /* VFS only */ +#define SQLITE_OPEN_SUPER_JOURNAL 0x00004000 /* VFS only */ +#define SQLITE_OPEN_NOMUTEX 0x00008000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_FULLMUTEX 0x00010000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_SHAREDCACHE 0x00020000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_PRIVATECACHE 0x00040000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_WAL 0x00080000 /* VFS only */ +#define SQLITE_OPEN_NOFOLLOW 0x01000000 /* Ok for sqlite3_open_v2() */ +#define SQLITE_OPEN_EXRESCODE 0x02000000 /* Extended result codes */ + +/* Reserved: 0x00F00000 */ +/* Legacy compatibility: */ +#define SQLITE_OPEN_MASTER_JOURNAL 0x00004000 /* VFS only */ + + +/* +** CAPI3REF: Device Characteristics +** +** The xDeviceCharacteristics method of the [sqlite3_io_methods] +** object returns an integer which is a vector of these +** bit values expressing I/O characteristics of the mass storage +** device that holds the file that the [sqlite3_io_methods] +** refers to. +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). The SQLITE_IOCAP_POWERSAFE_OVERWRITE property means that +** after reboot following a crash or power loss, the only bytes in a +** file that were written at the application level might have changed +** and that adjacent bytes, even bytes within the same sector are +** guaranteed to be unchanged. The SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN +** flag indicates that a file cannot be deleted when open. The +** SQLITE_IOCAP_IMMUTABLE flag indicates that the file is on +** read-only media and cannot be changed even by processes with +** elevated privileges. +** +** The SQLITE_IOCAP_BATCH_ATOMIC property means that the underlying +** filesystem supports doing multiple write operations atomically when those +** write operations are bracketed by [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] and +** [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. +** +** The SQLITE_IOCAP_SUBPAGE_READ property means that it is ok to read +** from the database file in amounts that are not a multiple of the +** page size and that do not begin at a page boundary. Without this +** property, SQLite is careful to only do full-page reads and write +** on aligned pages, with the one exception that it will do a sub-page +** read of the first page to access the database header. +*/ +#define SQLITE_IOCAP_ATOMIC 0x00000001 +#define SQLITE_IOCAP_ATOMIC512 0x00000002 +#define SQLITE_IOCAP_ATOMIC1K 0x00000004 +#define SQLITE_IOCAP_ATOMIC2K 0x00000008 +#define SQLITE_IOCAP_ATOMIC4K 0x00000010 +#define SQLITE_IOCAP_ATOMIC8K 0x00000020 +#define SQLITE_IOCAP_ATOMIC16K 0x00000040 +#define SQLITE_IOCAP_ATOMIC32K 0x00000080 +#define SQLITE_IOCAP_ATOMIC64K 0x00000100 +#define SQLITE_IOCAP_SAFE_APPEND 0x00000200 +#define SQLITE_IOCAP_SEQUENTIAL 0x00000400 +#define SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN 0x00000800 +#define SQLITE_IOCAP_POWERSAFE_OVERWRITE 0x00001000 +#define SQLITE_IOCAP_IMMUTABLE 0x00002000 +#define SQLITE_IOCAP_BATCH_ATOMIC 0x00004000 +#define SQLITE_IOCAP_SUBPAGE_READ 0x00008000 + +/* +** CAPI3REF: File Locking Levels +** +** SQLite uses one of these integer values as the second +** argument to calls it makes to the xLock() and xUnlock() methods +** of an [sqlite3_io_methods] object. These values are ordered from +** lest restrictive to most restrictive. +** +** The argument to xLock() is always SHARED or higher. The argument to +** xUnlock is either SHARED or NONE. +*/ +#define SQLITE_LOCK_NONE 0 /* xUnlock() only */ +#define SQLITE_LOCK_SHARED 1 /* xLock() or xUnlock() */ +#define SQLITE_LOCK_RESERVED 2 /* xLock() only */ +#define SQLITE_LOCK_PENDING 3 /* xLock() only */ +#define SQLITE_LOCK_EXCLUSIVE 4 /* xLock() only */ + +/* +** CAPI3REF: Synchronization Type Flags +** +** When SQLite invokes the xSync() method of an +** [sqlite3_io_methods] object it uses a combination of +** these integer values as the second argument. +** +** When the SQLITE_SYNC_DATAONLY flag is used, it means that the +** sync operation only needs to flush data to mass storage. Inode +** information need not be flushed. If the lower four bits of the flag +** equal SQLITE_SYNC_NORMAL, that means to use normal fsync() semantics. +** If the lower four bits equal SQLITE_SYNC_FULL, that means +** to use Mac OS X style fullsync instead of fsync(). +** +** Do not confuse the SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags +** with the [PRAGMA synchronous]=NORMAL and [PRAGMA synchronous]=FULL +** settings. The [synchronous pragma] determines when calls to the +** xSync VFS method occur and applies uniformly across all platforms. +** The SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL flags determine how +** energetic or rigorous or forceful the sync operations are and +** only make a difference on Mac OSX for the default SQLite code. +** (Third-party VFS implementations might also make the distinction +** between SQLITE_SYNC_NORMAL and SQLITE_SYNC_FULL, but among the +** operating systems natively supported by SQLite, only Mac OSX +** cares about the difference.) +*/ +#define SQLITE_SYNC_NORMAL 0x00002 +#define SQLITE_SYNC_FULL 0x00003 +#define SQLITE_SYNC_DATAONLY 0x00010 + +/* +** CAPI3REF: OS Interface Open File Handle +** +** An [sqlite3_file] object represents an open file in the +** [sqlite3_vfs | OS interface layer]. Individual OS interface +** implementations will +** want to subclass this object by appending additional fields +** for their own use. The pMethods entry is a pointer to an +** [sqlite3_io_methods] object that defines methods for performing +** I/O operations on the open file. +*/ +typedef struct sqlite3_file sqlite3_file; +struct sqlite3_file { + const struct sqlite3_io_methods *pMethods; /* Methods for an open file */ +}; + +/* +** CAPI3REF: OS Interface File Virtual Methods Object +** +** Every file opened by the [sqlite3_vfs.xOpen] method populates an +** [sqlite3_file] object (or, more commonly, a subclass of the +** [sqlite3_file] object) with a pointer to an instance of this object. +** This object defines the methods used to perform various operations +** against the open file represented by the [sqlite3_file] object. +** +** If the [sqlite3_vfs.xOpen] method sets the sqlite3_file.pMethods element +** to a non-NULL pointer, then the sqlite3_io_methods.xClose method +** may be invoked even if the [sqlite3_vfs.xOpen] reported that it failed. The +** only way to prevent a call to xClose following a failed [sqlite3_vfs.xOpen] +** is for the [sqlite3_vfs.xOpen] to set the sqlite3_file.pMethods element +** to NULL. +** +** The flags argument to xSync may be one of [SQLITE_SYNC_NORMAL] or +** [SQLITE_SYNC_FULL]. The first choice is the normal fsync(). +** The second choice is a Mac OS X style fullsync. The [SQLITE_SYNC_DATAONLY] +** flag may be ORed in to indicate that only the data of the file +** and not its inode needs to be synced. +** +** The integer values to xLock() and xUnlock() are one of +**
                  +**
                • [SQLITE_LOCK_NONE], +**
                • [SQLITE_LOCK_SHARED], +**
                • [SQLITE_LOCK_RESERVED], +**
                • [SQLITE_LOCK_PENDING], or +**
                • [SQLITE_LOCK_EXCLUSIVE]. +**
                +** xLock() upgrades the database file lock. In other words, xLock() moves the +** database file lock in the direction NONE toward EXCLUSIVE. The argument to +** xLock() is always one of SHARED, RESERVED, PENDING, or EXCLUSIVE, never +** SQLITE_LOCK_NONE. If the database file lock is already at or above the +** requested lock, then the call to xLock() is a no-op. +** xUnlock() downgrades the database file lock to either SHARED or NONE. +** If the lock is already at or below the requested lock state, then the call +** to xUnlock() is a no-op. +** The xCheckReservedLock() method checks whether any database connection, +** either in this process or in some other process, is holding a RESERVED, +** PENDING, or EXCLUSIVE lock on the file. It returns, via its output +** pointer parameter, true if such a lock exists and false otherwise. +** +** The xFileControl() method is a generic interface that allows custom +** VFS implementations to directly control an open file using the +** [sqlite3_file_control()] interface. The second "op" argument is an +** integer opcode. The third argument is a generic pointer intended to +** point to a structure that may contain arguments or space in which to +** write return values. Potential uses for xFileControl() might be +** functions to enable blocking locks with timeouts, to change the +** locking strategy (for example to use dot-file locks), to inquire +** about the status of a lock, or to break stale locks. The SQLite +** core reserves all opcodes less than 100 for its own use. +** A [file control opcodes | list of opcodes] less than 100 is available. +** Applications that define a custom xFileControl method should use opcodes +** greater than 100 to avoid conflicts. VFS implementations should +** return [SQLITE_NOTFOUND] for file control opcodes that they do not +** recognize. +** +** The xSectorSize() method returns the sector size of the +** device that underlies the file. The sector size is the +** minimum write that can be performed without disturbing +** other bytes in the file. The xDeviceCharacteristics() +** method returns a bit vector describing behaviors of the +** underlying device: +** +**
                  +**
                • [SQLITE_IOCAP_ATOMIC] +**
                • [SQLITE_IOCAP_ATOMIC512] +**
                • [SQLITE_IOCAP_ATOMIC1K] +**
                • [SQLITE_IOCAP_ATOMIC2K] +**
                • [SQLITE_IOCAP_ATOMIC4K] +**
                • [SQLITE_IOCAP_ATOMIC8K] +**
                • [SQLITE_IOCAP_ATOMIC16K] +**
                • [SQLITE_IOCAP_ATOMIC32K] +**
                • [SQLITE_IOCAP_ATOMIC64K] +**
                • [SQLITE_IOCAP_SAFE_APPEND] +**
                • [SQLITE_IOCAP_SEQUENTIAL] +**
                • [SQLITE_IOCAP_UNDELETABLE_WHEN_OPEN] +**
                • [SQLITE_IOCAP_POWERSAFE_OVERWRITE] +**
                • [SQLITE_IOCAP_IMMUTABLE] +**
                • [SQLITE_IOCAP_BATCH_ATOMIC] +**
                • [SQLITE_IOCAP_SUBPAGE_READ] +**
                +** +** The SQLITE_IOCAP_ATOMIC property means that all writes of +** any size are atomic. The SQLITE_IOCAP_ATOMICnnn values +** mean that writes of blocks that are nnn bytes in size and +** are aligned to an address which is an integer multiple of +** nnn are atomic. The SQLITE_IOCAP_SAFE_APPEND value means +** that when data is appended to a file, the data is appended +** first then the size of the file is extended, never the other +** way around. The SQLITE_IOCAP_SEQUENTIAL property means that +** information is written to disk in the same order as calls +** to xWrite(). +** +** If xRead() returns SQLITE_IOERR_SHORT_READ it must also fill +** in the unread portions of the buffer with zeros. A VFS that +** fails to zero-fill short reads might seem to work. However, +** failure to zero-fill short reads will eventually lead to +** database corruption. +*/ +typedef struct sqlite3_io_methods sqlite3_io_methods; +struct sqlite3_io_methods { + int iVersion; + int (*xClose)(sqlite3_file*); + int (*xRead)(sqlite3_file*, void*, int iAmt, sqlite3_int64 iOfst); + int (*xWrite)(sqlite3_file*, const void*, int iAmt, sqlite3_int64 iOfst); + int (*xTruncate)(sqlite3_file*, sqlite3_int64 size); + int (*xSync)(sqlite3_file*, int flags); + int (*xFileSize)(sqlite3_file*, sqlite3_int64 *pSize); + int (*xLock)(sqlite3_file*, int); + int (*xUnlock)(sqlite3_file*, int); + int (*xCheckReservedLock)(sqlite3_file*, int *pResOut); + int (*xFileControl)(sqlite3_file*, int op, void *pArg); + int (*xSectorSize)(sqlite3_file*); + int (*xDeviceCharacteristics)(sqlite3_file*); + /* Methods above are valid for version 1 */ + int (*xShmMap)(sqlite3_file*, int iPg, int pgsz, int, void volatile**); + int (*xShmLock)(sqlite3_file*, int offset, int n, int flags); + void (*xShmBarrier)(sqlite3_file*); + int (*xShmUnmap)(sqlite3_file*, int deleteFlag); + /* Methods above are valid for version 2 */ + int (*xFetch)(sqlite3_file*, sqlite3_int64 iOfst, int iAmt, void **pp); + int (*xUnfetch)(sqlite3_file*, sqlite3_int64 iOfst, void *p); + /* Methods above are valid for version 3 */ + /* Additional methods may be added in future releases */ +}; + +/* +** CAPI3REF: Standard File Control Opcodes +** KEYWORDS: {file control opcodes} {file control opcode} +** +** These integer constants are opcodes for the xFileControl method +** of the [sqlite3_io_methods] object and for the [sqlite3_file_control()] +** interface. +** +**
                  +**
                • [[SQLITE_FCNTL_LOCKSTATE]] +** The [SQLITE_FCNTL_LOCKSTATE] opcode is used for debugging. This +** opcode causes the xFileControl method to write the current state of +** the lock (one of [SQLITE_LOCK_NONE], [SQLITE_LOCK_SHARED], +** [SQLITE_LOCK_RESERVED], [SQLITE_LOCK_PENDING], or [SQLITE_LOCK_EXCLUSIVE]) +** into an integer that the pArg argument points to. +** This capability is only available if SQLite is compiled with [SQLITE_DEBUG]. +** +**
                • [[SQLITE_FCNTL_SIZE_HINT]] +** The [SQLITE_FCNTL_SIZE_HINT] opcode is used by SQLite to give the VFS +** layer a hint of how large the database file will grow to be during the +** current transaction. This hint is not guaranteed to be accurate but it +** is often close. The underlying VFS might choose to preallocate database +** file space based on this hint in order to help writes to the database +** file run faster. +** +**
                • [[SQLITE_FCNTL_SIZE_LIMIT]] +** The [SQLITE_FCNTL_SIZE_LIMIT] opcode is used by in-memory VFS that +** implements [sqlite3_deserialize()] to set an upper bound on the size +** of the in-memory database. The argument is a pointer to a [sqlite3_int64]. +** If the integer pointed to is negative, then it is filled in with the +** current limit. Otherwise the limit is set to the larger of the value +** of the integer pointed to and the current database size. The integer +** pointed to is set to the new limit. +** +**
                • [[SQLITE_FCNTL_CHUNK_SIZE]] +** The [SQLITE_FCNTL_CHUNK_SIZE] opcode is used to request that the VFS +** extends and truncates the database file in chunks of a size specified +** by the user. The fourth argument to [sqlite3_file_control()] should +** point to an integer (type int) containing the new chunk-size to use +** for the nominated database. Allocating database file space in large +** chunks (say 1MB at a time), may reduce file-system fragmentation and +** improve performance on some systems. +** +**
                • [[SQLITE_FCNTL_FILE_POINTER]] +** The [SQLITE_FCNTL_FILE_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with a particular database +** connection. See also [SQLITE_FCNTL_JOURNAL_POINTER]. +** +**
                • [[SQLITE_FCNTL_JOURNAL_POINTER]] +** The [SQLITE_FCNTL_JOURNAL_POINTER] opcode is used to obtain a pointer +** to the [sqlite3_file] object associated with the journal file (either +** the [rollback journal] or the [write-ahead log]) for a particular database +** connection. See also [SQLITE_FCNTL_FILE_POINTER]. +** +**
                • [[SQLITE_FCNTL_SYNC_OMITTED]] +** No longer in use. +** +**
                • [[SQLITE_FCNTL_SYNC]] +** The [SQLITE_FCNTL_SYNC] opcode is generated internally by SQLite and +** sent to the VFS immediately before the xSync method is invoked on a +** database file descriptor. Or, if the xSync method is not invoked +** because the user has configured SQLite with +** [PRAGMA synchronous | PRAGMA synchronous=OFF] it is invoked in place +** of the xSync method. In most cases, the pointer argument passed with +** this file-control is NULL. However, if the database file is being synced +** as part of a multi-database commit, the argument points to a nul-terminated +** string containing the transactions super-journal file name. VFSes that +** do not need this signal should silently ignore this opcode. Applications +** should not call [sqlite3_file_control()] with this opcode as doing so may +** disrupt the operation of the specialized VFSes that do require it. +** +**
                • [[SQLITE_FCNTL_COMMIT_PHASETWO]] +** The [SQLITE_FCNTL_COMMIT_PHASETWO] opcode is generated internally by SQLite +** and sent to the VFS after a transaction has been committed immediately +** but before the database is unlocked. VFSes that do not need this signal +** should silently ignore this opcode. Applications should not call +** [sqlite3_file_control()] with this opcode as doing so may disrupt the +** operation of the specialized VFSes that do require it. +** +**
                • [[SQLITE_FCNTL_WIN32_AV_RETRY]] +** ^The [SQLITE_FCNTL_WIN32_AV_RETRY] opcode is used to configure automatic +** retry counts and intervals for certain disk I/O operations for the +** windows [VFS] in order to provide robustness in the presence of +** anti-virus programs. By default, the windows VFS will retry file read, +** file write, and file delete operations up to 10 times, with a delay +** of 25 milliseconds before the first retry and with the delay increasing +** by an additional 25 milliseconds with each subsequent retry. This +** opcode allows these two values (10 retries and 25 milliseconds of delay) +** to be adjusted. The values are changed for all database connections +** within the same process. The argument is a pointer to an array of two +** integers where the first integer is the new retry count and the second +** integer is the delay. If either integer is negative, then the setting +** is not changed but instead the prior value of that setting is written +** into the array entry, allowing the current retry settings to be +** interrogated. The zDbName parameter is ignored. +** +**
                • [[SQLITE_FCNTL_PERSIST_WAL]] +** ^The [SQLITE_FCNTL_PERSIST_WAL] opcode is used to set or query the +** persistent [WAL | Write Ahead Log] setting. By default, the auxiliary +** write ahead log ([WAL file]) and shared memory +** files used for transaction control +** are automatically deleted when the latest connection to the database +** closes. Setting persistent WAL mode causes those files to persist after +** close. Persisting the files is useful when other processes that do not +** have write permission on the directory containing the database file want +** to read the database file, as the WAL and shared memory files must exist +** in order for the database to be readable. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable persistent WAL mode or 1 to enable persistent +** WAL mode. If the integer is -1, then it is overwritten with the current +** WAL persistence setting. +** +**
                • [[SQLITE_FCNTL_POWERSAFE_OVERWRITE]] +** ^The [SQLITE_FCNTL_POWERSAFE_OVERWRITE] opcode is used to set or query the +** persistent "powersafe-overwrite" or "PSOW" setting. The PSOW setting +** determines the [SQLITE_IOCAP_POWERSAFE_OVERWRITE] bit of the +** xDeviceCharacteristics methods. The fourth parameter to +** [sqlite3_file_control()] for this opcode should be a pointer to an integer. +** That integer is 0 to disable zero-damage mode or 1 to enable zero-damage +** mode. If the integer is -1, then it is overwritten with the current +** zero-damage mode setting. +** +**
                • [[SQLITE_FCNTL_OVERWRITE]] +** ^The [SQLITE_FCNTL_OVERWRITE] opcode is invoked by SQLite after opening +** a write transaction to indicate that, unless it is rolled back for some +** reason, the entire database file will be overwritten by the current +** transaction. This is used by VACUUM operations. +** +**
                • [[SQLITE_FCNTL_VFSNAME]] +** ^The [SQLITE_FCNTL_VFSNAME] opcode can be used to obtain the names of +** all [VFSes] in the VFS stack. The names are of all VFS shims and the +** final bottom-level VFS are written into memory obtained from +** [sqlite3_malloc()] and the result is stored in the char* variable +** that the fourth parameter of [sqlite3_file_control()] points to. +** The caller is responsible for freeing the memory when done. As with +** all file-control actions, there is no guarantee that this will actually +** do anything. Callers should initialize the char* variable to a NULL +** pointer in case this file-control is not implemented. This file-control +** is intended for diagnostic use only. +** +**
                • [[SQLITE_FCNTL_VFS_POINTER]] +** ^The [SQLITE_FCNTL_VFS_POINTER] opcode finds a pointer to the top-level +** [VFSes] currently in use. ^(The argument X in +** sqlite3_file_control(db,SQLITE_FCNTL_VFS_POINTER,X) must be +** of type "[sqlite3_vfs] **". This opcodes will set *X +** to a pointer to the top-level VFS.)^ +** ^When there are multiple VFS shims in the stack, this opcode finds the +** upper-most shim only. +** +**
                • [[SQLITE_FCNTL_PRAGMA]] +** ^Whenever a [PRAGMA] statement is parsed, an [SQLITE_FCNTL_PRAGMA] +** file control is sent to the open [sqlite3_file] object corresponding +** to the database file to which the pragma statement refers. ^The argument +** to the [SQLITE_FCNTL_PRAGMA] file control is an array of +** pointers to strings (char**) in which the second element of the array +** is the name of the pragma and the third element is the argument to the +** pragma or NULL if the pragma has no argument. ^The handler for an +** [SQLITE_FCNTL_PRAGMA] file control can optionally make the first element +** of the char** argument point to a string obtained from [sqlite3_mprintf()] +** or the equivalent and that string will become the result of the pragma or +** the error message if the pragma fails. ^If the +** [SQLITE_FCNTL_PRAGMA] file control returns [SQLITE_NOTFOUND], then normal +** [PRAGMA] processing continues. ^If the [SQLITE_FCNTL_PRAGMA] +** file control returns [SQLITE_OK], then the parser assumes that the +** VFS has handled the PRAGMA itself and the parser generates a no-op +** prepared statement if result string is NULL, or that returns a copy +** of the result string if the string is non-NULL. +** ^If the [SQLITE_FCNTL_PRAGMA] file control returns +** any result code other than [SQLITE_OK] or [SQLITE_NOTFOUND], that means +** that the VFS encountered an error while handling the [PRAGMA] and the +** compilation of the PRAGMA fails with an error. ^The [SQLITE_FCNTL_PRAGMA] +** file control occurs at the beginning of pragma statement analysis and so +** it is able to override built-in [PRAGMA] statements. +** +**
                • [[SQLITE_FCNTL_BUSYHANDLER]] +** ^The [SQLITE_FCNTL_BUSYHANDLER] +** file-control may be invoked by SQLite on the database file handle +** shortly after it is opened in order to provide a custom VFS with access +** to the connection's busy-handler callback. The argument is of type (void**) +** - an array of two (void *) values. The first (void *) actually points +** to a function of type (int (*)(void *)). In order to invoke the connection's +** busy-handler, this function should be invoked with the second (void *) in +** the array as the only argument. If it returns non-zero, then the operation +** should be retried. If it returns zero, the custom VFS should abandon the +** current operation. +** +**
                • [[SQLITE_FCNTL_TEMPFILENAME]] +** ^Applications can invoke the [SQLITE_FCNTL_TEMPFILENAME] file-control +** to have SQLite generate a +** temporary filename using the same algorithm that is followed to generate +** temporary filenames for TEMP tables and other internal uses. The +** argument should be a char** which will be filled with the filename +** written into memory obtained from [sqlite3_malloc()]. The caller should +** invoke [sqlite3_free()] on the result to avoid a memory leak. +** +**
                • [[SQLITE_FCNTL_MMAP_SIZE]] +** The [SQLITE_FCNTL_MMAP_SIZE] file control is used to query or set the +** maximum number of bytes that will be used for memory-mapped I/O. +** The argument is a pointer to a value of type sqlite3_int64 that +** is an advisory maximum number of bytes in the file to memory map. The +** pointer is overwritten with the old value. The limit is not changed if +** the value originally pointed to is negative, and so the current limit +** can be queried by passing in a pointer to a negative number. This +** file-control is used internally to implement [PRAGMA mmap_size]. +** +**
                • [[SQLITE_FCNTL_TRACE]] +** The [SQLITE_FCNTL_TRACE] file control provides advisory information +** to the VFS about what the higher layers of the SQLite stack are doing. +** This file control is used by some VFS activity tracing [shims]. +** The argument is a zero-terminated string. Higher layers in the +** SQLite stack may generate instances of this file control if +** the [SQLITE_USE_FCNTL_TRACE] compile-time option is enabled. +** +**
                • [[SQLITE_FCNTL_HAS_MOVED]] +** The [SQLITE_FCNTL_HAS_MOVED] file control interprets its argument as a +** pointer to an integer and it writes a boolean into that integer depending +** on whether or not the file has been renamed, moved, or deleted since it +** was first opened. +** +**
                • [[SQLITE_FCNTL_WIN32_GET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_GET_HANDLE] opcode can be used to obtain the +** underlying native file handle associated with a file handle. This file +** control interprets its argument as a pointer to a native file handle and +** writes the resulting value there. +** +**
                • [[SQLITE_FCNTL_WIN32_SET_HANDLE]] +** The [SQLITE_FCNTL_WIN32_SET_HANDLE] opcode is used for debugging. This +** opcode causes the xFileControl method to swap the file handle with the one +** pointed to by the pArg argument. This capability is used during testing +** and only needs to be supported when SQLITE_TEST is defined. +** +**
                • [[SQLITE_FCNTL_WAL_BLOCK]] +** The [SQLITE_FCNTL_WAL_BLOCK] is a signal to the VFS layer that it might +** be advantageous to block on the next WAL lock if the lock is not immediately +** available. The WAL subsystem issues this signal during rare +** circumstances in order to fix a problem with priority inversion. +** Applications should not use this file-control. +** +**
                • [[SQLITE_FCNTL_ZIPVFS]] +** The [SQLITE_FCNTL_ZIPVFS] opcode is implemented by zipvfs only. All other +** VFS should return SQLITE_NOTFOUND for this opcode. +** +**
                • [[SQLITE_FCNTL_RBU]] +** The [SQLITE_FCNTL_RBU] opcode is implemented by the special VFS used by +** the RBU extension only. All other VFS should return SQLITE_NOTFOUND for +** this opcode. +** +**
                • [[SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]] +** If the [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] opcode returns SQLITE_OK, then +** the file descriptor is placed in "batch write mode", which +** means all subsequent write operations will be deferred and done +** atomically at the next [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]. Systems +** that do not support batch atomic writes will return SQLITE_NOTFOUND. +** ^Following a successful SQLITE_FCNTL_BEGIN_ATOMIC_WRITE and prior to +** the closing [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] or +** [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE], SQLite will make +** no VFS interface calls on the same [sqlite3_file] file descriptor +** except for calls to the xWrite method and the xFileControl method +** with [SQLITE_FCNTL_SIZE_HINT]. +** +**
                • [[SQLITE_FCNTL_COMMIT_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_COMMIT_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be performed atomically. +** This file control returns [SQLITE_OK] if and only if the writes were +** all performed successfully and have been committed to persistent storage. +** ^Regardless of whether or not it is successful, this file control takes +** the file descriptor out of batch write mode so that all subsequent +** write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_COMMIT_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
                • [[SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE]] +** The [SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE] opcode causes all write +** operations since the previous successful call to +** [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE] to be rolled back. +** ^This file control takes the file descriptor out of batch write mode +** so that all subsequent write operations are independent. +** ^SQLite will never invoke SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE without +** a prior successful call to [SQLITE_FCNTL_BEGIN_ATOMIC_WRITE]. +** +**
                • [[SQLITE_FCNTL_LOCK_TIMEOUT]] +** The [SQLITE_FCNTL_LOCK_TIMEOUT] opcode is used to configure a VFS +** to block for up to M milliseconds before failing when attempting to +** obtain a file lock using the xLock or xShmLock methods of the VFS. +** The parameter is a pointer to a 32-bit signed integer that contains +** the value that M is to be set to. Before returning, the 32-bit signed +** integer is overwritten with the previous value of M. +** +**
                • [[SQLITE_FCNTL_DATA_VERSION]] +** The [SQLITE_FCNTL_DATA_VERSION] opcode is used to detect changes to +** a database file. The argument is a pointer to a 32-bit unsigned integer. +** The "data version" for the pager is written into the pointer. The +** "data version" changes whenever any change occurs to the corresponding +** database file, either through SQL statements on the same database +** connection or through transactions committed by separate database +** connections possibly in other processes. The [sqlite3_total_changes()] +** interface can be used to find if any database on the connection has changed, +** but that interface responds to changes on TEMP as well as MAIN and does +** not provide a mechanism to detect changes to MAIN only. Also, the +** [sqlite3_total_changes()] interface responds to internal changes only and +** omits changes made by other database connections. The +** [PRAGMA data_version] command provides a mechanism to detect changes to +** a single attached database that occur due to other database connections, +** but omits changes implemented by the database connection on which it is +** called. This file control is the only mechanism to detect changes that +** happen either internally or externally and that are associated with +** a particular attached database. +** +**
                • [[SQLITE_FCNTL_CKPT_START]] +** The [SQLITE_FCNTL_CKPT_START] opcode is invoked from within a checkpoint +** in wal mode before the client starts to copy pages from the wal +** file to the database file. +** +**
                • [[SQLITE_FCNTL_CKPT_DONE]] +** The [SQLITE_FCNTL_CKPT_DONE] opcode is invoked from within a checkpoint +** in wal mode after the client has finished copying pages from the wal +** file to the database file, but before the *-shm file is updated to +** record the fact that the pages have been checkpointed. +** +**
                • [[SQLITE_FCNTL_EXTERNAL_READER]] +** The EXPERIMENTAL [SQLITE_FCNTL_EXTERNAL_READER] opcode is used to detect +** whether or not there is a database client in another process with a wal-mode +** transaction open on the database or not. It is only available on unix.The +** (void*) argument passed with this file-control should be a pointer to a +** value of type (int). The integer value is set to 1 if the database is a wal +** mode database and there exists at least one client in another process that +** currently has an SQL transaction open on the database. It is set to 0 if +** the database is not a wal-mode db, or if there is no such connection in any +** other process. This opcode cannot be used to detect transactions opened +** by clients within the current process, only within other processes. +** +**
                • [[SQLITE_FCNTL_CKSM_FILE]] +** The [SQLITE_FCNTL_CKSM_FILE] opcode is for use internally by the +** [checksum VFS shim] only. +** +**
                • [[SQLITE_FCNTL_RESET_CACHE]] +** If there is currently no transaction open on the database, and the +** database is not a temp db, then the [SQLITE_FCNTL_RESET_CACHE] file-control +** purges the contents of the in-memory page cache. If there is an open +** transaction, or if the db is a temp-db, this opcode is a no-op, not an error. +**
                +*/ +#define SQLITE_FCNTL_LOCKSTATE 1 +#define SQLITE_FCNTL_GET_LOCKPROXYFILE 2 +#define SQLITE_FCNTL_SET_LOCKPROXYFILE 3 +#define SQLITE_FCNTL_LAST_ERRNO 4 +#define SQLITE_FCNTL_SIZE_HINT 5 +#define SQLITE_FCNTL_CHUNK_SIZE 6 +#define SQLITE_FCNTL_FILE_POINTER 7 +#define SQLITE_FCNTL_SYNC_OMITTED 8 +#define SQLITE_FCNTL_WIN32_AV_RETRY 9 +#define SQLITE_FCNTL_PERSIST_WAL 10 +#define SQLITE_FCNTL_OVERWRITE 11 +#define SQLITE_FCNTL_VFSNAME 12 +#define SQLITE_FCNTL_POWERSAFE_OVERWRITE 13 +#define SQLITE_FCNTL_PRAGMA 14 +#define SQLITE_FCNTL_BUSYHANDLER 15 +#define SQLITE_FCNTL_TEMPFILENAME 16 +#define SQLITE_FCNTL_MMAP_SIZE 18 +#define SQLITE_FCNTL_TRACE 19 +#define SQLITE_FCNTL_HAS_MOVED 20 +#define SQLITE_FCNTL_SYNC 21 +#define SQLITE_FCNTL_COMMIT_PHASETWO 22 +#define SQLITE_FCNTL_WIN32_SET_HANDLE 23 +#define SQLITE_FCNTL_WAL_BLOCK 24 +#define SQLITE_FCNTL_ZIPVFS 25 +#define SQLITE_FCNTL_RBU 26 +#define SQLITE_FCNTL_VFS_POINTER 27 +#define SQLITE_FCNTL_JOURNAL_POINTER 28 +#define SQLITE_FCNTL_WIN32_GET_HANDLE 29 +#define SQLITE_FCNTL_PDB 30 +#define SQLITE_FCNTL_BEGIN_ATOMIC_WRITE 31 +#define SQLITE_FCNTL_COMMIT_ATOMIC_WRITE 32 +#define SQLITE_FCNTL_ROLLBACK_ATOMIC_WRITE 33 +#define SQLITE_FCNTL_LOCK_TIMEOUT 34 +#define SQLITE_FCNTL_DATA_VERSION 35 +#define SQLITE_FCNTL_SIZE_LIMIT 36 +#define SQLITE_FCNTL_CKPT_DONE 37 +#define SQLITE_FCNTL_RESERVE_BYTES 38 +#define SQLITE_FCNTL_CKPT_START 39 +#define SQLITE_FCNTL_EXTERNAL_READER 40 +#define SQLITE_FCNTL_CKSM_FILE 41 +#define SQLITE_FCNTL_RESET_CACHE 42 + +/* deprecated names */ +#define SQLITE_GET_LOCKPROXYFILE SQLITE_FCNTL_GET_LOCKPROXYFILE +#define SQLITE_SET_LOCKPROXYFILE SQLITE_FCNTL_SET_LOCKPROXYFILE +#define SQLITE_LAST_ERRNO SQLITE_FCNTL_LAST_ERRNO + + +/* +** CAPI3REF: Mutex Handle +** +** The mutex module within SQLite defines [sqlite3_mutex] to be an +** abstract type for a mutex object. The SQLite core never looks +** at the internal representation of an [sqlite3_mutex]. It only +** deals with pointers to the [sqlite3_mutex] object. +** +** Mutexes are created using [sqlite3_mutex_alloc()]. +*/ +typedef struct sqlite3_mutex sqlite3_mutex; + +/* +** CAPI3REF: Loadable Extension Thunk +** +** A pointer to the opaque sqlite3_api_routines structure is passed as +** the third parameter to entry points of [loadable extensions]. This +** structure must be typedefed in order to work around compiler warnings +** on some platforms. +*/ +typedef struct sqlite3_api_routines sqlite3_api_routines; + +/* +** CAPI3REF: File Name +** +** Type [sqlite3_filename] is used by SQLite to pass filenames to the +** xOpen method of a [VFS]. It may be cast to (const char*) and treated +** as a normal, nul-terminated, UTF-8 buffer containing the filename, but +** may also be passed to special APIs such as: +** +**
                  +**
                • sqlite3_filename_database() +**
                • sqlite3_filename_journal() +**
                • sqlite3_filename_wal() +**
                • sqlite3_uri_parameter() +**
                • sqlite3_uri_boolean() +**
                • sqlite3_uri_int64() +**
                • sqlite3_uri_key() +**
                +*/ +typedef const char *sqlite3_filename; + +/* +** CAPI3REF: OS Interface Object +** +** An instance of the sqlite3_vfs object defines the interface between +** the SQLite core and the underlying operating system. The "vfs" +** in the name of the object stands for "virtual file system". See +** the [VFS | VFS documentation] for further information. +** +** The VFS interface is sometimes extended by adding new methods onto +** the end. Each time such an extension occurs, the iVersion field +** is incremented. The iVersion value started out as 1 in +** SQLite [version 3.5.0] on [dateof:3.5.0], then increased to 2 +** with SQLite [version 3.7.0] on [dateof:3.7.0], and then increased +** to 3 with SQLite [version 3.7.6] on [dateof:3.7.6]. Additional fields +** may be appended to the sqlite3_vfs object and the iVersion value +** may increase again in future versions of SQLite. +** Note that due to an oversight, the structure +** of the sqlite3_vfs object changed in the transition from +** SQLite [version 3.5.9] to [version 3.6.0] on [dateof:3.6.0] +** and yet the iVersion field was not increased. +** +** The szOsFile field is the size of the subclassed [sqlite3_file] +** structure used by this VFS. mxPathname is the maximum length of +** a pathname in this VFS. +** +** Registered sqlite3_vfs objects are kept on a linked list formed by +** the pNext pointer. The [sqlite3_vfs_register()] +** and [sqlite3_vfs_unregister()] interfaces manage this list +** in a thread-safe way. The [sqlite3_vfs_find()] interface +** searches the list. Neither the application code nor the VFS +** implementation should use the pNext pointer. +** +** The pNext field is the only field in the sqlite3_vfs +** structure that SQLite will ever modify. SQLite will only access +** or modify this field while holding a particular static mutex. +** The application should never modify anything within the sqlite3_vfs +** object once the object has been registered. +** +** The zName field holds the name of the VFS module. The name must +** be unique across all VFS modules. +** +** [[sqlite3_vfs.xOpen]] +** ^SQLite guarantees that the zFilename parameter to xOpen +** is either a NULL pointer or string obtained +** from xFullPathname() with an optional suffix added. +** ^If a suffix is added to the zFilename parameter, it will +** consist of a single "-" character followed by no more than +** 11 alphanumeric and/or "-" characters. +** ^SQLite further guarantees that +** the string will be valid and unchanged until xClose() is +** called. Because of the previous sentence, +** the [sqlite3_file] can safely store a pointer to the +** filename if it needs to remember the filename for some reason. +** If the zFilename parameter to xOpen is a NULL pointer then xOpen +** must invent its own temporary name for the file. ^Whenever the +** xFilename parameter is NULL it will also be the case that the +** flags parameter will include [SQLITE_OPEN_DELETEONCLOSE]. +** +** The flags argument to xOpen() includes all bits set in +** the flags argument to [sqlite3_open_v2()]. Or if [sqlite3_open()] +** or [sqlite3_open16()] is used, then flags includes at least +** [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]. +** If xOpen() opens a file read-only then it sets *pOutFlags to +** include [SQLITE_OPEN_READONLY]. Other bits in *pOutFlags may be set. +** +** ^(SQLite will also add one of the following flags to the xOpen() +** call, depending on the object being opened: +** +**
                  +**
                • [SQLITE_OPEN_MAIN_DB] +**
                • [SQLITE_OPEN_MAIN_JOURNAL] +**
                • [SQLITE_OPEN_TEMP_DB] +**
                • [SQLITE_OPEN_TEMP_JOURNAL] +**
                • [SQLITE_OPEN_TRANSIENT_DB] +**
                • [SQLITE_OPEN_SUBJOURNAL] +**
                • [SQLITE_OPEN_SUPER_JOURNAL] +**
                • [SQLITE_OPEN_WAL] +**
                )^ +** +** The file I/O implementation can use the object type flags to +** change the way it deals with files. For example, an application +** that does not care about crash recovery or rollback might make +** the open of a journal file a no-op. Writes to this journal would +** also be no-ops, and any attempt to read the journal would return +** SQLITE_IOERR. Or the implementation might recognize that a database +** file will be doing page-aligned sector reads and writes in a random +** order and set up its I/O subsystem accordingly. +** +** SQLite might also add one of the following flags to the xOpen method: +** +**
                  +**
                • [SQLITE_OPEN_DELETEONCLOSE] +**
                • [SQLITE_OPEN_EXCLUSIVE] +**
                +** +** The [SQLITE_OPEN_DELETEONCLOSE] flag means the file should be +** deleted when it is closed. ^The [SQLITE_OPEN_DELETEONCLOSE] +** will be set for TEMP databases and their journals, transient +** databases, and subjournals. +** +** ^The [SQLITE_OPEN_EXCLUSIVE] flag is always used in conjunction +** with the [SQLITE_OPEN_CREATE] flag, which are both directly +** analogous to the O_EXCL and O_CREAT flags of the POSIX open() +** API. The SQLITE_OPEN_EXCLUSIVE flag, when paired with the +** SQLITE_OPEN_CREATE, is used to indicate that file should always +** be created, and that it is an error if it already exists. +** It is not used to indicate the file should be opened +** for exclusive access. +** +** ^At least szOsFile bytes of memory are allocated by SQLite +** to hold the [sqlite3_file] structure passed as the third +** argument to xOpen. The xOpen method does not have to +** allocate the structure; it should just fill it in. Note that +** the xOpen method must set the sqlite3_file.pMethods to either +** a valid [sqlite3_io_methods] object or to NULL. xOpen must do +** this even if the open fails. SQLite expects that the sqlite3_file.pMethods +** element will be valid after xOpen returns regardless of the success +** or failure of the xOpen call. +** +** [[sqlite3_vfs.xAccess]] +** ^The flags argument to xAccess() may be [SQLITE_ACCESS_EXISTS] +** to test for the existence of a file, or [SQLITE_ACCESS_READWRITE] to +** test whether a file is readable and writable, or [SQLITE_ACCESS_READ] +** to test whether a file is at least readable. The SQLITE_ACCESS_READ +** flag is never actually used and is not implemented in the built-in +** VFSes of SQLite. The file is named by the second argument and can be a +** directory. The xAccess method returns [SQLITE_OK] on success or some +** non-zero error code if there is an I/O error or if the name of +** the file given in the second argument is illegal. If SQLITE_OK +** is returned, then non-zero or zero is written into *pResOut to indicate +** whether or not the file is accessible. +** +** ^SQLite will always allocate at least mxPathname+1 bytes for the +** output buffer xFullPathname. The exact size of the output buffer +** is also passed as a parameter to both methods. If the output buffer +** is not large enough, [SQLITE_CANTOPEN] should be returned. Since this is +** handled as a fatal error by SQLite, vfs implementations should endeavor +** to prevent this by setting mxPathname to a sufficiently large value. +** +** The xRandomness(), xSleep(), xCurrentTime(), and xCurrentTimeInt64() +** interfaces are not strictly a part of the filesystem, but they are +** included in the VFS structure for completeness. +** The xRandomness() function attempts to return nBytes bytes +** of good-quality randomness into zOut. The return value is +** the actual number of bytes of randomness obtained. +** The xSleep() method causes the calling thread to sleep for at +** least the number of microseconds given. ^The xCurrentTime() +** method returns a Julian Day Number for the current date and time as +** a floating point value. +** ^The xCurrentTimeInt64() method returns, as an integer, the Julian +** Day Number multiplied by 86400000 (the number of milliseconds in +** a 24-hour day). +** ^SQLite will use the xCurrentTimeInt64() method to get the current +** date and time if that method is available (if iVersion is 2 or +** greater and the function pointer is not NULL) and will fall back +** to xCurrentTime() if xCurrentTimeInt64() is unavailable. +** +** ^The xSetSystemCall(), xGetSystemCall(), and xNestSystemCall() interfaces +** are not used by the SQLite core. These optional interfaces are provided +** by some VFSes to facilitate testing of the VFS code. By overriding +** system calls with functions under its control, a test program can +** simulate faults and error conditions that would otherwise be difficult +** or impossible to induce. The set of system calls that can be overridden +** varies from one VFS to another, and from one version of the same VFS to the +** next. Applications that use these interfaces must be prepared for any +** or all of these interfaces to be NULL or for their behavior to change +** from one release to the next. Applications must not attempt to access +** any of these methods if the iVersion of the VFS is less than 3. +*/ +typedef struct sqlite3_vfs sqlite3_vfs; +typedef void (*sqlite3_syscall_ptr)(void); +struct sqlite3_vfs { + int iVersion; /* Structure version number (currently 3) */ + int szOsFile; /* Size of subclassed sqlite3_file */ + int mxPathname; /* Maximum file pathname length */ + sqlite3_vfs *pNext; /* Next registered VFS */ + const char *zName; /* Name of this virtual file system */ + void *pAppData; /* Pointer to application-specific data */ + int (*xOpen)(sqlite3_vfs*, sqlite3_filename zName, sqlite3_file*, + int flags, int *pOutFlags); + int (*xDelete)(sqlite3_vfs*, const char *zName, int syncDir); + int (*xAccess)(sqlite3_vfs*, const char *zName, int flags, int *pResOut); + int (*xFullPathname)(sqlite3_vfs*, const char *zName, int nOut, char *zOut); + void *(*xDlOpen)(sqlite3_vfs*, const char *zFilename); + void (*xDlError)(sqlite3_vfs*, int nByte, char *zErrMsg); + void (*(*xDlSym)(sqlite3_vfs*,void*, const char *zSymbol))(void); + void (*xDlClose)(sqlite3_vfs*, void*); + int (*xRandomness)(sqlite3_vfs*, int nByte, char *zOut); + int (*xSleep)(sqlite3_vfs*, int microseconds); + int (*xCurrentTime)(sqlite3_vfs*, double*); + int (*xGetLastError)(sqlite3_vfs*, int, char *); + /* + ** The methods above are in version 1 of the sqlite_vfs object + ** definition. Those that follow are added in version 2 or later + */ + int (*xCurrentTimeInt64)(sqlite3_vfs*, sqlite3_int64*); + /* + ** The methods above are in versions 1 and 2 of the sqlite_vfs object. + ** Those below are for version 3 and greater. + */ + int (*xSetSystemCall)(sqlite3_vfs*, const char *zName, sqlite3_syscall_ptr); + sqlite3_syscall_ptr (*xGetSystemCall)(sqlite3_vfs*, const char *zName); + const char *(*xNextSystemCall)(sqlite3_vfs*, const char *zName); + /* + ** The methods above are in versions 1 through 3 of the sqlite_vfs object. + ** New fields may be appended in future versions. The iVersion + ** value will increment whenever this happens. + */ +}; + +/* +** CAPI3REF: Flags for the xAccess VFS method +** +** These integer constants can be used as the third parameter to +** the xAccess method of an [sqlite3_vfs] object. They determine +** what kind of permissions the xAccess method is looking for. +** With SQLITE_ACCESS_EXISTS, the xAccess method +** simply checks whether the file exists. +** With SQLITE_ACCESS_READWRITE, the xAccess method +** checks whether the named directory is both readable and writable +** (in other words, if files can be added, removed, and renamed within +** the directory). +** The SQLITE_ACCESS_READWRITE constant is currently used only by the +** [temp_store_directory pragma], though this could change in a future +** release of SQLite. +** With SQLITE_ACCESS_READ, the xAccess method +** checks whether the file is readable. The SQLITE_ACCESS_READ constant is +** currently unused, though it might be used in a future release of +** SQLite. +*/ +#define SQLITE_ACCESS_EXISTS 0 +#define SQLITE_ACCESS_READWRITE 1 /* Used by PRAGMA temp_store_directory */ +#define SQLITE_ACCESS_READ 2 /* Unused */ + +/* +** CAPI3REF: Flags for the xShmLock VFS method +** +** These integer constants define the various locking operations +** allowed by the xShmLock method of [sqlite3_io_methods]. The +** following are the only legal combinations of flags to the +** xShmLock method: +** +**
                  +**
                • SQLITE_SHM_LOCK | SQLITE_SHM_SHARED +**
                • SQLITE_SHM_LOCK | SQLITE_SHM_EXCLUSIVE +**
                • SQLITE_SHM_UNLOCK | SQLITE_SHM_SHARED +**
                • SQLITE_SHM_UNLOCK | SQLITE_SHM_EXCLUSIVE +**
                +** +** When unlocking, the same SHARED or EXCLUSIVE flag must be supplied as +** was given on the corresponding lock. +** +** The xShmLock method can transition between unlocked and SHARED or +** between unlocked and EXCLUSIVE. It cannot transition between SHARED +** and EXCLUSIVE. +*/ +#define SQLITE_SHM_UNLOCK 1 +#define SQLITE_SHM_LOCK 2 +#define SQLITE_SHM_SHARED 4 +#define SQLITE_SHM_EXCLUSIVE 8 + +/* +** CAPI3REF: Maximum xShmLock index +** +** The xShmLock method on [sqlite3_io_methods] may use values +** between 0 and this upper bound as its "offset" argument. +** The SQLite core will never attempt to acquire or release a +** lock outside of this range +*/ +#define SQLITE_SHM_NLOCK 8 + + +/* +** CAPI3REF: Initialize The SQLite Library +** +** ^The sqlite3_initialize() routine initializes the +** SQLite library. ^The sqlite3_shutdown() routine +** deallocates any resources that were allocated by sqlite3_initialize(). +** These routines are designed to aid in process initialization and +** shutdown on embedded systems. Workstation applications using +** SQLite normally do not need to invoke either of these routines. +** +** A call to sqlite3_initialize() is an "effective" call if it is +** the first time sqlite3_initialize() is invoked during the lifetime of +** the process, or if it is the first time sqlite3_initialize() is invoked +** following a call to sqlite3_shutdown(). ^(Only an effective call +** of sqlite3_initialize() does any initialization. All other calls +** are harmless no-ops.)^ +** +** A call to sqlite3_shutdown() is an "effective" call if it is the first +** call to sqlite3_shutdown() since the last sqlite3_initialize(). ^(Only +** an effective call to sqlite3_shutdown() does any deinitialization. +** All other valid calls to sqlite3_shutdown() are harmless no-ops.)^ +** +** The sqlite3_initialize() interface is threadsafe, but sqlite3_shutdown() +** is not. The sqlite3_shutdown() interface must only be called from a +** single thread. All open [database connections] must be closed and all +** other SQLite resources must be deallocated prior to invoking +** sqlite3_shutdown(). +** +** Among other things, ^sqlite3_initialize() will invoke +** sqlite3_os_init(). Similarly, ^sqlite3_shutdown() +** will invoke sqlite3_os_end(). +** +** ^The sqlite3_initialize() routine returns [SQLITE_OK] on success. +** ^If for some reason, sqlite3_initialize() is unable to initialize +** the library (perhaps it is unable to allocate a needed resource such +** as a mutex) it returns an [error code] other than [SQLITE_OK]. +** +** ^The sqlite3_initialize() routine is called internally by many other +** SQLite interfaces so that an application usually does not need to +** invoke sqlite3_initialize() directly. For example, [sqlite3_open()] +** calls sqlite3_initialize() so the SQLite library will be automatically +** initialized when [sqlite3_open()] is called if it has not be initialized +** already. ^However, if SQLite is compiled with the [SQLITE_OMIT_AUTOINIT] +** compile-time option, then the automatic calls to sqlite3_initialize() +** are omitted and the application must call sqlite3_initialize() directly +** prior to using any other SQLite interface. For maximum portability, +** it is recommended that applications always invoke sqlite3_initialize() +** directly prior to using any other SQLite interface. Future releases +** of SQLite may require this. In other words, the behavior exhibited +** when SQLite is compiled with [SQLITE_OMIT_AUTOINIT] might become the +** default behavior in some future release of SQLite. +** +** The sqlite3_os_init() routine does operating-system specific +** initialization of the SQLite library. The sqlite3_os_end() +** routine undoes the effect of sqlite3_os_init(). Typical tasks +** performed by these routines include allocation or deallocation +** of static resources, initialization of global variables, +** setting up a default [sqlite3_vfs] module, or setting up +** a default configuration using [sqlite3_config()]. +** +** The application should never invoke either sqlite3_os_init() +** or sqlite3_os_end() directly. The application should only invoke +** sqlite3_initialize() and sqlite3_shutdown(). The sqlite3_os_init() +** interface is called automatically by sqlite3_initialize() and +** sqlite3_os_end() is called by sqlite3_shutdown(). Appropriate +** implementations for sqlite3_os_init() and sqlite3_os_end() +** are built into SQLite when it is compiled for Unix, Windows, or OS/2. +** When [custom builds | built for other platforms] +** (using the [SQLITE_OS_OTHER=1] compile-time +** option) the application must supply a suitable implementation for +** sqlite3_os_init() and sqlite3_os_end(). An application-supplied +** implementation of sqlite3_os_init() or sqlite3_os_end() +** must return [SQLITE_OK] on success and some other [error code] upon +** failure. +*/ +SQLITE_API int sqlite3_initialize(void); +SQLITE_API int sqlite3_shutdown(void); +SQLITE_API int sqlite3_os_init(void); +SQLITE_API int sqlite3_os_end(void); + +/* +** CAPI3REF: Configuring The SQLite Library +** +** The sqlite3_config() interface is used to make global configuration +** changes to SQLite in order to tune SQLite to the specific needs of +** the application. The default configuration is recommended for most +** applications and so this routine is usually not necessary. It is +** provided to support rare applications with unusual needs. +** +** The sqlite3_config() interface is not threadsafe. The application +** must ensure that no other SQLite interfaces are invoked by other +** threads while sqlite3_config() is running. +** +** The first argument to sqlite3_config() is an integer +** [configuration option] that determines +** what property of SQLite is to be configured. Subsequent arguments +** vary depending on the [configuration option] +** in the first argument. +** +** For most configuration options, the sqlite3_config() interface +** may only be invoked prior to library initialization using +** [sqlite3_initialize()] or after shutdown by [sqlite3_shutdown()]. +** The exceptional configuration options that may be invoked at any time +** are called "anytime configuration options". +** ^If sqlite3_config() is called after [sqlite3_initialize()] and before +** [sqlite3_shutdown()] with a first argument that is not an anytime +** configuration option, then the sqlite3_config() call will return SQLITE_MISUSE. +** Note, however, that ^sqlite3_config() can be called as part of the +** implementation of an application-defined [sqlite3_os_init()]. +** +** ^When a configuration option is set, sqlite3_config() returns [SQLITE_OK]. +** ^If the option is unknown or SQLite is unable to set the option +** then this routine returns a non-zero [error code]. +*/ +SQLITE_API int sqlite3_config(int, ...); + +/* +** CAPI3REF: Configure database connections +** METHOD: sqlite3 +** +** The sqlite3_db_config() interface is used to make configuration +** changes to a [database connection]. The interface is similar to +** [sqlite3_config()] except that the changes apply to a single +** [database connection] (specified in the first argument). +** +** The second argument to sqlite3_db_config(D,V,...) is the +** [SQLITE_DBCONFIG_LOOKASIDE | configuration verb] - an integer code +** that indicates what aspect of the [database connection] is being configured. +** Subsequent arguments vary depending on the configuration verb. +** +** ^Calls to sqlite3_db_config() return SQLITE_OK if and only if +** the call is considered successful. +*/ +SQLITE_API int sqlite3_db_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Memory Allocation Routines +** +** An instance of this object defines the interface between SQLite +** and low-level memory allocation routines. +** +** This object is used in only one place in the SQLite interface. +** A pointer to an instance of this object is the argument to +** [sqlite3_config()] when the configuration option is +** [SQLITE_CONFIG_MALLOC] or [SQLITE_CONFIG_GETMALLOC]. +** By creating an instance of this object +** and passing it to [sqlite3_config]([SQLITE_CONFIG_MALLOC]) +** during configuration, an application can specify an alternative +** memory allocation subsystem for SQLite to use for all of its +** dynamic memory needs. +** +** Note that SQLite comes with several [built-in memory allocators] +** that are perfectly adequate for the overwhelming majority of applications +** and that this object is only useful to a tiny minority of applications +** with specialized memory allocation requirements. This object is +** also used during testing of SQLite in order to specify an alternative +** memory allocator that simulates memory out-of-memory conditions in +** order to verify that SQLite recovers gracefully from such +** conditions. +** +** The xMalloc, xRealloc, and xFree methods must work like the +** malloc(), realloc() and free() functions from the standard C library. +** ^SQLite guarantees that the second argument to +** xRealloc is always a value returned by a prior call to xRoundup. +** +** xSize should return the allocated size of a memory allocation +** previously obtained from xMalloc or xRealloc. The allocated size +** is always at least as big as the requested size but may be larger. +** +** The xRoundup method returns what would be the allocated size of +** a memory allocation given a particular requested size. Most memory +** allocators round up memory allocations at least to the next multiple +** of 8. Some allocators round up to a larger multiple or to a power of 2. +** Every memory allocation request coming in through [sqlite3_malloc()] +** or [sqlite3_realloc()] first calls xRoundup. If xRoundup returns 0, +** that causes the corresponding memory allocation to fail. +** +** The xInit method initializes the memory allocator. For example, +** it might allocate any required mutexes or initialize internal data +** structures. The xShutdown method is invoked (indirectly) by +** [sqlite3_shutdown()] and should deallocate any resources acquired +** by xInit. The pAppData pointer is used as the only parameter to +** xInit and xShutdown. +** +** SQLite holds the [SQLITE_MUTEX_STATIC_MAIN] mutex when it invokes +** the xInit method, so the xInit method need not be threadsafe. The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. For all other methods, SQLite +** holds the [SQLITE_MUTEX_STATIC_MEM] mutex as long as the +** [SQLITE_CONFIG_MEMSTATUS] configuration option is turned on (which +** it is by default) and so the methods are automatically serialized. +** However, if [SQLITE_CONFIG_MEMSTATUS] is disabled, then the other +** methods must be threadsafe or else make their own arrangements for +** serialization. +** +** SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +*/ +typedef struct sqlite3_mem_methods sqlite3_mem_methods; +struct sqlite3_mem_methods { + void *(*xMalloc)(int); /* Memory allocation function */ + void (*xFree)(void*); /* Free a prior allocation */ + void *(*xRealloc)(void*,int); /* Resize an allocation */ + int (*xSize)(void*); /* Return the size of an allocation */ + int (*xRoundup)(int); /* Round up request size to allocation size */ + int (*xInit)(void*); /* Initialize the memory allocator */ + void (*xShutdown)(void*); /* Deinitialize the memory allocator */ + void *pAppData; /* Argument to xInit() and xShutdown() */ +}; + +/* +** CAPI3REF: Configuration Options +** KEYWORDS: {configuration option} +** +** These constants are the available integer configuration options that +** can be passed as the first argument to the [sqlite3_config()] interface. +** +** Most of the configuration options for sqlite3_config() +** will only work if invoked prior to [sqlite3_initialize()] or after +** [sqlite3_shutdown()]. The few exceptions to this rule are called +** "anytime configuration options". +** ^Calling [sqlite3_config()] with a first argument that is not an +** anytime configuration option in between calls to [sqlite3_initialize()] and +** [sqlite3_shutdown()] is a no-op that returns SQLITE_MISUSE. +** +** The set of anytime configuration options can change (by insertions +** and/or deletions) from one release of SQLite to the next. +** As of SQLite version 3.42.0, the complete set of anytime configuration +** options is: +**
                  +**
                • SQLITE_CONFIG_LOG +**
                • SQLITE_CONFIG_PCACHE_HDRSZ +**
                +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_config()] to make sure that +** the call worked. The [sqlite3_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
                +** [[SQLITE_CONFIG_SINGLETHREAD]]
                SQLITE_CONFIG_SINGLETHREAD
                +**
                There are no arguments to this option. ^This option sets the +** [threading mode] to Single-thread. In other words, it disables +** all mutexing and puts SQLite into a mode where it can only be used +** by a single thread. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to change the [threading mode] from its default +** value of Single-thread and so [sqlite3_config()] will return +** [SQLITE_ERROR] if called with the SQLITE_CONFIG_SINGLETHREAD +** configuration option.
                +** +** [[SQLITE_CONFIG_MULTITHREAD]]
                SQLITE_CONFIG_MULTITHREAD
                +**
                There are no arguments to this option. ^This option sets the +** [threading mode] to Multi-thread. In other words, it disables +** mutexing on [database connection] and [prepared statement] objects. +** The application is responsible for serializing access to +** [database connections] and [prepared statements]. But other mutexes +** are enabled so that SQLite will be safe to use in a multi-threaded +** environment as long as no two threads attempt to use the same +** [database connection] at the same time. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Multi-thread [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_MULTITHREAD configuration option.
                +** +** [[SQLITE_CONFIG_SERIALIZED]]
                SQLITE_CONFIG_SERIALIZED
                +**
                There are no arguments to this option. ^This option sets the +** [threading mode] to Serialized. In other words, this option enables +** all mutexes including the recursive +** mutexes on [database connection] and [prepared statement] objects. +** In this mode (which is the default when SQLite is compiled with +** [SQLITE_THREADSAFE=1]) the SQLite library will itself serialize access +** to [database connections] and [prepared statements] so that the +** application is free to use the same [database connection] or the +** same [prepared statement] in different threads at the same time. +** ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** it is not possible to set the Serialized [threading mode] and +** [sqlite3_config()] will return [SQLITE_ERROR] if called with the +** SQLITE_CONFIG_SERIALIZED configuration option.
                +** +** [[SQLITE_CONFIG_MALLOC]]
                SQLITE_CONFIG_MALLOC
                +**
                ^(The SQLITE_CONFIG_MALLOC option takes a single argument which is +** a pointer to an instance of the [sqlite3_mem_methods] structure. +** The argument specifies +** alternative low-level memory allocation routines to be used in place of +** the memory allocation routines built into SQLite.)^ ^SQLite makes +** its own private copy of the content of the [sqlite3_mem_methods] structure +** before the [sqlite3_config()] call returns.
                +** +** [[SQLITE_CONFIG_GETMALLOC]]
                SQLITE_CONFIG_GETMALLOC
                +**
                ^(The SQLITE_CONFIG_GETMALLOC option takes a single argument which +** is a pointer to an instance of the [sqlite3_mem_methods] structure. +** The [sqlite3_mem_methods] +** structure is filled with the currently defined memory allocation routines.)^ +** This option can be used to overload the default memory allocation +** routines with a wrapper that simulations memory allocation failure or +** tracks memory usage, for example.
                +** +** [[SQLITE_CONFIG_SMALL_MALLOC]]
                SQLITE_CONFIG_SMALL_MALLOC
                +**
                ^The SQLITE_CONFIG_SMALL_MALLOC option takes single argument of +** type int, interpreted as a boolean, which if true provides a hint to +** SQLite that it should avoid large memory allocations if possible. +** SQLite will run faster if it is free to make large memory allocations, +** but some application might prefer to run slower in exchange for +** guarantees about memory fragmentation that are possible if large +** allocations are avoided. This hint is normally off. +**
                +** +** [[SQLITE_CONFIG_MEMSTATUS]]
                SQLITE_CONFIG_MEMSTATUS
                +**
                ^The SQLITE_CONFIG_MEMSTATUS option takes single argument of type int, +** interpreted as a boolean, which enables or disables the collection of +** memory allocation statistics. ^(When memory allocation statistics are +** disabled, the following SQLite interfaces become non-operational: +**
                  +**
                • [sqlite3_hard_heap_limit64()] +**
                • [sqlite3_memory_used()] +**
                • [sqlite3_memory_highwater()] +**
                • [sqlite3_soft_heap_limit64()] +**
                • [sqlite3_status64()] +**
                )^ +** ^Memory allocation statistics are enabled by default unless SQLite is +** compiled with [SQLITE_DEFAULT_MEMSTATUS]=0 in which case memory +** allocation statistics are disabled by default. +**
                +** +** [[SQLITE_CONFIG_SCRATCH]]
                SQLITE_CONFIG_SCRATCH
                +**
                The SQLITE_CONFIG_SCRATCH option is no longer used. +**
                +** +** [[SQLITE_CONFIG_PAGECACHE]]
                SQLITE_CONFIG_PAGECACHE
                +**
                ^The SQLITE_CONFIG_PAGECACHE option specifies a memory pool +** that SQLite can use for the database page cache with the default page +** cache implementation. +** This configuration option is a no-op if an application-defined page +** cache implementation is loaded using the [SQLITE_CONFIG_PCACHE2]. +** ^There are three arguments to SQLITE_CONFIG_PAGECACHE: A pointer to +** 8-byte aligned memory (pMem), the size of each page cache line (sz), +** and the number of cache lines (N). +** The sz argument should be the size of the largest database page +** (a power of two between 512 and 65536) plus some extra bytes for each +** page header. ^The number of extra bytes needed by the page header +** can be determined using [SQLITE_CONFIG_PCACHE_HDRSZ]. +** ^It is harmless, apart from the wasted memory, +** for the sz parameter to be larger than necessary. The pMem +** argument must be either a NULL pointer or a pointer to an 8-byte +** aligned block of memory of at least sz*N bytes, otherwise +** subsequent behavior is undefined. +** ^When pMem is not NULL, SQLite will strive to use the memory provided +** to satisfy page cache needs, falling back to [sqlite3_malloc()] if +** a page cache line is larger than sz bytes or if all of the pMem buffer +** is exhausted. +** ^If pMem is NULL and N is non-zero, then each database connection +** does an initial bulk allocation for page cache memory +** from [sqlite3_malloc()] sufficient for N cache lines if N is positive or +** of -1024*N bytes if N is negative, . ^If additional +** page cache memory is needed beyond what is provided by the initial +** allocation, then SQLite goes to [sqlite3_malloc()] separately for each +** additional cache line.
                +** +** [[SQLITE_CONFIG_HEAP]]
                SQLITE_CONFIG_HEAP
                +**
                ^The SQLITE_CONFIG_HEAP option specifies a static memory buffer +** that SQLite will use for all of its dynamic memory allocation needs +** beyond those provided for by [SQLITE_CONFIG_PAGECACHE]. +** ^The SQLITE_CONFIG_HEAP option is only available if SQLite is compiled +** with either [SQLITE_ENABLE_MEMSYS3] or [SQLITE_ENABLE_MEMSYS5] and returns +** [SQLITE_ERROR] if invoked otherwise. +** ^There are three arguments to SQLITE_CONFIG_HEAP: +** An 8-byte aligned pointer to the memory, +** the number of bytes in the memory buffer, and the minimum allocation size. +** ^If the first pointer (the memory pointer) is NULL, then SQLite reverts +** to using its default memory allocator (the system malloc() implementation), +** undoing any prior invocation of [SQLITE_CONFIG_MALLOC]. ^If the +** memory pointer is not NULL then the alternative memory +** allocator is engaged to handle all of SQLites memory allocation needs. +** The first pointer (the memory pointer) must be aligned to an 8-byte +** boundary or subsequent behavior of SQLite will be undefined. +** The minimum allocation size is capped at 2**12. Reasonable values +** for the minimum allocation size are 2**5 through 2**8.
                +** +** [[SQLITE_CONFIG_MUTEX]]
                SQLITE_CONFIG_MUTEX
                +**
                ^(The SQLITE_CONFIG_MUTEX option takes a single argument which is a +** pointer to an instance of the [sqlite3_mutex_methods] structure. +** The argument specifies alternative low-level mutex routines to be used +** in place the mutex routines built into SQLite.)^ ^SQLite makes a copy of +** the content of the [sqlite3_mutex_methods] structure before the call to +** [sqlite3_config()] returns. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_MUTEX configuration option will +** return [SQLITE_ERROR].
                +** +** [[SQLITE_CONFIG_GETMUTEX]]
                SQLITE_CONFIG_GETMUTEX
                +**
                ^(The SQLITE_CONFIG_GETMUTEX option takes a single argument which +** is a pointer to an instance of the [sqlite3_mutex_methods] structure. The +** [sqlite3_mutex_methods] +** structure is filled with the currently defined mutex routines.)^ +** This option can be used to overload the default mutex allocation +** routines with a wrapper used to track mutex usage for performance +** profiling or testing, for example. ^If SQLite is compiled with +** the [SQLITE_THREADSAFE | SQLITE_THREADSAFE=0] compile-time option then +** the entire mutexing subsystem is omitted from the build and hence calls to +** [sqlite3_config()] with the SQLITE_CONFIG_GETMUTEX configuration option will +** return [SQLITE_ERROR].
                +** +** [[SQLITE_CONFIG_LOOKASIDE]]
                SQLITE_CONFIG_LOOKASIDE
                +**
                ^(The SQLITE_CONFIG_LOOKASIDE option takes two arguments that determine +** the default size of lookaside memory on each [database connection]. +** The first argument is the +** size of each lookaside buffer slot and the second is the number of +** slots allocated to each database connection.)^ ^(SQLITE_CONFIG_LOOKASIDE +** sets the default lookaside size. The [SQLITE_DBCONFIG_LOOKASIDE] +** option to [sqlite3_db_config()] can be used to change the lookaside +** configuration on individual connections.)^
                +** +** [[SQLITE_CONFIG_PCACHE2]]
                SQLITE_CONFIG_PCACHE2
                +**
                ^(The SQLITE_CONFIG_PCACHE2 option takes a single argument which is +** a pointer to an [sqlite3_pcache_methods2] object. This object specifies +** the interface to a custom page cache implementation.)^ +** ^SQLite makes a copy of the [sqlite3_pcache_methods2] object.
                +** +** [[SQLITE_CONFIG_GETPCACHE2]]
                SQLITE_CONFIG_GETPCACHE2
                +**
                ^(The SQLITE_CONFIG_GETPCACHE2 option takes a single argument which +** is a pointer to an [sqlite3_pcache_methods2] object. SQLite copies of +** the current page cache implementation into that object.)^
                +** +** [[SQLITE_CONFIG_LOG]]
                SQLITE_CONFIG_LOG
                +**
                The SQLITE_CONFIG_LOG option is used to configure the SQLite +** global [error log]. +** (^The SQLITE_CONFIG_LOG option takes two arguments: a pointer to a +** function with a call signature of void(*)(void*,int,const char*), +** and a pointer to void. ^If the function pointer is not NULL, it is +** invoked by [sqlite3_log()] to process each logging event. ^If the +** function pointer is NULL, the [sqlite3_log()] interface becomes a no-op. +** ^The void pointer that is the second argument to SQLITE_CONFIG_LOG is +** passed through as the first parameter to the application-defined logger +** function whenever that function is invoked. ^The second parameter to +** the logger function is a copy of the first parameter to the corresponding +** [sqlite3_log()] call and is intended to be a [result code] or an +** [extended result code]. ^The third parameter passed to the logger is +** log message after formatting via [sqlite3_snprintf()]. +** The SQLite logging interface is not reentrant; the logger function +** supplied by the application must not invoke any SQLite interface. +** In a multi-threaded application, the application-defined logger +** function must be threadsafe.
                +** +** [[SQLITE_CONFIG_URI]]
                SQLITE_CONFIG_URI +**
                ^(The SQLITE_CONFIG_URI option takes a single argument of type int. +** If non-zero, then URI handling is globally enabled. If the parameter is zero, +** then URI handling is globally disabled.)^ ^If URI handling is globally +** enabled, all filenames passed to [sqlite3_open()], [sqlite3_open_v2()], +** [sqlite3_open16()] or +** specified as part of [ATTACH] commands are interpreted as URIs, regardless +** of whether or not the [SQLITE_OPEN_URI] flag is set when the database +** connection is opened. ^If it is globally disabled, filenames are +** only interpreted as URIs if the SQLITE_OPEN_URI flag is set when the +** database connection is opened. ^(By default, URI handling is globally +** disabled. The default value may be changed by compiling with the +** [SQLITE_USE_URI] symbol defined.)^ +** +** [[SQLITE_CONFIG_COVERING_INDEX_SCAN]]
                SQLITE_CONFIG_COVERING_INDEX_SCAN +**
                ^The SQLITE_CONFIG_COVERING_INDEX_SCAN option takes a single integer +** argument which is interpreted as a boolean in order to enable or disable +** the use of covering indices for full table scans in the query optimizer. +** ^The default setting is determined +** by the [SQLITE_ALLOW_COVERING_INDEX_SCAN] compile-time option, or is "on" +** if that compile-time option is omitted. +** The ability to disable the use of covering indices for full table scans +** is because some incorrectly coded legacy applications might malfunction +** when the optimization is enabled. Providing the ability to +** disable the optimization allows the older, buggy application code to work +** without change even with newer versions of SQLite. +** +** [[SQLITE_CONFIG_PCACHE]] [[SQLITE_CONFIG_GETPCACHE]] +**
                SQLITE_CONFIG_PCACHE and SQLITE_CONFIG_GETPCACHE +**
                These options are obsolete and should not be used by new code. +** They are retained for backwards compatibility but are now no-ops. +**
                +** +** [[SQLITE_CONFIG_SQLLOG]] +**
                SQLITE_CONFIG_SQLLOG +**
                This option is only available if sqlite is compiled with the +** [SQLITE_ENABLE_SQLLOG] pre-processor macro defined. The first argument should +** be a pointer to a function of type void(*)(void*,sqlite3*,const char*, int). +** The second should be of type (void*). The callback is invoked by the library +** in three separate circumstances, identified by the value passed as the +** fourth parameter. If the fourth parameter is 0, then the database connection +** passed as the second argument has just been opened. The third argument +** points to a buffer containing the name of the main database file. If the +** fourth parameter is 1, then the SQL statement that the third parameter +** points to has just been executed. Or, if the fourth parameter is 2, then +** the connection being passed as the second parameter is being closed. The +** third parameter is passed NULL In this case. An example of using this +** configuration option can be seen in the "test_sqllog.c" source file in +** the canonical SQLite source tree.
                +** +** [[SQLITE_CONFIG_MMAP_SIZE]] +**
                SQLITE_CONFIG_MMAP_SIZE +**
                ^SQLITE_CONFIG_MMAP_SIZE takes two 64-bit integer (sqlite3_int64) values +** that are the default mmap size limit (the default setting for +** [PRAGMA mmap_size]) and the maximum allowed mmap size limit. +** ^The default setting can be overridden by each database connection using +** either the [PRAGMA mmap_size] command, or by using the +** [SQLITE_FCNTL_MMAP_SIZE] file control. ^(The maximum allowed mmap size +** will be silently truncated if necessary so that it does not exceed the +** compile-time maximum mmap size set by the +** [SQLITE_MAX_MMAP_SIZE] compile-time option.)^ +** ^If either argument to this option is negative, then that argument is +** changed to its compile-time default. +** +** [[SQLITE_CONFIG_WIN32_HEAPSIZE]] +**
                SQLITE_CONFIG_WIN32_HEAPSIZE +**
                ^The SQLITE_CONFIG_WIN32_HEAPSIZE option is only available if SQLite is +** compiled for Windows with the [SQLITE_WIN32_MALLOC] pre-processor macro +** defined. ^SQLITE_CONFIG_WIN32_HEAPSIZE takes a 32-bit unsigned integer value +** that specifies the maximum size of the created heap. +** +** [[SQLITE_CONFIG_PCACHE_HDRSZ]] +**
                SQLITE_CONFIG_PCACHE_HDRSZ +**
                ^The SQLITE_CONFIG_PCACHE_HDRSZ option takes a single parameter which +** is a pointer to an integer and writes into that integer the number of extra +** bytes per page required for each page in [SQLITE_CONFIG_PAGECACHE]. +** The amount of extra space required can change depending on the compiler, +** target platform, and SQLite version. +** +** [[SQLITE_CONFIG_PMASZ]] +**
                SQLITE_CONFIG_PMASZ +**
                ^The SQLITE_CONFIG_PMASZ option takes a single parameter which +** is an unsigned integer and sets the "Minimum PMA Size" for the multithreaded +** sorter to that integer. The default minimum PMA Size is set by the +** [SQLITE_SORTER_PMASZ] compile-time option. New threads are launched +** to help with sort operations when multithreaded sorting +** is enabled (using the [PRAGMA threads] command) and the amount of content +** to be sorted exceeds the page size times the minimum of the +** [PRAGMA cache_size] setting and this value. +** +** [[SQLITE_CONFIG_STMTJRNL_SPILL]] +**
                SQLITE_CONFIG_STMTJRNL_SPILL +**
                ^The SQLITE_CONFIG_STMTJRNL_SPILL option takes a single parameter which +** becomes the [statement journal] spill-to-disk threshold. +** [Statement journals] are held in memory until their size (in bytes) +** exceeds this threshold, at which point they are written to disk. +** Or if the threshold is -1, statement journals are always held +** exclusively in memory. +** Since many statement journals never become large, setting the spill +** threshold to a value such as 64KiB can greatly reduce the amount of +** I/O required to support statement rollback. +** The default value for this setting is controlled by the +** [SQLITE_STMTJRNL_SPILL] compile-time option. +** +** [[SQLITE_CONFIG_SORTERREF_SIZE]] +**
                SQLITE_CONFIG_SORTERREF_SIZE +**
                The SQLITE_CONFIG_SORTERREF_SIZE option accepts a single parameter +** of type (int) - the new value of the sorter-reference size threshold. +** Usually, when SQLite uses an external sort to order records according +** to an ORDER BY clause, all fields required by the caller are present in the +** sorted records. However, if SQLite determines based on the declared type +** of a table column that its values are likely to be very large - larger +** than the configured sorter-reference size threshold - then a reference +** is stored in each sorted record and the required column values loaded +** from the database as records are returned in sorted order. The default +** value for this option is to never use this optimization. Specifying a +** negative value for this option restores the default behavior. +** This option is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SORTER_REFERENCES] compile-time option. +** +** [[SQLITE_CONFIG_MEMDB_MAXSIZE]] +**
                SQLITE_CONFIG_MEMDB_MAXSIZE +**
                The SQLITE_CONFIG_MEMDB_MAXSIZE option accepts a single parameter +** [sqlite3_int64] parameter which is the default maximum size for an in-memory +** database created using [sqlite3_deserialize()]. This default maximum +** size can be adjusted up or down for individual databases using the +** [SQLITE_FCNTL_SIZE_LIMIT] [sqlite3_file_control|file-control]. If this +** configuration setting is never used, then the default maximum is determined +** by the [SQLITE_MEMDB_DEFAULT_MAXSIZE] compile-time option. If that +** compile-time option is not set, then the default maximum is 1073741824. +** +** [[SQLITE_CONFIG_ROWID_IN_VIEW]] +**
                SQLITE_CONFIG_ROWID_IN_VIEW +**
                The SQLITE_CONFIG_ROWID_IN_VIEW option enables or disables the ability +** for VIEWs to have a ROWID. The capability can only be enabled if SQLite is +** compiled with -DSQLITE_ALLOW_ROWID_IN_VIEW, in which case the capability +** defaults to on. This configuration option queries the current setting or +** changes the setting to off or on. The argument is a pointer to an integer. +** If that integer initially holds a value of 1, then the ability for VIEWs to +** have ROWIDs is activated. If the integer initially holds zero, then the +** ability is deactivated. Any other initial value for the integer leaves the +** setting unchanged. After changes, if any, the integer is written with +** a 1 or 0, if the ability for VIEWs to have ROWIDs is on or off. If SQLite +** is compiled without -DSQLITE_ALLOW_ROWID_IN_VIEW (which is the usual and +** recommended case) then the integer is always filled with zero, regardless +** if its initial value. +**
                +*/ +#define SQLITE_CONFIG_SINGLETHREAD 1 /* nil */ +#define SQLITE_CONFIG_MULTITHREAD 2 /* nil */ +#define SQLITE_CONFIG_SERIALIZED 3 /* nil */ +#define SQLITE_CONFIG_MALLOC 4 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_GETMALLOC 5 /* sqlite3_mem_methods* */ +#define SQLITE_CONFIG_SCRATCH 6 /* No longer used */ +#define SQLITE_CONFIG_PAGECACHE 7 /* void*, int sz, int N */ +#define SQLITE_CONFIG_HEAP 8 /* void*, int nByte, int min */ +#define SQLITE_CONFIG_MEMSTATUS 9 /* boolean */ +#define SQLITE_CONFIG_MUTEX 10 /* sqlite3_mutex_methods* */ +#define SQLITE_CONFIG_GETMUTEX 11 /* sqlite3_mutex_methods* */ +/* previously SQLITE_CONFIG_CHUNKALLOC 12 which is now unused. */ +#define SQLITE_CONFIG_LOOKASIDE 13 /* int int */ +#define SQLITE_CONFIG_PCACHE 14 /* no-op */ +#define SQLITE_CONFIG_GETPCACHE 15 /* no-op */ +#define SQLITE_CONFIG_LOG 16 /* xFunc, void* */ +#define SQLITE_CONFIG_URI 17 /* int */ +#define SQLITE_CONFIG_PCACHE2 18 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_GETPCACHE2 19 /* sqlite3_pcache_methods2* */ +#define SQLITE_CONFIG_COVERING_INDEX_SCAN 20 /* int */ +#define SQLITE_CONFIG_SQLLOG 21 /* xSqllog, void* */ +#define SQLITE_CONFIG_MMAP_SIZE 22 /* sqlite3_int64, sqlite3_int64 */ +#define SQLITE_CONFIG_WIN32_HEAPSIZE 23 /* int nByte */ +#define SQLITE_CONFIG_PCACHE_HDRSZ 24 /* int *psz */ +#define SQLITE_CONFIG_PMASZ 25 /* unsigned int szPma */ +#define SQLITE_CONFIG_STMTJRNL_SPILL 26 /* int nByte */ +#define SQLITE_CONFIG_SMALL_MALLOC 27 /* boolean */ +#define SQLITE_CONFIG_SORTERREF_SIZE 28 /* int nByte */ +#define SQLITE_CONFIG_MEMDB_MAXSIZE 29 /* sqlite3_int64 */ +#define SQLITE_CONFIG_ROWID_IN_VIEW 30 /* int* */ + +/* +** CAPI3REF: Database Connection Configuration Options +** +** These constants are the available integer configuration options that +** can be passed as the second argument to the [sqlite3_db_config()] interface. +** +** New configuration options may be added in future releases of SQLite. +** Existing configuration options might be discontinued. Applications +** should check the return code from [sqlite3_db_config()] to make sure that +** the call worked. ^The [sqlite3_db_config()] interface will return a +** non-zero [error code] if a discontinued or unsupported configuration option +** is invoked. +** +**
                +** [[SQLITE_DBCONFIG_LOOKASIDE]] +**
                SQLITE_DBCONFIG_LOOKASIDE
                +**
                ^This option takes three additional arguments that determine the +** [lookaside memory allocator] configuration for the [database connection]. +** ^The first argument (the third parameter to [sqlite3_db_config()] is a +** pointer to a memory buffer to use for lookaside memory. +** ^The first argument after the SQLITE_DBCONFIG_LOOKASIDE verb +** may be NULL in which case SQLite will allocate the +** lookaside buffer itself using [sqlite3_malloc()]. ^The second argument is the +** size of each lookaside buffer slot. ^The third argument is the number of +** slots. The size of the buffer in the first argument must be greater than +** or equal to the product of the second and third arguments. The buffer +** must be aligned to an 8-byte boundary. ^If the second argument to +** SQLITE_DBCONFIG_LOOKASIDE is not a multiple of 8, it is internally +** rounded down to the next smaller multiple of 8. ^(The lookaside memory +** configuration for a database connection can only be changed when that +** connection is not currently using lookaside memory, or in other words +** when the "current value" returned by +** [sqlite3_db_status](D,[SQLITE_DBSTATUS_LOOKASIDE_USED],...) is zero. +** Any attempt to change the lookaside memory configuration when lookaside +** memory is in use leaves the configuration unchanged and returns +** [SQLITE_BUSY].)^
                +** +** [[SQLITE_DBCONFIG_ENABLE_FKEY]] +**
                SQLITE_DBCONFIG_ENABLE_FKEY
                +**
                ^This option is used to enable or disable the enforcement of +** [foreign key constraints]. There should be two additional arguments. +** The first argument is an integer which is 0 to disable FK enforcement, +** positive to enable FK enforcement or negative to leave FK enforcement +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether FK enforcement is off or on +** following this call. The second parameter may be a NULL pointer, in +** which case the FK enforcement setting is not reported back.
                +** +** [[SQLITE_DBCONFIG_ENABLE_TRIGGER]] +**
                SQLITE_DBCONFIG_ENABLE_TRIGGER
                +**
                ^This option is used to enable or disable [CREATE TRIGGER | triggers]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable triggers, +** positive to enable triggers or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether triggers are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the trigger setting is not reported back. +** +**

                Originally this option disabled all triggers. ^(However, since +** SQLite version 3.35.0, TEMP triggers are still allowed even if +** this option is off. So, in other words, this option now only disables +** triggers in the main database schema or in the schemas of ATTACH-ed +** databases.)^

                +** +** [[SQLITE_DBCONFIG_ENABLE_VIEW]] +**
                SQLITE_DBCONFIG_ENABLE_VIEW
                +**
                ^This option is used to enable or disable [CREATE VIEW | views]. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable views, +** positive to enable views or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether views are disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the view setting is not reported back. +** +**

                Originally this option disabled all views. ^(However, since +** SQLite version 3.35.0, TEMP views are still allowed even if +** this option is off. So, in other words, this option now only disables +** views in the main database schema or in the schemas of ATTACH-ed +** databases.)^

                +** +** [[SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER]] +**
                SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER
                +**
                ^This option is used to enable or disable the +** [fts3_tokenizer()] function which is part of the +** [FTS3] full-text search engine extension. +** There should be two additional arguments. +** The first argument is an integer which is 0 to disable fts3_tokenizer() or +** positive to enable fts3_tokenizer() or negative to leave the setting +** unchanged. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether fts3_tokenizer is disabled or enabled +** following this call. The second parameter may be a NULL pointer, in +** which case the new setting is not reported back.
                +** +** [[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION]] +**
                SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION
                +**
                ^This option is used to enable or disable the [sqlite3_load_extension()] +** interface independently of the [load_extension()] SQL function. +** The [sqlite3_enable_load_extension()] API enables or disables both the +** C-API [sqlite3_load_extension()] and the SQL function [load_extension()]. +** There should be two additional arguments. +** When the first argument to this interface is 1, then only the C-API is +** enabled and the SQL function remains disabled. If the first argument to +** this interface is 0, then both the C-API and the SQL function are disabled. +** If the first argument is -1, then no changes are made to state of either the +** C-API or the SQL function. +** The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether [sqlite3_load_extension()] interface +** is disabled or enabled following this call. The second parameter may +** be a NULL pointer, in which case the new setting is not reported back. +**
                +** +** [[SQLITE_DBCONFIG_MAINDBNAME]]
                SQLITE_DBCONFIG_MAINDBNAME
                +**
                ^This option is used to change the name of the "main" database +** schema. ^The sole argument is a pointer to a constant UTF8 string +** which will become the new schema name in place of "main". ^SQLite +** does not make a copy of the new main schema name string, so the application +** must ensure that the argument passed into this DBCONFIG option is unchanged +** until after the database connection closes. +**
                +** +** [[SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE]] +**
                SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE
                +**
                Usually, when a database in wal mode is closed or detached from a +** database handle, SQLite checks if this will mean that there are now no +** connections at all to the database. If so, it performs a checkpoint +** operation before closing the connection. This option may be used to +** override this behavior. The first parameter passed to this operation +** is an integer - positive to disable checkpoints-on-close, or zero (the +** default) to enable them, and negative to leave the setting unchanged. +** The second parameter is a pointer to an integer +** into which is written 0 or 1 to indicate whether checkpoints-on-close +** have been disabled - 0 if they are not disabled, 1 if they are. +**
                +** +** [[SQLITE_DBCONFIG_ENABLE_QPSG]]
                SQLITE_DBCONFIG_ENABLE_QPSG
                +**
                ^(The SQLITE_DBCONFIG_ENABLE_QPSG option activates or deactivates +** the [query planner stability guarantee] (QPSG). When the QPSG is active, +** a single SQL query statement will always use the same algorithm regardless +** of values of [bound parameters].)^ The QPSG disables some query optimizations +** that look at the values of bound parameters, which can make some queries +** slower. But the QPSG has the advantage of more predictable behavior. With +** the QPSG active, SQLite will always use the same query plan in the field as +** was used during testing in the lab. +** The first argument to this setting is an integer which is 0 to disable +** the QPSG, positive to enable QPSG, or negative to leave the setting +** unchanged. The second parameter is a pointer to an integer into which +** is written 0 or 1 to indicate whether the QPSG is disabled or enabled +** following this call. +**
                +** +** [[SQLITE_DBCONFIG_TRIGGER_EQP]]
                SQLITE_DBCONFIG_TRIGGER_EQP
                +**
                By default, the output of EXPLAIN QUERY PLAN commands does not +** include output for any operations performed by trigger programs. This +** option is used to set or clear (the default) a flag that governs this +** behavior. The first parameter passed to this operation is an integer - +** positive to enable output for trigger programs, or zero to disable it, +** or negative to leave the setting unchanged. +** The second parameter is a pointer to an integer into which is written +** 0 or 1 to indicate whether output-for-triggers has been disabled - 0 if +** it is not disabled, 1 if it is. +**
                +** +** [[SQLITE_DBCONFIG_RESET_DATABASE]]
                SQLITE_DBCONFIG_RESET_DATABASE
                +**
                Set the SQLITE_DBCONFIG_RESET_DATABASE flag and then run +** [VACUUM] in order to reset a database back to an empty database +** with no schema and no content. The following process works even for +** a badly corrupted database file: +**
                  +**
                1. If the database connection is newly opened, make sure it has read the +** database schema by preparing then discarding some query against the +** database, or calling sqlite3_table_column_metadata(), ignoring any +** errors. This step is only necessary if the application desires to keep +** the database in WAL mode after the reset if it was in WAL mode before +** the reset. +**
                2. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 1, 0); +**
                3. [sqlite3_exec](db, "[VACUUM]", 0, 0, 0); +**
                4. sqlite3_db_config(db, SQLITE_DBCONFIG_RESET_DATABASE, 0, 0); +**
                +** Because resetting a database is destructive and irreversible, the +** process requires the use of this obscure API and multiple steps to +** help ensure that it does not happen by accident. Because this +** feature must be capable of resetting corrupt databases, and +** shutting down virtual tables may require access to that corrupt +** storage, the library must abandon any installed virtual tables +** without calling their xDestroy() methods. +** +** [[SQLITE_DBCONFIG_DEFENSIVE]]
                SQLITE_DBCONFIG_DEFENSIVE
                +**
                The SQLITE_DBCONFIG_DEFENSIVE option activates or deactivates the +** "defensive" flag for a database connection. When the defensive +** flag is enabled, language features that allow ordinary SQL to +** deliberately corrupt the database file are disabled. The disabled +** features include but are not limited to the following: +**
                  +**
                • The [PRAGMA writable_schema=ON] statement. +**
                • The [PRAGMA journal_mode=OFF] statement. +**
                • The [PRAGMA schema_version=N] statement. +**
                • Writes to the [sqlite_dbpage] virtual table. +**
                • Direct writes to [shadow tables]. +**
                +**
                +** +** [[SQLITE_DBCONFIG_WRITABLE_SCHEMA]]
                SQLITE_DBCONFIG_WRITABLE_SCHEMA
                +**
                The SQLITE_DBCONFIG_WRITABLE_SCHEMA option activates or deactivates the +** "writable_schema" flag. This has the same effect and is logically equivalent +** to setting [PRAGMA writable_schema=ON] or [PRAGMA writable_schema=OFF]. +** The first argument to this setting is an integer which is 0 to disable +** the writable_schema, positive to enable writable_schema, or negative to +** leave the setting unchanged. The second parameter is a pointer to an +** integer into which is written 0 or 1 to indicate whether the writable_schema +** is enabled or disabled following this call. +**
                +** +** [[SQLITE_DBCONFIG_LEGACY_ALTER_TABLE]] +**
                SQLITE_DBCONFIG_LEGACY_ALTER_TABLE
                +**
                The SQLITE_DBCONFIG_LEGACY_ALTER_TABLE option activates or deactivates +** the legacy behavior of the [ALTER TABLE RENAME] command such it +** behaves as it did prior to [version 3.24.0] (2018-06-04). See the +** "Compatibility Notice" on the [ALTER TABLE RENAME documentation] for +** additional information. This feature can also be turned on and off +** using the [PRAGMA legacy_alter_table] statement. +**
                +** +** [[SQLITE_DBCONFIG_DQS_DML]] +**
                SQLITE_DBCONFIG_DQS_DML
                +**
                The SQLITE_DBCONFIG_DQS_DML option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DML statements +** only, that is DELETE, INSERT, SELECT, and UPDATE statements. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
                +** +** [[SQLITE_DBCONFIG_DQS_DDL]] +**
                SQLITE_DBCONFIG_DQS_DDL
                +**
                The SQLITE_DBCONFIG_DQS option activates or deactivates +** the legacy [double-quoted string literal] misfeature for DDL statements, +** such as CREATE TABLE and CREATE INDEX. The +** default value of this setting is determined by the [-DSQLITE_DQS] +** compile-time option. +**
                +** +** [[SQLITE_DBCONFIG_TRUSTED_SCHEMA]] +**
                SQLITE_DBCONFIG_TRUSTED_SCHEMA
                +**
                The SQLITE_DBCONFIG_TRUSTED_SCHEMA option tells SQLite to +** assume that database schemas are untainted by malicious content. +** When the SQLITE_DBCONFIG_TRUSTED_SCHEMA option is disabled, SQLite +** takes additional defensive steps to protect the application from harm +** including: +**
                  +**
                • Prohibit the use of SQL functions inside triggers, views, +** CHECK constraints, DEFAULT clauses, expression indexes, +** partial indexes, or generated columns +** unless those functions are tagged with [SQLITE_INNOCUOUS]. +**
                • Prohibit the use of virtual tables inside of triggers or views +** unless those virtual tables are tagged with [SQLITE_VTAB_INNOCUOUS]. +**
                +** This setting defaults to "on" for legacy compatibility, however +** all applications are advised to turn it off if possible. This setting +** can also be controlled using the [PRAGMA trusted_schema] statement. +**
                +** +** [[SQLITE_DBCONFIG_LEGACY_FILE_FORMAT]] +**
                SQLITE_DBCONFIG_LEGACY_FILE_FORMAT
                +**
                The SQLITE_DBCONFIG_LEGACY_FILE_FORMAT option activates or deactivates +** the legacy file format flag. When activated, this flag causes all newly +** created database file to have a schema format version number (the 4-byte +** integer found at offset 44 into the database header) of 1. This in turn +** means that the resulting database file will be readable and writable by +** any SQLite version back to 3.0.0 ([dateof:3.0.0]). Without this setting, +** newly created databases are generally not understandable by SQLite versions +** prior to 3.3.0 ([dateof:3.3.0]). As these words are written, there +** is now scarcely any need to generate database files that are compatible +** all the way back to version 3.0.0, and so this setting is of little +** practical use, but is provided so that SQLite can continue to claim the +** ability to generate new database files that are compatible with version +** 3.0.0. +**

                Note that when the SQLITE_DBCONFIG_LEGACY_FILE_FORMAT setting is on, +** the [VACUUM] command will fail with an obscure error when attempting to +** process a table with generated columns and a descending index. This is +** not considered a bug since SQLite versions 3.3.0 and earlier do not support +** either generated columns or descending indexes. +**

                +** +** [[SQLITE_DBCONFIG_STMT_SCANSTATUS]] +**
                SQLITE_DBCONFIG_STMT_SCANSTATUS
                +**
                The SQLITE_DBCONFIG_STMT_SCANSTATUS option is only useful in +** SQLITE_ENABLE_STMT_SCANSTATUS builds. In this case, it sets or clears +** a flag that enables collection of the sqlite3_stmt_scanstatus_v2() +** statistics. For statistics to be collected, the flag must be set on +** the database handle both when the SQL statement is prepared and when it +** is stepped. The flag is set (collection of statistics is enabled) +** by default. This option takes two arguments: an integer and a pointer to +** an integer.. The first argument is 1, 0, or -1 to enable, disable, or +** leave unchanged the statement scanstatus option. If the second argument +** is not NULL, then the value of the statement scanstatus setting after +** processing the first argument is written into the integer that the second +** argument points to. +**
                +** +** [[SQLITE_DBCONFIG_REVERSE_SCANORDER]] +**
                SQLITE_DBCONFIG_REVERSE_SCANORDER
                +**
                The SQLITE_DBCONFIG_REVERSE_SCANORDER option changes the default order +** in which tables and indexes are scanned so that the scans start at the end +** and work toward the beginning rather than starting at the beginning and +** working toward the end. Setting SQLITE_DBCONFIG_REVERSE_SCANORDER is the +** same as setting [PRAGMA reverse_unordered_selects]. This option takes +** two arguments which are an integer and a pointer to an integer. The first +** argument is 1, 0, or -1 to enable, disable, or leave unchanged the +** reverse scan order flag, respectively. If the second argument is not NULL, +** then 0 or 1 is written into the integer that the second argument points to +** depending on if the reverse scan order flag is set after processing the +** first argument. +**
                +** +**
                +*/ +#define SQLITE_DBCONFIG_MAINDBNAME 1000 /* const char* */ +#define SQLITE_DBCONFIG_LOOKASIDE 1001 /* void* int int */ +#define SQLITE_DBCONFIG_ENABLE_FKEY 1002 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_TRIGGER 1003 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_FTS3_TOKENIZER 1004 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION 1005 /* int int* */ +#define SQLITE_DBCONFIG_NO_CKPT_ON_CLOSE 1006 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_QPSG 1007 /* int int* */ +#define SQLITE_DBCONFIG_TRIGGER_EQP 1008 /* int int* */ +#define SQLITE_DBCONFIG_RESET_DATABASE 1009 /* int int* */ +#define SQLITE_DBCONFIG_DEFENSIVE 1010 /* int int* */ +#define SQLITE_DBCONFIG_WRITABLE_SCHEMA 1011 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_ALTER_TABLE 1012 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DML 1013 /* int int* */ +#define SQLITE_DBCONFIG_DQS_DDL 1014 /* int int* */ +#define SQLITE_DBCONFIG_ENABLE_VIEW 1015 /* int int* */ +#define SQLITE_DBCONFIG_LEGACY_FILE_FORMAT 1016 /* int int* */ +#define SQLITE_DBCONFIG_TRUSTED_SCHEMA 1017 /* int int* */ +#define SQLITE_DBCONFIG_STMT_SCANSTATUS 1018 /* int int* */ +#define SQLITE_DBCONFIG_REVERSE_SCANORDER 1019 /* int int* */ +#define SQLITE_DBCONFIG_MAX 1019 /* Largest DBCONFIG */ /* ** CAPI3REF: Enable Or Disable Extended Result Codes +** METHOD: sqlite3 ** -** This routine enables or disables the -** [SQLITE_IOERR_READ | extended result codes] feature. -** By default, SQLite API routines return one of only 26 integer -** [SQLITE_OK | result codes]. When extended result codes -** are enabled by this routine, the repetoire of result codes can be -** much larger and can (hopefully) provide more detailed information -** about the cause of an error. -** -** The second argument is a boolean value that turns extended result -** codes on and off. Extended result codes are off by default for -** backwards compatibility with older versions of SQLite. +** ^The sqlite3_extended_result_codes() routine enables or disables the +** [extended result codes] feature of SQLite. ^The extended result +** codes are disabled by default for historical compatibility. */ -int sqlite3_extended_result_codes(sqlite3*, int onoff); +SQLITE_API int sqlite3_extended_result_codes(sqlite3*, int onoff); /* ** CAPI3REF: Last Insert Rowid +** METHOD: sqlite3 ** -** Each entry in an SQLite table has a unique 64-bit signed integer key -** called the "rowid". The rowid is always available as an undeclared -** column named ROWID, OID, or _ROWID_. If the table has a column of -** type INTEGER PRIMARY KEY then that column is another an alias for the -** rowid. +** ^Each entry in most SQLite tables (except for [WITHOUT ROWID] tables) +** has a unique 64-bit signed +** integer key called the [ROWID | "rowid"]. ^The rowid is always available +** as an undeclared column named ROWID, OID, or _ROWID_ as long as those +** names are not also used by explicitly declared columns. ^If +** the table has a column of type [INTEGER PRIMARY KEY] then that column +** is another alias for the rowid. ** -** This routine returns the rowid of the most recent INSERT into -** the database from the database connection given in the first -** argument. If no inserts have ever occurred on this database -** connection, zero is returned. +** ^The sqlite3_last_insert_rowid(D) interface usually returns the [rowid] of +** the most recent successful [INSERT] into a rowid table or [virtual table] +** on database connection D. ^Inserts into [WITHOUT ROWID] tables are not +** recorded. ^If no successful [INSERT]s into rowid tables have ever occurred +** on the database connection D, then sqlite3_last_insert_rowid(D) returns +** zero. ** -** If an INSERT occurs within a trigger, then the rowid of the -** inserted row is returned by this routine as long as the trigger -** is running. But once the trigger terminates, the value returned -** by this routine reverts to the last value inserted before the -** trigger fired. +** As well as being set automatically as rows are inserted into database +** tables, the value returned by this function may be set explicitly by +** [sqlite3_set_last_insert_rowid()] +** +** Some virtual table implementations may INSERT rows into rowid tables as +** part of committing a transaction (e.g. to flush data accumulated in memory +** to disk). In this case subsequent calls to this function return the rowid +** associated with these internal INSERT operations, which leads to +** unintuitive results. Virtual table implementations that do write to rowid +** tables in this way can avoid this problem by restoring the original +** rowid value using [sqlite3_set_last_insert_rowid()] before returning +** control to the user. +** +** ^(If an [INSERT] occurs within a trigger then this routine will +** return the [rowid] of the inserted row as long as the trigger is +** running. Once the trigger program ends, the value returned +** by this routine reverts to what it was before the trigger was fired.)^ +** +** ^An [INSERT] that fails due to a constraint violation is not a +** successful [INSERT] and does not change the value returned by this +** routine. ^Thus INSERT OR FAIL, INSERT OR IGNORE, INSERT OR ROLLBACK, +** and INSERT OR ABORT make no changes to the return value of this +** routine when their insertion fails. ^(When INSERT OR REPLACE +** encounters a constraint violation, it does not fail. The +** INSERT continues to completion after deleting rows that caused +** the constraint problem so INSERT OR REPLACE will always change +** the return value of this interface.)^ +** +** ^For the purposes of this routine, an [INSERT] is considered to +** be successful even if it is subsequently rolled back. +** +** This function is accessible to SQL statements via the +** [last_insert_rowid() SQL function]. +** +** If a separate thread performs a new [INSERT] on the same +** database connection while the [sqlite3_last_insert_rowid()] +** function is running and thus changes the last insert [rowid], +** then the value returned by [sqlite3_last_insert_rowid()] is +** unpredictable and might not equal either the old or the new +** last insert [rowid]. */ -sqlite_int64 sqlite3_last_insert_rowid(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_last_insert_rowid(sqlite3*); + +/* +** CAPI3REF: Set the Last Insert Rowid value. +** METHOD: sqlite3 +** +** The sqlite3_set_last_insert_rowid(D, R) method allows the application to +** set the value returned by calling sqlite3_last_insert_rowid(D) to R +** without inserting a row into the database. +*/ +SQLITE_API void sqlite3_set_last_insert_rowid(sqlite3*,sqlite3_int64); /* ** CAPI3REF: Count The Number Of Rows Modified +** METHOD: sqlite3 ** -** This function returns the number of database rows that were changed -** (or inserted or deleted) by the most recent SQL statement. Only -** changes that are directly specified by the INSERT, UPDATE, or -** DELETE statement are counted. Auxiliary changes caused by -** triggers are not counted. Use the [sqlite3_total_changes()] function -** to find the total number of changes including changes caused by triggers. +** ^These functions return the number of rows modified, inserted or +** deleted by the most recently completed INSERT, UPDATE or DELETE +** statement on the database connection specified by the only parameter. +** The two functions are identical except for the type of the return value +** and that if the number of rows modified by the most recent INSERT, UPDATE +** or DELETE is greater than the maximum value supported by type "int", then +** the return value of sqlite3_changes() is undefined. ^Executing any other +** type of SQL statement does not modify the value returned by these functions. ** -** Within the body of a trigger, the sqlite3_changes() interface can be -** called to find the number of -** changes in the most recently completed INSERT, UPDATE, or DELETE -** statement within the body of the trigger. +** ^Only changes made directly by the INSERT, UPDATE or DELETE statement are +** considered - auxiliary changes caused by [CREATE TRIGGER | triggers], +** [foreign key actions] or [REPLACE] constraint resolution are not counted. ** -** All changes are counted, even if they were later undone by a -** ROLLBACK or ABORT. Except, changes associated with creating and -** dropping tables are not counted. +** Changes to a view that are intercepted by +** [INSTEAD OF trigger | INSTEAD OF triggers] are not counted. ^The value +** returned by sqlite3_changes() immediately after an INSERT, UPDATE or +** DELETE statement run on a view is always zero. Only changes made to real +** tables are counted. ** -** If a callback invokes [sqlite3_exec()] or [sqlite3_step()] recursively, -** then the changes in the inner, recursive call are counted together -** with the changes in the outer call. +** Things are more complicated if the sqlite3_changes() function is +** executed while a trigger program is running. This may happen if the +** program uses the [changes() SQL function], or if some other callback +** function invokes sqlite3_changes() directly. Essentially: ** -** SQLite implements the command "DELETE FROM table" without a WHERE clause -** by dropping and recreating the table. (This is much faster than going -** through and deleting individual elements form the table.) Because of -** this optimization, the change count for "DELETE FROM table" will be -** zero regardless of the number of elements that were originally in the -** table. To get an accurate count of the number of rows deleted, use -** "DELETE FROM table WHERE 1" instead. +**
                  +**
                • ^(Before entering a trigger program the value returned by +** sqlite3_changes() function is saved. After the trigger program +** has finished, the original value is restored.)^ +** +**
                • ^(Within a trigger program each INSERT, UPDATE and DELETE +** statement sets the value returned by sqlite3_changes() +** upon completion as normal. Of course, this value will not include +** any changes performed by sub-triggers, as the sqlite3_changes() +** value will be saved and restored after each sub-trigger has run.)^ +**
                +** +** ^This means that if the changes() SQL function (or similar) is used +** by the first INSERT, UPDATE or DELETE statement within a trigger, it +** returns the value as set when the calling statement began executing. +** ^If it is used by the second or subsequent such statement within a trigger +** program, the value returned reflects the number of rows modified by the +** previous INSERT, UPDATE or DELETE statement within the same trigger. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_changes()] is running then the value returned +** is unpredictable and not meaningful. +** +** See also: +**
                  +**
                • the [sqlite3_total_changes()] interface +**
                • the [count_changes pragma] +**
                • the [changes() SQL function] +**
                • the [data_version pragma] +**
                */ -int sqlite3_changes(sqlite3*); +SQLITE_API int sqlite3_changes(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_changes64(sqlite3*); /* ** CAPI3REF: Total Number Of Rows Modified -*** -** This function returns the number of database rows that have been -** modified by INSERT, UPDATE or DELETE statements since the database handle -** was opened. This includes UPDATE, INSERT and DELETE statements executed -** as part of trigger programs. All changes are counted as soon as the -** statement that makes them is completed (when the statement handle is -** passed to [sqlite3_reset()] or [sqlite_finalise()]). +** METHOD: sqlite3 ** -** See also the [sqlite3_change()] interface. +** ^These functions return the total number of rows inserted, modified or +** deleted by all [INSERT], [UPDATE] or [DELETE] statements completed +** since the database connection was opened, including those executed as +** part of trigger programs. The two functions are identical except for the +** type of the return value and that if the number of rows modified by the +** connection exceeds the maximum value supported by type "int", then +** the return value of sqlite3_total_changes() is undefined. ^Executing +** any other type of SQL statement does not affect the value returned by +** sqlite3_total_changes(). ** -** SQLite implements the command "DELETE FROM table" without a WHERE clause -** by dropping and recreating the table. (This is much faster than going -** through and deleting individual elements form the table.) Because of -** this optimization, the change count for "DELETE FROM table" will be -** zero regardless of the number of elements that were originally in the -** table. To get an accurate count of the number of rows deleted, use -** "DELETE FROM table WHERE 1" instead. +** ^Changes made as part of [foreign key actions] are included in the +** count, but those made as part of REPLACE constraint resolution are +** not. ^Changes to a view that are intercepted by INSTEAD OF triggers +** are not counted. +** +** The [sqlite3_total_changes(D)] interface only reports the number +** of rows that changed due to SQL statement run against database +** connection D. Any changes by other database connections are ignored. +** To detect changes against a database file from other database +** connections use the [PRAGMA data_version] command or the +** [SQLITE_FCNTL_DATA_VERSION] [file control]. +** +** If a separate thread makes changes on the same database connection +** while [sqlite3_total_changes()] is running then the value +** returned is unpredictable and not meaningful. +** +** See also: +**
                  +**
                • the [sqlite3_changes()] interface +**
                • the [count_changes pragma] +**
                • the [changes() SQL function] +**
                • the [data_version pragma] +**
                • the [SQLITE_FCNTL_DATA_VERSION] [file control] +**
                */ -int sqlite3_total_changes(sqlite3*); +SQLITE_API int sqlite3_total_changes(sqlite3*); +SQLITE_API sqlite3_int64 sqlite3_total_changes64(sqlite3*); /* ** CAPI3REF: Interrupt A Long-Running Query +** METHOD: sqlite3 ** -** This function causes any pending database operation to abort and -** return at its earliest opportunity. This routine is typically +** ^This function causes any pending database operation to abort and +** return at its earliest opportunity. This routine is typically ** called in response to a user action such as pressing "Cancel" ** or Ctrl-C where the user wants a long query operation to halt ** immediately. ** -** It is safe to call this routine from a thread different from the -** thread that is currently running the database operation. +** ^It is safe to call this routine from a thread different from the +** thread that is currently running the database operation. But it +** is not safe to call this routine with a [database connection] that +** is closed or might close before sqlite3_interrupt() returns. ** -** The SQL operation that is interrupted will return [SQLITE_INTERRUPT]. -** If an interrupted operation was an update that is inside an -** explicit transaction, then the entire transaction will be rolled -** back automatically. +** ^If an SQL operation is very nearly finished at the time when +** sqlite3_interrupt() is called, then it might not have an opportunity +** to be interrupted and might continue to completion. +** +** ^An SQL operation that is interrupted will return [SQLITE_INTERRUPT]. +** ^If the interrupted SQL operation is an INSERT, UPDATE, or DELETE +** that is inside an explicit transaction, then the entire transaction +** will be rolled back automatically. +** +** ^The sqlite3_interrupt(D) call is in effect until all currently running +** SQL statements on [database connection] D complete. ^Any new SQL statements +** that are started after the sqlite3_interrupt() call and before the +** running statement count reaches zero are interrupted as if they had been +** running prior to the sqlite3_interrupt() call. ^New SQL statements +** that are started after the running statement count reaches zero are +** not effected by the sqlite3_interrupt(). +** ^A call to sqlite3_interrupt(D) that occurs when there are no running +** SQL statements is a no-op and has no effect on SQL statements +** that are started after the sqlite3_interrupt() call returns. +** +** ^The [sqlite3_is_interrupted(D)] interface can be used to determine whether +** or not an interrupt is currently in effect for [database connection] D. +** It returns 1 if an interrupt is currently in effect, or 0 otherwise. */ -void sqlite3_interrupt(sqlite3*); +SQLITE_API void sqlite3_interrupt(sqlite3*); +SQLITE_API int sqlite3_is_interrupted(sqlite3*); /* ** CAPI3REF: Determine If An SQL Statement Is Complete ** -** These functions return true if the given input string comprises -** one or more complete SQL statements. For the sqlite3_complete() call, -** the parameter must be a nul-terminated UTF-8 string. For -** sqlite3_complete16(), a nul-terminated machine byte order UTF-16 string -** is required. +** These routines are useful during command-line input to determine if the +** currently entered text seems to form a complete SQL statement or +** if additional input is needed before sending the text into +** SQLite for parsing. ^These routines return 1 if the input string +** appears to be a complete SQL statement. ^A statement is judged to be +** complete if it ends with a semicolon token and is not a prefix of a +** well-formed CREATE TRIGGER statement. ^Semicolons that are embedded within +** string literals or quoted identifier names or comments are not +** independent tokens (they are part of the token in which they are +** embedded) and thus do not count as a statement terminator. ^Whitespace +** and comments that follow the final semicolon are ignored. ** -** These routines are useful for command-line input to determine if the -** currently entered text forms one or more complete SQL statements or -** if additional input is needed before sending the statements into -** SQLite for parsing. The algorithm is simple. If the -** last token other than spaces and comments is a semicolon, then return -** true. Actually, the algorithm is a little more complicated than that -** in order to deal with triggers, but the basic idea is the same: the -** statement is not complete unless it ends in a semicolon. +** ^These routines return 0 if the statement is incomplete. ^If a +** memory allocation fails, then SQLITE_NOMEM is returned. +** +** ^These routines do not parse the SQL statements thus +** will not detect syntactically incorrect SQL. +** +** ^(If SQLite has not been initialized using [sqlite3_initialize()] prior +** to invoking sqlite3_complete16() then sqlite3_initialize() is invoked +** automatically by sqlite3_complete16(). If that initialization fails, +** then the return value from sqlite3_complete16() will be non-zero +** regardless of whether or not the input SQL is complete.)^ +** +** The input to [sqlite3_complete()] must be a zero-terminated +** UTF-8 string. +** +** The input to [sqlite3_complete16()] must be a zero-terminated +** UTF-16 string in native byte order. */ -int sqlite3_complete(const char *sql); -int sqlite3_complete16(const void *sql); +SQLITE_API int sqlite3_complete(const char *sql); +SQLITE_API int sqlite3_complete16(const void *sql); /* ** CAPI3REF: Register A Callback To Handle SQLITE_BUSY Errors +** KEYWORDS: {busy-handler callback} {busy handler} +** METHOD: sqlite3 ** -** This routine identifies a callback function that might be invoked -** whenever an attempt is made to open a database table -** that another thread or process has locked. -** If the busy callback is NULL, then [SQLITE_BUSY] -** (or sometimes [SQLITE_IOERR_BLOCKED]) -** is returned immediately upon encountering the lock. -** If the busy callback is not NULL, then the -** callback will be invoked with two arguments. The -** first argument to the handler is a copy of the void* pointer which -** is the third argument to this routine. The second argument to -** the handler is the number of times that the busy handler has -** been invoked for this locking event. If the +** ^The sqlite3_busy_handler(D,X,P) routine sets a callback function X +** that might be invoked with argument P whenever +** an attempt is made to access a database table associated with +** [database connection] D when another thread +** or process has the table locked. +** The sqlite3_busy_handler() interface is used to implement +** [sqlite3_busy_timeout()] and [PRAGMA busy_timeout]. +** +** ^If the busy callback is NULL, then [SQLITE_BUSY] +** is returned immediately upon encountering the lock. ^If the busy callback +** is not NULL, then the callback might be invoked with two arguments. +** +** ^The first argument to the busy handler is a copy of the void* pointer which +** is the third argument to sqlite3_busy_handler(). ^The second argument to +** the busy handler callback is the number of times that the busy handler has +** been invoked previously for the same locking event. ^If the ** busy callback returns 0, then no additional attempts are made to -** access the database and [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED] is returned. -** If the callback returns non-zero, then another attempt is made to open the -** database for reading and the cycle repeats. +** access the database and [SQLITE_BUSY] is returned +** to the application. +** ^If the callback returns non-zero, then another attempt +** is made to access the database and the cycle repeats. ** -** The presence of a busy handler does not guarantee that -** it will be invoked when there is lock contention. -** If SQLite determines that invoking the busy handler could result in -** a deadlock, it will return [SQLITE_BUSY] instead. +** The presence of a busy handler does not guarantee that it will be invoked +** when there is lock contention. ^If SQLite determines that invoking the busy +** handler could result in a deadlock, it will go ahead and return [SQLITE_BUSY] +** to the application instead of invoking the +** busy handler. ** Consider a scenario where one process is holding a read lock that ** it is trying to promote to a reserved lock and ** a second process is holding a reserved lock that it is trying @@ -474,273 +2848,396 @@ int sqlite3_complete16(const void *sql); ** will induce the first process to release its read lock and allow ** the second process to proceed. ** -** The default busy callback is NULL. +** ^The default busy callback is NULL. ** -** The [SQLITE_BUSY] error is converted to [SQLITE_IOERR_BLOCKED] when -** SQLite is in the middle of a large transaction where all the -** changes will not fit into the in-memory cache. SQLite will -** already hold a RESERVED lock on the database file, but it needs -** to promote this lock to EXCLUSIVE so that it can spill cache -** pages into the database file without harm to concurrent -** readers. If it is unable to promote the lock, then the in-memory -** cache will be left in an inconsistent state and so the error -** code is promoted from the relatively benign [SQLITE_BUSY] to -** the more severe [SQLITE_IOERR_BLOCKED]. This error code promotion -** forces an automatic rollback of the changes. See the -** -** CorruptionFollowingBusyError wiki page for a discussion of why -** this is important. -** -** Sqlite is re-entrant, so the busy handler may start a new query. -** (It is not clear why anyone would every want to do this, but it -** is allowed, in theory.) But the busy handler may not close the -** database. Closing the database from a busy handler will delete -** data structures out from under the executing query and will -** probably result in a segmentation fault or other runtime error. +** ^(There can only be a single busy handler defined for each +** [database connection]. Setting a new busy handler clears any +** previously set handler.)^ ^Note that calling [sqlite3_busy_timeout()] +** or evaluating [PRAGMA busy_timeout=N] will change the +** busy handler and thus clear any previously set busy handler. ** -** There can only be a single busy handler defined for each database -** connection. Setting a new busy handler clears any previous one. -** Note that calling [sqlite3_busy_timeout()] will also set or clear -** the busy handler. +** The busy callback should not take any actions which modify the +** database connection that invoked the busy handler. In other words, +** the busy handler is not reentrant. Any such actions +** result in undefined behavior. +** +** A busy handler must not close the database connection +** or [prepared statement] that invoked the busy handler. */ -int sqlite3_busy_handler(sqlite3*, int(*)(void*,int), void*); +SQLITE_API int sqlite3_busy_handler(sqlite3*,int(*)(void*,int),void*); /* ** CAPI3REF: Set A Busy Timeout +** METHOD: sqlite3 ** -** This routine sets a busy handler that sleeps for a while when a -** table is locked. The handler will sleep multiple times until -** at least "ms" milliseconds of sleeping have been done. After -** "ms" milliseconds of sleeping, the handler returns 0 which -** causes [sqlite3_step()] to return [SQLITE_BUSY] or [SQLITE_IOERR_BLOCKED]. +** ^This routine sets a [sqlite3_busy_handler | busy handler] that sleeps +** for a specified amount of time when a table is locked. ^The handler +** will sleep multiple times until at least "ms" milliseconds of sleeping +** have accumulated. ^After at least "ms" milliseconds of sleeping, +** the handler returns 0 which causes [sqlite3_step()] to return +** [SQLITE_BUSY]. ** -** Calling this routine with an argument less than or equal to zero +** ^Calling this routine with an argument less than or equal to zero ** turns off all busy handlers. ** -** There can only be a single busy handler for a particular database -** connection. If another busy handler was defined -** (using [sqlite3_busy_handler()]) prior to calling -** this routine, that other busy handler is cleared. +** ^(There can only be a single busy handler for a particular +** [database connection] at any given moment. If another busy handler +** was defined (using [sqlite3_busy_handler()]) prior to calling +** this routine, that other busy handler is cleared.)^ +** +** See also: [PRAGMA busy_timeout] */ -int sqlite3_busy_timeout(sqlite3*, int ms); +SQLITE_API int sqlite3_busy_timeout(sqlite3*, int ms); /* ** CAPI3REF: Convenience Routines For Running Queries +** METHOD: sqlite3 ** -** This next routine is a convenience wrapper around [sqlite3_exec()]. -** Instead of invoking a user-supplied callback for each row of the -** result, this routine remembers each row of the result in memory -** obtained from [sqlite3_malloc()], then returns all of the result after the -** query has finished. +** This is a legacy interface that is preserved for backwards compatibility. +** Use of this interface is not recommended. ** -** As an example, suppose the query result where this table: +** Definition: A result table is memory data structure created by the +** [sqlite3_get_table()] interface. A result table records the +** complete query results from one or more queries. ** -** 
                +** The table conceptually has a number of rows and columns.  But
+** these numbers are not part of the result table itself.  These
+** numbers are obtained separately.  Let N be the number of rows
+** and M be the number of columns.
+**
+** A result table is an array of pointers to zero-terminated UTF-8 strings.
+** There are (N+1)*M elements in the array.  The first M pointers point
+** to zero-terminated strings that  contain the names of the columns.
+** The remaining entries all point to query results.  NULL values result
+** in NULL pointers.  All other values are in their UTF-8 zero-terminated
+** string representation as returned by [sqlite3_column_text()].
+**
+** A result table might consist of one or more memory allocations.
+** It is not safe to pass a result table directly to [sqlite3_free()].
+** A result table should be deallocated using [sqlite3_free_table()].
+**
+** ^(As an example of the result table format, suppose a query result
+** is as follows:
+**
+** 
                 **        Name        | Age
 **        -----------------------
 **        Alice       | 43
 **        Bob         | 28
 **        Cindy       | 21
-** 
                
+**
                ** -** If the 3rd argument were &azResult then after the function returns -** azResult will contain the following data: +** There are two columns (M==2) and three rows (N==3). Thus the +** result table has 8 entries. Suppose the result table is stored +** in an array named azResult. Then azResult holds this content: ** -** 
                -**        azResult[0] = "Name";
-**        azResult[1] = "Age";
-**        azResult[2] = "Alice";
-**        azResult[3] = "43";
-**        azResult[4] = "Bob";
-**        azResult[5] = "28";
-**        azResult[6] = "Cindy";
-**        azResult[7] = "21";
-**
                +** 
                +**        azResult[0] = "Name";
+**        azResult[1] = "Age";
+**        azResult[2] = "Alice";
+**        azResult[3] = "43";
+**        azResult[4] = "Bob";
+**        azResult[5] = "28";
+**        azResult[6] = "Cindy";
+**        azResult[7] = "21";
+**
                )^ ** -** Notice that there is an extra row of data containing the column -** headers. But the *nrow return value is still 3. *ncolumn is -** set to 2. In general, the number of values inserted into azResult -** will be ((*nrow) + 1)*(*ncolumn). +** ^The sqlite3_get_table() function evaluates one or more +** semicolon-separated SQL statements in the zero-terminated UTF-8 +** string of its 2nd parameter and returns a result table to the +** pointer given in its 3rd parameter. ** -** After the calling function has finished using the result, it should -** pass the result data pointer to sqlite3_free_table() in order to -** release the memory that was malloc-ed. Because of the way the -** [sqlite3_malloc()] happens, the calling function must not try to call -** [sqlite3_free()] directly. Only [sqlite3_free_table()] is able to release -** the memory properly and safely. +** After the application has finished with the result from sqlite3_get_table(), +** it must pass the result table pointer to sqlite3_free_table() in order to +** release the memory that was malloced. Because of the way the +** [sqlite3_malloc()] happens within sqlite3_get_table(), the calling +** function must not try to call [sqlite3_free()] directly. Only +** [sqlite3_free_table()] is able to release the memory properly and safely. ** -** The return value of this routine is the same as from [sqlite3_exec()]. +** The sqlite3_get_table() interface is implemented as a wrapper around +** [sqlite3_exec()]. The sqlite3_get_table() routine does not have access +** to any internal data structures of SQLite. It uses only the public +** interface defined here. As a consequence, errors that occur in the +** wrapper layer outside of the internal [sqlite3_exec()] call are not +** reflected in subsequent calls to [sqlite3_errcode()] or +** [sqlite3_errmsg()]. */ -int sqlite3_get_table( - sqlite3*, /* An open database */ - const char *sql, /* SQL to be executed */ - char ***resultp, /* Result written to a char *[] that this points to */ - int *nrow, /* Number of result rows written here */ - int *ncolumn, /* Number of result columns written here */ - char **errmsg /* Error msg written here */ +SQLITE_API int sqlite3_get_table( + sqlite3 *db, /* An open database */ + const char *zSql, /* SQL to be evaluated */ + char ***pazResult, /* Results of the query */ + int *pnRow, /* Number of result rows written here */ + int *pnColumn, /* Number of result columns written here */ + char **pzErrmsg /* Error msg written here */ ); -void sqlite3_free_table(char **result); +SQLITE_API void sqlite3_free_table(char **result); /* ** CAPI3REF: Formatted String Printing Functions ** -** These routines are workalikes of the "printf()" family of functions +** These routines are work-alikes of the "printf()" family of functions ** from the standard C library. +** These routines understand most of the common formatting options from +** the standard library printf() +** plus some additional non-standard formats ([%q], [%Q], [%w], and [%z]). +** See the [built-in printf()] documentation for details. ** -** The sqlite3_mprintf() and sqlite3_vmprintf() routines write their -** results into memory obtained from [sqlite_malloc()]. +** ^The sqlite3_mprintf() and sqlite3_vmprintf() routines write their +** results into memory obtained from [sqlite3_malloc64()]. ** The strings returned by these two routines should be -** released by [sqlite3_free()]. Both routines return a -** NULL pointer if [sqlite3_malloc()] is unable to allocate enough +** released by [sqlite3_free()]. ^Both routines return a +** NULL pointer if [sqlite3_malloc64()] is unable to allocate enough ** memory to hold the resulting string. ** -** In sqlite3_snprintf() routine is similar to "snprintf()" from +** ^(The sqlite3_snprintf() routine is similar to "snprintf()" from ** the standard C library. The result is written into the ** buffer supplied as the second parameter whose size is given by -** the first parameter. Note that the order of the -** first two parameters is reversed from snprintf(). This is an +** the first parameter. Note that the order of the +** first two parameters is reversed from snprintf().)^ This is an ** historical accident that cannot be fixed without breaking -** backwards compatibility. Note also that sqlite3_snprintf() +** backwards compatibility. ^(Note also that sqlite3_snprintf() ** returns a pointer to its buffer instead of the number of -** characters actually written into the buffer. We admit that +** characters actually written into the buffer.)^ We admit that ** the number of characters written would be a more useful return ** value but we cannot change the implementation of sqlite3_snprintf() ** now without breaking compatibility. ** -** As long as the buffer size is greater than zero, sqlite3_snprintf() -** guarantees that the buffer is always zero-terminated. The first +** ^As long as the buffer size is greater than zero, sqlite3_snprintf() +** guarantees that the buffer is always zero-terminated. ^The first ** parameter "n" is the total size of the buffer, including space for ** the zero terminator. So the longest string that can be completely ** written will be n-1 characters. ** -** These routines all implement some additional formatting -** options that are useful for constructing SQL statements. -** All of the usual printf formatting options apply. In addition, there -** is are "%q" and "%Q" options. +** ^The sqlite3_vsnprintf() routine is a varargs version of sqlite3_snprintf(). ** -** The %q option works like %s in that it substitutes a null-terminated -** string from the argument list. But %q also doubles every '\'' character. -** %q is designed for use inside a string literal. By doubling each '\'' -** character it escapes that character and allows it to be inserted into -** the string. -** -** For example, so some string variable contains text as follows: -** -** 
                -**  char *zText = "It's a happy day!";
-**
                -** -** One can use this text in an SQL statement as follows: -** -** 
                -**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES('%q')", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-**
                -** -** Because the %q format string is used, the '\'' character in zText -** is escaped and the SQL generated is as follows: -** -** 
                -**  INSERT INTO table1 VALUES('It''s a happy day!')
-**
                -** -** This is correct. Had we used %s instead of %q, the generated SQL -** would have looked like this: -** -** 
                -**  INSERT INTO table1 VALUES('It's a happy day!');
-**
                -** -** This second example is an SQL syntax error. As a general rule you -** should always use %q instead of %s when inserting text into a string -** literal. -** -** The %Q option works like %q except it also adds single quotes around -** the outside of the total string. Or if the parameter in the argument -** list is a NULL pointer, %Q substitutes the text "NULL" (without single -** quotes) in place of the %Q option. So, for example, one could say: -** -** 
                -**  char *zSQL = sqlite3_mprintf("INSERT INTO table VALUES(%Q)", zText);
-**  sqlite3_exec(db, zSQL, 0, 0, 0);
-**  sqlite3_free(zSQL);
-**
                -** -** The code above will render a correct SQL statement in the zSQL -** variable even if the zText variable is a NULL pointer. +** See also: [built-in printf()], [printf() SQL function] */ -char *sqlite3_mprintf(const char*,...); -char *sqlite3_vmprintf(const char*, va_list); -char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_mprintf(const char*,...); +SQLITE_API char *sqlite3_vmprintf(const char*, va_list); +SQLITE_API char *sqlite3_snprintf(int,char*,const char*, ...); +SQLITE_API char *sqlite3_vsnprintf(int,char*,const char*, va_list); /* -** CAPI3REF: Memory Allocation Functions +** CAPI3REF: Memory Allocation Subsystem ** -** SQLite uses its own memory allocator. On some installations, this -** memory allocator is identical to the standard malloc()/realloc()/free() -** and can be used interchangable. On others, the implementations are -** different. For maximum portability, it is best not to mix calls -** to the standard malloc/realloc/free with the sqlite versions. +** The SQLite core uses these three routines for all of its own +** internal memory allocation needs. "Core" in the previous sentence +** does not include operating-system specific [VFS] implementation. The +** Windows VFS uses native malloc() and free() for some operations. +** +** ^The sqlite3_malloc() routine returns a pointer to a block +** of memory at least N bytes in length, where N is the parameter. +** ^If sqlite3_malloc() is unable to obtain sufficient free +** memory, it returns a NULL pointer. ^If the parameter N to +** sqlite3_malloc() is zero or negative then sqlite3_malloc() returns +** a NULL pointer. +** +** ^The sqlite3_malloc64(N) routine works just like +** sqlite3_malloc(N) except that N is an unsigned 64-bit integer instead +** of a signed 32-bit integer. +** +** ^Calling sqlite3_free() with a pointer previously returned +** by sqlite3_malloc() or sqlite3_realloc() releases that memory so +** that it might be reused. ^The sqlite3_free() routine is +** a no-op if is called with a NULL pointer. Passing a NULL pointer +** to sqlite3_free() is harmless. After being freed, memory +** should neither be read nor written. Even reading previously freed +** memory might result in a segmentation fault or other severe error. +** Memory corruption, a segmentation fault, or other severe error +** might result if sqlite3_free() is called with a non-NULL pointer that +** was not obtained from sqlite3_malloc() or sqlite3_realloc(). +** +** ^The sqlite3_realloc(X,N) interface attempts to resize a +** prior memory allocation X to be at least N bytes. +** ^If the X parameter to sqlite3_realloc(X,N) +** is a NULL pointer then its behavior is identical to calling +** sqlite3_malloc(N). +** ^If the N parameter to sqlite3_realloc(X,N) is zero or +** negative then the behavior is exactly the same as calling +** sqlite3_free(X). +** ^sqlite3_realloc(X,N) returns a pointer to a memory allocation +** of at least N bytes in size or NULL if insufficient memory is available. +** ^If M is the size of the prior allocation, then min(N,M) bytes +** of the prior allocation are copied into the beginning of buffer returned +** by sqlite3_realloc(X,N) and the prior allocation is freed. +** ^If sqlite3_realloc(X,N) returns NULL and N is positive, then the +** prior allocation is not freed. +** +** ^The sqlite3_realloc64(X,N) interfaces works the same as +** sqlite3_realloc(X,N) except that N is a 64-bit unsigned integer instead +** of a 32-bit signed integer. +** +** ^If X is a memory allocation previously obtained from sqlite3_malloc(), +** sqlite3_malloc64(), sqlite3_realloc(), or sqlite3_realloc64(), then +** sqlite3_msize(X) returns the size of that memory allocation in bytes. +** ^The value returned by sqlite3_msize(X) might be larger than the number +** of bytes requested when X was allocated. ^If X is a NULL pointer then +** sqlite3_msize(X) returns zero. If X points to something that is not +** the beginning of memory allocation, or if it points to a formerly +** valid memory allocation that has now been freed, then the behavior +** of sqlite3_msize(X) is undefined and possibly harmful. +** +** ^The memory returned by sqlite3_malloc(), sqlite3_realloc(), +** sqlite3_malloc64(), and sqlite3_realloc64() +** is always aligned to at least an 8 byte boundary, or to a +** 4 byte boundary if the [SQLITE_4_BYTE_ALIGNED_MALLOC] compile-time +** option is used. +** +** The pointer arguments to [sqlite3_free()] and [sqlite3_realloc()] +** must be either NULL or else pointers obtained from a prior +** invocation of [sqlite3_malloc()] or [sqlite3_realloc()] that have +** not yet been released. +** +** The application must not read or write any part of +** a block of memory after it has been released using +** [sqlite3_free()] or [sqlite3_realloc()]. */ -void *sqlite3_malloc(int); -void *sqlite3_realloc(void*, int); -void sqlite3_free(void*); +SQLITE_API void *sqlite3_malloc(int); +SQLITE_API void *sqlite3_malloc64(sqlite3_uint64); +SQLITE_API void *sqlite3_realloc(void*, int); +SQLITE_API void *sqlite3_realloc64(void*, sqlite3_uint64); +SQLITE_API void sqlite3_free(void*); +SQLITE_API sqlite3_uint64 sqlite3_msize(void*); + +/* +** CAPI3REF: Memory Allocator Statistics +** +** SQLite provides these two interfaces for reporting on the status +** of the [sqlite3_malloc()], [sqlite3_free()], and [sqlite3_realloc()] +** routines, which form the built-in memory allocation subsystem. +** +** ^The [sqlite3_memory_used()] routine returns the number of bytes +** of memory currently outstanding (malloced but not freed). +** ^The [sqlite3_memory_highwater()] routine returns the maximum +** value of [sqlite3_memory_used()] since the high-water mark +** was last reset. ^The values returned by [sqlite3_memory_used()] and +** [sqlite3_memory_highwater()] include any overhead +** added by SQLite in its implementation of [sqlite3_malloc()], +** but not overhead added by the any underlying system library +** routines that [sqlite3_malloc()] may call. +** +** ^The memory high-water mark is reset to the current value of +** [sqlite3_memory_used()] if and only if the parameter to +** [sqlite3_memory_highwater()] is true. ^The value returned +** by [sqlite3_memory_highwater(1)] is the high-water mark +** prior to the reset. +*/ +SQLITE_API sqlite3_int64 sqlite3_memory_used(void); +SQLITE_API sqlite3_int64 sqlite3_memory_highwater(int resetFlag); + +/* +** CAPI3REF: Pseudo-Random Number Generator +** +** SQLite contains a high-quality pseudo-random number generator (PRNG) used to +** select random [ROWID | ROWIDs] when inserting new records into a table that +** already uses the largest possible [ROWID]. The PRNG is also used for +** the built-in random() and randomblob() SQL functions. This interface allows +** applications to access the same PRNG for other purposes. +** +** ^A call to this routine stores N bytes of randomness into buffer P. +** ^The P parameter can be a NULL pointer. +** +** ^If this routine has not been previously called or if the previous +** call had N less than one or a NULL pointer for P, then the PRNG is +** seeded using randomness obtained from the xRandomness method of +** the default [sqlite3_vfs] object. +** ^If the previous call to this routine had an N of 1 or more and a +** non-NULL P then the pseudo-randomness is generated +** internally and without recourse to the [sqlite3_vfs] xRandomness +** method. +*/ +SQLITE_API void sqlite3_randomness(int N, void *P); /* ** CAPI3REF: Compile-Time Authorization Callbacks -*** -** This routine registers a authorizer callback with the SQLite library. -** The authorizer callback is invoked as SQL statements are being compiled +** METHOD: sqlite3 +** KEYWORDS: {authorizer callback} +** +** ^This routine registers an authorizer callback with a particular +** [database connection], supplied in the first argument. +** ^The authorizer callback is invoked as SQL statements are being compiled ** by [sqlite3_prepare()] or its variants [sqlite3_prepare_v2()], -** [sqlite3_prepare16()] and [sqlite3_prepare16_v2()]. At various +** [sqlite3_prepare_v3()], [sqlite3_prepare16()], [sqlite3_prepare16_v2()], +** and [sqlite3_prepare16_v3()]. ^At various ** points during the compilation process, as logic is being created ** to perform various actions, the authorizer callback is invoked to -** see if those actions are allowed. The authorizer callback should -** return SQLITE_OK to allow the action, [SQLITE_IGNORE] to disallow the +** see if those actions are allowed. ^The authorizer callback should +** return [SQLITE_OK] to allow the action, [SQLITE_IGNORE] to disallow the ** specific action but allow the SQL statement to continue to be ** compiled, or [SQLITE_DENY] to cause the entire SQL statement to be -** rejected with an error. +** rejected with an error. ^If the authorizer callback returns +** any value other than [SQLITE_IGNORE], [SQLITE_OK], or [SQLITE_DENY] +** then the [sqlite3_prepare_v2()] or equivalent call that triggered +** the authorizer will fail with an error message. ** -** Depending on the action, the [SQLITE_IGNORE] and [SQLITE_DENY] return -** codes might mean something different or they might mean the same -** thing. If the action is, for example, to perform a delete opertion, -** then [SQLITE_IGNORE] and [SQLITE_DENY] both cause the statement compilation -** to fail with an error. But if the action is to read a specific column -** from a specific table, then [SQLITE_DENY] will cause the entire -** statement to fail but [SQLITE_IGNORE] will cause a NULL value to be -** read instead of the actual column value. +** When the callback returns [SQLITE_OK], that means the operation +** requested is ok. ^When the callback returns [SQLITE_DENY], the +** [sqlite3_prepare_v2()] or equivalent call that triggered the +** authorizer will fail with an error message explaining that +** access is denied. ** -** The first parameter to the authorizer callback is a copy of -** the third parameter to the sqlite3_set_authorizer() interface. -** The second parameter to the callback is an integer -** [SQLITE_COPY | action code] that specifies the particular action -** to be authorized. The available action codes are -** [SQLITE_COPY | documented separately]. The third through sixth -** parameters to the callback are strings that contain additional -** details about the action to be authorized. +** ^The first parameter to the authorizer callback is a copy of the third +** parameter to the sqlite3_set_authorizer() interface. ^The second parameter +** to the callback is an integer [SQLITE_COPY | action code] that specifies +** the particular action to be authorized. ^The third through sixth parameters +** to the callback are either NULL pointers or zero-terminated strings +** that contain additional details about the action to be authorized. +** Applications must always be prepared to encounter a NULL pointer in any +** of the third through the sixth parameters of the authorization callback. ** -** An authorizer is used when preparing SQL statements from an untrusted -** source, to ensure that the SQL statements do not try to access data -** that they are not allowed to see, or that they do not try to -** execute malicious statements that damage the database. For +** ^If the action code is [SQLITE_READ] +** and the callback returns [SQLITE_IGNORE] then the +** [prepared statement] statement is constructed to substitute +** a NULL value in place of the table column that would have +** been read if [SQLITE_OK] had been returned. The [SQLITE_IGNORE] +** return can be used to deny an untrusted user access to individual +** columns of a table. +** ^When a table is referenced by a [SELECT] but no column values are +** extracted from that table (for example in a query like +** "SELECT count(*) FROM tab") then the [SQLITE_READ] authorizer callback +** is invoked once for that table with a column name that is an empty string. +** ^If the action code is [SQLITE_DELETE] and the callback returns +** [SQLITE_IGNORE] then the [DELETE] operation proceeds but the +** [truncate optimization] is disabled and all rows are deleted individually. +** +** An authorizer is used when [sqlite3_prepare | preparing] +** SQL statements from an untrusted source, to ensure that the SQL statements +** do not try to access data they are not allowed to see, or that they do not +** try to execute malicious statements that damage the database. For ** example, an application may allow a user to enter arbitrary ** SQL queries for evaluation by a database. But the application does ** not want the user to be able to make arbitrary changes to the ** database. An authorizer could then be put in place while the -** user-entered SQL is being prepared that disallows everything -** except SELECT statements. +** user-entered SQL is being [sqlite3_prepare | prepared] that +** disallows everything except [SELECT] statements. ** -** Only a single authorizer can be in place on a database connection +** Applications that need to process SQL from untrusted sources +** might also consider lowering resource limits using [sqlite3_limit()] +** and limiting database size using the [max_page_count] [PRAGMA] +** in addition to using an authorizer. +** +** ^(Only a single authorizer can be in place on a database connection ** at a time. Each call to sqlite3_set_authorizer overrides the -** previous call. A NULL authorizer means that no authorization -** callback is invoked. The default authorizer is NULL. +** previous call.)^ ^Disable the authorizer by installing a NULL callback. +** The authorizer is disabled by default. ** -** Note that the authorizer callback is invoked only during +** The authorizer callback must not do anything that will modify +** the database connection that invoked the authorizer callback. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^When [sqlite3_prepare_v2()] is used to prepare a statement, the +** statement might be re-prepared during [sqlite3_step()] due to a +** schema change. Hence, the application should ensure that the +** correct authorizer callback remains in place during the [sqlite3_step()]. +** +** ^Note that the authorizer callback is invoked only during ** [sqlite3_prepare()] or its variants. Authorization is not -** performed during statement evaluation in [sqlite3_step()]. +** performed during statement evaluation in [sqlite3_step()], unless +** as stated in the previous paragraph, sqlite3_step() invokes +** sqlite3_prepare_v2() to reprepare a statement after a schema change. */ -int sqlite3_set_authorizer( +SQLITE_API int sqlite3_set_authorizer( sqlite3*, int (*xAuth)(void*,int,const char*,const char*,const char*,const char*), void *pUserData @@ -754,6 +3251,9 @@ int sqlite3_set_authorizer( ** to signal SQLite whether or not the action is permitted. See the ** [sqlite3_set_authorizer | authorizer documentation] for additional ** information. +** +** Note that SQLITE_IGNORE is also used as a [conflict resolution mode] +** returned from the [sqlite3_vtab_on_conflict()] interface. */ #define SQLITE_DENY 1 /* Abort the SQL statement with an error */ #define SQLITE_IGNORE 2 /* Don't allow access, but don't generate an error */ @@ -762,19 +3262,19 @@ int sqlite3_set_authorizer( ** CAPI3REF: Authorizer Action Codes ** ** The [sqlite3_set_authorizer()] interface registers a callback function -** that is invoked to authorizer certain SQL statement actions. The +** that is invoked to authorize certain SQL statement actions. The ** second parameter to the callback is an integer code that specifies ** what action is being authorized. These are the integer action codes that ** the authorizer callback may be passed. ** -** These action code values signify what kind of operation is to be -** authorized. The 3rd and 4th parameters to the authorization callback -** function will be parameters or NULL depending on which of these -** codes is used as the second parameter. The 5th parameter to the -** authorizer callback is the name of the database ("main", "temp", -** etc.) if applicable. The 6th parameter to the authorizer callback +** These action code values signify what kind of operation is to be +** authorized. The 3rd and 4th parameters to the authorization +** callback function will be parameters or NULL depending on which of these +** codes is used as the second parameter. ^(The 5th parameter to the +** authorizer callback is the name of the database ("main", "temp", +** etc.) if applicable.)^ ^The 6th parameter to the authorizer callback ** is the name of the inner-most trigger or view that is responsible for -** the access attempt or NULL if this access attempt is directly from +** the access attempt or NULL if this access attempt is directly from ** top-level SQL code. */ /******************************************* 3rd ************ 4th ***********/ @@ -799,7 +3299,7 @@ int sqlite3_set_authorizer( #define SQLITE_PRAGMA 19 /* Pragma Name 1st arg or NULL */ #define SQLITE_READ 20 /* Table Name Column Name */ #define SQLITE_SELECT 21 /* NULL NULL */ -#define SQLITE_TRANSACTION 22 /* NULL NULL */ +#define SQLITE_TRANSACTION 22 /* Operation NULL */ #define SQLITE_UPDATE 23 /* Table Name Column Name */ #define SQLITE_ATTACH 24 /* Filename NULL */ #define SQLITE_DETACH 25 /* Database Name NULL */ @@ -808,586 +3308,1740 @@ int sqlite3_set_authorizer( #define SQLITE_ANALYZE 28 /* Table Name NULL */ #define SQLITE_CREATE_VTABLE 29 /* Table Name Module Name */ #define SQLITE_DROP_VTABLE 30 /* Table Name Module Name */ -#define SQLITE_FUNCTION 31 /* Function Name NULL */ +#define SQLITE_FUNCTION 31 /* NULL Function Name */ +#define SQLITE_SAVEPOINT 32 /* Operation Savepoint Name */ #define SQLITE_COPY 0 /* No longer used */ +#define SQLITE_RECURSIVE 33 /* NULL NULL */ /* -** CAPI3REF: Tracing And Profiling Functions +** CAPI3REF: Deprecated Tracing And Profiling Functions +** DEPRECATED +** +** These routines are deprecated. Use the [sqlite3_trace_v2()] interface +** instead of the routines described here. ** ** These routines register callback functions that can be used for ** tracing and profiling the execution of SQL statements. -** The callback function registered by sqlite3_trace() is invoked -** at the first [sqlite3_step()] for the evaluation of an SQL statement. -** The callback function registered by sqlite3_profile() is invoked -** as each SQL statement finishes and includes -** information on how long that statement ran. ** -** The sqlite3_profile() API is currently considered experimental and -** is subject to change. +** ^The callback function registered by sqlite3_trace() is invoked at +** various times when an SQL statement is being run by [sqlite3_step()]. +** ^The sqlite3_trace() callback is invoked with a UTF-8 rendering of the +** SQL statement text as the statement first begins executing. +** ^(Additional sqlite3_trace() callbacks might occur +** as each triggered subprogram is entered. The callbacks for triggers +** contain a UTF-8 SQL comment that identifies the trigger.)^ +** +** The [SQLITE_TRACE_SIZE_LIMIT] compile-time option can be used to limit +** the length of [bound parameter] expansion in the output of sqlite3_trace(). +** +** ^The callback function registered by sqlite3_profile() is invoked +** as each SQL statement finishes. ^The profile callback contains +** the original statement text and an estimate of wall-clock time +** of how long that statement took to run. ^The profile callback +** time is in units of nanoseconds, however the current implementation +** is only capable of millisecond resolution so the six least significant +** digits in the time are meaningless. Future versions of SQLite +** might provide greater resolution on the profiler callback. Invoking +** either [sqlite3_trace()] or [sqlite3_trace_v2()] will cancel the +** profile callback. */ -void *sqlite3_trace(sqlite3*, void(*xTrace)(void*,const char*), void*); -void *sqlite3_profile(sqlite3*, - void(*xProfile)(void*,const char*,sqlite_uint64), void*); +SQLITE_API SQLITE_DEPRECATED void *sqlite3_trace(sqlite3*, + void(*xTrace)(void*,const char*), void*); +SQLITE_API SQLITE_DEPRECATED void *sqlite3_profile(sqlite3*, + void(*xProfile)(void*,const char*,sqlite3_uint64), void*); + +/* +** CAPI3REF: SQL Trace Event Codes +** KEYWORDS: SQLITE_TRACE +** +** These constants identify classes of events that can be monitored +** using the [sqlite3_trace_v2()] tracing logic. The M argument +** to [sqlite3_trace_v2(D,M,X,P)] is an OR-ed combination of one or more of +** the following constants. ^The first argument to the trace callback +** is one of the following constants. +** +** New tracing constants may be added in future releases. +** +** ^A trace callback has four arguments: xCallback(T,C,P,X). +** ^The T argument is one of the integer type codes above. +** ^The C argument is a copy of the context pointer passed in as the +** fourth argument to [sqlite3_trace_v2()]. +** The P and X arguments are pointers whose meanings depend on T. +** +**
                +** [[SQLITE_TRACE_STMT]]
                SQLITE_TRACE_STMT
                +**
                ^An SQLITE_TRACE_STMT callback is invoked when a prepared statement +** first begins running and possibly at other times during the +** execution of the prepared statement, such as at the start of each +** trigger subprogram. ^The P argument is a pointer to the +** [prepared statement]. ^The X argument is a pointer to a string which +** is the unexpanded SQL text of the prepared statement or an SQL comment +** that indicates the invocation of a trigger. ^The callback can compute +** the same text that would have been returned by the legacy [sqlite3_trace()] +** interface by using the X argument when X begins with "--" and invoking +** [sqlite3_expanded_sql(P)] otherwise. +** +** [[SQLITE_TRACE_PROFILE]]
                SQLITE_TRACE_PROFILE
                +**
                ^An SQLITE_TRACE_PROFILE callback provides approximately the same +** information as is provided by the [sqlite3_profile()] callback. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument points to a 64-bit integer which is approximately +** the number of nanoseconds that the prepared statement took to run. +** ^The SQLITE_TRACE_PROFILE callback is invoked when the statement finishes. +** +** [[SQLITE_TRACE_ROW]]
                SQLITE_TRACE_ROW
                +**
                ^An SQLITE_TRACE_ROW callback is invoked whenever a prepared +** statement generates a single row of result. +** ^The P argument is a pointer to the [prepared statement] and the +** X argument is unused. +** +** [[SQLITE_TRACE_CLOSE]]
                SQLITE_TRACE_CLOSE
                +**
                ^An SQLITE_TRACE_CLOSE callback is invoked when a database +** connection closes. +** ^The P argument is a pointer to the [database connection] object +** and the X argument is unused. +**
                +*/ +#define SQLITE_TRACE_STMT 0x01 +#define SQLITE_TRACE_PROFILE 0x02 +#define SQLITE_TRACE_ROW 0x04 +#define SQLITE_TRACE_CLOSE 0x08 + +/* +** CAPI3REF: SQL Trace Hook +** METHOD: sqlite3 +** +** ^The sqlite3_trace_v2(D,M,X,P) interface registers a trace callback +** function X against [database connection] D, using property mask M +** and context pointer P. ^If the X callback is +** NULL or if the M mask is zero, then tracing is disabled. The +** M argument should be the bitwise OR-ed combination of +** zero or more [SQLITE_TRACE] constants. +** +** ^Each call to either sqlite3_trace(D,X,P) or sqlite3_trace_v2(D,M,X,P) +** overrides (cancels) all prior calls to sqlite3_trace(D,X,P) or +** sqlite3_trace_v2(D,M,X,P) for the [database connection] D. Each +** database connection may have at most one trace callback. +** +** ^The X callback is invoked whenever any of the events identified by +** mask M occur. ^The integer return value from the callback is currently +** ignored, though this may change in future releases. Callback +** implementations should return zero to ensure future compatibility. +** +** ^A trace callback is invoked with four arguments: callback(T,C,P,X). +** ^The T argument is one of the [SQLITE_TRACE] +** constants to indicate why the callback was invoked. +** ^The C argument is a copy of the context pointer. +** The P and X arguments are pointers whose meanings depend on T. +** +** The sqlite3_trace_v2() interface is intended to replace the legacy +** interfaces [sqlite3_trace()] and [sqlite3_profile()], both of which +** are deprecated. +*/ +SQLITE_API int sqlite3_trace_v2( + sqlite3*, + unsigned uMask, + int(*xCallback)(unsigned,void*,void*,void*), + void *pCtx +); /* ** CAPI3REF: Query Progress Callbacks +** METHOD: sqlite3 ** -** This routine configures a callback function - the progress callback - that -** is invoked periodically during long running calls to [sqlite3_exec()], -** [sqlite3_step()] and [sqlite3_get_table()]. An example use for this +** ^The sqlite3_progress_handler(D,N,X,P) interface causes the callback +** function X to be invoked periodically during long running calls to +** [sqlite3_step()] and [sqlite3_prepare()] and similar for +** database connection D. An example use for this ** interface is to keep a GUI updated during a large query. ** -** The progress callback is invoked once for every N virtual machine opcodes, -** where N is the second argument to this function. The progress callback -** itself is identified by the third argument to this function. The fourth -** argument to this function is a void pointer passed to the progress callback -** function each time it is invoked. +** ^The parameter P is passed through as the only parameter to the +** callback function X. ^The parameter N is the approximate number of +** [virtual machine instructions] that are evaluated between successive +** invocations of the callback X. ^If N is less than one then the progress +** handler is disabled. ** -** If a call to [sqlite3_exec()], [sqlite3_step()], or [sqlite3_get_table()] -** results in fewer than N opcodes being executed, then the progress -** callback is never invoked. -** -** Only a single progress callback function may be registered for each -** open database connection. Every call to sqlite3_progress_handler() -** overwrites the results of the previous call. -** To remove the progress callback altogether, pass NULL as the third -** argument to this function. +** ^Only a single progress handler may be defined at one time per +** [database connection]; setting a new progress handler cancels the +** old one. ^Setting parameter X to NULL disables the progress handler. +** ^The progress handler is also disabled by setting N to a value less +** than 1. ** -** If the progress callback returns a result other than 0, then the current -** query is immediately terminated and any database changes rolled back. -** The containing [sqlite3_exec()], [sqlite3_step()], or -** [sqlite3_get_table()] call returns SQLITE_INTERRUPT. This feature -** can be used, for example, to implement the "Cancel" button on a -** progress dialog box in a GUI. +** ^If the progress callback returns non-zero, the operation is +** interrupted. This feature can be used to implement a +** "Cancel" button on a GUI progress dialog box. +** +** The progress handler callback must not do anything that will modify +** the database connection that invoked the progress handler. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** The progress handler callback would originally only be invoked from the +** bytecode engine. It still might be invoked during [sqlite3_prepare()] +** and similar because those routines might force a reparse of the schema +** which involves running the bytecode engine. However, beginning with +** SQLite version 3.41.0, the progress handler callback might also be +** invoked directly from [sqlite3_prepare()] while analyzing and generating +** code for complex queries. */ -void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); +SQLITE_API void sqlite3_progress_handler(sqlite3*, int, int(*)(void*), void*); /* ** CAPI3REF: Opening A New Database Connection +** CONSTRUCTOR: sqlite3 ** -** Open the sqlite database file "filename". The "filename" is UTF-8 -** encoded for sqlite3_open() and UTF-16 encoded in the native byte order -** for sqlite3_open16(). An [sqlite3*] handle is returned in *ppDb, even -** if an error occurs. If the database is opened (or created) successfully, -** then SQLITE_OK is returned. Otherwise an error code is returned. The -** sqlite3_errmsg() or sqlite3_errmsg16() routines can be used to obtain -** an English language description of the error. +** ^These routines open an SQLite database file as specified by the +** filename argument. ^The filename argument is interpreted as UTF-8 for +** sqlite3_open() and sqlite3_open_v2() and as UTF-16 in the native byte +** order for sqlite3_open16(). ^(A [database connection] handle is usually +** returned in *ppDb, even if an error occurs. The only exception is that +** if SQLite is unable to allocate memory to hold the [sqlite3] object, +** a NULL will be written into *ppDb instead of a pointer to the [sqlite3] +** object.)^ ^(If the database is opened (and/or created) successfully, then +** [SQLITE_OK] is returned. Otherwise an [error code] is returned.)^ ^The +** [sqlite3_errmsg()] or [sqlite3_errmsg16()] routines can be used to obtain +** an English language description of the error following a failure of any +** of the sqlite3_open() routines. ** -** If the database file does not exist, then a new database will be created -** as needed. The default encoding for the database will be UTF-8 if -** sqlite3_open() is called and UTF-16 if sqlite3_open16 is used. +** ^The default encoding will be UTF-8 for databases created using +** sqlite3_open() or sqlite3_open_v2(). ^The default encoding for databases +** created using sqlite3_open16() will be UTF-16 in the native byte order. ** -** Whether or not an error occurs when it is opened, resources associated -** with the [sqlite3*] handle should be released by passing it to -** sqlite3_close() when it is no longer required. +** Whether or not an error occurs when it is opened, resources +** associated with the [database connection] handle should be released by +** passing it to [sqlite3_close()] when it is no longer required. ** -** Note to windows users: The encoding used for the filename argument -** of sqlite3_open() must be UTF-8, not whatever codepage is currently -** defined. Filenames containing international characters must be converted -** to UTF-8 prior to passing them into sqlite3_open(). +** The sqlite3_open_v2() interface works like sqlite3_open() +** except that it accepts two additional parameters for additional control +** over the new database connection. ^(The flags parameter to +** sqlite3_open_v2() must include, at a minimum, one of the following +** three flag combinations:)^ +** +**
                +** ^(
                [SQLITE_OPEN_READONLY]
                +**
                The database is opened in read-only mode. If the database does +** not already exist, an error is returned.
                )^ +** +** ^(
                [SQLITE_OPEN_READWRITE]
                +**
                The database is opened for reading and writing if possible, or +** reading only if the file is write protected by the operating +** system. In either case the database must already exist, otherwise +** an error is returned. For historical reasons, if opening in +** read-write mode fails due to OS-level permissions, an attempt is +** made to open it in read-only mode. [sqlite3_db_readonly()] can be +** used to determine whether the database is actually +** read-write.
                )^ +** +** ^(
                [SQLITE_OPEN_READWRITE] | [SQLITE_OPEN_CREATE]
                +**
                The database is opened for reading and writing, and is created if +** it does not already exist. This is the behavior that is always used for +** sqlite3_open() and sqlite3_open16().
                )^ +**
                +** +** In addition to the required flags, the following optional flags are +** also supported: +** +**
                +** ^(
                [SQLITE_OPEN_URI]
                +**
                The filename can be interpreted as a URI if this flag is set.
                )^ +** +** ^(
                [SQLITE_OPEN_MEMORY]
                +**
                The database will be opened as an in-memory database. The database +** is named by the "filename" argument for the purposes of cache-sharing, +** if shared cache mode is enabled, but the "filename" is otherwise ignored. +**
                )^ +** +** ^(
                [SQLITE_OPEN_NOMUTEX]
                +**
                The new database connection will use the "multi-thread" +** [threading mode].)^ This means that separate threads are allowed +** to use SQLite at the same time, as long as each thread is using +** a different [database connection]. +** +** ^(
                [SQLITE_OPEN_FULLMUTEX]
                +**
                The new database connection will use the "serialized" +** [threading mode].)^ This means the multiple threads can safely +** attempt to use the same database connection at the same time. +** (Mutexes will block any actual concurrency, but in this mode +** there is no harm in trying.) +** +** ^(
                [SQLITE_OPEN_SHAREDCACHE]
                +**
                The database is opened [shared cache] enabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** The [use of shared cache mode is discouraged] and hence shared cache +** capabilities may be omitted from many builds of SQLite. In such cases, +** this option is a no-op. +** +** ^(
                [SQLITE_OPEN_PRIVATECACHE]
                +**
                The database is opened [shared cache] disabled, overriding +** the default shared cache setting provided by +** [sqlite3_enable_shared_cache()].)^ +** +** [[OPEN_EXRESCODE]] ^(
                [SQLITE_OPEN_EXRESCODE]
                +**
                The database connection comes up in "extended result code mode". +** In other words, the database behaves as if +** [sqlite3_extended_result_codes(db,1)] were called on the database +** connection as soon as the connection is created. In addition to setting +** the extended result code mode, this flag also causes [sqlite3_open_v2()] +** to return an extended result code.
                +** +** [[OPEN_NOFOLLOW]] ^(
                [SQLITE_OPEN_NOFOLLOW]
                +**
                The database filename is not allowed to contain a symbolic link
                +**
                )^ +** +** If the 3rd parameter to sqlite3_open_v2() is not one of the +** required combinations shown above optionally combined with other +** [SQLITE_OPEN_READONLY | SQLITE_OPEN_* bits] +** then the behavior is undefined. Historic versions of SQLite +** have silently ignored surplus bits in the flags parameter to +** sqlite3_open_v2(), however that behavior might not be carried through +** into future versions of SQLite and so applications should not rely +** upon it. Note in particular that the SQLITE_OPEN_EXCLUSIVE flag is a no-op +** for sqlite3_open_v2(). The SQLITE_OPEN_EXCLUSIVE does *not* cause +** the open to fail if the database already exists. The SQLITE_OPEN_EXCLUSIVE +** flag is intended for use by the [sqlite3_vfs|VFS interface] only, and not +** by sqlite3_open_v2(). +** +** ^The fourth parameter to sqlite3_open_v2() is the name of the +** [sqlite3_vfs] object that defines the operating system interface that +** the new database connection should use. ^If the fourth parameter is +** a NULL pointer then the default [sqlite3_vfs] object is used. +** +** ^If the filename is ":memory:", then a private, temporary in-memory database +** is created for the connection. ^This in-memory database will vanish when +** the database connection is closed. Future versions of SQLite might +** make use of additional special filenames that begin with the ":" character. +** It is recommended that when a database filename actually does begin with +** a ":" character you should prefix the filename with a pathname such as +** "./" to avoid ambiguity. +** +** ^If the filename is an empty string, then a private, temporary +** on-disk database will be created. ^This private database will be +** automatically deleted as soon as the database connection is closed. +** +** [[URI filenames in sqlite3_open()]]

                URI Filenames

                +** +** ^If [URI filename] interpretation is enabled, and the filename argument +** begins with "file:", then the filename is interpreted as a URI. ^URI +** filename interpretation is enabled if the [SQLITE_OPEN_URI] flag is +** set in the third argument to sqlite3_open_v2(), or if it has +** been enabled globally using the [SQLITE_CONFIG_URI] option with the +** [sqlite3_config()] method or by the [SQLITE_USE_URI] compile-time option. +** URI filename interpretation is turned off +** by default, but future releases of SQLite might enable URI filename +** interpretation by default. See "[URI filenames]" for additional +** information. +** +** URI filenames are parsed according to RFC 3986. ^If the URI contains an +** authority, then it must be either an empty string or the string +** "localhost". ^If the authority is not an empty string or "localhost", an +** error is returned to the caller. ^The fragment component of a URI, if +** present, is ignored. +** +** ^SQLite uses the path component of the URI as the name of the disk file +** which contains the database. ^If the path begins with a '/' character, +** then it is interpreted as an absolute path. ^If the path does not begin +** with a '/' (meaning that the authority section is omitted from the URI) +** then the path is interpreted as a relative path. +** ^(On windows, the first component of an absolute path +** is a drive specification (e.g. "C:").)^ +** +** [[core URI query parameters]] +** The query component of a URI may contain parameters that are interpreted +** either by SQLite itself, or by a [VFS | custom VFS implementation]. +** SQLite and its built-in [VFSes] interpret the +** following query parameters: +** +**
                  +**
                • vfs: ^The "vfs" parameter may be used to specify the name of +** a VFS object that provides the operating system interface that should +** be used to access the database file on disk. ^If this option is set to +** an empty string the default VFS object is used. ^Specifying an unknown +** VFS is an error. ^If sqlite3_open_v2() is used and the vfs option is +** present, then the VFS specified by the option takes precedence over +** the value passed as the fourth parameter to sqlite3_open_v2(). +** +**
                • mode: ^(The mode parameter may be set to either "ro", "rw", +** "rwc", or "memory". Attempting to set it to any other value is +** an error)^. +** ^If "ro" is specified, then the database is opened for read-only +** access, just as if the [SQLITE_OPEN_READONLY] flag had been set in the +** third argument to sqlite3_open_v2(). ^If the mode option is set to +** "rw", then the database is opened for read-write (but not create) +** access, as if SQLITE_OPEN_READWRITE (but not SQLITE_OPEN_CREATE) had +** been set. ^Value "rwc" is equivalent to setting both +** SQLITE_OPEN_READWRITE and SQLITE_OPEN_CREATE. ^If the mode option is +** set to "memory" then a pure [in-memory database] that never reads +** or writes from disk is used. ^It is an error to specify a value for +** the mode parameter that is less restrictive than that specified by +** the flags passed in the third parameter to sqlite3_open_v2(). +** +**
                • cache: ^The cache parameter may be set to either "shared" or +** "private". ^Setting it to "shared" is equivalent to setting the +** SQLITE_OPEN_SHAREDCACHE bit in the flags argument passed to +** sqlite3_open_v2(). ^Setting the cache parameter to "private" is +** equivalent to setting the SQLITE_OPEN_PRIVATECACHE bit. +** ^If sqlite3_open_v2() is used and the "cache" parameter is present in +** a URI filename, its value overrides any behavior requested by setting +** SQLITE_OPEN_PRIVATECACHE or SQLITE_OPEN_SHAREDCACHE flag. +** +**
                • psow: ^The psow parameter indicates whether or not the +** [powersafe overwrite] property does or does not apply to the +** storage media on which the database file resides. +** +**
                • nolock: ^The nolock parameter is a boolean query parameter +** which if set disables file locking in rollback journal modes. This +** is useful for accessing a database on a filesystem that does not +** support locking. Caution: Database corruption might result if two +** or more processes write to the same database and any one of those +** processes uses nolock=1. +** +**
                • immutable: ^The immutable parameter is a boolean query +** parameter that indicates that the database file is stored on +** read-only media. ^When immutable is set, SQLite assumes that the +** database file cannot be changed, even by a process with higher +** privilege, and so the database is opened read-only and all locking +** and change detection is disabled. Caution: Setting the immutable +** property on a database file that does in fact change can result +** in incorrect query results and/or [SQLITE_CORRUPT] errors. +** See also: [SQLITE_IOCAP_IMMUTABLE]. +** +**
                +** +** ^Specifying an unknown parameter in the query component of a URI is not an +** error. Future versions of SQLite might understand additional query +** parameters. See "[query parameters with special meaning to SQLite]" for +** additional information. +** +** [[URI filename examples]]

                URI filename examples

                +** +**
            • +**
            URI filenames Results +**
            file:data.db +** Open the file "data.db" in the current directory. +**
            file:/home/fred/data.db
            +** file:///home/fred/data.db
            +** file://localhost/home/fred/data.db
            		+** Open the database file "/home/fred/data.db". +**
            file://darkstar/home/fred/data.db +** An error. "darkstar" is not a recognized authority. +**
            +** file:///C:/Documents%20and%20Settings/fred/Desktop/data.db +** Windows only: Open the file "data.db" on fred's desktop on drive +** C:. Note that the %20 escaping in this example is not strictly +** necessary - space characters can be used literally +** in URI filenames. +**
            file:data.db?mode=ro&cache=private +** Open file "data.db" in the current directory for read-only access. +** Regardless of whether or not shared-cache mode is enabled by +** default, use a private cache. +**
            file:/home/fred/data.db?vfs=unix-dotfile +** Open file "/home/fred/data.db". Use the special VFS "unix-dotfile" +** that uses dot-files in place of posix advisory locking. +**
            file:data.db?mode=readonly +** An error. "readonly" is not a valid option for the "mode" parameter. +** Use "ro" instead: "file:data.db?mode=ro". +**
            +** +** ^URI hexadecimal escape sequences (%HH) are supported within the path and +** query components of a URI. A hexadecimal escape sequence consists of a +** percent sign - "%" - followed by exactly two hexadecimal digits +** specifying an octet value. ^Before the path or query components of a +** URI filename are interpreted, they are encoded using UTF-8 and all +** hexadecimal escape sequences replaced by a single byte containing the +** corresponding octet. If this process generates an invalid UTF-8 encoding, +** the results are undefined. +** +** Note to Windows users: The encoding used for the filename argument +** of sqlite3_open() and sqlite3_open_v2() must be UTF-8, not whatever +** codepage is currently defined. Filenames containing international +** characters must be converted to UTF-8 prior to passing them into +** sqlite3_open() or sqlite3_open_v2(). +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling sqlite3_open() or sqlite3_open_v2(). Otherwise, various +** features that require the use of temporary files may fail. +** +** See also: [sqlite3_temp_directory] */ -int sqlite3_open( +SQLITE_API int sqlite3_open( const char *filename, /* Database filename (UTF-8) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); -int sqlite3_open16( +SQLITE_API int sqlite3_open16( const void *filename, /* Database filename (UTF-16) */ sqlite3 **ppDb /* OUT: SQLite db handle */ ); +SQLITE_API int sqlite3_open_v2( + const char *filename, /* Database filename (UTF-8) */ + sqlite3 **ppDb, /* OUT: SQLite db handle */ + int flags, /* Flags */ + const char *zVfs /* Name of VFS module to use */ +); + +/* +** CAPI3REF: Obtain Values For URI Parameters +** +** These are utility routines, useful to [VFS|custom VFS implementations], +** that check if a database file was a URI that contained a specific query +** parameter, and if so obtains the value of that query parameter. +** +** The first parameter to these interfaces (hereafter referred to +** as F) must be one of: +**
              +**
            • A database filename pointer created by the SQLite core and +** passed into the xOpen() method of a VFS implementation, or +**
            • A filename obtained from [sqlite3_db_filename()], or +**
            • A new filename constructed using [sqlite3_create_filename()]. +**
            +** If the F parameter is not one of the above, then the behavior is +** undefined and probably undesirable. Older versions of SQLite were +** more tolerant of invalid F parameters than newer versions. +** +** If F is a suitable filename (as described in the previous paragraph) +** and if P is the name of the query parameter, then +** sqlite3_uri_parameter(F,P) returns the value of the P +** parameter if it exists or a NULL pointer if P does not appear as a +** query parameter on F. If P is a query parameter of F and it +** has no explicit value, then sqlite3_uri_parameter(F,P) returns +** a pointer to an empty string. +** +** The sqlite3_uri_boolean(F,P,B) routine assumes that P is a boolean +** parameter and returns true (1) or false (0) according to the value +** of P. The sqlite3_uri_boolean(F,P,B) routine returns true (1) if the +** value of query parameter P is one of "yes", "true", or "on" in any +** case or if the value begins with a non-zero number. The +** sqlite3_uri_boolean(F,P,B) routines returns false (0) if the value of +** query parameter P is one of "no", "false", or "off" in any case or +** if the value begins with a numeric zero. If P is not a query +** parameter on F or if the value of P does not match any of the +** above, then sqlite3_uri_boolean(F,P,B) returns (B!=0). +** +** The sqlite3_uri_int64(F,P,D) routine converts the value of P into a +** 64-bit signed integer and returns that integer, or D if P does not +** exist. If the value of P is something other than an integer, then +** zero is returned. +** +** The sqlite3_uri_key(F,N) returns a pointer to the name (not +** the value) of the N-th query parameter for filename F, or a NULL +** pointer if N is less than zero or greater than the number of query +** parameters minus 1. The N value is zero-based so N should be 0 to obtain +** the name of the first query parameter, 1 for the second parameter, and +** so forth. +** +** If F is a NULL pointer, then sqlite3_uri_parameter(F,P) returns NULL and +** sqlite3_uri_boolean(F,P,B) returns B. If F is not a NULL pointer and +** is not a database file pathname pointer that the SQLite core passed +** into the xOpen VFS method, then the behavior of this routine is undefined +** and probably undesirable. +** +** Beginning with SQLite [version 3.31.0] ([dateof:3.31.0]) the input F +** parameter can also be the name of a rollback journal file or WAL file +** in addition to the main database file. Prior to version 3.31.0, these +** routines would only work if F was the name of the main database file. +** When the F parameter is the name of the rollback journal or WAL file, +** it has access to all the same query parameters as were found on the +** main database file. +** +** See the [URI filename] documentation for additional information. +*/ +SQLITE_API const char *sqlite3_uri_parameter(sqlite3_filename z, const char *zParam); +SQLITE_API int sqlite3_uri_boolean(sqlite3_filename z, const char *zParam, int bDefault); +SQLITE_API sqlite3_int64 sqlite3_uri_int64(sqlite3_filename, const char*, sqlite3_int64); +SQLITE_API const char *sqlite3_uri_key(sqlite3_filename z, int N); + +/* +** CAPI3REF: Translate filenames +** +** These routines are available to [VFS|custom VFS implementations] for +** translating filenames between the main database file, the journal file, +** and the WAL file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, then sqlite3_filename_database(F) +** returns the name of the corresponding database file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** passed by the SQLite core into the VFS, or if F is a database filename +** obtained from [sqlite3_db_filename()], then sqlite3_filename_journal(F) +** returns the name of the corresponding rollback journal file. +** +** If F is the name of an sqlite database file, journal file, or WAL file +** that was passed by the SQLite core into the VFS, or if F is a database +** filename obtained from [sqlite3_db_filename()], then +** sqlite3_filename_wal(F) returns the name of the corresponding +** WAL file. +** +** In all of the above, if F is not the name of a database, journal or WAL +** filename passed into the VFS from the SQLite core and F is not the +** return value from [sqlite3_db_filename()], then the result is +** undefined and is likely a memory access violation. +*/ +SQLITE_API const char *sqlite3_filename_database(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_journal(sqlite3_filename); +SQLITE_API const char *sqlite3_filename_wal(sqlite3_filename); + +/* +** CAPI3REF: Database File Corresponding To A Journal +** +** ^If X is the name of a rollback or WAL-mode journal file that is +** passed into the xOpen method of [sqlite3_vfs], then +** sqlite3_database_file_object(X) returns a pointer to the [sqlite3_file] +** object that represents the main database file. +** +** This routine is intended for use in custom [VFS] implementations +** only. It is not a general-purpose interface. +** The argument sqlite3_file_object(X) must be a filename pointer that +** has been passed into [sqlite3_vfs].xOpen method where the +** flags parameter to xOpen contains one of the bits +** [SQLITE_OPEN_MAIN_JOURNAL] or [SQLITE_OPEN_WAL]. Any other use +** of this routine results in undefined and probably undesirable +** behavior. +*/ +SQLITE_API sqlite3_file *sqlite3_database_file_object(const char*); + +/* +** CAPI3REF: Create and Destroy VFS Filenames +** +** These interfaces are provided for use by [VFS shim] implementations and +** are not useful outside of that context. +** +** The sqlite3_create_filename(D,J,W,N,P) allocates memory to hold a version of +** database filename D with corresponding journal file J and WAL file W and +** with N URI parameters key/values pairs in the array P. The result from +** sqlite3_create_filename(D,J,W,N,P) is a pointer to a database filename that +** is safe to pass to routines like: +**
              +**
            • [sqlite3_uri_parameter()], +**
            • [sqlite3_uri_boolean()], +**
            • [sqlite3_uri_int64()], +**
            • [sqlite3_uri_key()], +**
            • [sqlite3_filename_database()], +**
            • [sqlite3_filename_journal()], or +**
            • [sqlite3_filename_wal()]. +**
            +** If a memory allocation error occurs, sqlite3_create_filename() might +** return a NULL pointer. The memory obtained from sqlite3_create_filename(X) +** must be released by a corresponding call to sqlite3_free_filename(Y). +** +** The P parameter in sqlite3_create_filename(D,J,W,N,P) should be an array +** of 2*N pointers to strings. Each pair of pointers in this array corresponds +** to a key and value for a query parameter. The P parameter may be a NULL +** pointer if N is zero. None of the 2*N pointers in the P array may be +** NULL pointers and key pointers should not be empty strings. +** None of the D, J, or W parameters to sqlite3_create_filename(D,J,W,N,P) may +** be NULL pointers, though they can be empty strings. +** +** The sqlite3_free_filename(Y) routine releases a memory allocation +** previously obtained from sqlite3_create_filename(). Invoking +** sqlite3_free_filename(Y) where Y is a NULL pointer is a harmless no-op. +** +** If the Y parameter to sqlite3_free_filename(Y) is anything other +** than a NULL pointer or a pointer previously acquired from +** sqlite3_create_filename(), then bad things such as heap +** corruption or segfaults may occur. The value Y should not be +** used again after sqlite3_free_filename(Y) has been called. This means +** that if the [sqlite3_vfs.xOpen()] method of a VFS has been called using Y, +** then the corresponding [sqlite3_module.xClose() method should also be +** invoked prior to calling sqlite3_free_filename(Y). +*/ +SQLITE_API sqlite3_filename sqlite3_create_filename( + const char *zDatabase, + const char *zJournal, + const char *zWal, + int nParam, + const char **azParam +); +SQLITE_API void sqlite3_free_filename(sqlite3_filename); /* ** CAPI3REF: Error Codes And Messages +** METHOD: sqlite3 ** -** The sqlite3_errcode() interface returns the numeric -** [SQLITE_OK | result code] or [SQLITE_IOERR_READ | extended result code] -** for the most recent failed sqlite3_* API call associated -** with [sqlite3] handle 'db'. If a prior API call failed but the -** most recent API call succeeded, the return value from sqlite3_errcode() -** is undefined. +** ^If the most recent sqlite3_* API call associated with +** [database connection] D failed, then the sqlite3_errcode(D) interface +** returns the numeric [result code] or [extended result code] for that +** API call. +** ^The sqlite3_extended_errcode() +** interface is the same except that it always returns the +** [extended result code] even when extended result codes are +** disabled. ** -** The sqlite3_errmsg() and sqlite3_errmsg16() return English-langauge -** text that describes the error, as either UTF8 or UTF16 respectively. -** Memory to hold the error message string is managed internally. The -** string may be overwritten or deallocated by subsequent calls to SQLite -** interface functions. +** The values returned by sqlite3_errcode() and/or +** sqlite3_extended_errcode() might change with each API call. +** Except, there are some interfaces that are guaranteed to never +** change the value of the error code. The error-code preserving +** interfaces include the following: ** -** Calls to many sqlite3_* functions set the error code and string returned -** by [sqlite3_errcode()], [sqlite3_errmsg()], and [sqlite3_errmsg16()] -** (overwriting the previous values). Note that calls to [sqlite3_errcode()], -** [sqlite3_errmsg()], and [sqlite3_errmsg16()] themselves do not affect the -** results of future invocations. Calls to API routines that do not return -** an error code (examples: [sqlite3_data_count()] or [sqlite3_mprintf()]) do -** not change the error code returned by this routine. +**
              +**
            • sqlite3_errcode() +**
            • sqlite3_extended_errcode() +**
            • sqlite3_errmsg() +**
            • sqlite3_errmsg16() +**
            • sqlite3_error_offset() +**
            ** -** Assuming no other intervening sqlite3_* API calls are made, the error -** code returned by this function is associated with the same error as -** the strings returned by [sqlite3_errmsg()] and [sqlite3_errmsg16()]. +** ^The sqlite3_errmsg() and sqlite3_errmsg16() return English-language +** text that describes the error, as either UTF-8 or UTF-16 respectively, +** or NULL if no error message is available. +** (See how SQLite handles [invalid UTF] for exceptions to this rule.) +** ^(Memory to hold the error message string is managed internally. +** The application does not need to worry about freeing the result. +** However, the error string might be overwritten or deallocated by +** subsequent calls to other SQLite interface functions.)^ +** +** ^The sqlite3_errstr(E) interface returns the English-language text +** that describes the [result code] E, as UTF-8, or NULL if E is not an +** result code for which a text error message is available. +** ^(Memory to hold the error message string is managed internally +** and must not be freed by the application)^. +** +** ^If the most recent error references a specific token in the input +** SQL, the sqlite3_error_offset() interface returns the byte offset +** of the start of that token. ^The byte offset returned by +** sqlite3_error_offset() assumes that the input SQL is UTF8. +** ^If the most recent error does not reference a specific token in the input +** SQL, then the sqlite3_error_offset() function returns -1. +** +** When the serialized [threading mode] is in use, it might be the +** case that a second error occurs on a separate thread in between +** the time of the first error and the call to these interfaces. +** When that happens, the second error will be reported since these +** interfaces always report the most recent result. To avoid +** this, each thread can obtain exclusive use of the [database connection] D +** by invoking [sqlite3_mutex_enter]([sqlite3_db_mutex](D)) before beginning +** to use D and invoking [sqlite3_mutex_leave]([sqlite3_db_mutex](D)) after +** all calls to the interfaces listed here are completed. +** +** If an interface fails with SQLITE_MISUSE, that means the interface +** was invoked incorrectly by the application. In that case, the +** error code and message may or may not be set. */ -int sqlite3_errcode(sqlite3 *db); -const char *sqlite3_errmsg(sqlite3*); -const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API int sqlite3_errcode(sqlite3 *db); +SQLITE_API int sqlite3_extended_errcode(sqlite3 *db); +SQLITE_API const char *sqlite3_errmsg(sqlite3*); +SQLITE_API const void *sqlite3_errmsg16(sqlite3*); +SQLITE_API const char *sqlite3_errstr(int); +SQLITE_API int sqlite3_error_offset(sqlite3 *db); /* -** CAPI3REF: SQL Statement Object +** CAPI3REF: Prepared Statement Object +** KEYWORDS: {prepared statement} {prepared statements} ** -** Instance of this object represent single SQL statements. This -** is variously known as a "prepared statement" or a -** "compiled SQL statement" or simply as a "statement". -** -** The life of a statement object goes something like this: +** An instance of this object represents a single SQL statement that +** has been compiled into binary form and is ready to be evaluated. +** +** Think of each SQL statement as a separate computer program. The +** original SQL text is source code. A prepared statement object +** is the compiled object code. All SQL must be converted into a +** prepared statement before it can be run. +** +** The life-cycle of a prepared statement object usually goes like this: ** **
              -**
            1. Create the object using [sqlite3_prepare_v2()] or a related -** function. -**
            2. Bind values to host parameters using -** [sqlite3_bind_blob | sqlite3_bind_* interfaces]. +**
            3. Create the prepared statement object using [sqlite3_prepare_v2()]. +**
            4. Bind values to [parameters] using the sqlite3_bind_*() +** interfaces. **
            5. Run the SQL by calling [sqlite3_step()] one or more times. -**
            6. Reset the statement using [sqlite3_reset()] then go back +**
            7. Reset the prepared statement using [sqlite3_reset()] then go back ** to step 2. Do this zero or more times. **
            8. Destroy the object using [sqlite3_finalize()]. **
            -** -** Refer to documentation on individual methods above for additional -** information. */ typedef struct sqlite3_stmt sqlite3_stmt; +/* +** CAPI3REF: Run-time Limits +** METHOD: sqlite3 +** +** ^(This interface allows the size of various constructs to be limited +** on a connection by connection basis. The first parameter is the +** [database connection] whose limit is to be set or queried. The +** second parameter is one of the [limit categories] that define a +** class of constructs to be size limited. The third parameter is the +** new limit for that construct.)^ +** +** ^If the new limit is a negative number, the limit is unchanged. +** ^(For each limit category SQLITE_LIMIT_NAME there is a +** [limits | hard upper bound] +** set at compile-time by a C preprocessor macro called +** [limits | SQLITE_MAX_NAME]. +** (The "_LIMIT_" in the name is changed to "_MAX_".))^ +** ^Attempts to increase a limit above its hard upper bound are +** silently truncated to the hard upper bound. +** +** ^Regardless of whether or not the limit was changed, the +** [sqlite3_limit()] interface returns the prior value of the limit. +** ^Hence, to find the current value of a limit without changing it, +** simply invoke this interface with the third parameter set to -1. +** +** Run-time limits are intended for use in applications that manage +** both their own internal database and also databases that are controlled +** by untrusted external sources. An example application might be a +** web browser that has its own databases for storing history and +** separate databases controlled by JavaScript applications downloaded +** off the Internet. The internal databases can be given the +** large, default limits. Databases managed by external sources can +** be given much smaller limits designed to prevent a denial of service +** attack. Developers might also want to use the [sqlite3_set_authorizer()] +** interface to further control untrusted SQL. The size of the database +** created by an untrusted script can be contained using the +** [max_page_count] [PRAGMA]. +** +** New run-time limit categories may be added in future releases. +*/ +SQLITE_API int sqlite3_limit(sqlite3*, int id, int newVal); + +/* +** CAPI3REF: Run-Time Limit Categories +** KEYWORDS: {limit category} {*limit categories} +** +** These constants define various performance limits +** that can be lowered at run-time using [sqlite3_limit()]. +** The synopsis of the meanings of the various limits is shown below. +** Additional information is available at [limits | Limits in SQLite]. +** +**
            +** [[SQLITE_LIMIT_LENGTH]] ^(
            SQLITE_LIMIT_LENGTH
            +**
            The maximum size of any string or BLOB or table row, in bytes.
            )^ +** +** [[SQLITE_LIMIT_SQL_LENGTH]] ^(
            SQLITE_LIMIT_SQL_LENGTH
            +**
            The maximum length of an SQL statement, in bytes.
            )^ +** +** [[SQLITE_LIMIT_COLUMN]] ^(
            SQLITE_LIMIT_COLUMN
            +**
            The maximum number of columns in a table definition or in the +** result set of a [SELECT] or the maximum number of columns in an index +** or in an ORDER BY or GROUP BY clause.
            )^ +** +** [[SQLITE_LIMIT_EXPR_DEPTH]] ^(
            SQLITE_LIMIT_EXPR_DEPTH
            +**
            The maximum depth of the parse tree on any expression.
            )^ +** +** [[SQLITE_LIMIT_COMPOUND_SELECT]] ^(
            SQLITE_LIMIT_COMPOUND_SELECT
            +**
            The maximum number of terms in a compound SELECT statement.
            )^ +** +** [[SQLITE_LIMIT_VDBE_OP]] ^(
            SQLITE_LIMIT_VDBE_OP
            +**
            The maximum number of instructions in a virtual machine program +** used to implement an SQL statement. If [sqlite3_prepare_v2()] or +** the equivalent tries to allocate space for more than this many opcodes +** in a single prepared statement, an SQLITE_NOMEM error is returned.
            )^ +** +** [[SQLITE_LIMIT_FUNCTION_ARG]] ^(
            SQLITE_LIMIT_FUNCTION_ARG
            +**
            The maximum number of arguments on a function.
            )^ +** +** [[SQLITE_LIMIT_ATTACHED]] ^(
            SQLITE_LIMIT_ATTACHED
            +**
            The maximum number of [ATTACH | attached databases].)^
            +** +** [[SQLITE_LIMIT_LIKE_PATTERN_LENGTH]] +** ^(
            SQLITE_LIMIT_LIKE_PATTERN_LENGTH
            +**
            The maximum length of the pattern argument to the [LIKE] or +** [GLOB] operators.
            )^ +** +** [[SQLITE_LIMIT_VARIABLE_NUMBER]] +** ^(
            SQLITE_LIMIT_VARIABLE_NUMBER
            +**
            The maximum index number of any [parameter] in an SQL statement.)^ +** +** [[SQLITE_LIMIT_TRIGGER_DEPTH]] ^(
            SQLITE_LIMIT_TRIGGER_DEPTH
            +**
            The maximum depth of recursion for triggers.
            )^ +** +** [[SQLITE_LIMIT_WORKER_THREADS]] ^(
            SQLITE_LIMIT_WORKER_THREADS
            +**
            The maximum number of auxiliary worker threads that a single +** [prepared statement] may start.
            )^ +**
            +*/ +#define SQLITE_LIMIT_LENGTH 0 +#define SQLITE_LIMIT_SQL_LENGTH 1 +#define SQLITE_LIMIT_COLUMN 2 +#define SQLITE_LIMIT_EXPR_DEPTH 3 +#define SQLITE_LIMIT_COMPOUND_SELECT 4 +#define SQLITE_LIMIT_VDBE_OP 5 +#define SQLITE_LIMIT_FUNCTION_ARG 6 +#define SQLITE_LIMIT_ATTACHED 7 +#define SQLITE_LIMIT_LIKE_PATTERN_LENGTH 8 +#define SQLITE_LIMIT_VARIABLE_NUMBER 9 +#define SQLITE_LIMIT_TRIGGER_DEPTH 10 +#define SQLITE_LIMIT_WORKER_THREADS 11 + +/* +** CAPI3REF: Prepare Flags +** +** These constants define various flags that can be passed into +** "prepFlags" parameter of the [sqlite3_prepare_v3()] and +** [sqlite3_prepare16_v3()] interfaces. +** +** New flags may be added in future releases of SQLite. +** +**
            +** [[SQLITE_PREPARE_PERSISTENT]] ^(
            SQLITE_PREPARE_PERSISTENT
            +**
            The SQLITE_PREPARE_PERSISTENT flag is a hint to the query planner +** that the prepared statement will be retained for a long time and +** probably reused many times.)^ ^Without this flag, [sqlite3_prepare_v3()] +** and [sqlite3_prepare16_v3()] assume that the prepared statement will +** be used just once or at most a few times and then destroyed using +** [sqlite3_finalize()] relatively soon. The current implementation acts +** on this hint by avoiding the use of [lookaside memory] so as not to +** deplete the limited store of lookaside memory. Future versions of +** SQLite may act on this hint differently. +** +** [[SQLITE_PREPARE_NORMALIZE]]
            SQLITE_PREPARE_NORMALIZE
            +**
            The SQLITE_PREPARE_NORMALIZE flag is a no-op. This flag used +** to be required for any prepared statement that wanted to use the +** [sqlite3_normalized_sql()] interface. However, the +** [sqlite3_normalized_sql()] interface is now available to all +** prepared statements, regardless of whether or not they use this +** flag. +** +** [[SQLITE_PREPARE_NO_VTAB]]
            SQLITE_PREPARE_NO_VTAB
            +**
            The SQLITE_PREPARE_NO_VTAB flag causes the SQL compiler +** to return an error (error code SQLITE_ERROR) if the statement uses +** any virtual tables. +**
            +*/ +#define SQLITE_PREPARE_PERSISTENT 0x01 +#define SQLITE_PREPARE_NORMALIZE 0x02 +#define SQLITE_PREPARE_NO_VTAB 0x04 + /* ** CAPI3REF: Compiling An SQL Statement +** KEYWORDS: {SQL statement compiler} +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_stmt ** -** To execute an SQL query, it must first be compiled into a byte-code -** program using one of these routines. +** To execute an SQL statement, it must first be compiled into a byte-code +** program using one of these routines. Or, in other words, these routines +** are constructors for the [prepared statement] object. ** -** The first argument "db" is an [sqlite3 | SQLite database handle] -** obtained from a prior call to [sqlite3_open()] or [sqlite3_open16()]. -** The second argument "zSql" is the statement to be compiled, encoded -** as either UTF-8 or UTF-16. The sqlite3_prepare() and sqlite3_prepare_v2() -** interfaces uses UTF-8 and sqlite3_prepare16() and sqlite3_prepare16_v2() -** use UTF-16. If the next argument, "nBytes", is less -** than zero, then zSql is read up to the first zero terminator. If -** "nBytes" is not less than zero, then it is the length of the string zSql -** in bytes (not characters). +** The preferred routine to use is [sqlite3_prepare_v2()]. The +** [sqlite3_prepare()] interface is legacy and should be avoided. +** [sqlite3_prepare_v3()] has an extra "prepFlags" option that is used +** for special purposes. ** -** *pzTail is made to point to the first byte past the end of the first -** SQL statement in zSql. This routine only compiles the first statement -** in zSql, so *pzTail is left pointing to what remains uncompiled. +** The use of the UTF-8 interfaces is preferred, as SQLite currently +** does all parsing using UTF-8. The UTF-16 interfaces are provided +** as a convenience. The UTF-16 interfaces work by converting the +** input text into UTF-8, then invoking the corresponding UTF-8 interface. ** -** *ppStmt is left pointing to a compiled -** [sqlite3_stmt | SQL statement structure] that can be -** executed using [sqlite3_step()]. Or if there is an error, *ppStmt may be -** set to NULL. If the input text contained no SQL (if the input is and -** empty string or a comment) then *ppStmt is set to NULL. The calling -** procedure is responsible for deleting the compiled SQL statement -** using [sqlite3_finalize()] after it has finished with it. +** The first argument, "db", is a [database connection] obtained from a +** prior successful call to [sqlite3_open()], [sqlite3_open_v2()] or +** [sqlite3_open16()]. The database connection must not have been closed. ** -** On success, [SQLITE_OK] is returned. Otherwise an -** [SQLITE_ERROR | error code] is returned. +** The second argument, "zSql", is the statement to be compiled, encoded +** as either UTF-8 or UTF-16. The sqlite3_prepare(), sqlite3_prepare_v2(), +** and sqlite3_prepare_v3() +** interfaces use UTF-8, and sqlite3_prepare16(), sqlite3_prepare16_v2(), +** and sqlite3_prepare16_v3() use UTF-16. ** -** The sqlite3_prepare_v2() and sqlite3_prepare16_v2() interfaces are -** recommended for all new programs. The two older interfaces are retained -** for backwards compatibility, but their use is discouraged. -** In the "v2" interfaces, the prepared statement -** that is returned (the [sqlite3_stmt] object) contains a copy of the +** ^If the nByte argument is negative, then zSql is read up to the +** first zero terminator. ^If nByte is positive, then it is the maximum +** number of bytes read from zSql. When nByte is positive, zSql is read +** up to the first zero terminator or until the nByte bytes have been read, +** whichever comes first. ^If nByte is zero, then no prepared +** statement is generated. +** If the caller knows that the supplied string is nul-terminated, then +** there is a small performance advantage to passing an nByte parameter that +** is the number of bytes in the input string including +** the nul-terminator. +** Note that nByte measure the length of the input in bytes, not +** characters, even for the UTF-16 interfaces. +** +** ^If pzTail is not NULL then *pzTail is made to point to the first byte +** past the end of the first SQL statement in zSql. These routines only +** compile the first statement in zSql, so *pzTail is left pointing to +** what remains uncompiled. +** +** ^*ppStmt is left pointing to a compiled [prepared statement] that can be +** executed using [sqlite3_step()]. ^If there is an error, *ppStmt is set +** to NULL. ^If the input text contains no SQL (if the input is an empty +** string or a comment) then *ppStmt is set to NULL. +** The calling procedure is responsible for deleting the compiled +** SQL statement using [sqlite3_finalize()] after it has finished with it. +** ppStmt may not be NULL. +** +** ^On success, the sqlite3_prepare() family of routines return [SQLITE_OK]; +** otherwise an [error code] is returned. +** +** The sqlite3_prepare_v2(), sqlite3_prepare_v3(), sqlite3_prepare16_v2(), +** and sqlite3_prepare16_v3() interfaces are recommended for all new programs. +** The older interfaces (sqlite3_prepare() and sqlite3_prepare16()) +** are retained for backwards compatibility, but their use is discouraged. +** ^In the "vX" interfaces, the prepared statement +** that is returned (the [sqlite3_stmt] object) contains a copy of the ** original SQL text. This causes the [sqlite3_step()] interface to -** behave a differently in two ways: +** behave differently in three ways: ** **
              **
            1. -** If the database schema changes, instead of returning [SQLITE_SCHEMA] as it +** ^If the database schema changes, instead of returning [SQLITE_SCHEMA] as it ** always used to do, [sqlite3_step()] will automatically recompile the SQL -** statement and try to run it again. If the schema has changed in a way -** that makes the statement no longer valid, [sqlite3_step()] will still -** return [SQLITE_SCHEMA]. But unlike the legacy behavior, [SQLITE_SCHEMA] is -** now a fatal error. Calling [sqlite3_prepare_v2()] again will not make the -** error go away. Note: use [sqlite3_errmsg()] to find the text of the parsing -** error that results in an [SQLITE_SCHEMA] return. +** statement and try to run it again. As many as [SQLITE_MAX_SCHEMA_RETRY] +** retries will occur before sqlite3_step() gives up and returns an error. **
              2. ** **
            2. -** When an error occurs, -** [sqlite3_step()] will return one of the detailed -** [SQLITE_ERROR | result codes] or -** [SQLITE_IOERR_READ | extended result codes] such as directly. -** The legacy behavior was that [sqlite3_step()] would only return a generic -** [SQLITE_ERROR] result code and you would have to make a second call to -** [sqlite3_reset()] in order to find the underlying cause of the problem. -** With the "v2" prepare interfaces, the underlying reason for the error is -** returned immediately. +** ^When an error occurs, [sqlite3_step()] will return one of the detailed +** [error codes] or [extended error codes]. ^The legacy behavior was that +** [sqlite3_step()] would only return a generic [SQLITE_ERROR] result code +** and the application would have to make a second call to [sqlite3_reset()] +** in order to find the underlying cause of the problem. With the "v2" prepare +** interfaces, the underlying reason for the error is returned immediately. +**
              3. +** +**
            3. +** ^If the specific value bound to a [parameter | host parameter] in the +** WHERE clause might influence the choice of query plan for a statement, +** then the statement will be automatically recompiled, as if there had been +** a schema change, on the first [sqlite3_step()] call following any change +** to the [sqlite3_bind_text | bindings] of that [parameter]. +** ^The specific value of a WHERE-clause [parameter] might influence the +** choice of query plan if the parameter is the left-hand side of a [LIKE] +** or [GLOB] operator or if the parameter is compared to an indexed column +** and the [SQLITE_ENABLE_STAT4] compile-time option is enabled. **
              4. **
            -*/ -int sqlite3_prepare( - sqlite3 *db, /* Database handle */ - const char *zSql, /* SQL statement, UTF-8 encoded */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: Statement handle */ - const char **pzTail /* OUT: Pointer to unused portion of zSql */ -); -int sqlite3_prepare_v2( - sqlite3 *db, /* Database handle */ - const char *zSql, /* SQL statement, UTF-8 encoded */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: Statement handle */ - const char **pzTail /* OUT: Pointer to unused portion of zSql */ -); -int sqlite3_prepare16( - sqlite3 *db, /* Database handle */ - const void *zSql, /* SQL statement, UTF-16 encoded */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: Statement handle */ - const void **pzTail /* OUT: Pointer to unused portion of zSql */ -); -int sqlite3_prepare16_v2( - sqlite3 *db, /* Database handle */ - const void *zSql, /* SQL statement, UTF-16 encoded */ - int nBytes, /* Length of zSql in bytes. */ - sqlite3_stmt **ppStmt, /* OUT: Statement handle */ - const void **pzTail /* OUT: Pointer to unused portion of zSql */ -); - -/* -** CAPI3REF: Dynamically Typed Value Object ** -** SQLite uses dynamic typing for the values it stores. Values can -** be integers, floating point values, strings, BLOBs, or NULL. When -** passing around values internally, each value is represented as -** an instance of the sqlite3_value object. +**

            ^sqlite3_prepare_v3() differs from sqlite3_prepare_v2() only in having +** the extra prepFlags parameter, which is a bit array consisting of zero or +** more of the [SQLITE_PREPARE_PERSISTENT|SQLITE_PREPARE_*] flags. ^The +** sqlite3_prepare_v2() interface works exactly the same as +** sqlite3_prepare_v3() with a zero prepFlags parameter. */ -typedef struct Mem sqlite3_value; +SQLITE_API int sqlite3_prepare( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v2( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare_v3( + sqlite3 *db, /* Database handle */ + const char *zSql, /* SQL statement, UTF-8 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const char **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v2( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); +SQLITE_API int sqlite3_prepare16_v3( + sqlite3 *db, /* Database handle */ + const void *zSql, /* SQL statement, UTF-16 encoded */ + int nByte, /* Maximum length of zSql in bytes. */ + unsigned int prepFlags, /* Zero or more SQLITE_PREPARE_ flags */ + sqlite3_stmt **ppStmt, /* OUT: Statement handle */ + const void **pzTail /* OUT: Pointer to unused portion of zSql */ +); /* -** CAPI3REF: SQL Function Context Object +** CAPI3REF: Retrieving Statement SQL +** METHOD: sqlite3_stmt +** +** ^The sqlite3_sql(P) interface returns a pointer to a copy of the UTF-8 +** SQL text used to create [prepared statement] P if P was +** created by [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], +** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()]. +** ^The sqlite3_expanded_sql(P) interface returns a pointer to a UTF-8 +** string containing the SQL text of prepared statement P with +** [bound parameters] expanded. +** ^The sqlite3_normalized_sql(P) interface returns a pointer to a UTF-8 +** string containing the normalized SQL text of prepared statement P. The +** semantics used to normalize a SQL statement are unspecified and subject +** to change. At a minimum, literal values will be replaced with suitable +** placeholders. +** +** ^(For example, if a prepared statement is created using the SQL +** text "SELECT $abc,:xyz" and if parameter $abc is bound to integer 2345 +** and parameter :xyz is unbound, then sqlite3_sql() will return +** the original string, "SELECT $abc,:xyz" but sqlite3_expanded_sql() +** will return "SELECT 2345,NULL".)^ +** +** ^The sqlite3_expanded_sql() interface returns NULL if insufficient memory +** is available to hold the result, or if the result would exceed the +** the maximum string length determined by the [SQLITE_LIMIT_LENGTH]. +** +** ^The [SQLITE_TRACE_SIZE_LIMIT] compile-time option limits the size of +** bound parameter expansions. ^The [SQLITE_OMIT_TRACE] compile-time +** option causes sqlite3_expanded_sql() to always return NULL. +** +** ^The strings returned by sqlite3_sql(P) and sqlite3_normalized_sql(P) +** are managed by SQLite and are automatically freed when the prepared +** statement is finalized. +** ^The string returned by sqlite3_expanded_sql(P), on the other hand, +** is obtained from [sqlite3_malloc()] and must be freed by the application +** by passing it to [sqlite3_free()]. +** +** ^The sqlite3_normalized_sql() interface is only available if +** the [SQLITE_ENABLE_NORMALIZE] compile-time option is defined. +*/ +SQLITE_API const char *sqlite3_sql(sqlite3_stmt *pStmt); +SQLITE_API char *sqlite3_expanded_sql(sqlite3_stmt *pStmt); +#ifdef SQLITE_ENABLE_NORMALIZE +SQLITE_API const char *sqlite3_normalized_sql(sqlite3_stmt *pStmt); +#endif + +/* +** CAPI3REF: Determine If An SQL Statement Writes The Database +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_readonly(X) interface returns true (non-zero) if +** and only if the [prepared statement] X makes no direct changes to +** the content of the database file. +** +** Note that [application-defined SQL functions] or +** [virtual tables] might change the database indirectly as a side effect. +** ^(For example, if an application defines a function "eval()" that +** calls [sqlite3_exec()], then the following SQL statement would +** change the database file through side-effects: +** +** 

            +**    SELECT eval('DELETE FROM t1') FROM t2;
+**
            +** +** But because the [SELECT] statement does not change the database file +** directly, sqlite3_stmt_readonly() would still return true.)^ +** +** ^Transaction control statements such as [BEGIN], [COMMIT], [ROLLBACK], +** [SAVEPOINT], and [RELEASE] cause sqlite3_stmt_readonly() to return true, +** since the statements themselves do not actually modify the database but +** rather they control the timing of when other statements modify the +** database. ^The [ATTACH] and [DETACH] statements also cause +** sqlite3_stmt_readonly() to return true since, while those statements +** change the configuration of a database connection, they do not make +** changes to the content of the database files on disk. +** ^The sqlite3_stmt_readonly() interface returns true for [BEGIN] since +** [BEGIN] merely sets internal flags, but the [BEGIN|BEGIN IMMEDIATE] and +** [BEGIN|BEGIN EXCLUSIVE] commands do touch the database and so +** sqlite3_stmt_readonly() returns false for those commands. +** +** ^This routine returns false if there is any possibility that the +** statement might change the database file. ^A false return does +** not guarantee that the statement will change the database file. +** ^For example, an UPDATE statement might have a WHERE clause that +** makes it a no-op, but the sqlite3_stmt_readonly() result would still +** be false. ^Similarly, a CREATE TABLE IF NOT EXISTS statement is a +** read-only no-op if the table already exists, but +** sqlite3_stmt_readonly() still returns false for such a statement. +** +** ^If prepared statement X is an [EXPLAIN] or [EXPLAIN QUERY PLAN] +** statement, then sqlite3_stmt_readonly(X) returns the same value as +** if the EXPLAIN or EXPLAIN QUERY PLAN prefix were omitted. +*/ +SQLITE_API int sqlite3_stmt_readonly(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Query The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_isexplain(S) interface returns 1 if the +** prepared statement S is an EXPLAIN statement, or 2 if the +** statement S is an EXPLAIN QUERY PLAN. +** ^The sqlite3_stmt_isexplain(S) interface returns 0 if S is +** an ordinary statement or a NULL pointer. +*/ +SQLITE_API int sqlite3_stmt_isexplain(sqlite3_stmt *pStmt); + +/* +** CAPI3REF: Change The EXPLAIN Setting For A Prepared Statement +** METHOD: sqlite3_stmt +** +** The sqlite3_stmt_explain(S,E) interface changes the EXPLAIN +** setting for [prepared statement] S. If E is zero, then S becomes +** a normal prepared statement. If E is 1, then S behaves as if +** its SQL text began with "[EXPLAIN]". If E is 2, then S behaves as if +** its SQL text began with "[EXPLAIN QUERY PLAN]". +** +** Calling sqlite3_stmt_explain(S,E) might cause S to be reprepared. +** SQLite tries to avoid a reprepare, but a reprepare might be necessary +** on the first transition into EXPLAIN or EXPLAIN QUERY PLAN mode. +** +** Because of the potential need to reprepare, a call to +** sqlite3_stmt_explain(S,E) will fail with SQLITE_ERROR if S cannot be +** reprepared because it was created using [sqlite3_prepare()] instead of +** the newer [sqlite3_prepare_v2()] or [sqlite3_prepare_v3()] interfaces and +** hence has no saved SQL text with which to reprepare. +** +** Changing the explain setting for a prepared statement does not change +** the original SQL text for the statement. Hence, if the SQL text originally +** began with EXPLAIN or EXPLAIN QUERY PLAN, but sqlite3_stmt_explain(S,0) +** is called to convert the statement into an ordinary statement, the EXPLAIN +** or EXPLAIN QUERY PLAN keywords will still appear in the sqlite3_sql(S) +** output, even though the statement now acts like a normal SQL statement. +** +** This routine returns SQLITE_OK if the explain mode is successfully +** changed, or an error code if the explain mode could not be changed. +** The explain mode cannot be changed while a statement is active. +** Hence, it is good practice to call [sqlite3_reset(S)] +** immediately prior to calling sqlite3_stmt_explain(S,E). +*/ +SQLITE_API int sqlite3_stmt_explain(sqlite3_stmt *pStmt, int eMode); + +/* +** CAPI3REF: Determine If A Prepared Statement Has Been Reset +** METHOD: sqlite3_stmt +** +** ^The sqlite3_stmt_busy(S) interface returns true (non-zero) if the +** [prepared statement] S has been stepped at least once using +** [sqlite3_step(S)] but has neither run to completion (returned +** [SQLITE_DONE] from [sqlite3_step(S)]) nor +** been reset using [sqlite3_reset(S)]. ^The sqlite3_stmt_busy(S) +** interface returns false if S is a NULL pointer. If S is not a +** NULL pointer and is not a pointer to a valid [prepared statement] +** object, then the behavior is undefined and probably undesirable. +** +** This interface can be used in combination [sqlite3_next_stmt()] +** to locate all prepared statements associated with a database +** connection that are in need of being reset. This can be used, +** for example, in diagnostic routines to search for prepared +** statements that are holding a transaction open. +*/ +SQLITE_API int sqlite3_stmt_busy(sqlite3_stmt*); + +/* +** CAPI3REF: Dynamically Typed Value Object +** KEYWORDS: {protected sqlite3_value} {unprotected sqlite3_value} +** +** SQLite uses the sqlite3_value object to represent all values +** that can be stored in a database table. SQLite uses dynamic typing +** for the values it stores. ^Values stored in sqlite3_value objects +** can be integers, floating point values, strings, BLOBs, or NULL. +** +** An sqlite3_value object may be either "protected" or "unprotected". +** Some interfaces require a protected sqlite3_value. Other interfaces +** will accept either a protected or an unprotected sqlite3_value. +** Every interface that accepts sqlite3_value arguments specifies +** whether or not it requires a protected sqlite3_value. The +** [sqlite3_value_dup()] interface can be used to construct a new +** protected sqlite3_value from an unprotected sqlite3_value. +** +** The terms "protected" and "unprotected" refer to whether or not +** a mutex is held. An internal mutex is held for a protected +** sqlite3_value object but no mutex is held for an unprotected +** sqlite3_value object. If SQLite is compiled to be single-threaded +** (with [SQLITE_THREADSAFE=0] and with [sqlite3_threadsafe()] returning 0) +** or if SQLite is run in one of reduced mutex modes +** [SQLITE_CONFIG_SINGLETHREAD] or [SQLITE_CONFIG_MULTITHREAD] +** then there is no distinction between protected and unprotected +** sqlite3_value objects and they can be used interchangeably. However, +** for maximum code portability it is recommended that applications +** still make the distinction between protected and unprotected +** sqlite3_value objects even when not strictly required. +** +** ^The sqlite3_value objects that are passed as parameters into the +** implementation of [application-defined SQL functions] are protected. +** ^The sqlite3_value objects returned by [sqlite3_vtab_rhs_value()] +** are protected. +** ^The sqlite3_value object returned by +** [sqlite3_column_value()] is unprotected. +** Unprotected sqlite3_value objects may only be used as arguments +** to [sqlite3_result_value()], [sqlite3_bind_value()], and +** [sqlite3_value_dup()]. +** The [sqlite3_value_blob | sqlite3_value_type()] family of +** interfaces require protected sqlite3_value objects. +*/ +typedef struct sqlite3_value sqlite3_value; + +/* +** CAPI3REF: SQL Function Context Object ** ** The context in which an SQL function executes is stored in an -** sqlite3_context object. A pointer to such an object is the -** first parameter to user-defined SQL functions. +** sqlite3_context object. ^A pointer to an sqlite3_context object +** is always first parameter to [application-defined SQL functions]. +** The application-defined SQL function implementation will pass this +** pointer through into calls to [sqlite3_result_int | sqlite3_result()], +** [sqlite3_aggregate_context()], [sqlite3_user_data()], +** [sqlite3_context_db_handle()], [sqlite3_get_auxdata()], +** and/or [sqlite3_set_auxdata()]. */ typedef struct sqlite3_context sqlite3_context; /* -** CAPI3REF: Binding Values To Prepared Statements +** CAPI3REF: Binding Values To Prepared Statements +** KEYWORDS: {host parameter} {host parameters} {host parameter name} +** KEYWORDS: {SQL parameter} {SQL parameters} {parameter binding} +** METHOD: sqlite3_stmt ** -** In the SQL strings input to [sqlite3_prepare_v2()] and its variants, -** one or more literals can be replace by a parameter in one of these -** forms: +** ^(In the SQL statement text input to [sqlite3_prepare_v2()] and its variants, +** literals may be replaced by a [parameter] that matches one of following +** templates: ** **
              **
            • ? **
            • ?NNN -**
            • :AAA -**
            • @AAA +**
            • :VVV +**
            • @VVV **
            • $VVV **
            ** -** In the parameter forms shown above NNN is an integer literal, -** AAA is an alphanumeric identifier and VVV is a variable name according -** to the syntax rules of the TCL programming language. -** The values of these parameters (also called "host parameter names") +** In the templates above, NNN represents an integer literal, +** and VVV represents an alphanumeric identifier.)^ ^The values of these +** parameters (also called "host parameter names" or "SQL parameters") ** can be set using the sqlite3_bind_*() routines defined here. ** -** The first argument to the sqlite3_bind_*() routines always is a pointer -** to the [sqlite3_stmt] object returned from [sqlite3_prepare_v2()] or -** its variants. The second -** argument is the index of the parameter to be set. The first parameter has -** an index of 1. When the same named parameter is used more than once, second -** and subsequent -** occurrences have the same index as the first occurrence. The index for -** named parameters can be looked up using the -** [sqlite3_bind_parameter_name()] API if desired. The index for "?NNN" -** parametes is the value of NNN. -** The NNN value must be between 1 and the compile-time -** parameter SQLITE_MAX_VARIABLE_NUMBER (default value: 999). -** See limits.html for additional information. +** ^The first argument to the sqlite3_bind_*() routines is always +** a pointer to the [sqlite3_stmt] object returned from +** [sqlite3_prepare_v2()] or its variants. ** -** The third argument is the value to bind to the parameter. +** ^The second argument is the index of the SQL parameter to be set. +** ^The leftmost SQL parameter has an index of 1. ^When the same named +** SQL parameter is used more than once, second and subsequent +** occurrences have the same index as the first occurrence. +** ^The index for named parameters can be looked up using the +** [sqlite3_bind_parameter_index()] API if desired. ^The index +** for "?NNN" parameters is the value of NNN. +** ^The NNN value must be between 1 and the [sqlite3_limit()] +** parameter [SQLITE_LIMIT_VARIABLE_NUMBER] (default value: 32766). ** -** In those -** routines that have a fourth argument, its value is the number of bytes -** in the parameter. To be clear: the value is the number of bytes in the -** string, not the number of characters. The number -** of bytes does not include the zero-terminator at the end of strings. -** If the fourth parameter is negative, the length of the string is -** number of bytes up to the first zero terminator. +** ^The third argument is the value to bind to the parameter. +** ^If the third parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** or sqlite3_bind_blob() is a NULL pointer then the fourth parameter +** is ignored and the end result is the same as sqlite3_bind_null(). +** ^If the third parameter to sqlite3_bind_text() is not NULL, then +** it should be a pointer to well-formed UTF8 text. +** ^If the third parameter to sqlite3_bind_text16() is not NULL, then +** it should be a pointer to well-formed UTF16 text. +** ^If the third parameter to sqlite3_bind_text64() is not NULL, then +** it should be a pointer to a well-formed unicode string that is +** either UTF8 if the sixth parameter is SQLITE_UTF8, or UTF16 +** otherwise. ** -** The fifth argument to sqlite3_bind_blob(), sqlite3_bind_text(), and -** sqlite3_bind_text16() is a destructor used to dispose of the BLOB or -** text after SQLite has finished with it. If the fifth argument is the -** special value [SQLITE_STATIC], then the library assumes that the information -** is in static, unmanaged space and does not need to be freed. If the -** fifth argument has the value [SQLITE_TRANSIENT], then SQLite makes its -** own private copy of the data immediately, before the sqlite3_bind_*() -** routine returns. +** [[byte-order determination rules]] ^The byte-order of +** UTF16 input text is determined by the byte-order mark (BOM, U+FEFF) +** found in first character, which is removed, or in the absence of a BOM +** the byte order is the native byte order of the host +** machine for sqlite3_bind_text16() or the byte order specified in +** the 6th parameter for sqlite3_bind_text64().)^ +** ^If UTF16 input text contains invalid unicode +** characters, then SQLite might change those invalid characters +** into the unicode replacement character: U+FFFD. ** -** The sqlite3_bind_zeroblob() routine binds a BLOB of length n that -** is filled with zeros. A zeroblob uses a fixed amount of memory -** (just an integer to hold it size) while it is being processed. -** Zeroblobs are intended to serve as place-holders for BLOBs whose -** content is later written using -** [sqlite3_blob_open | increment BLOB I/O] routines. +** ^(In those routines that have a fourth argument, its value is the +** number of bytes in the parameter. To be clear: the value is the +** number of bytes in the value, not the number of characters.)^ +** ^If the fourth parameter to sqlite3_bind_text() or sqlite3_bind_text16() +** is negative, then the length of the string is +** the number of bytes up to the first zero terminator. +** If the fourth parameter to sqlite3_bind_blob() is negative, then +** the behavior is undefined. +** If a non-negative fourth parameter is provided to sqlite3_bind_text() +** or sqlite3_bind_text16() or sqlite3_bind_text64() then +** that parameter must be the byte offset +** where the NUL terminator would occur assuming the string were NUL +** terminated. If any NUL characters occurs at byte offsets less than +** the value of the fourth parameter then the resulting string value will +** contain embedded NULs. The result of expressions involving strings +** with embedded NULs is undefined. ** -** The sqlite3_bind_*() routines must be called after -** [sqlite3_prepare_v2()] (and its variants) or [sqlite3_reset()] and -** before [sqlite3_step()]. -** Bindings are not cleared by the [sqlite3_reset()] routine. -** Unbound parameters are interpreted as NULL. +** ^The fifth argument to the BLOB and string binding interfaces controls +** or indicates the lifetime of the object referenced by the third parameter. +** These three options exist: +** ^ (1) A destructor to dispose of the BLOB or string after SQLite has finished +** with it may be passed. ^It is called to dispose of the BLOB or string even +** if the call to the bind API fails, except the destructor is not called if +** the third parameter is a NULL pointer or the fourth parameter is negative. +** ^ (2) The special constant, [SQLITE_STATIC], may be passed to indicate that +** the application remains responsible for disposing of the object. ^In this +** case, the object and the provided pointer to it must remain valid until +** either the prepared statement is finalized or the same SQL parameter is +** bound to something else, whichever occurs sooner. +** ^ (3) The constant, [SQLITE_TRANSIENT], may be passed to indicate that the +** object is to be copied prior to the return from sqlite3_bind_*(). ^The +** object and pointer to it must remain valid until then. ^SQLite will then +** manage the lifetime of its private copy. ** -** These routines return [SQLITE_OK] on success or an error code if -** anything goes wrong. [SQLITE_RANGE] is returned if the parameter -** index is out of range. [SQLITE_NOMEM] is returned if malloc fails. -** [SQLITE_MISUSE] is returned if these routines are called on a virtual -** machine that is the wrong state or which has already been finalized. +** ^The sixth argument to sqlite3_bind_text64() must be one of +** [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE] +** to specify the encoding of the text in the third parameter. If +** the sixth argument to sqlite3_bind_text64() is not one of the +** allowed values shown above, or if the text encoding is different +** from the encoding specified by the sixth parameter, then the behavior +** is undefined. +** +** ^The sqlite3_bind_zeroblob() routine binds a BLOB of length N that +** is filled with zeroes. ^A zeroblob uses a fixed amount of memory +** (just an integer to hold its size) while it is being processed. +** Zeroblobs are intended to serve as placeholders for BLOBs whose +** content is later written using +** [sqlite3_blob_open | incremental BLOB I/O] routines. +** ^A negative value for the zeroblob results in a zero-length BLOB. +** +** ^The sqlite3_bind_pointer(S,I,P,T,D) routine causes the I-th parameter in +** [prepared statement] S to have an SQL value of NULL, but to also be +** associated with the pointer P of type T. ^D is either a NULL pointer or +** a pointer to a destructor function for P. ^SQLite will invoke the +** destructor D with a single argument of P when it is finished using +** P. The T parameter should be a static string, preferably a string +** literal. The sqlite3_bind_pointer() routine is part of the +** [pointer passing interface] added for SQLite 3.20.0. +** +** ^If any of the sqlite3_bind_*() routines are called with a NULL pointer +** for the [prepared statement] or with a prepared statement for which +** [sqlite3_step()] has been called more recently than [sqlite3_reset()], +** then the call will return [SQLITE_MISUSE]. If any sqlite3_bind_() +** routine is passed a [prepared statement] that has been finalized, the +** result is undefined and probably harmful. +** +** ^Bindings are not cleared by the [sqlite3_reset()] routine. +** ^Unbound parameters are interpreted as NULL. +** +** ^The sqlite3_bind_* routines return [SQLITE_OK] on success or an +** [error code] if anything goes wrong. +** ^[SQLITE_TOOBIG] might be returned if the size of a string or BLOB +** exceeds limits imposed by [sqlite3_limit]([SQLITE_LIMIT_LENGTH]) or +** [SQLITE_MAX_LENGTH]. +** ^[SQLITE_RANGE] is returned if the parameter +** index is out of range. ^[SQLITE_NOMEM] is returned if malloc() fails. +** +** See also: [sqlite3_bind_parameter_count()], +** [sqlite3_bind_parameter_name()], and [sqlite3_bind_parameter_index()]. */ -int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); -int sqlite3_bind_double(sqlite3_stmt*, int, double); -int sqlite3_bind_int(sqlite3_stmt*, int, int); -int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite_int64); -int sqlite3_bind_null(sqlite3_stmt*, int); -int sqlite3_bind_text(sqlite3_stmt*, int, const char*, int n, void(*)(void*)); -int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); -int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); -int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int sqlite3_bind_blob(sqlite3_stmt*, int, const void*, int n, void(*)(void*)); +SQLITE_API int sqlite3_bind_blob64(sqlite3_stmt*, int, const void*, sqlite3_uint64, + void(*)(void*)); +SQLITE_API int sqlite3_bind_double(sqlite3_stmt*, int, double); +SQLITE_API int sqlite3_bind_int(sqlite3_stmt*, int, int); +SQLITE_API int sqlite3_bind_int64(sqlite3_stmt*, int, sqlite3_int64); +SQLITE_API int sqlite3_bind_null(sqlite3_stmt*, int); +SQLITE_API int sqlite3_bind_text(sqlite3_stmt*,int,const char*,int,void(*)(void*)); +SQLITE_API int sqlite3_bind_text16(sqlite3_stmt*, int, const void*, int, void(*)(void*)); +SQLITE_API int sqlite3_bind_text64(sqlite3_stmt*, int, const char*, sqlite3_uint64, + void(*)(void*), unsigned char encoding); +SQLITE_API int sqlite3_bind_value(sqlite3_stmt*, int, const sqlite3_value*); +SQLITE_API int sqlite3_bind_pointer(sqlite3_stmt*, int, void*, const char*,void(*)(void*)); +SQLITE_API int sqlite3_bind_zeroblob(sqlite3_stmt*, int, int n); +SQLITE_API int sqlite3_bind_zeroblob64(sqlite3_stmt*, int, sqlite3_uint64); /* -** CAPI3REF: Number Of Host Parameters +** CAPI3REF: Number Of SQL Parameters +** METHOD: sqlite3_stmt ** -** Return the largest host parameter index in the precompiled statement given -** as the argument. When the host parameters are of the forms like ":AAA" -** or "?", then they are assigned sequential increasing numbers beginning -** with one, so the value returned is the number of parameters. However -** if the same host parameter name is used multiple times, each occurrance -** is given the same number, so the value returned in that case is the number -** of unique host parameter names. If host parameters of the form "?NNN" -** are used (where NNN is an integer) then there might be gaps in the -** numbering and the value returned by this interface is the index of the -** host parameter with the largest index value. +** ^This routine can be used to find the number of [SQL parameters] +** in a [prepared statement]. SQL parameters are tokens of the +** form "?", "?NNN", ":AAA", "$AAA", or "@AAA" that serve as +** placeholders for values that are [sqlite3_bind_blob | bound] +** to the parameters at a later time. +** +** ^(This routine actually returns the index of the largest (rightmost) +** parameter. For all forms except ?NNN, this will correspond to the +** number of unique parameters. If parameters of the ?NNN form are used, +** there may be gaps in the list.)^ +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_name()], and +** [sqlite3_bind_parameter_index()]. */ -int sqlite3_bind_parameter_count(sqlite3_stmt*); +SQLITE_API int sqlite3_bind_parameter_count(sqlite3_stmt*); /* ** CAPI3REF: Name Of A Host Parameter +** METHOD: sqlite3_stmt ** -** This routine returns a pointer to the name of the n-th parameter in a -** [sqlite3_stmt | prepared statement]. -** Host parameters of the form ":AAA" or "@AAA" or "$VVV" have a name -** which is the string ":AAA" or "@AAA" or "$VVV". -** In other words, the initial ":" or "$" or "@" -** is included as part of the name. -** Parameters of the form "?" or "?NNN" have no name. +** ^The sqlite3_bind_parameter_name(P,N) interface returns +** the name of the N-th [SQL parameter] in the [prepared statement] P. +** ^(SQL parameters of the form "?NNN" or ":AAA" or "@AAA" or "$AAA" +** have a name which is the string "?NNN" or ":AAA" or "@AAA" or "$AAA" +** respectively. +** In other words, the initial ":" or "$" or "@" or "?" +** is included as part of the name.)^ +** ^Parameters of the form "?" without a following integer have no name +** and are referred to as "nameless" or "anonymous parameters". ** -** The first bound parameter has an index of 1, not 0. +** ^The first host parameter has an index of 1, not 0. ** -** If the value n is out of range or if the n-th parameter is nameless, -** then NULL is returned. The returned string is always in the -** UTF-8 encoding even if the named parameter was originally specified -** as UTF-16 in [sqlite3_prepare16()] or [sqlite3_prepare16_v2()]. +** ^If the value N is out of range or if the N-th parameter is +** nameless, then NULL is returned. ^The returned string is +** always in UTF-8 encoding even if the named parameter was +** originally specified as UTF-16 in [sqlite3_prepare16()], +** [sqlite3_prepare16_v2()], or [sqlite3_prepare16_v3()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_index()]. */ -const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); +SQLITE_API const char *sqlite3_bind_parameter_name(sqlite3_stmt*, int); /* ** CAPI3REF: Index Of A Parameter With A Given Name +** METHOD: sqlite3_stmt ** -** This routine returns the index of a host parameter with the given name. -** The name must match exactly. If no parameter with the given name is -** found, return 0. Parameter names must be UTF8. +** ^Return the index of an SQL parameter given its name. ^The +** index value returned is suitable for use as the second +** parameter to [sqlite3_bind_blob|sqlite3_bind()]. ^A zero +** is returned if no matching parameter is found. ^The parameter +** name must be given in UTF-8 even if the original statement +** was prepared from UTF-16 text using [sqlite3_prepare16_v2()] or +** [sqlite3_prepare16_v3()]. +** +** See also: [sqlite3_bind_blob|sqlite3_bind()], +** [sqlite3_bind_parameter_count()], and +** [sqlite3_bind_parameter_name()]. */ -int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); +SQLITE_API int sqlite3_bind_parameter_index(sqlite3_stmt*, const char *zName); /* ** CAPI3REF: Reset All Bindings On A Prepared Statement +** METHOD: sqlite3_stmt ** -** Contrary to the intuition of many, [sqlite3_reset()] does not -** reset the [sqlite3_bind_blob | bindings] on a -** [sqlite3_stmt | prepared statement]. Use this routine to -** reset all host parameters to NULL. +** ^Contrary to the intuition of many, [sqlite3_reset()] does not reset +** the [sqlite3_bind_blob | bindings] on a [prepared statement]. +** ^Use this routine to reset all host parameters to NULL. */ -int sqlite3_clear_bindings(sqlite3_stmt*); +SQLITE_API int sqlite3_clear_bindings(sqlite3_stmt*); /* ** CAPI3REF: Number Of Columns In A Result Set +** METHOD: sqlite3_stmt ** -** Return the number of columns in the result set returned by the -** [sqlite3_stmt | compiled SQL statement]. This routine returns 0 -** if pStmt is an SQL statement that does not return data (for -** example an UPDATE). +** ^Return the number of columns in the result set returned by the +** [prepared statement]. ^If this routine returns 0, that means the +** [prepared statement] returns no data (for example an [UPDATE]). +** ^However, just because this routine returns a positive number does not +** mean that one or more rows of data will be returned. ^A SELECT statement +** will always have a positive sqlite3_column_count() but depending on the +** WHERE clause constraints and the table content, it might return no rows. +** +** See also: [sqlite3_data_count()] */ -int sqlite3_column_count(sqlite3_stmt *pStmt); +SQLITE_API int sqlite3_column_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Column Names In A Result Set +** METHOD: sqlite3_stmt ** -** These routines return the name assigned to a particular column -** in the result set of a SELECT statement. The sqlite3_column_name() -** interface returns a pointer to a UTF8 string and sqlite3_column_name16() -** returns a pointer to a UTF16 string. The first parameter is the -** [sqlite_stmt | prepared statement] that implements the SELECT statement. -** The second parameter is the column number. The left-most column is -** number 0. +** ^These routines return the name assigned to a particular column +** in the result set of a [SELECT] statement. ^The sqlite3_column_name() +** interface returns a pointer to a zero-terminated UTF-8 string +** and sqlite3_column_name16() returns a pointer to a zero-terminated +** UTF-16 string. ^The first parameter is the [prepared statement] +** that implements the [SELECT] statement. ^The second parameter is the +** column number. ^The leftmost column is number 0. ** -** The returned string pointer is valid until either the -** [sqlite_stmt | prepared statement] is destroyed by [sqlite3_finalize()] -** or until the next call sqlite3_column_name() or sqlite3_column_name16() -** on the same column. +** ^The returned string pointer is valid until either the [prepared statement] +** is destroyed by [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the next call to +** sqlite3_column_name() or sqlite3_column_name16() on the same column. +** +** ^If sqlite3_malloc() fails during the processing of either routine +** (for example during a conversion from UTF-8 to UTF-16) then a +** NULL pointer is returned. +** +** ^The name of a result column is the value of the "AS" clause for +** that column, if there is an AS clause. If there is no AS clause +** then the name of the column is unspecified and may change from +** one release of SQLite to the next. */ -const char *sqlite3_column_name(sqlite3_stmt*, int N); -const void *sqlite3_column_name16(sqlite3_stmt*, int N); +SQLITE_API const char *sqlite3_column_name(sqlite3_stmt*, int N); +SQLITE_API const void *sqlite3_column_name16(sqlite3_stmt*, int N); /* ** CAPI3REF: Source Of Data In A Query Result +** METHOD: sqlite3_stmt ** -** These routines provide a means to determine what column of what -** table in which database a result of a SELECT statement comes from. -** The name of the database or table or column can be returned as -** either a UTF8 or UTF16 string. The returned string is valid until -** the [sqlite3_stmt | prepared statement] is destroyed using -** [sqlite3_finalize()] or until the same information is requested -** again about the same column. +** ^These routines provide a means to determine the database, table, and +** table column that is the origin of a particular result column in +** [SELECT] statement. +** ^The name of the database or table or column can be returned as +** either a UTF-8 or UTF-16 string. ^The _database_ routines return +** the database name, the _table_ routines return the table name, and +** the origin_ routines return the column name. +** ^The returned string is valid until the [prepared statement] is destroyed +** using [sqlite3_finalize()] or until the statement is automatically +** reprepared by the first call to [sqlite3_step()] for a particular run +** or until the same information is requested +** again in a different encoding. ** -** The first argument to the following calls is a -** [sqlite3_stmt | compiled SQL statement]. -** These functions return information about the Nth column returned by +** ^The names returned are the original un-aliased names of the +** database, table, and column. +** +** ^The first argument to these interfaces is a [prepared statement]. +** ^These functions return information about the Nth result column returned by ** the statement, where N is the second function argument. +** ^The left-most column is column 0 for these routines. ** -** If the Nth column returned by the statement is an expression -** or subquery and is not a column value, then all of these functions -** return NULL. Otherwise, they return the -** name of the attached database, table and column that query result -** column was extracted from. +** ^If the Nth column returned by the statement is an expression or +** subquery and is not a column value, then all of these functions return +** NULL. ^These routines might also return NULL if a memory allocation error +** occurs. ^Otherwise, they return the name of the attached database, table, +** or column that query result column was extracted from. ** -** As with all other SQLite APIs, those postfixed with "16" return UTF-16 -** encoded strings, the other functions return UTF-8. +** ^As with all other SQLite APIs, those whose names end with "16" return +** UTF-16 encoded strings and the other functions return UTF-8. ** -** These APIs are only available if the library was compiled with the -** SQLITE_ENABLE_COLUMN_METADATA preprocessor symbol defined. +** ^These APIs are only available if the library was compiled with the +** [SQLITE_ENABLE_COLUMN_METADATA] C-preprocessor symbol. +** +** If two or more threads call one or more +** [sqlite3_column_database_name | column metadata interfaces] +** for the same [prepared statement] and result column +** at the same time then the results are undefined. */ -const char *sqlite3_column_database_name(sqlite3_stmt*,int); -const void *sqlite3_column_database_name16(sqlite3_stmt*,int); -const char *sqlite3_column_table_name(sqlite3_stmt*,int); -const void *sqlite3_column_table_name16(sqlite3_stmt*,int); -const char *sqlite3_column_origin_name(sqlite3_stmt*,int); -const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_database_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_database_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_table_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_table_name16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_origin_name(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_origin_name16(sqlite3_stmt*,int); /* ** CAPI3REF: Declared Datatype Of A Query Result +** METHOD: sqlite3_stmt ** -** The first parameter is a [sqlite3_stmt | compiled SQL statement]. -** If this statement is a SELECT statement and the Nth column of the -** returned result set of that SELECT is a table column (not an +** ^(The first parameter is a [prepared statement]. +** If this statement is a [SELECT] statement and the Nth column of the +** returned result set of that [SELECT] is a table column (not an ** expression or subquery) then the declared type of the table -** column is returned. If the Nth column of the result set is an +** column is returned.)^ ^If the Nth column of the result set is an ** expression or subquery, then a NULL pointer is returned. -** The returned string is always UTF-8 encoded. For example, in -** the database schema: +** ^The returned string is always UTF-8 encoded. +** +** ^(For example, given the database schema: ** ** CREATE TABLE t1(c1 VARIANT); ** -** And the following statement compiled: +** and the following statement to be compiled: ** ** SELECT c1 + 1, c1 FROM t1; ** -** Then this routine would return the string "VARIANT" for the second -** result column (i==1), and a NULL pointer for the first result column -** (i==0). +** this routine would return the string "VARIANT" for the second result +** column (i==1), and a NULL pointer for the first result column (i==0).)^ ** -** SQLite uses dynamic run-time typing. So just because a column +** ^SQLite uses dynamic run-time typing. ^So just because a column ** is declared to contain a particular type does not mean that the ** data stored in that column is of the declared type. SQLite is -** strongly typed, but the typing is dynamic not static. Type +** strongly typed, but the typing is dynamic not static. ^Type ** is associated with individual values, not with the containers ** used to hold those values. */ -const char *sqlite3_column_decltype(sqlite3_stmt *, int i); -const void *sqlite3_column_decltype16(sqlite3_stmt*,int); +SQLITE_API const char *sqlite3_column_decltype(sqlite3_stmt*,int); +SQLITE_API const void *sqlite3_column_decltype16(sqlite3_stmt*,int); -/* -** CAPI3REF: Evaluate An SQL Statement +/* +** CAPI3REF: Evaluate An SQL Statement +** METHOD: sqlite3_stmt ** -** After an [sqlite3_stmt | SQL statement] has been prepared with a call -** to either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] or to one of -** the legacy interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], -** then this function must be called one or more times to evaluate the -** statement. +** After a [prepared statement] has been prepared using any of +** [sqlite3_prepare_v2()], [sqlite3_prepare_v3()], [sqlite3_prepare16_v2()], +** or [sqlite3_prepare16_v3()] or one of the legacy +** interfaces [sqlite3_prepare()] or [sqlite3_prepare16()], this function +** must be called one or more times to evaluate the statement. ** -** The details of the behavior of this sqlite3_step() interface depend -** on whether the statement was prepared using the newer "v2" interface -** [sqlite3_prepare_v2()] and [sqlite3_prepare16_v2()] or the older legacy -** interface [sqlite3_prepare()] and [sqlite3_prepare16()]. The use of the -** new "v2" interface is recommended for new applications but the legacy +** The details of the behavior of the sqlite3_step() interface depend +** on whether the statement was prepared using the newer "vX" interfaces +** [sqlite3_prepare_v3()], [sqlite3_prepare_v2()], [sqlite3_prepare16_v3()], +** [sqlite3_prepare16_v2()] or the older legacy +** interfaces [sqlite3_prepare()] and [sqlite3_prepare16()]. The use of the +** new "vX" interface is recommended for new applications but the legacy ** interface will continue to be supported. ** -** In the lagacy interface, the return value will be either [SQLITE_BUSY], +** ^In the legacy interface, the return value will be either [SQLITE_BUSY], ** [SQLITE_DONE], [SQLITE_ROW], [SQLITE_ERROR], or [SQLITE_MISUSE]. -** With the "v2" interface, any of the other [SQLITE_OK | result code] -** or [SQLITE_IOERR_READ | extended result code] might be returned as -** well. +** ^With the "v2" interface, any of the other [result codes] or +** [extended result codes] might be returned as well. ** -** [SQLITE_BUSY] means that the database engine was unable to acquire the -** database locks it needs to do its job. If the statement is a COMMIT +** ^[SQLITE_BUSY] means that the database engine was unable to acquire the +** database locks it needs to do its job. ^If the statement is a [COMMIT] ** or occurs outside of an explicit transaction, then you can retry the -** statement. If the statement is not a COMMIT and occurs within a +** statement. If the statement is not a [COMMIT] and occurs within an ** explicit transaction then you should rollback the transaction before ** continuing. ** -** [SQLITE_DONE] means that the statement has finished executing +** ^[SQLITE_DONE] means that the statement has finished executing ** successfully. sqlite3_step() should not be called again on this virtual ** machine without first calling [sqlite3_reset()] to reset the virtual ** machine back to its initial state. ** -** If the SQL statement being executed returns any data, then -** [SQLITE_ROW] is returned each time a new row of data is ready -** for processing by the caller. The values may be accessed using -** the [sqlite3_column_int | column access functions]. +** ^If the SQL statement being executed returns any data, then [SQLITE_ROW] +** is returned each time a new row of data is ready for processing by the +** caller. The values may be accessed using the [column access functions]. ** sqlite3_step() is called again to retrieve the next row of data. -** -** [SQLITE_ERROR] means that a run-time error (such as a constraint +** +** ^[SQLITE_ERROR] means that a run-time error (such as a constraint ** violation) has occurred. sqlite3_step() should not be called again on ** the VM. More information may be found by calling [sqlite3_errmsg()]. -** With the legacy interface, a more specific error code (example: +** ^With the legacy interface, a more specific error code (for example, ** [SQLITE_INTERRUPT], [SQLITE_SCHEMA], [SQLITE_CORRUPT], and so forth) ** can be obtained by calling [sqlite3_reset()] on the -** [sqlite_stmt | prepared statement]. In the "v2" interface, +** [prepared statement]. ^In the "v2" interface, ** the more specific error code is returned directly by sqlite3_step(). ** ** [SQLITE_MISUSE] means that the this routine was called inappropriately. -** Perhaps it was called on a [sqlite_stmt | prepared statement] that has -** already been [sqlite3_finalize | finalized] or on one that had +** Perhaps it was called on a [prepared statement] that has +** already been [sqlite3_finalize | finalized] or on one that had ** previously returned [SQLITE_ERROR] or [SQLITE_DONE]. Or it could ** be the case that the same database connection is being used by two or ** more threads at the same moment in time. ** -** Goofy Interface Alert: -** In the legacy interface, -** the sqlite3_step() API always returns a generic error code, -** [SQLITE_ERROR], following any error other than [SQLITE_BUSY] -** and [SQLITE_MISUSE]. You must call [sqlite3_reset()] or -** [sqlite3_finalize()] in order to find one of the specific -** [SQLITE_ERROR | result codes] that better describes the error. +** For all versions of SQLite up to and including 3.6.23.1, a call to +** [sqlite3_reset()] was required after sqlite3_step() returned anything +** other than [SQLITE_ROW] before any subsequent invocation of +** sqlite3_step(). Failure to reset the prepared statement using +** [sqlite3_reset()] would result in an [SQLITE_MISUSE] return from +** sqlite3_step(). But after [version 3.6.23.1] ([dateof:3.6.23.1], +** sqlite3_step() began +** calling [sqlite3_reset()] automatically in this circumstance rather +** than returning [SQLITE_MISUSE]. This is not considered a compatibility +** break because any application that ever receives an SQLITE_MISUSE error +** is broken by definition. The [SQLITE_OMIT_AUTORESET] compile-time option +** can be used to restore the legacy behavior. +** +** Goofy Interface Alert: In the legacy interface, the sqlite3_step() +** API always returns a generic error code, [SQLITE_ERROR], following any +** error other than [SQLITE_BUSY] and [SQLITE_MISUSE]. You must call +** [sqlite3_reset()] or [sqlite3_finalize()] in order to find one of the +** specific [error codes] that better describes the error. ** We admit that this is a goofy design. The problem has been fixed ** with the "v2" interface. If you prepare all of your SQL statements -** using either [sqlite3_prepare_v2()] or [sqlite3_prepare16_v2()] instead -** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()], then the -** more specific [SQLITE_ERROR | result codes] are returned directly -** by sqlite3_step(). The use of the "v2" interface is recommended. +** using [sqlite3_prepare_v3()] or [sqlite3_prepare_v2()] +** or [sqlite3_prepare16_v2()] or [sqlite3_prepare16_v3()] instead +** of the legacy [sqlite3_prepare()] and [sqlite3_prepare16()] interfaces, +** then the more specific [error codes] are returned directly +** by sqlite3_step(). The use of the "vX" interfaces is recommended. */ -int sqlite3_step(sqlite3_stmt*); +SQLITE_API int sqlite3_step(sqlite3_stmt*); /* -** CAPI3REF: +** CAPI3REF: Number of columns in a result set +** METHOD: sqlite3_stmt ** -** Return the number of values in the current row of the result set. +** ^The sqlite3_data_count(P) interface returns the number of columns in the +** current row of the result set of [prepared statement] P. +** ^If prepared statement P does not have results ready to return +** (via calls to the [sqlite3_column_int | sqlite3_column()] family of +** interfaces) then sqlite3_data_count(P) returns 0. +** ^The sqlite3_data_count(P) routine also returns 0 if P is a NULL pointer. +** ^The sqlite3_data_count(P) routine returns 0 if the previous call to +** [sqlite3_step](P) returned [SQLITE_DONE]. ^The sqlite3_data_count(P) +** will return non-zero if previous call to [sqlite3_step](P) returned +** [SQLITE_ROW], except in the case of the [PRAGMA incremental_vacuum] +** where it always returns zero since each step of that multi-step +** pragma returns 0 columns of data. ** -** After a call to [sqlite3_step()] that returns [SQLITE_ROW], this routine -** will return the same value as the [sqlite3_column_count()] function. -** After [sqlite3_step()] has returned an [SQLITE_DONE], [SQLITE_BUSY], or -** a [SQLITE_ERROR | error code], or before [sqlite3_step()] has been -** called on the [sqlite_stmt | prepared statement] for the first time, -** this routine returns zero. +** See also: [sqlite3_column_count()] */ -int sqlite3_data_count(sqlite3_stmt *pStmt); +SQLITE_API int sqlite3_data_count(sqlite3_stmt *pStmt); /* ** CAPI3REF: Fundamental Datatypes +** KEYWORDS: SQLITE_TEXT ** -** Every value in SQLite has one of five fundamental datatypes: +** ^(Every value in SQLite has one of five fundamental datatypes: ** **
              **
            • 64-bit signed integer @@ -1395,13 +5049,13 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); **
            • string **
            • BLOB **
            • NULL -**
            +**
          )^ ** ** These constants are codes for each of those types. ** ** Note that the SQLITE_TEXT constant was also used in SQLite version 2 ** for a completely different meaning. Software that links against both -** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT not +** SQLite version 2 and SQLite version 3 should use SQLITE3_TEXT, not ** SQLITE_TEXT. */ #define SQLITE_INTEGER 1 @@ -1416,246 +5070,463 @@ int sqlite3_data_count(sqlite3_stmt *pStmt); #define SQLITE3_TEXT 3 /* -** CAPI3REF: Results Values From A Query +** CAPI3REF: Result Values From A Query +** KEYWORDS: {column access functions} +** METHOD: sqlite3_stmt ** -** These routines return information about the information -** in a single column of the current result row of a query. In every -** case the first argument is a pointer to the -** [sqlite3_stmt | SQL statement] that is being -** evaluate (the [sqlite_stmt*] that was returned from -** [sqlite3_prepare_v2()] or one of its variants) and -** the second argument is the index of the column for which information -** should be returned. The left-most column has an index of 0. +** Summary: +**
          +**
          sqlite3_column_blob→BLOB result +**
          sqlite3_column_double→REAL result +**
          sqlite3_column_int→32-bit INTEGER result +**
          sqlite3_column_int64→64-bit INTEGER result +**
          sqlite3_column_text→UTF-8 TEXT result +**
          sqlite3_column_text16→UTF-16 TEXT result +**
          sqlite3_column_value→The result as an +** [sqlite3_value|unprotected sqlite3_value] object. +**
              +**
          sqlite3_column_bytes→Size of a BLOB +** or a UTF-8 TEXT result in bytes +**
          sqlite3_column_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
          sqlite3_column_type→Default +** datatype of the result +**
          ** -** If the SQL statement is not currently point to a valid row, or if the -** the column index is out of range, the result is undefined. +** Details: ** -** The sqlite3_column_type() routine returns +** ^These routines return information about a single column of the current +** result row of a query. ^In every case the first argument is a pointer +** to the [prepared statement] that is being evaluated (the [sqlite3_stmt*] +** that was returned from [sqlite3_prepare_v2()] or one of its variants) +** and the second argument is the index of the column for which information +** should be returned. ^The leftmost column of the result set has the index 0. +** ^The number of columns in the result can be determined using +** [sqlite3_column_count()]. +** +** If the SQL statement does not currently point to a valid row, or if the +** column index is out of range, the result is undefined. +** These routines may only be called when the most recent call to +** [sqlite3_step()] has returned [SQLITE_ROW] and neither +** [sqlite3_reset()] nor [sqlite3_finalize()] have been called subsequently. +** If any of these routines are called after [sqlite3_reset()] or +** [sqlite3_finalize()] or after [sqlite3_step()] has returned +** something other than [SQLITE_ROW], the results are undefined. +** If [sqlite3_step()] or [sqlite3_reset()] or [sqlite3_finalize()] +** are called from a different thread while any of these routines +** are pending, then the results are undefined. +** +** The first six interfaces (_blob, _double, _int, _int64, _text, and _text16) +** each return the value of a result column in a specific data format. If +** the result column is not initially in the requested format (for example, +** if the query returns an integer but the sqlite3_column_text() interface +** is used to extract the value) then an automatic type conversion is performed. +** +** ^The sqlite3_column_type() routine returns the ** [SQLITE_INTEGER | datatype code] for the initial data type -** of the result column. The returned value is one of [SQLITE_INTEGER], -** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL]. The value -** returned by sqlite3_column_type() is only meaningful if no type -** conversions have occurred as described below. After a type conversion, -** the value returned by sqlite3_column_type() is undefined. Future +** of the result column. ^The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL]. +** The return value of sqlite3_column_type() can be used to decide which +** of the first six interface should be used to extract the column value. +** The value returned by sqlite3_column_type() is only meaningful if no +** automatic type conversions have occurred for the value in question. +** After a type conversion, the result of calling sqlite3_column_type() +** is undefined, though harmless. Future ** versions of SQLite may change the behavior of sqlite3_column_type() ** following a type conversion. ** -*** The sqlite3_column_nm +** If the result is a BLOB or a TEXT string, then the sqlite3_column_bytes() +** or sqlite3_column_bytes16() interfaces can be used to determine the size +** of that BLOB or string. ** -** If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() +** ^If the result is a BLOB or UTF-8 string then the sqlite3_column_bytes() ** routine returns the number of bytes in that BLOB or string. -** If the result is a UTF-16 string, then sqlite3_column_bytes() converts +** ^If the result is a UTF-16 string, then sqlite3_column_bytes() converts ** the string to UTF-8 and then returns the number of bytes. -** If the result is a numeric value then sqlite3_column_bytes() uses +** ^If the result is a numeric value then sqlite3_column_bytes() uses ** [sqlite3_snprintf()] to convert that value to a UTF-8 string and returns ** the number of bytes in that string. -** The value returned does not include the zero terminator at the end -** of the string. For clarity: the value returned is the number of +** ^If the result is NULL, then sqlite3_column_bytes() returns zero. +** +** ^If the result is a BLOB or UTF-16 string then the sqlite3_column_bytes16() +** routine returns the number of bytes in that BLOB or string. +** ^If the result is a UTF-8 string, then sqlite3_column_bytes16() converts +** the string to UTF-16 and then returns the number of bytes. +** ^If the result is a numeric value then sqlite3_column_bytes16() uses +** [sqlite3_snprintf()] to convert that value to a UTF-16 string and returns +** the number of bytes in that string. +** ^If the result is NULL, then sqlite3_column_bytes16() returns zero. +** +** ^The values returned by [sqlite3_column_bytes()] and +** [sqlite3_column_bytes16()] do not include the zero terminators at the end +** of the string. ^For clarity: the values returned by +** [sqlite3_column_bytes()] and [sqlite3_column_bytes16()] are the number of ** bytes in the string, not the number of characters. ** -** The sqlite3_column_bytes16() routine is similar to sqlite3_column_bytes() -** but leaves the result in UTF-16 instead of UTF-8. -** The zero terminator is not included in this count. +** ^Strings returned by sqlite3_column_text() and sqlite3_column_text16(), +** even empty strings, are always zero-terminated. ^The return +** value from sqlite3_column_blob() for a zero-length BLOB is a NULL pointer. ** -** These routines attempt to convert the value where appropriate. For -** example, if the internal representation is FLOAT and a text result -** is requested, [sqlite3_snprintf()] is used internally to do the conversion -** automatically. The following table details the conversions that -** are applied: +** ^Strings returned by sqlite3_column_text16() always have the endianness +** which is native to the platform, regardless of the text encoding set +** for the database. +** +** Warning: ^The object returned by [sqlite3_column_value()] is an +** [unprotected sqlite3_value] object. In a multithreaded environment, +** an unprotected sqlite3_value object may only be used safely with +** [sqlite3_bind_value()] and [sqlite3_result_value()]. +** If the [unprotected sqlite3_value] object returned by +** [sqlite3_column_value()] is used in any other way, including calls +** to routines like [sqlite3_value_int()], [sqlite3_value_text()], +** or [sqlite3_value_bytes()], the behavior is not threadsafe. +** Hence, the sqlite3_column_value() interface +** is normally only useful within the implementation of +** [application-defined SQL functions] or [virtual tables], not within +** top-level application code. +** +** These routines may attempt to convert the datatype of the result. +** ^For example, if the internal representation is FLOAT and a text result +** is requested, [sqlite3_snprintf()] is used internally to perform the +** conversion automatically. ^(The following table details the conversions +** that are applied: ** **
          ** -**
          Internal Requested -**
          Type Type Conversion +**
          Internal
          Type           		Requested
          Type           		Conversion ** **
          NULL INTEGER Result is 0 **
          NULL FLOAT Result is 0.0 -**
          NULL TEXT Result is NULL pointer -**
          NULL BLOB Result is NULL pointer +**
          NULL TEXT Result is a NULL pointer +**
          NULL BLOB Result is a NULL pointer **
          INTEGER FLOAT Convert from integer to float **
          INTEGER TEXT ASCII rendering of the integer -**
          INTEGER BLOB Same as for INTEGER->TEXT -**
          FLOAT INTEGER Convert from float to integer +**
          INTEGER BLOB Same as INTEGER->TEXT +**
          FLOAT INTEGER [CAST] to INTEGER **
          FLOAT TEXT ASCII rendering of the float -**
          FLOAT BLOB Same as FLOAT->TEXT -**
          TEXT INTEGER Use atoi() -**
          TEXT FLOAT Use atof() +**
          FLOAT BLOB [CAST] to BLOB +**
          TEXT INTEGER [CAST] to INTEGER +**
          TEXT FLOAT [CAST] to REAL **
          TEXT BLOB No change -**
          BLOB INTEGER Convert to TEXT then use atoi() -**
          BLOB FLOAT Convert to TEXT then use atof() -**
          BLOB TEXT Add a zero terminator if needed +**
          BLOB INTEGER [CAST] to INTEGER +**
          BLOB FLOAT [CAST] to REAL +**
          BLOB TEXT [CAST] to TEXT, ensure zero terminator **
          -**
          -** -** The table above makes reference to standard C library functions atoi() -** and atof(). SQLite does not really use these functions. It has its -** on equavalent internal routines. The atoi() and atof() names are -** used in the table for brevity and because they are familiar to most -** C programmers. +** )^ ** ** Note that when type conversions occur, pointers returned by prior ** calls to sqlite3_column_blob(), sqlite3_column_text(), and/or -** sqlite3_column_text16() may be invalidated. +** sqlite3_column_text16() may be invalidated. ** Type conversions and pointer invalidations might occur ** in the following cases: ** **
            -**

          • The initial content is a BLOB and sqlite3_column_text() -** or sqlite3_column_text16() is called. A zero-terminator might -** need to be added to the string.

            • -** -**

          • The initial content is UTF-8 text and sqlite3_column_bytes16() or -** sqlite3_column_text16() is called. The content must be converted -** to UTF-16.

            • -** -**

          • The initial content is UTF-16 text and sqlite3_column_bytes() or -** sqlite3_column_text() is called. The content must be converted -** to UTF-8.

            • +**
          • The initial content is a BLOB and sqlite3_column_text() or +** sqlite3_column_text16() is called. A zero-terminator might +** need to be added to the string.
            • +**
          • The initial content is UTF-8 text and sqlite3_column_bytes16() or +** sqlite3_column_text16() is called. The content must be converted +** to UTF-16.
            • +**
          • The initial content is UTF-16 text and sqlite3_column_bytes() or +** sqlite3_column_text() is called. The content must be converted +** to UTF-8.
            • **
          ** -** Conversions between UTF-16be and UTF-16le are always done in place and do +** ^Conversions between UTF-16be and UTF-16le are always done in place and do ** not invalidate a prior pointer, though of course the content of the buffer -** that the prior pointer points to will have been modified. Other kinds -** of conversion are done in place when it is possible, but sometime it is -** not possible and in those cases prior pointers are invalidated. +** that the prior pointer references will have been modified. Other kinds +** of conversion are done in place when it is possible, but sometimes they +** are not possible and in those cases prior pointers are invalidated. ** -** The safest and easiest to remember policy is to invoke these routines +** The safest policy is to invoke these routines ** in one of the following ways: ** -**
            +**
              **
            • sqlite3_column_text() followed by sqlite3_column_bytes()
              • **
            • sqlite3_column_blob() followed by sqlite3_column_bytes()
              • **
            • sqlite3_column_text16() followed by sqlite3_column_bytes16()
              • -**
            +**
          ** -** In other words, you should call sqlite3_column_text(), sqlite3_column_blob(), -** or sqlite3_column_text16() first to force the result into the desired -** format, then invoke sqlite3_column_bytes() or sqlite3_column_bytes16() to -** find the size of the result. Do not mix call to sqlite3_column_text() or -** sqlite3_column_blob() with calls to sqlite3_column_bytes16(). And do not -** mix calls to sqlite3_column_text16() with calls to sqlite3_column_bytes(). +** In other words, you should call sqlite3_column_text(), +** sqlite3_column_blob(), or sqlite3_column_text16() first to force the result +** into the desired format, then invoke sqlite3_column_bytes() or +** sqlite3_column_bytes16() to find the size of the result. Do not mix calls +** to sqlite3_column_text() or sqlite3_column_blob() with calls to +** sqlite3_column_bytes16(), and do not mix calls to sqlite3_column_text16() +** with calls to sqlite3_column_bytes(). +** +** ^The pointers returned are valid until a type conversion occurs as +** described above, or until [sqlite3_step()] or [sqlite3_reset()] or +** [sqlite3_finalize()] is called. ^The memory space used to hold strings +** and BLOBs is freed automatically. Do not pass the pointers returned +** from [sqlite3_column_blob()], [sqlite3_column_text()], etc. into +** [sqlite3_free()]. +** +** As long as the input parameters are correct, these routines will only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
            +**
          • sqlite3_column_blob() +**
          • sqlite3_column_text() +**
          • sqlite3_column_text16() +**
          • sqlite3_column_bytes() +**
          • sqlite3_column_bytes16() +**
          +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ -const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); -int sqlite3_column_bytes(sqlite3_stmt*, int iCol); -int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); -double sqlite3_column_double(sqlite3_stmt*, int iCol); -int sqlite3_column_int(sqlite3_stmt*, int iCol); -sqlite_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); -const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); -const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); -int sqlite3_column_type(sqlite3_stmt*, int iCol); -sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_blob(sqlite3_stmt*, int iCol); +SQLITE_API double sqlite3_column_double(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_int(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_int64 sqlite3_column_int64(sqlite3_stmt*, int iCol); +SQLITE_API const unsigned char *sqlite3_column_text(sqlite3_stmt*, int iCol); +SQLITE_API const void *sqlite3_column_text16(sqlite3_stmt*, int iCol); +SQLITE_API sqlite3_value *sqlite3_column_value(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_bytes16(sqlite3_stmt*, int iCol); +SQLITE_API int sqlite3_column_type(sqlite3_stmt*, int iCol); /* ** CAPI3REF: Destroy A Prepared Statement Object +** DESTRUCTOR: sqlite3_stmt ** -** The sqlite3_finalize() function is called to delete a -** [sqlite3_stmt | compiled SQL statement]. If the statement was -** executed successfully, or not executed at all, then SQLITE_OK is returned. -** If execution of the statement failed then an -** [SQLITE_ERROR | error code] or [SQLITE_IOERR_READ | extended error code] -** is returned. +** ^The sqlite3_finalize() function is called to delete a [prepared statement]. +** ^If the most recent evaluation of the statement encountered no errors +** or if the statement is never been evaluated, then sqlite3_finalize() returns +** SQLITE_OK. ^If the most recent evaluation of statement S failed, then +** sqlite3_finalize(S) returns the appropriate [error code] or +** [extended error code]. ** -** This routine can be called at any point during the execution of the -** [sqlite3_stmt | virtual machine]. If the virtual machine has not -** completed execution when this routine is called, that is like -** encountering an error or an interrupt. (See [sqlite3_interrupt()].) -** Incomplete updates may be rolled back and transactions cancelled, -** depending on the circumstances, and the -** [SQLITE_ERROR | result code] returned will be [SQLITE_ABORT]. +** ^The sqlite3_finalize(S) routine can be called at any point during +** the life cycle of [prepared statement] S: +** before statement S is ever evaluated, after +** one or more calls to [sqlite3_reset()], or after any call +** to [sqlite3_step()] regardless of whether or not the statement has +** completed execution. +** +** ^Invoking sqlite3_finalize() on a NULL pointer is a harmless no-op. +** +** The application must finalize every [prepared statement] in order to avoid +** resource leaks. It is a grievous error for the application to try to use +** a prepared statement after it has been finalized. Any use of a prepared +** statement after it has been finalized can result in undefined and +** undesirable behavior such as segfaults and heap corruption. */ -int sqlite3_finalize(sqlite3_stmt *pStmt); +SQLITE_API int sqlite3_finalize(sqlite3_stmt *pStmt); /* ** CAPI3REF: Reset A Prepared Statement Object +** METHOD: sqlite3_stmt ** -** The sqlite3_reset() function is called to reset a -** [sqlite_stmt | compiled SQL statement] object. -** back to it's initial state, ready to be re-executed. -** Any SQL statement variables that had values bound to them using +** The sqlite3_reset() function is called to reset a [prepared statement] +** object back to its initial state, ready to be re-executed. +** ^Any SQL statement variables that had values bound to them using ** the [sqlite3_bind_blob | sqlite3_bind_*() API] retain their values. ** Use [sqlite3_clear_bindings()] to reset the bindings. +** +** ^The [sqlite3_reset(S)] interface resets the [prepared statement] S +** back to the beginning of its program. +** +** ^The return code from [sqlite3_reset(S)] indicates whether or not +** the previous evaluation of prepared statement S completed successfully. +** ^If [sqlite3_step(S)] has never before been called on S or if +** [sqlite3_step(S)] has not been called since the previous call +** to [sqlite3_reset(S)], then [sqlite3_reset(S)] will return +** [SQLITE_OK]. +** +** ^If the most recent call to [sqlite3_step(S)] for the +** [prepared statement] S indicated an error, then +** [sqlite3_reset(S)] returns an appropriate [error code]. +** ^The [sqlite3_reset(S)] interface might also return an [error code] +** if there were no prior errors but the process of resetting +** the prepared statement caused a new error. ^For example, if an +** [INSERT] statement with a [RETURNING] clause is only stepped one time, +** that one call to [sqlite3_step(S)] might return SQLITE_ROW but +** the overall statement might still fail and the [sqlite3_reset(S)] call +** might return SQLITE_BUSY if locking constraints prevent the +** database change from committing. Therefore, it is important that +** applications check the return code from [sqlite3_reset(S)] even if +** no prior call to [sqlite3_step(S)] indicated a problem. +** +** ^The [sqlite3_reset(S)] interface does not change the values +** of any [sqlite3_bind_blob|bindings] on the [prepared statement] S. */ -int sqlite3_reset(sqlite3_stmt *pStmt); +SQLITE_API int sqlite3_reset(sqlite3_stmt *pStmt); + /* ** CAPI3REF: Create Or Redefine SQL Functions +** KEYWORDS: {function creation routines} +** METHOD: sqlite3 ** -** The following two functions are used to add SQL functions or aggregates -** or to redefine the behavior of existing SQL functions or aggregates. The -** difference only between the two is that the second parameter, the -** name of the (scalar) function or aggregate, is encoded in UTF-8 for -** sqlite3_create_function() and UTF-16 for sqlite3_create_function16(). +** ^These functions (collectively known as "function creation routines") +** are used to add SQL functions or aggregates or to redefine the behavior +** of existing SQL functions or aggregates. The only differences between +** the three "sqlite3_create_function*" routines are the text encoding +** expected for the second parameter (the name of the function being +** created) and the presence or absence of a destructor callback for +** the application data pointer. Function sqlite3_create_window_function() +** is similar, but allows the user to supply the extra callback functions +** needed by [aggregate window functions]. ** -** The first argument is the [sqlite3 | database handle] that holds the -** SQL function or aggregate is to be added or redefined. If a single -** program uses more than one database handle internally, then SQL -** functions or aggregates must be added individually to each database -** handle with which they will be used. +** ^The first parameter is the [database connection] to which the SQL +** function is to be added. ^If an application uses more than one database +** connection then application-defined SQL functions must be added +** to each database connection separately. ** -** The second parameter is the name of the SQL function to be created -** or redefined. -** The length of the name is limited to 255 bytes, exclusive of the -** zero-terminator. Note that the name length limit is in bytes, not -** characters. Any attempt to create a function with a longer name -** will result in an SQLITE_ERROR error. +** ^The second parameter is the name of the SQL function to be created or +** redefined. ^The length of the name is limited to 255 bytes in a UTF-8 +** representation, exclusive of the zero-terminator. ^Note that the name +** length limit is in UTF-8 bytes, not characters nor UTF-16 bytes. +** ^Any attempt to create a function with a longer name +** will result in [SQLITE_MISUSE] being returned. ** -** The third parameter is the number of arguments that the SQL function or -** aggregate takes. If this parameter is negative, then the SQL function or -** aggregate may take any number of arguments. +** ^The third parameter (nArg) +** is the number of arguments that the SQL function or +** aggregate takes. ^If this parameter is -1, then the SQL function or +** aggregate may take any number of arguments between 0 and the limit +** set by [sqlite3_limit]([SQLITE_LIMIT_FUNCTION_ARG]). If the third +** parameter is less than -1 or greater than 127 then the behavior is +** undefined. ** -** The fourth parameter, eTextRep, specifies what +** ^The fourth parameter, eTextRep, specifies what ** [SQLITE_UTF8 | text encoding] this SQL function prefers for -** its parameters. Any SQL function implementation should be able to work -** work with UTF-8, UTF-16le, or UTF-16be. But some implementations may be -** more efficient with one encoding than another. It is allowed to -** invoke sqlite_create_function() or sqlite3_create_function16() multiple -** times with the same function but with different values of eTextRep. -** When multiple implementations of the same function are available, SQLite +** its parameters. The application should set this parameter to +** [SQLITE_UTF16LE] if the function implementation invokes +** [sqlite3_value_text16le()] on an input, or [SQLITE_UTF16BE] if the +** implementation invokes [sqlite3_value_text16be()] on an input, or +** [SQLITE_UTF16] if [sqlite3_value_text16()] is used, or [SQLITE_UTF8] +** otherwise. ^The same SQL function may be registered multiple times using +** different preferred text encodings, with different implementations for +** each encoding. +** ^When multiple implementations of the same function are available, SQLite ** will pick the one that involves the least amount of data conversion. -** If there is only a single implementation which does not care what -** text encoding is used, then the fourth argument should be -** [SQLITE_ANY]. ** -** The fifth parameter is an arbitrary pointer. The implementation -** of the function can gain access to this pointer using -** [sqlite_user_data()]. +** ^The fourth parameter may optionally be ORed with [SQLITE_DETERMINISTIC] +** to signal that the function will always return the same result given +** the same inputs within a single SQL statement. Most SQL functions are +** deterministic. The built-in [random()] SQL function is an example of a +** function that is not deterministic. The SQLite query planner is able to +** perform additional optimizations on deterministic functions, so use +** of the [SQLITE_DETERMINISTIC] flag is recommended where possible. ** -** The seventh, eighth and ninth parameters, xFunc, xStep and xFinal, are -** pointers to C-language functions that implement the SQL -** function or aggregate. A scalar SQL function requires an implementation of -** the xFunc callback only, NULL pointers should be passed as the xStep -** and xFinal parameters. An aggregate SQL function requires an implementation -** of xStep and xFinal and NULL should be passed for xFunc. To delete an -** existing SQL function or aggregate, pass NULL for all three function -** callback. +** ^The fourth parameter may also optionally include the [SQLITE_DIRECTONLY] +** flag, which if present prevents the function from being invoked from +** within VIEWs, TRIGGERs, CHECK constraints, generated column expressions, +** index expressions, or the WHERE clause of partial indexes. ** -** It is permitted to register multiple implementations of the same +** For best security, the [SQLITE_DIRECTONLY] flag is recommended for +** all application-defined SQL functions that do not need to be +** used inside of triggers, view, CHECK constraints, or other elements of +** the database schema. This flags is especially recommended for SQL +** functions that have side effects or reveal internal application state. +** Without this flag, an attacker might be able to modify the schema of +** a database file to include invocations of the function with parameters +** chosen by the attacker, which the application will then execute when +** the database file is opened and read. +** +** ^(The fifth parameter is an arbitrary pointer. The implementation of the +** function can gain access to this pointer using [sqlite3_user_data()].)^ +** +** ^The sixth, seventh and eighth parameters passed to the three +** "sqlite3_create_function*" functions, xFunc, xStep and xFinal, are +** pointers to C-language functions that implement the SQL function or +** aggregate. ^A scalar SQL function requires an implementation of the xFunc +** callback only; NULL pointers must be passed as the xStep and xFinal +** parameters. ^An aggregate SQL function requires an implementation of xStep +** and xFinal and NULL pointer must be passed for xFunc. ^To delete an existing +** SQL function or aggregate, pass NULL pointers for all three function +** callbacks. +** +** ^The sixth, seventh, eighth and ninth parameters (xStep, xFinal, xValue +** and xInverse) passed to sqlite3_create_window_function are pointers to +** C-language callbacks that implement the new function. xStep and xFinal +** must both be non-NULL. xValue and xInverse may either both be NULL, in +** which case a regular aggregate function is created, or must both be +** non-NULL, in which case the new function may be used as either an aggregate +** or aggregate window function. More details regarding the implementation +** of aggregate window functions are +** [user-defined window functions|available here]. +** +** ^(If the final parameter to sqlite3_create_function_v2() or +** sqlite3_create_window_function() is not NULL, then it is destructor for +** the application data pointer. The destructor is invoked when the function +** is deleted, either by being overloaded or when the database connection +** closes.)^ ^The destructor is also invoked if the call to +** sqlite3_create_function_v2() fails. ^When the destructor callback is +** invoked, it is passed a single argument which is a copy of the application +** data pointer which was the fifth parameter to sqlite3_create_function_v2(). +** +** ^It is permitted to register multiple implementations of the same ** functions with the same name but with either differing numbers of -** arguments or differing perferred text encodings. SQLite will use -** the implementation most closely matches the way in which the -** SQL function is used. +** arguments or differing preferred text encodings. ^SQLite will use +** the implementation that most closely matches the way in which the +** SQL function is used. ^A function implementation with a non-negative +** nArg parameter is a better match than a function implementation with +** a negative nArg. ^A function where the preferred text encoding +** matches the database encoding is a better +** match than a function where the encoding is different. +** ^A function where the encoding difference is between UTF16le and UTF16be +** is a closer match than a function where the encoding difference is +** between UTF8 and UTF16. +** +** ^Built-in functions may be overloaded by new application-defined functions. +** +** ^An application-defined function is permitted to call other +** SQLite interfaces. However, such calls must not +** close the database connection nor finalize or reset the prepared +** statement in which the function is running. */ -int sqlite3_create_function( - sqlite3 *, +SQLITE_API int sqlite3_create_function( + sqlite3 *db, const char *zFunctionName, int nArg, int eTextRep, - void*, + void *pApp, void (*xFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); -int sqlite3_create_function16( - sqlite3*, +SQLITE_API int sqlite3_create_function16( + sqlite3 *db, const void *zFunctionName, int nArg, int eTextRep, - void*, + void *pApp, void (*xFunc)(sqlite3_context*,int,sqlite3_value**), void (*xStep)(sqlite3_context*,int,sqlite3_value**), void (*xFinal)(sqlite3_context*) ); +SQLITE_API int sqlite3_create_function_v2( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xFunc)(sqlite3_context*,int,sqlite3_value**), + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void(*xDestroy)(void*) +); +SQLITE_API int sqlite3_create_window_function( + sqlite3 *db, + const char *zFunctionName, + int nArg, + int eTextRep, + void *pApp, + void (*xStep)(sqlite3_context*,int,sqlite3_value**), + void (*xFinal)(sqlite3_context*), + void (*xValue)(sqlite3_context*), + void (*xInverse)(sqlite3_context*,int,sqlite3_value**), + void(*xDestroy)(void*) +); /* ** CAPI3REF: Text Encodings @@ -1663,157 +5534,552 @@ int sqlite3_create_function16( ** These constant define integer codes that represent the various ** text encodings supported by SQLite. */ -#define SQLITE_UTF8 1 -#define SQLITE_UTF16LE 2 -#define SQLITE_UTF16BE 3 +#define SQLITE_UTF8 1 /* IMP: R-37514-35566 */ +#define SQLITE_UTF16LE 2 /* IMP: R-03371-37637 */ +#define SQLITE_UTF16BE 3 /* IMP: R-51971-34154 */ #define SQLITE_UTF16 4 /* Use native byte order */ -#define SQLITE_ANY 5 /* sqlite3_create_function only */ +#define SQLITE_ANY 5 /* Deprecated */ #define SQLITE_UTF16_ALIGNED 8 /* sqlite3_create_collation only */ /* -** CAPI3REF: Obsolete Functions +** CAPI3REF: Function Flags ** -** These functions are all now obsolete. In order to maintain -** backwards compatibility with older code, we continue to support -** these functions. However, new development projects should avoid -** the use of these functions. To help encourage people to avoid -** using these functions, we are not going to tell you want they do. +** These constants may be ORed together with the +** [SQLITE_UTF8 | preferred text encoding] as the fourth argument +** to [sqlite3_create_function()], [sqlite3_create_function16()], or +** [sqlite3_create_function_v2()]. +** +**
          +** [[SQLITE_DETERMINISTIC]]
          SQLITE_DETERMINISTIC
          +** The SQLITE_DETERMINISTIC flag means that the new function always gives +** the same output when the input parameters are the same. +** The [abs|abs() function] is deterministic, for example, but +** [randomblob|randomblob()] is not. Functions must +** be deterministic in order to be used in certain contexts such as +** with the WHERE clause of [partial indexes] or in [generated columns]. +** SQLite might also optimize deterministic functions by factoring them +** out of inner loops. +**
          +** +** [[SQLITE_DIRECTONLY]]
          SQLITE_DIRECTONLY
          +** The SQLITE_DIRECTONLY flag means that the function may only be invoked +** from top-level SQL, and cannot be used in VIEWs or TRIGGERs nor in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], or [generated columns]. +**

          +** The SQLITE_DIRECTONLY flag is recommended for any +** [application-defined SQL function] +** that has side-effects or that could potentially leak sensitive information. +** This will prevent attacks in which an application is tricked +** into using a database file that has had its schema surreptitiously +** modified to invoke the application-defined function in ways that are +** harmful. +**

          +** Some people say it is good practice to set SQLITE_DIRECTONLY on all +** [application-defined SQL functions], regardless of whether or not they +** are security sensitive, as doing so prevents those functions from being used +** inside of the database schema, and thus ensures that the database +** can be inspected and modified using generic tools (such as the [CLI]) +** that do not have access to the application-defined functions. +**

          +** +** [[SQLITE_INNOCUOUS]]
          SQLITE_INNOCUOUS
          +** The SQLITE_INNOCUOUS flag means that the function is unlikely +** to cause problems even if misused. An innocuous function should have +** no side effects and should not depend on any values other than its +** input parameters. The [abs|abs() function] is an example of an +** innocuous function. +** The [load_extension() SQL function] is not innocuous because of its +** side effects. +**

          SQLITE_INNOCUOUS is similar to SQLITE_DETERMINISTIC, but is not +** exactly the same. The [random|random() function] is an example of a +** function that is innocuous but not deterministic. +**

          Some heightened security settings +** ([SQLITE_DBCONFIG_TRUSTED_SCHEMA] and [PRAGMA trusted_schema=OFF]) +** disable the use of SQL functions inside views and triggers and in +** schema structures such as [CHECK constraints], [DEFAULT clauses], +** [expression indexes], [partial indexes], and [generated columns] unless +** the function is tagged with SQLITE_INNOCUOUS. Most built-in functions +** are innocuous. Developers are advised to avoid using the +** SQLITE_INNOCUOUS flag for application-defined functions unless the +** function has been carefully audited and found to be free of potentially +** security-adverse side-effects and information-leaks. +**

          +** +** [[SQLITE_SUBTYPE]]
          SQLITE_SUBTYPE
          +** The SQLITE_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_value_subtype()] to inspect the sub-types of its arguments. +** This flag instructs SQLite to omit some corner-case optimizations that +** might disrupt the operation of the [sqlite3_value_subtype()] function, +** causing it to return zero rather than the correct subtype(). +** All SQL functions that invoke [sqlite3_value_subtype()] should have this +** property. If the SQLITE_SUBTYPE property is omitted, then the return +** value from [sqlite3_value_subtype()] might sometimes be zero even though +** a non-zero subtype was specified by the function argument expression. +** +** [[SQLITE_RESULT_SUBTYPE]]
          SQLITE_RESULT_SUBTYPE
          +** The SQLITE_RESULT_SUBTYPE flag indicates to SQLite that a function might call +** [sqlite3_result_subtype()] to cause a sub-type to be associated with its +** result. +** Every function that invokes [sqlite3_result_subtype()] should have this +** property. If it does not, then the call to [sqlite3_result_subtype()] +** might become a no-op if the function is used as term in an +** [expression index]. On the other hand, SQL functions that never invoke +** [sqlite3_result_subtype()] should avoid setting this property, as the +** purpose of this property is to disable certain optimizations that are +** incompatible with subtypes. +** +** [[SQLITE_SELFORDER1]]
          SQLITE_SELFORDER1
          +** The SQLITE_SELFORDER1 flag indicates that the function is an aggregate +** that internally orders the values provided to the first argument. The +** ordered-set aggregate SQL notation with a single ORDER BY term can be +** used to invoke this function. If the ordered-set aggregate notation is +** used on a function that lacks this flag, then an error is raised. Note +** that the ordered-set aggregate syntax is only available if SQLite is +** built using the -DSQLITE_ENABLE_ORDERED_SET_AGGREGATES compile-time option. +**
          +**
          */ -int sqlite3_aggregate_count(sqlite3_context*); -int sqlite3_expired(sqlite3_stmt*); -int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); -int sqlite3_global_recover(void); - +#define SQLITE_DETERMINISTIC 0x000000800 +#define SQLITE_DIRECTONLY 0x000080000 +#define SQLITE_SUBTYPE 0x000100000 +#define SQLITE_INNOCUOUS 0x000200000 +#define SQLITE_RESULT_SUBTYPE 0x001000000 +#define SQLITE_SELFORDER1 0x002000000 /* -** CAPI3REF: Obtaining SQL Function Parameter Values +** CAPI3REF: Deprecated Functions +** DEPRECATED ** -** The C-language implementation of SQL functions and aggregates uses -** this set of interface routines to access the parameter values on -** the function or aggregate. +** These functions are [deprecated]. In order to maintain +** backwards compatibility with older code, these functions continue +** to be supported. However, new applications should avoid +** the use of these functions. To encourage programmers to avoid +** these functions, we will not explain what they do. +*/ +#ifndef SQLITE_OMIT_DEPRECATED +SQLITE_API SQLITE_DEPRECATED int sqlite3_aggregate_count(sqlite3_context*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_expired(sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_transfer_bindings(sqlite3_stmt*, sqlite3_stmt*); +SQLITE_API SQLITE_DEPRECATED int sqlite3_global_recover(void); +SQLITE_API SQLITE_DEPRECATED void sqlite3_thread_cleanup(void); +SQLITE_API SQLITE_DEPRECATED int sqlite3_memory_alarm(void(*)(void*,sqlite3_int64,int), + void*,sqlite3_int64); +#endif + +/* +** CAPI3REF: Obtaining SQL Values +** METHOD: sqlite3_value ** -** The xFunc (for scalar functions) or xStep (for aggregates) parameters -** to [sqlite3_create_function()] and [sqlite3_create_function16()] -** define callbacks that implement the SQL functions and aggregates. -** The 4th parameter to these callbacks is an array of pointers to -** [sqlite3_value] objects. There is one [sqlite3_value] object for -** each parameter to the SQL function. These routines are used to -** extract values from the [sqlite3_value] objects. +** Summary: +**
          +**
          sqlite3_value_blob→BLOB value +**
          sqlite3_value_double→REAL value +**
          sqlite3_value_int→32-bit INTEGER value +**
          sqlite3_value_int64→64-bit INTEGER value +**
          sqlite3_value_pointer→Pointer value +**
          sqlite3_value_text→UTF-8 TEXT value +**
          sqlite3_value_text16→UTF-16 TEXT value in +** the native byteorder +**
          sqlite3_value_text16be→UTF-16be TEXT value +**
          sqlite3_value_text16le→UTF-16le TEXT value +**
              +**
          sqlite3_value_bytes→Size of a BLOB +** or a UTF-8 TEXT in bytes +**
          sqlite3_value_bytes16   +** →  Size of UTF-16 +** TEXT in bytes +**
          sqlite3_value_type→Default +** datatype of the value +**
          sqlite3_value_numeric_type   +** →  Best numeric datatype of the value +**
          sqlite3_value_nochange   +** →  True if the column is unchanged in an UPDATE +** against a virtual table. +**
          sqlite3_value_frombind   +** →  True if value originated from a [bound parameter] +**
          ** -** These routines work just like the corresponding -** [sqlite3_column_blob | sqlite3_column_* routines] except that -** these routines take a single [sqlite3_value*] pointer instead -** of an [sqlite3_stmt*] pointer and an integer column number. +** Details: ** -** The sqlite3_value_text16() interface extracts a UTF16 string -** in the native byte-order of the host machine. The +** These routines extract type, size, and content information from +** [protected sqlite3_value] objects. Protected sqlite3_value objects +** are used to pass parameter information into the functions that +** implement [application-defined SQL functions] and [virtual tables]. +** +** These routines work only with [protected sqlite3_value] objects. +** Any attempt to use these routines on an [unprotected sqlite3_value] +** is not threadsafe. +** +** ^These routines work just like the corresponding [column access functions] +** except that these routines take a single [protected sqlite3_value] object +** pointer instead of a [sqlite3_stmt*] pointer and an integer column number. +** +** ^The sqlite3_value_text16() interface extracts a UTF-16 string +** in the native byte-order of the host machine. ^The ** sqlite3_value_text16be() and sqlite3_value_text16le() interfaces -** extract UTF16 strings as big-endian and little-endian respectively. +** extract UTF-16 strings as big-endian and little-endian respectively. ** -** The sqlite3_value_numeric_type() interface attempts to apply +** ^If [sqlite3_value] object V was initialized +** using [sqlite3_bind_pointer(S,I,P,X,D)] or [sqlite3_result_pointer(C,P,X,D)] +** and if X and Y are strings that compare equal according to strcmp(X,Y), +** then sqlite3_value_pointer(V,Y) will return the pointer P. ^Otherwise, +** sqlite3_value_pointer(V,Y) returns a NULL. The sqlite3_bind_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** ^(The sqlite3_value_type(V) interface returns the +** [SQLITE_INTEGER | datatype code] for the initial datatype of the +** [sqlite3_value] object V. The returned value is one of [SQLITE_INTEGER], +** [SQLITE_FLOAT], [SQLITE_TEXT], [SQLITE_BLOB], or [SQLITE_NULL].)^ +** Other interfaces might change the datatype for an sqlite3_value object. +** For example, if the datatype is initially SQLITE_INTEGER and +** sqlite3_value_text(V) is called to extract a text value for that +** integer, then subsequent calls to sqlite3_value_type(V) might return +** SQLITE_TEXT. Whether or not a persistent internal datatype conversion +** occurs is undefined and may change from one release of SQLite to the next. +** +** ^(The sqlite3_value_numeric_type() interface attempts to apply ** numeric affinity to the value. This means that an attempt is ** made to convert the value to an integer or floating point. If -** such a conversion is possible without loss of information (in order -** words if the value is original a string that looks like a number) -** then it is done. Otherwise no conversion occurs. The -** [SQLITE_INTEGER | datatype] after conversion is returned. +** such a conversion is possible without loss of information (in other +** words, if the value is a string that looks like a number) +** then the conversion is performed. Otherwise no conversion occurs. +** The [SQLITE_INTEGER | datatype] after conversion is returned.)^ ** -** Please pay particular attention to the fact that the pointer that -** is returned from [sqlite3_value_blob()], [sqlite3_value_text()], or +** ^Within the [xUpdate] method of a [virtual table], the +** sqlite3_value_nochange(X) interface returns true if and only if +** the column corresponding to X is unchanged by the UPDATE operation +** that the xUpdate method call was invoked to implement and if +** and the prior [xColumn] method call that was invoked to extracted +** the value for that column returned without setting a result (probably +** because it queried [sqlite3_vtab_nochange()] and found that the column +** was unchanging). ^Within an [xUpdate] method, any value for which +** sqlite3_value_nochange(X) is true will in all other respects appear +** to be a NULL value. If sqlite3_value_nochange(X) is invoked anywhere other +** than within an [xUpdate] method call for an UPDATE statement, then +** the return value is arbitrary and meaningless. +** +** ^The sqlite3_value_frombind(X) interface returns non-zero if the +** value X originated from one of the [sqlite3_bind_int|sqlite3_bind()] +** interfaces. ^If X comes from an SQL literal value, or a table column, +** or an expression, then sqlite3_value_frombind(X) returns zero. +** +** Please pay particular attention to the fact that the pointer returned +** from [sqlite3_value_blob()], [sqlite3_value_text()], or ** [sqlite3_value_text16()] can be invalidated by a subsequent call to -** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite_value_text()], -** or [sqlite3_value_text16()]. +** [sqlite3_value_bytes()], [sqlite3_value_bytes16()], [sqlite3_value_text()], +** or [sqlite3_value_text16()]. +** +** These routines must be called from the same thread as +** the SQL function that supplied the [sqlite3_value*] parameters. +** +** As long as the input parameter is correct, these routines can only +** fail if an out-of-memory error occurs during a format conversion. +** Only the following subset of interfaces are subject to out-of-memory +** errors: +** +**
            +**
          • sqlite3_value_blob() +**
          • sqlite3_value_text() +**
          • sqlite3_value_text16() +**
          • sqlite3_value_text16le() +**
          • sqlite3_value_text16be() +**
          • sqlite3_value_bytes() +**
          • sqlite3_value_bytes16() +**
          +** +** If an out-of-memory error occurs, then the return value from these +** routines is the same as if the column had contained an SQL NULL value. +** Valid SQL NULL returns can be distinguished from out-of-memory errors +** by invoking the [sqlite3_errcode()] immediately after the suspect +** return value is obtained and before any +** other SQLite interface is called on the same [database connection]. */ -const void *sqlite3_value_blob(sqlite3_value*); -int sqlite3_value_bytes(sqlite3_value*); -int sqlite3_value_bytes16(sqlite3_value*); -double sqlite3_value_double(sqlite3_value*); -int sqlite3_value_int(sqlite3_value*); -sqlite_int64 sqlite3_value_int64(sqlite3_value*); -const unsigned char *sqlite3_value_text(sqlite3_value*); -const void *sqlite3_value_text16(sqlite3_value*); -const void *sqlite3_value_text16le(sqlite3_value*); -const void *sqlite3_value_text16be(sqlite3_value*); -int sqlite3_value_type(sqlite3_value*); -int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API const void *sqlite3_value_blob(sqlite3_value*); +SQLITE_API double sqlite3_value_double(sqlite3_value*); +SQLITE_API int sqlite3_value_int(sqlite3_value*); +SQLITE_API sqlite3_int64 sqlite3_value_int64(sqlite3_value*); +SQLITE_API void *sqlite3_value_pointer(sqlite3_value*, const char*); +SQLITE_API const unsigned char *sqlite3_value_text(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16le(sqlite3_value*); +SQLITE_API const void *sqlite3_value_text16be(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes(sqlite3_value*); +SQLITE_API int sqlite3_value_bytes16(sqlite3_value*); +SQLITE_API int sqlite3_value_type(sqlite3_value*); +SQLITE_API int sqlite3_value_numeric_type(sqlite3_value*); +SQLITE_API int sqlite3_value_nochange(sqlite3_value*); +SQLITE_API int sqlite3_value_frombind(sqlite3_value*); + +/* +** CAPI3REF: Report the internal text encoding state of an sqlite3_value object +** METHOD: sqlite3_value +** +** ^(The sqlite3_value_encoding(X) interface returns one of [SQLITE_UTF8], +** [SQLITE_UTF16BE], or [SQLITE_UTF16LE] according to the current text encoding +** of the value X, assuming that X has type TEXT.)^ If sqlite3_value_type(X) +** returns something other than SQLITE_TEXT, then the return value from +** sqlite3_value_encoding(X) is meaningless. ^Calls to +** [sqlite3_value_text(X)], [sqlite3_value_text16(X)], [sqlite3_value_text16be(X)], +** [sqlite3_value_text16le(X)], [sqlite3_value_bytes(X)], or +** [sqlite3_value_bytes16(X)] might change the encoding of the value X and +** thus change the return from subsequent calls to sqlite3_value_encoding(X). +** +** This routine is intended for used by applications that test and validate +** the SQLite implementation. This routine is inquiring about the opaque +** internal state of an [sqlite3_value] object. Ordinary applications should +** not need to know what the internal state of an sqlite3_value object is and +** hence should not need to use this interface. +*/ +SQLITE_API int sqlite3_value_encoding(sqlite3_value*); + +/* +** CAPI3REF: Finding The Subtype Of SQL Values +** METHOD: sqlite3_value +** +** The sqlite3_value_subtype(V) function returns the subtype for +** an [application-defined SQL function] argument V. The subtype +** information can be used to pass a limited amount of context from +** one SQL function to another. Use the [sqlite3_result_subtype()] +** routine to set the subtype for the return value of an SQL function. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_SUBTYPE] property in the text +** encoding argument when the function is [sqlite3_create_function|registered]. +** If the [SQLITE_SUBTYPE] property is omitted, then sqlite3_value_subtype() +** might return zero instead of the upstream subtype in some corner cases. +*/ +SQLITE_API unsigned int sqlite3_value_subtype(sqlite3_value*); + +/* +** CAPI3REF: Copy And Free SQL Values +** METHOD: sqlite3_value +** +** ^The sqlite3_value_dup(V) interface makes a copy of the [sqlite3_value] +** object D and returns a pointer to that copy. ^The [sqlite3_value] returned +** is a [protected sqlite3_value] object even if the input is not. +** ^The sqlite3_value_dup(V) interface returns NULL if V is NULL or if a +** memory allocation fails. ^If V is a [pointer value], then the result +** of sqlite3_value_dup(V) is a NULL value. +** +** ^The sqlite3_value_free(V) interface frees an [sqlite3_value] object +** previously obtained from [sqlite3_value_dup()]. ^If V is a NULL pointer +** then sqlite3_value_free(V) is a harmless no-op. +*/ +SQLITE_API sqlite3_value *sqlite3_value_dup(const sqlite3_value*); +SQLITE_API void sqlite3_value_free(sqlite3_value*); /* ** CAPI3REF: Obtain Aggregate Function Context +** METHOD: sqlite3_context ** -** The implementation of aggregate SQL functions use this routine to allocate -** a structure for storing their state. The first time this routine -** is called for a particular aggregate, a new structure of size nBytes -** is allocated, zeroed, and returned. On subsequent calls (for the -** same aggregate instance) the same buffer is returned. The implementation -** of the aggregate can use the returned buffer to accumulate data. +** Implementations of aggregate SQL functions use this +** routine to allocate memory for storing their state. ** -** The buffer allocated is freed automatically by SQLite whan the aggregate -** query concludes. +** ^The first time the sqlite3_aggregate_context(C,N) routine is called +** for a particular aggregate function, SQLite allocates +** N bytes of memory, zeroes out that memory, and returns a pointer +** to the new memory. ^On second and subsequent calls to +** sqlite3_aggregate_context() for the same aggregate function instance, +** the same buffer is returned. Sqlite3_aggregate_context() is normally +** called once for each invocation of the xStep callback and then one +** last time when the xFinal callback is invoked. ^(When no rows match +** an aggregate query, the xStep() callback of the aggregate function +** implementation is never called and xFinal() is called exactly once. +** In those cases, sqlite3_aggregate_context() might be called for the +** first time from within xFinal().)^ ** -** The first parameter should be a copy of the -** [sqlite3_context | SQL function context] that is the first -** parameter to the callback routine that implements the aggregate +** ^The sqlite3_aggregate_context(C,N) routine returns a NULL pointer +** when first called if N is less than or equal to zero or if a memory +** allocation error occurs. +** +** ^(The amount of space allocated by sqlite3_aggregate_context(C,N) is +** determined by the N parameter on first successful call. Changing the +** value of N in any subsequent call to sqlite3_aggregate_context() within +** the same aggregate function instance will not resize the memory +** allocation.)^ Within the xFinal callback, it is customary to set +** N=0 in calls to sqlite3_aggregate_context(C,N) so that no +** pointless memory allocations occur. +** +** ^SQLite automatically frees the memory allocated by +** sqlite3_aggregate_context() when the aggregate query concludes. +** +** The first parameter must be a copy of the +** [sqlite3_context | SQL function context] that is the first parameter +** to the xStep or xFinal callback routine that implements the aggregate ** function. +** +** This routine must be called from the same thread in which +** the aggregate SQL function is running. */ -void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); +SQLITE_API void *sqlite3_aggregate_context(sqlite3_context*, int nBytes); /* ** CAPI3REF: User Data For Functions +** METHOD: sqlite3_context ** -** The pUserData parameter to the [sqlite3_create_function()] -** and [sqlite3_create_function16()] routines -** used to register user functions is available to -** the implementation of the function using this call. +** ^The sqlite3_user_data() interface returns a copy of +** the pointer that was the pUserData parameter (the 5th parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +** +** This routine must be called from the same thread in which +** the application-defined function is running. */ -void *sqlite3_user_data(sqlite3_context*); +SQLITE_API void *sqlite3_user_data(sqlite3_context*); + +/* +** CAPI3REF: Database Connection For Functions +** METHOD: sqlite3_context +** +** ^The sqlite3_context_db_handle() interface returns a copy of +** the pointer to the [database connection] (the 1st parameter) +** of the [sqlite3_create_function()] +** and [sqlite3_create_function16()] routines that originally +** registered the application defined function. +*/ +SQLITE_API sqlite3 *sqlite3_context_db_handle(sqlite3_context*); /* ** CAPI3REF: Function Auxiliary Data +** METHOD: sqlite3_context ** -** The following two functions may be used by scalar SQL functions to -** associate meta-data with argument values. If the same value is passed to -** multiple invocations of the same SQL function during query execution, under -** some circumstances the associated meta-data may be preserved. This may -** be used, for example, to add a regular-expression matching scalar -** function. The compiled version of the regular expression is stored as -** meta-data associated with the SQL value passed as the regular expression -** pattern. The compiled regular expression can be reused on multiple -** invocations of the same function so that the original pattern string -** does not need to be recompiled on each invocation. +** These functions may be used by (non-aggregate) SQL functions to +** associate auxiliary data with argument values. If the same argument +** value is passed to multiple invocations of the same SQL function during +** query execution, under some circumstances the associated auxiliary data +** might be preserved. An example of where this might be useful is in a +** regular-expression matching function. The compiled version of the regular +** expression can be stored as auxiliary data associated with the pattern string. +** Then as long as the pattern string remains the same, +** the compiled regular expression can be reused on multiple +** invocations of the same function. ** -** The sqlite3_get_auxdata() interface returns a pointer to the meta-data -** associated with the Nth argument value to the current SQL function -** call, where N is the second parameter. If no meta-data has been set for -** that value, then a NULL pointer is returned. +** ^The sqlite3_get_auxdata(C,N) interface returns a pointer to the auxiliary data +** associated by the sqlite3_set_auxdata(C,N,P,X) function with the Nth argument +** value to the application-defined function. ^N is zero for the left-most +** function argument. ^If there is no auxiliary data +** associated with the function argument, the sqlite3_get_auxdata(C,N) interface +** returns a NULL pointer. ** -** The sqlite3_set_auxdata() is used to associate meta-data with an SQL -** function argument. The third parameter is a pointer to the meta-data -** to be associated with the Nth user function argument value. The fourth -** parameter specifies a destructor that will be called on the meta- -** data pointer to release it when it is no longer required. If the -** destructor is NULL, it is not invoked. +** ^The sqlite3_set_auxdata(C,N,P,X) interface saves P as auxiliary data for the +** N-th argument of the application-defined function. ^Subsequent +** calls to sqlite3_get_auxdata(C,N) return P from the most recent +** sqlite3_set_auxdata(C,N,P,X) call if the auxiliary data is still valid or +** NULL if the auxiliary data has been discarded. +** ^After each call to sqlite3_set_auxdata(C,N,P,X) where X is not NULL, +** SQLite will invoke the destructor function X with parameter P exactly +** once, when the auxiliary data is discarded. +** SQLite is free to discard the auxiliary data at any time, including:
            +**
          • ^(when the corresponding function parameter changes)^, or +**
          • ^(when [sqlite3_reset()] or [sqlite3_finalize()] is called for the +** SQL statement)^, or +**
          • ^(when sqlite3_set_auxdata() is invoked again on the same +** parameter)^, or +**
          • ^(during the original sqlite3_set_auxdata() call when a memory +** allocation error occurs.)^ +**
          • ^(during the original sqlite3_set_auxdata() call if the function +** is evaluated during query planning instead of during query execution, +** as sometimes happens with [SQLITE_ENABLE_STAT4].)^
          ** -** In practice, meta-data is preserved between function calls for -** expressions that are constant at compile time. This includes literal -** values and SQL variables. +** Note the last two bullets in particular. The destructor X in +** sqlite3_set_auxdata(C,N,P,X) might be called immediately, before the +** sqlite3_set_auxdata() interface even returns. Hence sqlite3_set_auxdata() +** should be called near the end of the function implementation and the +** function implementation should not make any use of P after +** sqlite3_set_auxdata() has been called. Furthermore, a call to +** sqlite3_get_auxdata() that occurs immediately after a corresponding call +** to sqlite3_set_auxdata() might still return NULL if an out-of-memory +** condition occurred during the sqlite3_set_auxdata() call or if the +** function is being evaluated during query planning rather than during +** query execution. +** +** ^(In practice, auxiliary data is preserved between function calls for +** function parameters that are compile-time constants, including literal +** values and [parameters] and expressions composed from the same.)^ +** +** The value of the N parameter to these interfaces should be non-negative. +** Future enhancements may make use of negative N values to define new +** kinds of function caching behavior. +** +** These routines must be called from the same thread in which +** the SQL function is running. +** +** See also: [sqlite3_get_clientdata()] and [sqlite3_set_clientdata()]. */ -void *sqlite3_get_auxdata(sqlite3_context*, int); -void sqlite3_set_auxdata(sqlite3_context*, int, void*, void (*)(void*)); +SQLITE_API void *sqlite3_get_auxdata(sqlite3_context*, int N); +SQLITE_API void sqlite3_set_auxdata(sqlite3_context*, int N, void*, void (*)(void*)); +/* +** CAPI3REF: Database Connection Client Data +** METHOD: sqlite3 +** +** These functions are used to associate one or more named pointers +** with a [database connection]. +** A call to sqlite3_set_clientdata(D,N,P,X) causes the pointer P +** to be attached to [database connection] D using name N. Subsequent +** calls to sqlite3_get_clientdata(D,N) will return a copy of pointer P +** or a NULL pointer if there were no prior calls to +** sqlite3_set_clientdata() with the same values of D and N. +** Names are compared using strcmp() and are thus case sensitive. +** +** If P and X are both non-NULL, then the destructor X is invoked with +** argument P on the first of the following occurrences: +**
            +**
          • An out-of-memory error occurs during the call to +** sqlite3_set_clientdata() which attempts to register pointer P. +**
          • A subsequent call to sqlite3_set_clientdata(D,N,P,X) is made +** with the same D and N parameters. +**
          • The database connection closes. SQLite does not make any guarantees +** about the order in which destructors are called, only that all +** destructors will be called exactly once at some point during the +** database connection closing process. +**
          +** +** SQLite does not do anything with client data other than invoke +** destructors on the client data at the appropriate time. The intended +** use for client data is to provide a mechanism for wrapper libraries +** to store additional information about an SQLite database connection. +** +** There is no limit (other than available memory) on the number of different +** client data pointers (with different names) that can be attached to a +** single database connection. However, the implementation is optimized +** for the case of having only one or two different client data names. +** Applications and wrapper libraries are discouraged from using more than +** one client data name each. +** +** There is no way to enumerate the client data pointers +** associated with a database connection. The N parameter can be thought +** of as a secret key such that only code that knows the secret key is able +** to access the associated data. +** +** Security Warning: These interfaces should not be exposed in scripting +** languages or in other circumstances where it might be possible for an +** an attacker to invoke them. Any agent that can invoke these interfaces +** can probably also take control of the process. +** +** Database connection client data is only available for SQLite +** version 3.44.0 ([dateof:3.44.0]) and later. +** +** See also: [sqlite3_set_auxdata()] and [sqlite3_get_auxdata()]. +*/ +SQLITE_API void *sqlite3_get_clientdata(sqlite3*,const char*); +SQLITE_API int sqlite3_set_clientdata(sqlite3*, const char*, void*, void(*)(void*)); /* ** CAPI3REF: Constants Defining Special Destructor Behavior ** -** These are special value for the destructor that is passed in as the -** final argument to routines like [sqlite3_result_blob()]. If the destructor +** These are special values for the destructor that is passed in as the +** final argument to routines like [sqlite3_result_blob()]. ^If the destructor ** argument is SQLITE_STATIC, it means that the content pointer is constant -** and will never change. It does not need to be destroyed. The +** and will never change. It does not need to be destroyed. ^The ** SQLITE_TRANSIENT value means that the content will likely change in ** the near future and that SQLite should make its own private copy of ** the content before returning. ** ** The typedef is necessary to work around problems in certain -** C++ compilers. See ticket #2191. +** C++ compilers. */ typedef void (*sqlite3_destructor_type)(void*); #define SQLITE_STATIC ((sqlite3_destructor_type)0) @@ -1821,424 +6087,1067 @@ typedef void (*sqlite3_destructor_type)(void*); /* ** CAPI3REF: Setting The Result Of An SQL Function +** METHOD: sqlite3_context ** ** These routines are used by the xFunc or xFinal callbacks that ** implement SQL functions and aggregates. See ** [sqlite3_create_function()] and [sqlite3_create_function16()] ** for additional information. ** -** These functions work very much like the -** [sqlite3_bind_blob | sqlite3_bind_*] family of functions used -** to bind values to host parameters in prepared statements. -** Refer to the -** [sqlite3_bind_blob | sqlite3_bind_* documentation] for -** additional information. +** These functions work very much like the [parameter binding] family of +** functions used to bind values to host parameters in prepared statements. +** Refer to the [SQL parameter] documentation for additional information. ** -** The sqlite3_result_error() and sqlite3_result_error16() functions -** cause the implemented SQL function to throw an exception. The -** parameter to sqlite3_result_error() or sqlite3_result_error16() -** is the text of an error message. +** ^The sqlite3_result_blob() interface sets the result from +** an application-defined function to be the BLOB whose content is pointed +** to by the second parameter and which is N bytes long where N is the +** third parameter. ** -** The sqlite3_result_toobig() cause the function implementation -** to throw and error indicating that a string or BLOB is to long -** to represent. +** ^The sqlite3_result_zeroblob(C,N) and sqlite3_result_zeroblob64(C,N) +** interfaces set the result of the application-defined function to be +** a BLOB containing all zero bytes and N bytes in size. +** +** ^The sqlite3_result_double() interface sets the result from +** an application-defined function to be a floating point value specified +** by its 2nd argument. +** +** ^The sqlite3_result_error() and sqlite3_result_error16() functions +** cause the implemented SQL function to throw an exception. +** ^SQLite uses the string pointed to by the +** 2nd parameter of sqlite3_result_error() or sqlite3_result_error16() +** as the text of an error message. ^SQLite interprets the error +** message string from sqlite3_result_error() as UTF-8. ^SQLite +** interprets the string from sqlite3_result_error16() as UTF-16 using +** the same [byte-order determination rules] as [sqlite3_bind_text16()]. +** ^If the third parameter to sqlite3_result_error() +** or sqlite3_result_error16() is negative then SQLite takes as the error +** message all text up through the first zero character. +** ^If the third parameter to sqlite3_result_error() or +** sqlite3_result_error16() is non-negative then SQLite takes that many +** bytes (not characters) from the 2nd parameter as the error message. +** ^The sqlite3_result_error() and sqlite3_result_error16() +** routines make a private copy of the error message text before +** they return. Hence, the calling function can deallocate or +** modify the text after they return without harm. +** ^The sqlite3_result_error_code() function changes the error code +** returned by SQLite as a result of an error in a function. ^By default, +** the error code is SQLITE_ERROR. ^A subsequent call to sqlite3_result_error() +** or sqlite3_result_error16() resets the error code to SQLITE_ERROR. +** +** ^The sqlite3_result_error_toobig() interface causes SQLite to throw an +** error indicating that a string or BLOB is too long to represent. +** +** ^The sqlite3_result_error_nomem() interface causes SQLite to throw an +** error indicating that a memory allocation failed. +** +** ^The sqlite3_result_int() interface sets the return value +** of the application-defined function to be the 32-bit signed integer +** value given in the 2nd argument. +** ^The sqlite3_result_int64() interface sets the return value +** of the application-defined function to be the 64-bit signed integer +** value given in the 2nd argument. +** +** ^The sqlite3_result_null() interface sets the return value +** of the application-defined function to be NULL. +** +** ^The sqlite3_result_text(), sqlite3_result_text16(), +** sqlite3_result_text16le(), and sqlite3_result_text16be() interfaces +** set the return value of the application-defined function to be +** a text string which is represented as UTF-8, UTF-16 native byte order, +** UTF-16 little endian, or UTF-16 big endian, respectively. +** ^The sqlite3_result_text64() interface sets the return value of an +** application-defined function to be a text string in an encoding +** specified by the fifth (and last) parameter, which must be one +** of [SQLITE_UTF8], [SQLITE_UTF16], [SQLITE_UTF16BE], or [SQLITE_UTF16LE]. +** ^SQLite takes the text result from the application from +** the 2nd parameter of the sqlite3_result_text* interfaces. +** ^If the 3rd parameter to any of the sqlite3_result_text* interfaces +** other than sqlite3_result_text64() is negative, then SQLite computes +** the string length itself by searching the 2nd parameter for the first +** zero character. +** ^If the 3rd parameter to the sqlite3_result_text* interfaces +** is non-negative, then as many bytes (not characters) of the text +** pointed to by the 2nd parameter are taken as the application-defined +** function result. If the 3rd parameter is non-negative, then it +** must be the byte offset into the string where the NUL terminator would +** appear if the string where NUL terminated. If any NUL characters occur +** in the string at a byte offset that is less than the value of the 3rd +** parameter, then the resulting string will contain embedded NULs and the +** result of expressions operating on strings with embedded NULs is undefined. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is a non-NULL pointer, then SQLite calls that +** function as the destructor on the text or BLOB result when it has +** finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces or to +** sqlite3_result_blob is the special constant SQLITE_STATIC, then SQLite +** assumes that the text or BLOB result is in constant space and does not +** copy the content of the parameter nor call a destructor on the content +** when it has finished using that result. +** ^If the 4th parameter to the sqlite3_result_text* interfaces +** or sqlite3_result_blob is the special constant SQLITE_TRANSIENT +** then SQLite makes a copy of the result into space obtained +** from [sqlite3_malloc()] before it returns. +** +** ^For the sqlite3_result_text16(), sqlite3_result_text16le(), and +** sqlite3_result_text16be() routines, and for sqlite3_result_text64() +** when the encoding is not UTF8, if the input UTF16 begins with a +** byte-order mark (BOM, U+FEFF) then the BOM is removed from the +** string and the rest of the string is interpreted according to the +** byte-order specified by the BOM. ^The byte-order specified by +** the BOM at the beginning of the text overrides the byte-order +** specified by the interface procedure. ^So, for example, if +** sqlite3_result_text16le() is invoked with text that begins +** with bytes 0xfe, 0xff (a big-endian byte-order mark) then the +** first two bytes of input are skipped and the remaining input +** is interpreted as UTF16BE text. +** +** ^For UTF16 input text to the sqlite3_result_text16(), +** sqlite3_result_text16be(), sqlite3_result_text16le(), and +** sqlite3_result_text64() routines, if the text contains invalid +** UTF16 characters, the invalid characters might be converted +** into the unicode replacement character, U+FFFD. +** +** ^The sqlite3_result_value() interface sets the result of +** the application-defined function to be a copy of the +** [unprotected sqlite3_value] object specified by the 2nd parameter. ^The +** sqlite3_result_value() interface makes a copy of the [sqlite3_value] +** so that the [sqlite3_value] specified in the parameter may change or +** be deallocated after sqlite3_result_value() returns without harm. +** ^A [protected sqlite3_value] object may always be used where an +** [unprotected sqlite3_value] object is required, so either +** kind of [sqlite3_value] object can be used with this interface. +** +** ^The sqlite3_result_pointer(C,P,T,D) interface sets the result to an +** SQL NULL value, just like [sqlite3_result_null(C)], except that it +** also associates the host-language pointer P or type T with that +** NULL value such that the pointer can be retrieved within an +** [application-defined SQL function] using [sqlite3_value_pointer()]. +** ^If the D parameter is not NULL, then it is a pointer to a destructor +** for the P parameter. ^SQLite invokes D with P as its only argument +** when SQLite is finished with P. The T parameter should be a static +** string and preferably a string literal. The sqlite3_result_pointer() +** routine is part of the [pointer passing interface] added for SQLite 3.20.0. +** +** If these routines are called from within the different thread +** than the one containing the application-defined function that received +** the [sqlite3_context] pointer, the results are undefined. */ -void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); -void sqlite3_result_double(sqlite3_context*, double); -void sqlite3_result_error(sqlite3_context*, const char*, int); -void sqlite3_result_error16(sqlite3_context*, const void*, int); -void sqlite3_result_error_toobig(sqlite3_context*); -void sqlite3_result_int(sqlite3_context*, int); -void sqlite3_result_int64(sqlite3_context*, sqlite_int64); -void sqlite3_result_null(sqlite3_context*); -void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); -void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); -void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); -void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); -void sqlite3_result_value(sqlite3_context*, sqlite3_value*); -void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API void sqlite3_result_blob(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_blob64(sqlite3_context*,const void*, + sqlite3_uint64,void(*)(void*)); +SQLITE_API void sqlite3_result_double(sqlite3_context*, double); +SQLITE_API void sqlite3_result_error(sqlite3_context*, const char*, int); +SQLITE_API void sqlite3_result_error16(sqlite3_context*, const void*, int); +SQLITE_API void sqlite3_result_error_toobig(sqlite3_context*); +SQLITE_API void sqlite3_result_error_nomem(sqlite3_context*); +SQLITE_API void sqlite3_result_error_code(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int(sqlite3_context*, int); +SQLITE_API void sqlite3_result_int64(sqlite3_context*, sqlite3_int64); +SQLITE_API void sqlite3_result_null(sqlite3_context*); +SQLITE_API void sqlite3_result_text(sqlite3_context*, const char*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text64(sqlite3_context*, const char*,sqlite3_uint64, + void(*)(void*), unsigned char encoding); +SQLITE_API void sqlite3_result_text16(sqlite3_context*, const void*, int, void(*)(void*)); +SQLITE_API void sqlite3_result_text16le(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_text16be(sqlite3_context*, const void*, int,void(*)(void*)); +SQLITE_API void sqlite3_result_value(sqlite3_context*, sqlite3_value*); +SQLITE_API void sqlite3_result_pointer(sqlite3_context*, void*,const char*,void(*)(void*)); +SQLITE_API void sqlite3_result_zeroblob(sqlite3_context*, int n); +SQLITE_API int sqlite3_result_zeroblob64(sqlite3_context*, sqlite3_uint64 n); + + +/* +** CAPI3REF: Setting The Subtype Of An SQL Function +** METHOD: sqlite3_context +** +** The sqlite3_result_subtype(C,T) function causes the subtype of +** the result from the [application-defined SQL function] with +** [sqlite3_context] C to be the value T. Only the lower 8 bits +** of the subtype T are preserved in current versions of SQLite; +** higher order bits are discarded. +** The number of subtype bytes preserved by SQLite might increase +** in future releases of SQLite. +** +** Every [application-defined SQL function] that invokes this interface +** should include the [SQLITE_RESULT_SUBTYPE] property in its +** text encoding argument when the SQL function is +** [sqlite3_create_function|registered]. If the [SQLITE_RESULT_SUBTYPE] +** property is omitted from the function that invokes sqlite3_result_subtype(), +** then in some cases the sqlite3_result_subtype() might fail to set +** the result subtype. +** +** If SQLite is compiled with -DSQLITE_STRICT_SUBTYPE=1, then any +** SQL function that invokes the sqlite3_result_subtype() interface +** and that does not have the SQLITE_RESULT_SUBTYPE property will raise +** an error. Future versions of SQLite might enable -DSQLITE_STRICT_SUBTYPE=1 +** by default. +*/ +SQLITE_API void sqlite3_result_subtype(sqlite3_context*,unsigned int); /* ** CAPI3REF: Define New Collating Sequences +** METHOD: sqlite3 ** -** These functions are used to add new collation sequences to the -** [sqlite3*] handle specified as the first argument. +** ^These functions add, remove, or modify a [collation] associated +** with the [database connection] specified as the first argument. ** -** The name of the new collation sequence is specified as a UTF-8 string +** ^The name of the collation is a UTF-8 string ** for sqlite3_create_collation() and sqlite3_create_collation_v2() -** and a UTF-16 string for sqlite3_create_collation16(). In all cases -** the name is passed as the second function argument. +** and a UTF-16 string in native byte order for sqlite3_create_collation16(). +** ^Collation names that compare equal according to [sqlite3_strnicmp()] are +** considered to be the same name. ** -** The third argument must be one of the constants [SQLITE_UTF8], -** [SQLITE_UTF16LE] or [SQLITE_UTF16BE], indicating that the user-supplied -** routine expects to be passed pointers to strings encoded using UTF-8, -** UTF-16 little-endian or UTF-16 big-endian respectively. +** ^(The third argument (eTextRep) must be one of the constants: +**
            +**
          • [SQLITE_UTF8], +**
          • [SQLITE_UTF16LE], +**
          • [SQLITE_UTF16BE], +**
          • [SQLITE_UTF16], or +**
          • [SQLITE_UTF16_ALIGNED]. +**
          )^ +** ^The eTextRep argument determines the encoding of strings passed +** to the collating function callback, xCompare. +** ^The [SQLITE_UTF16] and [SQLITE_UTF16_ALIGNED] values for eTextRep +** force strings to be UTF16 with native byte order. +** ^The [SQLITE_UTF16_ALIGNED] value for eTextRep forces strings to begin +** on an even byte address. ** -** A pointer to the user supplied routine must be passed as the fifth -** argument. If it is NULL, this is the same as deleting the collation -** sequence (so that SQLite cannot call it anymore). Each time the user -** supplied function is invoked, it is passed a copy of the void* passed as -** the fourth argument to sqlite3_create_collation() or -** sqlite3_create_collation16() as its first parameter. +** ^The fourth argument, pArg, is an application data pointer that is passed +** through as the first argument to the collating function callback. ** -** The remaining arguments to the user-supplied routine are two strings, -** each represented by a [length, data] pair and encoded in the encoding -** that was passed as the third argument when the collation sequence was -** registered. The user routine should return negative, zero or positive if -** the first string is less than, equal to, or greater than the second -** string. i.e. (STRING1 - STRING2). +** ^The fifth argument, xCompare, is a pointer to the collating function. +** ^Multiple collating functions can be registered using the same name but +** with different eTextRep parameters and SQLite will use whichever +** function requires the least amount of data transformation. +** ^If the xCompare argument is NULL then the collating function is +** deleted. ^When all collating functions having the same name are deleted, +** that collation is no longer usable. ** -** The sqlite3_create_collation_v2() works like sqlite3_create_collation() -** excapt that it takes an extra argument which is a destructor for -** the collation. The destructor is called when the collation is -** destroyed and is passed a copy of the fourth parameter void* pointer -** of the sqlite3_create_collation_v2(). Collations are destroyed when -** they are overridden by later calls to the collation creation functions -** or when the [sqlite3*] database handle is closed using [sqlite3_close()]. +** ^The collating function callback is invoked with a copy of the pArg +** application data pointer and with two strings in the encoding specified +** by the eTextRep argument. The two integer parameters to the collating +** function callback are the length of the two strings, in bytes. The collating +** function must return an integer that is negative, zero, or positive +** if the first string is less than, equal to, or greater than the second, +** respectively. A collating function must always return the same answer +** given the same inputs. If two or more collating functions are registered +** to the same collation name (using different eTextRep values) then all +** must give an equivalent answer when invoked with equivalent strings. +** The collating function must obey the following properties for all +** strings A, B, and C: ** -** The sqlite3_create_collation_v2() interface is experimental and -** subject to change in future releases. The other collation creation -** functions are stable. +**
            +**
          1. If A==B then B==A. +**
          2. If A==B and B==C then A==C. +**
          3. If A<B THEN B>A. +**
          4. If A<B and B<C then A<C. +**
          +** +** If a collating function fails any of the above constraints and that +** collating function is registered and used, then the behavior of SQLite +** is undefined. +** +** ^The sqlite3_create_collation_v2() works like sqlite3_create_collation() +** with the addition that the xDestroy callback is invoked on pArg when +** the collating function is deleted. +** ^Collating functions are deleted when they are overridden by later +** calls to the collation creation functions or when the +** [database connection] is closed using [sqlite3_close()]. +** +** ^The xDestroy callback is not called if the +** sqlite3_create_collation_v2() function fails. Applications that invoke +** sqlite3_create_collation_v2() with a non-NULL xDestroy argument should +** check the return code and dispose of the application data pointer +** themselves rather than expecting SQLite to deal with it for them. +** This is different from every other SQLite interface. The inconsistency +** is unfortunate but cannot be changed without breaking backwards +** compatibility. +** +** See also: [sqlite3_collation_needed()] and [sqlite3_collation_needed16()]. */ -int sqlite3_create_collation( - sqlite3*, - const char *zName, - int eTextRep, - void*, +SQLITE_API int sqlite3_create_collation( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); -int sqlite3_create_collation_v2( - sqlite3*, - const char *zName, - int eTextRep, - void*, +SQLITE_API int sqlite3_create_collation_v2( + sqlite3*, + const char *zName, + int eTextRep, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*), void(*xDestroy)(void*) ); -int sqlite3_create_collation16( - sqlite3*, - const char *zName, - int eTextRep, - void*, +SQLITE_API int sqlite3_create_collation16( + sqlite3*, + const void *zName, + int eTextRep, + void *pArg, int(*xCompare)(void*,int,const void*,int,const void*) ); /* ** CAPI3REF: Collation Needed Callbacks +** METHOD: sqlite3 ** -** To avoid having to register all collation sequences before a database +** ^To avoid having to register all collation sequences before a database ** can be used, a single callback function may be registered with the -** database handle to be called whenever an undefined collation sequence is -** required. +** [database connection] to be invoked whenever an undefined collation +** sequence is required. ** -** If the function is registered using the sqlite3_collation_needed() API, +** ^If the function is registered using the sqlite3_collation_needed() API, ** then it is passed the names of undefined collation sequences as strings -** encoded in UTF-8. If sqlite3_collation_needed16() is used, the names -** are passed as UTF-16 in machine native byte order. A call to either -** function replaces any existing callback. +** encoded in UTF-8. ^If sqlite3_collation_needed16() is used, +** the names are passed as UTF-16 in machine native byte order. +** ^A call to either function replaces the existing collation-needed callback. ** -** When the callback is invoked, the first argument passed is a copy +** ^(When the callback is invoked, the first argument passed is a copy ** of the second argument to sqlite3_collation_needed() or -** sqlite3_collation_needed16(). The second argument is the database -** handle. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], or -** [SQLITE_UTF16LE], indicating the most desirable form of the collation -** sequence function required. The fourth parameter is the name of the -** required collation sequence. +** sqlite3_collation_needed16(). The second argument is the database +** connection. The third argument is one of [SQLITE_UTF8], [SQLITE_UTF16BE], +** or [SQLITE_UTF16LE], indicating the most desirable form of the collation +** sequence function required. The fourth parameter is the name of the +** required collation sequence.)^ ** ** The callback function should register the desired collation using ** [sqlite3_create_collation()], [sqlite3_create_collation16()], or ** [sqlite3_create_collation_v2()]. */ -int sqlite3_collation_needed( - sqlite3*, - void*, +SQLITE_API int sqlite3_collation_needed( + sqlite3*, + void*, void(*)(void*,sqlite3*,int eTextRep,const char*) ); -int sqlite3_collation_needed16( - sqlite3*, +SQLITE_API int sqlite3_collation_needed16( + sqlite3*, void*, void(*)(void*,sqlite3*,int eTextRep,const void*) ); +#ifdef SQLITE_ENABLE_CEROD /* -** Specify the key for an encrypted database. This routine should be -** called right after sqlite3_open(). -** -** The code to implement this API is not available in the public release -** of SQLite. +** Specify the activation key for a CEROD database. Unless +** activated, none of the CEROD routines will work. */ -int sqlite3_key( - sqlite3 *db, /* Database to be rekeyed */ - const void *pKey, int nKey /* The key */ +SQLITE_API void sqlite3_activate_cerod( + const char *zPassPhrase /* Activation phrase */ ); +#endif /* -** Change the key on an open database. If the current database is not -** encrypted, this routine will encrypt it. If pNew==0 or nNew==0, the -** database is decrypted. +** CAPI3REF: Suspend Execution For A Short Time ** -** The code to implement this API is not available in the public release -** of SQLite. -*/ -int sqlite3_rekey( - sqlite3 *db, /* Database to be rekeyed */ - const void *pKey, int nKey /* The new key */ -); - -/* -** CAPI3REF: Suspend Execution For A Short Time +** The sqlite3_sleep() function causes the current thread to suspend execution +** for at least a number of milliseconds specified in its parameter. ** -** This function causes the current thread to suspect execution -** a number of milliseconds specified in its parameter. -** -** If the operating system does not support sleep requests with -** millisecond time resolution, then the time will be rounded up to -** the nearest second. The number of milliseconds of sleep actually +** If the operating system does not support sleep requests with +** millisecond time resolution, then the time will be rounded up to +** the nearest second. The number of milliseconds of sleep actually ** requested from the operating system is returned. +** +** ^SQLite implements this interface by calling the xSleep() +** method of the default [sqlite3_vfs] object. If the xSleep() method +** of the default VFS is not implemented correctly, or not implemented at +** all, then the behavior of sqlite3_sleep() may deviate from the description +** in the previous paragraphs. +** +** If a negative argument is passed to sqlite3_sleep() the results vary by +** VFS and operating system. Some system treat a negative argument as an +** instruction to sleep forever. Others understand it to mean do not sleep +** at all. ^In SQLite version 3.42.0 and later, a negative +** argument passed into sqlite3_sleep() is changed to zero before it is relayed +** down into the xSleep method of the VFS. */ -int sqlite3_sleep(int); +SQLITE_API int sqlite3_sleep(int); /* -** CAPI3REF: Name Of The Folder Holding Temporary Files +** CAPI3REF: Name Of The Folder Holding Temporary Files ** -** If this global variable is made to point to a string which is -** the name of a folder (a.ka. directory), then all temporary files -** created by SQLite will be placed in that directory. If this variable -** is NULL pointer, then SQLite does a search for an appropriate temporary -** file directory. +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all temporary files +** created by SQLite when using a built-in [sqlite3_vfs | VFS] +** will be placed in that directory.)^ ^If this variable +** is a NULL pointer, then SQLite performs a search for an appropriate +** temporary file directory. ** -** Once [sqlite3_open()] has been called, changing this variable will -** invalidate the current temporary database, if any. Generally speaking, -** it is not safe to invoke this routine after [sqlite3_open()] has -** been called. +** Applications are strongly discouraged from using this global variable. +** It is required to set a temporary folder on Windows Runtime (WinRT). +** But for all other platforms, it is highly recommended that applications +** neither read nor write this variable. This global variable is a relic +** that exists for backwards compatibility of legacy applications and should +** be avoided in new projects. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [temp_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [temp_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [temp_store_directory pragma] should be avoided. +** Except when requested by the [temp_store_directory pragma], SQLite +** does not free the memory that sqlite3_temp_directory points to. If +** the application wants that memory to be freed, it must do +** so itself, taking care to only do so after all [database connection] +** objects have been destroyed. +** +** Note to Windows Runtime users: The temporary directory must be set +** prior to calling [sqlite3_open] or [sqlite3_open_v2]. Otherwise, various +** features that require the use of temporary files may fail. Here is an +** example of how to do this using C++ with the Windows Runtime: +** +** 
          +** LPCWSTR zPath = Windows::Storage::ApplicationData::Current->
+**       TemporaryFolder->Path->Data();
+** char zPathBuf[MAX_PATH + 1];
+** memset(zPathBuf, 0, sizeof(zPathBuf));
+** WideCharToMultiByte(CP_UTF8, 0, zPath, -1, zPathBuf, sizeof(zPathBuf),
+**       NULL, NULL);
+** sqlite3_temp_directory = sqlite3_mprintf("%s", zPathBuf);
+**
          */ -extern char *sqlite3_temp_directory; +SQLITE_API SQLITE_EXTERN char *sqlite3_temp_directory; /* -** CAPI3REF: Test To See If The Databse Is In Auto-Commit Mode +** CAPI3REF: Name Of The Folder Holding Database Files ** -** Test to see whether or not the database connection is in autocommit -** mode. Return TRUE if it is and FALSE if not. Autocommit mode is on -** by default. Autocommit is disabled by a BEGIN statement and reenabled -** by the next COMMIT or ROLLBACK. +** ^(If this global variable is made to point to a string which is +** the name of a folder (a.k.a. directory), then all database files +** specified with a relative pathname and created or accessed by +** SQLite when using a built-in windows [sqlite3_vfs | VFS] will be assumed +** to be relative to that directory.)^ ^If this variable is a NULL +** pointer, then SQLite assumes that all database files specified +** with a relative pathname are relative to the current directory +** for the process. Only the windows VFS makes use of this global +** variable; it is ignored by the unix VFS. +** +** Changing the value of this variable while a database connection is +** open can result in a corrupt database. +** +** It is not safe to read or modify this variable in more than one +** thread at a time. It is not safe to read or modify this variable +** if a [database connection] is being used at the same time in a separate +** thread. +** It is intended that this variable be set once +** as part of process initialization and before any SQLite interface +** routines have been called and that this variable remain unchanged +** thereafter. +** +** ^The [data_store_directory pragma] may modify this variable and cause +** it to point to memory obtained from [sqlite3_malloc]. ^Furthermore, +** the [data_store_directory pragma] always assumes that any string +** that this variable points to is held in memory obtained from +** [sqlite3_malloc] and the pragma may attempt to free that memory +** using [sqlite3_free]. +** Hence, if this variable is modified directly, either it should be +** made NULL or made to point to memory obtained from [sqlite3_malloc] +** or else the use of the [data_store_directory pragma] should be avoided. */ -int sqlite3_get_autocommit(sqlite3*); +SQLITE_API SQLITE_EXTERN char *sqlite3_data_directory; /* -** CAPI3REF: Find The Database Handle Associated With A Prepared Statement +** CAPI3REF: Win32 Specific Interface ** -** Return the [sqlite3*] database handle to which a -** [sqlite3_stmt | prepared statement] belongs. -** This is the same database handle that was -** the first argument to the [sqlite3_prepare_v2()] or its variants -** that was used to create the statement in the first place. +** These interfaces are available only on Windows. The +** [sqlite3_win32_set_directory] interface is used to set the value associated +** with the [sqlite3_temp_directory] or [sqlite3_data_directory] variable, to +** zValue, depending on the value of the type parameter. The zValue parameter +** should be NULL to cause the previous value to be freed via [sqlite3_free]; +** a non-NULL value will be copied into memory obtained from [sqlite3_malloc] +** prior to being used. The [sqlite3_win32_set_directory] interface returns +** [SQLITE_OK] to indicate success, [SQLITE_ERROR] if the type is unsupported, +** or [SQLITE_NOMEM] if memory could not be allocated. The value of the +** [sqlite3_data_directory] variable is intended to act as a replacement for +** the current directory on the sub-platforms of Win32 where that concept is +** not present, e.g. WinRT and UWP. The [sqlite3_win32_set_directory8] and +** [sqlite3_win32_set_directory16] interfaces behave exactly the same as the +** sqlite3_win32_set_directory interface except the string parameter must be +** UTF-8 or UTF-16, respectively. */ -sqlite3 *sqlite3_db_handle(sqlite3_stmt*); +SQLITE_API int sqlite3_win32_set_directory( + unsigned long type, /* Identifier for directory being set or reset */ + void *zValue /* New value for directory being set or reset */ +); +SQLITE_API int sqlite3_win32_set_directory8(unsigned long type, const char *zValue); +SQLITE_API int sqlite3_win32_set_directory16(unsigned long type, const void *zValue); +/* +** CAPI3REF: Win32 Directory Types +** +** These macros are only available on Windows. They define the allowed values +** for the type argument to the [sqlite3_win32_set_directory] interface. +*/ +#define SQLITE_WIN32_DATA_DIRECTORY_TYPE 1 +#define SQLITE_WIN32_TEMP_DIRECTORY_TYPE 2 + +/* +** CAPI3REF: Test For Auto-Commit Mode +** KEYWORDS: {autocommit mode} +** METHOD: sqlite3 +** +** ^The sqlite3_get_autocommit() interface returns non-zero or +** zero if the given database connection is or is not in autocommit mode, +** respectively. ^Autocommit mode is on by default. +** ^Autocommit mode is disabled by a [BEGIN] statement. +** ^Autocommit mode is re-enabled by a [COMMIT] or [ROLLBACK]. +** +** If certain kinds of errors occur on a statement within a multi-statement +** transaction (errors including [SQLITE_FULL], [SQLITE_IOERR], +** [SQLITE_NOMEM], [SQLITE_BUSY], and [SQLITE_INTERRUPT]) then the +** transaction might be rolled back automatically. The only way to +** find out whether SQLite automatically rolled back the transaction after +** an error is to use this function. +** +** If another thread changes the autocommit status of the database +** connection while this routine is running, then the return value +** is undefined. +*/ +SQLITE_API int sqlite3_get_autocommit(sqlite3*); + +/* +** CAPI3REF: Find The Database Handle Of A Prepared Statement +** METHOD: sqlite3_stmt +** +** ^The sqlite3_db_handle interface returns the [database connection] handle +** to which a [prepared statement] belongs. ^The [database connection] +** returned by sqlite3_db_handle is the same [database connection] +** that was the first argument +** to the [sqlite3_prepare_v2()] call (or its variants) that was used to +** create the statement in the first place. +*/ +SQLITE_API sqlite3 *sqlite3_db_handle(sqlite3_stmt*); + +/* +** CAPI3REF: Return The Schema Name For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_name(D,N) interface returns a pointer to the schema name +** for the N-th database on database connection D, or a NULL pointer of N is +** out of range. An N value of 0 means the main database file. An N of 1 is +** the "temp" schema. Larger values of N correspond to various ATTACH-ed +** databases. +** +** Space to hold the string that is returned by sqlite3_db_name() is managed +** by SQLite itself. The string might be deallocated by any operation that +** changes the schema, including [ATTACH] or [DETACH] or calls to +** [sqlite3_serialize()] or [sqlite3_deserialize()], even operations that +** occur on a different thread. Applications that need to +** remember the string long-term should make their own copy. Applications that +** are accessing the same database connection simultaneously on multiple +** threads should mutex-protect calls to this API and should make their own +** private copy of the result prior to releasing the mutex. +*/ +SQLITE_API const char *sqlite3_db_name(sqlite3 *db, int N); + +/* +** CAPI3REF: Return The Filename For A Database Connection +** METHOD: sqlite3 +** +** ^The sqlite3_db_filename(D,N) interface returns a pointer to the filename +** associated with database N of connection D. +** ^If there is no attached database N on the database +** connection D, or if database N is a temporary or in-memory database, then +** this function will return either a NULL pointer or an empty string. +** +** ^The string value returned by this routine is owned and managed by +** the database connection. ^The value will be valid until the database N +** is [DETACH]-ed or until the database connection closes. +** +** ^The filename returned by this function is the output of the +** xFullPathname method of the [VFS]. ^In other words, the filename +** will be an absolute pathname, even if the filename used +** to open the database originally was a URI or relative pathname. +** +** If the filename pointer returned by this routine is not NULL, then it +** can be used as the filename input parameter to these routines: +**
            +**
          • [sqlite3_uri_parameter()] +**
          • [sqlite3_uri_boolean()] +**
          • [sqlite3_uri_int64()] +**
          • [sqlite3_filename_database()] +**
          • [sqlite3_filename_journal()] +**
          • [sqlite3_filename_wal()] +**
          +*/ +SQLITE_API sqlite3_filename sqlite3_db_filename(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine if a database is read-only +** METHOD: sqlite3 +** +** ^The sqlite3_db_readonly(D,N) interface returns 1 if the database N +** of connection D is read-only, 0 if it is read/write, or -1 if N is not +** the name of a database on connection D. +*/ +SQLITE_API int sqlite3_db_readonly(sqlite3 *db, const char *zDbName); + +/* +** CAPI3REF: Determine the transaction state of a database +** METHOD: sqlite3 +** +** ^The sqlite3_txn_state(D,S) interface returns the current +** [transaction state] of schema S in database connection D. ^If S is NULL, +** then the highest transaction state of any schema on database connection D +** is returned. Transaction states are (in order of lowest to highest): +**
            +**
          1. SQLITE_TXN_NONE +**
          2. SQLITE_TXN_READ +**
          3. SQLITE_TXN_WRITE +**
          +** ^If the S argument to sqlite3_txn_state(D,S) is not the name of +** a valid schema, then -1 is returned. +*/ +SQLITE_API int sqlite3_txn_state(sqlite3*,const char *zSchema); + +/* +** CAPI3REF: Allowed return values from sqlite3_txn_state() +** KEYWORDS: {transaction state} +** +** These constants define the current transaction state of a database file. +** ^The [sqlite3_txn_state(D,S)] interface returns one of these +** constants in order to describe the transaction state of schema S +** in [database connection] D. +** +**
          +** [[SQLITE_TXN_NONE]]
          SQLITE_TXN_NONE
          +**
          The SQLITE_TXN_NONE state means that no transaction is currently +** pending.
          +** +** [[SQLITE_TXN_READ]]
          SQLITE_TXN_READ
          +**
          The SQLITE_TXN_READ state means that the database is currently +** in a read transaction. Content has been read from the database file +** but nothing in the database file has changed. The transaction state +** will advanced to SQLITE_TXN_WRITE if any changes occur and there are +** no other conflicting concurrent write transactions. The transaction +** state will revert to SQLITE_TXN_NONE following a [ROLLBACK] or +** [COMMIT].
          +** +** [[SQLITE_TXN_WRITE]]
          SQLITE_TXN_WRITE
          +**
          The SQLITE_TXN_WRITE state means that the database is currently +** in a write transaction. Content has been written to the database file +** but has not yet committed. The transaction state will change to +** to SQLITE_TXN_NONE at the next [ROLLBACK] or [COMMIT].
          +*/ +#define SQLITE_TXN_NONE 0 +#define SQLITE_TXN_READ 1 +#define SQLITE_TXN_WRITE 2 + +/* +** CAPI3REF: Find the next prepared statement +** METHOD: sqlite3 +** +** ^This interface returns a pointer to the next [prepared statement] after +** pStmt associated with the [database connection] pDb. ^If pStmt is NULL +** then this interface returns a pointer to the first prepared statement +** associated with the database connection pDb. ^If no prepared statement +** satisfies the conditions of this routine, it returns NULL. +** +** The [database connection] pointer D in a call to +** [sqlite3_next_stmt(D,S)] must refer to an open database +** connection and in particular must not be a NULL pointer. +*/ +SQLITE_API sqlite3_stmt *sqlite3_next_stmt(sqlite3 *pDb, sqlite3_stmt *pStmt); /* ** CAPI3REF: Commit And Rollback Notification Callbacks +** METHOD: sqlite3 ** -** These routines -** register callback functions to be invoked whenever a transaction -** is committed or rolled back. The pArg argument is passed through -** to the callback. If the callback on a commit hook function -** returns non-zero, then the commit is converted into a rollback. +** ^The sqlite3_commit_hook() interface registers a callback +** function to be invoked whenever a transaction is [COMMIT | committed]. +** ^Any callback set by a previous call to sqlite3_commit_hook() +** for the same database connection is overridden. +** ^The sqlite3_rollback_hook() interface registers a callback +** function to be invoked whenever a transaction is [ROLLBACK | rolled back]. +** ^Any callback set by a previous call to sqlite3_rollback_hook() +** for the same database connection is overridden. +** ^The pArg argument is passed through to the callback. +** ^If the callback on a commit hook function returns non-zero, +** then the commit is converted into a rollback. ** -** If another function was previously registered, its pArg value is returned. -** Otherwise NULL is returned. +** ^The sqlite3_commit_hook(D,C,P) and sqlite3_rollback_hook(D,C,P) functions +** return the P argument from the previous call of the same function +** on the same [database connection] D, or NULL for +** the first call for each function on D. ** -** Registering a NULL function disables the callback. +** The commit and rollback hook callbacks are not reentrant. +** The callback implementation must not do anything that will modify +** the database connection that invoked the callback. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the commit +** or rollback hook in the first place. +** Note that running any other SQL statements, including SELECT statements, +** or merely calling [sqlite3_prepare_v2()] and [sqlite3_step()] will modify +** the database connections for the meaning of "modify" in this paragraph. ** -** For the purposes of this API, a transaction is said to have been +** ^Registering a NULL function disables the callback. +** +** ^When the commit hook callback routine returns zero, the [COMMIT] +** operation is allowed to continue normally. ^If the commit hook +** returns non-zero, then the [COMMIT] is converted into a [ROLLBACK]. +** ^The rollback hook is invoked on a rollback that results from a commit +** hook returning non-zero, just as it would be with any other rollback. +** +** ^For the purposes of this API, a transaction is said to have been ** rolled back if an explicit "ROLLBACK" statement is executed, or -** an error or constraint causes an implicit rollback to occur. The -** callback is not invoked if a transaction is automatically rolled -** back because the database connection is closed. +** an error or constraint causes an implicit rollback to occur. +** ^The rollback callback is not invoked if a transaction is +** automatically rolled back because the database connection is closed. ** -** These are experimental interfaces and are subject to change. +** See also the [sqlite3_update_hook()] interface. */ -void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); -void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); +SQLITE_API void *sqlite3_commit_hook(sqlite3*, int(*)(void*), void*); +SQLITE_API void *sqlite3_rollback_hook(sqlite3*, void(*)(void *), void*); + +/* +** CAPI3REF: Autovacuum Compaction Amount Callback +** METHOD: sqlite3 +** +** ^The sqlite3_autovacuum_pages(D,C,P,X) interface registers a callback +** function C that is invoked prior to each autovacuum of the database +** file. ^The callback is passed a copy of the generic data pointer (P), +** the schema-name of the attached database that is being autovacuumed, +** the size of the database file in pages, the number of free pages, +** and the number of bytes per page, respectively. The callback should +** return the number of free pages that should be removed by the +** autovacuum. ^If the callback returns zero, then no autovacuum happens. +** ^If the value returned is greater than or equal to the number of +** free pages, then a complete autovacuum happens. +** +**

          ^If there are multiple ATTACH-ed database files that are being +** modified as part of a transaction commit, then the autovacuum pages +** callback is invoked separately for each file. +** +**

          The callback is not reentrant. The callback function should +** not attempt to invoke any other SQLite interface. If it does, bad +** things may happen, including segmentation faults and corrupt database +** files. The callback function should be a simple function that +** does some arithmetic on its input parameters and returns a result. +** +** ^The X parameter to sqlite3_autovacuum_pages(D,C,P,X) is an optional +** destructor for the P parameter. ^If X is not NULL, then X(P) is +** invoked whenever the database connection closes or when the callback +** is overwritten by another invocation of sqlite3_autovacuum_pages(). +** +**

          ^There is only one autovacuum pages callback per database connection. +** ^Each call to the sqlite3_autovacuum_pages() interface overrides all +** previous invocations for that database connection. ^If the callback +** argument (C) to sqlite3_autovacuum_pages(D,C,P,X) is a NULL pointer, +** then the autovacuum steps callback is canceled. The return value +** from sqlite3_autovacuum_pages() is normally SQLITE_OK, but might +** be some other error code if something goes wrong. The current +** implementation will only return SQLITE_OK or SQLITE_MISUSE, but other +** return codes might be added in future releases. +** +**

          If no autovacuum pages callback is specified (the usual case) or +** a NULL pointer is provided for the callback, +** then the default behavior is to vacuum all free pages. So, in other +** words, the default behavior is the same as if the callback function +** were something like this: +** +** 

          +**     unsigned int demonstration_autovac_pages_callback(
+**       void *pClientData,
+**       const char *zSchema,
+**       unsigned int nDbPage,
+**       unsigned int nFreePage,
+**       unsigned int nBytePerPage
+**     ){
+**       return nFreePage;
+**     }
+**
          +*/ +SQLITE_API int sqlite3_autovacuum_pages( + sqlite3 *db, + unsigned int(*)(void*,const char*,unsigned int,unsigned int,unsigned int), + void*, + void(*)(void*) +); + /* ** CAPI3REF: Data Change Notification Callbacks +** METHOD: sqlite3 ** -** Register a callback function with the database connection identified by the -** first argument to be invoked whenever a row is updated, inserted or deleted. -** Any callback set by a previous call to this function for the same -** database connection is overridden. +** ^The sqlite3_update_hook() interface registers a callback function +** with the [database connection] identified by the first argument +** to be invoked whenever a row is updated, inserted or deleted in +** a [rowid table]. +** ^Any callback set by a previous call to this function +** for the same database connection is overridden. ** -** The second argument is a pointer to the function to invoke when a -** row is updated, inserted or deleted. The first argument to the callback is -** a copy of the third argument to sqlite3_update_hook(). The second callback -** argument is one of SQLITE_INSERT, SQLITE_DELETE or SQLITE_UPDATE, depending -** on the operation that caused the callback to be invoked. The third and -** fourth arguments to the callback contain pointers to the database and -** table name containing the affected row. The final callback parameter is -** the rowid of the row. In the case of an update, this is the rowid after -** the update takes place. +** ^The second argument is a pointer to the function to invoke when a +** row is updated, inserted or deleted in a rowid table. +** ^The first argument to the callback is a copy of the third argument +** to sqlite3_update_hook(). +** ^The second callback argument is one of [SQLITE_INSERT], [SQLITE_DELETE], +** or [SQLITE_UPDATE], depending on the operation that caused the callback +** to be invoked. +** ^The third and fourth arguments to the callback contain pointers to the +** database and table name containing the affected row. +** ^The final callback parameter is the [rowid] of the row. +** ^In the case of an update, this is the [rowid] after the update takes place. ** -** The update hook is not invoked when internal system tables are -** modified (i.e. sqlite_master and sqlite_sequence). +** ^(The update hook is not invoked when internal system tables are +** modified (i.e. sqlite_sequence).)^ +** ^The update hook is not invoked when [WITHOUT ROWID] tables are modified. ** -** If another function was previously registered, its pArg value is returned. -** Otherwise NULL is returned. +** ^In the current implementation, the update hook +** is not invoked when conflicting rows are deleted because of an +** [ON CONFLICT | ON CONFLICT REPLACE] clause. ^Nor is the update hook +** invoked when rows are deleted using the [truncate optimization]. +** The exceptions defined in this paragraph might change in a future +** release of SQLite. +** +** Whether the update hook is invoked before or after the +** corresponding change is currently unspecified and may differ +** depending on the type of change. Do not rely on the order of the +** hook call with regards to the final result of the operation which +** triggers the hook. +** +** The update hook implementation must not do anything that will modify +** the database connection that invoked the update hook. Any actions +** to modify the database connection must be deferred until after the +** completion of the [sqlite3_step()] call that triggered the update hook. +** Note that [sqlite3_prepare_v2()] and [sqlite3_step()] both modify their +** database connections for the meaning of "modify" in this paragraph. +** +** ^The sqlite3_update_hook(D,C,P) function +** returns the P argument from the previous call +** on the same [database connection] D, or NULL for +** the first call on D. +** +** See also the [sqlite3_commit_hook()], [sqlite3_rollback_hook()], +** and [sqlite3_preupdate_hook()] interfaces. */ -void *sqlite3_update_hook( - sqlite3*, - void(*)(void *,int ,char const *,char const *,sqlite_int64), +SQLITE_API void *sqlite3_update_hook( + sqlite3*, + void(*)(void *,int ,char const *,char const *,sqlite3_int64), void* ); /* -** CAPI3REF: Enable Or Disable Shared Pager Cache +** CAPI3REF: Enable Or Disable Shared Pager Cache ** -** This routine enables or disables the sharing of the database cache -** and schema data structures between connections to the same database. -** Sharing is enabled if the argument is true and disabled if the argument -** is false. +** ^(This routine enables or disables the sharing of the database cache +** and schema data structures between [database connection | connections] +** to the same database. Sharing is enabled if the argument is true +** and disabled if the argument is false.)^ ** -** Cache sharing is enabled and disabled on a thread-by-thread basis. -** Each call to this routine enables or disables cache sharing only for -** connections created in the same thread in which this routine is called. -** There is no mechanism for sharing cache between database connections -** running in different threads. +** This interface is omitted if SQLite is compiled with +** [-DSQLITE_OMIT_SHARED_CACHE]. The [-DSQLITE_OMIT_SHARED_CACHE] +** compile-time option is recommended because the +** [use of shared cache mode is discouraged]. ** -** Sharing must be disabled prior to shutting down a thread or else -** the thread will leak memory. Call this routine with an argument of -** 0 to turn off sharing. Or use the sqlite3_thread_cleanup() API. +** ^Cache sharing is enabled and disabled for an entire process. +** This is a change as of SQLite [version 3.5.0] ([dateof:3.5.0]). +** In prior versions of SQLite, +** sharing was enabled or disabled for each thread separately. ** -** This routine must not be called when any database connections -** are active in the current thread. Enabling or disabling shared -** cache while there are active database connections will result -** in memory corruption. +** ^(The cache sharing mode set by this interface effects all subsequent +** calls to [sqlite3_open()], [sqlite3_open_v2()], and [sqlite3_open16()]. +** Existing database connections continue to use the sharing mode +** that was in effect at the time they were opened.)^ ** -** When the shared cache is enabled, the -** following routines must always be called from the same thread: -** [sqlite3_open()], [sqlite3_prepare_v2()], [sqlite3_step()], -** [sqlite3_reset()], [sqlite3_finalize()], and [sqlite3_close()]. -** This is due to the fact that the shared cache makes use of -** thread-specific storage so that it will be available for sharing -** with other connections. +** ^(This routine returns [SQLITE_OK] if shared cache was enabled or disabled +** successfully. An [error code] is returned otherwise.)^ ** -** Virtual tables cannot be used with a shared cache. When shared -** cache is enabled, the sqlite3_create_module() API used to register -** virtual tables will always return an error. +** ^Shared cache is disabled by default. It is recommended that it stay +** that way. In other words, do not use this routine. This interface +** continues to be provided for historical compatibility, but its use is +** discouraged. Any use of shared cache is discouraged. If shared cache +** must be used, it is recommended that shared cache only be enabled for +** individual database connections using the [sqlite3_open_v2()] interface +** with the [SQLITE_OPEN_SHAREDCACHE] flag. ** -** This routine returns [SQLITE_OK] if shared cache was -** enabled or disabled successfully. An [SQLITE_ERROR | error code] -** is returned otherwise. +** Note: This method is disabled on MacOS X 10.7 and iOS version 5.0 +** and will always return SQLITE_MISUSE. On those systems, +** shared cache mode should be enabled per-database connection via +** [sqlite3_open_v2()] with [SQLITE_OPEN_SHAREDCACHE]. ** -** Shared cache is disabled by default for backward compatibility. +** This interface is threadsafe on processors where writing a +** 32-bit integer is atomic. +** +** See Also: [SQLite Shared-Cache Mode] */ -int sqlite3_enable_shared_cache(int); +SQLITE_API int sqlite3_enable_shared_cache(int); /* -** CAPI3REF: Attempt To Free Heap Memory +** CAPI3REF: Attempt To Free Heap Memory ** -** Attempt to free N bytes of heap memory by deallocating non-essential -** memory allocations held by the database library (example: memory -** used to cache database pages to improve performance). +** ^The sqlite3_release_memory() interface attempts to free N bytes +** of heap memory by deallocating non-essential memory allocations +** held by the database library. Memory used to cache database +** pages to improve performance is an example of non-essential memory. +** ^sqlite3_release_memory() returns the number of bytes actually freed, +** which might be more or less than the amount requested. +** ^The sqlite3_release_memory() routine is a no-op returning zero +** if SQLite is not compiled with [SQLITE_ENABLE_MEMORY_MANAGEMENT]. ** -** This function is not a part of standard builds. It is only created -** if SQLite is compiled with the SQLITE_ENABLE_MEMORY_MANAGEMENT macro. +** See also: [sqlite3_db_release_memory()] */ -int sqlite3_release_memory(int); +SQLITE_API int sqlite3_release_memory(int); /* -** CAPI3REF: Impose A Limit On Heap Size +** CAPI3REF: Free Memory Used By A Database Connection +** METHOD: sqlite3 ** -** Place a "soft" limit on the amount of heap memory that may be allocated by -** SQLite within the current thread. If an internal allocation is requested -** that would exceed the specified limit, [sqlite3_release_memory()] is invoked -** one or more times to free up some space before the allocation is made. +** ^The sqlite3_db_release_memory(D) interface attempts to free as much heap +** memory as possible from database connection D. Unlike the +** [sqlite3_release_memory()] interface, this interface is in effect even +** when the [SQLITE_ENABLE_MEMORY_MANAGEMENT] compile-time option is +** omitted. ** -** The limit is called "soft", because if [sqlite3_release_memory()] cannot free -** sufficient memory to prevent the limit from being exceeded, the memory is -** allocated anyway and the current operation proceeds. -** -** Prior to shutting down a thread sqlite3_soft_heap_limit() must be set to -** zero (the default) or else the thread will leak memory. Alternatively, use -** the [sqlite3_thread_cleanup()] API. -** -** A negative or zero value for N means that there is no soft heap limit and -** [sqlite3_release_memory()] will only be called when memory is exhaused. -** The default value for the soft heap limit is zero. -** -** SQLite makes a best effort to honor the soft heap limit. But if it -** is unable to reduce memory usage below the soft limit, execution will -** continue without error or notification. This is why the limit is -** called a "soft" limit. It is advisory only. -** -** This function is only available if the library was compiled with the -** SQLITE_ENABLE_MEMORY_MANAGEMENT option set. -** memory-management has been enabled. +** See also: [sqlite3_release_memory()] */ -void sqlite3_soft_heap_limit(int); +SQLITE_API int sqlite3_db_release_memory(sqlite3*); /* -** CAPI3REF: Clean Up Thread Local Storage +** CAPI3REF: Impose A Limit On Heap Size ** -** This routine makes sure that all thread-local storage has been -** deallocated for the current thread. +** These interfaces impose limits on the amount of heap memory that will be +** by all database connections within a single process. ** -** This routine is not technically necessary. All thread-local storage -** will be automatically deallocated once memory-management and -** shared-cache are disabled and the soft heap limit has been set -** to zero. This routine is provided as a convenience for users who -** want to make absolutely sure they have not forgotten something -** prior to killing off a thread. +** ^The sqlite3_soft_heap_limit64() interface sets and/or queries the +** soft limit on the amount of heap memory that may be allocated by SQLite. +** ^SQLite strives to keep heap memory utilization below the soft heap +** limit by reducing the number of pages held in the page cache +** as heap memory usages approaches the limit. +** ^The soft heap limit is "soft" because even though SQLite strives to stay +** below the limit, it will exceed the limit rather than generate +** an [SQLITE_NOMEM] error. In other words, the soft heap limit +** is advisory only. +** +** ^The sqlite3_hard_heap_limit64(N) interface sets a hard upper bound of +** N bytes on the amount of memory that will be allocated. ^The +** sqlite3_hard_heap_limit64(N) interface is similar to +** sqlite3_soft_heap_limit64(N) except that memory allocations will fail +** when the hard heap limit is reached. +** +** ^The return value from both sqlite3_soft_heap_limit64() and +** sqlite3_hard_heap_limit64() is the size of +** the heap limit prior to the call, or negative in the case of an +** error. ^If the argument N is negative +** then no change is made to the heap limit. Hence, the current +** size of heap limits can be determined by invoking +** sqlite3_soft_heap_limit64(-1) or sqlite3_hard_heap_limit(-1). +** +** ^Setting the heap limits to zero disables the heap limiter mechanism. +** +** ^The soft heap limit may not be greater than the hard heap limit. +** ^If the hard heap limit is enabled and if sqlite3_soft_heap_limit(N) +** is invoked with a value of N that is greater than the hard heap limit, +** the soft heap limit is set to the value of the hard heap limit. +** ^The soft heap limit is automatically enabled whenever the hard heap +** limit is enabled. ^When sqlite3_hard_heap_limit64(N) is invoked and +** the soft heap limit is outside the range of 1..N, then the soft heap +** limit is set to N. ^Invoking sqlite3_soft_heap_limit64(0) when the +** hard heap limit is enabled makes the soft heap limit equal to the +** hard heap limit. +** +** The memory allocation limits can also be adjusted using +** [PRAGMA soft_heap_limit] and [PRAGMA hard_heap_limit]. +** +** ^(The heap limits are not enforced in the current implementation +** if one or more of following conditions are true: +** +**
            +**
          • The limit value is set to zero. +**
          • Memory accounting is disabled using a combination of the +** [sqlite3_config]([SQLITE_CONFIG_MEMSTATUS],...) start-time option and +** the [SQLITE_DEFAULT_MEMSTATUS] compile-time option. +**
          • An alternative page cache implementation is specified using +** [sqlite3_config]([SQLITE_CONFIG_PCACHE2],...). +**
          • The page cache allocates from its own memory pool supplied +** by [sqlite3_config]([SQLITE_CONFIG_PAGECACHE],...) rather than +** from the heap. +**
          )^ +** +** The circumstances under which SQLite will enforce the heap limits may +** changes in future releases of SQLite. */ -void sqlite3_thread_cleanup(void); +SQLITE_API sqlite3_int64 sqlite3_soft_heap_limit64(sqlite3_int64 N); +SQLITE_API sqlite3_int64 sqlite3_hard_heap_limit64(sqlite3_int64 N); /* -** CAPI3REF: Extract Metadata About A Column Of A Table +** CAPI3REF: Deprecated Soft Heap Limit Interface +** DEPRECATED ** -** This routine -** returns meta-data about a specific column of a specific database -** table accessible using the connection handle passed as the first function -** argument. +** This is a deprecated version of the [sqlite3_soft_heap_limit64()] +** interface. This routine is provided for historical compatibility +** only. All new applications should use the +** [sqlite3_soft_heap_limit64()] interface rather than this one. +*/ +SQLITE_API SQLITE_DEPRECATED void sqlite3_soft_heap_limit(int N); + + +/* +** CAPI3REF: Extract Metadata About A Column Of A Table +** METHOD: sqlite3 ** -** The column is identified by the second, third and fourth parameters to -** this function. The second parameter is either the name of the database -** (i.e. "main", "temp" or an attached database) containing the specified -** table or NULL. If it is NULL, then all attached databases are searched -** for the table using the same algorithm as the database engine uses to +** ^(The sqlite3_table_column_metadata(X,D,T,C,....) routine returns +** information about column C of table T in database D +** on [database connection] X.)^ ^The sqlite3_table_column_metadata() +** interface returns SQLITE_OK and fills in the non-NULL pointers in +** the final five arguments with appropriate values if the specified +** column exists. ^The sqlite3_table_column_metadata() interface returns +** SQLITE_ERROR if the specified column does not exist. +** ^If the column-name parameter to sqlite3_table_column_metadata() is a +** NULL pointer, then this routine simply checks for the existence of the +** table and returns SQLITE_OK if the table exists and SQLITE_ERROR if it +** does not. If the table name parameter T in a call to +** sqlite3_table_column_metadata(X,D,T,C,...) is NULL then the result is +** undefined behavior. +** +** ^The column is identified by the second, third and fourth parameters to +** this function. ^(The second parameter is either the name of the database +** (i.e. "main", "temp", or an attached database) containing the specified +** table or NULL.)^ ^If it is NULL, then all attached databases are searched +** for the table using the same algorithm used by the database engine to ** resolve unqualified table references. ** -** The third and fourth parameters to this function are the table and column -** name of the desired column, respectively. Neither of these parameters -** may be NULL. +** ^The third and fourth parameters to this function are the table and column +** name of the desired column, respectively. ** -** Meta information is returned by writing to the memory locations passed as -** the 5th and subsequent parameters to this function. Any of these -** arguments may be NULL, in which case the corresponding element of meta -** information is ommitted. +** ^Metadata is returned by writing to the memory locations passed as the 5th +** and subsequent parameters to this function. ^Any of these arguments may be +** NULL, in which case the corresponding element of metadata is omitted. ** -** 
          -** Parameter     Output Type      Description
-** -----------------------------------
+** ^(
          
+** 
+** 
           Parameter  Output
          Type           		  Description
 **
-**   5th         const char*      Data type
-**   6th         const char*      Name of the default collation sequence 
-**   7th         int              True if the column has a NOT NULL constraint
-**   8th         int              True if the column is part of the PRIMARY KEY
-**   9th         int              True if the column is AUTOINCREMENT
-** 
+** 
           5th  const char*  Data type
+** 
           6th  const char*  Name of default collation sequence
+** 
           7th  int          True if column has a NOT NULL constraint
+** 
           8th  int          True if column is part of the PRIMARY KEY
+** 
           9th  int          True if column is [AUTOINCREMENT]
+** 
          
+** 
          )^
 **
+** ^The memory pointed to by the character pointers returned for the
+** declaration type and collation sequence is valid until the next
+** call to any SQLite API function.
 **
-** The memory pointed to by the character pointers returned for the 
-** declaration type and collation sequence is valid only until the next 
-** call to any sqlite API function.
+** ^If the specified table is actually a view, an [error code] is returned.
 **
-** If the specified table is actually a view, then an error is returned.
-**
-** If the specified column is "rowid", "oid" or "_rowid_" and an 
-** INTEGER PRIMARY KEY column has been explicitly declared, then the output 
-** parameters are set for the explicitly declared column. If there is no
-** explicitly declared IPK column, then the output parameters are set as 
-** follows:
+** ^If the specified column is "rowid", "oid" or "_rowid_" and the table
+** is not a [WITHOUT ROWID] table and an
+** [INTEGER PRIMARY KEY] column has been explicitly declared, then the output
+** parameters are set for the explicitly declared column. ^(If there is no
+** [INTEGER PRIMARY KEY] column, then the outputs
+** for the [rowid] are set as follows:
 **
 ** 
           **     data type: "INTEGER"
@@ -2246,17 +7155,13 @@ void sqlite3_thread_cleanup(void);
 **     not null: 0
 **     primary key: 1
 **     auto increment: 0
-** 
          
+**
          )^ ** -** This function may load one or more schemas from database files. If an -** error occurs during this process, or if the requested table or column -** cannot be found, an SQLITE error code is returned and an error message -** left in the database handle (to be retrieved using sqlite3_errmsg()). -** -** This API is only available if the library was compiled with the -** SQLITE_ENABLE_COLUMN_METADATA preprocessor symbol defined. +** ^This function causes all database schemas to be read from disk and +** parsed, if that has not already been done, and returns an error if +** any errors are encountered while loading the schema. */ -int sqlite3_table_column_metadata( +SQLITE_API int sqlite3_table_column_metadata( sqlite3 *db, /* Connection handle */ const char *zDbName, /* Database name or NULL */ const char *zTableName, /* Table name */ @@ -2265,26 +7170,54 @@ int sqlite3_table_column_metadata( char const **pzCollSeq, /* OUTPUT: Collation sequence name */ int *pNotNull, /* OUTPUT: True if NOT NULL constraint exists */ int *pPrimaryKey, /* OUTPUT: True if column part of PK */ - int *pAutoinc /* OUTPUT: True if colums is auto-increment */ + int *pAutoinc /* OUTPUT: True if column is auto-increment */ ); /* ** CAPI3REF: Load An Extension +** METHOD: sqlite3 ** -** Attempt to load an SQLite extension library contained in the file -** zFile. The entry point is zProc. zProc may be 0 in which case the -** name of the entry point defaults to "sqlite3_extension_init". +** ^This interface loads an SQLite extension library from the named file. ** -** Return [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^The sqlite3_load_extension() interface attempts to load an +** [SQLite extension] library contained in the file zFile. If +** the file cannot be loaded directly, attempts are made to load +** with various operating-system specific extensions added. +** So for example, if "samplelib" cannot be loaded, then names like +** "samplelib.so" or "samplelib.dylib" or "samplelib.dll" might +** be tried also. ** -** If an error occurs and pzErrMsg is not 0, then fill *pzErrMsg with -** error message text. The calling function should free this memory -** by calling [sqlite3_free()]. +** ^The entry point is zProc. +** ^(zProc may be 0, in which case SQLite will try to come up with an +** entry point name on its own. It first tries "sqlite3_extension_init". +** If that does not work, it constructs a name "sqlite3_X_init" where the +** X is consists of the lower-case equivalent of all ASCII alphabetic +** characters in the filename from the last "/" to the first following +** "." and omitting any initial "lib".)^ +** ^The sqlite3_load_extension() interface returns +** [SQLITE_OK] on success and [SQLITE_ERROR] if something goes wrong. +** ^If an error occurs and pzErrMsg is not 0, then the +** [sqlite3_load_extension()] interface shall attempt to +** fill *pzErrMsg with error message text stored in memory +** obtained from [sqlite3_malloc()]. The calling function +** should free this memory by calling [sqlite3_free()]. ** -** Extension loading must be enabled using [sqlite3_enable_load_extension()] -** prior to calling this API or an error will be returned. +** ^Extension loading must be enabled using +** [sqlite3_enable_load_extension()] or +** [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],1,NULL) +** prior to calling this API, +** otherwise an error will be returned. +** +** Security warning: It is recommended that the +** [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method be used to enable only this +** interface. The use of the [sqlite3_enable_load_extension()] interface +** should be avoided. This will keep the SQL function [load_extension()] +** disabled and prevent SQL injections from giving attackers +** access to extension loading capabilities. +** +** See also the [load_extension() SQL function]. */ -int sqlite3_load_extension( +SQLITE_API int sqlite3_load_extension( sqlite3 *db, /* Load the extension into this database connection */ const char *zFile, /* Name of the shared library containing extension */ const char *zProc, /* Entry point. Derived from zFile if 0 */ @@ -2292,72 +7225,89 @@ int sqlite3_load_extension( ); /* -** CAPI3REF: Enable Or Disable Extension Loading +** CAPI3REF: Enable Or Disable Extension Loading +** METHOD: sqlite3 ** -** So as not to open security holes in older applications that are -** unprepared to deal with extension loading, and as a means of disabling -** extension loading while evaluating user-entered SQL, the following -** API is provided to turn the [sqlite3_load_extension()] mechanism on and -** off. It is off by default. See ticket #1863. +** ^So as not to open security holes in older applications that are +** unprepared to deal with [extension loading], and as a means of disabling +** [extension loading] while evaluating user-entered SQL, the following API +** is provided to turn the [sqlite3_load_extension()] mechanism on and off. ** -** Call this routine with onoff==1 to turn extension loading on -** and call it with onoff==0 to turn it back off again. +** ^Extension loading is off by default. +** ^Call the sqlite3_enable_load_extension() routine with onoff==1 +** to turn extension loading on and call it with onoff==0 to turn +** it back off again. +** +** ^This interface enables or disables both the C-API +** [sqlite3_load_extension()] and the SQL function [load_extension()]. +** ^(Use [sqlite3_db_config](db,[SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION],..) +** to enable or disable only the C-API.)^ +** +** Security warning: It is recommended that extension loading +** be enabled using the [SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION] method +** rather than this interface, so the [load_extension()] SQL function +** remains disabled. This will prevent SQL injections from giving attackers +** access to extension loading capabilities. */ -int sqlite3_enable_load_extension(sqlite3 *db, int onoff); +SQLITE_API int sqlite3_enable_load_extension(sqlite3 *db, int onoff); /* -** CAPI3REF: Make Arrangements To Automatically Load An Extension +** CAPI3REF: Automatically Load Statically Linked Extensions ** -** Register an extension entry point that is automatically invoked -** whenever a new database connection is opened using -** [sqlite3_open()] or [sqlite3_open16()]. +** ^This interface causes the xEntryPoint() function to be invoked for +** each new [database connection] that is created. The idea here is that +** xEntryPoint() is the entry point for a statically linked [SQLite extension] +** that is to be automatically loaded into all new database connections. ** -** This API can be invoked at program startup in order to register -** one or more statically linked extensions that will be available -** to all new database connections. +** ^(Even though the function prototype shows that xEntryPoint() takes +** no arguments and returns void, SQLite invokes xEntryPoint() with three +** arguments and expects an integer result as if the signature of the +** entry point where as follows: ** -** Duplicate extensions are detected so calling this routine multiple -** times with the same extension is harmless. +** 
          +**    int xEntryPoint(
+**      sqlite3 *db,
+**      const char **pzErrMsg,
+**      const struct sqlite3_api_routines *pThunk
+**    );
+**
          )^ ** -** This routine stores a pointer to the extension in an array -** that is obtained from malloc(). If you run a memory leak -** checker on your program and it reports a leak because of this -** array, then invoke [sqlite3_automatic_extension_reset()] prior -** to shutdown to free the memory. +** If the xEntryPoint routine encounters an error, it should make *pzErrMsg +** point to an appropriate error message (obtained from [sqlite3_mprintf()]) +** and return an appropriate [error code]. ^SQLite ensures that *pzErrMsg +** is NULL before calling the xEntryPoint(). ^SQLite will invoke +** [sqlite3_free()] on *pzErrMsg after xEntryPoint() returns. ^If any +** xEntryPoint() returns an error, the [sqlite3_open()], [sqlite3_open16()], +** or [sqlite3_open_v2()] call that provoked the xEntryPoint() will fail. ** -** Automatic extensions apply across all threads. +** ^Calling sqlite3_auto_extension(X) with an entry point X that is already +** on the list of automatic extensions is a harmless no-op. ^No entry point +** will be called more than once for each database connection that is opened. ** -** This interface is experimental and is subject to change or -** removal in future releases of SQLite. +** See also: [sqlite3_reset_auto_extension()] +** and [sqlite3_cancel_auto_extension()] */ -int sqlite3_auto_extension(void *xEntryPoint); +SQLITE_API int sqlite3_auto_extension(void(*xEntryPoint)(void)); +/* +** CAPI3REF: Cancel Automatic Extension Loading +** +** ^The [sqlite3_cancel_auto_extension(X)] interface unregisters the +** initialization routine X that was registered using a prior call to +** [sqlite3_auto_extension(X)]. ^The [sqlite3_cancel_auto_extension(X)] +** routine returns 1 if initialization routine X was successfully +** unregistered and it returns 0 if X was not on the list of initialization +** routines. +*/ +SQLITE_API int sqlite3_cancel_auto_extension(void(*xEntryPoint)(void)); /* ** CAPI3REF: Reset Automatic Extension Loading ** -** Disable all previously registered automatic extensions. This -** routine undoes the effect of all prior [sqlite3_automatic_extension()] -** calls. -** -** This call disabled automatic extensions in all threads. -** -** This interface is experimental and is subject to change or -** removal in future releases of SQLite. -*/ -void sqlite3_reset_auto_extension(void); - - -/* -****** EXPERIMENTAL - subject to change without notice ************** -** -** The interface to the virtual-table mechanism is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stablizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. +** ^This interface disables all automatic extensions previously +** registered using [sqlite3_auto_extension()]. */ +SQLITE_API void sqlite3_reset_auto_extension(void); /* ** Structures used by the virtual table interface @@ -2368,9 +7318,20 @@ typedef struct sqlite3_vtab_cursor sqlite3_vtab_cursor; typedef struct sqlite3_module sqlite3_module; /* -** A module is a class of virtual tables. Each module is defined -** by an instance of the following structure. This structure consists -** mostly of methods for the module. +** CAPI3REF: Virtual Table Object +** KEYWORDS: sqlite3_module {virtual table module} +** +** This structure, sometimes called a "virtual table module", +** defines the implementation of a [virtual table]. +** This structure consists mostly of methods for the module. +** +** ^A virtual table module is created by filling in a persistent +** instance of this structure and passing a pointer to that instance +** to [sqlite3_create_module()] or [sqlite3_create_module_v2()]. +** ^The registration remains valid until it is replaced by a different +** module or until the [database connection] closes. The content +** of this structure must not change while it is registered with +** any database connection. */ struct sqlite3_module { int iVersion; @@ -2390,8 +7351,8 @@ struct sqlite3_module { int (*xNext)(sqlite3_vtab_cursor*); int (*xEof)(sqlite3_vtab_cursor*); int (*xColumn)(sqlite3_vtab_cursor*, sqlite3_context*, int); - int (*xRowid)(sqlite3_vtab_cursor*, sqlite_int64 *pRowid); - int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite_int64 *); + int (*xRowid)(sqlite3_vtab_cursor*, sqlite3_int64 *pRowid); + int (*xUpdate)(sqlite3_vtab *, int, sqlite3_value **, sqlite3_int64 *); int (*xBegin)(sqlite3_vtab *pVTab); int (*xSync)(sqlite3_vtab *pVTab); int (*xCommit)(sqlite3_vtab *pVTab); @@ -2399,128 +7360,324 @@ struct sqlite3_module { int (*xFindFunction)(sqlite3_vtab *pVtab, int nArg, const char *zName, void (**pxFunc)(sqlite3_context*,int,sqlite3_value**), void **ppArg); + int (*xRename)(sqlite3_vtab *pVtab, const char *zNew); + /* The methods above are in version 1 of the sqlite_module object. Those + ** below are for version 2 and greater. */ + int (*xSavepoint)(sqlite3_vtab *pVTab, int); + int (*xRelease)(sqlite3_vtab *pVTab, int); + int (*xRollbackTo)(sqlite3_vtab *pVTab, int); + /* The methods above are in versions 1 and 2 of the sqlite_module object. + ** Those below are for version 3 and greater. */ + int (*xShadowName)(const char*); + /* The methods above are in versions 1 through 3 of the sqlite_module object. + ** Those below are for version 4 and greater. */ + int (*xIntegrity)(sqlite3_vtab *pVTab, const char *zSchema, + const char *zTabName, int mFlags, char **pzErr); }; /* -** The sqlite3_index_info structure and its substructures is used to -** pass information into and receive the reply from the xBestIndex -** method of an sqlite3_module. The fields under **Inputs** are the +** CAPI3REF: Virtual Table Indexing Information +** KEYWORDS: sqlite3_index_info +** +** The sqlite3_index_info structure and its substructures is used as part +** of the [virtual table] interface to +** pass information into and receive the reply from the [xBestIndex] +** method of a [virtual table module]. The fields under **Inputs** are the ** inputs to xBestIndex and are read-only. xBestIndex inserts its ** results into the **Outputs** fields. ** -** The aConstraint[] array records WHERE clause constraints of the -** form: +** ^(The aConstraint[] array records WHERE clause constraints of the form: ** -** column OP expr +**
          column OP expr
          ** -** Where OP is =, <, <=, >, or >=. The particular operator is stored -** in aConstraint[].op. The index of the column is stored in -** aConstraint[].iColumn. aConstraint[].usable is TRUE if the +** where OP is =, <, <=, >, or >=.)^ ^(The particular operator is +** stored in aConstraint[].op using one of the +** [SQLITE_INDEX_CONSTRAINT_EQ | SQLITE_INDEX_CONSTRAINT_ values].)^ +** ^(The index of the column is stored in +** aConstraint[].iColumn.)^ ^(aConstraint[].usable is TRUE if the ** expr on the right-hand side can be evaluated (and thus the constraint -** is usable) and false if it cannot. +** is usable) and false if it cannot.)^ ** -** The optimizer automatically inverts terms of the form "expr OP column" -** and makes other simplificatinos to the WHERE clause in an attempt to +** ^The optimizer automatically inverts terms of the form "expr OP column" +** and makes other simplifications to the WHERE clause in an attempt to ** get as many WHERE clause terms into the form shown above as possible. -** The aConstraint[] array only reports WHERE clause terms in the correct -** form that refer to the particular virtual table being queried. +** ^The aConstraint[] array only reports WHERE clause terms that are +** relevant to the particular virtual table being queried. ** -** Information about the ORDER BY clause is stored in aOrderBy[]. -** Each term of aOrderBy records a column of the ORDER BY clause. +** ^Information about the ORDER BY clause is stored in aOrderBy[]. +** ^Each term of aOrderBy records a column of the ORDER BY clause. ** -** The xBestIndex method must fill aConstraintUsage[] with information -** about what parameters to pass to xFilter. If argvIndex>0 then +** The colUsed field indicates which columns of the virtual table may be +** required by the current scan. Virtual table columns are numbered from +** zero in the order in which they appear within the CREATE TABLE statement +** passed to sqlite3_declare_vtab(). For the first 63 columns (columns 0-62), +** the corresponding bit is set within the colUsed mask if the column may be +** required by SQLite. If the table has at least 64 columns and any column +** to the right of the first 63 is required, then bit 63 of colUsed is also +** set. In other words, column iCol may be required if the expression +** (colUsed & ((sqlite3_uint64)1 << (iCol>=63 ? 63 : iCol))) evaluates to +** non-zero. +** +** The [xBestIndex] method must fill aConstraintUsage[] with information +** about what parameters to pass to xFilter. ^If argvIndex>0 then ** the right-hand side of the corresponding aConstraint[] is evaluated -** and becomes the argvIndex-th entry in argv. If aConstraintUsage[].omit +** and becomes the argvIndex-th entry in argv. ^(If aConstraintUsage[].omit ** is true, then the constraint is assumed to be fully handled by the -** virtual table and is not checked again by SQLite. +** virtual table and might not be checked again by the byte code.)^ ^(The +** aConstraintUsage[].omit flag is an optimization hint. When the omit flag +** is left in its default setting of false, the constraint will always be +** checked separately in byte code. If the omit flag is change to true, then +** the constraint may or may not be checked in byte code. In other words, +** when the omit flag is true there is no guarantee that the constraint will +** not be checked again using byte code.)^ ** -** The idxNum and idxPtr values are recorded and passed into xFilter. -** sqlite3_free() is used to free idxPtr if needToFreeIdxPtr is true. +** ^The idxNum and idxStr values are recorded and passed into the +** [xFilter] method. +** ^[sqlite3_free()] is used to free idxStr if and only if +** needToFreeIdxStr is true. ** -** The orderByConsumed means that output from xFilter will occur in +** ^The orderByConsumed means that output from [xFilter]/[xNext] will occur in ** the correct order to satisfy the ORDER BY clause so that no separate ** sorting step is required. ** -** The estimatedCost value is an estimate of the cost of doing the -** particular lookup. A full scan of a table with N entries should have -** a cost of N. A binary search of a table of N entries should have a -** cost of approximately log(N). +** ^The estimatedCost value is an estimate of the cost of a particular +** strategy. A cost of N indicates that the cost of the strategy is similar +** to a linear scan of an SQLite table with N rows. A cost of log(N) +** indicates that the expense of the operation is similar to that of a +** binary search on a unique indexed field of an SQLite table with N rows. +** +** ^The estimatedRows value is an estimate of the number of rows that +** will be returned by the strategy. +** +** The xBestIndex method may optionally populate the idxFlags field with a +** mask of SQLITE_INDEX_SCAN_* flags. One such flag is +** [SQLITE_INDEX_SCAN_HEX], which if set causes the [EXPLAIN QUERY PLAN] +** output to show the idxNum has hex instead of as decimal. Another flag is +** SQLITE_INDEX_SCAN_UNIQUE, which if set indicates that the query plan will +** return at most one row. +** +** Additionally, if xBestIndex sets the SQLITE_INDEX_SCAN_UNIQUE flag, then +** SQLite also assumes that if a call to the xUpdate() method is made as +** part of the same statement to delete or update a virtual table row and the +** implementation returns SQLITE_CONSTRAINT, then there is no need to rollback +** any database changes. In other words, if the xUpdate() returns +** SQLITE_CONSTRAINT, the database contents must be exactly as they were +** before xUpdate was called. By contrast, if SQLITE_INDEX_SCAN_UNIQUE is not +** set and xUpdate returns SQLITE_CONSTRAINT, any database changes made by +** the xUpdate method are automatically rolled back by SQLite. +** +** IMPORTANT: The estimatedRows field was added to the sqlite3_index_info +** structure for SQLite [version 3.8.2] ([dateof:3.8.2]). +** If a virtual table extension is +** used with an SQLite version earlier than 3.8.2, the results of attempting +** to read or write the estimatedRows field are undefined (but are likely +** to include crashing the application). The estimatedRows field should +** therefore only be used if [sqlite3_libversion_number()] returns a +** value greater than or equal to 3008002. Similarly, the idxFlags field +** was added for [version 3.9.0] ([dateof:3.9.0]). +** It may therefore only be used if +** sqlite3_libversion_number() returns a value greater than or equal to +** 3009000. */ struct sqlite3_index_info { /* Inputs */ - const int nConstraint; /* Number of entries in aConstraint */ - const struct sqlite3_index_constraint { - int iColumn; /* Column on left-hand side of constraint */ + int nConstraint; /* Number of entries in aConstraint */ + struct sqlite3_index_constraint { + int iColumn; /* Column constrained. -1 for ROWID */ unsigned char op; /* Constraint operator */ unsigned char usable; /* True if this constraint is usable */ int iTermOffset; /* Used internally - xBestIndex should ignore */ - } *const aConstraint; /* Table of WHERE clause constraints */ - const int nOrderBy; /* Number of terms in the ORDER BY clause */ - const struct sqlite3_index_orderby { + } *aConstraint; /* Table of WHERE clause constraints */ + int nOrderBy; /* Number of terms in the ORDER BY clause */ + struct sqlite3_index_orderby { int iColumn; /* Column number */ unsigned char desc; /* True for DESC. False for ASC. */ - } *const aOrderBy; /* The ORDER BY clause */ - + } *aOrderBy; /* The ORDER BY clause */ /* Outputs */ struct sqlite3_index_constraint_usage { int argvIndex; /* if >0, constraint is part of argv to xFilter */ unsigned char omit; /* Do not code a test for this constraint */ - } *const aConstraintUsage; + } *aConstraintUsage; int idxNum; /* Number used to identify the index */ char *idxStr; /* String, possibly obtained from sqlite3_malloc */ int needToFreeIdxStr; /* Free idxStr using sqlite3_free() if true */ int orderByConsumed; /* True if output is already ordered */ - double estimatedCost; /* Estimated cost of using this index */ + double estimatedCost; /* Estimated cost of using this index */ + /* Fields below are only available in SQLite 3.8.2 and later */ + sqlite3_int64 estimatedRows; /* Estimated number of rows returned */ + /* Fields below are only available in SQLite 3.9.0 and later */ + int idxFlags; /* Mask of SQLITE_INDEX_SCAN_* flags */ + /* Fields below are only available in SQLite 3.10.0 and later */ + sqlite3_uint64 colUsed; /* Input: Mask of columns used by statement */ }; -#define SQLITE_INDEX_CONSTRAINT_EQ 2 -#define SQLITE_INDEX_CONSTRAINT_GT 4 -#define SQLITE_INDEX_CONSTRAINT_LE 8 -#define SQLITE_INDEX_CONSTRAINT_LT 16 -#define SQLITE_INDEX_CONSTRAINT_GE 32 -#define SQLITE_INDEX_CONSTRAINT_MATCH 64 /* -** This routine is used to register a new module name with an SQLite -** connection. Module names must be registered before creating new -** virtual tables on the module, or before using preexisting virtual -** tables of the module. +** CAPI3REF: Virtual Table Scan Flags +** +** Virtual table implementations are allowed to set the +** [sqlite3_index_info].idxFlags field to some combination of +** these bits. */ -int sqlite3_create_module( +#define SQLITE_INDEX_SCAN_UNIQUE 0x00000001 /* Scan visits at most 1 row */ +#define SQLITE_INDEX_SCAN_HEX 0x00000002 /* Display idxNum as hex */ + /* in EXPLAIN QUERY PLAN */ + +/* +** CAPI3REF: Virtual Table Constraint Operator Codes +** +** These macros define the allowed values for the +** [sqlite3_index_info].aConstraint[].op field. Each value represents +** an operator that is part of a constraint term in the WHERE clause of +** a query that uses a [virtual table]. +** +** ^The left-hand operand of the operator is given by the corresponding +** aConstraint[].iColumn field. ^An iColumn of -1 indicates the left-hand +** operand is the rowid. +** The SQLITE_INDEX_CONSTRAINT_LIMIT and SQLITE_INDEX_CONSTRAINT_OFFSET +** operators have no left-hand operand, and so for those operators the +** corresponding aConstraint[].iColumn is meaningless and should not be +** used. +** +** All operator values from SQLITE_INDEX_CONSTRAINT_FUNCTION through +** value 255 are reserved to represent functions that are overloaded +** by the [xFindFunction|xFindFunction method] of the virtual table +** implementation. +** +** The right-hand operands for each constraint might be accessible using +** the [sqlite3_vtab_rhs_value()] interface. Usually the right-hand +** operand is only available if it appears as a single constant literal +** in the input SQL. If the right-hand operand is another column or an +** expression (even a constant expression) or a parameter, then the +** sqlite3_vtab_rhs_value() probably will not be able to extract it. +** ^The SQLITE_INDEX_CONSTRAINT_ISNULL and +** SQLITE_INDEX_CONSTRAINT_ISNOTNULL operators have no right-hand operand +** and hence calls to sqlite3_vtab_rhs_value() for those operators will +** always return SQLITE_NOTFOUND. +** +** The collating sequence to be used for comparison can be found using +** the [sqlite3_vtab_collation()] interface. For most real-world virtual +** tables, the collating sequence of constraints does not matter (for example +** because the constraints are numeric) and so the sqlite3_vtab_collation() +** interface is not commonly needed. +*/ +#define SQLITE_INDEX_CONSTRAINT_EQ 2 +#define SQLITE_INDEX_CONSTRAINT_GT 4 +#define SQLITE_INDEX_CONSTRAINT_LE 8 +#define SQLITE_INDEX_CONSTRAINT_LT 16 +#define SQLITE_INDEX_CONSTRAINT_GE 32 +#define SQLITE_INDEX_CONSTRAINT_MATCH 64 +#define SQLITE_INDEX_CONSTRAINT_LIKE 65 +#define SQLITE_INDEX_CONSTRAINT_GLOB 66 +#define SQLITE_INDEX_CONSTRAINT_REGEXP 67 +#define SQLITE_INDEX_CONSTRAINT_NE 68 +#define SQLITE_INDEX_CONSTRAINT_ISNOT 69 +#define SQLITE_INDEX_CONSTRAINT_ISNOTNULL 70 +#define SQLITE_INDEX_CONSTRAINT_ISNULL 71 +#define SQLITE_INDEX_CONSTRAINT_IS 72 +#define SQLITE_INDEX_CONSTRAINT_LIMIT 73 +#define SQLITE_INDEX_CONSTRAINT_OFFSET 74 +#define SQLITE_INDEX_CONSTRAINT_FUNCTION 150 + +/* +** CAPI3REF: Register A Virtual Table Implementation +** METHOD: sqlite3 +** +** ^These routines are used to register a new [virtual table module] name. +** ^Module names must be registered before +** creating a new [virtual table] using the module and before using a +** preexisting [virtual table] for the module. +** +** ^The module name is registered on the [database connection] specified +** by the first parameter. ^The name of the module is given by the +** second parameter. ^The third parameter is a pointer to +** the implementation of the [virtual table module]. ^The fourth +** parameter is an arbitrary client data pointer that is passed through +** into the [xCreate] and [xConnect] methods of the virtual table module +** when a new virtual table is be being created or reinitialized. +** +** ^The sqlite3_create_module_v2() interface has a fifth parameter which +** is a pointer to a destructor for the pClientData. ^SQLite will +** invoke the destructor function (if it is not NULL) when SQLite +** no longer needs the pClientData pointer. ^The destructor will also +** be invoked if the call to sqlite3_create_module_v2() fails. +** ^The sqlite3_create_module() +** interface is equivalent to sqlite3_create_module_v2() with a NULL +** destructor. +** +** ^If the third parameter (the pointer to the sqlite3_module object) is +** NULL then no new module is created and any existing modules with the +** same name are dropped. +** +** See also: [sqlite3_drop_modules()] +*/ +SQLITE_API int sqlite3_create_module( sqlite3 *db, /* SQLite connection to register module with */ const char *zName, /* Name of the module */ - const sqlite3_module *, /* Methods for the module */ - void * /* Client data for xCreate/xConnect */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData /* Client data for xCreate/xConnect */ +); +SQLITE_API int sqlite3_create_module_v2( + sqlite3 *db, /* SQLite connection to register module with */ + const char *zName, /* Name of the module */ + const sqlite3_module *p, /* Methods for the module */ + void *pClientData, /* Client data for xCreate/xConnect */ + void(*xDestroy)(void*) /* Module destructor function */ ); /* -** Every module implementation uses a subclass of the following structure -** to describe a particular instance of the module. Each subclass will -** be taylored to the specific needs of the module implementation. The -** purpose of this superclass is to define certain fields that are common -** to all module implementations. +** CAPI3REF: Remove Unnecessary Virtual Table Implementations +** METHOD: sqlite3 ** -** Virtual tables methods can set an error message by assigning a -** string obtained from sqlite3_mprintf() to zErrMsg. The method should -** take care that any prior string is freed by a call to sqlite3_free() -** prior to assigning a new string to zErrMsg. After the error message +** ^The sqlite3_drop_modules(D,L) interface removes all virtual +** table modules from database connection D except those named on list L. +** The L parameter must be either NULL or a pointer to an array of pointers +** to strings where the array is terminated by a single NULL pointer. +** ^If the L parameter is NULL, then all virtual table modules are removed. +** +** See also: [sqlite3_create_module()] +*/ +SQLITE_API int sqlite3_drop_modules( + sqlite3 *db, /* Remove modules from this connection */ + const char **azKeep /* Except, do not remove the ones named here */ +); + +/* +** CAPI3REF: Virtual Table Instance Object +** KEYWORDS: sqlite3_vtab +** +** Every [virtual table module] implementation uses a subclass +** of this object to describe a particular instance +** of the [virtual table]. Each subclass will +** be tailored to the specific needs of the module implementation. +** The purpose of this superclass is to define certain fields that are +** common to all module implementations. +** +** ^Virtual tables methods can set an error message by assigning a +** string obtained from [sqlite3_mprintf()] to zErrMsg. The method should +** take care that any prior string is freed by a call to [sqlite3_free()] +** prior to assigning a new string to zErrMsg. ^After the error message ** is delivered up to the client application, the string will be automatically -** freed by sqlite3_free() and the zErrMsg field will be zeroed. Note -** that sqlite3_mprintf() and sqlite3_free() are used on the zErrMsg field -** since virtual tables are commonly implemented in loadable extensions which -** do not have access to sqlite3MPrintf() or sqlite3Free(). +** freed by sqlite3_free() and the zErrMsg field will be zeroed. */ struct sqlite3_vtab { const sqlite3_module *pModule; /* The module for this virtual table */ - int nRef; /* Used internally */ + int nRef; /* Number of open cursors */ char *zErrMsg; /* Error message from sqlite3_mprintf() */ /* Virtual table implementations will typically add additional fields */ }; -/* Every module implementation uses a subclass of the following structure -** to describe cursors that point into the virtual table and are used +/* +** CAPI3REF: Virtual Table Cursor Object +** KEYWORDS: sqlite3_vtab_cursor {virtual table cursor} +** +** Every [virtual table module] implementation uses a subclass of the +** following structure to describe cursors that point into the +** [virtual table] and are used ** to loop through the virtual table. Cursors are created using the -** xOpen method of the module. Each module implementation will define +** [sqlite3_module.xOpen | xOpen] method of the module and are destroyed +** by the [sqlite3_module.xClose | xClose] method. Cursors are used +** by the [xFilter], [xNext], [xEof], [xColumn], and [xRowid] methods +** of the module. Each module implementation will define ** the content of a cursor structure to suit its own needs. ** ** This superclass exists in order to define fields of the cursor that @@ -2532,138 +7689,3183 @@ struct sqlite3_vtab_cursor { }; /* -** The xCreate and xConnect methods of a module use the following API +** CAPI3REF: Declare The Schema Of A Virtual Table +** +** ^The [xCreate] and [xConnect] methods of a +** [virtual table module] call this interface ** to declare the format (the names and datatypes of the columns) of ** the virtual tables they implement. */ -int sqlite3_declare_vtab(sqlite3*, const char *zCreateTable); +SQLITE_API int sqlite3_declare_vtab(sqlite3*, const char *zSQL); /* -** Virtual tables can provide alternative implementations of functions -** using the xFindFunction method. But global versions of those functions -** must exist in order to be overloaded. +** CAPI3REF: Overload A Function For A Virtual Table +** METHOD: sqlite3 ** -** This API makes sure a global version of a function with a particular +** ^(Virtual tables can provide alternative implementations of functions +** using the [xFindFunction] method of the [virtual table module]. +** But global versions of those functions +** must exist in order to be overloaded.)^ +** +** ^(This API makes sure a global version of a function with a particular ** name and number of parameters exists. If no such function exists -** before this API is called, a new function is created. The implementation +** before this API is called, a new function is created.)^ ^The implementation ** of the new function always causes an exception to be thrown. So ** the new function is not good for anything by itself. Its only -** purpose is to be a place-holder function that can be overloaded -** by virtual tables. -** -** This API should be considered part of the virtual table interface, -** which is experimental and subject to change. -*/ -int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); - -/* -** The interface to the virtual-table mechanism defined above (back up -** to a comment remarkably similar to this one) is currently considered -** to be experimental. The interface might change in incompatible ways. -** If this is a problem for you, do not use the interface at this time. -** -** When the virtual-table mechanism stablizes, we will declare the -** interface fixed, support it indefinitely, and remove this comment. -** -****** EXPERIMENTAL - subject to change without notice ************** +** purpose is to be a placeholder function that can be overloaded +** by a [virtual table]. */ +SQLITE_API int sqlite3_overload_function(sqlite3*, const char *zFuncName, int nArg); /* ** CAPI3REF: A Handle To An Open BLOB +** KEYWORDS: {BLOB handle} {BLOB handles} ** -** An instance of the following opaque structure is used to -** represent an blob-handle. A blob-handle is created by -** [sqlite3_blob_open()] and destroyed by [sqlite3_blob_close()]. -** The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces -** can be used to read or write small subsections of the blob. -** The [sqltie3_blob_size()] interface returns the size of the -** blob in bytes. +** An instance of this object represents an open BLOB on which +** [sqlite3_blob_open | incremental BLOB I/O] can be performed. +** ^Objects of this type are created by [sqlite3_blob_open()] +** and destroyed by [sqlite3_blob_close()]. +** ^The [sqlite3_blob_read()] and [sqlite3_blob_write()] interfaces +** can be used to read or write small subsections of the BLOB. +** ^The [sqlite3_blob_bytes()] interface returns the size of the BLOB in bytes. */ typedef struct sqlite3_blob sqlite3_blob; /* ** CAPI3REF: Open A BLOB For Incremental I/O +** METHOD: sqlite3 +** CONSTRUCTOR: sqlite3_blob ** -** Open a handle to the blob located in row iRow,, column zColumn, -** table zTable in database zDb. i.e. the same blob that would -** be selected by: +** ^(This interfaces opens a [BLOB handle | handle] to the BLOB located +** in row iRow, column zColumn, table zTable in database zDb; +** in other words, the same BLOB that would be selected by: ** ** 
          -**     SELECT zColumn FROM zDb.zTable WHERE rowid = iRow;
-**
          +** SELECT zColumn FROM zDb.zTable WHERE [rowid] = iRow; +** )^ ** -** If the flags parameter is non-zero, the blob is opened for -** read and write access. If it is zero, the blob is opened for read -** access. +** ^(Parameter zDb is not the filename that contains the database, but +** rather the symbolic name of the database. For attached databases, this is +** the name that appears after the AS keyword in the [ATTACH] statement. +** For the main database file, the database name is "main". For TEMP +** tables, the database name is "temp".)^ ** -** On success, [SQLITE_OK] is returned and the new -** [sqlite3_blob | blob handle] is written to *ppBlob. -** Otherwise an error code is returned and -** any value written to *ppBlob should not be used by the caller. -** This function sets the database-handle error code and message -** accessible via [sqlite3_errcode()] and [sqlite3_errmsg()]. +** ^If the flags parameter is non-zero, then the BLOB is opened for read +** and write access. ^If the flags parameter is zero, the BLOB is opened for +** read-only access. +** +** ^(On success, [SQLITE_OK] is returned and the new [BLOB handle] is stored +** in *ppBlob. Otherwise an [error code] is returned and, unless the error +** code is SQLITE_MISUSE, *ppBlob is set to NULL.)^ ^This means that, provided +** the API is not misused, it is always safe to call [sqlite3_blob_close()] +** on *ppBlob after this function it returns. +** +** This function fails with SQLITE_ERROR if any of the following are true: +**
            +**
          • ^(Database zDb does not exist)^, +**
          • ^(Table zTable does not exist within database zDb)^, +**
          • ^(Table zTable is a WITHOUT ROWID table)^, +**
          • ^(Column zColumn does not exist)^, +**
          • ^(Row iRow is not present in the table)^, +**
          • ^(The specified column of row iRow contains a value that is not +** a TEXT or BLOB value)^, +**
          • ^(Column zColumn is part of an index, PRIMARY KEY or UNIQUE +** constraint and the blob is being opened for read/write access)^, +**
          • ^([foreign key constraints | Foreign key constraints] are enabled, +** column zColumn is part of a [child key] definition and the blob is +** being opened for read/write access)^. +**
          +** +** ^Unless it returns SQLITE_MISUSE, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** +** A BLOB referenced by sqlite3_blob_open() may be read using the +** [sqlite3_blob_read()] interface and modified by using +** [sqlite3_blob_write()]. The [BLOB handle] can be moved to a +** different row of the same table using the [sqlite3_blob_reopen()] +** interface. However, the column, table, or database of a [BLOB handle] +** cannot be changed after the [BLOB handle] is opened. +** +** ^(If the row that a BLOB handle points to is modified by an +** [UPDATE], [DELETE], or by [ON CONFLICT] side-effects +** then the BLOB handle is marked as "expired". +** This is true if any column of the row is changed, even a column +** other than the one the BLOB handle is open on.)^ +** ^Calls to [sqlite3_blob_read()] and [sqlite3_blob_write()] for +** an expired BLOB handle fail with a return code of [SQLITE_ABORT]. +** ^(Changes written into a BLOB prior to the BLOB expiring are not +** rolled back by the expiration of the BLOB. Such changes will eventually +** commit if the transaction continues to completion.)^ +** +** ^Use the [sqlite3_blob_bytes()] interface to determine the size of +** the opened blob. ^The size of a blob may not be changed by this +** interface. Use the [UPDATE] SQL command to change the size of a +** blob. +** +** ^The [sqlite3_bind_zeroblob()] and [sqlite3_result_zeroblob()] interfaces +** and the built-in [zeroblob] SQL function may be used to create a +** zero-filled blob to read or write using the incremental-blob interface. +** +** To avoid a resource leak, every open [BLOB handle] should eventually +** be released by a call to [sqlite3_blob_close()]. +** +** See also: [sqlite3_blob_close()], +** [sqlite3_blob_reopen()], [sqlite3_blob_read()], +** [sqlite3_blob_bytes()], [sqlite3_blob_write()]. */ -int sqlite3_blob_open( +SQLITE_API int sqlite3_blob_open( sqlite3*, const char *zDb, const char *zTable, const char *zColumn, - sqlite_int64 iRow, + sqlite3_int64 iRow, int flags, sqlite3_blob **ppBlob ); /* -** CAPI3REF: Close A BLOB Handle +** CAPI3REF: Move a BLOB Handle to a New Row +** METHOD: sqlite3_blob ** -** Close an open [sqlite3_blob | blob handle]. +** ^This function is used to move an existing [BLOB handle] so that it points +** to a different row of the same database table. ^The new row is identified +** by the rowid value passed as the second argument. Only the row can be +** changed. ^The database, table and column on which the blob handle is open +** remain the same. Moving an existing [BLOB handle] to a new row is +** faster than closing the existing handle and opening a new one. +** +** ^(The new row must meet the same criteria as for [sqlite3_blob_open()] - +** it must exist and there must be either a blob or text value stored in +** the nominated column.)^ ^If the new row is not present in the table, or if +** it does not contain a blob or text value, or if another error occurs, an +** SQLite error code is returned and the blob handle is considered aborted. +** ^All subsequent calls to [sqlite3_blob_read()], [sqlite3_blob_write()] or +** [sqlite3_blob_reopen()] on an aborted blob handle immediately return +** SQLITE_ABORT. ^Calling [sqlite3_blob_bytes()] on an aborted blob handle +** always returns zero. +** +** ^This function sets the database handle error code and message. */ -int sqlite3_blob_close(sqlite3_blob *); +SQLITE_API int sqlite3_blob_reopen(sqlite3_blob *, sqlite3_int64); /* -** CAPI3REF: Return The Size Of An Open BLOB +** CAPI3REF: Close A BLOB Handle +** DESTRUCTOR: sqlite3_blob ** -** Return the size in bytes of the blob accessible via the open -** [sqlite3_blob | blob-handle] passed as an argument. +** ^This function closes an open [BLOB handle]. ^(The BLOB handle is closed +** unconditionally. Even if this routine returns an error code, the +** handle is still closed.)^ +** +** ^If the blob handle being closed was opened for read-write access, and if +** the database is in auto-commit mode and there are no other open read-write +** blob handles or active write statements, the current transaction is +** committed. ^If an error occurs while committing the transaction, an error +** code is returned and the transaction rolled back. +** +** Calling this function with an argument that is not a NULL pointer or an +** open blob handle results in undefined behavior. ^Calling this routine +** with a null pointer (such as would be returned by a failed call to +** [sqlite3_blob_open()]) is a harmless no-op. ^Otherwise, if this function +** is passed a valid open blob handle, the values returned by the +** sqlite3_errcode() and sqlite3_errmsg() functions are set before returning. */ -int sqlite3_blob_bytes(sqlite3_blob *); +SQLITE_API int sqlite3_blob_close(sqlite3_blob *); /* -** CAPI3REF: Read Data From A BLOB Incrementally +** CAPI3REF: Return The Size Of An Open BLOB +** METHOD: sqlite3_blob ** -** This function is used to read data from an open -** [sqlite3_blob | blob-handle] into a caller supplied buffer. -** n bytes of data are copied into buffer -** z from the open blob, starting at offset iOffset. +** ^Returns the size in bytes of the BLOB accessible via the +** successfully opened [BLOB handle] in its only argument. ^The +** incremental blob I/O routines can only read or overwriting existing +** blob content; they cannot change the size of a blob. ** -** On success, SQLITE_OK is returned. Otherwise, an -** [SQLITE_ERROR | SQLite error code] or an -** [SQLITE_IOERR_READ | extended error code] is returned. +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. */ -int sqlite3_blob_read(sqlite3_blob *, void *z, int n, int iOffset); +SQLITE_API int sqlite3_blob_bytes(sqlite3_blob *); /* -** CAPI3REF: Write Data Into A BLOB Incrementally +** CAPI3REF: Read Data From A BLOB Incrementally +** METHOD: sqlite3_blob ** -** This function is used to write data into an open -** [sqlite3_blob | blob-handle] from a user supplied buffer. -** n bytes of data are copied from the buffer -** pointed to by z into the open blob, starting at offset iOffset. +** ^(This function is used to read data from an open [BLOB handle] into a +** caller-supplied buffer. N bytes of data are copied into buffer Z +** from the open BLOB, starting at offset iOffset.)^ ** -** If the [sqlite3_blob | blob-handle] passed as the first argument -** was not opened for writing (the flags parameter to [sqlite3_blob_open()] -*** was zero), this function returns [SQLITE_READONLY]. +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is read. ^If N or iOffset is +** less than zero, [SQLITE_ERROR] is returned and no data is read. +** ^The size of the blob (and hence the maximum value of N+iOffset) +** can be determined using the [sqlite3_blob_bytes()] interface. ** -** This function may only modify the contents of the blob, it is -** not possible to increase the size of a blob using this API. If -** offset iOffset is less than n bytes from the end of the blob, -** [SQLITE_ERROR] is returned and no data is written. +** ^An attempt to read from an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ** -** On success, SQLITE_OK is returned. Otherwise, an -** [SQLITE_ERROR | SQLite error code] or an -** [SQLITE_IOERR_READ | extended error code] is returned. +** ^(On success, sqlite3_blob_read() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_write()]. */ -int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); +SQLITE_API int sqlite3_blob_read(sqlite3_blob *, void *Z, int N, int iOffset); + +/* +** CAPI3REF: Write Data Into A BLOB Incrementally +** METHOD: sqlite3_blob +** +** ^(This function is used to write data into an open [BLOB handle] from a +** caller-supplied buffer. N bytes of data are copied from the buffer Z +** into the open BLOB, starting at offset iOffset.)^ +** +** ^(On success, sqlite3_blob_write() returns SQLITE_OK. +** Otherwise, an [error code] or an [extended error code] is returned.)^ +** ^Unless SQLITE_MISUSE is returned, this function sets the +** [database connection] error code and message accessible via +** [sqlite3_errcode()] and [sqlite3_errmsg()] and related functions. +** +** ^If the [BLOB handle] passed as the first argument was not opened for +** writing (the flags parameter to [sqlite3_blob_open()] was zero), +** this function returns [SQLITE_READONLY]. +** +** This function may only modify the contents of the BLOB; it is +** not possible to increase the size of a BLOB using this API. +** ^If offset iOffset is less than N bytes from the end of the BLOB, +** [SQLITE_ERROR] is returned and no data is written. The size of the +** BLOB (and hence the maximum value of N+iOffset) can be determined +** using the [sqlite3_blob_bytes()] interface. ^If N or iOffset are less +** than zero [SQLITE_ERROR] is returned and no data is written. +** +** ^An attempt to write to an expired [BLOB handle] fails with an +** error code of [SQLITE_ABORT]. ^Writes to the BLOB that occurred +** before the [BLOB handle] expired are not rolled back by the +** expiration of the handle, though of course those changes might +** have been overwritten by the statement that expired the BLOB handle +** or by other independent statements. +** +** This routine only works on a [BLOB handle] which has been created +** by a prior successful call to [sqlite3_blob_open()] and which has not +** been closed by [sqlite3_blob_close()]. Passing any other pointer in +** to this routine results in undefined and probably undesirable behavior. +** +** See also: [sqlite3_blob_read()]. +*/ +SQLITE_API int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); + +/* +** CAPI3REF: Virtual File System Objects +** +** A virtual filesystem (VFS) is an [sqlite3_vfs] object +** that SQLite uses to interact +** with the underlying operating system. Most SQLite builds come with a +** single default VFS that is appropriate for the host computer. +** New VFSes can be registered and existing VFSes can be unregistered. +** The following interfaces are provided. +** +** ^The sqlite3_vfs_find() interface returns a pointer to a VFS given its name. +** ^Names are case sensitive. +** ^Names are zero-terminated UTF-8 strings. +** ^If there is no match, a NULL pointer is returned. +** ^If zVfsName is NULL then the default VFS is returned. +** +** ^New VFSes are registered with sqlite3_vfs_register(). +** ^Each new VFS becomes the default VFS if the makeDflt flag is set. +** ^The same VFS can be registered multiple times without injury. +** ^To make an existing VFS into the default VFS, register it again +** with the makeDflt flag set. If two different VFSes with the +** same name are registered, the behavior is undefined. If a +** VFS is registered with a name that is NULL or an empty string, +** then the behavior is undefined. +** +** ^Unregister a VFS with the sqlite3_vfs_unregister() interface. +** ^(If the default VFS is unregistered, another VFS is chosen as +** the default. The choice for the new VFS is arbitrary.)^ +*/ +SQLITE_API sqlite3_vfs *sqlite3_vfs_find(const char *zVfsName); +SQLITE_API int sqlite3_vfs_register(sqlite3_vfs*, int makeDflt); +SQLITE_API int sqlite3_vfs_unregister(sqlite3_vfs*); + +/* +** CAPI3REF: Mutexes +** +** The SQLite core uses these routines for thread +** synchronization. Though they are intended for internal +** use by SQLite, code that links against SQLite is +** permitted to use any of these routines. +** +** The SQLite source code contains multiple implementations +** of these mutex routines. An appropriate implementation +** is selected automatically at compile-time. The following +** implementations are available in the SQLite core: +** +**
            +**
          • SQLITE_MUTEX_PTHREADS +**
          • SQLITE_MUTEX_W32 +**
          • SQLITE_MUTEX_NOOP +**
          +** +** The SQLITE_MUTEX_NOOP implementation is a set of routines +** that does no real locking and is appropriate for use in +** a single-threaded application. The SQLITE_MUTEX_PTHREADS and +** SQLITE_MUTEX_W32 implementations are appropriate for use on Unix +** and Windows. +** +** If SQLite is compiled with the SQLITE_MUTEX_APPDEF preprocessor +** macro defined (with "-DSQLITE_MUTEX_APPDEF=1"), then no mutex +** implementation is included with the library. In this case the +** application must supply a custom mutex implementation using the +** [SQLITE_CONFIG_MUTEX] option of the sqlite3_config() function +** before calling sqlite3_initialize() or any other public sqlite3_ +** function that calls sqlite3_initialize(). +** +** ^The sqlite3_mutex_alloc() routine allocates a new +** mutex and returns a pointer to it. ^The sqlite3_mutex_alloc() +** routine returns NULL if it is unable to allocate the requested +** mutex. The argument to sqlite3_mutex_alloc() must one of these +** integer constants: +** +**
            +**
          • SQLITE_MUTEX_FAST +**
          • SQLITE_MUTEX_RECURSIVE +**
          • SQLITE_MUTEX_STATIC_MAIN +**
          • SQLITE_MUTEX_STATIC_MEM +**
          • SQLITE_MUTEX_STATIC_OPEN +**
          • SQLITE_MUTEX_STATIC_PRNG +**
          • SQLITE_MUTEX_STATIC_LRU +**
          • SQLITE_MUTEX_STATIC_PMEM +**
          • SQLITE_MUTEX_STATIC_APP1 +**
          • SQLITE_MUTEX_STATIC_APP2 +**
          • SQLITE_MUTEX_STATIC_APP3 +**
          • SQLITE_MUTEX_STATIC_VFS1 +**
          • SQLITE_MUTEX_STATIC_VFS2 +**
          • SQLITE_MUTEX_STATIC_VFS3 +**
          +** +** ^The first two constants (SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) +** cause sqlite3_mutex_alloc() to create +** a new mutex. ^The new mutex is recursive when SQLITE_MUTEX_RECURSIVE +** is used but not necessarily so when SQLITE_MUTEX_FAST is used. +** The mutex implementation does not need to make a distinction +** between SQLITE_MUTEX_RECURSIVE and SQLITE_MUTEX_FAST if it does +** not want to. SQLite will only request a recursive mutex in +** cases where it really needs one. If a faster non-recursive mutex +** implementation is available on the host platform, the mutex subsystem +** might return such a mutex in response to SQLITE_MUTEX_FAST. +** +** ^The other allowed parameters to sqlite3_mutex_alloc() (anything other +** than SQLITE_MUTEX_FAST and SQLITE_MUTEX_RECURSIVE) each return +** a pointer to a static preexisting mutex. ^Nine static mutexes are +** used by the current version of SQLite. Future versions of SQLite +** may add additional static mutexes. Static mutexes are for internal +** use by SQLite only. Applications that use SQLite mutexes should +** use only the dynamic mutexes returned by SQLITE_MUTEX_FAST or +** SQLITE_MUTEX_RECURSIVE. +** +** ^Note that if one of the dynamic mutex parameters (SQLITE_MUTEX_FAST +** or SQLITE_MUTEX_RECURSIVE) is used then sqlite3_mutex_alloc() +** returns a different mutex on every call. ^For the static +** mutex types, the same mutex is returned on every call that has +** the same type number. +** +** ^The sqlite3_mutex_free() routine deallocates a previously +** allocated dynamic mutex. Attempting to deallocate a static +** mutex results in undefined behavior. +** +** ^The sqlite3_mutex_enter() and sqlite3_mutex_try() routines attempt +** to enter a mutex. ^If another thread is already within the mutex, +** sqlite3_mutex_enter() will block and sqlite3_mutex_try() will return +** SQLITE_BUSY. ^The sqlite3_mutex_try() interface returns [SQLITE_OK] +** upon successful entry. ^(Mutexes created using +** SQLITE_MUTEX_RECURSIVE can be entered multiple times by the same thread. +** In such cases, the +** mutex must be exited an equal number of times before another thread +** can enter.)^ If the same thread tries to enter any mutex other +** than an SQLITE_MUTEX_RECURSIVE more than once, the behavior is undefined. +** +** ^(Some systems (for example, Windows 95) do not support the operation +** implemented by sqlite3_mutex_try(). On those systems, sqlite3_mutex_try() +** will always return SQLITE_BUSY. In most cases the SQLite core only uses +** sqlite3_mutex_try() as an optimization, so this is acceptable +** behavior. The exceptions are unix builds that set the +** SQLITE_ENABLE_SETLK_TIMEOUT build option. In that case a working +** sqlite3_mutex_try() is required.)^ +** +** ^The sqlite3_mutex_leave() routine exits a mutex that was +** previously entered by the same thread. The behavior +** is undefined if the mutex is not currently entered by the +** calling thread or is not currently allocated. +** +** ^If the argument to sqlite3_mutex_enter(), sqlite3_mutex_try(), +** sqlite3_mutex_leave(), or sqlite3_mutex_free() is a NULL pointer, +** then any of the four routines behaves as a no-op. +** +** See also: [sqlite3_mutex_held()] and [sqlite3_mutex_notheld()]. +*/ +SQLITE_API sqlite3_mutex *sqlite3_mutex_alloc(int); +SQLITE_API void sqlite3_mutex_free(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_enter(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_try(sqlite3_mutex*); +SQLITE_API void sqlite3_mutex_leave(sqlite3_mutex*); + +/* +** CAPI3REF: Mutex Methods Object +** +** An instance of this structure defines the low-level routines +** used to allocate and use mutexes. +** +** Usually, the default mutex implementations provided by SQLite are +** sufficient, however the application has the option of substituting a custom +** implementation for specialized deployments or systems for which SQLite +** does not provide a suitable implementation. In this case, the application +** creates and populates an instance of this structure to pass +** to sqlite3_config() along with the [SQLITE_CONFIG_MUTEX] option. +** Additionally, an instance of this structure can be used as an +** output variable when querying the system for the current mutex +** implementation, using the [SQLITE_CONFIG_GETMUTEX] option. +** +** ^The xMutexInit method defined by this structure is invoked as +** part of system initialization by the sqlite3_initialize() function. +** ^The xMutexInit routine is called by SQLite exactly once for each +** effective call to [sqlite3_initialize()]. +** +** ^The xMutexEnd method defined by this structure is invoked as +** part of system shutdown by the sqlite3_shutdown() function. The +** implementation of this method is expected to release all outstanding +** resources obtained by the mutex methods implementation, especially +** those obtained by the xMutexInit method. ^The xMutexEnd() +** interface is invoked exactly once for each call to [sqlite3_shutdown()]. +** +** ^(The remaining seven methods defined by this structure (xMutexAlloc, +** xMutexFree, xMutexEnter, xMutexTry, xMutexLeave, xMutexHeld and +** xMutexNotheld) implement the following interfaces (respectively): +** +**
            +**
          • [sqlite3_mutex_alloc()]
            • +**
          • [sqlite3_mutex_free()]
            • +**
          • [sqlite3_mutex_enter()]
            • +**
          • [sqlite3_mutex_try()]
            • +**
          • [sqlite3_mutex_leave()]
            • +**
          • [sqlite3_mutex_held()]
            • +**
          • [sqlite3_mutex_notheld()]
            • +**
          )^ +** +** The only difference is that the public sqlite3_XXX functions enumerated +** above silently ignore any invocations that pass a NULL pointer instead +** of a valid mutex handle. The implementations of the methods defined +** by this structure are not required to handle this case. The results +** of passing a NULL pointer instead of a valid mutex handle are undefined +** (i.e. it is acceptable to provide an implementation that segfaults if +** it is passed a NULL pointer). +** +** The xMutexInit() method must be threadsafe. It must be harmless to +** invoke xMutexInit() multiple times within the same process and without +** intervening calls to xMutexEnd(). Second and subsequent calls to +** xMutexInit() must be no-ops. +** +** xMutexInit() must not use SQLite memory allocation ([sqlite3_malloc()] +** and its associates). Similarly, xMutexAlloc() must not use SQLite memory +** allocation for a static mutex. ^However xMutexAlloc() may use SQLite +** memory allocation for a fast or recursive mutex. +** +** ^SQLite will invoke the xMutexEnd() method when [sqlite3_shutdown()] is +** called, but only if the prior call to xMutexInit returned SQLITE_OK. +** If xMutexInit fails in any way, it is expected to clean up after itself +** prior to returning. +*/ +typedef struct sqlite3_mutex_methods sqlite3_mutex_methods; +struct sqlite3_mutex_methods { + int (*xMutexInit)(void); + int (*xMutexEnd)(void); + sqlite3_mutex *(*xMutexAlloc)(int); + void (*xMutexFree)(sqlite3_mutex *); + void (*xMutexEnter)(sqlite3_mutex *); + int (*xMutexTry)(sqlite3_mutex *); + void (*xMutexLeave)(sqlite3_mutex *); + int (*xMutexHeld)(sqlite3_mutex *); + int (*xMutexNotheld)(sqlite3_mutex *); +}; + +/* +** CAPI3REF: Mutex Verification Routines +** +** The sqlite3_mutex_held() and sqlite3_mutex_notheld() routines +** are intended for use inside assert() statements. The SQLite core +** never uses these routines except inside an assert() and applications +** are advised to follow the lead of the core. The SQLite core only +** provides implementations for these routines when it is compiled +** with the SQLITE_DEBUG flag. External mutex implementations +** are only required to provide these routines if SQLITE_DEBUG is +** defined and if NDEBUG is not defined. +** +** These routines should return true if the mutex in their argument +** is held or not held, respectively, by the calling thread. +** +** The implementation is not required to provide versions of these +** routines that actually work. If the implementation does not provide working +** versions of these routines, it should at least provide stubs that always +** return true so that one does not get spurious assertion failures. +** +** If the argument to sqlite3_mutex_held() is a NULL pointer then +** the routine should return 1. This seems counter-intuitive since +** clearly the mutex cannot be held if it does not exist. But +** the reason the mutex does not exist is because the build is not +** using mutexes. And we do not want the assert() containing the +** call to sqlite3_mutex_held() to fail, so a non-zero return is +** the appropriate thing to do. The sqlite3_mutex_notheld() +** interface should also return 1 when given a NULL pointer. +*/ +#ifndef NDEBUG +SQLITE_API int sqlite3_mutex_held(sqlite3_mutex*); +SQLITE_API int sqlite3_mutex_notheld(sqlite3_mutex*); +#endif + +/* +** CAPI3REF: Mutex Types +** +** The [sqlite3_mutex_alloc()] interface takes a single argument +** which is one of these integer constants. +** +** The set of static mutexes may change from one SQLite release to the +** next. Applications that override the built-in mutex logic must be +** prepared to accommodate additional static mutexes. +*/ +#define SQLITE_MUTEX_FAST 0 +#define SQLITE_MUTEX_RECURSIVE 1 +#define SQLITE_MUTEX_STATIC_MAIN 2 +#define SQLITE_MUTEX_STATIC_MEM 3 /* sqlite3_malloc() */ +#define SQLITE_MUTEX_STATIC_MEM2 4 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_OPEN 4 /* sqlite3BtreeOpen() */ +#define SQLITE_MUTEX_STATIC_PRNG 5 /* sqlite3_randomness() */ +#define SQLITE_MUTEX_STATIC_LRU 6 /* lru page list */ +#define SQLITE_MUTEX_STATIC_LRU2 7 /* NOT USED */ +#define SQLITE_MUTEX_STATIC_PMEM 7 /* sqlite3PageMalloc() */ +#define SQLITE_MUTEX_STATIC_APP1 8 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP2 9 /* For use by application */ +#define SQLITE_MUTEX_STATIC_APP3 10 /* For use by application */ +#define SQLITE_MUTEX_STATIC_VFS1 11 /* For use by built-in VFS */ +#define SQLITE_MUTEX_STATIC_VFS2 12 /* For use by extension VFS */ +#define SQLITE_MUTEX_STATIC_VFS3 13 /* For use by application VFS */ + +/* Legacy compatibility: */ +#define SQLITE_MUTEX_STATIC_MASTER 2 + + +/* +** CAPI3REF: Retrieve the mutex for a database connection +** METHOD: sqlite3 +** +** ^This interface returns a pointer the [sqlite3_mutex] object that +** serializes access to the [database connection] given in the argument +** when the [threading mode] is Serialized. +** ^If the [threading mode] is Single-thread or Multi-thread then this +** routine returns a NULL pointer. +*/ +SQLITE_API sqlite3_mutex *sqlite3_db_mutex(sqlite3*); + +/* +** CAPI3REF: Low-Level Control Of Database Files +** METHOD: sqlite3 +** KEYWORDS: {file control} +** +** ^The [sqlite3_file_control()] interface makes a direct call to the +** xFileControl method for the [sqlite3_io_methods] object associated +** with a particular database identified by the second argument. ^The +** name of the database is "main" for the main database or "temp" for the +** TEMP database, or the name that appears after the AS keyword for +** databases that are added using the [ATTACH] SQL command. +** ^A NULL pointer can be used in place of "main" to refer to the +** main database file. +** ^The third and fourth parameters to this routine +** are passed directly through to the second and third parameters of +** the xFileControl method. ^The return value of the xFileControl +** method becomes the return value of this routine. +** +** A few opcodes for [sqlite3_file_control()] are handled directly +** by the SQLite core and never invoke the +** sqlite3_io_methods.xFileControl method. +** ^The [SQLITE_FCNTL_FILE_POINTER] value for the op parameter causes +** a pointer to the underlying [sqlite3_file] object to be written into +** the space pointed to by the 4th parameter. The +** [SQLITE_FCNTL_JOURNAL_POINTER] works similarly except that it returns +** the [sqlite3_file] object associated with the journal file instead of +** the main database. The [SQLITE_FCNTL_VFS_POINTER] opcode returns +** a pointer to the underlying [sqlite3_vfs] object for the file. +** The [SQLITE_FCNTL_DATA_VERSION] returns the data version counter +** from the pager. +** +** ^If the second parameter (zDbName) does not match the name of any +** open database file, then SQLITE_ERROR is returned. ^This error +** code is not remembered and will not be recalled by [sqlite3_errcode()] +** or [sqlite3_errmsg()]. The underlying xFileControl method might +** also return SQLITE_ERROR. There is no way to distinguish between +** an incorrect zDbName and an SQLITE_ERROR return from the underlying +** xFileControl method. +** +** See also: [file control opcodes] +*/ +SQLITE_API int sqlite3_file_control(sqlite3*, const char *zDbName, int op, void*); + +/* +** CAPI3REF: Testing Interface +** +** ^The sqlite3_test_control() interface is used to read out internal +** state of SQLite and to inject faults into SQLite for testing +** purposes. ^The first parameter is an operation code that determines +** the number, meaning, and operation of all subsequent parameters. +** +** This interface is not for use by applications. It exists solely +** for verifying the correct operation of the SQLite library. Depending +** on how the SQLite library is compiled, this interface might not exist. +** +** The details of the operation codes, their meanings, the parameters +** they take, and what they do are all subject to change without notice. +** Unlike most of the SQLite API, this function is not guaranteed to +** operate consistently from one release to the next. +*/ +SQLITE_API int sqlite3_test_control(int op, ...); + +/* +** CAPI3REF: Testing Interface Operation Codes +** +** These constants are the valid operation code parameters used +** as the first argument to [sqlite3_test_control()]. +** +** These parameters and their meanings are subject to change +** without notice. These values are for testing purposes only. +** Applications should not use any of these parameters or the +** [sqlite3_test_control()] interface. +*/ +#define SQLITE_TESTCTRL_FIRST 5 +#define SQLITE_TESTCTRL_PRNG_SAVE 5 +#define SQLITE_TESTCTRL_PRNG_RESTORE 6 +#define SQLITE_TESTCTRL_PRNG_RESET 7 /* NOT USED */ +#define SQLITE_TESTCTRL_FK_NO_ACTION 7 +#define SQLITE_TESTCTRL_BITVEC_TEST 8 +#define SQLITE_TESTCTRL_FAULT_INSTALL 9 +#define SQLITE_TESTCTRL_BENIGN_MALLOC_HOOKS 10 +#define SQLITE_TESTCTRL_PENDING_BYTE 11 +#define SQLITE_TESTCTRL_ASSERT 12 +#define SQLITE_TESTCTRL_ALWAYS 13 +#define SQLITE_TESTCTRL_RESERVE 14 /* NOT USED */ +#define SQLITE_TESTCTRL_JSON_SELFCHECK 14 +#define SQLITE_TESTCTRL_OPTIMIZATIONS 15 +#define SQLITE_TESTCTRL_ISKEYWORD 16 /* NOT USED */ +#define SQLITE_TESTCTRL_GETOPT 16 +#define SQLITE_TESTCTRL_SCRATCHMALLOC 17 /* NOT USED */ +#define SQLITE_TESTCTRL_INTERNAL_FUNCTIONS 17 +#define SQLITE_TESTCTRL_LOCALTIME_FAULT 18 +#define SQLITE_TESTCTRL_EXPLAIN_STMT 19 /* NOT USED */ +#define SQLITE_TESTCTRL_ONCE_RESET_THRESHOLD 19 +#define SQLITE_TESTCTRL_NEVER_CORRUPT 20 +#define SQLITE_TESTCTRL_VDBE_COVERAGE 21 +#define SQLITE_TESTCTRL_BYTEORDER 22 +#define SQLITE_TESTCTRL_ISINIT 23 +#define SQLITE_TESTCTRL_SORTER_MMAP 24 +#define SQLITE_TESTCTRL_IMPOSTER 25 +#define SQLITE_TESTCTRL_PARSER_COVERAGE 26 +#define SQLITE_TESTCTRL_RESULT_INTREAL 27 +#define SQLITE_TESTCTRL_PRNG_SEED 28 +#define SQLITE_TESTCTRL_EXTRA_SCHEMA_CHECKS 29 +#define SQLITE_TESTCTRL_SEEK_COUNT 30 +#define SQLITE_TESTCTRL_TRACEFLAGS 31 +#define SQLITE_TESTCTRL_TUNE 32 +#define SQLITE_TESTCTRL_LOGEST 33 +#define SQLITE_TESTCTRL_USELONGDOUBLE 34 /* NOT USED */ +#define SQLITE_TESTCTRL_LAST 34 /* Largest TESTCTRL */ + +/* +** CAPI3REF: SQL Keyword Checking +** +** These routines provide access to the set of SQL language keywords +** recognized by SQLite. Applications can uses these routines to determine +** whether or not a specific identifier needs to be escaped (for example, +** by enclosing in double-quotes) so as not to confuse the parser. +** +** The sqlite3_keyword_count() interface returns the number of distinct +** keywords understood by SQLite. +** +** The sqlite3_keyword_name(N,Z,L) interface finds the 0-based N-th keyword and +** makes *Z point to that keyword expressed as UTF8 and writes the number +** of bytes in the keyword into *L. The string that *Z points to is not +** zero-terminated. The sqlite3_keyword_name(N,Z,L) routine returns +** SQLITE_OK if N is within bounds and SQLITE_ERROR if not. If either Z +** or L are NULL or invalid pointers then calls to +** sqlite3_keyword_name(N,Z,L) result in undefined behavior. +** +** The sqlite3_keyword_check(Z,L) interface checks to see whether or not +** the L-byte UTF8 identifier that Z points to is a keyword, returning non-zero +** if it is and zero if not. +** +** The parser used by SQLite is forgiving. It is often possible to use +** a keyword as an identifier as long as such use does not result in a +** parsing ambiguity. For example, the statement +** "CREATE TABLE BEGIN(REPLACE,PRAGMA,END);" is accepted by SQLite, and +** creates a new table named "BEGIN" with three columns named +** "REPLACE", "PRAGMA", and "END". Nevertheless, best practice is to avoid +** using keywords as identifiers. Common techniques used to avoid keyword +** name collisions include: +**
            +**
          • Put all identifier names inside double-quotes. This is the official +** SQL way to escape identifier names. +**
          • Put identifier names inside [...]. This is not standard SQL, +** but it is what SQL Server does and so lots of programmers use this +** technique. +**
          • Begin every identifier with the letter "Z" as no SQL keywords start +** with "Z". +**
          • Include a digit somewhere in every identifier name. +**
          +** +** Note that the number of keywords understood by SQLite can depend on +** compile-time options. For example, "VACUUM" is not a keyword if +** SQLite is compiled with the [-DSQLITE_OMIT_VACUUM] option. Also, +** new keywords may be added to future releases of SQLite. +*/ +SQLITE_API int sqlite3_keyword_count(void); +SQLITE_API int sqlite3_keyword_name(int,const char**,int*); +SQLITE_API int sqlite3_keyword_check(const char*,int); + +/* +** CAPI3REF: Dynamic String Object +** KEYWORDS: {dynamic string} +** +** An instance of the sqlite3_str object contains a dynamically-sized +** string under construction. +** +** The lifecycle of an sqlite3_str object is as follows: +**
            +**
          1. ^The sqlite3_str object is created using [sqlite3_str_new()]. +**
          2. ^Text is appended to the sqlite3_str object using various +** methods, such as [sqlite3_str_appendf()]. +**
          3. ^The sqlite3_str object is destroyed and the string it created +** is returned using the [sqlite3_str_finish()] interface. +**
          +*/ +typedef struct sqlite3_str sqlite3_str; + +/* +** CAPI3REF: Create A New Dynamic String Object +** CONSTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_new(D)] interface allocates and initializes +** a new [sqlite3_str] object. To avoid memory leaks, the object returned by +** [sqlite3_str_new()] must be freed by a subsequent call to +** [sqlite3_str_finish(X)]. +** +** ^The [sqlite3_str_new(D)] interface always returns a pointer to a +** valid [sqlite3_str] object, though in the event of an out-of-memory +** error the returned object might be a special singleton that will +** silently reject new text, always return SQLITE_NOMEM from +** [sqlite3_str_errcode()], always return 0 for +** [sqlite3_str_length()], and always return NULL from +** [sqlite3_str_finish(X)]. It is always safe to use the value +** returned by [sqlite3_str_new(D)] as the sqlite3_str parameter +** to any of the other [sqlite3_str] methods. +** +** The D parameter to [sqlite3_str_new(D)] may be NULL. If the +** D parameter in [sqlite3_str_new(D)] is not NULL, then the maximum +** length of the string contained in the [sqlite3_str] object will be +** the value set for [sqlite3_limit](D,[SQLITE_LIMIT_LENGTH]) instead +** of [SQLITE_MAX_LENGTH]. +*/ +SQLITE_API sqlite3_str *sqlite3_str_new(sqlite3*); + +/* +** CAPI3REF: Finalize A Dynamic String +** DESTRUCTOR: sqlite3_str +** +** ^The [sqlite3_str_finish(X)] interface destroys the sqlite3_str object X +** and returns a pointer to a memory buffer obtained from [sqlite3_malloc64()] +** that contains the constructed string. The calling application should +** pass the returned value to [sqlite3_free()] to avoid a memory leak. +** ^The [sqlite3_str_finish(X)] interface may return a NULL pointer if any +** errors were encountered during construction of the string. ^The +** [sqlite3_str_finish(X)] interface will also return a NULL pointer if the +** string in [sqlite3_str] object X is zero bytes long. +*/ +SQLITE_API char *sqlite3_str_finish(sqlite3_str*); + +/* +** CAPI3REF: Add Content To A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces add content to an sqlite3_str object previously obtained +** from [sqlite3_str_new()]. +** +** ^The [sqlite3_str_appendf(X,F,...)] and +** [sqlite3_str_vappendf(X,F,V)] interfaces uses the [built-in printf] +** functionality of SQLite to append formatted text onto the end of +** [sqlite3_str] object X. +** +** ^The [sqlite3_str_append(X,S,N)] method appends exactly N bytes from string S +** onto the end of the [sqlite3_str] object X. N must be non-negative. +** S must contain at least N non-zero bytes of content. To append a +** zero-terminated string in its entirety, use the [sqlite3_str_appendall()] +** method instead. +** +** ^The [sqlite3_str_appendall(X,S)] method appends the complete content of +** zero-terminated string S onto the end of [sqlite3_str] object X. +** +** ^The [sqlite3_str_appendchar(X,N,C)] method appends N copies of the +** single-byte character C onto the end of [sqlite3_str] object X. +** ^This method can be used, for example, to add whitespace indentation. +** +** ^The [sqlite3_str_reset(X)] method resets the string under construction +** inside [sqlite3_str] object X back to zero bytes in length. +** +** These methods do not return a result code. ^If an error occurs, that fact +** is recorded in the [sqlite3_str] object and can be recovered by a +** subsequent call to [sqlite3_str_errcode(X)]. +*/ +SQLITE_API void sqlite3_str_appendf(sqlite3_str*, const char *zFormat, ...); +SQLITE_API void sqlite3_str_vappendf(sqlite3_str*, const char *zFormat, va_list); +SQLITE_API void sqlite3_str_append(sqlite3_str*, const char *zIn, int N); +SQLITE_API void sqlite3_str_appendall(sqlite3_str*, const char *zIn); +SQLITE_API void sqlite3_str_appendchar(sqlite3_str*, int N, char C); +SQLITE_API void sqlite3_str_reset(sqlite3_str*); + +/* +** CAPI3REF: Status Of A Dynamic String +** METHOD: sqlite3_str +** +** These interfaces return the current status of an [sqlite3_str] object. +** +** ^If any prior errors have occurred while constructing the dynamic string +** in sqlite3_str X, then the [sqlite3_str_errcode(X)] method will return +** an appropriate error code. ^The [sqlite3_str_errcode(X)] method returns +** [SQLITE_NOMEM] following any out-of-memory error, or +** [SQLITE_TOOBIG] if the size of the dynamic string exceeds +** [SQLITE_MAX_LENGTH], or [SQLITE_OK] if there have been no errors. +** +** ^The [sqlite3_str_length(X)] method returns the current length, in bytes, +** of the dynamic string under construction in [sqlite3_str] object X. +** ^The length returned by [sqlite3_str_length(X)] does not include the +** zero-termination byte. +** +** ^The [sqlite3_str_value(X)] method returns a pointer to the current +** content of the dynamic string under construction in X. The value +** returned by [sqlite3_str_value(X)] is managed by the sqlite3_str object X +** and might be freed or altered by any subsequent method on the same +** [sqlite3_str] object. Applications must not used the pointer returned +** [sqlite3_str_value(X)] after any subsequent method call on the same +** object. ^Applications may change the content of the string returned +** by [sqlite3_str_value(X)] as long as they do not write into any bytes +** outside the range of 0 to [sqlite3_str_length(X)] and do not read or +** write any byte after any subsequent sqlite3_str method call. +*/ +SQLITE_API int sqlite3_str_errcode(sqlite3_str*); +SQLITE_API int sqlite3_str_length(sqlite3_str*); +SQLITE_API char *sqlite3_str_value(sqlite3_str*); + +/* +** CAPI3REF: SQLite Runtime Status +** +** ^These interfaces are used to retrieve runtime status information +** about the performance of SQLite, and optionally to reset various +** highwater marks. ^The first argument is an integer code for +** the specific parameter to measure. ^(Recognized integer codes +** are of the form [status parameters | SQLITE_STATUS_...].)^ +** ^The current value of the parameter is returned into *pCurrent. +** ^The highest recorded value is returned in *pHighwater. ^If the +** resetFlag is true, then the highest record value is reset after +** *pHighwater is written. ^(Some parameters do not record the highest +** value. For those parameters +** nothing is written into *pHighwater and the resetFlag is ignored.)^ +** ^(Other parameters record only the highwater mark and not the current +** value. For these latter parameters nothing is written into *pCurrent.)^ +** +** ^The sqlite3_status() and sqlite3_status64() routines return +** SQLITE_OK on success and a non-zero [error code] on failure. +** +** If either the current value or the highwater mark is too large to +** be represented by a 32-bit integer, then the values returned by +** sqlite3_status() are undefined. +** +** See also: [sqlite3_db_status()] +*/ +SQLITE_API int sqlite3_status(int op, int *pCurrent, int *pHighwater, int resetFlag); +SQLITE_API int sqlite3_status64( + int op, + sqlite3_int64 *pCurrent, + sqlite3_int64 *pHighwater, + int resetFlag +); + + +/* +** CAPI3REF: Status Parameters +** KEYWORDS: {status parameters} +** +** These integer constants designate various run-time status parameters +** that can be returned by [sqlite3_status()]. +** +**
          +** [[SQLITE_STATUS_MEMORY_USED]] ^(
          SQLITE_STATUS_MEMORY_USED
          +**
          This parameter is the current amount of memory checked out +** using [sqlite3_malloc()], either directly or indirectly. The +** figure includes calls made to [sqlite3_malloc()] by the application +** and internal memory usage by the SQLite library. Auxiliary page-cache +** memory controlled by [SQLITE_CONFIG_PAGECACHE] is not included in +** this parameter. The amount returned is the sum of the allocation +** sizes as reported by the xSize method in [sqlite3_mem_methods].
          )^ +** +** [[SQLITE_STATUS_MALLOC_SIZE]] ^(
          SQLITE_STATUS_MALLOC_SIZE
          +**
          This parameter records the largest memory allocation request +** handed to [sqlite3_malloc()] or [sqlite3_realloc()] (or their +** internal equivalents). Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
          )^ +** +** [[SQLITE_STATUS_MALLOC_COUNT]] ^(
          SQLITE_STATUS_MALLOC_COUNT
          +**
          This parameter records the number of separate memory allocations +** currently checked out.
          )^ +** +** [[SQLITE_STATUS_PAGECACHE_USED]] ^(
          SQLITE_STATUS_PAGECACHE_USED
          +**
          This parameter returns the number of pages used out of the +** [pagecache memory allocator] that was configured using +** [SQLITE_CONFIG_PAGECACHE]. The +** value returned is in pages, not in bytes.
          )^ +** +** [[SQLITE_STATUS_PAGECACHE_OVERFLOW]] +** ^(
          SQLITE_STATUS_PAGECACHE_OVERFLOW
          +**
          This parameter returns the number of bytes of page cache +** allocation which could not be satisfied by the [SQLITE_CONFIG_PAGECACHE] +** buffer and where forced to overflow to [sqlite3_malloc()]. The +** returned value includes allocations that overflowed because they +** where too large (they were larger than the "sz" parameter to +** [SQLITE_CONFIG_PAGECACHE]) and allocations that overflowed because +** no space was left in the page cache.
          )^ +** +** [[SQLITE_STATUS_PAGECACHE_SIZE]] ^(
          SQLITE_STATUS_PAGECACHE_SIZE
          +**
          This parameter records the largest memory allocation request +** handed to the [pagecache memory allocator]. Only the value returned in the +** *pHighwater parameter to [sqlite3_status()] is of interest. +** The value written into the *pCurrent parameter is undefined.
          )^ +** +** [[SQLITE_STATUS_SCRATCH_USED]]
          SQLITE_STATUS_SCRATCH_USED
          +**
          No longer used.
          +** +** [[SQLITE_STATUS_SCRATCH_OVERFLOW]] ^(
          SQLITE_STATUS_SCRATCH_OVERFLOW
          +**
          No longer used.
          +** +** [[SQLITE_STATUS_SCRATCH_SIZE]]
          SQLITE_STATUS_SCRATCH_SIZE
          +**
          No longer used.
          +** +** [[SQLITE_STATUS_PARSER_STACK]] ^(
          SQLITE_STATUS_PARSER_STACK
          +**
          The *pHighwater parameter records the deepest parser stack. +** The *pCurrent value is undefined. The *pHighwater value is only +** meaningful if SQLite is compiled with [YYTRACKMAXSTACKDEPTH].
          )^ +**
          +** +** New status parameters may be added from time to time. +*/ +#define SQLITE_STATUS_MEMORY_USED 0 +#define SQLITE_STATUS_PAGECACHE_USED 1 +#define SQLITE_STATUS_PAGECACHE_OVERFLOW 2 +#define SQLITE_STATUS_SCRATCH_USED 3 /* NOT USED */ +#define SQLITE_STATUS_SCRATCH_OVERFLOW 4 /* NOT USED */ +#define SQLITE_STATUS_MALLOC_SIZE 5 +#define SQLITE_STATUS_PARSER_STACK 6 +#define SQLITE_STATUS_PAGECACHE_SIZE 7 +#define SQLITE_STATUS_SCRATCH_SIZE 8 /* NOT USED */ +#define SQLITE_STATUS_MALLOC_COUNT 9 + +/* +** CAPI3REF: Database Connection Status +** METHOD: sqlite3 +** +** ^This interface is used to retrieve runtime status information +** about a single [database connection]. ^The first argument is the +** database connection object to be interrogated. ^The second argument +** is an integer constant, taken from the set of +** [SQLITE_DBSTATUS options], that +** determines the parameter to interrogate. The set of +** [SQLITE_DBSTATUS options] is likely +** to grow in future releases of SQLite. +** +** ^The current value of the requested parameter is written into *pCur +** and the highest instantaneous value is written into *pHiwtr. ^If +** the resetFlg is true, then the highest instantaneous value is +** reset back down to the current value. +** +** ^The sqlite3_db_status() routine returns SQLITE_OK on success and a +** non-zero [error code] on failure. +** +** See also: [sqlite3_status()] and [sqlite3_stmt_status()]. +*/ +SQLITE_API int sqlite3_db_status(sqlite3*, int op, int *pCur, int *pHiwtr, int resetFlg); + +/* +** CAPI3REF: Status Parameters for database connections +** KEYWORDS: {SQLITE_DBSTATUS options} +** +** These constants are the available integer "verbs" that can be passed as +** the second argument to the [sqlite3_db_status()] interface. +** +** New verbs may be added in future releases of SQLite. Existing verbs +** might be discontinued. Applications should check the return code from +** [sqlite3_db_status()] to make sure that the call worked. +** The [sqlite3_db_status()] interface will return a non-zero error code +** if a discontinued or unsupported verb is invoked. +** +**
          +** [[SQLITE_DBSTATUS_LOOKASIDE_USED]] ^(
          SQLITE_DBSTATUS_LOOKASIDE_USED
          +**
          This parameter returns the number of lookaside memory slots currently +** checked out.
          )^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_HIT]] ^(
          SQLITE_DBSTATUS_LOOKASIDE_HIT
          +**
          This parameter returns the number of malloc attempts that were +** satisfied using lookaside memory. Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE]] +** ^(
          SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE
          +**
          This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to the amount of +** memory requested being larger than the lookaside slot size. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL]] +** ^(
          SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL
          +**
          This parameter returns the number malloc attempts that might have +** been satisfied using lookaside memory but failed due to all lookaside +** memory already being in use. +** Only the high-water value is meaningful; +** the current value is always zero.)^ +** +** [[SQLITE_DBSTATUS_CACHE_USED]] ^(
          SQLITE_DBSTATUS_CACHE_USED
          +**
          This parameter returns the approximate number of bytes of heap +** memory used by all pager caches associated with the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_USED is always 0. +** +** [[SQLITE_DBSTATUS_CACHE_USED_SHARED]] +** ^(
          SQLITE_DBSTATUS_CACHE_USED_SHARED
          +**
          This parameter is similar to DBSTATUS_CACHE_USED, except that if a +** pager cache is shared between two or more connections the bytes of heap +** memory used by that pager cache is divided evenly between the attached +** connections.)^ In other words, if none of the pager caches associated +** with the database connection are shared, this request returns the same +** value as DBSTATUS_CACHE_USED. Or, if one or more or the pager caches are +** shared, the value returned by this call will be smaller than that returned +** by DBSTATUS_CACHE_USED. ^The highwater mark associated with +** SQLITE_DBSTATUS_CACHE_USED_SHARED is always 0. +** +** [[SQLITE_DBSTATUS_SCHEMA_USED]] ^(
          SQLITE_DBSTATUS_SCHEMA_USED
          +**
          This parameter returns the approximate number of bytes of heap +** memory used to store the schema for all databases associated +** with the connection - main, temp, and any [ATTACH]-ed databases.)^ +** ^The full amount of memory used by the schemas is reported, even if the +** schema memory is shared with other database connections due to +** [shared cache mode] being enabled. +** ^The highwater mark associated with SQLITE_DBSTATUS_SCHEMA_USED is always 0. +** +** [[SQLITE_DBSTATUS_STMT_USED]] ^(
          SQLITE_DBSTATUS_STMT_USED
          +**
          This parameter returns the approximate number of bytes of heap +** and lookaside memory used by all prepared statements associated with +** the database connection.)^ +** ^The highwater mark associated with SQLITE_DBSTATUS_STMT_USED is always 0. +**
          +** +** [[SQLITE_DBSTATUS_CACHE_HIT]] ^(
          SQLITE_DBSTATUS_CACHE_HIT
          +**
          This parameter returns the number of pager cache hits that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_HIT +** is always 0. +**
          +** +** [[SQLITE_DBSTATUS_CACHE_MISS]] ^(
          SQLITE_DBSTATUS_CACHE_MISS
          +**
          This parameter returns the number of pager cache misses that have +** occurred.)^ ^The highwater mark associated with SQLITE_DBSTATUS_CACHE_MISS +** is always 0. +**
          +** +** [[SQLITE_DBSTATUS_CACHE_WRITE]] ^(
          SQLITE_DBSTATUS_CACHE_WRITE
          +**
          This parameter returns the number of dirty cache entries that have +** been written to disk. Specifically, the number of pages written to the +** wal file in wal mode databases, or the number of pages written to the +** database file in rollback mode databases. Any pages written as part of +** transaction rollback or database recovery operations are not included. +** If an IO or other error occurs while writing a page to disk, the effect +** on subsequent SQLITE_DBSTATUS_CACHE_WRITE requests is undefined.)^ ^The +** highwater mark associated with SQLITE_DBSTATUS_CACHE_WRITE is always 0. +**
          +** +** [[SQLITE_DBSTATUS_CACHE_SPILL]] ^(
          SQLITE_DBSTATUS_CACHE_SPILL
          +**
          This parameter returns the number of dirty cache entries that have +** been written to disk in the middle of a transaction due to the page +** cache overflowing. Transactions are more efficient if they are written +** to disk all at once. When pages spill mid-transaction, that introduces +** additional overhead. This parameter can be used help identify +** inefficiencies that can be resolved by increasing the cache size. +**
          +** +** [[SQLITE_DBSTATUS_DEFERRED_FKS]] ^(
          SQLITE_DBSTATUS_DEFERRED_FKS
          +**
          This parameter returns zero for the current value if and only if +** all foreign key constraints (deferred or immediate) have been +** resolved.)^ ^The highwater mark is always 0. +**
          +**
          +*/ +#define SQLITE_DBSTATUS_LOOKASIDE_USED 0 +#define SQLITE_DBSTATUS_CACHE_USED 1 +#define SQLITE_DBSTATUS_SCHEMA_USED 2 +#define SQLITE_DBSTATUS_STMT_USED 3 +#define SQLITE_DBSTATUS_LOOKASIDE_HIT 4 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_SIZE 5 +#define SQLITE_DBSTATUS_LOOKASIDE_MISS_FULL 6 +#define SQLITE_DBSTATUS_CACHE_HIT 7 +#define SQLITE_DBSTATUS_CACHE_MISS 8 +#define SQLITE_DBSTATUS_CACHE_WRITE 9 +#define SQLITE_DBSTATUS_DEFERRED_FKS 10 +#define SQLITE_DBSTATUS_CACHE_USED_SHARED 11 +#define SQLITE_DBSTATUS_CACHE_SPILL 12 +#define SQLITE_DBSTATUS_MAX 12 /* Largest defined DBSTATUS */ + + +/* +** CAPI3REF: Prepared Statement Status +** METHOD: sqlite3_stmt +** +** ^(Each prepared statement maintains various +** [SQLITE_STMTSTATUS counters] that measure the number +** of times it has performed specific operations.)^ These counters can +** be used to monitor the performance characteristics of the prepared +** statements. For example, if the number of table steps greatly exceeds +** the number of table searches or result rows, that would tend to indicate +** that the prepared statement is using a full table scan rather than +** an index. +** +** ^(This interface is used to retrieve and reset counter values from +** a [prepared statement]. The first argument is the prepared statement +** object to be interrogated. The second argument +** is an integer code for a specific [SQLITE_STMTSTATUS counter] +** to be interrogated.)^ +** ^The current value of the requested counter is returned. +** ^If the resetFlg is true, then the counter is reset to zero after this +** interface call returns. +** +** See also: [sqlite3_status()] and [sqlite3_db_status()]. +*/ +SQLITE_API int sqlite3_stmt_status(sqlite3_stmt*, int op,int resetFlg); + +/* +** CAPI3REF: Status Parameters for prepared statements +** KEYWORDS: {SQLITE_STMTSTATUS counter} {SQLITE_STMTSTATUS counters} +** +** These preprocessor macros define integer codes that name counter +** values associated with the [sqlite3_stmt_status()] interface. +** The meanings of the various counters are as follows: +** +**
          +** [[SQLITE_STMTSTATUS_FULLSCAN_STEP]]
          SQLITE_STMTSTATUS_FULLSCAN_STEP
          +**
          ^This is the number of times that SQLite has stepped forward in +** a table as part of a full table scan. Large numbers for this counter +** may indicate opportunities for performance improvement through +** careful use of indices.
          +** +** [[SQLITE_STMTSTATUS_SORT]]
          SQLITE_STMTSTATUS_SORT
          +**
          ^This is the number of sort operations that have occurred. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance through careful use of indices.
          +** +** [[SQLITE_STMTSTATUS_AUTOINDEX]]
          SQLITE_STMTSTATUS_AUTOINDEX
          +**
          ^This is the number of rows inserted into transient indices that +** were created automatically in order to help joins run faster. +** A non-zero value in this counter may indicate an opportunity to +** improvement performance by adding permanent indices that do not +** need to be reinitialized each time the statement is run.
          +** +** [[SQLITE_STMTSTATUS_VM_STEP]]
          SQLITE_STMTSTATUS_VM_STEP
          +**
          ^This is the number of virtual machine operations executed +** by the prepared statement if that number is less than or equal +** to 2147483647. The number of virtual machine operations can be +** used as a proxy for the total work done by the prepared statement. +** If the number of virtual machine operations exceeds 2147483647 +** then the value returned by this statement status code is undefined. +** +** [[SQLITE_STMTSTATUS_REPREPARE]]
          SQLITE_STMTSTATUS_REPREPARE
          +**
          ^This is the number of times that the prepare statement has been +** automatically regenerated due to schema changes or changes to +** [bound parameters] that might affect the query plan. +** +** [[SQLITE_STMTSTATUS_RUN]]
          SQLITE_STMTSTATUS_RUN
          +**
          ^This is the number of times that the prepared statement has +** been run. A single "run" for the purposes of this counter is one +** or more calls to [sqlite3_step()] followed by a call to [sqlite3_reset()]. +** The counter is incremented on the first [sqlite3_step()] call of each +** cycle. +** +** [[SQLITE_STMTSTATUS_FILTER_MISS]] +** [[SQLITE_STMTSTATUS_FILTER HIT]] +**
          SQLITE_STMTSTATUS_FILTER_HIT
          +** SQLITE_STMTSTATUS_FILTER_MISS
          +**
          ^SQLITE_STMTSTATUS_FILTER_HIT is the number of times that a join +** step was bypassed because a Bloom filter returned not-found. The +** corresponding SQLITE_STMTSTATUS_FILTER_MISS value is the number of +** times that the Bloom filter returned a find, and thus the join step +** had to be processed as normal. +** +** [[SQLITE_STMTSTATUS_MEMUSED]]
          SQLITE_STMTSTATUS_MEMUSED
          +**
          ^This is the approximate number of bytes of heap memory +** used to store the prepared statement. ^This value is not actually +** a counter, and so the resetFlg parameter to sqlite3_stmt_status() +** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. +**
          +**
          +*/ +#define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 +#define SQLITE_STMTSTATUS_SORT 2 +#define SQLITE_STMTSTATUS_AUTOINDEX 3 +#define SQLITE_STMTSTATUS_VM_STEP 4 +#define SQLITE_STMTSTATUS_REPREPARE 5 +#define SQLITE_STMTSTATUS_RUN 6 +#define SQLITE_STMTSTATUS_FILTER_MISS 7 +#define SQLITE_STMTSTATUS_FILTER_HIT 8 +#define SQLITE_STMTSTATUS_MEMUSED 99 + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache type is opaque. It is implemented by +** the pluggable module. The SQLite core has no knowledge of +** its size or internal structure and never deals with the +** sqlite3_pcache object except by holding and passing pointers +** to the object. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache sqlite3_pcache; + +/* +** CAPI3REF: Custom Page Cache Object +** +** The sqlite3_pcache_page object represents a single page in the +** page cache. The page cache will allocate instances of this +** object. Various methods of the page cache use pointers to instances +** of this object as parameters or as their return value. +** +** See [sqlite3_pcache_methods2] for additional information. +*/ +typedef struct sqlite3_pcache_page sqlite3_pcache_page; +struct sqlite3_pcache_page { + void *pBuf; /* The content of the page */ + void *pExtra; /* Extra information associated with the page */ +}; + +/* +** CAPI3REF: Application Defined Page Cache. +** KEYWORDS: {page cache} +** +** ^(The [sqlite3_config]([SQLITE_CONFIG_PCACHE2], ...) interface can +** register an alternative page cache implementation by passing in an +** instance of the sqlite3_pcache_methods2 structure.)^ +** In many applications, most of the heap memory allocated by +** SQLite is used for the page cache. +** By implementing a +** custom page cache using this API, an application can better control +** the amount of memory consumed by SQLite, the way in which +** that memory is allocated and released, and the policies used to +** determine exactly which parts of a database file are cached and for +** how long. +** +** The alternative page cache mechanism is an +** extreme measure that is only needed by the most demanding applications. +** The built-in page cache is recommended for most uses. +** +** ^(The contents of the sqlite3_pcache_methods2 structure are copied to an +** internal buffer by SQLite within the call to [sqlite3_config]. Hence +** the application may discard the parameter after the call to +** [sqlite3_config()] returns.)^ +** +** [[the xInit() page cache method]] +** ^(The xInit() method is called once for each effective +** call to [sqlite3_initialize()])^ +** (usually only once during the lifetime of the process). ^(The xInit() +** method is passed a copy of the sqlite3_pcache_methods2.pArg value.)^ +** The intent of the xInit() method is to set up global data structures +** required by the custom page cache implementation. +** ^(If the xInit() method is NULL, then the +** built-in default page cache is used instead of the application defined +** page cache.)^ +** +** [[the xShutdown() page cache method]] +** ^The xShutdown() method is called by [sqlite3_shutdown()]. +** It can be used to clean up +** any outstanding resources before process shutdown, if required. +** ^The xShutdown() method may be NULL. +** +** ^SQLite automatically serializes calls to the xInit method, +** so the xInit method need not be threadsafe. ^The +** xShutdown method is only called from [sqlite3_shutdown()] so it does +** not need to be threadsafe either. All other methods must be threadsafe +** in multithreaded applications. +** +** ^SQLite will never invoke xInit() more than once without an intervening +** call to xShutdown(). +** +** [[the xCreate() page cache methods]] +** ^SQLite invokes the xCreate() method to construct a new cache instance. +** SQLite will typically create one cache instance for each open database file, +** though this is not guaranteed. ^The +** first parameter, szPage, is the size in bytes of the pages that must +** be allocated by the cache. ^szPage will always a power of two. ^The +** second parameter szExtra is a number of bytes of extra storage +** associated with each page cache entry. ^The szExtra parameter will +** a number less than 250. SQLite will use the +** extra szExtra bytes on each page to store metadata about the underlying +** database page on disk. The value passed into szExtra depends +** on the SQLite version, the target platform, and how SQLite was compiled. +** ^The third argument to xCreate(), bPurgeable, is true if the cache being +** created will be used to cache database pages of a file stored on disk, or +** false if it is used for an in-memory database. The cache implementation +** does not have to do anything special based with the value of bPurgeable; +** it is purely advisory. ^On a cache where bPurgeable is false, SQLite will +** never invoke xUnpin() except to deliberately delete a page. +** ^In other words, calls to xUnpin() on a cache with bPurgeable set to +** false will always have the "discard" flag set to true. +** ^Hence, a cache created with bPurgeable false will +** never contain any unpinned pages. +** +** [[the xCachesize() page cache method]] +** ^(The xCachesize() method may be called at any time by SQLite to set the +** suggested maximum cache-size (number of pages stored by) the cache +** instance passed as the first argument. This is the value configured using +** the SQLite "[PRAGMA cache_size]" command.)^ As with the bPurgeable +** parameter, the implementation is not required to do anything with this +** value; it is advisory only. +** +** [[the xPagecount() page cache methods]] +** The xPagecount() method must return the number of pages currently +** stored in the cache, both pinned and unpinned. +** +** [[the xFetch() page cache methods]] +** The xFetch() method locates a page in the cache and returns a pointer to +** an sqlite3_pcache_page object associated with that page, or a NULL pointer. +** The pBuf element of the returned sqlite3_pcache_page object will be a +** pointer to a buffer of szPage bytes used to store the content of a +** single database page. The pExtra element of sqlite3_pcache_page will be +** a pointer to the szExtra bytes of extra storage that SQLite has requested +** for each entry in the page cache. +** +** The page to be fetched is determined by the key. ^The minimum key value +** is 1. After it has been retrieved using xFetch, the page is considered +** to be "pinned". +** +** If the requested page is already in the page cache, then the page cache +** implementation must return a pointer to the page buffer with its content +** intact. If the requested page is not already in the cache, then the +** cache implementation should use the value of the createFlag +** parameter to help it determined what action to take: +** +** +**
          createFlag Behavior when page is not already in cache +**
          0 Do not allocate a new page. Return NULL. +**
          1 Allocate a new page if it easy and convenient to do so. +** Otherwise return NULL. +**
          2 Make every effort to allocate a new page. Only return +** NULL if allocating a new page is effectively impossible. +**
          +** +** ^(SQLite will normally invoke xFetch() with a createFlag of 0 or 1. SQLite +** will only use a createFlag of 2 after a prior call with a createFlag of 1 +** failed.)^ In between the xFetch() calls, SQLite may +** attempt to unpin one or more cache pages by spilling the content of +** pinned pages to disk and synching the operating system disk cache. +** +** [[the xUnpin() page cache method]] +** ^xUnpin() is called by SQLite with a pointer to a currently pinned page +** as its second argument. If the third parameter, discard, is non-zero, +** then the page must be evicted from the cache. +** ^If the discard parameter is +** zero, then the page may be discarded or retained at the discretion of +** page cache implementation. ^The page cache implementation +** may choose to evict unpinned pages at any time. +** +** The cache must not perform any reference counting. A single +** call to xUnpin() unpins the page regardless of the number of prior calls +** to xFetch(). +** +** [[the xRekey() page cache methods]] +** The xRekey() method is used to change the key value associated with the +** page passed as the second argument. If the cache +** previously contains an entry associated with newKey, it must be +** discarded. ^Any prior cache entry associated with newKey is guaranteed not +** to be pinned. +** +** When SQLite calls the xTruncate() method, the cache must discard all +** existing cache entries with page numbers (keys) greater than or equal +** to the value of the iLimit parameter passed to xTruncate(). If any +** of these pages are pinned, they are implicitly unpinned, meaning that +** they can be safely discarded. +** +** [[the xDestroy() page cache method]] +** ^The xDestroy() method is used to delete a cache allocated by xCreate(). +** All resources associated with the specified cache should be freed. ^After +** calling the xDestroy() method, SQLite considers the [sqlite3_pcache*] +** handle invalid, and will not use it with any other sqlite3_pcache_methods2 +** functions. +** +** [[the xShrink() page cache method]] +** ^SQLite invokes the xShrink() method when it wants the page cache to +** free up as much of heap memory as possible. The page cache implementation +** is not obligated to free any memory, but well-behaved implementations should +** do their best. +*/ +typedef struct sqlite3_pcache_methods2 sqlite3_pcache_methods2; +struct sqlite3_pcache_methods2 { + int iVersion; + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int szExtra, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + sqlite3_pcache_page *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, sqlite3_pcache_page*, int discard); + void (*xRekey)(sqlite3_pcache*, sqlite3_pcache_page*, + unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); + void (*xShrink)(sqlite3_pcache*); +}; + +/* +** This is the obsolete pcache_methods object that has now been replaced +** by sqlite3_pcache_methods2. This object is not used by SQLite. It is +** retained in the header file for backwards compatibility only. +*/ +typedef struct sqlite3_pcache_methods sqlite3_pcache_methods; +struct sqlite3_pcache_methods { + void *pArg; + int (*xInit)(void*); + void (*xShutdown)(void*); + sqlite3_pcache *(*xCreate)(int szPage, int bPurgeable); + void (*xCachesize)(sqlite3_pcache*, int nCachesize); + int (*xPagecount)(sqlite3_pcache*); + void *(*xFetch)(sqlite3_pcache*, unsigned key, int createFlag); + void (*xUnpin)(sqlite3_pcache*, void*, int discard); + void (*xRekey)(sqlite3_pcache*, void*, unsigned oldKey, unsigned newKey); + void (*xTruncate)(sqlite3_pcache*, unsigned iLimit); + void (*xDestroy)(sqlite3_pcache*); +}; + + +/* +** CAPI3REF: Online Backup Object +** +** The sqlite3_backup object records state information about an ongoing +** online backup operation. ^The sqlite3_backup object is created by +** a call to [sqlite3_backup_init()] and is destroyed by a call to +** [sqlite3_backup_finish()]. +** +** See Also: [Using the SQLite Online Backup API] +*/ +typedef struct sqlite3_backup sqlite3_backup; + +/* +** CAPI3REF: Online Backup API. +** +** The backup API copies the content of one database into another. +** It is useful either for creating backups of databases or +** for copying in-memory databases to or from persistent files. +** +** See Also: [Using the SQLite Online Backup API] +** +** ^SQLite holds a write transaction open on the destination database file +** for the duration of the backup operation. +** ^The source database is read-locked only while it is being read; +** it is not locked continuously for the entire backup operation. +** ^Thus, the backup may be performed on a live source database without +** preventing other database connections from +** reading or writing to the source database while the backup is underway. +** +** ^(To perform a backup operation: +**
            +**
          1. sqlite3_backup_init() is called once to initialize the +** backup, +**
          2. sqlite3_backup_step() is called one or more times to transfer +** the data between the two databases, and finally +**
          3. sqlite3_backup_finish() is called to release all resources +** associated with the backup operation. +**
          )^ +** There should be exactly one call to sqlite3_backup_finish() for each +** successful call to sqlite3_backup_init(). +** +** [[sqlite3_backup_init()]] sqlite3_backup_init() +** +** ^The D and N arguments to sqlite3_backup_init(D,N,S,M) are the +** [database connection] associated with the destination database +** and the database name, respectively. +** ^The database name is "main" for the main database, "temp" for the +** temporary database, or the name specified after the AS keyword in +** an [ATTACH] statement for an attached database. +** ^The S and M arguments passed to +** sqlite3_backup_init(D,N,S,M) identify the [database connection] +** and database name of the source database, respectively. +** ^The source and destination [database connections] (parameters S and D) +** must be different or else sqlite3_backup_init(D,N,S,M) will fail with +** an error. +** +** ^A call to sqlite3_backup_init() will fail, returning NULL, if +** there is already a read or read-write transaction open on the +** destination database. +** +** ^If an error occurs within sqlite3_backup_init(D,N,S,M), then NULL is +** returned and an error code and error message are stored in the +** destination [database connection] D. +** ^The error code and message for the failed call to sqlite3_backup_init() +** can be retrieved using the [sqlite3_errcode()], [sqlite3_errmsg()], and/or +** [sqlite3_errmsg16()] functions. +** ^A successful call to sqlite3_backup_init() returns a pointer to an +** [sqlite3_backup] object. +** ^The [sqlite3_backup] object may be used with the sqlite3_backup_step() and +** sqlite3_backup_finish() functions to perform the specified backup +** operation. +** +** [[sqlite3_backup_step()]] sqlite3_backup_step() +** +** ^Function sqlite3_backup_step(B,N) will copy up to N pages between +** the source and destination databases specified by [sqlite3_backup] object B. +** ^If N is negative, all remaining source pages are copied. +** ^If sqlite3_backup_step(B,N) successfully copies N pages and there +** are still more pages to be copied, then the function returns [SQLITE_OK]. +** ^If sqlite3_backup_step(B,N) successfully finishes copying all pages +** from source to destination, then it returns [SQLITE_DONE]. +** ^If an error occurs while running sqlite3_backup_step(B,N), +** then an [error code] is returned. ^As well as [SQLITE_OK] and +** [SQLITE_DONE], a call to sqlite3_backup_step() may return [SQLITE_READONLY], +** [SQLITE_NOMEM], [SQLITE_BUSY], [SQLITE_LOCKED], or an +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX] extended error code. +** +** ^(The sqlite3_backup_step() might return [SQLITE_READONLY] if +**
            +**
          1. the destination database was opened read-only, or +**
          2. the destination database is using write-ahead-log journaling +** and the destination and source page sizes differ, or +**
          3. the destination database is an in-memory database and the +** destination and source page sizes differ. +**
          )^ +** +** ^If sqlite3_backup_step() cannot obtain a required file-system lock, then +** the [sqlite3_busy_handler | busy-handler function] +** is invoked (if one is specified). ^If the +** busy-handler returns non-zero before the lock is available, then +** [SQLITE_BUSY] is returned to the caller. ^In this case the call to +** sqlite3_backup_step() can be retried later. ^If the source +** [database connection] +** is being used to write to the source database when sqlite3_backup_step() +** is called, then [SQLITE_LOCKED] is returned immediately. ^Again, in this +** case the call to sqlite3_backup_step() can be retried later on. ^(If +** [SQLITE_IOERR_ACCESS | SQLITE_IOERR_XXX], [SQLITE_NOMEM], or +** [SQLITE_READONLY] is returned, then +** there is no point in retrying the call to sqlite3_backup_step(). These +** errors are considered fatal.)^ The application must accept +** that the backup operation has failed and pass the backup operation handle +** to the sqlite3_backup_finish() to release associated resources. +** +** ^The first call to sqlite3_backup_step() obtains an exclusive lock +** on the destination file. ^The exclusive lock is not released until either +** sqlite3_backup_finish() is called or the backup operation is complete +** and sqlite3_backup_step() returns [SQLITE_DONE]. ^Every call to +** sqlite3_backup_step() obtains a [shared lock] on the source database that +** lasts for the duration of the sqlite3_backup_step() call. +** ^Because the source database is not locked between calls to +** sqlite3_backup_step(), the source database may be modified mid-way +** through the backup process. ^If the source database is modified by an +** external process or via a database connection other than the one being +** used by the backup operation, then the backup will be automatically +** restarted by the next call to sqlite3_backup_step(). ^If the source +** database is modified by the using the same database connection as is used +** by the backup operation, then the backup database is automatically +** updated at the same time. +** +** [[sqlite3_backup_finish()]] sqlite3_backup_finish() +** +** When sqlite3_backup_step() has returned [SQLITE_DONE], or when the +** application wishes to abandon the backup operation, the application +** should destroy the [sqlite3_backup] by passing it to sqlite3_backup_finish(). +** ^The sqlite3_backup_finish() interfaces releases all +** resources associated with the [sqlite3_backup] object. +** ^If sqlite3_backup_step() has not yet returned [SQLITE_DONE], then any +** active write-transaction on the destination database is rolled back. +** The [sqlite3_backup] object is invalid +** and may not be used following a call to sqlite3_backup_finish(). +** +** ^The value returned by sqlite3_backup_finish is [SQLITE_OK] if no +** sqlite3_backup_step() errors occurred, regardless or whether or not +** sqlite3_backup_step() completed. +** ^If an out-of-memory condition or IO error occurred during any prior +** sqlite3_backup_step() call on the same [sqlite3_backup] object, then +** sqlite3_backup_finish() returns the corresponding [error code]. +** +** ^A return of [SQLITE_BUSY] or [SQLITE_LOCKED] from sqlite3_backup_step() +** is not a permanent error and does not affect the return value of +** sqlite3_backup_finish(). +** +** [[sqlite3_backup_remaining()]] [[sqlite3_backup_pagecount()]] +** sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** +** ^The sqlite3_backup_remaining() routine returns the number of pages still +** to be backed up at the conclusion of the most recent sqlite3_backup_step(). +** ^The sqlite3_backup_pagecount() routine returns the total number of pages +** in the source database at the conclusion of the most recent +** sqlite3_backup_step(). +** ^(The values returned by these functions are only updated by +** sqlite3_backup_step(). If the source database is modified in a way that +** changes the size of the source database or the number of pages remaining, +** those changes are not reflected in the output of sqlite3_backup_pagecount() +** and sqlite3_backup_remaining() until after the next +** sqlite3_backup_step().)^ +** +** Concurrent Usage of Database Handles +** +** ^The source [database connection] may be used by the application for other +** purposes while a backup operation is underway or being initialized. +** ^If SQLite is compiled and configured to support threadsafe database +** connections, then the source database connection may be used concurrently +** from within other threads. +** +** However, the application must guarantee that the destination +** [database connection] is not passed to any other API (by any thread) after +** sqlite3_backup_init() is called and before the corresponding call to +** sqlite3_backup_finish(). SQLite does not currently check to see +** if the application incorrectly accesses the destination [database connection] +** and so no error code is reported, but the operations may malfunction +** nevertheless. Use of the destination database connection while a +** backup is in progress might also cause a mutex deadlock. +** +** If running in [shared cache mode], the application must +** guarantee that the shared cache used by the destination database +** is not accessed while the backup is running. In practice this means +** that the application must guarantee that the disk file being +** backed up to is not accessed by any connection within the process, +** not just the specific connection that was passed to sqlite3_backup_init(). +** +** The [sqlite3_backup] object itself is partially threadsafe. Multiple +** threads may safely make multiple concurrent calls to sqlite3_backup_step(). +** However, the sqlite3_backup_remaining() and sqlite3_backup_pagecount() +** APIs are not strictly speaking threadsafe. If they are invoked at the +** same time as another thread is invoking sqlite3_backup_step() it is +** possible that they return invalid values. +** +** Alternatives To Using The Backup API +** +** Other techniques for safely creating a consistent backup of an SQLite +** database include: +** +**
            +**
          • The [VACUUM INTO] command. +**
          • The [sqlite3_rsync] utility program. +**
          +*/ +SQLITE_API sqlite3_backup *sqlite3_backup_init( + sqlite3 *pDest, /* Destination database handle */ + const char *zDestName, /* Destination database name */ + sqlite3 *pSource, /* Source database handle */ + const char *zSourceName /* Source database name */ +); +SQLITE_API int sqlite3_backup_step(sqlite3_backup *p, int nPage); +SQLITE_API int sqlite3_backup_finish(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_remaining(sqlite3_backup *p); +SQLITE_API int sqlite3_backup_pagecount(sqlite3_backup *p); + +/* +** CAPI3REF: Unlock Notification +** METHOD: sqlite3 +** +** ^When running in shared-cache mode, a database operation may fail with +** an [SQLITE_LOCKED] error if the required locks on the shared-cache or +** individual tables within the shared-cache cannot be obtained. See +** [SQLite Shared-Cache Mode] for a description of shared-cache locking. +** ^This API may be used to register a callback that SQLite will invoke +** when the connection currently holding the required lock relinquishes it. +** ^This API is only available if the library was compiled with the +** [SQLITE_ENABLE_UNLOCK_NOTIFY] C-preprocessor symbol defined. +** +** See Also: [Using the SQLite Unlock Notification Feature]. +** +** ^Shared-cache locks are released when a database connection concludes +** its current transaction, either by committing it or rolling it back. +** +** ^When a connection (known as the blocked connection) fails to obtain a +** shared-cache lock and SQLITE_LOCKED is returned to the caller, the +** identity of the database connection (the blocking connection) that +** has locked the required resource is stored internally. ^After an +** application receives an SQLITE_LOCKED error, it may call the +** sqlite3_unlock_notify() method with the blocked connection handle as +** the first argument to register for a callback that will be invoked +** when the blocking connections current transaction is concluded. ^The +** callback is invoked from within the [sqlite3_step] or [sqlite3_close] +** call that concludes the blocking connection's transaction. +** +** ^(If sqlite3_unlock_notify() is called in a multi-threaded application, +** there is a chance that the blocking connection will have already +** concluded its transaction by the time sqlite3_unlock_notify() is invoked. +** If this happens, then the specified callback is invoked immediately, +** from within the call to sqlite3_unlock_notify().)^ +** +** ^If the blocked connection is attempting to obtain a write-lock on a +** shared-cache table, and more than one other connection currently holds +** a read-lock on the same table, then SQLite arbitrarily selects one of +** the other connections to use as the blocking connection. +** +** ^(There may be at most one unlock-notify callback registered by a +** blocked connection. If sqlite3_unlock_notify() is called when the +** blocked connection already has a registered unlock-notify callback, +** then the new callback replaces the old.)^ ^If sqlite3_unlock_notify() is +** called with a NULL pointer as its second argument, then any existing +** unlock-notify callback is canceled. ^The blocked connections +** unlock-notify callback may also be canceled by closing the blocked +** connection using [sqlite3_close()]. +** +** The unlock-notify callback is not reentrant. If an application invokes +** any sqlite3_xxx API functions from within an unlock-notify callback, a +** crash or deadlock may be the result. +** +** ^Unless deadlock is detected (see below), sqlite3_unlock_notify() always +** returns SQLITE_OK. +** +** Callback Invocation Details +** +** When an unlock-notify callback is registered, the application provides a +** single void* pointer that is passed to the callback when it is invoked. +** However, the signature of the callback function allows SQLite to pass +** it an array of void* context pointers. The first argument passed to +** an unlock-notify callback is a pointer to an array of void* pointers, +** and the second is the number of entries in the array. +** +** When a blocking connection's transaction is concluded, there may be +** more than one blocked connection that has registered for an unlock-notify +** callback. ^If two or more such blocked connections have specified the +** same callback function, then instead of invoking the callback function +** multiple times, it is invoked once with the set of void* context pointers +** specified by the blocked connections bundled together into an array. +** This gives the application an opportunity to prioritize any actions +** related to the set of unblocked database connections. +** +** Deadlock Detection +** +** Assuming that after registering for an unlock-notify callback a +** database waits for the callback to be issued before taking any further +** action (a reasonable assumption), then using this API may cause the +** application to deadlock. For example, if connection X is waiting for +** connection Y's transaction to be concluded, and similarly connection +** Y is waiting on connection X's transaction, then neither connection +** will proceed and the system may remain deadlocked indefinitely. +** +** To avoid this scenario, the sqlite3_unlock_notify() performs deadlock +** detection. ^If a given call to sqlite3_unlock_notify() would put the +** system in a deadlocked state, then SQLITE_LOCKED is returned and no +** unlock-notify callback is registered. The system is said to be in +** a deadlocked state if connection A has registered for an unlock-notify +** callback on the conclusion of connection B's transaction, and connection +** B has itself registered for an unlock-notify callback when connection +** A's transaction is concluded. ^Indirect deadlock is also detected, so +** the system is also considered to be deadlocked if connection B has +** registered for an unlock-notify callback on the conclusion of connection +** C's transaction, where connection C is waiting on connection A. ^Any +** number of levels of indirection are allowed. +** +** The "DROP TABLE" Exception +** +** When a call to [sqlite3_step()] returns SQLITE_LOCKED, it is almost +** always appropriate to call sqlite3_unlock_notify(). There is however, +** one exception. When executing a "DROP TABLE" or "DROP INDEX" statement, +** SQLite checks if there are any currently executing SELECT statements +** that belong to the same connection. If there are, SQLITE_LOCKED is +** returned. In this case there is no "blocking connection", so invoking +** sqlite3_unlock_notify() results in the unlock-notify callback being +** invoked immediately. If the application then re-attempts the "DROP TABLE" +** or "DROP INDEX" query, an infinite loop might be the result. +** +** One way around this problem is to check the extended error code returned +** by an sqlite3_step() call. ^(If there is a blocking connection, then the +** extended error code is set to SQLITE_LOCKED_SHAREDCACHE. Otherwise, in +** the special "DROP TABLE/INDEX" case, the extended error code is just +** SQLITE_LOCKED.)^ +*/ +SQLITE_API int sqlite3_unlock_notify( + sqlite3 *pBlocked, /* Waiting connection */ + void (*xNotify)(void **apArg, int nArg), /* Callback function to invoke */ + void *pNotifyArg /* Argument to pass to xNotify */ +); + + +/* +** CAPI3REF: String Comparison +** +** ^The [sqlite3_stricmp()] and [sqlite3_strnicmp()] APIs allow applications +** and extensions to compare the contents of two buffers containing UTF-8 +** strings in a case-independent fashion, using the same definition of "case +** independence" that SQLite uses internally when comparing identifiers. +*/ +SQLITE_API int sqlite3_stricmp(const char *, const char *); +SQLITE_API int sqlite3_strnicmp(const char *, const char *, int); + +/* +** CAPI3REF: String Globbing +* +** ^The [sqlite3_strglob(P,X)] interface returns zero if and only if +** string X matches the [GLOB] pattern P. +** ^The definition of [GLOB] pattern matching used in +** [sqlite3_strglob(P,X)] is the same as for the "X GLOB P" operator in the +** SQL dialect understood by SQLite. ^The [sqlite3_strglob(P,X)] function +** is case sensitive. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strlike()]. +*/ +SQLITE_API int sqlite3_strglob(const char *zGlob, const char *zStr); + +/* +** CAPI3REF: String LIKE Matching +* +** ^The [sqlite3_strlike(P,X,E)] interface returns zero if and only if +** string X matches the [LIKE] pattern P with escape character E. +** ^The definition of [LIKE] pattern matching used in +** [sqlite3_strlike(P,X,E)] is the same as for the "X LIKE P ESCAPE E" +** operator in the SQL dialect understood by SQLite. ^For "X LIKE P" without +** the ESCAPE clause, set the E parameter of [sqlite3_strlike(P,X,E)] to 0. +** ^As with the LIKE operator, the [sqlite3_strlike(P,X,E)] function is case +** insensitive - equivalent upper and lower case ASCII characters match +** one another. +** +** ^The [sqlite3_strlike(P,X,E)] function matches Unicode characters, though +** only ASCII characters are case folded. +** +** Note that this routine returns zero on a match and non-zero if the strings +** do not match, the same as [sqlite3_stricmp()] and [sqlite3_strnicmp()]. +** +** See also: [sqlite3_strglob()]. +*/ +SQLITE_API int sqlite3_strlike(const char *zGlob, const char *zStr, unsigned int cEsc); + +/* +** CAPI3REF: Error Logging Interface +** +** ^The [sqlite3_log()] interface writes a message into the [error log] +** established by the [SQLITE_CONFIG_LOG] option to [sqlite3_config()]. +** ^If logging is enabled, the zFormat string and subsequent arguments are +** used with [sqlite3_snprintf()] to generate the final output string. +** +** The sqlite3_log() interface is intended for use by extensions such as +** virtual tables, collating functions, and SQL functions. While there is +** nothing to prevent an application from calling sqlite3_log(), doing so +** is considered bad form. +** +** The zFormat string must not be NULL. +** +** To avoid deadlocks and other threading problems, the sqlite3_log() routine +** will not use dynamically allocated memory. The log message is stored in +** a fixed-length buffer on the stack. If the log message is longer than +** a few hundred characters, it will be truncated to the length of the +** buffer. +*/ +SQLITE_API void sqlite3_log(int iErrCode, const char *zFormat, ...); + +/* +** CAPI3REF: Write-Ahead Log Commit Hook +** METHOD: sqlite3 +** +** ^The [sqlite3_wal_hook()] function is used to register a callback that +** is invoked each time data is committed to a database in wal mode. +** +** ^(The callback is invoked by SQLite after the commit has taken place and +** the associated write-lock on the database released)^, so the implementation +** may read, write or [checkpoint] the database as required. +** +** ^The first parameter passed to the callback function when it is invoked +** is a copy of the third parameter passed to sqlite3_wal_hook() when +** registering the callback. ^The second is a copy of the database handle. +** ^The third parameter is the name of the database that was written to - +** either "main" or the name of an [ATTACH]-ed database. ^The fourth parameter +** is the number of pages currently in the write-ahead log file, +** including those that were just committed. +** +** The callback function should normally return [SQLITE_OK]. ^If an error +** code is returned, that error will propagate back up through the +** SQLite code base to cause the statement that provoked the callback +** to report an error, though the commit will have still occurred. If the +** callback returns [SQLITE_ROW] or [SQLITE_DONE], or if it returns a value +** that does not correspond to any valid SQLite error code, the results +** are undefined. +** +** A single database handle may have at most a single write-ahead log callback +** registered at one time. ^Calling [sqlite3_wal_hook()] replaces any +** previously registered write-ahead log callback. ^The return value is +** a copy of the third parameter from the previous call, if any, or 0. +** ^Note that the [sqlite3_wal_autocheckpoint()] interface and the +** [wal_autocheckpoint pragma] both invoke [sqlite3_wal_hook()] and will +** overwrite any prior [sqlite3_wal_hook()] settings. +*/ +SQLITE_API void *sqlite3_wal_hook( + sqlite3*, + int(*)(void *,sqlite3*,const char*,int), + void* +); + +/* +** CAPI3REF: Configure an auto-checkpoint +** METHOD: sqlite3 +** +** ^The [sqlite3_wal_autocheckpoint(D,N)] is a wrapper around +** [sqlite3_wal_hook()] that causes any database on [database connection] D +** to automatically [checkpoint] +** after committing a transaction if there are N or +** more frames in the [write-ahead log] file. ^Passing zero or +** a negative value as the nFrame parameter disables automatic +** checkpoints entirely. +** +** ^The callback registered by this function replaces any existing callback +** registered using [sqlite3_wal_hook()]. ^Likewise, registering a callback +** using [sqlite3_wal_hook()] disables the automatic checkpoint mechanism +** configured by this function. +** +** ^The [wal_autocheckpoint pragma] can be used to invoke this interface +** from SQL. +** +** ^Checkpoints initiated by this mechanism are +** [sqlite3_wal_checkpoint_v2|PASSIVE]. +** +** ^Every new [database connection] defaults to having the auto-checkpoint +** enabled with a threshold of 1000 or [SQLITE_DEFAULT_WAL_AUTOCHECKPOINT] +** pages. The use of this interface +** is only necessary if the default setting is found to be suboptimal +** for a particular application. +*/ +SQLITE_API int sqlite3_wal_autocheckpoint(sqlite3 *db, int N); + +/* +** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 +** +** ^(The sqlite3_wal_checkpoint(D,X) is equivalent to +** [sqlite3_wal_checkpoint_v2](D,X,[SQLITE_CHECKPOINT_PASSIVE],0,0).)^ +** +** In brief, sqlite3_wal_checkpoint(D,X) causes the content in the +** [write-ahead log] for database X on [database connection] D to be +** transferred into the database file and for the write-ahead log to +** be reset. See the [checkpointing] documentation for addition +** information. +** +** This interface used to be the only way to cause a checkpoint to +** occur. But then the newer and more powerful [sqlite3_wal_checkpoint_v2()] +** interface was added. This interface is retained for backwards +** compatibility and as a convenience for applications that need to manually +** start a callback but which do not need the full power (and corresponding +** complication) of [sqlite3_wal_checkpoint_v2()]. +*/ +SQLITE_API int sqlite3_wal_checkpoint(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Checkpoint a database +** METHOD: sqlite3 +** +** ^(The sqlite3_wal_checkpoint_v2(D,X,M,L,C) interface runs a checkpoint +** operation on database X of [database connection] D in mode M. Status +** information is written back into integers pointed to by L and C.)^ +** ^(The M parameter must be a valid [checkpoint mode]:)^ +** +**
          +**
          SQLITE_CHECKPOINT_PASSIVE
          +** ^Checkpoint as many frames as possible without waiting for any database +** readers or writers to finish, then sync the database file if all frames +** in the log were checkpointed. ^The [busy-handler callback] +** is never invoked in the SQLITE_CHECKPOINT_PASSIVE mode. +** ^On the other hand, passive mode might leave the checkpoint unfinished +** if there are concurrent readers or writers. +** +**
          SQLITE_CHECKPOINT_FULL
          +** ^This mode blocks (it invokes the +** [sqlite3_busy_handler|busy-handler callback]) until there is no +** database writer and all readers are reading from the most recent database +** snapshot. ^It then checkpoints all frames in the log file and syncs the +** database file. ^This mode blocks new database writers while it is pending, +** but new database readers are allowed to continue unimpeded. +** +**
          SQLITE_CHECKPOINT_RESTART
          +** ^This mode works the same way as SQLITE_CHECKPOINT_FULL with the addition +** that after checkpointing the log file it blocks (calls the +** [busy-handler callback]) +** until all readers are reading from the database file only. ^This ensures +** that the next writer will restart the log file from the beginning. +** ^Like SQLITE_CHECKPOINT_FULL, this mode blocks new +** database writer attempts while it is pending, but does not impede readers. +** +**
          SQLITE_CHECKPOINT_TRUNCATE
          +** ^This mode works the same way as SQLITE_CHECKPOINT_RESTART with the +** addition that it also truncates the log file to zero bytes just prior +** to a successful return. +**
          +** +** ^If pnLog is not NULL, then *pnLog is set to the total number of frames in +** the log file or to -1 if the checkpoint could not run because +** of an error or because the database is not in [WAL mode]. ^If pnCkpt is not +** NULL,then *pnCkpt is set to the total number of checkpointed frames in the +** log file (including any that were already checkpointed before the function +** was called) or to -1 if the checkpoint could not run due to an error or +** because the database is not in WAL mode. ^Note that upon successful +** completion of an SQLITE_CHECKPOINT_TRUNCATE, the log file will have been +** truncated to zero bytes and so both *pnLog and *pnCkpt will be set to zero. +** +** ^All calls obtain an exclusive "checkpoint" lock on the database file. ^If +** any other process is running a checkpoint operation at the same time, the +** lock cannot be obtained and SQLITE_BUSY is returned. ^Even if there is a +** busy-handler configured, it will not be invoked in this case. +** +** ^The SQLITE_CHECKPOINT_FULL, RESTART and TRUNCATE modes also obtain the +** exclusive "writer" lock on the database file. ^If the writer lock cannot be +** obtained immediately, and a busy-handler is configured, it is invoked and +** the writer lock retried until either the busy-handler returns 0 or the lock +** is successfully obtained. ^The busy-handler is also invoked while waiting for +** database readers as described above. ^If the busy-handler returns 0 before +** the writer lock is obtained or while waiting for database readers, the +** checkpoint operation proceeds from that point in the same way as +** SQLITE_CHECKPOINT_PASSIVE - checkpointing as many frames as possible +** without blocking any further. ^SQLITE_BUSY is returned in this case. +** +** ^If parameter zDb is NULL or points to a zero length string, then the +** specified operation is attempted on all WAL databases [attached] to +** [database connection] db. In this case the +** values written to output parameters *pnLog and *pnCkpt are undefined. ^If +** an SQLITE_BUSY error is encountered when processing one or more of the +** attached WAL databases, the operation is still attempted on any remaining +** attached databases and SQLITE_BUSY is returned at the end. ^If any other +** error occurs while processing an attached database, processing is abandoned +** and the error code is returned to the caller immediately. ^If no error +** (SQLITE_BUSY or otherwise) is encountered while processing the attached +** databases, SQLITE_OK is returned. +** +** ^If database zDb is the name of an attached database that is not in WAL +** mode, SQLITE_OK is returned and both *pnLog and *pnCkpt set to -1. ^If +** zDb is not NULL (or a zero length string) and is not the name of any +** attached database, SQLITE_ERROR is returned to the caller. +** +** ^Unless it returns SQLITE_MISUSE, +** the sqlite3_wal_checkpoint_v2() interface +** sets the error information that is queried by +** [sqlite3_errcode()] and [sqlite3_errmsg()]. +** +** ^The [PRAGMA wal_checkpoint] command can be used to invoke this interface +** from SQL. +*/ +SQLITE_API int sqlite3_wal_checkpoint_v2( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of attached database (or NULL) */ + int eMode, /* SQLITE_CHECKPOINT_* value */ + int *pnLog, /* OUT: Size of WAL log in frames */ + int *pnCkpt /* OUT: Total number of frames checkpointed */ +); + +/* +** CAPI3REF: Checkpoint Mode Values +** KEYWORDS: {checkpoint mode} +** +** These constants define all valid values for the "checkpoint mode" passed +** as the third parameter to the [sqlite3_wal_checkpoint_v2()] interface. +** See the [sqlite3_wal_checkpoint_v2()] documentation for details on the +** meaning of each of these checkpoint modes. +*/ +#define SQLITE_CHECKPOINT_PASSIVE 0 /* Do as much as possible w/o blocking */ +#define SQLITE_CHECKPOINT_FULL 1 /* Wait for writers, then checkpoint */ +#define SQLITE_CHECKPOINT_RESTART 2 /* Like FULL but wait for readers */ +#define SQLITE_CHECKPOINT_TRUNCATE 3 /* Like RESTART but also truncate WAL */ + +/* +** CAPI3REF: Virtual Table Interface Configuration +** +** This function may be called by either the [xConnect] or [xCreate] method +** of a [virtual table] implementation to configure +** various facets of the virtual table interface. +** +** If this interface is invoked outside the context of an xConnect or +** xCreate virtual table method then the behavior is undefined. +** +** In the call sqlite3_vtab_config(D,C,...) the D parameter is the +** [database connection] in which the virtual table is being created and +** which is passed in as the first argument to the [xConnect] or [xCreate] +** method that is invoking sqlite3_vtab_config(). The C parameter is one +** of the [virtual table configuration options]. The presence and meaning +** of parameters after C depend on which [virtual table configuration option] +** is used. +*/ +SQLITE_API int sqlite3_vtab_config(sqlite3*, int op, ...); + +/* +** CAPI3REF: Virtual Table Configuration Options +** KEYWORDS: {virtual table configuration options} +** KEYWORDS: {virtual table configuration option} +** +** These macros define the various options to the +** [sqlite3_vtab_config()] interface that [virtual table] implementations +** can use to customize and optimize their behavior. +** +**
          +** [[SQLITE_VTAB_CONSTRAINT_SUPPORT]] +**
          SQLITE_VTAB_CONSTRAINT_SUPPORT
          +**
          Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_CONSTRAINT_SUPPORT,X) are supported, +** where X is an integer. If X is zero, then the [virtual table] whose +** [xCreate] or [xConnect] method invoked [sqlite3_vtab_config()] does not +** support constraints. In this configuration (which is the default) if +** a call to the [xUpdate] method returns [SQLITE_CONSTRAINT], then the entire +** statement is rolled back as if [ON CONFLICT | OR ABORT] had been +** specified as part of the users SQL statement, regardless of the actual +** ON CONFLICT mode specified. +** +** If X is non-zero, then the virtual table implementation guarantees +** that if [xUpdate] returns [SQLITE_CONSTRAINT], it will do so before +** any modifications to internal or persistent data structures have been made. +** If the [ON CONFLICT] mode is ABORT, FAIL, IGNORE or ROLLBACK, SQLite +** is able to roll back a statement or database transaction, and abandon +** or continue processing the current SQL statement as appropriate. +** If the ON CONFLICT mode is REPLACE and the [xUpdate] method returns +** [SQLITE_CONSTRAINT], SQLite handles this as if the ON CONFLICT mode +** had been ABORT. +** +** Virtual table implementations that are required to handle OR REPLACE +** must do so within the [xUpdate] method. If a call to the +** [sqlite3_vtab_on_conflict()] function indicates that the current ON +** CONFLICT policy is REPLACE, the virtual table implementation should +** silently replace the appropriate rows within the xUpdate callback and +** return SQLITE_OK. Or, if this is not possible, it may return +** SQLITE_CONSTRAINT, in which case SQLite falls back to OR ABORT +** constraint handling. +**
          +** +** [[SQLITE_VTAB_DIRECTONLY]]
          SQLITE_VTAB_DIRECTONLY
          +**
          Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_DIRECTONLY) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** prohibits that virtual table from being used from within triggers and +** views. +**
          +** +** [[SQLITE_VTAB_INNOCUOUS]]
          SQLITE_VTAB_INNOCUOUS
          +**
          Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_INNOCUOUS) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** identify that virtual table as being safe to use from within triggers +** and views. Conceptually, the SQLITE_VTAB_INNOCUOUS tag means that the +** virtual table can do no serious harm even if it is controlled by a +** malicious hacker. Developers should avoid setting the SQLITE_VTAB_INNOCUOUS +** flag unless absolutely necessary. +**
          +** +** [[SQLITE_VTAB_USES_ALL_SCHEMAS]]
          SQLITE_VTAB_USES_ALL_SCHEMAS
          +**
          Calls of the form +** [sqlite3_vtab_config](db,SQLITE_VTAB_USES_ALL_SCHEMA) from within the +** the [xConnect] or [xCreate] methods of a [virtual table] implementation +** instruct the query planner to begin at least a read transaction on +** all schemas ("main", "temp", and any ATTACH-ed databases) whenever the +** virtual table is used. +**
          +**
          +*/ +#define SQLITE_VTAB_CONSTRAINT_SUPPORT 1 +#define SQLITE_VTAB_INNOCUOUS 2 +#define SQLITE_VTAB_DIRECTONLY 3 +#define SQLITE_VTAB_USES_ALL_SCHEMAS 4 + +/* +** CAPI3REF: Determine The Virtual Table Conflict Policy +** +** This function may only be called from within a call to the [xUpdate] method +** of a [virtual table] implementation for an INSERT or UPDATE operation. ^The +** value returned is one of [SQLITE_ROLLBACK], [SQLITE_IGNORE], [SQLITE_FAIL], +** [SQLITE_ABORT], or [SQLITE_REPLACE], according to the [ON CONFLICT] mode +** of the SQL statement that triggered the call to the [xUpdate] method of the +** [virtual table]. +*/ +SQLITE_API int sqlite3_vtab_on_conflict(sqlite3 *); + +/* +** CAPI3REF: Determine If Virtual Table Column Access Is For UPDATE +** +** If the sqlite3_vtab_nochange(X) routine is called within the [xColumn] +** method of a [virtual table], then it might return true if the +** column is being fetched as part of an UPDATE operation during which the +** column value will not change. The virtual table implementation can use +** this hint as permission to substitute a return value that is less +** expensive to compute and that the corresponding +** [xUpdate] method understands as a "no-change" value. +** +** If the [xColumn] method calls sqlite3_vtab_nochange() and finds that +** the column is not changed by the UPDATE statement, then the xColumn +** method can optionally return without setting a result, without calling +** any of the [sqlite3_result_int|sqlite3_result_xxxxx() interfaces]. +** In that case, [sqlite3_value_nochange(X)] will return true for the +** same column in the [xUpdate] method. +** +** The sqlite3_vtab_nochange() routine is an optimization. Virtual table +** implementations should continue to give a correct answer even if the +** sqlite3_vtab_nochange() interface were to always return false. In the +** current implementation, the sqlite3_vtab_nochange() interface does always +** returns false for the enhanced [UPDATE FROM] statement. +*/ +SQLITE_API int sqlite3_vtab_nochange(sqlite3_context*); + +/* +** CAPI3REF: Determine The Collation For a Virtual Table Constraint +** METHOD: sqlite3_index_info +** +** This function may only be called from within a call to the [xBestIndex] +** method of a [virtual table]. This function returns a pointer to a string +** that is the name of the appropriate collation sequence to use for text +** comparisons on the constraint identified by its arguments. +** +** The first argument must be the pointer to the [sqlite3_index_info] object +** that is the first parameter to the xBestIndex() method. The second argument +** must be an index into the aConstraint[] array belonging to the +** sqlite3_index_info structure passed to xBestIndex. +** +** Important: +** The first parameter must be the same pointer that is passed into the +** xBestMethod() method. The first parameter may not be a pointer to a +** different [sqlite3_index_info] object, even an exact copy. +** +** The return value is computed as follows: +** +**
            +**

          1. If the constraint comes from a WHERE clause expression that contains +** a [COLLATE operator], then the name of the collation specified by +** that COLLATE operator is returned. +**

          2. If there is no COLLATE operator, but the column that is the subject +** of the constraint specifies an alternative collating sequence via +** a [COLLATE clause] on the column definition within the CREATE TABLE +** statement that was passed into [sqlite3_declare_vtab()], then the +** name of that alternative collating sequence is returned. +**

          3. Otherwise, "BINARY" is returned. +**

          +*/ +SQLITE_API const char *sqlite3_vtab_collation(sqlite3_index_info*,int); + +/* +** CAPI3REF: Determine if a virtual table query is DISTINCT +** METHOD: sqlite3_index_info +** +** This API may only be used from within an [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this +** interface from outside of xBestIndex() is undefined and probably harmful. +** +** ^The sqlite3_vtab_distinct() interface returns an integer between 0 and +** 3. The integer returned by sqlite3_vtab_distinct() +** gives the virtual table additional information about how the query +** planner wants the output to be ordered. As long as the virtual table +** can meet the ordering requirements of the query planner, it may set +** the "orderByConsumed" flag. +** +**

          1. +** ^If the sqlite3_vtab_distinct() interface returns 0, that means +** that the query planner needs the virtual table to return all rows in the +** sort order defined by the "nOrderBy" and "aOrderBy" fields of the +** [sqlite3_index_info] object. This is the default expectation. If the +** virtual table outputs all rows in sorted order, then it is always safe for +** the xBestIndex method to set the "orderByConsumed" flag, regardless of +** the return value from sqlite3_vtab_distinct(). +**

          2. +** ^(If the sqlite3_vtab_distinct() interface returns 1, that means +** that the query planner does not need the rows to be returned in sorted order +** as long as all rows with the same values in all columns identified by the +** "aOrderBy" field are adjacent.)^ This mode is used when the query planner +** is doing a GROUP BY. +**

          3. +** ^(If the sqlite3_vtab_distinct() interface returns 2, that means +** that the query planner does not need the rows returned in any particular +** order, as long as rows with the same values in all columns identified +** by "aOrderBy" are adjacent.)^ ^(Furthermore, when two or more rows +** contain the same values for all columns identified by "colUsed", all but +** one such row may optionally be omitted from the result.)^ +** The virtual table is not required to omit rows that are duplicates +** over the "colUsed" columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for a DISTINCT query. +**

          4. +** ^(If the sqlite3_vtab_distinct() interface returns 3, that means the +** virtual table must return rows in the order defined by "aOrderBy" as +** if the sqlite3_vtab_distinct() interface had returned 0. However if +** two or more rows in the result have the same values for all columns +** identified by "colUsed", then all but one such row may optionally be +** omitted.)^ Like when the return value is 2, the virtual table +** is not required to omit rows that are duplicates over the "colUsed" +** columns, but if the virtual table can do that without +** too much extra effort, it could potentially help the query to run faster. +** This mode is used for queries +** that have both DISTINCT and ORDER BY clauses. +**

          +** +**

          The following table summarizes the conditions under which the +** virtual table is allowed to set the "orderByConsumed" flag based on +** the value returned by sqlite3_vtab_distinct(). This table is a +** restatement of the previous four paragraphs: +** +** +** +**
          sqlite3_vtab_distinct() return value +** Rows are returned in aOrderBy order +** Rows with the same value in all aOrderBy columns are adjacent +** Duplicates over all colUsed columns may be omitted +**
          0yesyesno +**
          1noyesno +**
          2noyesyes +**
          3yesyesyes +**
          +** +** ^For the purposes of comparing virtual table output values to see if the +** values are same value for sorting purposes, two NULL values are considered +** to be the same. In other words, the comparison operator is "IS" +** (or "IS NOT DISTINCT FROM") and not "==". +** +** If a virtual table implementation is unable to meet the requirements +** specified above, then it must not set the "orderByConsumed" flag in the +** [sqlite3_index_info] object or an incorrect answer may result. +** +** ^A virtual table implementation is always free to return rows in any order +** it wants, as long as the "orderByConsumed" flag is not set. ^When the +** the "orderByConsumed" flag is unset, the query planner will add extra +** [bytecode] to ensure that the final results returned by the SQL query are +** ordered correctly. The use of the "orderByConsumed" flag and the +** sqlite3_vtab_distinct() interface is merely an optimization. ^Careful +** use of the sqlite3_vtab_distinct() interface and the "orderByConsumed" +** flag might help queries against a virtual table to run faster. Being +** overly aggressive and setting the "orderByConsumed" flag when it is not +** valid to do so, on the other hand, might cause SQLite to return incorrect +** results. +*/ +SQLITE_API int sqlite3_vtab_distinct(sqlite3_index_info*); + +/* +** CAPI3REF: Identify and handle IN constraints in xBestIndex +** +** This interface may only be used from within an +** [xBestIndex|xBestIndex() method] of a [virtual table] implementation. +** The result of invoking this interface from any other context is +** undefined and probably harmful. +** +** ^(A constraint on a virtual table of the form +** "[IN operator|column IN (...)]" is +** communicated to the xBestIndex method as a +** [SQLITE_INDEX_CONSTRAINT_EQ] constraint.)^ If xBestIndex wants to use +** this constraint, it must set the corresponding +** aConstraintUsage[].argvIndex to a positive integer. ^(Then, under +** the usual mode of handling IN operators, SQLite generates [bytecode] +** that invokes the [xFilter|xFilter() method] once for each value +** on the right-hand side of the IN operator.)^ Thus the virtual table +** only sees a single value from the right-hand side of the IN operator +** at a time. +** +** In some cases, however, it would be advantageous for the virtual +** table to see all values on the right-hand of the IN operator all at +** once. The sqlite3_vtab_in() interfaces facilitates this in two ways: +** +**

            +**

          1. +** ^A call to sqlite3_vtab_in(P,N,-1) will return true (non-zero) +** if and only if the [sqlite3_index_info|P->aConstraint][N] constraint +** is an [IN operator] that can be processed all at once. ^In other words, +** sqlite3_vtab_in() with -1 in the third argument is a mechanism +** by which the virtual table can ask SQLite if all-at-once processing +** of the IN operator is even possible. +** +**

          2. +** ^A call to sqlite3_vtab_in(P,N,F) with F==1 or F==0 indicates +** to SQLite that the virtual table does or does not want to process +** the IN operator all-at-once, respectively. ^Thus when the third +** parameter (F) is non-negative, this interface is the mechanism by +** which the virtual table tells SQLite how it wants to process the +** IN operator. +**

          +** +** ^The sqlite3_vtab_in(P,N,F) interface can be invoked multiple times +** within the same xBestIndex method call. ^For any given P,N pair, +** the return value from sqlite3_vtab_in(P,N,F) will always be the same +** within the same xBestIndex call. ^If the interface returns true +** (non-zero), that means that the constraint is an IN operator +** that can be processed all-at-once. ^If the constraint is not an IN +** operator or cannot be processed all-at-once, then the interface returns +** false. +** +** ^(All-at-once processing of the IN operator is selected if both of the +** following conditions are met: +** +**
            +**

          1. The P->aConstraintUsage[N].argvIndex value is set to a positive +** integer. This is how the virtual table tells SQLite that it wants to +** use the N-th constraint. +** +**

          2. The last call to sqlite3_vtab_in(P,N,F) for which F was +** non-negative had F>=1. +**

          )^ +** +** ^If either or both of the conditions above are false, then SQLite uses +** the traditional one-at-a-time processing strategy for the IN constraint. +** ^If both conditions are true, then the argvIndex-th parameter to the +** xFilter method will be an [sqlite3_value] that appears to be NULL, +** but which can be passed to [sqlite3_vtab_in_first()] and +** [sqlite3_vtab_in_next()] to find all values on the right-hand side +** of the IN constraint. +*/ +SQLITE_API int sqlite3_vtab_in(sqlite3_index_info*, int iCons, int bHandle); + +/* +** CAPI3REF: Find all elements on the right-hand side of an IN constraint. +** +** These interfaces are only useful from within the +** [xFilter|xFilter() method] of a [virtual table] implementation. +** The result of invoking these interfaces from any other context +** is undefined and probably harmful. +** +** The X parameter in a call to sqlite3_vtab_in_first(X,P) or +** sqlite3_vtab_in_next(X,P) should be one of the parameters to the +** xFilter method which invokes these routines, and specifically +** a parameter that was previously selected for all-at-once IN constraint +** processing use the [sqlite3_vtab_in()] interface in the +** [xBestIndex|xBestIndex method]. ^(If the X parameter is not +** an xFilter argument that was selected for all-at-once IN constraint +** processing, then these routines return [SQLITE_ERROR].)^ +** +** ^(Use these routines to access all values on the right-hand side +** of the IN constraint using code like the following: +** +** 
          +**    for(rc=sqlite3_vtab_in_first(pList, &pVal);
+**        rc==SQLITE_OK && pVal;
+**        rc=sqlite3_vtab_in_next(pList, &pVal)
+**    ){
+**      // do something with pVal
+**    }
+**    if( rc!=SQLITE_OK ){
+**      // an error has occurred
+**    }
+**
          )^ +** +** ^On success, the sqlite3_vtab_in_first(X,P) and sqlite3_vtab_in_next(X,P) +** routines return SQLITE_OK and set *P to point to the first or next value +** on the RHS of the IN constraint. ^If there are no more values on the +** right hand side of the IN constraint, then *P is set to NULL and these +** routines return [SQLITE_DONE]. ^The return value might be +** some other value, such as SQLITE_NOMEM, in the event of a malfunction. +** +** The *ppOut values returned by these routines are only valid until the +** next call to either of these routines or until the end of the xFilter +** method from which these routines were called. If the virtual table +** implementation needs to retain the *ppOut values for longer, it must make +** copies. The *ppOut values are [protected sqlite3_value|protected]. +*/ +SQLITE_API int sqlite3_vtab_in_first(sqlite3_value *pVal, sqlite3_value **ppOut); +SQLITE_API int sqlite3_vtab_in_next(sqlite3_value *pVal, sqlite3_value **ppOut); + +/* +** CAPI3REF: Constraint values in xBestIndex() +** METHOD: sqlite3_index_info +** +** This API may only be used from within the [xBestIndex|xBestIndex method] +** of a [virtual table] implementation. The result of calling this interface +** from outside of an xBestIndex method are undefined and probably harmful. +** +** ^When the sqlite3_vtab_rhs_value(P,J,V) interface is invoked from within +** the [xBestIndex] method of a [virtual table] implementation, with P being +** a copy of the [sqlite3_index_info] object pointer passed into xBestIndex and +** J being a 0-based index into P->aConstraint[], then this routine +** attempts to set *V to the value of the right-hand operand of +** that constraint if the right-hand operand is known. ^If the +** right-hand operand is not known, then *V is set to a NULL pointer. +** ^The sqlite3_vtab_rhs_value(P,J,V) interface returns SQLITE_OK if +** and only if *V is set to a value. ^The sqlite3_vtab_rhs_value(P,J,V) +** inteface returns SQLITE_NOTFOUND if the right-hand side of the J-th +** constraint is not available. ^The sqlite3_vtab_rhs_value() interface +** can return an result code other than SQLITE_OK or SQLITE_NOTFOUND if +** something goes wrong. +** +** The sqlite3_vtab_rhs_value() interface is usually only successful if +** the right-hand operand of a constraint is a literal value in the original +** SQL statement. If the right-hand operand is an expression or a reference +** to some other column or a [host parameter], then sqlite3_vtab_rhs_value() +** will probably return [SQLITE_NOTFOUND]. +** +** ^(Some constraints, such as [SQLITE_INDEX_CONSTRAINT_ISNULL] and +** [SQLITE_INDEX_CONSTRAINT_ISNOTNULL], have no right-hand operand. For such +** constraints, sqlite3_vtab_rhs_value() always returns SQLITE_NOTFOUND.)^ +** +** ^The [sqlite3_value] object returned in *V is a protected sqlite3_value +** and remains valid for the duration of the xBestIndex method call. +** ^When xBestIndex returns, the sqlite3_value object returned by +** sqlite3_vtab_rhs_value() is automatically deallocated. +** +** The "_rhs_" in the name of this routine is an abbreviation for +** "Right-Hand Side". +*/ +SQLITE_API int sqlite3_vtab_rhs_value(sqlite3_index_info*, int, sqlite3_value **ppVal); + +/* +** CAPI3REF: Conflict resolution modes +** KEYWORDS: {conflict resolution mode} +** +** These constants are returned by [sqlite3_vtab_on_conflict()] to +** inform a [virtual table] implementation what the [ON CONFLICT] mode +** is for the SQL statement being evaluated. +** +** Note that the [SQLITE_IGNORE] constant is also used as a potential +** return value from the [sqlite3_set_authorizer()] callback and that +** [SQLITE_ABORT] is also a [result code]. +*/ +#define SQLITE_ROLLBACK 1 +/* #define SQLITE_IGNORE 2 // Also used by sqlite3_authorizer() callback */ +#define SQLITE_FAIL 3 +/* #define SQLITE_ABORT 4 // Also an error code */ +#define SQLITE_REPLACE 5 + +/* +** CAPI3REF: Prepared Statement Scan Status Opcodes +** KEYWORDS: {scanstatus options} +** +** The following constants can be used for the T parameter to the +** [sqlite3_stmt_scanstatus(S,X,T,V)] interface. Each constant designates a +** different metric for sqlite3_stmt_scanstatus() to return. +** +** When the value returned to V is a string, space to hold that string is +** managed by the prepared statement S and will be automatically freed when +** S is finalized. +** +** Not all values are available for all query elements. When a value is +** not available, the output variable is set to -1 if the value is numeric, +** or to NULL if it is a string (SQLITE_SCANSTAT_NAME). +** +**
          +** [[SQLITE_SCANSTAT_NLOOP]]
          SQLITE_SCANSTAT_NLOOP
          +**
          ^The [sqlite3_int64] variable pointed to by the V parameter will be +** set to the total number of times that the X-th loop has run.
          +** +** [[SQLITE_SCANSTAT_NVISIT]]
          SQLITE_SCANSTAT_NVISIT
          +**
          ^The [sqlite3_int64] variable pointed to by the V parameter will be set +** to the total number of rows examined by all iterations of the X-th loop.
          +** +** [[SQLITE_SCANSTAT_EST]]
          SQLITE_SCANSTAT_EST
          +**
          ^The "double" variable pointed to by the V parameter will be set to the +** query planner's estimate for the average number of rows output from each +** iteration of the X-th loop. If the query planner's estimates was accurate, +** then this value will approximate the quotient NVISIT/NLOOP and the +** product of this value for all prior loops with the same SELECTID will +** be the NLOOP value for the current loop. +** +** [[SQLITE_SCANSTAT_NAME]]
          SQLITE_SCANSTAT_NAME
          +**
          ^The "const char *" variable pointed to by the V parameter will be set +** to a zero-terminated UTF-8 string containing the name of the index or table +** used for the X-th loop. +** +** [[SQLITE_SCANSTAT_EXPLAIN]]
          SQLITE_SCANSTAT_EXPLAIN
          +**
          ^The "const char *" variable pointed to by the V parameter will be set +** to a zero-terminated UTF-8 string containing the [EXPLAIN QUERY PLAN] +** description for the X-th loop. +** +** [[SQLITE_SCANSTAT_SELECTID]]
          SQLITE_SCANSTAT_SELECTID
          +**
          ^The "int" variable pointed to by the V parameter will be set to the +** id for the X-th query plan element. The id value is unique within the +** statement. The select-id is the same value as is output in the first +** column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_PARENTID]]
          SQLITE_SCANSTAT_PARENTID
          +**
          The "int" variable pointed to by the V parameter will be set to the +** the id of the parent of the current query element, if applicable, or +** to zero if the query element has no parent. This is the same value as +** returned in the second column of an [EXPLAIN QUERY PLAN] query. +** +** [[SQLITE_SCANSTAT_NCYCLE]]
          SQLITE_SCANSTAT_NCYCLE
          +**
          The sqlite3_int64 output value is set to the number of cycles, +** according to the processor time-stamp counter, that elapsed while the +** query element was being processed. This value is not available for +** all query elements - if it is unavailable the output variable is +** set to -1. +**
          +*/ +#define SQLITE_SCANSTAT_NLOOP 0 +#define SQLITE_SCANSTAT_NVISIT 1 +#define SQLITE_SCANSTAT_EST 2 +#define SQLITE_SCANSTAT_NAME 3 +#define SQLITE_SCANSTAT_EXPLAIN 4 +#define SQLITE_SCANSTAT_SELECTID 5 +#define SQLITE_SCANSTAT_PARENTID 6 +#define SQLITE_SCANSTAT_NCYCLE 7 + +/* +** CAPI3REF: Prepared Statement Scan Status +** METHOD: sqlite3_stmt +** +** These interfaces return information about the predicted and measured +** performance for pStmt. Advanced applications can use this +** interface to compare the predicted and the measured performance and +** issue warnings and/or rerun [ANALYZE] if discrepancies are found. +** +** Since this interface is expected to be rarely used, it is only +** available if SQLite is compiled using the [SQLITE_ENABLE_STMT_SCANSTATUS] +** compile-time option. +** +** The "iScanStatusOp" parameter determines which status information to return. +** The "iScanStatusOp" must be one of the [scanstatus options] or the behavior +** of this interface is undefined. ^The requested measurement is written into +** a variable pointed to by the "pOut" parameter. +** +** The "flags" parameter must be passed a mask of flags. At present only +** one flag is defined - SQLITE_SCANSTAT_COMPLEX. If SQLITE_SCANSTAT_COMPLEX +** is specified, then status information is available for all elements +** of a query plan that are reported by "EXPLAIN QUERY PLAN" output. If +** SQLITE_SCANSTAT_COMPLEX is not specified, then only query plan elements +** that correspond to query loops (the "SCAN..." and "SEARCH..." elements of +** the EXPLAIN QUERY PLAN output) are available. Invoking API +** sqlite3_stmt_scanstatus() is equivalent to calling +** sqlite3_stmt_scanstatus_v2() with a zeroed flags parameter. +** +** Parameter "idx" identifies the specific query element to retrieve statistics +** for. Query elements are numbered starting from zero. A value of -1 may be +** to query for statistics regarding the entire query. ^If idx is out of range +** - less than -1 or greater than or equal to the total number of query +** elements used to implement the statement - a non-zero value is returned and +** the variable that pOut points to is unchanged. +** +** See also: [sqlite3_stmt_scanstatus_reset()] +*/ +SQLITE_API int sqlite3_stmt_scanstatus( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + void *pOut /* Result written here */ +); +SQLITE_API int sqlite3_stmt_scanstatus_v2( + sqlite3_stmt *pStmt, /* Prepared statement for which info desired */ + int idx, /* Index of loop to report on */ + int iScanStatusOp, /* Information desired. SQLITE_SCANSTAT_* */ + int flags, /* Mask of flags defined below */ + void *pOut /* Result written here */ +); + +/* +** CAPI3REF: Prepared Statement Scan Status +** KEYWORDS: {scan status flags} +*/ +#define SQLITE_SCANSTAT_COMPLEX 0x0001 + +/* +** CAPI3REF: Zero Scan-Status Counters +** METHOD: sqlite3_stmt +** +** ^Zero all [sqlite3_stmt_scanstatus()] related event counters. +** +** This API is only available if the library is built with pre-processor +** symbol [SQLITE_ENABLE_STMT_SCANSTATUS] defined. +*/ +SQLITE_API void sqlite3_stmt_scanstatus_reset(sqlite3_stmt*); + +/* +** CAPI3REF: Flush caches to disk mid-transaction +** METHOD: sqlite3 +** +** ^If a write-transaction is open on [database connection] D when the +** [sqlite3_db_cacheflush(D)] interface invoked, any dirty +** pages in the pager-cache that are not currently in use are written out +** to disk. A dirty page may be in use if a database cursor created by an +** active SQL statement is reading from it, or if it is page 1 of a database +** file (page 1 is always "in use"). ^The [sqlite3_db_cacheflush(D)] +** interface flushes caches for all schemas - "main", "temp", and +** any [attached] databases. +** +** ^If this function needs to obtain extra database locks before dirty pages +** can be flushed to disk, it does so. ^If those locks cannot be obtained +** immediately and there is a busy-handler callback configured, it is invoked +** in the usual manner. ^If the required lock still cannot be obtained, then +** the database is skipped and an attempt made to flush any dirty pages +** belonging to the next (if any) database. ^If any databases are skipped +** because locks cannot be obtained, but no other error occurs, this +** function returns SQLITE_BUSY. +** +** ^If any other error occurs while flushing dirty pages to disk (for +** example an IO error or out-of-memory condition), then processing is +** abandoned and an SQLite [error code] is returned to the caller immediately. +** +** ^Otherwise, if no error occurs, [sqlite3_db_cacheflush()] returns SQLITE_OK. +** +** ^This function does not set the database handle error code or message +** returned by the [sqlite3_errcode()] and [sqlite3_errmsg()] functions. +*/ +SQLITE_API int sqlite3_db_cacheflush(sqlite3*); + +/* +** CAPI3REF: The pre-update hook. +** METHOD: sqlite3 +** +** ^These interfaces are only available if SQLite is compiled using the +** [SQLITE_ENABLE_PREUPDATE_HOOK] compile-time option. +** +** ^The [sqlite3_preupdate_hook()] interface registers a callback function +** that is invoked prior to each [INSERT], [UPDATE], and [DELETE] operation +** on a database table. +** ^At most one preupdate hook may be registered at a time on a single +** [database connection]; each call to [sqlite3_preupdate_hook()] overrides +** the previous setting. +** ^The preupdate hook is disabled by invoking [sqlite3_preupdate_hook()] +** with a NULL pointer as the second parameter. +** ^The third parameter to [sqlite3_preupdate_hook()] is passed through as +** the first parameter to callbacks. +** +** ^The preupdate hook only fires for changes to real database tables; the +** preupdate hook is not invoked for changes to [virtual tables] or to +** system tables like sqlite_sequence or sqlite_stat1. +** +** ^The second parameter to the preupdate callback is a pointer to +** the [database connection] that registered the preupdate hook. +** ^The third parameter to the preupdate callback is one of the constants +** [SQLITE_INSERT], [SQLITE_DELETE], or [SQLITE_UPDATE] to identify the +** kind of update operation that is about to occur. +** ^(The fourth parameter to the preupdate callback is the name of the +** database within the database connection that is being modified. This +** will be "main" for the main database or "temp" for TEMP tables or +** the name given after the AS keyword in the [ATTACH] statement for attached +** databases.)^ +** ^The fifth parameter to the preupdate callback is the name of the +** table that is being modified. +** +** For an UPDATE or DELETE operation on a [rowid table], the sixth +** parameter passed to the preupdate callback is the initial [rowid] of the +** row being modified or deleted. For an INSERT operation on a rowid table, +** or any operation on a WITHOUT ROWID table, the value of the sixth +** parameter is undefined. For an INSERT or UPDATE on a rowid table the +** seventh parameter is the final rowid value of the row being inserted +** or updated. The value of the seventh parameter passed to the callback +** function is not defined for operations on WITHOUT ROWID tables, or for +** DELETE operations on rowid tables. +** +** ^The sqlite3_preupdate_hook(D,C,P) function returns the P argument from +** the previous call on the same [database connection] D, or NULL for +** the first call on D. +** +** The [sqlite3_preupdate_old()], [sqlite3_preupdate_new()], +** [sqlite3_preupdate_count()], and [sqlite3_preupdate_depth()] interfaces +** provide additional information about a preupdate event. These routines +** may only be called from within a preupdate callback. Invoking any of +** these routines from outside of a preupdate callback or with a +** [database connection] pointer that is different from the one supplied +** to the preupdate callback results in undefined and probably undesirable +** behavior. +** +** ^The [sqlite3_preupdate_count(D)] interface returns the number of columns +** in the row that is being inserted, updated, or deleted. +** +** ^The [sqlite3_preupdate_old(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row before it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_UPDATE and SQLITE_DELETE +** preupdate callbacks; if it is used by an SQLITE_INSERT callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_new(D,N,P)] interface writes into P a pointer to +** a [protected sqlite3_value] that contains the value of the Nth column of +** the table row after it is updated. The N parameter must be between 0 +** and one less than the number of columns or the behavior will be +** undefined. This must only be used within SQLITE_INSERT and SQLITE_UPDATE +** preupdate callbacks; if it is used by an SQLITE_DELETE callback then the +** behavior is undefined. The [sqlite3_value] that P points to +** will be destroyed when the preupdate callback returns. +** +** ^The [sqlite3_preupdate_depth(D)] interface returns 0 if the preupdate +** callback was invoked as a result of a direct insert, update, or delete +** operation; or 1 for inserts, updates, or deletes invoked by top-level +** triggers; or 2 for changes resulting from triggers called by top-level +** triggers; and so forth. +** +** When the [sqlite3_blob_write()] API is used to update a blob column, +** the pre-update hook is invoked with SQLITE_DELETE. This is because the +** in this case the new values are not available. In this case, when a +** callback made with op==SQLITE_DELETE is actually a write using the +** sqlite3_blob_write() API, the [sqlite3_preupdate_blobwrite()] returns +** the index of the column being written. In other cases, where the +** pre-update hook is being invoked for some other reason, including a +** regular DELETE, sqlite3_preupdate_blobwrite() returns -1. +** +** See also: [sqlite3_update_hook()] +*/ +#if defined(SQLITE_ENABLE_PREUPDATE_HOOK) +SQLITE_API void *sqlite3_preupdate_hook( + sqlite3 *db, + void(*xPreUpdate)( + void *pCtx, /* Copy of third arg to preupdate_hook() */ + sqlite3 *db, /* Database handle */ + int op, /* SQLITE_UPDATE, DELETE or INSERT */ + char const *zDb, /* Database name */ + char const *zName, /* Table name */ + sqlite3_int64 iKey1, /* Rowid of row about to be deleted/updated */ + sqlite3_int64 iKey2 /* New rowid value (for a rowid UPDATE) */ + ), + void* +); +SQLITE_API int sqlite3_preupdate_old(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_count(sqlite3 *); +SQLITE_API int sqlite3_preupdate_depth(sqlite3 *); +SQLITE_API int sqlite3_preupdate_new(sqlite3 *, int, sqlite3_value **); +SQLITE_API int sqlite3_preupdate_blobwrite(sqlite3 *); +#endif + +/* +** CAPI3REF: Low-level system error code +** METHOD: sqlite3 +** +** ^Attempt to return the underlying operating system error code or error +** number that caused the most recent I/O error or failure to open a file. +** The return value is OS-dependent. For example, on unix systems, after +** [sqlite3_open_v2()] returns [SQLITE_CANTOPEN], this interface could be +** called to get back the underlying "errno" that caused the problem, such +** as ENOSPC, EAUTH, EISDIR, and so forth. +*/ +SQLITE_API int sqlite3_system_errno(sqlite3*); + +/* +** CAPI3REF: Database Snapshot +** KEYWORDS: {snapshot} {sqlite3_snapshot} +** +** An instance of the snapshot object records the state of a [WAL mode] +** database for some specific point in history. +** +** In [WAL mode], multiple [database connections] that are open on the +** same database file can each be reading a different historical version +** of the database file. When a [database connection] begins a read +** transaction, that connection sees an unchanging copy of the database +** as it existed for the point in time when the transaction first started. +** Subsequent changes to the database from other connections are not seen +** by the reader until a new read transaction is started. +** +** The sqlite3_snapshot object records state information about an historical +** version of the database file so that it is possible to later open a new read +** transaction that sees that historical version of the database rather than +** the most recent version. +*/ +typedef struct sqlite3_snapshot { + unsigned char hidden[48]; +} sqlite3_snapshot; + +/* +** CAPI3REF: Record A Database Snapshot +** CONSTRUCTOR: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_get(D,S,P)] interface attempts to make a +** new [sqlite3_snapshot] object that records the current state of +** schema S in database connection D. ^On success, the +** [sqlite3_snapshot_get(D,S,P)] interface writes a pointer to the newly +** created [sqlite3_snapshot] object into *P and returns SQLITE_OK. +** If there is not already a read-transaction open on schema S when +** this function is called, one is opened automatically. +** +** If a read-transaction is opened by this function, then it is guaranteed +** that the returned snapshot object may not be invalidated by a database +** writer or checkpointer until after the read-transaction is closed. This +** is not guaranteed if a read-transaction is already open when this +** function is called. In that case, any subsequent write or checkpoint +** operation on the database may invalidate the returned snapshot handle, +** even while the read-transaction remains open. +** +** The following must be true for this function to succeed. If any of +** the following statements are false when sqlite3_snapshot_get() is +** called, SQLITE_ERROR is returned. The final value of *P is undefined +** in this case. +** +**
            +**
          • The database handle must not be in [autocommit mode]. +** +**
          • Schema S of [database connection] D must be a [WAL mode] database. +** +**
          • There must not be a write transaction open on schema S of database +** connection D. +** +**
          • One or more transactions must have been written to the current wal +** file since it was created on disk (by any connection). This means +** that a snapshot cannot be taken on a wal mode database with no wal +** file immediately after it is first opened. At least one transaction +** must be written to it first. +**
          +** +** This function may also return SQLITE_NOMEM. If it is called with the +** database handle in autocommit mode but fails for some other reason, +** whether or not a read transaction is opened on schema S is undefined. +** +** The [sqlite3_snapshot] object returned from a successful call to +** [sqlite3_snapshot_get()] must be freed using [sqlite3_snapshot_free()] +** to avoid a memory leak. +** +** The [sqlite3_snapshot_get()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_get( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot **ppSnapshot +); + +/* +** CAPI3REF: Start a read transaction on an historical snapshot +** METHOD: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_open(D,S,P)] interface either starts a new read +** transaction or upgrades an existing one for schema S of +** [database connection] D such that the read transaction refers to +** historical [snapshot] P, rather than the most recent change to the +** database. ^The [sqlite3_snapshot_open()] interface returns SQLITE_OK +** on success or an appropriate [error code] if it fails. +** +** ^In order to succeed, the database connection must not be in +** [autocommit mode] when [sqlite3_snapshot_open(D,S,P)] is called. If there +** is already a read transaction open on schema S, then the database handle +** must have no active statements (SELECT statements that have been passed +** to sqlite3_step() but not sqlite3_reset() or sqlite3_finalize()). +** SQLITE_ERROR is returned if either of these conditions is violated, or +** if schema S does not exist, or if the snapshot object is invalid. +** +** ^A call to sqlite3_snapshot_open() will fail to open if the specified +** snapshot has been overwritten by a [checkpoint]. In this case +** SQLITE_ERROR_SNAPSHOT is returned. +** +** If there is already a read transaction open when this function is +** invoked, then the same read transaction remains open (on the same +** database snapshot) if SQLITE_ERROR, SQLITE_BUSY or SQLITE_ERROR_SNAPSHOT +** is returned. If another error code - for example SQLITE_PROTOCOL or an +** SQLITE_IOERR error code - is returned, then the final state of the +** read transaction is undefined. If SQLITE_OK is returned, then the +** read transaction is now open on database snapshot P. +** +** ^(A call to [sqlite3_snapshot_open(D,S,P)] will fail if the +** database connection D does not know that the database file for +** schema S is in [WAL mode]. A database connection might not know +** that the database file is in [WAL mode] if there has been no prior +** I/O on that database connection, or if the database entered [WAL mode] +** after the most recent I/O on the database connection.)^ +** (Hint: Run "[PRAGMA application_id]" against a newly opened +** database connection in order to make it ready to use snapshots.) +** +** The [sqlite3_snapshot_open()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_open( + sqlite3 *db, + const char *zSchema, + sqlite3_snapshot *pSnapshot +); + +/* +** CAPI3REF: Destroy a snapshot +** DESTRUCTOR: sqlite3_snapshot +** +** ^The [sqlite3_snapshot_free(P)] interface destroys [sqlite3_snapshot] P. +** The application must eventually free every [sqlite3_snapshot] object +** using this routine to avoid a memory leak. +** +** The [sqlite3_snapshot_free()] interface is only available when the +** [SQLITE_ENABLE_SNAPSHOT] compile-time option is used. +*/ +SQLITE_API SQLITE_EXPERIMENTAL void sqlite3_snapshot_free(sqlite3_snapshot*); + +/* +** CAPI3REF: Compare the ages of two snapshot handles. +** METHOD: sqlite3_snapshot +** +** The sqlite3_snapshot_cmp(P1, P2) interface is used to compare the ages +** of two valid snapshot handles. +** +** If the two snapshot handles are not associated with the same database +** file, the result of the comparison is undefined. +** +** Additionally, the result of the comparison is only valid if both of the +** snapshot handles were obtained by calling sqlite3_snapshot_get() since the +** last time the wal file was deleted. The wal file is deleted when the +** database is changed back to rollback mode or when the number of database +** clients drops to zero. If either snapshot handle was obtained before the +** wal file was last deleted, the value returned by this function +** is undefined. +** +** Otherwise, this API returns a negative value if P1 refers to an older +** snapshot than P2, zero if the two handles refer to the same database +** snapshot, and a positive value if P1 is a newer snapshot than P2. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_cmp( + sqlite3_snapshot *p1, + sqlite3_snapshot *p2 +); + +/* +** CAPI3REF: Recover snapshots from a wal file +** METHOD: sqlite3_snapshot +** +** If a [WAL file] remains on disk after all database connections close +** (either through the use of the [SQLITE_FCNTL_PERSIST_WAL] [file control] +** or because the last process to have the database opened exited without +** calling [sqlite3_close()]) and a new connection is subsequently opened +** on that database and [WAL file], the [sqlite3_snapshot_open()] interface +** will only be able to open the last transaction added to the WAL file +** even though the WAL file contains other valid transactions. +** +** This function attempts to scan the WAL file associated with database zDb +** of database handle db and make all valid snapshots available to +** sqlite3_snapshot_open(). It is an error if there is already a read +** transaction open on the database, or if the database is not a WAL mode +** database. +** +** SQLITE_OK is returned if successful, or an SQLite error code otherwise. +** +** This interface is only available if SQLite is compiled with the +** [SQLITE_ENABLE_SNAPSHOT] option. +*/ +SQLITE_API SQLITE_EXPERIMENTAL int sqlite3_snapshot_recover(sqlite3 *db, const char *zDb); + +/* +** CAPI3REF: Serialize a database +** +** The sqlite3_serialize(D,S,P,F) interface returns a pointer to memory +** that is a serialization of the S database on [database connection] D. +** If P is not a NULL pointer, then the size of the database in bytes +** is written into *P. +** +** For an ordinary on-disk database file, the serialization is just a +** copy of the disk file. For an in-memory database or a "TEMP" database, +** the serialization is the same sequence of bytes which would be written +** to disk if that database where backed up to disk. +** +** The usual case is that sqlite3_serialize() copies the serialization of +** the database into memory obtained from [sqlite3_malloc64()] and returns +** a pointer to that memory. The caller is responsible for freeing the +** returned value to avoid a memory leak. However, if the F argument +** contains the SQLITE_SERIALIZE_NOCOPY bit, then no memory allocations +** are made, and the sqlite3_serialize() function will return a pointer +** to the contiguous memory representation of the database that SQLite +** is currently using for that database, or NULL if the no such contiguous +** memory representation of the database exists. A contiguous memory +** representation of the database will usually only exist if there has +** been a prior call to [sqlite3_deserialize(D,S,...)] with the same +** values of D and S. +** The size of the database is written into *P even if the +** SQLITE_SERIALIZE_NOCOPY bit is set but no contiguous copy +** of the database exists. +** +** After the call, if the SQLITE_SERIALIZE_NOCOPY bit had been set, +** the returned buffer content will remain accessible and unchanged +** until either the next write operation on the connection or when +** the connection is closed, and applications must not modify the +** buffer. If the bit had been clear, the returned buffer will not +** be accessed by SQLite after the call. +** +** A call to sqlite3_serialize(D,S,P,F) might return NULL even if the +** SQLITE_SERIALIZE_NOCOPY bit is omitted from argument F if a memory +** allocation error occurs. +** +** This interface is omitted if SQLite is compiled with the +** [SQLITE_OMIT_DESERIALIZE] option. +*/ +SQLITE_API unsigned char *sqlite3_serialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to serialize. ex: "main", "temp", ... */ + sqlite3_int64 *piSize, /* Write size of the DB here, if not NULL */ + unsigned int mFlags /* Zero or more SQLITE_SERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_serialize +** +** Zero or more of the following constants can be OR-ed together for +** the F argument to [sqlite3_serialize(D,S,P,F)]. +** +** SQLITE_SERIALIZE_NOCOPY means that [sqlite3_serialize()] will return +** a pointer to contiguous in-memory database that it is currently using, +** without making a copy of the database. If SQLite is not currently using +** a contiguous in-memory database, then this option causes +** [sqlite3_serialize()] to return a NULL pointer. SQLite will only be +** using a contiguous in-memory database if it has been initialized by a +** prior call to [sqlite3_deserialize()]. +*/ +#define SQLITE_SERIALIZE_NOCOPY 0x001 /* Do no memory allocations */ + +/* +** CAPI3REF: Deserialize a database +** +** The sqlite3_deserialize(D,S,P,N,M,F) interface causes the +** [database connection] D to disconnect from database S and then +** reopen S as an in-memory database based on the serialization contained +** in P. The serialized database P is N bytes in size. M is the size of +** the buffer P, which might be larger than N. If M is larger than N, and +** the SQLITE_DESERIALIZE_READONLY bit is not set in F, then SQLite is +** permitted to add content to the in-memory database as long as the total +** size does not exceed M bytes. +** +** If the SQLITE_DESERIALIZE_FREEONCLOSE bit is set in F, then SQLite will +** invoke sqlite3_free() on the serialization buffer when the database +** connection closes. If the SQLITE_DESERIALIZE_RESIZEABLE bit is set, then +** SQLite will try to increase the buffer size using sqlite3_realloc64() +** if writes on the database cause it to grow larger than M bytes. +** +** Applications must not modify the buffer P or invalidate it before +** the database connection D is closed. +** +** The sqlite3_deserialize() interface will fail with SQLITE_BUSY if the +** database is currently in a read transaction or is involved in a backup +** operation. +** +** It is not possible to deserialized into the TEMP database. If the +** S argument to sqlite3_deserialize(D,S,P,N,M,F) is "temp" then the +** function returns SQLITE_ERROR. +** +** The deserialized database should not be in [WAL mode]. If the database +** is in WAL mode, then any attempt to use the database file will result +** in an [SQLITE_CANTOPEN] error. The application can set the +** [file format version numbers] (bytes 18 and 19) of the input database P +** to 0x01 prior to invoking sqlite3_deserialize(D,S,P,N,M,F) to force the +** database file into rollback mode and work around this limitation. +** +** If sqlite3_deserialize(D,S,P,N,M,F) fails for any reason and if the +** SQLITE_DESERIALIZE_FREEONCLOSE bit is set in argument F, then +** [sqlite3_free()] is invoked on argument P prior to returning. +** +** This interface is omitted if SQLite is compiled with the +** [SQLITE_OMIT_DESERIALIZE] option. +*/ +SQLITE_API int sqlite3_deserialize( + sqlite3 *db, /* The database connection */ + const char *zSchema, /* Which DB to reopen with the deserialization */ + unsigned char *pData, /* The serialized database content */ + sqlite3_int64 szDb, /* Number bytes in the deserialization */ + sqlite3_int64 szBuf, /* Total size of buffer pData[] */ + unsigned mFlags /* Zero or more SQLITE_DESERIALIZE_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3_deserialize() +** +** The following are allowed values for 6th argument (the F argument) to +** the [sqlite3_deserialize(D,S,P,N,M,F)] interface. +** +** The SQLITE_DESERIALIZE_FREEONCLOSE means that the database serialization +** in the P argument is held in memory obtained from [sqlite3_malloc64()] +** and that SQLite should take ownership of this memory and automatically +** free it when it has finished using it. Without this flag, the caller +** is responsible for freeing any dynamically allocated memory. +** +** The SQLITE_DESERIALIZE_RESIZEABLE flag means that SQLite is allowed to +** grow the size of the database using calls to [sqlite3_realloc64()]. This +** flag should only be used if SQLITE_DESERIALIZE_FREEONCLOSE is also used. +** Without this flag, the deserialized database cannot increase in size beyond +** the number of bytes specified by the M parameter. +** +** The SQLITE_DESERIALIZE_READONLY flag means that the deserialized database +** should be treated as read-only. +*/ +#define SQLITE_DESERIALIZE_FREEONCLOSE 1 /* Call sqlite3_free() on close */ +#define SQLITE_DESERIALIZE_RESIZEABLE 2 /* Resize using sqlite3_realloc64() */ +#define SQLITE_DESERIALIZE_READONLY 4 /* Database is read-only */ /* ** Undo the hack that converts floating point types to integer for @@ -2673,7 +10875,2709 @@ int sqlite3_blob_write(sqlite3_blob *, const void *z, int n, int iOffset); # undef double #endif +#if defined(__wasi__) +# undef SQLITE_WASI +# define SQLITE_WASI 1 +# ifndef SQLITE_OMIT_LOAD_EXTENSION +# define SQLITE_OMIT_LOAD_EXTENSION +# endif +# ifndef SQLITE_THREADSAFE +# define SQLITE_THREADSAFE 0 +# endif +#endif + #ifdef __cplusplus } /* End of the 'extern "C"' block */ #endif +#endif /* SQLITE3_H */ + +/******** Begin file sqlite3rtree.h *********/ +/* +** 2010 August 30 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +************************************************************************* +*/ + +#ifndef _SQLITE3RTREE_H_ +#define _SQLITE3RTREE_H_ + + +#ifdef __cplusplus +extern "C" { #endif + +typedef struct sqlite3_rtree_geometry sqlite3_rtree_geometry; +typedef struct sqlite3_rtree_query_info sqlite3_rtree_query_info; + +/* The double-precision datatype used by RTree depends on the +** SQLITE_RTREE_INT_ONLY compile-time option. +*/ +#ifdef SQLITE_RTREE_INT_ONLY + typedef sqlite3_int64 sqlite3_rtree_dbl; +#else + typedef double sqlite3_rtree_dbl; +#endif + +/* +** Register a geometry callback named zGeom that can be used as part of an +** R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zGeom(... params ...) +*/ +SQLITE_API int sqlite3_rtree_geometry_callback( + sqlite3 *db, + const char *zGeom, + int (*xGeom)(sqlite3_rtree_geometry*, int, sqlite3_rtree_dbl*,int*), + void *pContext +); + + +/* +** A pointer to a structure of the following type is passed as the first +** argument to callbacks registered using rtree_geometry_callback(). +*/ +struct sqlite3_rtree_geometry { + void *pContext; /* Copy of pContext passed to s_r_g_c() */ + int nParam; /* Size of array aParam[] */ + sqlite3_rtree_dbl *aParam; /* Parameters passed to SQL geom function */ + void *pUser; /* Callback implementation user data */ + void (*xDelUser)(void *); /* Called by SQLite to clean up pUser */ +}; + +/* +** Register a 2nd-generation geometry callback named zScore that can be +** used as part of an R-Tree geometry query as follows: +** +** SELECT ... FROM WHERE MATCH $zQueryFunc(... params ...) +*/ +SQLITE_API int sqlite3_rtree_query_callback( + sqlite3 *db, + const char *zQueryFunc, + int (*xQueryFunc)(sqlite3_rtree_query_info*), + void *pContext, + void (*xDestructor)(void*) +); + + +/* +** A pointer to a structure of the following type is passed as the +** argument to scored geometry callback registered using +** sqlite3_rtree_query_callback(). +** +** Note that the first 5 fields of this structure are identical to +** sqlite3_rtree_geometry. This structure is a subclass of +** sqlite3_rtree_geometry. +*/ +struct sqlite3_rtree_query_info { + void *pContext; /* pContext from when function registered */ + int nParam; /* Number of function parameters */ + sqlite3_rtree_dbl *aParam; /* value of function parameters */ + void *pUser; /* callback can use this, if desired */ + void (*xDelUser)(void*); /* function to free pUser */ + sqlite3_rtree_dbl *aCoord; /* Coordinates of node or entry to check */ + unsigned int *anQueue; /* Number of pending entries in the queue */ + int nCoord; /* Number of coordinates */ + int iLevel; /* Level of current node or entry */ + int mxLevel; /* The largest iLevel value in the tree */ + sqlite3_int64 iRowid; /* Rowid for current entry */ + sqlite3_rtree_dbl rParentScore; /* Score of parent node */ + int eParentWithin; /* Visibility of parent node */ + int eWithin; /* OUT: Visibility */ + sqlite3_rtree_dbl rScore; /* OUT: Write the score here */ + /* The following fields are only available in 3.8.11 and later */ + sqlite3_value **apSqlParam; /* Original SQL values of parameters */ +}; + +/* +** Allowed values for sqlite3_rtree_query.eWithin and .eParentWithin. +*/ +#define NOT_WITHIN 0 /* Object completely outside of query region */ +#define PARTLY_WITHIN 1 /* Object partially overlaps query region */ +#define FULLY_WITHIN 2 /* Object fully contained within query region */ + + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* ifndef _SQLITE3RTREE_H_ */ + +/******** End of sqlite3rtree.h *********/ +/******** Begin file sqlite3session.h *********/ + +#if !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) +#define __SQLITESESSION_H_ 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +extern "C" { +#endif + + +/* +** CAPI3REF: Session Object Handle +** +** An instance of this object is a [session] that can be used to +** record changes to a database. +*/ +typedef struct sqlite3_session sqlite3_session; + +/* +** CAPI3REF: Changeset Iterator Handle +** +** An instance of this object acts as a cursor for iterating +** over the elements of a [changeset] or [patchset]. +*/ +typedef struct sqlite3_changeset_iter sqlite3_changeset_iter; + +/* +** CAPI3REF: Create A New Session Object +** CONSTRUCTOR: sqlite3_session +** +** Create a new session object attached to database handle db. If successful, +** a pointer to the new object is written to *ppSession and SQLITE_OK is +** returned. If an error occurs, *ppSession is set to NULL and an SQLite +** error code (e.g. SQLITE_NOMEM) is returned. +** +** It is possible to create multiple session objects attached to a single +** database handle. +** +** Session objects created using this function should be deleted using the +** [sqlite3session_delete()] function before the database handle that they +** are attached to is itself closed. If the database handle is closed before +** the session object is deleted, then the results of calling any session +** module function, including [sqlite3session_delete()] on the session object +** are undefined. +** +** Because the session module uses the [sqlite3_preupdate_hook()] API, it +** is not possible for an application to register a pre-update hook on a +** database handle that has one or more session objects attached. Nor is +** it possible to create a session object attached to a database handle for +** which a pre-update hook is already defined. The results of attempting +** either of these things are undefined. +** +** The session object will be used to create changesets for tables in +** database zDb, where zDb is either "main", or "temp", or the name of an +** attached database. It is not an error if database zDb is not attached +** to the database when the session object is created. +*/ +SQLITE_API int sqlite3session_create( + sqlite3 *db, /* Database handle */ + const char *zDb, /* Name of db (e.g. "main") */ + sqlite3_session **ppSession /* OUT: New session object */ +); + +/* +** CAPI3REF: Delete A Session Object +** DESTRUCTOR: sqlite3_session +** +** Delete a session object previously allocated using +** [sqlite3session_create()]. Once a session object has been deleted, the +** results of attempting to use pSession with any other session module +** function are undefined. +** +** Session objects must be deleted before the database handle to which they +** are attached is closed. Refer to the documentation for +** [sqlite3session_create()] for details. +*/ +SQLITE_API void sqlite3session_delete(sqlite3_session *pSession); + +/* +** CAPI3REF: Configure a Session Object +** METHOD: sqlite3_session +** +** This method is used to configure a session object after it has been +** created. At present the only valid values for the second parameter are +** [SQLITE_SESSION_OBJCONFIG_SIZE] and [SQLITE_SESSION_OBJCONFIG_ROWID]. +** +*/ +SQLITE_API int sqlite3session_object_config(sqlite3_session*, int op, void *pArg); + +/* +** CAPI3REF: Options for sqlite3session_object_config +** +** The following values may passed as the the 2nd parameter to +** sqlite3session_object_config(). +** +**
          SQLITE_SESSION_OBJCONFIG_SIZE
          +** This option is used to set, clear or query the flag that enables +** the [sqlite3session_changeset_size()] API. Because it imposes some +** computational overhead, this API is disabled by default. Argument +** pArg must point to a value of type (int). If the value is initially +** 0, then the sqlite3session_changeset_size() API is disabled. If it +** is greater than 0, then the same API is enabled. Or, if the initial +** value is less than zero, no change is made. In all cases the (int) +** variable is set to 1 if the sqlite3session_changeset_size() API is +** enabled following the current call, or 0 otherwise. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. +** +**
          SQLITE_SESSION_OBJCONFIG_ROWID
          +** This option is used to set, clear or query the flag that enables +** collection of data for tables with no explicit PRIMARY KEY. +** +** Normally, tables with no explicit PRIMARY KEY are simply ignored +** by the sessions module. However, if this flag is set, it behaves +** as if such tables have a column "_rowid_ INTEGER PRIMARY KEY" inserted +** as their leftmost columns. +** +** It is an error (SQLITE_MISUSE) to attempt to modify this setting after +** the first table has been attached to the session object. +*/ +#define SQLITE_SESSION_OBJCONFIG_SIZE 1 +#define SQLITE_SESSION_OBJCONFIG_ROWID 2 + +/* +** CAPI3REF: Enable Or Disable A Session Object +** METHOD: sqlite3_session +** +** Enable or disable the recording of changes by a session object. When +** enabled, a session object records changes made to the database. When +** disabled - it does not. A newly created session object is enabled. +** Refer to the documentation for [sqlite3session_changeset()] for further +** details regarding how enabling and disabling a session object affects +** the eventual changesets. +** +** Passing zero to this function disables the session. Passing a value +** greater than zero enables it. Passing a value less than zero is a +** no-op, and may be used to query the current state of the session. +** +** The return value indicates the final state of the session object: 0 if +** the session is disabled, or 1 if it is enabled. +*/ +SQLITE_API int sqlite3session_enable(sqlite3_session *pSession, int bEnable); + +/* +** CAPI3REF: Set Or Clear the Indirect Change Flag +** METHOD: sqlite3_session +** +** Each change recorded by a session object is marked as either direct or +** indirect. A change is marked as indirect if either: +** +**
            +**
          • The session object "indirect" flag is set when the change is +** made, or +**
          • The change is made by an SQL trigger or foreign key action +** instead of directly as a result of a users SQL statement. +**
          +** +** If a single row is affected by more than one operation within a session, +** then the change is considered indirect if all operations meet the criteria +** for an indirect change above, or direct otherwise. +** +** This function is used to set, clear or query the session object indirect +** flag. If the second argument passed to this function is zero, then the +** indirect flag is cleared. If it is greater than zero, the indirect flag +** is set. Passing a value less than zero does not modify the current value +** of the indirect flag, and may be used to query the current state of the +** indirect flag for the specified session object. +** +** The return value indicates the final state of the indirect flag: 0 if +** it is clear, or 1 if it is set. +*/ +SQLITE_API int sqlite3session_indirect(sqlite3_session *pSession, int bIndirect); + +/* +** CAPI3REF: Attach A Table To A Session Object +** METHOD: sqlite3_session +** +** If argument zTab is not NULL, then it is the name of a table to attach +** to the session object passed as the first argument. All subsequent changes +** made to the table while the session object is enabled will be recorded. See +** documentation for [sqlite3session_changeset()] for further details. +** +** Or, if argument zTab is NULL, then changes are recorded for all tables +** in the database. If additional tables are added to the database (by +** executing "CREATE TABLE" statements) after this call is made, changes for +** the new tables are also recorded. +** +** Changes can only be recorded for tables that have a PRIMARY KEY explicitly +** defined as part of their CREATE TABLE statement. It does not matter if the +** PRIMARY KEY is an "INTEGER PRIMARY KEY" (rowid alias) or not. The PRIMARY +** KEY may consist of a single column, or may be a composite key. +** +** It is not an error if the named table does not exist in the database. Nor +** is it an error if the named table does not have a PRIMARY KEY. However, +** no changes will be recorded in either of these scenarios. +** +** Changes are not recorded for individual rows that have NULL values stored +** in one or more of their PRIMARY KEY columns. +** +** SQLITE_OK is returned if the call completes without error. Or, if an error +** occurs, an SQLite error code (e.g. SQLITE_NOMEM) is returned. +** +**

          Special sqlite_stat1 Handling

          +** +** As of SQLite version 3.22.0, the "sqlite_stat1" table is an exception to +** some of the rules above. In SQLite, the schema of sqlite_stat1 is: +** 
          +**        CREATE TABLE sqlite_stat1(tbl,idx,stat)
+**
          +** +** Even though sqlite_stat1 does not have a PRIMARY KEY, changes are +** recorded for it as if the PRIMARY KEY is (tbl,idx). Additionally, changes +** are recorded for rows for which (idx IS NULL) is true. However, for such +** rows a zero-length blob (SQL value X'') is stored in the changeset or +** patchset instead of a NULL value. This allows such changesets to be +** manipulated by legacy implementations of sqlite3changeset_invert(), +** concat() and similar. +** +** The sqlite3changeset_apply() function automatically converts the +** zero-length blob back to a NULL value when updating the sqlite_stat1 +** table. However, if the application calls sqlite3changeset_new(), +** sqlite3changeset_old() or sqlite3changeset_conflict on a changeset +** iterator directly (including on a changeset iterator passed to a +** conflict-handler callback) then the X'' value is returned. The application +** must translate X'' to NULL itself if required. +** +** Legacy (older than 3.22.0) versions of the sessions module cannot capture +** changes made to the sqlite_stat1 table. Legacy versions of the +** sqlite3changeset_apply() function silently ignore any modifications to the +** sqlite_stat1 table that are part of a changeset or patchset. +*/ +SQLITE_API int sqlite3session_attach( + sqlite3_session *pSession, /* Session object */ + const char *zTab /* Table name */ +); + +/* +** CAPI3REF: Set a table filter on a Session Object. +** METHOD: sqlite3_session +** +** The second argument (xFilter) is the "filter callback". For changes to rows +** in tables that are not attached to the Session object, the filter is called +** to determine whether changes to the table's rows should be tracked or not. +** If xFilter returns 0, changes are not tracked. Note that once a table is +** attached, xFilter will not be called again. +*/ +SQLITE_API void sqlite3session_table_filter( + sqlite3_session *pSession, /* Session object */ + int(*xFilter)( + void *pCtx, /* Copy of third arg to _filter_table() */ + const char *zTab /* Table name */ + ), + void *pCtx /* First argument passed to xFilter */ +); + +/* +** CAPI3REF: Generate A Changeset From A Session Object +** METHOD: sqlite3_session +** +** Obtain a changeset containing changes to the tables attached to the +** session object passed as the first argument. If successful, +** set *ppChangeset to point to a buffer containing the changeset +** and *pnChangeset to the size of the changeset in bytes before returning +** SQLITE_OK. If an error occurs, set both *ppChangeset and *pnChangeset to +** zero and return an SQLite error code. +** +** A changeset consists of zero or more INSERT, UPDATE and/or DELETE changes, +** each representing a change to a single row of an attached table. An INSERT +** change contains the values of each field of a new database row. A DELETE +** contains the original values of each field of a deleted database row. An +** UPDATE change contains the original values of each field of an updated +** database row along with the updated values for each updated non-primary-key +** column. It is not possible for an UPDATE change to represent a change that +** modifies the values of primary key columns. If such a change is made, it +** is represented in a changeset as a DELETE followed by an INSERT. +** +** Changes are not recorded for rows that have NULL values stored in one or +** more of their PRIMARY KEY columns. If such a row is inserted or deleted, +** no corresponding change is present in the changesets returned by this +** function. If an existing row with one or more NULL values stored in +** PRIMARY KEY columns is updated so that all PRIMARY KEY columns are non-NULL, +** only an INSERT is appears in the changeset. Similarly, if an existing row +** with non-NULL PRIMARY KEY values is updated so that one or more of its +** PRIMARY KEY columns are set to NULL, the resulting changeset contains a +** DELETE change only. +** +** The contents of a changeset may be traversed using an iterator created +** using the [sqlite3changeset_start()] API. A changeset may be applied to +** a database with a compatible schema using the [sqlite3changeset_apply()] +** API. +** +** Within a changeset generated by this function, all changes related to a +** single table are grouped together. In other words, when iterating through +** a changeset or when applying a changeset to a database, all changes related +** to a single table are processed before moving on to the next table. Tables +** are sorted in the same order in which they were attached (or auto-attached) +** to the sqlite3_session object. The order in which the changes related to +** a single table are stored is undefined. +** +** Following a successful call to this function, it is the responsibility of +** the caller to eventually free the buffer that *ppChangeset points to using +** [sqlite3_free()]. +** +**

          Changeset Generation

          +** +** Once a table has been attached to a session object, the session object +** records the primary key values of all new rows inserted into the table. +** It also records the original primary key and other column values of any +** deleted or updated rows. For each unique primary key value, data is only +** recorded once - the first time a row with said primary key is inserted, +** updated or deleted in the lifetime of the session. +** +** There is one exception to the previous paragraph: when a row is inserted, +** updated or deleted, if one or more of its primary key columns contain a +** NULL value, no record of the change is made. +** +** The session object therefore accumulates two types of records - those +** that consist of primary key values only (created when the user inserts +** a new record) and those that consist of the primary key values and the +** original values of other table columns (created when the users deletes +** or updates a record). +** +** When this function is called, the requested changeset is created using +** both the accumulated records and the current contents of the database +** file. Specifically: +** +**
            +**
          • For each record generated by an insert, the database is queried +** for a row with a matching primary key. If one is found, an INSERT +** change is added to the changeset. If no such row is found, no change +** is added to the changeset. +** +**
          • For each record generated by an update or delete, the database is +** queried for a row with a matching primary key. If such a row is +** found and one or more of the non-primary key fields have been +** modified from their original values, an UPDATE change is added to +** the changeset. Or, if no such row is found in the table, a DELETE +** change is added to the changeset. If there is a row with a matching +** primary key in the database, but all fields contain their original +** values, no change is added to the changeset. +**
          +** +** This means, amongst other things, that if a row is inserted and then later +** deleted while a session object is active, neither the insert nor the delete +** will be present in the changeset. Or if a row is deleted and then later a +** row with the same primary key values inserted while a session object is +** active, the resulting changeset will contain an UPDATE change instead of +** a DELETE and an INSERT. +** +** When a session object is disabled (see the [sqlite3session_enable()] API), +** it does not accumulate records when rows are inserted, updated or deleted. +** This may appear to have some counter-intuitive effects if a single row +** is written to more than once during a session. For example, if a row +** is inserted while a session object is enabled, then later deleted while +** the same session object is disabled, no INSERT record will appear in the +** changeset, even though the delete took place while the session was disabled. +** Or, if one field of a row is updated while a session is disabled, and +** another field of the same row is updated while the session is enabled, the +** resulting changeset will contain an UPDATE change that updates both fields. +*/ +SQLITE_API int sqlite3session_changeset( + sqlite3_session *pSession, /* Session object */ + int *pnChangeset, /* OUT: Size of buffer at *ppChangeset */ + void **ppChangeset /* OUT: Buffer containing changeset */ +); + +/* +** CAPI3REF: Return An Upper-limit For The Size Of The Changeset +** METHOD: sqlite3_session +** +** By default, this function always returns 0. For it to return +** a useful result, the sqlite3_session object must have been configured +** to enable this API using sqlite3session_object_config() with the +** SQLITE_SESSION_OBJCONFIG_SIZE verb. +** +** When enabled, this function returns an upper limit, in bytes, for the size +** of the changeset that might be produced if sqlite3session_changeset() were +** called. The final changeset size might be equal to or smaller than the +** size in bytes returned by this function. +*/ +SQLITE_API sqlite3_int64 sqlite3session_changeset_size(sqlite3_session *pSession); + +/* +** CAPI3REF: Load The Difference Between Tables Into A Session +** METHOD: sqlite3_session +** +** If it is not already attached to the session object passed as the first +** argument, this function attaches table zTbl in the same manner as the +** [sqlite3session_attach()] function. If zTbl does not exist, or if it +** does not have a primary key, this function is a no-op (but does not return +** an error). +** +** Argument zFromDb must be the name of a database ("main", "temp" etc.) +** attached to the same database handle as the session object that contains +** a table compatible with the table attached to the session by this function. +** A table is considered compatible if it: +** +**
            +**
          • Has the same name, +**
          • Has the same set of columns declared in the same order, and +**
          • Has the same PRIMARY KEY definition. +**
          +** +** If the tables are not compatible, SQLITE_SCHEMA is returned. If the tables +** are compatible but do not have any PRIMARY KEY columns, it is not an error +** but no changes are added to the session object. As with other session +** APIs, tables without PRIMARY KEYs are simply ignored. +** +** This function adds a set of changes to the session object that could be +** used to update the table in database zFrom (call this the "from-table") +** so that its content is the same as the table attached to the session +** object (call this the "to-table"). Specifically: +** +**
            +**
          • For each row (primary key) that exists in the to-table but not in +** the from-table, an INSERT record is added to the session object. +** +**
          • For each row (primary key) that exists in the to-table but not in +** the from-table, a DELETE record is added to the session object. +** +**
          • For each row (primary key) that exists in both tables, but features +** different non-PK values in each, an UPDATE record is added to the +** session. +**
          +** +** To clarify, if this function is called and then a changeset constructed +** using [sqlite3session_changeset()], then after applying that changeset to +** database zFrom the contents of the two compatible tables would be +** identical. +** +** It an error if database zFrom does not exist or does not contain the +** required compatible table. +** +** If the operation is successful, SQLITE_OK is returned. Otherwise, an SQLite +** error code. In this case, if argument pzErrMsg is not NULL, *pzErrMsg +** may be set to point to a buffer containing an English language error +** message. It is the responsibility of the caller to free this buffer using +** sqlite3_free(). +*/ +SQLITE_API int sqlite3session_diff( + sqlite3_session *pSession, + const char *zFromDb, + const char *zTbl, + char **pzErrMsg +); + + +/* +** CAPI3REF: Generate A Patchset From A Session Object +** METHOD: sqlite3_session +** +** The differences between a patchset and a changeset are that: +** +**
            +**
          • DELETE records consist of the primary key fields only. The +** original values of other fields are omitted. +**
          • The original values of any modified fields are omitted from +** UPDATE records. +**
          +** +** A patchset blob may be used with up to date versions of all +** sqlite3changeset_xxx API functions except for sqlite3changeset_invert(), +** which returns SQLITE_CORRUPT if it is passed a patchset. Similarly, +** attempting to use a patchset blob with old versions of the +** sqlite3changeset_xxx APIs also provokes an SQLITE_CORRUPT error. +** +** Because the non-primary key "old.*" fields are omitted, no +** SQLITE_CHANGESET_DATA conflicts can be detected or reported if a patchset +** is passed to the sqlite3changeset_apply() API. Other conflict types work +** in the same way as for changesets. +** +** Changes within a patchset are ordered in the same way as for changesets +** generated by the sqlite3session_changeset() function (i.e. all changes for +** a single table are grouped together, tables appear in the order in which +** they were attached to the session object). +*/ +SQLITE_API int sqlite3session_patchset( + sqlite3_session *pSession, /* Session object */ + int *pnPatchset, /* OUT: Size of buffer at *ppPatchset */ + void **ppPatchset /* OUT: Buffer containing patchset */ +); + +/* +** CAPI3REF: Test if a changeset has recorded any changes. +** +** Return non-zero if no changes to attached tables have been recorded by +** the session object passed as the first argument. Otherwise, if one or +** more changes have been recorded, return zero. +** +** Even if this function returns zero, it is possible that calling +** [sqlite3session_changeset()] on the session handle may still return a +** changeset that contains no changes. This can happen when a row in +** an attached table is modified and then later on the original values +** are restored. However, if this function returns non-zero, then it is +** guaranteed that a call to sqlite3session_changeset() will return a +** changeset containing zero changes. +*/ +SQLITE_API int sqlite3session_isempty(sqlite3_session *pSession); + +/* +** CAPI3REF: Query for the amount of heap memory used by a session object. +** +** This API returns the total amount of heap memory in bytes currently +** used by the session object passed as the only argument. +*/ +SQLITE_API sqlite3_int64 sqlite3session_memory_used(sqlite3_session *pSession); + +/* +** CAPI3REF: Create An Iterator To Traverse A Changeset +** CONSTRUCTOR: sqlite3_changeset_iter +** +** Create an iterator used to iterate through the contents of a changeset. +** If successful, *pp is set to point to the iterator handle and SQLITE_OK +** is returned. Otherwise, if an error occurs, *pp is set to zero and an +** SQLite error code is returned. +** +** The following functions can be used to advance and query a changeset +** iterator created by this function: +** +**
            +**
          • [sqlite3changeset_next()] +**
          • [sqlite3changeset_op()] +**
          • [sqlite3changeset_new()] +**
          • [sqlite3changeset_old()] +**
          +** +** It is the responsibility of the caller to eventually destroy the iterator +** by passing it to [sqlite3changeset_finalize()]. The buffer containing the +** changeset (pChangeset) must remain valid until after the iterator is +** destroyed. +** +** Assuming the changeset blob was created by one of the +** [sqlite3session_changeset()], [sqlite3changeset_concat()] or +** [sqlite3changeset_invert()] functions, all changes within the changeset +** that apply to a single table are grouped together. This means that when +** an application iterates through a changeset using an iterator created by +** this function, all changes that relate to a single table are visited +** consecutively. There is no chance that the iterator will visit a change +** the applies to table X, then one for table Y, and then later on visit +** another change for table X. +** +** The behavior of sqlite3changeset_start_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETSTART_INVERT | supported flags] as the 4th parameter. +** +** Note that the sqlite3changeset_start_v2() API is still experimental +** and therefore subject to change. +*/ +SQLITE_API int sqlite3changeset_start( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset /* Pointer to blob containing changeset */ +); +SQLITE_API int sqlite3changeset_start_v2( + sqlite3_changeset_iter **pp, /* OUT: New changeset iterator handle */ + int nChangeset, /* Size of changeset blob in bytes */ + void *pChangeset, /* Pointer to blob containing changeset */ + int flags /* SESSION_CHANGESETSTART_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_start_v2 +** +** The following flags may passed via the 4th parameter to +** [sqlite3changeset_start_v2] and [sqlite3changeset_start_v2_strm]: +** +**
          SQLITE_CHANGESETAPPLY_INVERT
          +** Invert the changeset while iterating through it. This is equivalent to +** inverting a changeset using sqlite3changeset_invert() before applying it. +** It is an error to specify this flag with a patchset. +*/ +#define SQLITE_CHANGESETSTART_INVERT 0x0002 + + +/* +** CAPI3REF: Advance A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function may only be used with iterators created by the function +** [sqlite3changeset_start()]. If it is called on an iterator passed to +** a conflict-handler callback by [sqlite3changeset_apply()], SQLITE_MISUSE +** is returned and the call has no effect. +** +** Immediately after an iterator is created by sqlite3changeset_start(), it +** does not point to any change in the changeset. Assuming the changeset +** is not empty, the first call to this function advances the iterator to +** point to the first change in the changeset. Each subsequent call advances +** the iterator to point to the next change in the changeset (if any). If +** no error occurs and the iterator points to a valid change after a call +** to sqlite3changeset_next() has advanced it, SQLITE_ROW is returned. +** Otherwise, if all changes in the changeset have already been visited, +** SQLITE_DONE is returned. +** +** If an error occurs, an SQLite error code is returned. Possible error +** codes include SQLITE_CORRUPT (if the changeset buffer is corrupt) or +** SQLITE_NOMEM. +*/ +SQLITE_API int sqlite3changeset_next(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Obtain The Current Operation From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned [SQLITE_ROW]. If this +** is not the case, this function returns [SQLITE_MISUSE]. +** +** Arguments pOp, pnCol and pzTab may not be NULL. Upon return, three +** outputs are set through these pointers: +** +** *pOp is set to one of [SQLITE_INSERT], [SQLITE_DELETE] or [SQLITE_UPDATE], +** depending on the type of change that the iterator currently points to; +** +** *pnCol is set to the number of columns in the table affected by the change; and +** +** *pzTab is set to point to a nul-terminated utf-8 encoded string containing +** the name of the table affected by the current change. The buffer remains +** valid until either sqlite3changeset_next() is called on the iterator +** or until the conflict-handler function returns. +** +** If pbIndirect is not NULL, then *pbIndirect is set to true (1) if the change +** is an indirect change, or false (0) otherwise. See the documentation for +** [sqlite3session_indirect()] for a description of direct and indirect +** changes. +** +** If no error occurs, SQLITE_OK is returned. If an error does occur, an +** SQLite error code is returned. The values of the output variables may not +** be trusted in this case. +*/ +SQLITE_API int sqlite3changeset_op( + sqlite3_changeset_iter *pIter, /* Iterator object */ + const char **pzTab, /* OUT: Pointer to table name */ + int *pnCol, /* OUT: Number of columns in table */ + int *pOp, /* OUT: SQLITE_INSERT, DELETE or UPDATE */ + int *pbIndirect /* OUT: True for an 'indirect' change */ +); + +/* +** CAPI3REF: Obtain The Primary Key Definition Of A Table +** METHOD: sqlite3_changeset_iter +** +** For each modified table, a changeset includes the following: +** +**
            +**
          • The number of columns in the table, and +**
          • Which of those columns make up the tables PRIMARY KEY. +**
          +** +** This function is used to find which columns comprise the PRIMARY KEY of +** the table modified by the change that iterator pIter currently points to. +** If successful, *pabPK is set to point to an array of nCol entries, where +** nCol is the number of columns in the table. Elements of *pabPK are set to +** 0x01 if the corresponding column is part of the tables primary key, or +** 0x00 if it is not. +** +** If argument pnCol is not NULL, then *pnCol is set to the number of columns +** in the table. +** +** If this function is called when the iterator does not point to a valid +** entry, SQLITE_MISUSE is returned and the output variables zeroed. Otherwise, +** SQLITE_OK is returned and the output variables populated as described +** above. +*/ +SQLITE_API int sqlite3changeset_pk( + sqlite3_changeset_iter *pIter, /* Iterator object */ + unsigned char **pabPK, /* OUT: Array of boolean - true for PK cols */ + int *pnCol /* OUT: Number of entries in output array */ +); + +/* +** CAPI3REF: Obtain old.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_DELETE] or [SQLITE_UPDATE]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** original row values stored as part of the UPDATE or DELETE change and +** returns SQLITE_OK. The name of the function comes from the fact that this +** is similar to the "old.*" columns available to update or delete triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_old( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Old value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain new.* Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** The pIter argument passed to this function may either be an iterator +** passed to a conflict-handler by [sqlite3changeset_apply()], or an iterator +** created by [sqlite3changeset_start()]. In the latter case, the most recent +** call to [sqlite3changeset_next()] must have returned SQLITE_ROW. +** Furthermore, it may only be called if the type of change that the iterator +** currently points to is either [SQLITE_UPDATE] or [SQLITE_INSERT]. Otherwise, +** this function returns [SQLITE_MISUSE] and sets *ppValue to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the vector of +** new row values stored as part of the UPDATE or INSERT change and +** returns SQLITE_OK. If the change is an UPDATE and does not include +** a new value for the requested column, *ppValue is set to NULL and +** SQLITE_OK returned. The name of the function comes from the fact that +** this is similar to the "new.*" columns available to update or delete +** triggers. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_new( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: New value (or NULL pointer) */ +); + +/* +** CAPI3REF: Obtain Conflicting Row Values From A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function should only be used with iterator objects passed to a +** conflict-handler callback by [sqlite3changeset_apply()] with either +** [SQLITE_CHANGESET_DATA] or [SQLITE_CHANGESET_CONFLICT]. If this function +** is called on any other iterator, [SQLITE_MISUSE] is returned and *ppValue +** is set to NULL. +** +** Argument iVal must be greater than or equal to 0, and less than the number +** of columns in the table affected by the current change. Otherwise, +** [SQLITE_RANGE] is returned and *ppValue is set to NULL. +** +** If successful, this function sets *ppValue to point to a protected +** sqlite3_value object containing the iVal'th value from the +** "conflicting row" associated with the current conflict-handler callback +** and returns SQLITE_OK. +** +** If some other error occurs (e.g. an OOM condition), an SQLite error code +** is returned and *ppValue is set to NULL. +*/ +SQLITE_API int sqlite3changeset_conflict( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int iVal, /* Column number */ + sqlite3_value **ppValue /* OUT: Value from conflicting row */ +); + +/* +** CAPI3REF: Determine The Number Of Foreign Key Constraint Violations +** METHOD: sqlite3_changeset_iter +** +** This function may only be called with an iterator passed to an +** SQLITE_CHANGESET_FOREIGN_KEY conflict handler callback. In this case +** it sets the output variable to the total number of known foreign key +** violations in the destination database and returns SQLITE_OK. +** +** In all other cases this function returns SQLITE_MISUSE. +*/ +SQLITE_API int sqlite3changeset_fk_conflicts( + sqlite3_changeset_iter *pIter, /* Changeset iterator */ + int *pnOut /* OUT: Number of FK violations */ +); + + +/* +** CAPI3REF: Finalize A Changeset Iterator +** METHOD: sqlite3_changeset_iter +** +** This function is used to finalize an iterator allocated with +** [sqlite3changeset_start()]. +** +** This function should only be called on iterators created using the +** [sqlite3changeset_start()] function. If an application calls this +** function with an iterator passed to a conflict-handler by +** [sqlite3changeset_apply()], [SQLITE_MISUSE] is immediately returned and the +** call has no effect. +** +** If an error was encountered within a call to an sqlite3changeset_xxx() +** function (for example an [SQLITE_CORRUPT] in [sqlite3changeset_next()] or an +** [SQLITE_NOMEM] in [sqlite3changeset_new()]) then an error code corresponding +** to that error is returned by this function. Otherwise, SQLITE_OK is +** returned. This is to allow the following pattern (pseudo-code): +** +** 
          +**   sqlite3changeset_start();
+**   while( SQLITE_ROW==sqlite3changeset_next() ){
+**     // Do something with change.
+**   }
+**   rc = sqlite3changeset_finalize();
+**   if( rc!=SQLITE_OK ){
+**     // An error has occurred
+**   }
+**
          +*/ +SQLITE_API int sqlite3changeset_finalize(sqlite3_changeset_iter *pIter); + +/* +** CAPI3REF: Invert A Changeset +** +** This function is used to "invert" a changeset object. Applying an inverted +** changeset to a database reverses the effects of applying the uninverted +** changeset. Specifically: +** +**
            +**
          • Each DELETE change is changed to an INSERT, and +**
          • Each INSERT change is changed to a DELETE, and +**
          • For each UPDATE change, the old.* and new.* values are exchanged. +**
          +** +** This function does not change the order in which changes appear within +** the changeset. It merely reverses the sense of each individual change. +** +** If successful, a pointer to a buffer containing the inverted changeset +** is stored in *ppOut, the size of the same buffer is stored in *pnOut, and +** SQLITE_OK is returned. If an error occurs, both *pnOut and *ppOut are +** zeroed and an SQLite error code returned. +** +** It is the responsibility of the caller to eventually call sqlite3_free() +** on the *ppOut pointer to free the buffer allocation following a successful +** call to this function. +** +** WARNING/TODO: This function currently assumes that the input is a valid +** changeset. If it is not, the results are undefined. +*/ +SQLITE_API int sqlite3changeset_invert( + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + +/* +** CAPI3REF: Concatenate Two Changeset Objects +** +** This function is used to concatenate two changesets, A and B, into a +** single changeset. The result is a changeset equivalent to applying +** changeset A followed by changeset B. +** +** This function combines the two input changesets using an +** sqlite3_changegroup object. Calling it produces similar results as the +** following code fragment: +** +** 
          +**   sqlite3_changegroup *pGrp;
+**   rc = sqlite3_changegroup_new(&pGrp);
+**   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nA, pA);
+**   if( rc==SQLITE_OK ) rc = sqlite3changegroup_add(pGrp, nB, pB);
+**   if( rc==SQLITE_OK ){
+**     rc = sqlite3changegroup_output(pGrp, pnOut, ppOut);
+**   }else{
+**     *ppOut = 0;
+**     *pnOut = 0;
+**   }
+**
          +** +** Refer to the sqlite3_changegroup documentation below for details. +*/ +SQLITE_API int sqlite3changeset_concat( + int nA, /* Number of bytes in buffer pA */ + void *pA, /* Pointer to buffer containing changeset A */ + int nB, /* Number of bytes in buffer pB */ + void *pB, /* Pointer to buffer containing changeset B */ + int *pnOut, /* OUT: Number of bytes in output changeset */ + void **ppOut /* OUT: Buffer containing output changeset */ +); + + +/* +** CAPI3REF: Upgrade the Schema of a Changeset/Patchset +*/ +SQLITE_API int sqlite3changeset_upgrade( + sqlite3 *db, + const char *zDb, + int nIn, const void *pIn, /* Input changeset */ + int *pnOut, void **ppOut /* OUT: Inverse of input */ +); + + + +/* +** CAPI3REF: Changegroup Handle +** +** A changegroup is an object used to combine two or more +** [changesets] or [patchsets] +*/ +typedef struct sqlite3_changegroup sqlite3_changegroup; + +/* +** CAPI3REF: Create A New Changegroup Object +** CONSTRUCTOR: sqlite3_changegroup +** +** An sqlite3_changegroup object is used to combine two or more changesets +** (or patchsets) into a single changeset (or patchset). A single changegroup +** object may combine changesets or patchsets, but not both. The output is +** always in the same format as the input. +** +** If successful, this function returns SQLITE_OK and populates (*pp) with +** a pointer to a new sqlite3_changegroup object before returning. The caller +** should eventually free the returned object using a call to +** sqlite3changegroup_delete(). If an error occurs, an SQLite error code +** (i.e. SQLITE_NOMEM) is returned and *pp is set to NULL. +** +** The usual usage pattern for an sqlite3_changegroup object is as follows: +** +**
            +**
          • It is created using a call to sqlite3changegroup_new(). +** +**
          • Zero or more changesets (or patchsets) are added to the object +** by calling sqlite3changegroup_add(). +** +**
          • The result of combining all input changesets together is obtained +** by the application via a call to sqlite3changegroup_output(). +** +**
          • The object is deleted using a call to sqlite3changegroup_delete(). +**
          +** +** Any number of calls to add() and output() may be made between the calls to +** new() and delete(), and in any order. +** +** As well as the regular sqlite3changegroup_add() and +** sqlite3changegroup_output() functions, also available are the streaming +** versions sqlite3changegroup_add_strm() and sqlite3changegroup_output_strm(). +*/ +SQLITE_API int sqlite3changegroup_new(sqlite3_changegroup **pp); + +/* +** CAPI3REF: Add a Schema to a Changegroup +** METHOD: sqlite3_changegroup_schema +** +** This method may be used to optionally enforce the rule that the changesets +** added to the changegroup handle must match the schema of database zDb +** ("main", "temp", or the name of an attached database). If +** sqlite3changegroup_add() is called to add a changeset that is not compatible +** with the configured schema, SQLITE_SCHEMA is returned and the changegroup +** object is left in an undefined state. +** +** A changeset schema is considered compatible with the database schema in +** the same way as for sqlite3changeset_apply(). Specifically, for each +** table in the changeset, there exists a database table with: +** +**
            +**
          • The name identified by the changeset, and +**
          • at least as many columns as recorded in the changeset, and +**
          • the primary key columns in the same position as recorded in +** the changeset. +**
          +** +** The output of the changegroup object always has the same schema as the +** database nominated using this function. In cases where changesets passed +** to sqlite3changegroup_add() have fewer columns than the corresponding table +** in the database schema, these are filled in using the default column +** values from the database schema. This makes it possible to combined +** changesets that have different numbers of columns for a single table +** within a changegroup, provided that they are otherwise compatible. +*/ +SQLITE_API int sqlite3changegroup_schema(sqlite3_changegroup*, sqlite3*, const char *zDb); + +/* +** CAPI3REF: Add A Changeset To A Changegroup +** METHOD: sqlite3_changegroup +** +** Add all changes within the changeset (or patchset) in buffer pData (size +** nData bytes) to the changegroup. +** +** If the buffer contains a patchset, then all prior calls to this function +** on the same changegroup object must also have specified patchsets. Or, if +** the buffer contains a changeset, so must have the earlier calls to this +** function. Otherwise, SQLITE_ERROR is returned and no changes are added +** to the changegroup. +** +** Rows within the changeset and changegroup are identified by the values in +** their PRIMARY KEY columns. A change in the changeset is considered to +** apply to the same row as a change already present in the changegroup if +** the two rows have the same primary key. +** +** Changes to rows that do not already appear in the changegroup are +** simply copied into it. Or, if both the new changeset and the changegroup +** contain changes that apply to a single row, the final contents of the +** changegroup depends on the type of each change, as follows: +** +** +** +** +**
          Existing Change New Change Output Change +**
          INSERT INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
          INSERT UPDATE +** The INSERT change remains in the changegroup. The values in the +** INSERT change are modified as if the row was inserted by the +** existing change and then updated according to the new change. +**
          INSERT DELETE +** The existing INSERT is removed from the changegroup. The DELETE is +** not added. +**
          UPDATE INSERT +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
          UPDATE UPDATE +** The existing UPDATE remains within the changegroup. It is amended +** so that the accompanying values are as if the row was updated once +** by the existing change and then again by the new change. +**
          UPDATE DELETE +** The existing UPDATE is replaced by the new DELETE within the +** changegroup. +**
          DELETE INSERT +** If one or more of the column values in the row inserted by the +** new change differ from those in the row deleted by the existing +** change, the existing DELETE is replaced by an UPDATE within the +** changegroup. Otherwise, if the inserted row is exactly the same +** as the deleted row, the existing DELETE is simply discarded. +**
          DELETE UPDATE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
          DELETE DELETE +** The new change is ignored. This case does not occur if the new +** changeset was recorded immediately after the changesets already +** added to the changegroup. +**
          +** +** If the new changeset contains changes to a table that is already present +** in the changegroup, then the number of columns and the position of the +** primary key columns for the table must be consistent. If this is not the +** case, this function fails with SQLITE_SCHEMA. Except, if the changegroup +** object has been configured with a database schema using the +** sqlite3changegroup_schema() API, then it is possible to combine changesets +** with different numbers of columns for a single table, provided that +** they are otherwise compatible. +** +** If the input changeset appears to be corrupt and the corruption is +** detected, SQLITE_CORRUPT is returned. Or, if an out-of-memory condition +** occurs during processing, this function returns SQLITE_NOMEM. +** +** In all cases, if an error occurs the state of the final contents of the +** changegroup is undefined. If no error occurs, SQLITE_OK is returned. +*/ +SQLITE_API int sqlite3changegroup_add(sqlite3_changegroup*, int nData, void *pData); + +/* +** CAPI3REF: Add A Single Change To A Changegroup +** METHOD: sqlite3_changegroup +** +** This function adds the single change currently indicated by the iterator +** passed as the second argument to the changegroup object. The rules for +** adding the change are just as described for [sqlite3changegroup_add()]. +** +** If the change is successfully added to the changegroup, SQLITE_OK is +** returned. Otherwise, an SQLite error code is returned. +** +** The iterator must point to a valid entry when this function is called. +** If it does not, SQLITE_ERROR is returned and no change is added to the +** changegroup. Additionally, the iterator must not have been opened with +** the SQLITE_CHANGESETAPPLY_INVERT flag. In this case SQLITE_ERROR is also +** returned. +*/ +SQLITE_API int sqlite3changegroup_add_change( + sqlite3_changegroup*, + sqlite3_changeset_iter* +); + + + +/* +** CAPI3REF: Obtain A Composite Changeset From A Changegroup +** METHOD: sqlite3_changegroup +** +** Obtain a buffer containing a changeset (or patchset) representing the +** current contents of the changegroup. If the inputs to the changegroup +** were themselves changesets, the output is a changeset. Or, if the +** inputs were patchsets, the output is also a patchset. +** +** As with the output of the sqlite3session_changeset() and +** sqlite3session_patchset() functions, all changes related to a single +** table are grouped together in the output of this function. Tables appear +** in the same order as for the very first changeset added to the changegroup. +** If the second or subsequent changesets added to the changegroup contain +** changes for tables that do not appear in the first changeset, they are +** appended onto the end of the output changeset, again in the order in +** which they are first encountered. +** +** If an error occurs, an SQLite error code is returned and the output +** variables (*pnData) and (*ppData) are set to 0. Otherwise, SQLITE_OK +** is returned and the output variables are set to the size of and a +** pointer to the output buffer, respectively. In this case it is the +** responsibility of the caller to eventually free the buffer using a +** call to sqlite3_free(). +*/ +SQLITE_API int sqlite3changegroup_output( + sqlite3_changegroup*, + int *pnData, /* OUT: Size of output buffer in bytes */ + void **ppData /* OUT: Pointer to output buffer */ +); + +/* +** CAPI3REF: Delete A Changegroup Object +** DESTRUCTOR: sqlite3_changegroup +*/ +SQLITE_API void sqlite3changegroup_delete(sqlite3_changegroup*); + +/* +** CAPI3REF: Apply A Changeset To A Database +** +** Apply a changeset or patchset to a database. These functions attempt to +** update the "main" database attached to handle db with the changes found in +** the changeset passed via the second and third arguments. +** +** The fourth argument (xFilter) passed to these functions is the "filter +** callback". If it is not NULL, then for each table affected by at least one +** change in the changeset, the filter callback is invoked with +** the table name as the second argument, and a copy of the context pointer +** passed as the sixth argument as the first. If the "filter callback" +** returns zero, then no attempt is made to apply any changes to the table. +** Otherwise, if the return value is non-zero or the xFilter argument to +** is NULL, all changes related to the table are attempted. +** +** For each table that is not excluded by the filter callback, this function +** tests that the target database contains a compatible table. A table is +** considered compatible if all of the following are true: +** +**
            +**
          • The table has the same name as the name recorded in the +** changeset, and +**
          • The table has at least as many columns as recorded in the +** changeset, and +**
          • The table has primary key columns in the same position as +** recorded in the changeset. +**
          +** +** If there is no compatible table, it is not an error, but none of the +** changes associated with the table are applied. A warning message is issued +** via the sqlite3_log() mechanism with the error code SQLITE_SCHEMA. At most +** one such warning is issued for each table in the changeset. +** +** For each change for which there is a compatible table, an attempt is made +** to modify the table contents according to the UPDATE, INSERT or DELETE +** change. If a change cannot be applied cleanly, the conflict handler +** function passed as the fifth argument to sqlite3changeset_apply() may be +** invoked. A description of exactly when the conflict handler is invoked for +** each type of change is below. +** +** Unlike the xFilter argument, xConflict may not be passed NULL. The results +** of passing anything other than a valid function pointer as the xConflict +** argument are undefined. +** +** Each time the conflict handler function is invoked, it must return one +** of [SQLITE_CHANGESET_OMIT], [SQLITE_CHANGESET_ABORT] or +** [SQLITE_CHANGESET_REPLACE]. SQLITE_CHANGESET_REPLACE may only be returned +** if the second argument passed to the conflict handler is either +** SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If the conflict-handler +** returns an illegal value, any changes already made are rolled back and +** the call to sqlite3changeset_apply() returns SQLITE_MISUSE. Different +** actions are taken by sqlite3changeset_apply() depending on the value +** returned by each invocation of the conflict-handler function. Refer to +** the documentation for the three +** [SQLITE_CHANGESET_OMIT|available return values] for details. +** +**
          +**
          DELETE Changes
          +** For each DELETE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all non-primary key columns also match the values stored in +** the changeset the row is deleted from the target database. +** +** If a row with matching primary key values is found, but one or more of +** the non-primary key fields contains a value different from the original +** row value stored in the changeset, the conflict-handler function is +** invoked with [SQLITE_CHANGESET_DATA] as the second argument. If the +** database table has more columns than are recorded in the changeset, +** only the values of those non-primary key fields are compared against +** the current database contents - any trailing database table columns +** are ignored. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the DELETE operation is attempted, but SQLite returns SQLITE_CONSTRAINT +** (which can only happen if a foreign key constraint is violated), the +** conflict-handler function is invoked with [SQLITE_CHANGESET_CONSTRAINT] +** passed as the second argument. This includes the case where the DELETE +** operation is attempted because an earlier call to the conflict handler +** function returned [SQLITE_CHANGESET_REPLACE]. +** +**
          INSERT Changes
          +** For each INSERT change, an attempt is made to insert the new row into +** the database. If the changeset row contains fewer fields than the +** database table, the trailing fields are populated with their default +** values. +** +** If the attempt to insert the row fails because the database already +** contains a row with the same primary key values, the conflict handler +** function is invoked with the second argument set to +** [SQLITE_CHANGESET_CONFLICT]. +** +** If the attempt to insert the row fails because of some other constraint +** violation (e.g. NOT NULL or UNIQUE), the conflict handler function is +** invoked with the second argument set to [SQLITE_CHANGESET_CONSTRAINT]. +** This includes the case where the INSERT operation is re-attempted because +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +** +**
          UPDATE Changes
          +** For each UPDATE change, the function checks if the target database +** contains a row with the same primary key value (or values) as the +** original row values stored in the changeset. If it does, and the values +** stored in all modified non-primary key columns also match the values +** stored in the changeset the row is updated within the target database. +** +** If a row with matching primary key values is found, but one or more of +** the modified non-primary key fields contains a value different from an +** original row value stored in the changeset, the conflict-handler function +** is invoked with [SQLITE_CHANGESET_DATA] as the second argument. Since +** UPDATE changes only contain values for non-primary key fields that are +** to be modified, only those fields need to match the original values to +** avoid the SQLITE_CHANGESET_DATA conflict-handler callback. +** +** If no row with matching primary key values is found in the database, +** the conflict-handler function is invoked with [SQLITE_CHANGESET_NOTFOUND] +** passed as the second argument. +** +** If the UPDATE operation is attempted, but SQLite returns +** SQLITE_CONSTRAINT, the conflict-handler function is invoked with +** [SQLITE_CHANGESET_CONSTRAINT] passed as the second argument. +** This includes the case where the UPDATE operation is attempted after +** an earlier call to the conflict handler function returned +** [SQLITE_CHANGESET_REPLACE]. +**
          +** +** It is safe to execute SQL statements, including those that write to the +** table that the callback related to, from within the xConflict callback. +** This can be used to further customize the application's conflict +** resolution strategy. +** +** All changes made by these functions are enclosed in a savepoint transaction. +** If any other error (aside from a constraint failure when attempting to +** write to the target database) occurs, then the savepoint transaction is +** rolled back, restoring the target database to its original state, and an +** SQLite error code returned. +** +** If the output parameters (ppRebase) and (pnRebase) are non-NULL and +** the input is a changeset (not a patchset), then sqlite3changeset_apply_v2() +** may set (*ppRebase) to point to a "rebase" that may be used with the +** sqlite3_rebaser APIs buffer before returning. In this case (*pnRebase) +** is set to the size of the buffer in bytes. It is the responsibility of the +** caller to eventually free any such buffer using sqlite3_free(). The buffer +** is only allocated and populated if one or more conflicts were encountered +** while applying the patchset. See comments surrounding the sqlite3_rebaser +** APIs for further details. +** +** The behavior of sqlite3changeset_apply_v2() and its streaming equivalent +** may be modified by passing a combination of +** [SQLITE_CHANGESETAPPLY_NOSAVEPOINT | supported flags] as the 9th parameter. +** +** Note that the sqlite3changeset_apply_v2() API is still experimental +** and therefore subject to change. +*/ +SQLITE_API int sqlite3changeset_apply( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +SQLITE_API int sqlite3changeset_apply_v2( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int nChangeset, /* Size of changeset in bytes */ + void *pChangeset, /* Changeset blob */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, /* OUT: Rebase data */ + int flags /* SESSION_CHANGESETAPPLY_* flags */ +); + +/* +** CAPI3REF: Flags for sqlite3changeset_apply_v2 +** +** The following flags may passed via the 9th parameter to +** [sqlite3changeset_apply_v2] and [sqlite3changeset_apply_v2_strm]: +** +**
          +**
          SQLITE_CHANGESETAPPLY_NOSAVEPOINT
          +** Usually, the sessions module encloses all operations performed by +** a single call to apply_v2() or apply_v2_strm() in a [SAVEPOINT]. The +** SAVEPOINT is committed if the changeset or patchset is successfully +** applied, or rolled back if an error occurs. Specifying this flag +** causes the sessions module to omit this savepoint. In this case, if the +** caller has an open transaction or savepoint when apply_v2() is called, +** it may revert the partially applied changeset by rolling it back. +** +**
          SQLITE_CHANGESETAPPLY_INVERT
          +** Invert the changeset before applying it. This is equivalent to inverting +** a changeset using sqlite3changeset_invert() before applying it. It is +** an error to specify this flag with a patchset. +** +**
          SQLITE_CHANGESETAPPLY_IGNORENOOP
          +** Do not invoke the conflict handler callback for any changes that +** would not actually modify the database even if they were applied. +** Specifically, this means that the conflict handler is not invoked +** for: +**
            +**
          • a delete change if the row being deleted cannot be found, +**
          • an update change if the modified fields are already set to +** their new values in the conflicting row, or +**
          • an insert change if all fields of the conflicting row match +** the row being inserted. +**
          +** +**
          SQLITE_CHANGESETAPPLY_FKNOACTION
          +** If this flag it set, then all foreign key constraints in the target +** database behave as if they were declared with "ON UPDATE NO ACTION ON +** DELETE NO ACTION", even if they are actually CASCADE, RESTRICT, SET NULL +** or SET DEFAULT. +*/ +#define SQLITE_CHANGESETAPPLY_NOSAVEPOINT 0x0001 +#define SQLITE_CHANGESETAPPLY_INVERT 0x0002 +#define SQLITE_CHANGESETAPPLY_IGNORENOOP 0x0004 +#define SQLITE_CHANGESETAPPLY_FKNOACTION 0x0008 + +/* +** CAPI3REF: Constants Passed To The Conflict Handler +** +** Values that may be passed as the second argument to a conflict-handler. +** +**
          +**
          SQLITE_CHANGESET_DATA
          +** The conflict handler is invoked with CHANGESET_DATA as the second argument +** when processing a DELETE or UPDATE change if a row with the required +** PRIMARY KEY fields is present in the database, but one or more other +** (non primary-key) fields modified by the update do not contain the +** expected "before" values. +** +** The conflicting row, in this case, is the database row with the matching +** primary key. +** +**
          SQLITE_CHANGESET_NOTFOUND
          +** The conflict handler is invoked with CHANGESET_NOTFOUND as the second +** argument when processing a DELETE or UPDATE change if a row with the +** required PRIMARY KEY fields is not present in the database. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
          SQLITE_CHANGESET_CONFLICT
          +** CHANGESET_CONFLICT is passed as the second argument to the conflict +** handler while processing an INSERT change if the operation would result +** in duplicate primary key values. +** +** The conflicting row in this case is the database row with the matching +** primary key. +** +**
          SQLITE_CHANGESET_FOREIGN_KEY
          +** If foreign key handling is enabled, and applying a changeset leaves the +** database in a state containing foreign key violations, the conflict +** handler is invoked with CHANGESET_FOREIGN_KEY as the second argument +** exactly once before the changeset is committed. If the conflict handler +** returns CHANGESET_OMIT, the changes, including those that caused the +** foreign key constraint violation, are committed. Or, if it returns +** CHANGESET_ABORT, the changeset is rolled back. +** +** No current or conflicting row information is provided. The only function +** it is possible to call on the supplied sqlite3_changeset_iter handle +** is sqlite3changeset_fk_conflicts(). +** +**
          SQLITE_CHANGESET_CONSTRAINT
          +** If any other constraint violation occurs while applying a change (i.e. +** a UNIQUE, CHECK or NOT NULL constraint), the conflict handler is +** invoked with CHANGESET_CONSTRAINT as the second argument. +** +** There is no conflicting row in this case. The results of invoking the +** sqlite3changeset_conflict() API are undefined. +** +**
          +*/ +#define SQLITE_CHANGESET_DATA 1 +#define SQLITE_CHANGESET_NOTFOUND 2 +#define SQLITE_CHANGESET_CONFLICT 3 +#define SQLITE_CHANGESET_CONSTRAINT 4 +#define SQLITE_CHANGESET_FOREIGN_KEY 5 + +/* +** CAPI3REF: Constants Returned By The Conflict Handler +** +** A conflict handler callback must return one of the following three values. +** +**
          +**
          SQLITE_CHANGESET_OMIT
          +** If a conflict handler returns this value no special action is taken. The +** change that caused the conflict is not applied. The session module +** continues to the next change in the changeset. +** +**
          SQLITE_CHANGESET_REPLACE
          +** This value may only be returned if the second argument to the conflict +** handler was SQLITE_CHANGESET_DATA or SQLITE_CHANGESET_CONFLICT. If this +** is not the case, any changes applied so far are rolled back and the +** call to sqlite3changeset_apply() returns SQLITE_MISUSE. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_DATA conflict +** handler, then the conflicting row is either updated or deleted, depending +** on the type of change. +** +** If CHANGESET_REPLACE is returned by an SQLITE_CHANGESET_CONFLICT conflict +** handler, then the conflicting row is removed from the database and a +** second attempt to apply the change is made. If this second attempt fails, +** the original row is restored to the database before continuing. +** +**
          SQLITE_CHANGESET_ABORT
          +** If this value is returned, any changes applied so far are rolled back +** and the call to sqlite3changeset_apply() returns SQLITE_ABORT. +**
          +*/ +#define SQLITE_CHANGESET_OMIT 0 +#define SQLITE_CHANGESET_REPLACE 1 +#define SQLITE_CHANGESET_ABORT 2 + +/* +** CAPI3REF: Rebasing changesets +** EXPERIMENTAL +** +** Suppose there is a site hosting a database in state S0. And that +** modifications are made that move that database to state S1 and a +** changeset recorded (the "local" changeset). Then, a changeset based +** on S0 is received from another site (the "remote" changeset) and +** applied to the database. The database is then in state +** (S1+"remote"), where the exact state depends on any conflict +** resolution decisions (OMIT or REPLACE) made while applying "remote". +** Rebasing a changeset is to update it to take those conflict +** resolution decisions into account, so that the same conflicts +** do not have to be resolved elsewhere in the network. +** +** For example, if both the local and remote changesets contain an +** INSERT of the same key on "CREATE TABLE t1(a PRIMARY KEY, b)": +** +** local: INSERT INTO t1 VALUES(1, 'v1'); +** remote: INSERT INTO t1 VALUES(1, 'v2'); +** +** and the conflict resolution is REPLACE, then the INSERT change is +** removed from the local changeset (it was overridden). Or, if the +** conflict resolution was "OMIT", then the local changeset is modified +** to instead contain: +** +** UPDATE t1 SET b = 'v2' WHERE a=1; +** +** Changes within the local changeset are rebased as follows: +** +**
          +**
          Local INSERT
          +** This may only conflict with a remote INSERT. If the conflict +** resolution was OMIT, then add an UPDATE change to the rebased +** changeset. Or, if the conflict resolution was REPLACE, add +** nothing to the rebased changeset. +** +**
          Local DELETE
          +** This may conflict with a remote UPDATE or DELETE. In both cases the +** only possible resolution is OMIT. If the remote operation was a +** DELETE, then add no change to the rebased changeset. If the remote +** operation was an UPDATE, then the old.* fields of change are updated +** to reflect the new.* values in the UPDATE. +** +**
          Local UPDATE
          +** This may conflict with a remote UPDATE or DELETE. If it conflicts +** with a DELETE, and the conflict resolution was OMIT, then the update +** is changed into an INSERT. Any undefined values in the new.* record +** from the update change are filled in using the old.* values from +** the conflicting DELETE. Or, if the conflict resolution was REPLACE, +** the UPDATE change is simply omitted from the rebased changeset. +** +** If conflict is with a remote UPDATE and the resolution is OMIT, then +** the old.* values are rebased using the new.* values in the remote +** change. Or, if the resolution is REPLACE, then the change is copied +** into the rebased changeset with updates to columns also updated by +** the conflicting remote UPDATE removed. If this means no columns would +** be updated, the change is omitted. +**
          +** +** A local change may be rebased against multiple remote changes +** simultaneously. If a single key is modified by multiple remote +** changesets, they are combined as follows before the local changeset +** is rebased: +** +**
            +**
          • If there has been one or more REPLACE resolutions on a +** key, it is rebased according to a REPLACE. +** +**
          • If there have been no REPLACE resolutions on a key, then +** the local changeset is rebased according to the most recent +** of the OMIT resolutions. +**
          +** +** Note that conflict resolutions from multiple remote changesets are +** combined on a per-field basis, not per-row. This means that in the +** case of multiple remote UPDATE operations, some fields of a single +** local change may be rebased for REPLACE while others are rebased for +** OMIT. +** +** In order to rebase a local changeset, the remote changeset must first +** be applied to the local database using sqlite3changeset_apply_v2() and +** the buffer of rebase information captured. Then: +** +**
            +**
          1. An sqlite3_rebaser object is created by calling +** sqlite3rebaser_create(). +**
          2. The new object is configured with the rebase buffer obtained from +** sqlite3changeset_apply_v2() by calling sqlite3rebaser_configure(). +** If the local changeset is to be rebased against multiple remote +** changesets, then sqlite3rebaser_configure() should be called +** multiple times, in the same order that the multiple +** sqlite3changeset_apply_v2() calls were made. +**
          3. Each local changeset is rebased by calling sqlite3rebaser_rebase(). +**
          4. The sqlite3_rebaser object is deleted by calling +** sqlite3rebaser_delete(). +**
          +*/ +typedef struct sqlite3_rebaser sqlite3_rebaser; + +/* +** CAPI3REF: Create a changeset rebaser object. +** EXPERIMENTAL +** +** Allocate a new changeset rebaser object. If successful, set (*ppNew) to +** point to the new object and return SQLITE_OK. Otherwise, if an error +** occurs, return an SQLite error code (e.g. SQLITE_NOMEM) and set (*ppNew) +** to NULL. +*/ +SQLITE_API int sqlite3rebaser_create(sqlite3_rebaser **ppNew); + +/* +** CAPI3REF: Configure a changeset rebaser object. +** EXPERIMENTAL +** +** Configure the changeset rebaser object to rebase changesets according +** to the conflict resolutions described by buffer pRebase (size nRebase +** bytes), which must have been obtained from a previous call to +** sqlite3changeset_apply_v2(). +*/ +SQLITE_API int sqlite3rebaser_configure( + sqlite3_rebaser*, + int nRebase, const void *pRebase +); + +/* +** CAPI3REF: Rebase a changeset +** EXPERIMENTAL +** +** Argument pIn must point to a buffer containing a changeset nIn bytes +** in size. This function allocates and populates a buffer with a copy +** of the changeset rebased according to the configuration of the +** rebaser object passed as the first argument. If successful, (*ppOut) +** is set to point to the new buffer containing the rebased changeset and +** (*pnOut) to its size in bytes and SQLITE_OK returned. It is the +** responsibility of the caller to eventually free the new buffer using +** sqlite3_free(). Otherwise, if an error occurs, (*ppOut) and (*pnOut) +** are set to zero and an SQLite error code returned. +*/ +SQLITE_API int sqlite3rebaser_rebase( + sqlite3_rebaser*, + int nIn, const void *pIn, + int *pnOut, void **ppOut +); + +/* +** CAPI3REF: Delete a changeset rebaser object. +** EXPERIMENTAL +** +** Delete the changeset rebaser object and all associated resources. There +** should be one call to this function for each successful invocation +** of sqlite3rebaser_create(). +*/ +SQLITE_API void sqlite3rebaser_delete(sqlite3_rebaser *p); + +/* +** CAPI3REF: Streaming Versions of API functions. +** +** The six streaming API xxx_strm() functions serve similar purposes to the +** corresponding non-streaming API functions: +** +** +** +**
          Streaming functionNon-streaming equivalent
          sqlite3changeset_apply_strm[sqlite3changeset_apply] +**
          sqlite3changeset_apply_strm_v2[sqlite3changeset_apply_v2] +**
          sqlite3changeset_concat_strm[sqlite3changeset_concat] +**
          sqlite3changeset_invert_strm[sqlite3changeset_invert] +**
          sqlite3changeset_start_strm[sqlite3changeset_start] +**
          sqlite3session_changeset_strm[sqlite3session_changeset] +**
          sqlite3session_patchset_strm[sqlite3session_patchset] +**
          +** +** Non-streaming functions that accept changesets (or patchsets) as input +** require that the entire changeset be stored in a single buffer in memory. +** Similarly, those that return a changeset or patchset do so by returning +** a pointer to a single large buffer allocated using sqlite3_malloc(). +** Normally this is convenient. However, if an application running in a +** low-memory environment is required to handle very large changesets, the +** large contiguous memory allocations required can become onerous. +** +** In order to avoid this problem, instead of a single large buffer, input +** is passed to a streaming API functions by way of a callback function that +** the sessions module invokes to incrementally request input data as it is +** required. In all cases, a pair of API function parameters such as +** +** 
          +**        int nChangeset,
+**        void *pChangeset,
+**
          +** +** Is replaced by: +** +** 
          +**        int (*xInput)(void *pIn, void *pData, int *pnData),
+**        void *pIn,
+**
          +** +** Each time the xInput callback is invoked by the sessions module, the first +** argument passed is a copy of the supplied pIn context pointer. The second +** argument, pData, points to a buffer (*pnData) bytes in size. Assuming no +** error occurs the xInput method should copy up to (*pnData) bytes of data +** into the buffer and set (*pnData) to the actual number of bytes copied +** before returning SQLITE_OK. If the input is completely exhausted, (*pnData) +** should be set to zero to indicate this. Or, if an error occurs, an SQLite +** error code should be returned. In all cases, if an xInput callback returns +** an error, all processing is abandoned and the streaming API function +** returns a copy of the error code to the caller. +** +** In the case of sqlite3changeset_start_strm(), the xInput callback may be +** invoked by the sessions module at any point during the lifetime of the +** iterator. If such an xInput callback returns an error, the iterator enters +** an error state, whereby all subsequent calls to iterator functions +** immediately fail with the same error code as returned by xInput. +** +** Similarly, streaming API functions that return changesets (or patchsets) +** return them in chunks by way of a callback function instead of via a +** pointer to a single large buffer. In this case, a pair of parameters such +** as: +** +** 
          +**        int *pnChangeset,
+**        void **ppChangeset,
+**
          +** +** Is replaced by: +** +** 
          +**        int (*xOutput)(void *pOut, const void *pData, int nData),
+**        void *pOut
+**
          +** +** The xOutput callback is invoked zero or more times to return data to +** the application. The first parameter passed to each call is a copy of the +** pOut pointer supplied by the application. The second parameter, pData, +** points to a buffer nData bytes in size containing the chunk of output +** data being returned. If the xOutput callback successfully processes the +** supplied data, it should return SQLITE_OK to indicate success. Otherwise, +** it should return some other SQLite error code. In this case processing +** is immediately abandoned and the streaming API function returns a copy +** of the xOutput error code to the application. +** +** The sessions module never invokes an xOutput callback with the third +** parameter set to a value less than or equal to zero. Other than this, +** no guarantees are made as to the size of the chunks of data returned. +*/ +SQLITE_API int sqlite3changeset_apply_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx /* First argument passed to xConflict */ +); +SQLITE_API int sqlite3changeset_apply_v2_strm( + sqlite3 *db, /* Apply change to "main" db of this handle */ + int (*xInput)(void *pIn, void *pData, int *pnData), /* Input function */ + void *pIn, /* First arg for xInput */ + int(*xFilter)( + void *pCtx, /* Copy of sixth arg to _apply() */ + const char *zTab /* Table name */ + ), + int(*xConflict)( + void *pCtx, /* Copy of sixth arg to _apply() */ + int eConflict, /* DATA, MISSING, CONFLICT, CONSTRAINT */ + sqlite3_changeset_iter *p /* Handle describing change and conflict */ + ), + void *pCtx, /* First argument passed to xConflict */ + void **ppRebase, int *pnRebase, + int flags +); +SQLITE_API int sqlite3changeset_concat_strm( + int (*xInputA)(void *pIn, void *pData, int *pnData), + void *pInA, + int (*xInputB)(void *pIn, void *pData, int *pnData), + void *pInB, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changeset_invert_strm( + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changeset_start_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +SQLITE_API int sqlite3changeset_start_v2_strm( + sqlite3_changeset_iter **pp, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int flags +); +SQLITE_API int sqlite3session_changeset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3session_patchset_strm( + sqlite3_session *pSession, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3changegroup_add_strm(sqlite3_changegroup*, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn +); +SQLITE_API int sqlite3changegroup_output_strm(sqlite3_changegroup*, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); +SQLITE_API int sqlite3rebaser_rebase_strm( + sqlite3_rebaser *pRebaser, + int (*xInput)(void *pIn, void *pData, int *pnData), + void *pIn, + int (*xOutput)(void *pOut, const void *pData, int nData), + void *pOut +); + +/* +** CAPI3REF: Configure global parameters +** +** The sqlite3session_config() interface is used to make global configuration +** changes to the sessions module in order to tune it to the specific needs +** of the application. +** +** The sqlite3session_config() interface is not threadsafe. If it is invoked +** while any other thread is inside any other sessions method then the +** results are undefined. Furthermore, if it is invoked after any sessions +** related objects have been created, the results are also undefined. +** +** The first argument to the sqlite3session_config() function must be one +** of the SQLITE_SESSION_CONFIG_XXX constants defined below. The +** interpretation of the (void*) value passed as the second parameter and +** the effect of calling this function depends on the value of the first +** parameter. +** +**
          +**
          SQLITE_SESSION_CONFIG_STRMSIZE
          +** By default, the sessions module streaming interfaces attempt to input +** and output data in approximately 1 KiB chunks. This operand may be used +** to set and query the value of this configuration setting. The pointer +** passed as the second argument must point to a value of type (int). +** If this value is greater than 0, it is used as the new streaming data +** chunk size for both input and output. Before returning, the (int) value +** pointed to by pArg is set to the final value of the streaming interface +** chunk size. +**
          +** +** This function returns SQLITE_OK if successful, or an SQLite error code +** otherwise. +*/ +SQLITE_API int sqlite3session_config(int op, void *pArg); + +/* +** CAPI3REF: Values for sqlite3session_config(). +*/ +#define SQLITE_SESSION_CONFIG_STRMSIZE 1 + +/* +** Make sure we can call this stuff from C++. +*/ +#ifdef __cplusplus +} +#endif + +#endif /* !defined(__SQLITESESSION_H_) && defined(SQLITE_ENABLE_SESSION) */ + +/******** End of sqlite3session.h *********/ +/******** Begin file fts5.h *********/ +/* +** 2014 May 31 +** +** The author disclaims copyright to this source code. In place of +** a legal notice, here is a blessing: +** +** May you do good and not evil. +** May you find forgiveness for yourself and forgive others. +** May you share freely, never taking more than you give. +** +****************************************************************************** +** +** Interfaces to extend FTS5. Using the interfaces defined in this file, +** FTS5 may be extended with: +** +** * custom tokenizers, and +** * custom auxiliary functions. +*/ + + +#ifndef _FTS5_H +#define _FTS5_H + + +#ifdef __cplusplus +extern "C" { +#endif + +/************************************************************************* +** CUSTOM AUXILIARY FUNCTIONS +** +** Virtual table implementations may overload SQL functions by implementing +** the sqlite3_module.xFindFunction() method. +*/ + +typedef struct Fts5ExtensionApi Fts5ExtensionApi; +typedef struct Fts5Context Fts5Context; +typedef struct Fts5PhraseIter Fts5PhraseIter; + +typedef void (*fts5_extension_function)( + const Fts5ExtensionApi *pApi, /* API offered by current FTS version */ + Fts5Context *pFts, /* First arg to pass to pApi functions */ + sqlite3_context *pCtx, /* Context for returning result/error */ + int nVal, /* Number of values in apVal[] array */ + sqlite3_value **apVal /* Array of trailing arguments */ +); + +struct Fts5PhraseIter { + const unsigned char *a; + const unsigned char *b; +}; + +/* +** EXTENSION API FUNCTIONS +** +** xUserData(pFts): +** Return a copy of the pUserData pointer passed to the xCreateFunction() +** API when the extension function was registered. +** +** xColumnTotalSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the FTS5 table. Or, if iCol is +** non-negative but less than the number of columns in the table, return +** the total number of tokens in column iCol, considering all rows in +** the FTS5 table. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** xColumnCount(pFts): +** Return the number of columns in the table. +** +** xColumnSize(pFts, iCol, pnToken): +** If parameter iCol is less than zero, set output variable *pnToken +** to the total number of tokens in the current row. Or, if iCol is +** non-negative but less than the number of columns in the table, set +** *pnToken to the number of tokens in column iCol of the current row. +** +** If parameter iCol is greater than or equal to the number of columns +** in the table, SQLITE_RANGE is returned. Or, if an error occurs (e.g. +** an OOM condition or IO error), an appropriate SQLite error code is +** returned. +** +** This function may be quite inefficient if used with an FTS5 table +** created with the "columnsize=0" option. +** +** xColumnText: +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the text of column iCol of +** the current document. If successful, (*pz) is set to point to a buffer +** containing the text in utf-8 encoding, (*pn) is set to the size in bytes +** (not characters) of the buffer and SQLITE_OK is returned. Otherwise, +** if an error occurs, an SQLite error code is returned and the final values +** of (*pz) and (*pn) are undefined. +** +** xPhraseCount: +** Returns the number of phrases in the current query expression. +** +** xPhraseSize: +** If parameter iCol is less than zero, or greater than or equal to the +** number of phrases in the current query, as returned by xPhraseCount, +** 0 is returned. Otherwise, this function returns the number of tokens in +** phrase iPhrase of the query. Phrases are numbered starting from zero. +** +** xInstCount: +** Set *pnInst to the total number of occurrences of all phrases within +** the query within the current row. Return SQLITE_OK if successful, or +** an error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always returns 0. +** +** xInst: +** Query for the details of phrase match iIdx within the current row. +** Phrase matches are numbered starting from zero, so the iIdx argument +** should be greater than or equal to zero and smaller than the value +** output by xInstCount(). If iIdx is less than zero or greater than +** or equal to the value returned by xInstCount(), SQLITE_RANGE is returned. +** +** Otherwise, output parameter *piPhrase is set to the phrase number, *piCol +** to the column in which it occurs and *piOff the token offset of the +** first token of the phrase. SQLITE_OK is returned if successful, or an +** error code (i.e. SQLITE_NOMEM) if an error occurs. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xRowid: +** Returns the rowid of the current row. +** +** xTokenize: +** Tokenize text using the tokenizer belonging to the FTS5 table. +** +** xQueryPhrase(pFts5, iPhrase, pUserData, xCallback): +** This API function is used to query the FTS table for phrase iPhrase +** of the current query. Specifically, a query equivalent to: +** +** ... FROM ftstable WHERE ftstable MATCH $p ORDER BY rowid +** +** with $p set to a phrase equivalent to the phrase iPhrase of the +** current query is executed. Any column filter that applies to +** phrase iPhrase of the current query is included in $p. For each +** row visited, the callback function passed as the fourth argument +** is invoked. The context and API objects passed to the callback +** function may be used to access the properties of each matched row. +** Invoking Api.xUserData() returns a copy of the pointer passed as +** the third argument to pUserData. +** +** If parameter iPhrase is less than zero, or greater than or equal to +** the number of phrases in the query, as returned by xPhraseCount(), +** this function returns SQLITE_RANGE. +** +** If the callback function returns any value other than SQLITE_OK, the +** query is abandoned and the xQueryPhrase function returns immediately. +** If the returned value is SQLITE_DONE, xQueryPhrase returns SQLITE_OK. +** Otherwise, the error code is propagated upwards. +** +** If the query runs to completion without incident, SQLITE_OK is returned. +** Or, if some error occurs before the query completes or is aborted by +** the callback, an SQLite error code is returned. +** +** +** xSetAuxdata(pFts5, pAux, xDelete) +** +** Save the pointer passed as the second argument as the extension function's +** "auxiliary data". The pointer may then be retrieved by the current or any +** future invocation of the same fts5 extension function made as part of +** the same MATCH query using the xGetAuxdata() API. +** +** Each extension function is allocated a single auxiliary data slot for +** each FTS query (MATCH expression). If the extension function is invoked +** more than once for a single FTS query, then all invocations share a +** single auxiliary data context. +** +** If there is already an auxiliary data pointer when this function is +** invoked, then it is replaced by the new pointer. If an xDelete callback +** was specified along with the original pointer, it is invoked at this +** point. +** +** The xDelete callback, if one is specified, is also invoked on the +** auxiliary data pointer after the FTS5 query has finished. +** +** If an error (e.g. an OOM condition) occurs within this function, +** the auxiliary data is set to NULL and an error code returned. If the +** xDelete parameter was not NULL, it is invoked on the auxiliary data +** pointer before returning. +** +** +** xGetAuxdata(pFts5, bClear) +** +** Returns the current auxiliary data pointer for the fts5 extension +** function. See the xSetAuxdata() method for details. +** +** If the bClear argument is non-zero, then the auxiliary data is cleared +** (set to NULL) before this function returns. In this case the xDelete, +** if any, is not invoked. +** +** +** xRowCount(pFts5, pnRow) +** +** This function is used to retrieve the total number of rows in the table. +** In other words, the same value that would be returned by: +** +** SELECT count(*) FROM ftstable; +** +** xPhraseFirst() +** This function is used, along with type Fts5PhraseIter and the xPhraseNext +** method, to iterate through all instances of a single query phrase within +** the current row. This is the same information as is accessible via the +** xInstCount/xInst APIs. While the xInstCount/xInst APIs are more convenient +** to use, this API may be faster under some circumstances. To iterate +** through instances of phrase iPhrase, use the following code: +** +** Fts5PhraseIter iter; +** int iCol, iOff; +** for(pApi->xPhraseFirst(pFts, iPhrase, &iter, &iCol, &iOff); +** iCol>=0; +** pApi->xPhraseNext(pFts, &iter, &iCol, &iOff) +** ){ +** // An instance of phrase iPhrase at offset iOff of column iCol +** } +** +** The Fts5PhraseIter structure is defined above. Applications should not +** modify this structure directly - it should only be used as shown above +** with the xPhraseFirst() and xPhraseNext() API methods (and by +** xPhraseFirstColumn() and xPhraseNextColumn() as illustrated below). +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. If the FTS5 table is created +** with either "detail=none" or "detail=column" and "content=" option +** (i.e. if it is a contentless table), then this API always iterates +** through an empty set (all calls to xPhraseFirst() set iCol to -1). +** +** In all cases, matches are visited in (column ASC, offset ASC) order. +** i.e. all those in column 0, sorted by offset, followed by those in +** column 1, etc. +** +** xPhraseNext() +** See xPhraseFirst above. +** +** xPhraseFirstColumn() +** This function and xPhraseNextColumn() are similar to the xPhraseFirst() +** and xPhraseNext() APIs described above. The difference is that instead +** of iterating through all instances of a phrase in the current row, these +** APIs are used to iterate through the set of columns in the current row +** that contain one or more instances of a specified phrase. For example: +** +** Fts5PhraseIter iter; +** int iCol; +** for(pApi->xPhraseFirstColumn(pFts, iPhrase, &iter, &iCol); +** iCol>=0; +** pApi->xPhraseNextColumn(pFts, &iter, &iCol) +** ){ +** // Column iCol contains at least one instance of phrase iPhrase +** } +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" option. If the FTS5 table is created with either +** "detail=none" "content=" option (i.e. if it is a contentless table), +** then this API always iterates through an empty set (all calls to +** xPhraseFirstColumn() set iCol to -1). +** +** The information accessed using this API and its companion +** xPhraseFirstColumn() may also be obtained using xPhraseFirst/xPhraseNext +** (or xInst/xInstCount). The chief advantage of this API is that it is +** significantly more efficient than those alternatives when used with +** "detail=column" tables. +** +** xPhraseNextColumn() +** See xPhraseFirstColumn above. +** +** xQueryToken(pFts5, iPhrase, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase iPhrase of the current +** query. Before returning, output parameter *ppToken is set to point +** to a buffer containing the requested token, and *pnToken to the +** size of this buffer in bytes. +** +** If iPhrase or iToken are less than zero, or if iPhrase is greater than +** or equal to the number of phrases in the query as reported by +** xPhraseCount(), or if iToken is equal to or greater than the number of +** tokens in the phrase, SQLITE_RANGE is returned and *ppToken and *pnToken + are both zeroed. +** +** The output text is not a copy of the query text that specified the +** token. It is the output of the tokenizer module. For tokendata=1 +** tables, this includes any embedded 0x00 and trailing data. +** +** xInstToken(pFts5, iIdx, iToken, ppToken, pnToken) +** This is used to access token iToken of phrase hit iIdx within the +** current row. If iIdx is less than zero or greater than or equal to the +** value returned by xInstCount(), SQLITE_RANGE is returned. Otherwise, +** output variable (*ppToken) is set to point to a buffer containing the +** matching document token, and (*pnToken) to the size of that buffer in +** bytes. This API is not available if the specified token matches a +** prefix query term. In that case both output variables are always set +** to 0. +** +** The output text is not a copy of the document text that was tokenized. +** It is the output of the tokenizer module. For tokendata=1 tables, this +** includes any embedded 0x00 and trailing data. +** +** This API can be quite slow if used with an FTS5 table created with the +** "detail=none" or "detail=column" option. +** +** xColumnLocale(pFts5, iIdx, pzLocale, pnLocale) +** If parameter iCol is less than zero, or greater than or equal to the +** number of columns in the table, SQLITE_RANGE is returned. +** +** Otherwise, this function attempts to retrieve the locale associated +** with column iCol of the current row. Usually, there is no associated +** locale, and output parameters (*pzLocale) and (*pnLocale) are set +** to NULL and 0, respectively. However, if the fts5_locale() function +** was used to associate a locale with the value when it was inserted +** into the fts5 table, then (*pzLocale) is set to point to a nul-terminated +** buffer containing the name of the locale in utf-8 encoding. (*pnLocale) +** is set to the size in bytes of the buffer, not including the +** nul-terminator. +** +** If successful, SQLITE_OK is returned. Or, if an error occurs, an +** SQLite error code is returned. The final value of the output parameters +** is undefined in this case. +** +** xTokenize_v2: +** Tokenize text using the tokenizer belonging to the FTS5 table. This +** API is the same as the xTokenize() API, except that it allows a tokenizer +** locale to be specified. +*/ +struct Fts5ExtensionApi { + int iVersion; /* Currently always set to 4 */ + + void *(*xUserData)(Fts5Context*); + + int (*xColumnCount)(Fts5Context*); + int (*xRowCount)(Fts5Context*, sqlite3_int64 *pnRow); + int (*xColumnTotalSize)(Fts5Context*, int iCol, sqlite3_int64 *pnToken); + + int (*xTokenize)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); + + int (*xPhraseCount)(Fts5Context*); + int (*xPhraseSize)(Fts5Context*, int iPhrase); + + int (*xInstCount)(Fts5Context*, int *pnInst); + int (*xInst)(Fts5Context*, int iIdx, int *piPhrase, int *piCol, int *piOff); + + sqlite3_int64 (*xRowid)(Fts5Context*); + int (*xColumnText)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xColumnSize)(Fts5Context*, int iCol, int *pnToken); + + int (*xQueryPhrase)(Fts5Context*, int iPhrase, void *pUserData, + int(*)(const Fts5ExtensionApi*,Fts5Context*,void*) + ); + int (*xSetAuxdata)(Fts5Context*, void *pAux, void(*xDelete)(void*)); + void *(*xGetAuxdata)(Fts5Context*, int bClear); + + int (*xPhraseFirst)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*, int*); + void (*xPhraseNext)(Fts5Context*, Fts5PhraseIter*, int *piCol, int *piOff); + + int (*xPhraseFirstColumn)(Fts5Context*, int iPhrase, Fts5PhraseIter*, int*); + void (*xPhraseNextColumn)(Fts5Context*, Fts5PhraseIter*, int *piCol); + + /* Below this point are iVersion>=3 only */ + int (*xQueryToken)(Fts5Context*, + int iPhrase, int iToken, + const char **ppToken, int *pnToken + ); + int (*xInstToken)(Fts5Context*, int iIdx, int iToken, const char**, int*); + + /* Below this point are iVersion>=4 only */ + int (*xColumnLocale)(Fts5Context*, int iCol, const char **pz, int *pn); + int (*xTokenize_v2)(Fts5Context*, + const char *pText, int nText, /* Text to tokenize */ + const char *pLocale, int nLocale, /* Locale to pass to tokenizer */ + void *pCtx, /* Context passed to xToken() */ + int (*xToken)(void*, int, const char*, int, int, int) /* Callback */ + ); +}; + +/* +** CUSTOM AUXILIARY FUNCTIONS +*************************************************************************/ + +/************************************************************************* +** CUSTOM TOKENIZERS +** +** Applications may also register custom tokenizer types. A tokenizer +** is registered by providing fts5 with a populated instance of the +** following structure. All structure methods must be defined, setting +** any member of the fts5_tokenizer struct to NULL leads to undefined +** behaviour. The structure methods are expected to function as follows: +** +** xCreate: +** This function is used to allocate and initialize a tokenizer instance. +** A tokenizer instance is required to actually tokenize text. +** +** The first argument passed to this function is a copy of the (void*) +** pointer provided by the application when the fts5_tokenizer_v2 object +** was registered with FTS5 (the third argument to xCreateTokenizer()). +** The second and third arguments are an array of nul-terminated strings +** containing the tokenizer arguments, if any, specified following the +** tokenizer name as part of the CREATE VIRTUAL TABLE statement used +** to create the FTS5 table. +** +** The final argument is an output variable. If successful, (*ppOut) +** should be set to point to the new tokenizer handle and SQLITE_OK +** returned. If an error occurs, some value other than SQLITE_OK should +** be returned. In this case, fts5 assumes that the final value of *ppOut +** is undefined. +** +** xDelete: +** This function is invoked to delete a tokenizer handle previously +** allocated using xCreate(). Fts5 guarantees that this function will +** be invoked exactly once for each successful call to xCreate(). +** +** xTokenize: +** This function is expected to tokenize the nText byte string indicated +** by argument pText. pText may or may not be nul-terminated. The first +** argument passed to this function is a pointer to an Fts5Tokenizer object +** returned by an earlier call to xCreate(). +** +** The third argument indicates the reason that FTS5 is requesting +** tokenization of the supplied text. This is always one of the following +** four values: +** +**
          • FTS5_TOKENIZE_DOCUMENT - A document is being inserted into +** or removed from the FTS table. The tokenizer is being invoked to +** determine the set of tokens to add to (or delete from) the +** FTS index. +** +**
          • FTS5_TOKENIZE_QUERY - A MATCH query is being executed +** against the FTS index. The tokenizer is being called to tokenize +** a bareword or quoted string specified as part of the query. +** +**
          • (FTS5_TOKENIZE_QUERY | FTS5_TOKENIZE_PREFIX) - Same as +** FTS5_TOKENIZE_QUERY, except that the bareword or quoted string is +** followed by a "*" character, indicating that the last token +** returned by the tokenizer will be treated as a token prefix. +** +**
          • FTS5_TOKENIZE_AUX - The tokenizer is being invoked to +** satisfy an fts5_api.xTokenize() request made by an auxiliary +** function. Or an fts5_api.xColumnSize() request made by the same +** on a columnsize=0 database. +**
          +** +** The sixth and seventh arguments passed to xTokenize() - pLocale and +** nLocale - are a pointer to a buffer containing the locale to use for +** tokenization (e.g. "en_US") and its size in bytes, respectively. The +** pLocale buffer is not nul-terminated. pLocale may be passed NULL (in +** which case nLocale is always 0) to indicate that the tokenizer should +** use its default locale. +** +** For each token in the input string, the supplied callback xToken() must +** be invoked. The first argument to it should be a copy of the pointer +** passed as the second argument to xTokenize(). The third and fourth +** arguments are a pointer to a buffer containing the token text, and the +** size of the token in bytes. The 4th and 5th arguments are the byte offsets +** of the first byte of and first byte immediately following the text from +** which the token is derived within the input. +** +** The second argument passed to the xToken() callback ("tflags") should +** normally be set to 0. The exception is if the tokenizer supports +** synonyms. In this case see the discussion below for details. +** +** FTS5 assumes the xToken() callback is invoked for each token in the +** order that they occur within the input text. +** +** If an xToken() callback returns any value other than SQLITE_OK, then +** the tokenization should be abandoned and the xTokenize() method should +** immediately return a copy of the xToken() return value. Or, if the +** input buffer is exhausted, xTokenize() should return SQLITE_OK. Finally, +** if an error occurs with the xTokenize() implementation itself, it +** may abandon the tokenization and return any error code other than +** SQLITE_OK or SQLITE_DONE. +** +** If the tokenizer is registered using an fts5_tokenizer_v2 object, +** then the xTokenize() method has two additional arguments - pLocale +** and nLocale. These specify the locale that the tokenizer should use +** for the current request. If pLocale and nLocale are both 0, then the +** tokenizer should use its default locale. Otherwise, pLocale points to +** an nLocale byte buffer containing the name of the locale to use as utf-8 +** text. pLocale is not nul-terminated. +** +** FTS5_TOKENIZER +** +** There is also an fts5_tokenizer object. This is an older, deprecated, +** version of fts5_tokenizer_v2. It is similar except that: +** +**
            +**
          • There is no "iVersion" field, and +**
          • The xTokenize() method does not take a locale argument. +**
          +** +** Legacy fts5_tokenizer tokenizers must be registered using the +** legacy xCreateTokenizer() function, instead of xCreateTokenizer_v2(). +** +** Tokenizer implementations registered using either API may be retrieved +** using both xFindTokenizer() and xFindTokenizer_v2(). +** +** SYNONYM SUPPORT +** +** Custom tokenizers may also support synonyms. Consider a case in which a +** user wishes to query for a phrase such as "first place". Using the +** built-in tokenizers, the FTS5 query 'first + place' will match instances +** of "first place" within the document set, but not alternative forms +** such as "1st place". In some applications, it would be better to match +** all instances of "first place" or "1st place" regardless of which form +** the user specified in the MATCH query text. +** +** There are several ways to approach this in FTS5: +** +**
          1. By mapping all synonyms to a single token. In this case, using +** the above example, this means that the tokenizer returns the +** same token for inputs "first" and "1st". Say that token is in +** fact "first", so that when the user inserts the document "I won +** 1st place" entries are added to the index for tokens "i", "won", +** "first" and "place". If the user then queries for '1st + place', +** the tokenizer substitutes "first" for "1st" and the query works +** as expected. +** +**
          2. By querying the index for all synonyms of each query term +** separately. In this case, when tokenizing query text, the +** tokenizer may provide multiple synonyms for a single term +** within the document. FTS5 then queries the index for each +** synonym individually. For example, faced with the query: +** +** +** ... MATCH 'first place' +** +** the tokenizer offers both "1st" and "first" as synonyms for the +** first token in the MATCH query and FTS5 effectively runs a query +** similar to: +** +** +** ... MATCH '(first OR 1st) place' +** +** except that, for the purposes of auxiliary functions, the query +** still appears to contain just two phrases - "(first OR 1st)" +** being treated as a single phrase. +** +**
          3. By adding multiple synonyms for a single term to the FTS index. +** Using this method, when tokenizing document text, the tokenizer +** provides multiple synonyms for each token. So that when a +** document such as "I won first place" is tokenized, entries are +** added to the FTS index for "i", "won", "first", "1st" and +** "place". +** +** This way, even if the tokenizer does not provide synonyms +** when tokenizing query text (it should not - to do so would be +** inefficient), it doesn't matter if the user queries for +** 'first + place' or '1st + place', as there are entries in the +** FTS index corresponding to both forms of the first token. +**
          +** +** Whether it is parsing document or query text, any call to xToken that +** specifies a tflags argument with the FTS5_TOKEN_COLOCATED bit +** is considered to supply a synonym for the previous token. For example, +** when parsing the document "I won first place", a tokenizer that supports +** synonyms would call xToken() 5 times, as follows: +** +** +** xToken(pCtx, 0, "i", 1, 0, 1); +** xToken(pCtx, 0, "won", 3, 2, 5); +** xToken(pCtx, 0, "first", 5, 6, 11); +** xToken(pCtx, FTS5_TOKEN_COLOCATED, "1st", 3, 6, 11); +** xToken(pCtx, 0, "place", 5, 12, 17); +** +** +** It is an error to specify the FTS5_TOKEN_COLOCATED flag the first time +** xToken() is called. Multiple synonyms may be specified for a single token +** by making multiple calls to xToken(FTS5_TOKEN_COLOCATED) in sequence. +** There is no limit to the number of synonyms that may be provided for a +** single token. +** +** In many cases, method (1) above is the best approach. It does not add +** extra data to the FTS index or require FTS5 to query for multiple terms, +** so it is efficient in terms of disk space and query speed. However, it +** does not support prefix queries very well. If, as suggested above, the +** token "first" is substituted for "1st" by the tokenizer, then the query: +** +** +** ... MATCH '1s*' +** +** will not match documents that contain the token "1st" (as the tokenizer +** will probably not map "1s" to any prefix of "first"). +** +** For full prefix support, method (3) may be preferred. In this case, +** because the index contains entries for both "first" and "1st", prefix +** queries such as 'fi*' or '1s*' will match correctly. However, because +** extra entries are added to the FTS index, this method uses more space +** within the database. +** +** Method (2) offers a midpoint between (1) and (3). Using this method, +** a query such as '1s*' will match documents that contain the literal +** token "1st", but not "first" (assuming the tokenizer is not able to +** provide synonyms for prefixes). However, a non-prefix query like '1st' +** will match against "1st" and "first". This method does not require +** extra disk space, as no extra entries are added to the FTS index. +** On the other hand, it may require more CPU cycles to run MATCH queries, +** as separate queries of the FTS index are required for each synonym. +** +** When using methods (2) or (3), it is important that the tokenizer only +** provide synonyms when tokenizing document text (method (3)) or query +** text (method (2)), not both. Doing so will not cause any errors, but is +** inefficient. +*/ +typedef struct Fts5Tokenizer Fts5Tokenizer; +typedef struct fts5_tokenizer_v2 fts5_tokenizer_v2; +struct fts5_tokenizer_v2 { + int iVersion; /* Currently always 2 */ + + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + const char *pLocale, int nLocale, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + +/* +** New code should use the fts5_tokenizer_v2 type to define tokenizer +** implementations. The following type is included for legacy applications +** that still use it. +*/ +typedef struct fts5_tokenizer fts5_tokenizer; +struct fts5_tokenizer { + int (*xCreate)(void*, const char **azArg, int nArg, Fts5Tokenizer **ppOut); + void (*xDelete)(Fts5Tokenizer*); + int (*xTokenize)(Fts5Tokenizer*, + void *pCtx, + int flags, /* Mask of FTS5_TOKENIZE_* flags */ + const char *pText, int nText, + int (*xToken)( + void *pCtx, /* Copy of 2nd argument to xTokenize() */ + int tflags, /* Mask of FTS5_TOKEN_* flags */ + const char *pToken, /* Pointer to buffer containing token */ + int nToken, /* Size of token in bytes */ + int iStart, /* Byte offset of token within input text */ + int iEnd /* Byte offset of end of token within input text */ + ) + ); +}; + + +/* Flags that may be passed as the third argument to xTokenize() */ +#define FTS5_TOKENIZE_QUERY 0x0001 +#define FTS5_TOKENIZE_PREFIX 0x0002 +#define FTS5_TOKENIZE_DOCUMENT 0x0004 +#define FTS5_TOKENIZE_AUX 0x0008 + +/* Flags that may be passed by the tokenizer implementation back to FTS5 +** as the third argument to the supplied xToken callback. */ +#define FTS5_TOKEN_COLOCATED 0x0001 /* Same position as prev. token */ + +/* +** END OF CUSTOM TOKENIZERS +*************************************************************************/ + +/************************************************************************* +** FTS5 EXTENSION REGISTRATION API +*/ +typedef struct fts5_api fts5_api; +struct fts5_api { + int iVersion; /* Currently always set to 3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer *pTokenizer + ); + + /* Create a new auxiliary function */ + int (*xCreateFunction)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_extension_function xFunction, + void (*xDestroy)(void*) + ); + + /* APIs below this point are only available if iVersion>=3 */ + + /* Create a new tokenizer */ + int (*xCreateTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void *pUserData, + fts5_tokenizer_v2 *pTokenizer, + void (*xDestroy)(void*) + ); + + /* Find an existing tokenizer */ + int (*xFindTokenizer_v2)( + fts5_api *pApi, + const char *zName, + void **ppUserData, + fts5_tokenizer_v2 **ppTokenizer + ); +}; + +/* +** END OF REGISTRATION API +*************************************************************************/ + +#ifdef __cplusplus +} /* end of the 'extern "C"' block */ +#endif + +#endif /* _FTS5_H */ + +/******** End of fts5.h *********/